vulkan: Allow the driver to manually enable threaded submit

This is useful for drivers that wish to be able to block inside their
vk_queue::driver_submit hook.

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15566>

src/vulkan/runtime/vk_device.c

@@ -186,6 +186,31 @@ vk_device_finish(UNUSED struct vk_device *device)
    vk_object_base_finish(&device->base);
 }
 
+void
+vk_device_enable_threaded_submit(struct vk_device *device)
+{
+   /* This must be called before any queues are created */
+   assert(list_is_empty(&device->queues));
+
+   /* In order to use threaded submit, we need every sync type that can be
+    * used as a wait fence for vkQueueSubmit() to support WAIT_PENDING.
+    * It's required for cross-thread/process submit re-ordering.
+    */
+   for (const struct vk_sync_type *const *t =
+        device->physical->supported_sync_types; *t; t++) {
+      if ((*t)->features & VK_SYNC_FEATURE_GPU_WAIT)
+         assert((*t)->features & VK_SYNC_FEATURE_WAIT_PENDING);
+   }
+
+   /* Any binary vk_sync types which will be used as permanent semaphore
+    * payloads also need to support vk_sync_type::move, but that's a lot
+    * harder to assert since it only applies to permanent semaphore payloads.
+    */
+
+   if (device->submit_mode != VK_QUEUE_SUBMIT_MODE_THREADED)
+      device->submit_mode = VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND;
+}
+
 VkResult
 vk_device_flush(struct vk_device *device)
 {

src/vulkan/runtime/vk_device.h

@@ -255,6 +255,17 @@ vk_device_set_drm_fd(struct vk_device *device, int drm_fd)
 void
 vk_device_finish(struct vk_device *device);
 
+/** Enables threaded submit on this device
+ *
+ * This doesn't ensure that threaded submit will be used.  It just disables
+ * the deferred submit option for emulated timeline semaphores and forces them
+ * to always use the threaded path.  It also does some checks that the vk_sync
+ * types used by the driver work for threaded submit.
+ *
+ * This must be called before any queues are created.
+ */
+void vk_device_enable_threaded_submit(struct vk_device *device);
+
 static inline bool
 vk_device_supports_threaded_submit(const struct vk_device *device)
 {

src/vulkan/runtime/vk_queue.c

@@ -44,7 +44,7 @@
 #include "vulkan/wsi/wsi_common.h"
 
 static VkResult
-vk_queue_enable_submit_thread(struct vk_queue *queue);
+vk_queue_start_submit_thread(struct vk_queue *queue);
 
 VkResult
 vk_queue_init(struct vk_queue *queue, struct vk_device *device,
@@ -90,7 +90,7 @@ vk_queue_init(struct vk_queue *queue, struct vk_device *device,
    }
 
    if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED) {
-      result = vk_queue_enable_submit_thread(queue);
+      result = vk_queue_start_submit_thread(queue);
       if (result != VK_SUCCESS)
          goto fail_thread;
    }
@@ -508,7 +508,7 @@ vk_queue_submit_thread_func(void *_data)
 }
 
 static VkResult
-vk_queue_enable_submit_thread(struct vk_queue *queue)
+vk_queue_start_submit_thread(struct vk_queue *queue)
 {
    int ret;
 
@@ -520,13 +520,11 @@ vk_queue_enable_submit_thread(struct vk_queue *queue)
    if (ret == thrd_error)
       return vk_errorf(queue, VK_ERROR_UNKNOWN, "thrd_create failed");
 
-   queue->submit.mode = VK_QUEUE_SUBMIT_MODE_THREADED;
-
    return VK_SUCCESS;
 }
 
 static void
-vk_queue_disable_submit_thread(struct vk_queue *queue)
+vk_queue_stop_submit_thread(struct vk_queue *queue)
 {
    vk_queue_drain(queue);
 
@@ -542,6 +540,23 @@ vk_queue_disable_submit_thread(struct vk_queue *queue)
    queue->submit.mode = VK_QUEUE_SUBMIT_MODE_IMMEDIATE;
 }
 
+VkResult
+vk_queue_enable_submit_thread(struct vk_queue *queue)
+{
+   assert(vk_device_supports_threaded_submit(queue->base.device));
+
+   if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED)
+      return VK_SUCCESS;
+
+   VkResult result = vk_queue_start_submit_thread(queue);
+   if (result != VK_SUCCESS)
+      return result;
+
+   queue->submit.mode = VK_QUEUE_SUBMIT_MODE_THREADED;
+
+   return VK_SUCCESS;
+}
+
 struct vulkan_submit_info {
    const void *pNext;
 
@@ -1075,7 +1090,7 @@ void
 vk_queue_finish(struct vk_queue *queue)
 {
    if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED)
-      vk_queue_disable_submit_thread(queue);
+      vk_queue_stop_submit_thread(queue);
 
    while (!list_is_empty(&queue->submit.submits)) {
       assert(vk_device_is_lost_no_report(queue->base.device));

src/vulkan/runtime/vk_queue.h

@@ -163,6 +163,15 @@ vk_queue_is_empty(struct vk_queue *queue)
    return list_is_empty(&queue->submit.submits);
 }
 
+/** Enables threaded submit on this queue
+ *
+ * This should be called by the driver if it wants to be able to block inside
+ * `vk_queue::driver_submit`.  Once this function has been called, the queue
+ * will always use a submit thread for all submissions.  You must have called
+ * vk_device_enabled_threaded_submit() before calling this function.
+ */
+VkResult vk_queue_enable_submit_thread(struct vk_queue *queue);
+
 VkResult vk_queue_flush(struct vk_queue *queue, uint32_t *submit_count_out);
 
 VkResult vk_queue_wait_before_present(struct vk_queue *queue,