vulkan/queue: Rework vk_queue_submit()

Instead of basing everything on the timeline mode, base it on the submit
mode of the queue.  This makes a lot more sense since it's what we
really care about anyway.

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/15566>

src/vulkan/runtime/vk_device.h

@@ -255,6 +255,13 @@ vk_device_set_drm_fd(struct vk_device *device, int drm_fd)
 void
 vk_device_finish(struct vk_device *device);
 
+static inline bool
+vk_device_supports_threaded_submit(const struct vk_device *device)
+{
+   return device->submit_mode == VK_QUEUE_SUBMIT_MODE_THREADED ||
+          device->submit_mode == VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND;
+}
+
 VkResult vk_device_flush(struct vk_device *device);
 
 VkResult PRINTFLIKE(4, 5)

src/vulkan/runtime/vk_queue.c

@@ -646,8 +646,7 @@ vk_queue_submit(struct vk_queue *queue,
          semaphore->temporary = NULL;
       } else {
          if (semaphore->type == VK_SEMAPHORE_TYPE_BINARY) {
-            if (queue->base.device->timeline_mode ==
+            if (vk_device_supports_threaded_submit(device))
-                VK_DEVICE_TIMELINE_MODE_ASSISTED)
                assert(semaphore->permanent.type->move);
             has_binary_permanent_semaphore_wait = true;
          }
@@ -810,9 +809,55 @@ vk_queue_submit(struct vk_queue *queue,
          goto fail;
    }
 
-   switch (queue->base.device->timeline_mode) {
+   switch (queue->submit.mode) {
-   case VK_DEVICE_TIMELINE_MODE_ASSISTED:
+   case VK_QUEUE_SUBMIT_MODE_IMMEDIATE:
-      if (queue->submit.mode == VK_QUEUE_SUBMIT_MODE_THREADED) {
+      result = vk_queue_submit_final(queue, submit);
+      if (unlikely(result != VK_SUCCESS))
+         goto fail;
+
+      /* If threaded submit is possible on this device, we need to ensure that
+       * binary semaphore payloads get reset so that any other threads can
+       * properly wait on them for dependency checking.  Because we don't
+       * currently have a submit thread, we can directly reset that binary
+       * semaphore payloads.
+       *
+       * If we the vk_sync is in our signal et, we can consider it to have
+       * been both reset and signaled by queue_submit_final().  A reset in
+       * this case would be wrong because it would throw away our signal
+       * operation.  If we don't signal the vk_sync, then we need to reset it.
+       */
+      if (vk_device_supports_threaded_submit(device) &&
+          has_binary_permanent_semaphore_wait) {
+         for (uint32_t i = 0; i < submit->wait_count; i++) {
+            if ((submit->waits[i].sync->flags & VK_SYNC_IS_TIMELINE) ||
+                submit->_wait_temps[i] != NULL)
+               continue;
+
+            bool was_signaled = false;
+            for (uint32_t j = 0; j < submit->signal_count; j++) {
+               if (submit->signals[j].sync == submit->waits[i].sync) {
+                  was_signaled = true;
+                  break;
+               }
+            }
+
+            if (!was_signaled) {
+               result = vk_sync_reset(queue->base.device,
+                                      submit->waits[i].sync);
+               if (unlikely(result != VK_SUCCESS))
+                  goto fail;
+            }
+         }
+      }
+
+      vk_queue_submit_destroy(queue, submit);
+      return result;
+
+   case VK_QUEUE_SUBMIT_MODE_DEFERRED:
+      vk_queue_push_submit(queue, submit);
+      return vk_device_flush(queue->base.device);
+
+   case VK_QUEUE_SUBMIT_MODE_THREADED:
       if (has_binary_permanent_semaphore_wait) {
          for (uint32_t i = 0; i < info->wait_count; i++) {
             VK_FROM_HANDLE(vk_semaphore, semaphore,
@@ -917,57 +962,11 @@ vk_queue_submit(struct vk_queue *queue,
       }
 
       return VK_SUCCESS;
-      } else {
-         result = vk_queue_submit_final(queue, submit);
-         if (unlikely(result != VK_SUCCESS))
-            goto fail;
 
-         /* If we don't have a submit thread, we can more directly ensure
+   case VK_QUEUE_SUBMIT_MODE_THREADED_ON_DEMAND:
-          * that binary semaphore payloads get reset.  If we also signal the
+      unreachable("Invalid vk_queue::submit.mode");
-          * vk_sync, then we can consider it to have been both reset and
-          * signaled.  A reset in this case would be wrong because it would
-          * throw away our signal operation.  If we don't signal the vk_sync,
-          * then we need to reset it.
-          */
-         if (has_binary_permanent_semaphore_wait) {
-            for (uint32_t i = 0; i < submit->wait_count; i++) {
-               if ((submit->waits[i].sync->flags & VK_SYNC_IS_TIMELINE) ||
-                   submit->_wait_temps[i] != NULL)
-                  continue;
-
-               bool was_signaled = false;
-               for (uint32_t j = 0; j < submit->signal_count; j++) {
-                  if (submit->signals[j].sync == submit->waits[i].sync) {
-                     was_signaled = true;
-                     break;
    }
-               }
+   unreachable("Invalid submit mode");
-
-               if (!was_signaled) {
-                  result = vk_sync_reset(queue->base.device,
-                                         submit->waits[i].sync);
-                  if (unlikely(result != VK_SUCCESS))
-                     goto fail;
-               }
-            }
-         }
-
-         vk_queue_submit_destroy(queue, submit);
-         return VK_SUCCESS;
-      }
-      unreachable("Should have returned");
-
-   case VK_DEVICE_TIMELINE_MODE_EMULATED:
-      vk_queue_push_submit(queue, submit);
-      return vk_device_flush(queue->base.device);
-
-   case VK_DEVICE_TIMELINE_MODE_NONE:
-   case VK_DEVICE_TIMELINE_MODE_NATIVE:
-      result = vk_queue_submit_final(queue, submit);
-      vk_queue_submit_destroy(queue, submit);
-      return result;
-   }
-   unreachable("Invalid timeline mode");
 
 fail:
    vk_queue_submit_destroy(queue, submit);