[dxvk] Implement asynchronous presentation

Off-loads the vkQueuePresentKHR call to the queue submission thread
to avoid synchronization with that thread on a present call.

src/d3d11/d3d11_swapchain.cpp

@@ -306,8 +306,10 @@ namespace dxvk {
         m_context->endRecording(),
         sync.acquire, sync.present);
       
-      status = m_device->presentImage(
-        m_presenter, sync.present);
+      m_device->presentImage(m_presenter,
+        sync.present, &m_presentStatus);
+      
+      status = m_device->waitForSubmission(&m_presentStatus);
       
       if (status != VK_SUCCESS)
         RecreateSwapChain(m_vsync);

src/d3d11/d3d11_swapchain.h

@@ -113,6 +113,8 @@ namespace dxvk {
 
     D3D11Texture2D*         m_backBuffer = nullptr;
 
+    DxvkSubmitStatus        m_presentStatus;
+
     std::vector<Rc<DxvkImageView>> m_imageViews;
 
     bool                    m_dirty = true;

src/dxvk/dxvk_device.cpp

@@ -206,20 +206,19 @@ namespace dxvk {
   }
   
   
-  VkResult DxvkDevice::presentImage(
+  void DxvkDevice::presentImage(
     const Rc<vk::Presenter>&        presenter,
-          VkSemaphore               semaphore) {
+          VkSemaphore               semaphore,
+          DxvkSubmitStatus*         status) {
+    status->result = VK_NOT_READY;
+
     DxvkPresentInfo presentInfo;
     presentInfo.presenter = presenter;
     presentInfo.waitSync  = semaphore;
-    VkResult status = m_submissionQueue.present(presentInfo);
-
-    if (status != VK_SUCCESS)
-      return status;
+    m_submissionQueue.present(presentInfo, status);
     
     std::lock_guard<sync::Spinlock> statLock(m_statLock);
     m_statCounters.addCtr(DxvkStatCounter::QueuePresentCount, 1);
-    return status;
   }
 
 
@@ -239,6 +238,18 @@ namespace dxvk {
   }
   
   
+  VkResult DxvkDevice::waitForSubmission(DxvkSubmitStatus* status) {
+    VkResult result = status->result.load();
+
+    if (result == VK_NOT_READY) {
+      m_submissionQueue.synchronizeSubmission(status);
+      result = status->result.load();
+    }
+
+    return result;
+  }
+  
+  
   void DxvkDevice::waitForIdle() {
     m_submissionQueue.synchronize();
 

src/dxvk/dxvk_device.h

@@ -328,15 +328,17 @@ namespace dxvk {
     /**
      * \brief Presents a swap chain image
      * 
-     * Locks the device queues and invokes the
-     * presenter's \c presentImage method.
+     * Invokes the presenter's \c presentImage method on
+     * the submission thread. The status of this operation
+     * can be retrieved with \ref waitForSubmission.
      * \param [in] presenter The presenter
      * \param [in] semaphore Sync semaphore
-     * \returns Status of the operation
+     * \param [out] status Present status
      */
-    VkResult presentImage(
+    void presentImage(
       const Rc<vk::Presenter>&        presenter,
-            VkSemaphore               semaphore);
+            VkSemaphore               semaphore,
+            DxvkSubmitStatus*         status);
     
     /**
      * \brief Submits a command list
@@ -384,6 +386,14 @@ namespace dxvk {
     uint32_t pendingSubmissions() const {
       return m_submissionQueue.pendingSubmissions();
     }
+
+    /**
+     * \brief Waits for a given submission
+     * 
+     * \param [in,out] status Submission status
+     * \returns Result of the submission
+     */
+    VkResult waitForSubmission(DxvkSubmitStatus* status);
     
     /**
      * \brief Waits until the device becomes idle

src/dxvk/dxvk_queue.cpp

@@ -31,17 +31,34 @@ namespace dxvk {
       return m_submitQueue.size() + m_finishQueue.size() <= MaxNumQueuedCommandBuffers;
     });
 
+    DxvkSubmitEntry entry = { };
+    entry.submit = std::move(submitInfo);
+
     m_pending += 1;
-    m_submitQueue.push(std::move(submitInfo));
+    m_submitQueue.push(std::move(entry));
     m_appendCond.notify_all();
   }
 
 
-  VkResult DxvkSubmissionQueue::present(DxvkPresentInfo presentInfo) {
-    this->synchronize();
-    
-    std::unique_lock<std::mutex> lock(m_mutexQueue);
-    return presentInfo.presenter->presentImage(presentInfo.waitSync);
+  void DxvkSubmissionQueue::present(DxvkPresentInfo presentInfo, DxvkSubmitStatus* status) {
+    std::unique_lock<std::mutex> lock(m_mutex);
+
+    DxvkSubmitEntry entry = { };
+    entry.status  = status;
+    entry.present = std::move(presentInfo);
+
+    m_submitQueue.push(std::move(entry));
+    m_appendCond.notify_all();
+  }
+
+
+  void DxvkSubmissionQueue::synchronizeSubmission(
+          DxvkSubmitStatus*   status) {
+    std::unique_lock<std::mutex> lock(m_mutex);
+
+    m_submitCond.wait(lock, [status] {
+      return status->result.load() != VK_NOT_READY;
+    });
   }
 
 
@@ -77,32 +94,39 @@ namespace dxvk {
       if (m_stopped.load())
         return;
       
-      DxvkSubmitInfo submitInfo = std::move(m_submitQueue.front());
+      DxvkSubmitEntry entry = std::move(m_submitQueue.front());
       lock.unlock();
 
       // Submit command buffer to device
-      VkResult status;
+      VkResult status = VK_NOT_READY;
 
       { std::lock_guard<std::mutex> lock(m_mutexQueue);
 
-        status = submitInfo.cmdList->submit(
-          submitInfo.waitSync,
-          submitInfo.wakeSync);
+        if (entry.submit.cmdList != nullptr) {
+          status = entry.submit.cmdList->submit(
+            entry.submit.waitSync,
+            entry.submit.wakeSync);
+        } else if (entry.present.presenter != nullptr) {
+          status = entry.present.presenter->presentImage(
+            entry.present.waitSync);
+        }
       }
+
+      if (entry.status)
+        entry.status->result = status;
       
       // On success, pass it on to the queue thread
       lock = std::unique_lock<std::mutex>(m_mutex);
 
       if (status == VK_SUCCESS) {
-        m_finishQueue.push(std::move(submitInfo));
-        m_submitQueue.pop();
-        m_submitCond.notify_all();
+        if (entry.submit.cmdList != nullptr)
+          m_finishQueue.push(std::move(entry));
       } else {
-        Logger::err(str::format(
-          "DxvkSubmissionQueue: Command submission failed with ",
-          status));
-        m_pending -= 1;
+        Logger::err(str::format("DxvkSubmissionQueue: Command submission failed: ", status));
       }
+
+      m_submitQueue.pop();
+      m_submitCond.notify_all();
     }
   }
   
@@ -120,16 +144,16 @@ namespace dxvk {
       if (m_stopped.load())
         return;
       
-      DxvkSubmitInfo submitInfo = std::move(m_finishQueue.front());
+      DxvkSubmitEntry entry = std::move(m_finishQueue.front());
       lock.unlock();
       
-      VkResult status = submitInfo.cmdList->synchronize();
+      VkResult status = entry.submit.cmdList->synchronize();
       
       if (status == VK_SUCCESS) {
-        submitInfo.cmdList->signalEvents();
-        submitInfo.cmdList->reset();
+        entry.submit.cmdList->signalEvents();
+        entry.submit.cmdList->reset();
         
-        m_device->recycleCommandList(submitInfo.cmdList);
+        m_device->recycleCommandList(entry.submit.cmdList);
       } else {
         Logger::err(str::format(
           "DxvkSubmissionQueue: Failed to sync fence: ",

src/dxvk/dxvk_queue.h

@@ -14,6 +14,17 @@ namespace dxvk {
   
   class DxvkDevice;
 
+  /**
+   * \brief Submission status
+   * 
+   * Stores the result of a queue
+   * submission or a present call.
+   */
+  struct DxvkSubmitStatus {
+    std::atomic<VkResult> result = { VK_SUCCESS };
+  };
+
+
   /**
    * \brief Queue submission info
    * 
@@ -39,6 +50,16 @@ namespace dxvk {
   };
 
 
+  /**
+   * \brief Submission queue entry
+   */
+  struct DxvkSubmitEntry {
+    DxvkSubmitStatus*   status;
+    DxvkSubmitInfo      submit;
+    DxvkPresentInfo     present;
+  };
+
+
   /**
    * \brief Submission queue
    */
@@ -69,7 +90,7 @@ namespace dxvk {
      * \param [in] submitInfo Submission parameters 
      */
     void submit(
-            DxvkSubmitInfo  submitInfo);
+            DxvkSubmitInfo      submitInfo);
     
     /**
      * \brief Presents an image synchronously
@@ -80,8 +101,19 @@ namespace dxvk {
      * \param [in] present Present parameters
      * \returns Status of the operation
      */
-    VkResult present(
-            DxvkPresentInfo present);
+    void present(
+            DxvkPresentInfo     presentInfo,
+            DxvkSubmitStatus*   status);
+    
+    /**
+     * \brief Synchronizes with one queue submission
+     * 
+     * Waits for the result of the given submission
+     * or present operation to become available.
+     * \param [in,out] status Submission status
+     */
+    void synchronizeSubmission(
+            DxvkSubmitStatus*   status);
     
     /**
      * \brief Synchronizes with queue submissions
@@ -123,8 +155,8 @@ namespace dxvk {
     std::condition_variable m_submitCond;
     std::condition_variable m_finishCond;
 
-    std::queue<DxvkSubmitInfo> m_submitQueue;
-    std::queue<DxvkSubmitInfo> m_finishQueue;
+    std::queue<DxvkSubmitEntry> m_submitQueue;
+    std::queue<DxvkSubmitEntry> m_finishQueue;
 
     dxvk::thread            m_submitThread;
     dxvk::thread            m_finishThread;