From 433262c2546833f1ad078c1ba2d3494f872fc5cd Mon Sep 17 00:00:00 2001
From: Hans-Kristian Arntzen <post@arntzen-software.no>
Date: Mon, 6 Jun 2022 12:53:33 +0200
Subject: [PATCH] tests: Add headless D3D12 RenderDoc capture support.

Signed-off-by: Hans-Kristian Arntzen <post@arntzen-software.no>
---
 tests/d3d12_test_utils.c | 20 ++++++++++++----
 tests/d3d12_test_utils.h | 51 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 4 deletions(-)

diff --git a/tests/d3d12_test_utils.c b/tests/d3d12_test_utils.c
index 00efea4b..4023ecb9 100644
--- a/tests/d3d12_test_utils.c
+++ b/tests/d3d12_test_utils.c
@@ -27,6 +27,10 @@ PFN_D3D12_GET_DEBUG_INTERFACE pfn_D3D12GetDebugInterface;
 const char *vkd3d_test_platform = "other";
 struct vkd3d_test_state_context vkd3d_test_state;
 
+#ifdef _WIN32
+RENDERDOC_API_1_0_0 *renderdoc_api;
+#endif
+
 bool compare_float(float f, float g, int ulps)
 {
     int x, y;
@@ -861,6 +865,7 @@ ID3D12CommandSignature *create_command_signature_(unsigned int line,
 
 bool init_compute_test_context_(unsigned int line, struct test_context *context)
 {
+    D3D12_COMMAND_LIST_TYPE command_list_type = D3D12_COMMAND_LIST_TYPE_COMPUTE;
     ID3D12Device *device;
     HRESULT hr;
 
@@ -873,14 +878,21 @@ bool init_compute_test_context_(unsigned int line, struct test_context *context)
     }
     device = context->device;
 
-    context->queue = create_command_queue_(line, device,
-            D3D12_COMMAND_LIST_TYPE_COMPUTE, D3D12_COMMAND_QUEUE_PRIORITY_NORMAL);
+#ifdef _WIN32
+    begin_renderdoc_capturing(device);
+    /* Workaround RenderDoc bug. It expects a DIRECT command queue to exist. */
+    if (renderdoc_api)
+        command_list_type = D3D12_COMMAND_LIST_TYPE_DIRECT;
+#endif
 
-    hr = ID3D12Device_CreateCommandAllocator(device, D3D12_COMMAND_LIST_TYPE_COMPUTE,
+    context->queue = create_command_queue_(line, device,
+            command_list_type, D3D12_COMMAND_QUEUE_PRIORITY_NORMAL);
+
+    hr = ID3D12Device_CreateCommandAllocator(device, command_list_type,
             &IID_ID3D12CommandAllocator, (void **)&context->allocator);
     ok_(line)(hr == S_OK, "Failed to create command allocator, hr %#x.\n", hr);
 
-    hr = ID3D12Device_CreateCommandList(device, 0, D3D12_COMMAND_LIST_TYPE_COMPUTE,
+    hr = ID3D12Device_CreateCommandList(device, 0, command_list_type,
             context->allocator, NULL, &IID_ID3D12GraphicsCommandList, (void **)&context->list);
     ok_(line)(hr == S_OK, "Failed to create command list, hr %#x.\n", hr);
 
diff --git a/tests/d3d12_test_utils.h b/tests/d3d12_test_utils.h
index 92354f19..7484f833 100644
--- a/tests/d3d12_test_utils.h
+++ b/tests/d3d12_test_utils.h
@@ -19,6 +19,10 @@
 #ifndef __VKD3D_D3D12_TEST_UTILS_H
 #define __VKD3D_D3D12_TEST_UTILS_H
 
+#ifdef _WIN32
+#include "renderdoc_app.h"
+#endif
+
 #define SHADER_BYTECODE(code) {code,sizeof(code)}
 
 #define wait_queue_idle(a, b) wait_queue_idle_(__LINE__, a, b)
@@ -1049,6 +1053,45 @@ static inline void create_render_target_(unsigned int line, struct test_context
         ID3D12Device_CreateRenderTargetView(context->device, *render_target, NULL, *rtv);
 }
 
+/* Utility code for capturing native D3D12 tests, which is why this only covers Win32.
+ * Launch the d3d12.exe test binary from RenderDoc UI.
+ * For Vulkan capturing, use VKD3D_AUTO_CAPTURE_COUNTS and friends instead. */
+#ifdef _WIN32
+extern RENDERDOC_API_1_0_0 *renderdoc_api;
+
+static inline void begin_renderdoc_capturing(ID3D12Device *device)
+{
+    pRENDERDOC_GetAPI get_api;
+    HANDLE renderdoc;
+    FARPROC fn_ptr;
+
+    if (!renderdoc_api)
+    {
+        renderdoc = GetModuleHandleA("renderdoc.dll");
+        if (renderdoc)
+        {
+            fn_ptr = GetProcAddress(renderdoc, "RENDERDOC_GetAPI");
+            if (fn_ptr)
+            {
+                /* Workaround compiler warnings about casting to function pointer. */
+                memcpy(&get_api, &fn_ptr, sizeof(fn_ptr));
+                if (!get_api(eRENDERDOC_API_Version_1_0_0, (void **)&renderdoc_api))
+                    renderdoc_api = NULL;
+            }
+        }
+    }
+
+    if (renderdoc_api)
+        renderdoc_api->StartFrameCapture(device, NULL);
+}
+
+static inline void end_renderdoc_capturing(ID3D12Device *device)
+{
+    if (renderdoc_api)
+        renderdoc_api->EndFrameCapture(device, NULL);
+}
+#endif
+
 #define init_test_context(context, desc) init_test_context_(__LINE__, context, desc)
 static inline bool init_test_context_(unsigned int line, struct test_context *context,
         const struct test_context_desc *desc)
@@ -1066,6 +1109,10 @@ static inline bool init_test_context_(unsigned int line, struct test_context *co
     }
     device = context->device;
 
+#ifdef _WIN32
+    begin_renderdoc_capturing(device);
+#endif
+
     context->queue = create_command_queue_(line, device, D3D12_COMMAND_LIST_TYPE_DIRECT, D3D12_COMMAND_QUEUE_PRIORITY_NORMAL);
 
     hr = ID3D12Device_CreateCommandAllocator(device, D3D12_COMMAND_LIST_TYPE_DIRECT,
@@ -1117,6 +1164,10 @@ static inline void destroy_test_context_(unsigned int line, struct test_context
 {
     ULONG refcount;
 
+#ifdef _WIN32
+    end_renderdoc_capturing(context->device);
+#endif
+
     if (context->pipeline_state)
         ID3D12PipelineState_Release(context->pipeline_state);
     if (context->root_signature)