vkd3d: Workaround broken barriers in DEATHLOOP.

In DEATHLOOP, there is a render pass which renders out a simple image,
which is then directly followed by a compute dispatch, reading that
image. The image is still in RENDER_TARGET state, and color buffers are
*not* flushed properly on at least RADV, manifesting as a very
distracting glitch pattern. This is a game bug, but for the time being,
we have to workaround it, *sigh*.

For a simple workaround, we can detect patterns where we see these
events in succession:

- Color RT is started
- StateBefore == RENDER_TARGET is not observed
- Dispatch()

In particular, when entering the options menu, highly distracting
glitches are observed in the background.

Signed-off-by: Hans-Kristian Arntzen <post@arntzen-software.no>

include/vkd3d.h

@@ -76,6 +76,7 @@ enum vkd3d_config_flags
     VKD3D_CONFIG_FLAG_FORCE_HOST_CACHED = 0x00002000,
     VKD3D_CONFIG_FLAG_DXR11 = 0x00004000,
     VKD3D_CONFIG_FLAG_FORCE_NO_INVARIANT_POSITION = 0x00008000,
+    VKD3D_CONFIG_FLAG_WORKAROUND_MISSING_COLOR_COMPUTE_BARRIERS = 0x00010000,
 };
 
 typedef HRESULT (*PFN_vkd3d_signal_event)(HANDLE event);

libs/vkd3d/command.c

@@ -2891,6 +2891,8 @@ static void d3d12_command_list_emit_render_pass_transition(struct d3d12_command_
             stage_mask |= new_stages;
             j++;
         }
+
+        list->workaround_state.has_pending_color_write = true;
     }
 
     dsv = &list->dsv;
@@ -4289,6 +4291,8 @@ static void d3d12_command_list_reset_api_state(struct d3d12_command_list *list,
     list->cbv_srv_uav_descriptors = NULL;
     list->vrs_image = NULL;
 
+    list->workaround_state.has_pending_color_write = false;
+
     ID3D12GraphicsCommandList_SetPipelineState(iface, initial_pipeline_state);
 }
 
@@ -5530,6 +5534,31 @@ static void STDMETHODCALLTYPE d3d12_command_list_DrawIndexedInstanced(d3d12_comm
         VK_CALL(vkCmdDrawIndexedIndirect(list->vk_command_buffer, scratch.buffer, scratch.offset, 1, 0));
 }
 
+static void d3d12_command_list_workaround_handle_missing_color_compute_barriers(struct d3d12_command_list *list)
+{
+    const struct vkd3d_vk_device_procs *vk_procs = &list->device->vk_procs;
+    VkMemoryBarrier barrier;
+
+    if (list->workaround_state.has_pending_color_write &&
+            (vkd3d_config_flags & VKD3D_CONFIG_FLAG_WORKAROUND_MISSING_COLOR_COMPUTE_BARRIERS))
+    {
+        /* Very specifically, every render pass which writes color sets
+         * has_pending_color_write = true.
+         * If we observe a StateBefore == RENDER_TARGET, clear the flag.
+         * If we come to a plain dispatch with this flag set,
+         * insert a simple memory barrier. The image will still have wrong layout,
+         * but this is good enough to workaround the game bug. */
+        barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
+        barrier.pNext = NULL;
+        barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
+        barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
+        VK_CALL(vkCmdPipelineBarrier(list->vk_command_buffer, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+                VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &barrier, 0, NULL, 0, NULL));
+        list->workaround_state.has_pending_color_write = false;
+        WARN("Injecting workaround barrier!\n");
+    }
+}
+
 static void STDMETHODCALLTYPE d3d12_command_list_Dispatch(d3d12_command_list_iface *iface,
         UINT x, UINT y, UINT z)
 {
@@ -5556,6 +5585,8 @@ static void STDMETHODCALLTYPE d3d12_command_list_Dispatch(d3d12_command_list_ifa
         return;
     }
 
+    d3d12_command_list_workaround_handle_missing_color_compute_barriers(list);
+
     if (!list->predicate_va)
         VK_CALL(vkCmdDispatch(list->vk_command_buffer, x, y, z));
     else
@@ -7010,6 +7041,9 @@ static void STDMETHODCALLTYPE d3d12_command_list_ResourceBarrier(d3d12_command_l
                 VkImageLayout old_layout = VK_IMAGE_LAYOUT_UNDEFINED;
                 VkImageLayout new_layout = VK_IMAGE_LAYOUT_UNDEFINED;
 
+                if (transition->StateBefore == D3D12_RESOURCE_STATE_RENDER_TARGET)
+                    list->workaround_state.has_pending_color_write = false;
+
                 if (!is_valid_resource_state(transition->StateBefore))
                 {
                     d3d12_command_list_mark_as_invalid(list,

libs/vkd3d/device.c

@@ -480,7 +480,9 @@ struct vkd3d_instance_application_meta
 static const struct vkd3d_instance_application_meta application_override[] = {
     /* MSVC fails to compile empty array. */
     { VKD3D_STRING_COMPARE_EXACT, "GravityMark.exe", VKD3D_CONFIG_FLAG_FORCE_MINIMUM_SUBGROUP_SIZE, 0 },
-    { VKD3D_STRING_COMPARE_EXACT, "Deathloop.exe", VKD3D_CONFIG_FLAG_IGNORE_RTV_HOST_VISIBLE, 0 },
+    /* The game forgets to do a barrier when going from render pass to compute. */
+    { VKD3D_STRING_COMPARE_EXACT, "Deathloop.exe",
+            VKD3D_CONFIG_FLAG_IGNORE_RTV_HOST_VISIBLE | VKD3D_CONFIG_FLAG_WORKAROUND_MISSING_COLOR_COMPUTE_BARRIERS, 0 },
     /* Shadow of the Tomb Raider (750920).
      * Invariant workarounds actually cause more issues than they resolve on NV.
      * RADV already has workarounds by default.

libs/vkd3d/vkd3d_private.h

@@ -1975,6 +1975,14 @@ struct d3d12_command_list
     size_t dsv_resource_tracking_count;
     size_t dsv_resource_tracking_size;
 
+    /* Hackery needed for game workarounds. */
+    struct
+    {
+        /* Used to keep track of COLOR write -> COMPUTE where game forget to insert barrier
+         * before the dispatch. */
+        bool has_pending_color_write;
+    } workaround_state;
+
     struct vkd3d_private_store private_store;
 };