vulkan: Copy the depth/stencil state optimization code from ANV

Instead of having stencil writes as an out parameter of the optimization
function, we add a new write_enable field for stencil that's equivalent
to the similarly named field for depth.  This doesn't mean drivers must
actually support disabling stencil writes independently but the
information may be helpful on some hardware.

Reviewed-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/17328>

docs/vulkan/graphics-state.rst

@@ -180,6 +180,12 @@ use a state even if it isn't dirty on this particular draw.
 .. doxygenfunction:: vk_dynamic_graphics_state_any_dirty
 
 
+Depth stencil state optimization
+--------------------------------
+
+.. doxygenfunction:: vk_optimize_depth_stencil_state
+
+
 Reference
 ---------
 

src/vulkan/runtime/vk_graphics_state.c

@@ -654,6 +654,7 @@ vk_depth_stencil_state_init(struct vk_depth_stencil_state *ds,
    ds->depth.bounds_test.max = ds_info->maxDepthBounds;
 
    ds->stencil.test_enable = ds_info->stencilTestEnable;
+   ds->stencil.write_enable = true;
    vk_stencil_test_face_state_init(&ds->stencil.front, &ds_info->front);
    vk_stencil_test_face_state_init(&ds->stencil.back, &ds_info->back);
 }
@@ -666,6 +667,100 @@ vk_dynamic_graphics_state_init_ds(struct vk_dynamic_graphics_state *dst,
    dst->ds = *ds;
 }
 
+static bool
+optimize_stencil_face(struct vk_stencil_test_face_state *face,
+                      VkCompareOp depthCompareOp)
+{
+   /* If compareOp is ALWAYS then the stencil test will never fail and failOp
+    * will never happen.  Set failOp to KEEP in this case.
+    */
+   if (face->op.compare == VK_COMPARE_OP_ALWAYS)
+      face->op.fail = VK_STENCIL_OP_KEEP;
+
+   /* If compareOp is NEVER or depthCompareOp is NEVER then one of the depth
+    * or stencil tests will fail and passOp will never happen.
+    */
+   if (face->op.compare == VK_COMPARE_OP_NEVER ||
+       depthCompareOp == VK_COMPARE_OP_NEVER)
+      face->op.pass = VK_STENCIL_OP_KEEP;
+
+   /* If compareOp is NEVER or depthCompareOp is ALWAYS then either the
+    * stencil test will fail or the depth test will pass.  In either case,
+    * depthFailOp will never happen.
+    */
+   if (face->op.compare == VK_COMPARE_OP_NEVER ||
+       depthCompareOp == VK_COMPARE_OP_ALWAYS)
+      face->op.depth_fail = VK_STENCIL_OP_KEEP;
+
+   return face->op.fail != VK_STENCIL_OP_KEEP ||
+          face->op.depth_fail != VK_STENCIL_OP_KEEP ||
+          face->op.pass != VK_STENCIL_OP_KEEP;
+}
+
+void
+vk_optimize_depth_stencil_state(struct vk_depth_stencil_state *ds,
+                                VkImageAspectFlags ds_aspects)
+{
+   /* stencil.write_enable is a dummy right now that should always be true */
+   assert(ds->stencil.write_enable);
+
+   /* If the depth test is disabled, we won't be writing anything. Make sure we
+    * treat the test as always passing later on as well.
+    *
+    * Also, the Vulkan spec requires that if either depth or stencil is not
+    * present, the pipeline is to act as if the test silently passes. In that
+    * case we won't write either.
+    */
+   if (!ds->depth.test_enable || !(ds_aspects & VK_IMAGE_ASPECT_DEPTH_BIT)) {
+      ds->depth.write_enable = false;
+      ds->depth.compare_op = VK_COMPARE_OP_ALWAYS;
+   }
+
+   if (!(ds_aspects & VK_IMAGE_ASPECT_STENCIL_BIT)) {
+      ds->stencil.write_enable = false;
+      ds->stencil.front.op.compare = VK_COMPARE_OP_ALWAYS;
+      ds->stencil.back.op.compare = VK_COMPARE_OP_ALWAYS;
+   }
+
+   /* If the stencil test is enabled and always fails, then we will never get
+    * to the depth test so we can just disable the depth test entirely.
+    */
+   if (ds->stencil.test_enable &&
+       ds->stencil.front.op.compare == VK_COMPARE_OP_NEVER &&
+       ds->stencil.back.op.compare == VK_COMPARE_OP_NEVER) {
+      ds->depth.test_enable = false;
+      ds->depth.write_enable = false;
+   }
+
+   /* If depthCompareOp is EQUAL then the value we would be writing to the
+    * depth buffer is the same as the value that's already there so there's no
+    * point in writing it.
+    */
+   if (ds->depth.compare_op == VK_COMPARE_OP_EQUAL)
+      ds->depth.write_enable = false;
+
+   /* If the stencil ops are such that we don't actually ever modify the
+    * stencil buffer, we should disable writes.
+    */
+   if (!optimize_stencil_face(&ds->stencil.front, ds->depth.compare_op) &&
+       !optimize_stencil_face(&ds->stencil.back, ds->depth.compare_op))
+      ds->stencil.write_enable = false;
+
+   /* If the depth test always passes and we never write out depth, that's the
+    * same as if the depth test is disabled entirely.
+    */
+   if (ds->depth.compare_op == VK_COMPARE_OP_ALWAYS && !ds->depth.write_enable)
+      ds->depth.test_enable = false;
+
+   /* If the stencil test always passes and we never write out stencil, that's
+    * the same as if the stencil test is disabled entirely.
+    */
+   if (ds->stencil.front.op.compare == VK_COMPARE_OP_ALWAYS &&
+       ds->stencil.back.op.compare == VK_COMPARE_OP_ALWAYS &&
+       !ds->stencil.write_enable)
+      ds->stencil.test_enable = false;
+}
+
 static void
 vk_color_blend_state_init(struct vk_color_blend_state *cb,
                           const BITSET_WORD *dynamic,
@@ -1232,6 +1327,7 @@ const struct vk_dynamic_graphics_state vk_default_dynamic_graphics_state = {
          },
       },
       .stencil = {
+         .write_enable = true,
          .front = {
             .compare_mask = -1,
             .write_mask = -1,

src/vulkan/runtime/vk_graphics_state.h

@@ -382,6 +382,16 @@ struct vk_depth_stencil_state {
        */
       bool test_enable;
 
+      /** Whether or not stencil is should be written
+       *
+       * This does not map directly to any particular Vulkan API state and is
+       * initialized to true.  If independent stencil disable ever becomes a
+       * thing, it will use this state.  vk_optimize_depth_stencil_state() may
+       * set this to false if it can prove that the stencil test will never
+       * alter the stencil value.
+       */
+      bool write_enable;
+
       /** VkPipelineDepthStencilStateCreateInfo::front */
       struct vk_stencil_test_face_state front;
 
@@ -390,6 +400,23 @@ struct vk_depth_stencil_state {
    } stencil;
 };
 
+/** Optimize a depth/stencil state
+ *
+ * The way depth and stencil testing is specified, there are many case where,
+ * regardless of depth/stencil writes being enabled, nothing actually gets
+ * written due to some other bit of state being set.  In the presence of
+ * discards, it's fairly easy to get into cases where early depth/stencil
+ * testing is disabled on some hardware, leading to a fairly big performance
+ * hit.  This function attempts to optimize the depth stencil state and
+ * disable writes and sometimes even testing whenever possible.
+ *
+ * @param[inout]  ds          The depth stencil state to optimize
+ * @param[in]     ds_aspects  Which image aspects are present in the render
+ *                            pass.
+ */
+void vk_optimize_depth_stencil_state(struct vk_depth_stencil_state *ds,
+                                     VkImageAspectFlags ds_aspects);
+
 struct vk_color_blend_attachment_state {
    /** VkPipelineColorBlendAttachmentState::blendEnable */
    bool blend_enable;