zink: emit some barriers out of renderpass where possible

when emitting barriers, we don't need to end the renderpass in some cases

this handles the case of emitting barriers for new resources which have
never before been accessed and thus do not require synchronization at a
specific point in the batch

Reviewed-by: Dave Airlie <airlied@redhat.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/10700>

src/gallium/drivers/zink/zink_batch.c

@@ -96,6 +96,7 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs)
     * before the state is reused
     */
    bs->fence.submitted = false;
+   bs->has_barriers = false;
    zink_screen_update_last_finished(screen, bs->fence.batch_id);
    bs->fence.batch_id = 0;
    bs->work_count[0] = bs->work_count[1] = 0;
@@ -135,6 +136,8 @@ zink_batch_state_destroy(struct zink_screen *screen, struct zink_batch_state *bs
 
    if (bs->cmdbuf)
       vkFreeCommandBuffers(screen->dev, bs->cmdpool, 1, &bs->cmdbuf);
+   if (bs->barrier_cmdbuf)
+      vkFreeCommandBuffers(screen->dev, bs->cmdpool, 1, &bs->barrier_cmdbuf);
    if (bs->cmdpool)
       vkDestroyCommandPool(screen->dev, bs->cmdpool, NULL);
 
@@ -171,6 +174,9 @@ create_batch_state(struct zink_context *ctx)
    if (vkAllocateCommandBuffers(screen->dev, &cbai, &bs->cmdbuf) != VK_SUCCESS)
       goto fail;
 
+   if (vkAllocateCommandBuffers(screen->dev, &cbai, &bs->barrier_cmdbuf) != VK_SUCCESS)
+      goto fail;
+
 #define SET_CREATE_OR_FAIL(ptr) \
    ptr = _mesa_pointer_set_create(bs); \
    if (!ptr) \
@@ -279,6 +285,8 @@ zink_start_batch(struct zink_context *ctx, struct zink_batch *batch)
    cbbi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
    if (vkBeginCommandBuffer(batch->state->cmdbuf, &cbbi) != VK_SUCCESS)
       debug_printf("vkBeginCommandBuffer failed\n");
+   if (vkBeginCommandBuffer(batch->state->barrier_cmdbuf, &cbbi) != VK_SUCCESS)
+      debug_printf("vkBeginCommandBuffer failed\n");
 
    batch->state->fence.batch_id = ctx->curr_batch;
    batch->state->fence.completed = false;
@@ -317,8 +325,12 @@ submit_queue(void *data, int thread_index)
    si.signalSemaphoreCount = 0;
    si.pSignalSemaphores = NULL;
    si.pWaitDstStageMask = NULL;
-   si.commandBufferCount = 1;
-   si.pCommandBuffers = &bs->cmdbuf;
+   si.commandBufferCount = bs->has_barriers ? 2 : 1;
+   VkCommandBuffer cmdbufs[2] = {
+      bs->barrier_cmdbuf,
+      bs->cmdbuf,
+   };
+   si.pCommandBuffers = bs->has_barriers ? cmdbufs : &cmdbufs[1];
 
    VkTimelineSemaphoreSubmitInfo tsi = {};
    if (bs->have_timelines) {
@@ -485,6 +497,10 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch)
       debug_printf("vkEndCommandBuffer failed\n");
       return;
    }
+   if (vkEndCommandBuffer(batch->state->barrier_cmdbuf) != VK_SUCCESS) {
+      debug_printf("vkEndCommandBuffer failed\n");
+      return;
+   }
    vkResetFences(zink_screen(ctx->base.screen)->dev, 1, &batch->state->fence.fence);
 
    struct zink_screen *screen = zink_screen(ctx->base.screen);

src/gallium/drivers/zink/zink_batch.h

@@ -55,6 +55,8 @@ struct zink_batch_state {
    struct zink_context *ctx;
    VkCommandPool cmdpool;
    VkCommandBuffer cmdbuf;
+   VkCommandBuffer barrier_cmdbuf;
+   bool has_barriers;
 
    VkQueue queue; //duplicated from batch for threading
    VkSemaphore sem;

src/gallium/drivers/zink/zink_context.c

@@ -1803,6 +1803,20 @@ zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource
    return zink_resource_image_needs_barrier(res, new_layout, flags, pipeline);
 }
 
+static inline VkCommandBuffer
+get_cmdbuf(struct zink_context *ctx, struct zink_resource *res)
+{
+   if ((res->access && !res->unordered_barrier) || !ctx->batch.in_rp) {
+      struct zink_batch *batch = zink_batch_no_rp(ctx);
+      assert(!batch->in_rp);
+      res->unordered_barrier = false;
+      return batch->state->cmdbuf;
+   }
+   res->unordered_barrier = true;
+   ctx->batch.state->has_barriers = true;
+   return ctx->batch.state->barrier_cmdbuf;
+}
+
 void
 zink_resource_image_barrier(struct zink_context *ctx, struct zink_batch *batch, struct zink_resource *res,
                       VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
@@ -1813,11 +1827,10 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_batch *batch,
    if (!pipeline)
       pipeline = pipeline_dst_stage(new_layout);
    /* only barrier if we're changing layout or doing something besides read -> read */
-   batch = zink_batch_no_rp(ctx);
-   assert(!batch->in_rp);
+   VkCommandBuffer cmdbuf = get_cmdbuf(ctx, res);
    assert(new_layout);
    vkCmdPipelineBarrier(
-      batch->state->cmdbuf,
+      cmdbuf,
       res->access_stage ? res->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
       pipeline,
       0,
@@ -1826,9 +1839,14 @@ zink_resource_image_barrier(struct zink_context *ctx, struct zink_batch *batch,
       1, &imb
    );
 
+   if (res->unordered_barrier) {
+      res->access |= imb.dstAccessMask;
+      res->access_stage |= pipeline;
+   } else {
+      res->access = imb.dstAccessMask;
+      res->access_stage = pipeline;
+   }
    res->layout = new_layout;
-   res->access_stage = pipeline;
-   res->access = imb.dstAccessMask;
 }
 
 
@@ -1908,11 +1926,13 @@ zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_batch *batch,
       return;
    if (!pipeline)
       pipeline = pipeline_access_stage(flags);
+   if (!zink_resource_buffer_needs_barrier(res, flags, pipeline))
+      return;
+
+   VkCommandBuffer cmdbuf = get_cmdbuf(ctx, res);
    /* only barrier if we're changing layout or doing something besides read -> read */
-   batch = zink_batch_no_rp(ctx);
-   assert(!batch->in_rp);
    vkCmdPipelineBarrier(
-      batch->state->cmdbuf,
+      cmdbuf,
       res->access_stage ? res->access_stage : pipeline_access_stage(res->access),
       pipeline,
       0,
@@ -1920,8 +1940,14 @@ zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_batch *batch,
       1, &bmb,
       0, NULL
    );
-   res->access = bmb.dstAccessMask;
-   res->access_stage = pipeline;
+
+   if (res->unordered_barrier) {
+      res->access |= bmb.dstAccessMask;
+      res->access_stage |= pipeline;
+   } else {
+      res->access = bmb.dstAccessMask;
+      res->access_stage = pipeline;
+   }
 }
 
 bool
@@ -2806,6 +2832,7 @@ zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resou
    zink_resource_object_reference(zink_screen(pctx->screen), &d->obj, s->obj);
    d->access = s->access;
    d->access_stage = s->access_stage;
+   d->unordered_barrier = s->unordered_barrier;
    zink_resource_rebind(zink_context(pctx), d);
 }
 

src/gallium/drivers/zink/zink_resource.c

@@ -718,6 +718,7 @@ invalidate_buffer(struct zink_context *ctx, struct zink_resource *res)
    res->obj = new_obj;
    res->access_stage = 0;
    res->access = 0;
+   res->unordered_barrier = false;
    zink_resource_rebind(ctx, res);
    zink_descriptor_set_refs_clear(&old_obj->desc_set_refs, old_obj);
    zink_resource_object_reference(screen, &old_obj, NULL);

src/gallium/drivers/zink/zink_resource.h

@@ -86,6 +86,7 @@ struct zink_resource {
 
    VkPipelineStageFlagBits access_stage;
    VkAccessFlags access;
+   bool unordered_barrier;
 
    struct zink_resource_object *obj;
    struct zink_resource_object *scanout_obj; //TODO: remove for wsi