From 21aec585c111ef53dce63ab13206a7eb9ee64037 Mon Sep 17 00:00:00 2001
From: Mark Collins <mark@igalia.com>
Date: Wed, 7 Sep 2022 02:56:27 +0000
Subject: [PATCH] tu: Retain allocated CSes in tu_autotune_on_submit

It was determined that a significant part of queue submission
overhead was from allocation/freeing of CSes constantly inside
`tu_autotune_on_submit`. This has been reduced by retaining
instances of `tu_submission_data` with their corresponding
CSes, this results in entirely eliminating that overhead as
resetting a CS is a very cheap operation compared to allocation
or even freeing it wholly.

Signed-off-by: Mark Collins <mark@igalia.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/18461>
---
 src/freedreno/vulkan/tu_autotune.c | 27 ++++++++++++++++++++++++---
 src/freedreno/vulkan/tu_autotune.h |  5 +++++
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/src/freedreno/vulkan/tu_autotune.c b/src/freedreno/vulkan/tu_autotune.c
index 3562813ca8677..3faafa73ff384 100644
--- a/src/freedreno/vulkan/tu_autotune.c
+++ b/src/freedreno/vulkan/tu_autotune.c
@@ -100,8 +100,14 @@ static struct tu_submission_data *
 create_submission_data(struct tu_device *dev, struct tu_autotune *at,
                        uint32_t fence)
 {
-   struct tu_submission_data *submission_data =
-      calloc(1, sizeof(struct tu_submission_data));
+   struct tu_submission_data *submission_data = NULL;
+   if (!list_is_empty(&at->submission_data_pool)) {
+      submission_data = list_first_entry(&at->submission_data_pool,
+                                         struct tu_submission_data, node);
+      list_del(&submission_data->node);
+   } else {
+      submission_data = calloc(1, sizeof(struct tu_submission_data));
+   }
    submission_data->fence = fence;
 
    struct tu_cs* fence_cs = &submission_data->fence_cs;
@@ -120,6 +126,15 @@ create_submission_data(struct tu_device *dev, struct tu_autotune *at,
    return submission_data;
 }
 
+static void
+finish_submission_data(struct tu_autotune *at,
+                       struct tu_submission_data *data)
+{
+   list_del(&data->node);
+   list_addtail(&data->node, &at->submission_data_pool);
+   tu_cs_reset(&data->fence_cs);
+}
+
 static void
 free_submission_data(struct tu_submission_data *data)
 {
@@ -265,7 +280,7 @@ process_results(struct tu_autotune *at, uint32_t current_fence)
       if (fence_before(current_fence, submission_data->fence))
          break;
 
-      free_submission_data(submission_data);
+      finish_submission_data(at, submission_data);
    }
 }
 
@@ -396,6 +411,7 @@ tu_autotune_init(struct tu_autotune *at, struct tu_device *dev)
 
    list_inithead(&at->pending_results);
    list_inithead(&at->pending_submission_data);
+   list_inithead(&at->submission_data_pool);
 
    /* start from 1 because tu6_global::autotune_fence is initialized to 0 */
    at->fence_counter = 1;
@@ -434,6 +450,11 @@ tu_autotune_fini(struct tu_autotune *at, struct tu_device *dev)
       free_submission_data(submission_data);
    }
 
+   list_for_each_entry_safe(struct tu_submission_data, submission_data,
+                            &at->submission_data_pool, node) {
+      free_submission_data(submission_data);
+   }
+
    _mesa_hash_table_destroy(at->ht, NULL);
    u_rwlock_destroy(&at->ht_lock);
 }
diff --git a/src/freedreno/vulkan/tu_autotune.h b/src/freedreno/vulkan/tu_autotune.h
index c47ade8ffc76b..f30338d14cc2d 100644
--- a/src/freedreno/vulkan/tu_autotune.h
+++ b/src/freedreno/vulkan/tu_autotune.h
@@ -74,6 +74,11 @@ struct tu_autotune {
     */
    struct list_head pending_submission_data;
 
+   /**
+    * List of per-submission data that has been finished and can be reused.
+    */
+   struct list_head submission_data_pool;
+
    uint32_t fence_counter;
    uint32_t idx_counter;
 };