iris: Enable u_threaded_context

This implements most of the remaining u_threaded_context support.  Most
of the heavy lifting was done in the previous patches which fixed things
up for the new thread safety requirements.  Only a few things remain.

u_threaded_context support can be disabled via an environment variable:

   GALLIUM_THREAD=0

On Felix's Tigerlake with the GPU at fixed frequency, enabling
u_threaded_context improves performance of several games:

   - Civilization VI: +17%
   - Shadow of Mordor: +6%
   - Bioshock Infinite +6%
   - Xonotic: +6%

Various microbenchmarks improve substantially as well:

   - GfxBench5 gl_driver2: +58%
   - SynMark2 OglBatch6: +54%
   - Piglit drawoverhead: +25%

Reviewed-by: Zoltán Böszörményi <zboszor@gmail.com>
Reviewed-by: Ian Romanick <ian.d.romanick@intel.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8964>

src/gallium/drivers/iris/iris_context.c

@@ -24,6 +24,7 @@
 #include <time.h>
 #include "pipe/p_defines.h"
 #include "pipe/p_state.h"
+#include "util/debug.h"
 #include "util/ralloc.h"
 #include "util/u_inlines.h"
 #include "util/format/u_format.h"
@@ -362,5 +363,15 @@ iris_create_context(struct pipe_screen *pscreen, void *priv, unsigned flags)
    screen->vtbl.init_render_context(&ice->batches[IRIS_BATCH_RENDER]);
    screen->vtbl.init_compute_context(&ice->batches[IRIS_BATCH_COMPUTE]);
 
-   return ctx;
+   if (!(flags & PIPE_CONTEXT_PREFER_THREADED))
+      return ctx;
+
+   /* Clover doesn't support u_threaded_context */
+   if (flags & PIPE_CONTEXT_COMPUTE_ONLY)
+      return ctx;
+
+   return threaded_context_create(ctx, &screen->transfer_pool,
+                                  iris_replace_buffer_storage,
+                                  NULL, /* TODO: asynchronous flushes? */
+                                  &ice->thrctx);
 }

src/gallium/drivers/iris/iris_context.h

@@ -563,6 +563,7 @@ struct iris_border_color_pool {
  */
 struct iris_context {
    struct pipe_context ctx;
+   struct threaded_context *thrctx;
 
    /** A debug callback for KHR_debug output. */
    struct pipe_debug_callback dbg;

src/gallium/drivers/iris/iris_fence.c

@@ -366,6 +366,8 @@ iris_fence_finish(struct pipe_screen *p_screen,
                   struct pipe_fence_handle *fence,
                   uint64_t timeout)
 {
+   ctx = threaded_context_unwrap_sync(ctx);
+
    struct iris_context *ice = (struct iris_context *)ctx;
    struct iris_screen *screen = (struct iris_screen *)p_screen;
 

src/gallium/drivers/iris/iris_resource.c

@@ -1271,6 +1271,8 @@ iris_resource_get_handle(struct pipe_screen *pscreen,
    bool mod_with_aux =
       res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE;
 
+   /* if ctx is ever used, do ctx = threaded_context_unwrap_sync(ctx) */
+
    iris_resource_disable_aux_on_first_query(resource, usage);
 
    struct iris_bo *bo;
@@ -1336,6 +1338,32 @@ resource_is_busy(struct iris_context *ice,
    return busy;
 }
 
+void
+iris_replace_buffer_storage(struct pipe_context *ctx,
+                            struct pipe_resource *p_dst,
+                            struct pipe_resource *p_src)
+{
+   struct iris_screen *screen = (void *) ctx->screen;
+   struct iris_context *ice = (void *) ctx;
+   struct iris_resource *dst = (void *) p_dst;
+   struct iris_resource *src = (void *) p_src;
+
+   assert(memcmp(&dst->surf, &src->surf, sizeof(dst->surf)) == 0);
+
+   struct iris_bo *old_bo = dst->bo;
+
+   /* Swap out the backing storage */
+   iris_bo_reference(src->bo);
+   dst->bo = src->bo;
+
+   /* Rebind the buffer, replacing any state referring to the old BO's
+    * address, and marking state dirty so it's reemitted.
+    */
+   screen->vtbl.rebind_buffer(ice, dst);
+
+   iris_bo_unreference(old_bo);
+}
+
 static void
 iris_invalidate_resource(struct pipe_context *ctx,
                          struct pipe_resource *resource)

src/gallium/drivers/iris/iris_resource.h

@@ -312,6 +312,11 @@ iris_resource_get_clear_color(const struct iris_resource *res,
                               struct iris_bo **clear_color_bo,
                               uint64_t *clear_color_offset);
 
+void iris_replace_buffer_storage(struct pipe_context *ctx,
+                                 struct pipe_resource *dst,
+                                 struct pipe_resource *src);
+
+
 void iris_init_screen_resource_functions(struct pipe_screen *pscreen);
 
 void iris_dirty_for_history(struct iris_context *ice,