gallium: implement compute pbo download

this reworks PIPE_CAP_PREFER_BLIT_BASED_TEXTURE_TRANSFER into an
enum as PIPE_CAP_TEXTURE_TRANSFER_MODES, enabling drivers to choose
a (sometimes) faster, compute-based download mechanism based on a new
pipe_screen hook

compute pbo download is implemented using shaders with a prolog to convert
the input format to generic rgb float values, then an epilog to convert
to the output value. the prolog and epilog are determined based on a vec4
of packed ubo data which is dynamically updated based on the API usage

currently, the only known limitations are:
* GL_ARB_texture_cube_map_array is broken somehow (and disabled)
* AMD hardware somehow can't do depth readback?

otherwise it should work for every possible case

Acked-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>

Acked-by: Marek Olšák <marek.olsak@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11984>

docs/gallium/screen.rst

@@ -181,7 +181,9 @@ The integer capabilities:
   that are supported for implementing a texture transfer which needs format conversions
   and swizzling in gallium frontends. Generally, all hardware drivers with
   dedicated memory should return PIPE_TEXTURE_TRANSFER_BLIT and all software rasterizers
-  should return PIPE_TEXTURE_TRANSFER_DEFAULT.
+  should return PIPE_TEXTURE_TRANSFER_DEFAULT. PIPE_TEXTURE_TRANSFER_COMPUTE requires drivers
+  to support 8bit and 16bit shader storage buffer writes and to implement
+  pipe_screen::is_compute_copy_faster.
 * ``PIPE_CAP_QUERY_PIPELINE_STATISTICS``: Whether PIPE_QUERY_PIPELINE_STATISTICS
   is supported.
 * ``PIPE_CAP_TEXTURE_BORDER_COLOR_QUIRK``: Bitmask indicating whether special

src/gallium/include/pipe/p_defines.h

@@ -1007,6 +1007,7 @@ enum pipe_cap
 enum pipe_texture_transfer_mode {
    PIPE_TEXTURE_TRANSFER_DEFAULT = 0,
    PIPE_TEXTURE_TRANSFER_BLIT = (1 << 0),
+   PIPE_TEXTURE_TRANSFER_COMPUTE = (1 << 1),
 };
 
 /**

src/mesa/meson.build

@@ -587,6 +587,7 @@ files_libmesa_gallium = files(
   'state_tracker/st_nir_lower_builtin.c',
   'state_tracker/st_nir_lower_tex_src_plane.c',
   'state_tracker/st_pbo.c',
+  'state_tracker/st_pbo_compute.c',
   'state_tracker/st_pbo.h',
   'state_tracker/st_program.c',
   'state_tracker/st_program.h',

src/mesa/state_tracker/st_cb_texture.c

@@ -51,6 +51,7 @@
 #include "state_tracker/st_debug.h"
 #include "state_tracker/st_context.h"
 #include "state_tracker/st_cb_bitmap.h"
+#include "state_tracker/st_cb_drawpixels.h"
 #include "state_tracker/st_cb_fbo.h"
 #include "state_tracker/st_cb_flush.h"
 #include "state_tracker/st_cb_texture.h"
@@ -2518,31 +2519,31 @@ st_GetTexSubImage(struct gl_context * ctx,
 
    if (!st->prefer_blit_based_texture_transfer &&
        !_mesa_is_format_compressed(texImage->TexFormat)) {
-      /* Try to avoid the fallback if we're doing texture decompression here */
-      goto fallback;
+      /* Try to avoid the non_blit_transfer if we're doing texture decompression here */
+      goto non_blit_transfer;
    }
 
    /* Handle non-finalized textures. */
    if (!stImage->pt || stImage->pt != stObj->pt || !src) {
-      goto fallback;
+      goto cpu_transfer;
    }
 
    /* XXX Fallback to _mesa_GetTexImage_sw for depth-stencil formats
     * due to an incomplete stencil blit implementation in some drivers. */
    if (format == GL_DEPTH_STENCIL || format == GL_STENCIL_INDEX) {
-      goto fallback;
+      goto non_blit_transfer;
    }
 
    /* If the base internal format and the texture format don't match, we have
     * to fall back to _mesa_GetTexImage_sw. */
    if (texImage->_BaseFormat !=
        _mesa_get_format_base_format(texImage->TexFormat)) {
-      goto fallback;
+      goto non_blit_transfer;
    }
 
    src_format = st_pbo_get_src_format(screen, stObj->surface_based ? stObj->surface_format : src->format, src);
    if (src_format == PIPE_FORMAT_NONE)
-      goto fallback;
+      goto non_blit_transfer;
 
    if (format == GL_DEPTH_COMPONENT || format == GL_DEPTH_STENCIL)
       bind = PIPE_BIND_DEPTH_STENCIL;
@@ -2552,7 +2553,7 @@ st_GetTexSubImage(struct gl_context * ctx,
    dst_format = st_pbo_get_dst_format(ctx, pipe_target, src_format, util_format_is_compressed(src->format),
                                format, type, bind);
    if (dst_format == PIPE_FORMAT_NONE)
-      goto fallback;
+      goto non_blit_transfer;
 
    if (st->pbo.download_enabled && ctx->Pack.BufferObj) {
       if (try_pbo_download(st, texImage,
@@ -2567,11 +2568,11 @@ st_GetTexSubImage(struct gl_context * ctx,
     * in which case the memcpy-based fast path will be used. */
    if (_mesa_format_matches_format_and_type(texImage->TexFormat, format,
                                             type, ctx->Pack.SwapBytes, NULL))
-      goto fallback;
+      goto non_blit_transfer;
 
    dst = create_dst_texture(ctx, dst_format, pipe_target, width, height, depth, gl_target, bind);
    if (!dst)
-      goto fallback;
+      goto non_blit_transfer;
 
    /* From now on, we need the gallium representation of dimensions. */
    if (gl_target == GL_TEXTURE_1D_ARRAY) {
@@ -2612,12 +2613,17 @@ st_GetTexSubImage(struct gl_context * ctx,
                            depth, format, type, pixels, texImage);
    pipe_resource_reference(&dst, NULL);
 
-fallback:
-   if (!done) {
-      _mesa_GetTexSubImage_sw(ctx, xoffset, yoffset, zoffset,
-                              width, height, depth,
-                              format, type, pixels, texImage);
+non_blit_transfer:
+   if (done)
+      return;
+   if (st->allow_compute_based_texture_transfer) {
+      if (st_GetTexSubImage_shader(ctx, xoffset, yoffset, zoffset, width, height, depth, format, type, pixels, texImage))
+         return;
    }
+cpu_transfer:
+   _mesa_GetTexSubImage_sw(ctx, xoffset, yoffset, zoffset,
+                           width, height, depth,
+                           format, type, pixels, texImage);
 }
 
 

src/mesa/state_tracker/st_context.c

@@ -625,6 +625,11 @@ st_create_context_priv(struct gl_context *ctx, struct pipe_context *pipe,
 
    st_init_atoms(st);
    st_init_clear(st);
+   {
+      enum pipe_texture_transfer_mode val = screen->get_param(screen, PIPE_CAP_TEXTURE_TRANSFER_MODES);
+      st->prefer_blit_based_texture_transfer = (val & PIPE_TEXTURE_TRANSFER_BLIT) != 0;
+      st->allow_compute_based_texture_transfer = (val & PIPE_TEXTURE_TRANSFER_COMPUTE) != 0;
+   }
    st_init_pbo_helpers(st);
 
    /* Choose texture target for glDrawPixels, glBitmap, renderbuffers */
@@ -689,10 +694,6 @@ st_create_context_priv(struct gl_context *ctx, struct pipe_context *pipe,
    st->has_astc_5x5_ldr =
       screen->is_format_supported(screen, PIPE_FORMAT_ASTC_5x5_SRGB,
                                   PIPE_TEXTURE_2D, 0, 0, PIPE_BIND_SAMPLER_VIEW);
-   {
-      enum pipe_texture_transfer_mode val = screen->get_param(screen, PIPE_CAP_TEXTURE_TRANSFER_MODES);
-      st->prefer_blit_based_texture_transfer = (val & PIPE_TEXTURE_TRANSFER_BLIT) == PIPE_TEXTURE_TRANSFER_BLIT;
-   }
    st->force_persample_in_shader =
       screen->get_param(screen, PIPE_CAP_SAMPLE_SHADING) &&
       !screen->get_param(screen, PIPE_CAP_FORCE_PERSAMPLE_INTERP);

src/mesa/state_tracker/st_context.h

@@ -147,6 +147,7 @@ struct st_context
    boolean has_astc_2d_ldr;
    boolean has_astc_5x5_ldr;
    boolean prefer_blit_based_texture_transfer;
+   boolean allow_compute_based_texture_transfer;
    boolean force_persample_in_shader;
    boolean has_shareable_shaders;
    boolean has_half_float_packing;
@@ -326,6 +327,7 @@ struct st_context
       void *gs;
       void *upload_fs[5][2];
       void *download_fs[5][PIPE_MAX_TEXTURE_TYPES][2];
+      struct hash_table *shaders;
       bool upload_enabled;
       bool download_enabled;
       bool rgba_only;

src/mesa/state_tracker/st_pbo.c

@@ -661,6 +661,9 @@ st_init_pbo_helpers(struct st_context *st)
    /* Rasterizer state */
    memset(&st->pbo.raster, 0, sizeof(struct pipe_rasterizer_state));
    st->pbo.raster.half_pixel_center = 1;
+
+   if (st->allow_compute_based_texture_transfer)
+      st->pbo.shaders = _mesa_hash_table_create_u32_keys(NULL);
 }
 
 void
@@ -697,4 +700,10 @@ st_destroy_pbo_helpers(struct st_context *st)
       st->pipe->delete_vs_state(st->pipe, st->pbo.vs);
       st->pbo.vs = NULL;
    }
+
+   if (st->pbo.shaders) {
+      hash_table_foreach(st->pbo.shaders, entry)
+         st->pipe->delete_compute_state(st->pipe, entry->data);
+      _mesa_hash_table_destroy(st->pbo.shaders, NULL);
+   }
 }

src/mesa/state_tracker/st_pbo.h

@@ -110,6 +110,12 @@ st_pbo_get_download_fs(struct st_context *st, enum pipe_texture_target target,
                        enum pipe_format dst_format,
                        bool need_layer);
 
+bool
+st_GetTexSubImage_shader(struct gl_context * ctx,
+                         GLint xoffset, GLint yoffset, GLint zoffset,
+                         GLsizei width, GLsizei height, GLint depth,
+                         GLenum format, GLenum type, void * pixels,
+                         struct gl_texture_image *texImage);
 
 enum pipe_format
 st_pbo_get_dst_format(struct gl_context *ctx, enum pipe_texture_target target,