freedreno/a6xx: Add r2d support for GMEM resolves

For cases where we need to do an MSAA resolve that BLIT events cannot
handle, fall back to CP_BLIT (r2d)

Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/4085
Signed-off-by: Rob Clark <robdclark@chromium.org>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/8509>

src/gallium/drivers/freedreno/a6xx/fd6_blitter.c

@@ -256,9 +256,6 @@ emit_blit_setup(struct fd_ringbuffer *ring,
 	bool is_srgb = util_format_is_srgb(pfmt);
 	enum a6xx_2d_ifmt ifmt = fd6_ifmt(fmt);
 
-	OUT_PKT7(ring, CP_SET_MARKER, 1);
-	OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE));
-
 	if (is_srgb) {
 		assert(ifmt == R2D_UNORM8);
 		ifmt = R2D_UNORM8_SRGB;
@@ -789,6 +786,77 @@ fd6_clear_surface(struct fd_context *ctx,
 	}
 }
 
+void
+fd6_resolve_tile(struct fd_batch *batch, struct fd_ringbuffer *ring,
+		uint32_t base, struct pipe_surface *psurf)
+{
+	const struct fd_gmem_stateobj *gmem = batch->gmem_state;
+	uint64_t gmem_base = batch->ctx->screen->gmem_base + base;
+	uint32_t gmem_pitch = gmem->bin_w * batch->framebuffer.samples *
+			util_format_get_blocksize(psurf->format);
+
+	OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2);
+	OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) | A6XX_GRAS_2D_DST_TL_Y(0));
+	OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(psurf->width - 1) |
+			A6XX_GRAS_2D_DST_BR_Y(psurf->height - 1));
+
+	OUT_PKT4(ring, REG_A6XX_GRAS_2D_SRC_TL_X, 4);
+	OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_X(0));
+	OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_X(psurf->width - 1));
+	OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_Y(0));
+	OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_Y(psurf->height - 1));
+
+	/* Enable scissor bit, which will take into account the window scissor
+	 * which is set per-tile
+	 */
+	emit_blit_setup(ring, psurf->format, true, NULL);
+
+	/* We shouldn't be using GMEM in the layered rendering case: */
+	assert(psurf->u.tex.first_layer == psurf->u.tex.last_layer);
+
+	emit_blit_dst(ring, psurf->texture, psurf->format, psurf->u.tex.level,
+			psurf->u.tex.first_layer);
+
+	enum a6xx_format sfmt = fd6_pipe2color(psurf->format);
+	enum a3xx_msaa_samples samples = fd_msaa_samples(batch->framebuffer.samples);
+
+	OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 10);
+	OUT_RING(ring, A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(sfmt) |
+			A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(TILE6_2) |
+			A6XX_SP_PS_2D_SRC_INFO_SAMPLES(samples) |
+			COND(samples > MSAA_ONE, A6XX_SP_PS_2D_SRC_INFO_SAMPLES_AVERAGE) |
+			COND(util_format_is_srgb(psurf->format), A6XX_SP_PS_2D_SRC_INFO_SRGB) |
+			A6XX_SP_PS_2D_SRC_INFO_UNK20 |
+			A6XX_SP_PS_2D_SRC_INFO_UNK22);
+	OUT_RING(ring, A6XX_SP_PS_2D_SRC_SIZE_WIDTH(psurf->width) |
+			A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(psurf->height));
+	OUT_RING(ring, gmem_base);                   /* SP_PS_2D_SRC_LO */
+	OUT_RING(ring, gmem_base >> 32);             /* SP_PS_2D_SRC_HI */
+	OUT_RING(ring, A6XX_SP_PS_2D_SRC_PITCH_PITCH(gmem_pitch));
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+	OUT_RING(ring, 0x00000000);
+
+	/* sync GMEM writes with CACHE. */
+	fd6_cache_inv(batch, ring);
+
+	/* Wait for CACHE_INVALIDATE to land */
+	fd_wfi(batch, ring);
+
+	OUT_PKT7(ring, CP_BLIT, 1);
+	OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE));
+
+	OUT_WFI5(ring);
+
+	/* CP_BLIT writes to the CCU, unlike CP_EVENT_WRITE::BLIT which writes to
+	 * sysmem, and we generally assume that GMEM renderpasses leave their
+	 * results in sysmem, so we need to flush manually here.
+	 */
+	fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true);
+}
+
 static bool
 handle_rgba_blit(struct fd_context *ctx, const struct pipe_blit_info *info)
 {

src/gallium/drivers/freedreno/a6xx/fd6_blitter.h

@@ -35,9 +35,15 @@
 void fd6_blitter_init(struct pipe_context *pctx);
 unsigned fd6_tile_mode(const struct pipe_resource *tmpl);
 
-void
-fd6_clear_surface(struct fd_context *ctx,
+/*
+ * Blitter APIs used by gmem for cases that need CP_BLIT's (r2d)
+ * instead of CP_EVENT_WRITE::BLITs
+ */
+
+void fd6_clear_surface(struct fd_context *ctx,
 		struct fd_ringbuffer *ring, struct pipe_surface *psurf,
 		uint32_t width, uint32_t height, union pipe_color_union *color);
+void fd6_resolve_tile(struct fd_batch *batch, struct fd_ringbuffer *ring,
+		uint32_t base, struct pipe_surface *psurf);
 
 #endif /* FD6_BLIT_H_ */

src/gallium/drivers/freedreno/a6xx/fd6_gmem.c

@@ -1129,6 +1129,47 @@ fd6_emit_tile_renderprep(struct fd_batch *batch, const struct fd_tile *tile)
 	trace_end_clear_restore(&batch->trace);
 }
 
+static bool
+blit_can_resolve(enum pipe_format format)
+{
+	const struct util_format_description *desc = util_format_description(format);
+
+	/* blit event can only do resolve for simple cases:
+	 * averaging samples as unsigned integers or choosing only one sample
+	 */
+	if (util_format_is_snorm(format) || util_format_is_srgb(format))
+		return false;
+
+	/* can't do formats with larger channel sizes
+	 * note: this includes all float formats
+	 * note2: single channel integer formats seem OK
+	 */
+	if (desc->channel[0].size > 10)
+		return false;
+
+	switch (format) {
+	/* for unknown reasons blit event can't msaa resolve these formats when tiled
+	 * likely related to these formats having different layout from other cpp=2 formats
+	 */
+	case PIPE_FORMAT_R8G8_UNORM:
+	case PIPE_FORMAT_R8G8_UINT:
+	case PIPE_FORMAT_R8G8_SINT:
+	/* TODO: this one should be able to work? */
+	case PIPE_FORMAT_Z24_UNORM_S8_UINT:
+		return false;
+	default:
+		break;
+	}
+
+	return true;
+}
+
+static bool
+needs_resolve(struct pipe_surface *psurf)
+{
+	return psurf->nr_samples && (psurf->nr_samples != psurf->texture->nr_samples);
+}
+
 static void
 emit_resolve_blit(struct fd_batch *batch,
 				  struct fd_ringbuffer *ring,
@@ -1142,6 +1183,18 @@ emit_resolve_blit(struct fd_batch *batch,
 	if (!fd_resource(psurf->texture)->valid)
 		return;
 
+	/* if we need to resolve, but cannot with BLIT event, we instead need
+	 * to generate per-tile CP_BLIT (r2d) commands:
+	 *
+	 * The separate-stencil is a special case, we might need to use CP_BLIT
+	 * for depth, but we can still resolve stencil with a BLIT event
+	 */
+	if (needs_resolve(psurf) && !blit_can_resolve(psurf->format) &&
+			(buffer != FD_BUFFER_STENCIL)) {
+		fd6_resolve_tile(batch, ring, base, psurf);
+		return;
+	}
+
 	switch (buffer) {
 	case FD_BUFFER_COLOR:
 		break;