From 42319c562642100a2c5d07dd8522f63e6ea6f0fb Mon Sep 17 00:00:00 2001 From: Alyssa Rosenzweig Date: Wed, 4 Nov 2020 08:37:55 -0500 Subject: [PATCH] pan/mdg: Move writeout lowering to common panfrost These will be used in the Bifrost compiler, albeit for a slightly different purpose. Signed-off-by: Alyssa Rosenzweig Part-of: --- src/panfrost/Makefile.sources | 1 + src/panfrost/midgard/compiler.h | 4 - src/panfrost/midgard/midgard_compile.c | 229 +--------------------- src/panfrost/util/meson.build | 1 + src/panfrost/util/pan_ir.h | 9 + src/panfrost/util/pan_lower_writeout.c | 255 +++++++++++++++++++++++++ 6 files changed, 268 insertions(+), 231 deletions(-) create mode 100644 src/panfrost/util/pan_lower_writeout.c diff --git a/src/panfrost/Makefile.sources b/src/panfrost/Makefile.sources index 82fbc5de9d8..4dec2629602 100644 --- a/src/panfrost/Makefile.sources +++ b/src/panfrost/Makefile.sources @@ -99,4 +99,5 @@ util_FILES := \ util/pan_ir.h \ util/pan_liveness.c \ util/pan_lower_framebuffer.c \ + util/pan_lower_writeout.c \ util/pan_sysval.c \ diff --git a/src/panfrost/midgard/compiler.h b/src/panfrost/midgard/compiler.h index 74a3532ab3c..03f463a4646 100644 --- a/src/panfrost/midgard/compiler.h +++ b/src/panfrost/midgard/compiler.h @@ -71,10 +71,6 @@ typedef struct midgard_branch { }; } midgard_branch; -#define PAN_WRITEOUT_C 1 -#define PAN_WRITEOUT_Z 2 -#define PAN_WRITEOUT_S 4 - /* Generic in-memory data type repesenting a single logical instruction, rather * than a single instruction group. This is the preferred form for code gen. * Multiple midgard_insturctions will later be combined during scheduling, diff --git a/src/panfrost/midgard/midgard_compile.c b/src/panfrost/midgard/midgard_compile.c index 8c64151fb91..4c1fd438a23 100644 --- a/src/panfrost/midgard/midgard_compile.c +++ b/src/panfrost/midgard/midgard_compile.c @@ -210,231 +210,6 @@ midgard_nir_lower_fdot2(nir_shader *shader) NULL); } -/* Midgard can write all of color, depth and stencil in a single writeout - * operation, so we merge depth/stencil stores with color stores. - * If there are no color stores, we add a write to the "depth RT". - */ -static bool -midgard_nir_lower_zs_store(nir_shader *nir) -{ - if (nir->info.stage != MESA_SHADER_FRAGMENT) - return false; - - nir_variable *z_var = NULL, *s_var = NULL; - - nir_foreach_shader_out_variable(var, nir) { - if (var->data.location == FRAG_RESULT_DEPTH) - z_var = var; - else if (var->data.location == FRAG_RESULT_STENCIL) - s_var = var; - } - - if (!z_var && !s_var) - return false; - - bool progress = false; - - nir_foreach_function(function, nir) { - if (!function->impl) continue; - - nir_intrinsic_instr *z_store = NULL, *s_store = NULL; - - nir_foreach_block(block, function->impl) { - nir_foreach_instr_safe(instr, block) { - if (instr->type != nir_instr_type_intrinsic) - continue; - - nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); - if (intr->intrinsic != nir_intrinsic_store_output) - continue; - - if (z_var && nir_intrinsic_base(intr) == z_var->data.driver_location) { - assert(!z_store); - z_store = intr; - } - - if (s_var && nir_intrinsic_base(intr) == s_var->data.driver_location) { - assert(!s_store); - s_store = intr; - } - } - } - - if (!z_store && !s_store) continue; - - bool replaced = false; - - nir_foreach_block(block, function->impl) { - nir_foreach_instr_safe(instr, block) { - if (instr->type != nir_instr_type_intrinsic) - continue; - - nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); - if (intr->intrinsic != nir_intrinsic_store_output) - continue; - - const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr)); - assert(var); - - if (var->data.location != FRAG_RESULT_COLOR && - var->data.location < FRAG_RESULT_DATA0) - continue; - - if (var->data.index) - continue; - - assert(nir_src_is_const(intr->src[1]) && "no indirect outputs"); - - nir_builder b; - nir_builder_init(&b, function->impl); - - assert(!z_store || z_store->instr.block == instr->block); - assert(!s_store || s_store->instr.block == instr->block); - b.cursor = nir_after_block_before_jump(instr->block); - - nir_intrinsic_instr *combined_store; - combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); - - combined_store->num_components = intr->src[0].ssa->num_components; - - nir_intrinsic_set_base(combined_store, nir_intrinsic_base(intr)); - - unsigned writeout = PAN_WRITEOUT_C; - if (z_store) - writeout |= PAN_WRITEOUT_Z; - if (s_store) - writeout |= PAN_WRITEOUT_S; - - nir_intrinsic_set_component(combined_store, writeout); - - struct nir_ssa_def *zero = nir_imm_int(&b, 0); - - struct nir_ssa_def *src[4] = { - intr->src[0].ssa, - intr->src[1].ssa, - z_store ? z_store->src[0].ssa : zero, - s_store ? s_store->src[0].ssa : zero, - }; - - for (int i = 0; i < 4; ++i) - combined_store->src[i] = nir_src_for_ssa(src[i]); - - nir_builder_instr_insert(&b, &combined_store->instr); - - nir_instr_remove(instr); - - replaced = true; - } - } - - /* Insert a store to the depth RT (0xff) if needed */ - if (!replaced) { - nir_builder b; - nir_builder_init(&b, function->impl); - - nir_block *block = NULL; - if (z_store && s_store) - assert(z_store->instr.block == s_store->instr.block); - - if (z_store) - block = z_store->instr.block; - else - block = s_store->instr.block; - - b.cursor = nir_after_block_before_jump(block); - - nir_intrinsic_instr *combined_store; - combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); - - combined_store->num_components = 4; - - unsigned base; - if (z_store) - base = nir_intrinsic_base(z_store); - else - base = nir_intrinsic_base(s_store); - nir_intrinsic_set_base(combined_store, base); - - unsigned writeout = 0; - if (z_store) - writeout |= PAN_WRITEOUT_Z; - if (s_store) - writeout |= PAN_WRITEOUT_S; - - nir_intrinsic_set_component(combined_store, writeout); - - struct nir_ssa_def *zero = nir_imm_int(&b, 0); - - struct nir_ssa_def *src[4] = { - nir_imm_vec4(&b, 0, 0, 0, 0), - zero, - z_store ? z_store->src[0].ssa : zero, - s_store ? s_store->src[0].ssa : zero, - }; - - for (int i = 0; i < 4; ++i) - combined_store->src[i] = nir_src_for_ssa(src[i]); - - nir_builder_instr_insert(&b, &combined_store->instr); - } - - if (z_store) - nir_instr_remove(&z_store->instr); - - if (s_store) - nir_instr_remove(&s_store->instr); - - nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); - progress = true; - } - - return progress; -} - -/* Real writeout stores, which break execution, need to be moved to after - * dual-source stores, which are just standard register writes. */ -static bool -midgard_nir_reorder_writeout(nir_shader *nir) -{ - bool progress = false; - - nir_foreach_function(function, nir) { - if (!function->impl) continue; - - nir_foreach_block(block, function->impl) { - nir_instr *last_writeout = NULL; - - nir_foreach_instr_reverse_safe(instr, block) { - if (instr->type != nir_instr_type_intrinsic) - continue; - - nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); - if (intr->intrinsic != nir_intrinsic_store_output) - continue; - - const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr)); - - if (var->data.index) { - if (!last_writeout) - last_writeout = instr; - continue; - } - - if (!last_writeout) - continue; - - /* This is a real store, so move it to after dual-source stores */ - exec_node_remove(&instr->node); - exec_node_insert_after(&last_writeout->node, &instr->node); - - progress = true; - } - } - } - - return progress; -} - static bool mdg_is_64(const nir_instr *instr, const void *_unused) { @@ -2988,13 +2763,13 @@ midgard_compile_shader_nir(void *mem_ctx, nir_shader *nir, NIR_PASS_V(nir, nir_lower_io, nir_var_shader_in | nir_var_shader_out, glsl_type_size, 0); NIR_PASS_V(nir, nir_lower_ssbo); - NIR_PASS_V(nir, midgard_nir_lower_zs_store); + NIR_PASS_V(nir, pan_nir_lower_zs_store); /* Optimisation passes */ optimise_nir(nir, ctx->quirks, inputs->is_blend); - NIR_PASS_V(nir, midgard_nir_reorder_writeout); + NIR_PASS_V(nir, pan_nir_reorder_writeout); if ((midgard_debug & MIDGARD_DBG_SHADERS) && !nir->info.internal) { nir_print_shader(nir, stdout); diff --git a/src/panfrost/util/meson.build b/src/panfrost/util/meson.build index 118722b02b4..8ac6ee99e43 100644 --- a/src/panfrost/util/meson.build +++ b/src/panfrost/util/meson.build @@ -28,6 +28,7 @@ libpanfrost_util_files = files( 'pan_ir.h', 'pan_liveness.c', 'pan_lower_framebuffer.c', + 'pan_lower_writeout.c', 'pan_sysval.c', ) diff --git a/src/panfrost/util/pan_ir.h b/src/panfrost/util/pan_ir.h index 51ba08056e8..d0a5208d388 100644 --- a/src/panfrost/util/pan_ir.h +++ b/src/panfrost/util/pan_ir.h @@ -233,4 +233,13 @@ void pan_print_alu_type(nir_alu_type t, FILE *fp); bool pan_has_source_mod(nir_alu_src *src, nir_op op); bool pan_has_dest_mod(nir_dest **dest, nir_op op); +/* NIR passes to do some backend-specific lowering */ + +#define PAN_WRITEOUT_C 1 +#define PAN_WRITEOUT_Z 2 +#define PAN_WRITEOUT_S 4 + +bool pan_nir_reorder_writeout(nir_shader *nir); +bool pan_nir_lower_zs_store(nir_shader *nir); + #endif diff --git a/src/panfrost/util/pan_lower_writeout.c b/src/panfrost/util/pan_lower_writeout.c new file mode 100644 index 00000000000..b4155407ffa --- /dev/null +++ b/src/panfrost/util/pan_lower_writeout.c @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2018-2020 Collabora, Ltd. + * Copyright (C) 2019-2020 Icecream95 + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "pan_ir.h" +#include "compiler/nir/nir_builder.h" + +/* Midgard can write all of color, depth and stencil in a single writeout + * operation, so we merge depth/stencil stores with color stores. + * If there are no color stores, we add a write to the "depth RT". + * + * For Bifrost, we want these combined so we can properly order + * +ZS_EMIT with respect to +ATEST and +BLEND, as well as combining + * depth/stencil stores into a single +ZS_EMIT op. + */ +bool +pan_nir_lower_zs_store(nir_shader *nir) +{ + if (nir->info.stage != MESA_SHADER_FRAGMENT) + return false; + + nir_variable *z_var = NULL, *s_var = NULL; + + nir_foreach_shader_out_variable(var, nir) { + if (var->data.location == FRAG_RESULT_DEPTH) + z_var = var; + else if (var->data.location == FRAG_RESULT_STENCIL) + s_var = var; + } + + if (!z_var && !s_var) + return false; + + bool progress = false; + + nir_foreach_function(function, nir) { + if (!function->impl) continue; + + nir_intrinsic_instr *z_store = NULL, *s_store = NULL; + + nir_foreach_block(block, function->impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + if (intr->intrinsic != nir_intrinsic_store_output) + continue; + + if (z_var && nir_intrinsic_base(intr) == z_var->data.driver_location) { + assert(!z_store); + z_store = intr; + } + + if (s_var && nir_intrinsic_base(intr) == s_var->data.driver_location) { + assert(!s_store); + s_store = intr; + } + } + } + + if (!z_store && !s_store) continue; + + bool replaced = false; + + nir_foreach_block(block, function->impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + if (intr->intrinsic != nir_intrinsic_store_output) + continue; + + const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr)); + assert(var); + + if (var->data.location != FRAG_RESULT_COLOR && + var->data.location < FRAG_RESULT_DATA0) + continue; + + if (var->data.index) + continue; + + assert(nir_src_is_const(intr->src[1]) && "no indirect outputs"); + + nir_builder b; + nir_builder_init(&b, function->impl); + + assert(!z_store || z_store->instr.block == instr->block); + assert(!s_store || s_store->instr.block == instr->block); + b.cursor = nir_after_block_before_jump(instr->block); + + nir_intrinsic_instr *combined_store; + combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); + + combined_store->num_components = intr->src[0].ssa->num_components; + + nir_intrinsic_set_base(combined_store, nir_intrinsic_base(intr)); + + unsigned writeout = PAN_WRITEOUT_C; + if (z_store) + writeout |= PAN_WRITEOUT_Z; + if (s_store) + writeout |= PAN_WRITEOUT_S; + + nir_intrinsic_set_component(combined_store, writeout); + + struct nir_ssa_def *zero = nir_imm_int(&b, 0); + + struct nir_ssa_def *src[4] = { + intr->src[0].ssa, + intr->src[1].ssa, + z_store ? z_store->src[0].ssa : zero, + s_store ? s_store->src[0].ssa : zero, + }; + + for (int i = 0; i < 4; ++i) + combined_store->src[i] = nir_src_for_ssa(src[i]); + + nir_builder_instr_insert(&b, &combined_store->instr); + + nir_instr_remove(instr); + + replaced = true; + } + } + + /* Insert a store to the depth RT (0xff) if needed */ + if (!replaced) { + nir_builder b; + nir_builder_init(&b, function->impl); + + nir_block *block = NULL; + if (z_store && s_store) + assert(z_store->instr.block == s_store->instr.block); + + if (z_store) + block = z_store->instr.block; + else + block = s_store->instr.block; + + b.cursor = nir_after_block_before_jump(block); + + nir_intrinsic_instr *combined_store; + combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan); + + combined_store->num_components = 4; + + unsigned base; + if (z_store) + base = nir_intrinsic_base(z_store); + else + base = nir_intrinsic_base(s_store); + nir_intrinsic_set_base(combined_store, base); + + unsigned writeout = 0; + if (z_store) + writeout |= PAN_WRITEOUT_Z; + if (s_store) + writeout |= PAN_WRITEOUT_S; + + nir_intrinsic_set_component(combined_store, writeout); + + struct nir_ssa_def *zero = nir_imm_int(&b, 0); + + struct nir_ssa_def *src[4] = { + nir_imm_vec4(&b, 0, 0, 0, 0), + zero, + z_store ? z_store->src[0].ssa : zero, + s_store ? s_store->src[0].ssa : zero, + }; + + for (int i = 0; i < 4; ++i) + combined_store->src[i] = nir_src_for_ssa(src[i]); + + nir_builder_instr_insert(&b, &combined_store->instr); + } + + if (z_store) + nir_instr_remove(&z_store->instr); + + if (s_store) + nir_instr_remove(&s_store->instr); + + nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance); + progress = true; + } + + return progress; +} + +/* Real writeout stores, which break execution, need to be moved to after + * dual-source stores, which are just standard register writes. */ +bool +pan_nir_reorder_writeout(nir_shader *nir) +{ + bool progress = false; + + nir_foreach_function(function, nir) { + if (!function->impl) continue; + + nir_foreach_block(block, function->impl) { + nir_instr *last_writeout = NULL; + + nir_foreach_instr_reverse_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr); + if (intr->intrinsic != nir_intrinsic_store_output) + continue; + + const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr)); + + if (var->data.index) { + if (!last_writeout) + last_writeout = instr; + continue; + } + + if (!last_writeout) + continue; + + /* This is a real store, so move it to after dual-source stores */ + exec_node_remove(&instr->node); + exec_node_insert_after(&last_writeout->node, &instr->node); + + progress = true; + } + } + } + + return progress; +}