nir_to_tgsi: Add support for HW atomics.
Found missing when taking virgl on the nir-to-tgsi path. Needs an SSBO fix as well to pass all of the atomic_counters tests. Reviewed-by: Adam Jackson <ajax@redhat.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11937>
This commit is contained in:
parent
be7d964ff0
commit
846a4512e9
|
@ -423,6 +423,10 @@ ntt_setup_uniforms(struct ntt_compile *c)
|
||||||
var->data.image.format,
|
var->data.image.format,
|
||||||
!(var->data.access & ACCESS_NON_WRITEABLE),
|
!(var->data.access & ACCESS_NON_WRITEABLE),
|
||||||
false);
|
false);
|
||||||
|
} else if (glsl_contains_atomic(var->type)) {
|
||||||
|
uint32_t offset = var->data.offset / 4;
|
||||||
|
uint32_t size = glsl_atomic_size(var->type) / 4;
|
||||||
|
ureg_DECL_hw_atomic(c->ureg, offset, offset + size - 1, var->data.binding, 0);
|
||||||
} else {
|
} else {
|
||||||
unsigned size;
|
unsigned size;
|
||||||
if (packed) {
|
if (packed) {
|
||||||
|
@ -1284,7 +1288,8 @@ ntt_emit_mem(struct ntt_compile *c, nir_intrinsic_instr *instr,
|
||||||
{
|
{
|
||||||
bool is_store = (instr->intrinsic == nir_intrinsic_store_ssbo ||
|
bool is_store = (instr->intrinsic == nir_intrinsic_store_ssbo ||
|
||||||
instr->intrinsic == nir_intrinsic_store_shared);
|
instr->intrinsic == nir_intrinsic_store_shared);
|
||||||
bool is_load = (instr->intrinsic == nir_intrinsic_load_ssbo ||
|
bool is_load = (instr->intrinsic == nir_intrinsic_atomic_counter_read ||
|
||||||
|
instr->intrinsic == nir_intrinsic_load_ssbo ||
|
||||||
instr->intrinsic == nir_intrinsic_load_shared);
|
instr->intrinsic == nir_intrinsic_load_shared);
|
||||||
unsigned opcode;
|
unsigned opcode;
|
||||||
struct ureg_src src[4];
|
struct ureg_src src[4];
|
||||||
|
@ -1302,6 +1307,14 @@ ntt_emit_mem(struct ntt_compile *c, nir_intrinsic_instr *instr,
|
||||||
memory = ureg_src_register(TGSI_FILE_MEMORY, 0);
|
memory = ureg_src_register(TGSI_FILE_MEMORY, 0);
|
||||||
nir_src = 0;
|
nir_src = 0;
|
||||||
break;
|
break;
|
||||||
|
case nir_var_uniform: { /* HW atomic buffers */
|
||||||
|
uint32_t offset = nir_src_as_uint(instr->src[0]);
|
||||||
|
memory = ureg_src_dimension(ureg_src_register(TGSI_FILE_HW_ATOMIC, offset / 4),
|
||||||
|
nir_intrinsic_base(instr));
|
||||||
|
nir_src = 0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
unreachable("unknown memory type");
|
unreachable("unknown memory type");
|
||||||
}
|
}
|
||||||
|
@ -1313,13 +1326,26 @@ ntt_emit_mem(struct ntt_compile *c, nir_intrinsic_instr *instr,
|
||||||
src[num_src++] = memory;
|
src[num_src++] = memory;
|
||||||
if (instr->intrinsic != nir_intrinsic_get_ssbo_size) {
|
if (instr->intrinsic != nir_intrinsic_get_ssbo_size) {
|
||||||
src[num_src++] = ntt_get_src(c, instr->src[nir_src++]); /* offset */
|
src[num_src++] = ntt_get_src(c, instr->src[nir_src++]); /* offset */
|
||||||
if (!is_load)
|
switch (instr->intrinsic) {
|
||||||
src[num_src++] = ntt_get_src(c, instr->src[nir_src++]); /* value */
|
case nir_intrinsic_atomic_counter_inc:
|
||||||
|
src[num_src++] = ureg_imm1i(c->ureg, 1);
|
||||||
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_post_dec:
|
||||||
|
src[num_src++] = ureg_imm1i(c->ureg, -1);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (!is_load)
|
||||||
|
src[num_src++] = ntt_get_src(c, instr->src[nir_src++]); /* value */
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
switch (instr->intrinsic) {
|
switch (instr->intrinsic) {
|
||||||
|
case nir_intrinsic_atomic_counter_add:
|
||||||
|
case nir_intrinsic_atomic_counter_inc:
|
||||||
|
case nir_intrinsic_atomic_counter_post_dec:
|
||||||
case nir_intrinsic_ssbo_atomic_add:
|
case nir_intrinsic_ssbo_atomic_add:
|
||||||
case nir_intrinsic_shared_atomic_add:
|
case nir_intrinsic_shared_atomic_add:
|
||||||
opcode = TGSI_OPCODE_ATOMUADD;
|
opcode = TGSI_OPCODE_ATOMUADD;
|
||||||
|
@ -1328,10 +1354,12 @@ ntt_emit_mem(struct ntt_compile *c, nir_intrinsic_instr *instr,
|
||||||
case nir_intrinsic_shared_atomic_fadd:
|
case nir_intrinsic_shared_atomic_fadd:
|
||||||
opcode = TGSI_OPCODE_ATOMFADD;
|
opcode = TGSI_OPCODE_ATOMFADD;
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_min:
|
||||||
case nir_intrinsic_ssbo_atomic_imin:
|
case nir_intrinsic_ssbo_atomic_imin:
|
||||||
case nir_intrinsic_shared_atomic_imin:
|
case nir_intrinsic_shared_atomic_imin:
|
||||||
opcode = TGSI_OPCODE_ATOMIMIN;
|
opcode = TGSI_OPCODE_ATOMIMIN;
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_max:
|
||||||
case nir_intrinsic_ssbo_atomic_imax:
|
case nir_intrinsic_ssbo_atomic_imax:
|
||||||
case nir_intrinsic_shared_atomic_imax:
|
case nir_intrinsic_shared_atomic_imax:
|
||||||
opcode = TGSI_OPCODE_ATOMIMAX;
|
opcode = TGSI_OPCODE_ATOMIMAX;
|
||||||
|
@ -1344,27 +1372,33 @@ ntt_emit_mem(struct ntt_compile *c, nir_intrinsic_instr *instr,
|
||||||
case nir_intrinsic_shared_atomic_umax:
|
case nir_intrinsic_shared_atomic_umax:
|
||||||
opcode = TGSI_OPCODE_ATOMUMAX;
|
opcode = TGSI_OPCODE_ATOMUMAX;
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_and:
|
||||||
case nir_intrinsic_ssbo_atomic_and:
|
case nir_intrinsic_ssbo_atomic_and:
|
||||||
case nir_intrinsic_shared_atomic_and:
|
case nir_intrinsic_shared_atomic_and:
|
||||||
opcode = TGSI_OPCODE_ATOMAND;
|
opcode = TGSI_OPCODE_ATOMAND;
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_or:
|
||||||
case nir_intrinsic_ssbo_atomic_or:
|
case nir_intrinsic_ssbo_atomic_or:
|
||||||
case nir_intrinsic_shared_atomic_or:
|
case nir_intrinsic_shared_atomic_or:
|
||||||
opcode = TGSI_OPCODE_ATOMOR;
|
opcode = TGSI_OPCODE_ATOMOR;
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_xor:
|
||||||
case nir_intrinsic_ssbo_atomic_xor:
|
case nir_intrinsic_ssbo_atomic_xor:
|
||||||
case nir_intrinsic_shared_atomic_xor:
|
case nir_intrinsic_shared_atomic_xor:
|
||||||
opcode = TGSI_OPCODE_ATOMXOR;
|
opcode = TGSI_OPCODE_ATOMXOR;
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_exchange:
|
||||||
case nir_intrinsic_ssbo_atomic_exchange:
|
case nir_intrinsic_ssbo_atomic_exchange:
|
||||||
case nir_intrinsic_shared_atomic_exchange:
|
case nir_intrinsic_shared_atomic_exchange:
|
||||||
opcode = TGSI_OPCODE_ATOMXCHG;
|
opcode = TGSI_OPCODE_ATOMXCHG;
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_comp_swap:
|
||||||
case nir_intrinsic_ssbo_atomic_comp_swap:
|
case nir_intrinsic_ssbo_atomic_comp_swap:
|
||||||
case nir_intrinsic_shared_atomic_comp_swap:
|
case nir_intrinsic_shared_atomic_comp_swap:
|
||||||
opcode = TGSI_OPCODE_ATOMCAS;
|
opcode = TGSI_OPCODE_ATOMCAS;
|
||||||
src[num_src++] = ntt_get_src(c, instr->src[nir_src++]);
|
src[num_src++] = ntt_get_src(c, instr->src[nir_src++]);
|
||||||
break;
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_read:
|
||||||
case nir_intrinsic_load_ssbo:
|
case nir_intrinsic_load_ssbo:
|
||||||
case nir_intrinsic_load_shared:
|
case nir_intrinsic_load_shared:
|
||||||
opcode = TGSI_OPCODE_LOAD;
|
opcode = TGSI_OPCODE_LOAD;
|
||||||
|
@ -1778,6 +1812,23 @@ ntt_emit_intrinsic(struct ntt_compile *c, nir_intrinsic_instr *instr)
|
||||||
ntt_emit_mem(c, instr, nir_var_mem_shared);
|
ntt_emit_mem(c, instr, nir_var_mem_shared);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case nir_intrinsic_atomic_counter_read:
|
||||||
|
case nir_intrinsic_atomic_counter_add:
|
||||||
|
case nir_intrinsic_atomic_counter_inc:
|
||||||
|
case nir_intrinsic_atomic_counter_post_dec:
|
||||||
|
case nir_intrinsic_atomic_counter_min:
|
||||||
|
case nir_intrinsic_atomic_counter_max:
|
||||||
|
case nir_intrinsic_atomic_counter_and:
|
||||||
|
case nir_intrinsic_atomic_counter_or:
|
||||||
|
case nir_intrinsic_atomic_counter_xor:
|
||||||
|
case nir_intrinsic_atomic_counter_exchange:
|
||||||
|
case nir_intrinsic_atomic_counter_comp_swap:
|
||||||
|
ntt_emit_mem(c, instr, nir_var_uniform);
|
||||||
|
break;
|
||||||
|
case nir_intrinsic_atomic_counter_pre_dec:
|
||||||
|
unreachable("Should be lowered by ntt_lower_atomic_pre_dec()");
|
||||||
|
break;
|
||||||
|
|
||||||
case nir_intrinsic_image_load:
|
case nir_intrinsic_image_load:
|
||||||
case nir_intrinsic_image_store:
|
case nir_intrinsic_image_store:
|
||||||
case nir_intrinsic_image_size:
|
case nir_intrinsic_image_size:
|
||||||
|
@ -2780,6 +2831,32 @@ ntt_fix_nir_options(struct pipe_screen *screen, struct nir_shader *s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool
|
||||||
|
ntt_lower_atomic_pre_dec_filter(const nir_instr *instr, const void *_data)
|
||||||
|
{
|
||||||
|
return (instr->type == nir_instr_type_intrinsic &&
|
||||||
|
nir_instr_as_intrinsic(instr)->intrinsic == nir_intrinsic_atomic_counter_pre_dec);
|
||||||
|
}
|
||||||
|
|
||||||
|
static nir_ssa_def *
|
||||||
|
ntt_lower_atomic_pre_dec_lower(nir_builder *b, nir_instr *instr, void *_data)
|
||||||
|
{
|
||||||
|
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
|
||||||
|
|
||||||
|
nir_ssa_def *old_result = &intr->dest.ssa;
|
||||||
|
intr->intrinsic = nir_intrinsic_atomic_counter_post_dec;
|
||||||
|
|
||||||
|
return nir_iadd_imm(b, old_result, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool
|
||||||
|
ntt_lower_atomic_pre_dec(nir_shader *s)
|
||||||
|
{
|
||||||
|
return nir_shader_lower_instructions(s,
|
||||||
|
ntt_lower_atomic_pre_dec_filter,
|
||||||
|
ntt_lower_atomic_pre_dec_lower, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
/* Lowers texture projectors if we can't do them as TGSI_OPCODE_TXP. */
|
/* Lowers texture projectors if we can't do them as TGSI_OPCODE_TXP. */
|
||||||
static void
|
static void
|
||||||
nir_to_tgsi_lower_txp(nir_shader *s)
|
nir_to_tgsi_lower_txp(nir_shader *s)
|
||||||
|
@ -2848,6 +2925,9 @@ nir_to_tgsi(struct nir_shader *s,
|
||||||
nir_to_tgsi_lower_txp(s);
|
nir_to_tgsi_lower_txp(s);
|
||||||
NIR_PASS_V(s, nir_to_tgsi_lower_tex);
|
NIR_PASS_V(s, nir_to_tgsi_lower_tex);
|
||||||
|
|
||||||
|
if (s->info.num_abos)
|
||||||
|
NIR_PASS_V(s, ntt_lower_atomic_pre_dec);
|
||||||
|
|
||||||
if (!original_options->lower_uniforms_to_ubo) {
|
if (!original_options->lower_uniforms_to_ubo) {
|
||||||
NIR_PASS_V(s, nir_lower_uniforms_to_ubo,
|
NIR_PASS_V(s, nir_lower_uniforms_to_ubo,
|
||||||
screen->get_param(screen, PIPE_CAP_PACKED_UNIFORMS),
|
screen->get_param(screen, PIPE_CAP_PACKED_UNIFORMS),
|
||||||
|
|
Loading…
Reference in New Issue