KonstantinSeurer
/
mesa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

r600: add support for hw atomic counters. (v3) 

This adds support for the evergreen/cayman atomic counters.

These are implemented using GDS append/consume counters. The values
for each counter are loaded before drawing and saved after each draw
using special CP packets.

v2: move hw atomic assignment into driver.
v3: fix messing up caps (Gert Wollny), only store ranges in driver,
drop buffers.

Signed-off-by: Dave Airlie <airlied@redhat.com>
Acked-by: Nicolai Hähnle <nicolai.haehnle@amd.com>
Tested-By: Gert Wollny <gw.fossdev@gmail.com>
This commit is contained in:
Dave Airlie 2017-11-02 10:26:51 +10:00
parent 9e62654d4b
commit 06993e4ee3
7 changed files with 480 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
src/gallium/drivers/r600/evergreen_state.c
View File

@ -3717,6 +3717,38 @@ static void evergreen_set_tess_state(struct pipe_context *ctx,
rctx->tess_state_dirty = true;
}
static void evergreen_set_hw_atomic_buffers(struct pipe_context *ctx,
unsigned start_slot,
unsigned count,
const struct pipe_shader_buffer *buffers)
{
struct r600_context *rctx = (struct r600_context *)ctx;
struct r600_atomic_buffer_state *astate;
int i, idx;
astate = &rctx->atomic_buffer_state;
/* we'd probably like to expand this to 8 later so put the logic in */
for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
const struct pipe_shader_buffer *buf;
struct pipe_shader_buffer *abuf;
abuf = &astate->buffer[i];
if (!buffers || !buffers[idx].buffer) {
pipe_resource_reference(&abuf->buffer, NULL);
astate->enabled_mask &= ~(1 << i);
continue;
}
buf = &buffers[idx];
pipe_resource_reference(&abuf->buffer, buf->buffer);
abuf->buffer_offset = buf->buffer_offset;
abuf->buffer_size = buf->buffer_size;
astate->enabled_mask |= (1 << i);
}
}
void evergreen_init_state_functions(struct r600_context *rctx)
{
unsigned id = 1;
@ -3802,6 +3834,7 @@ void evergreen_init_state_functions(struct r600_context *rctx)
rctx->b.b.set_polygon_stipple = evergreen_set_polygon_stipple;
rctx->b.b.set_min_samples = evergreen_set_min_samples;
rctx->b.b.set_tess_state = evergreen_set_tess_state;
rctx->b.b.set_hw_atomic_buffers = evergreen_set_hw_atomic_buffers;
if (rctx->b.chip_class == EVERGREEN)
rctx->b.b.get_sample_position = evergreen_get_sample_position;
else
@ -4108,3 +4141,129 @@ void eg_trace_emit(struct r600_context *rctx)
radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
radeon_emit(cs, AC_ENCODE_TRACE_POINT(rctx->trace_id));
}
bool evergreen_emit_atomic_buffer_setup(struct r600_context *rctx,
struct r600_shader_atomic *combined_atomics,
uint8_t *atomic_used_mask_p)
{
struct radeon_winsys_cs *cs = rctx->b.gfx.cs;
struct r600_atomic_buffer_state *astate = &rctx->atomic_buffer_state;
unsigned pkt_flags = 0;
uint8_t atomic_used_mask = 0;
int i, j, k;
for (i = 0; i < EG_NUM_HW_STAGES; i++) {
uint8_t num_atomic_stage;
struct r600_pipe_shader *pshader;
pshader = rctx->hw_shader_stages[i].shader;
if (!pshader)
continue;
num_atomic_stage = pshader->shader.nhwatomic_ranges;
if (!num_atomic_stage)
continue;
for (j = 0; j < num_atomic_stage; j++) {
struct r600_shader_atomic *atomic = &pshader->shader.atomics[j];
int natomics = atomic->end - atomic->start + 1;
for (k = 0; k < natomics; k++) {
/* seen this in a previous stage */
if (atomic_used_mask & (1u << (atomic->hw_idx + k)))
continue;
combined_atomics[atomic->hw_idx + k].hw_idx = atomic->hw_idx + k;
combined_atomics[atomic->hw_idx + k].buffer_id = atomic->buffer_id;
combined_atomics[atomic->hw_idx + k].start = atomic->start + k;
combined_atomics[atomic->hw_idx + k].end = combined_atomics[atomic->hw_idx + k].start + 1;
atomic_used_mask |= (1u << (atomic->hw_idx + k));
}
}
}
uint32_t mask = atomic_used_mask;
while (mask) {
unsigned atomic_index = u_bit_scan(&mask);
struct r600_shader_atomic *atomic = &combined_atomics[atomic_index];
struct r600_resource *resource = r600_resource(astate->buffer[atomic->buffer_id].buffer);
assert(resource);
unsigned reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx,
resource,
RADEON_USAGE_READ,
RADEON_PRIO_SHADER_RW_BUFFER);
uint64_t dst_offset = resource->gpu_address + (atomic->start * 4);
uint32_t base_reg_0 = R_02872C_GDS_APPEND_COUNT_0;
uint32_t reg_val = (base_reg_0 + atomic->hw_idx * 4 - EVERGREEN_CONTEXT_REG_OFFSET) >> 2;
radeon_emit(cs, PKT3(PKT3_SET_APPEND_CNT, 2, 0) | pkt_flags);
radeon_emit(cs, (reg_val << 16) | 0x3);
radeon_emit(cs, dst_offset & 0xfffffffc);
radeon_emit(cs, (dst_offset >> 32) & 0xff);
radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
radeon_emit(cs, reloc);
}
*atomic_used_mask_p = atomic_used_mask;
return true;
}
void evergreen_emit_atomic_buffer_save(struct r600_context *rctx,
struct r600_shader_atomic *combined_atomics,
uint8_t *atomic_used_mask_p)
{
struct radeon_winsys_cs *cs = rctx->b.gfx.cs;
struct r600_atomic_buffer_state *astate = &rctx->atomic_buffer_state;
uint32_t pkt_flags = 0;
uint32_t event = EVENT_TYPE_PS_DONE;
uint32_t mask = astate->enabled_mask;
uint64_t dst_offset;
unsigned reloc;
mask = *atomic_used_mask_p;
while (mask) {
unsigned atomic_index = u_bit_scan(&mask);
struct r600_shader_atomic *atomic = &combined_atomics[atomic_index];
struct r600_resource *resource = r600_resource(astate->buffer[atomic->buffer_id].buffer);
assert(resource);
uint32_t base_reg_0 = R_02872C_GDS_APPEND_COUNT_0;
reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx,
resource,
RADEON_USAGE_WRITE,
RADEON_PRIO_SHADER_RW_BUFFER);
dst_offset = resource->gpu_address + (atomic->start * 4);
uint32_t reg_val = (base_reg_0 + atomic->hw_idx * 4) >> 2;
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOS, 3, 0) | pkt_flags);
radeon_emit(cs, EVENT_TYPE(event) | EVENT_INDEX(6));
radeon_emit(cs, (dst_offset) & 0xffffffff);
radeon_emit(cs, (0 << 29) | ((dst_offset >> 32) & 0xff));
radeon_emit(cs, reg_val);
radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
radeon_emit(cs, reloc);
}
++rctx->append_fence_id;
reloc = radeon_add_to_buffer_list(&rctx->b, &rctx->b.gfx,
r600_resource(rctx->append_fence),
RADEON_USAGE_READWRITE,
RADEON_PRIO_SHADER_RW_BUFFER);
dst_offset = r600_resource(rctx->append_fence)->gpu_address;
radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOS, 3, 0) | pkt_flags);
radeon_emit(cs, EVENT_TYPE(event) | EVENT_INDEX(6));
radeon_emit(cs, dst_offset & 0xffffffff);
radeon_emit(cs, (2 << 29) | ((dst_offset >> 32) & 0xff));
radeon_emit(cs, rctx->append_fence_id);
radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
radeon_emit(cs, reloc);
radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0) | pkt_flags);
radeon_emit(cs, WAIT_REG_MEM_GEQUAL | WAIT_REG_MEM_MEMORY | (1 << 8));
radeon_emit(cs, dst_offset & 0xffffffff);
radeon_emit(cs, ((dst_offset >> 32) & 0xff));
radeon_emit(cs, rctx->append_fence_id);
radeon_emit(cs, 0xffffffff);
radeon_emit(cs, 0xa);
radeon_emit(cs, PKT3(PKT3_NOP, 0, 0));
radeon_emit(cs, reloc);
}

15
src/gallium/drivers/r600/r600_pipe.c
View File

@ -74,6 +74,8 @@ static void r600_destroy_context(struct pipe_context *context)
r600_resource_reference(&rctx->dummy_cmask, NULL);
r600_resource_reference(&rctx->dummy_fmask, NULL);
if (rctx->append_fence)
pipe_resource_reference((struct pipe_resource**)&rctx->append_fence, NULL);
for (sh = 0; sh < PIPE_SHADER_TYPES; sh++) {
rctx->b.b.set_constant_buffer(&rctx->b.b, sh, R600_BUFFER_INFO_CONST_BUFFER, NULL);
free(rctx->driver_consts[sh].constants);
@ -186,6 +188,9 @@ static struct pipe_context *r600_create_context(struct pipe_screen *screen,
rctx->b.family == CHIP_CAICOS ||
rctx->b.family == CHIP_CAYMAN ||
rctx->b.family == CHIP_ARUBA);
rctx->append_fence = pipe_buffer_create(rctx->b.b.screen, PIPE_BIND_CUSTOM,
PIPE_USAGE_DEFAULT, 32);
break;
default:
R600_ERR("Unsupported chip class %d.\n", rctx->b.chip_class);
@ -605,8 +610,17 @@ static int r600_get_shader_param(struct pipe_screen* pscreen,
case PIPE_SHADER_CAP_MAX_SHADER_IMAGES:
case PIPE_SHADER_CAP_LOWER_IF_THRESHOLD:
case PIPE_SHADER_CAP_TGSI_SKIP_MERGE_REGISTERS:
return 0;
case PIPE_SHADER_CAP_MAX_HW_ATOMIC_COUNTERS:
if (rscreen->b.family >= CHIP_CEDAR && rscreen->has_atomics)
return 8;
return 0;
case PIPE_SHADER_CAP_MAX_HW_ATOMIC_COUNTER_BUFFERS:
/* having to allocate the atomics out amongst shaders stages is messy,
so give compute 8 buffers and all the others one */
if (rscreen->b.family >= CHIP_CEDAR && rscreen->has_atomics) {
return EG_MAX_ATOMIC_BUFFERS;
}
return 0;
case PIPE_SHADER_CAP_MAX_UNROLL_ITERATIONS_HINT:
/* due to a bug in the shader compiler, some loops hang
@ -741,6 +755,7 @@ struct pipe_screen *r600_screen_create(struct radeon_winsys *ws,
/* Create the auxiliary context. This must be done last. */
rscreen->b.aux_context = rscreen->b.b.context_create(&rscreen->b.b, NULL, 0);
rscreen->has_atomics = rscreen->b.info.drm_minor >= 44;
#if 0 /* This is for testing whether aux_context and buffer clearing work correctly. */
struct pipe_resource templ = {};

22
src/gallium/drivers/r600/r600_pipe.h
View File

@ -64,6 +64,8 @@
#define R600_MAX_DRIVER_CONST_BUFFERS 3
#define R600_MAX_CONST_BUFFERS (R600_MAX_USER_CONST_BUFFERS + R600_MAX_DRIVER_CONST_BUFFERS)
#define EG_MAX_ATOMIC_BUFFERS 8
/* start driver buffers after user buffers */
#define R600_BUFFER_INFO_CONST_BUFFER (R600_MAX_USER_CONST_BUFFERS)
#define R600_UCP_SIZE (4*4*8)
@ -247,6 +249,7 @@ struct r600_screen {
struct r600_common_screen b;
bool has_msaa;
bool has_compressed_msaa_texturing;
bool has_atomics;
/*for compute global memory binding, we allocate stuff here, instead of
* buffers.
@ -416,6 +419,12 @@ struct r600_shader_state {
struct r600_pipe_shader *shader;
};
struct r600_atomic_buffer_state {
uint32_t enabled_mask;
uint32_t dirty_mask;
struct pipe_shader_buffer buffer[EG_MAX_ATOMIC_BUFFERS];
};
struct r600_context {
struct r600_common_context b;
struct r600_screen *screen;
@ -470,6 +479,7 @@ struct r600_context {
struct r600_config_state config_state;
struct r600_stencil_ref_state stencil_ref;
struct r600_vgt_state vgt_state;
struct r600_atomic_buffer_state atomic_buffer_state;
/* Shaders and shader resources. */
struct r600_cso_state vertex_fetch_shader;
struct r600_shader_state hw_shader_stages[EG_NUM_HW_STAGES];
@ -531,6 +541,9 @@ struct r600_context {
struct r600_resource *last_trace_buf;
struct r600_resource *trace_buf;
unsigned trace_id;
struct pipe_resource *append_fence;
uint32_t append_fence_id;
};
static inline void r600_emit_command_buffer(struct radeon_winsys_cs *cs,
@ -959,4 +972,13 @@ unsigned r600_conv_prim_to_gs_out(unsigned mode);
void eg_trace_emit(struct r600_context *rctx);
void eg_dump_debug_state(struct pipe_context *ctx, FILE *f,
unsigned flags);
struct r600_shader_atomic;
bool evergreen_emit_atomic_buffer_setup(struct r600_context *rctx,
struct r600_shader_atomic *combined_atomics,
uint8_t *atomic_used_mask_p);
void evergreen_emit_atomic_buffer_save(struct r600_context *rctx,
struct r600_shader_atomic *combined_atomics,
uint8_t *atomic_used_mask_p);
#endif

239
src/gallium/drivers/r600/r600_shader.c
View File

@ -194,6 +194,8 @@ int r600_pipe_shader_create(struct pipe_context *ctx,
/* disable SB for shaders using doubles */
use_sb &= !shader->shader.uses_doubles;
use_sb &= !shader->shader.uses_atomics;
/* Check if the bytecode has already been built. */
if (!shader->shader.bc.bytecode) {
r = r600_bytecode_build(&shader->shader.bc);
@ -407,6 +409,7 @@ static int tgsi_is_supported(struct r600_shader_ctx *ctx)
if (i->Src[j].Register.Dimension) {
switch (i->Src[j].Register.File) {
case TGSI_FILE_CONSTANT:
case TGSI_FILE_HW_ATOMIC:
break;
case TGSI_FILE_INPUT:
if (ctx->type == PIPE_SHADER_GEOMETRY ||
@ -966,6 +969,17 @@ static int tgsi_declaration(struct r600_shader_ctx *ctx)
case TGSI_FILE_ADDRESS:
break;
case TGSI_FILE_HW_ATOMIC:
i = ctx->shader->nhwatomic_ranges;
ctx->shader->atomics[i].start = d->Range.First;
ctx->shader->atomics[i].end = d->Range.Last;
ctx->shader->atomics[i].hw_idx = ctx->shader->atomic_base + ctx->shader->nhwatomic;
ctx->shader->atomics[i].array_id = d->Array.ArrayID;
ctx->shader->atomics[i].buffer_id = d->Dim.Index2D;
ctx->shader->nhwatomic_ranges++;
ctx->shader->nhwatomic += count;
break;
case TGSI_FILE_SYSTEM_VALUE:
if (d->Semantic.Name == TGSI_SEMANTIC_SAMPLEMASK ||
d->Semantic.Name == TGSI_SEMANTIC_SAMPLEID ||
@ -2946,6 +2960,7 @@ static int r600_shader_from_tgsi(struct r600_context *rctx,
shader->indirect_files = ctx.info.indirect_files;
shader->uses_doubles = ctx.info.uses_doubles;
shader->uses_atomics = ctx.info.file_mask[TGSI_FILE_HW_ATOMIC];
shader->nsys_inputs = 0;
indirect_gprs = ctx.info.indirect_files & ~((1 << TGSI_FILE_CONSTANT) | (1 << TGSI_FILE_SAMPLER));
@ -2959,6 +2974,7 @@ static int r600_shader_from_tgsi(struct r600_context *rctx,
shader->vs_as_gs_a = key.vs.as_gs_a;
shader->vs_as_es = key.vs.as_es;
shader->vs_as_ls = key.vs.as_ls;
shader->atomic_base = key.vs.first_atomic_counter;
if (shader->vs_as_es)
ring_outputs = true;
if (shader->vs_as_ls)
@ -2966,20 +2982,24 @@ static int r600_shader_from_tgsi(struct r600_context *rctx,
break;
case PIPE_SHADER_GEOMETRY:
ring_outputs = true;
shader->atomic_base = key.gs.first_atomic_counter;
break;
case PIPE_SHADER_TESS_CTRL:
shader->tcs_prim_mode = key.tcs.prim_mode;
shader->atomic_base = key.tcs.first_atomic_counter;
lds_outputs = true;
lds_inputs = true;
break;
case PIPE_SHADER_TESS_EVAL:
shader->tes_as_es = key.tes.as_es;
shader->atomic_base = key.tes.first_atomic_counter;
lds_inputs = true;
if (shader->tes_as_es)
ring_outputs = true;
break;
case PIPE_SHADER_FRAGMENT:
shader->two_side = key.ps.color_two_side;
shader->atomic_base = key.ps.first_atomic_counter;
break;
default:
break;
@ -7533,6 +7553,181 @@ static int tgsi_tex(struct r600_shader_ctx *ctx)
return 0;
}
static int find_hw_atomic_counter(struct r600_shader_ctx *ctx,
struct tgsi_full_src_register *src)
{
int i;
if (src->Register.Indirect) {
for (i = 0; i < ctx->shader->nhwatomic_ranges; i++) {
if (src->Indirect.ArrayID == ctx->shader->atomics[i].array_id)
return ctx->shader->atomics[i].hw_idx;
}
} else {
uint32_t index = src->Register.Index;
for (i = 0; i < ctx->shader->nhwatomic_ranges; i++) {
if (ctx->shader->atomics[i].buffer_id != src->Dimension.Index)
continue;
if (index > ctx->shader->atomics[i].end)
continue;
if (index < ctx->shader->atomics[i].start)
continue;
uint32_t offset = (index - ctx->shader->atomics[i].start);
return ctx->shader->atomics[i].hw_idx + offset;
}
}
assert(0);
return -1;
}
static int tgsi_load_gds(struct r600_shader_ctx *ctx)
{
struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction;
int r;
struct r600_bytecode_gds gds;
int uav_id = 0;
int uav_index_mode = 0;
uav_id = find_hw_atomic_counter(ctx, &inst->Src[0]);
if (inst->Src[0].Register.Indirect)
uav_index_mode = 2;
memset(&gds, 0, sizeof(struct r600_bytecode_gds));
gds.op = FETCH_OP_GDS_READ_RET;
gds.dst_gpr = ctx->file_offset[inst->Dst[0].Register.File] + inst->Dst[0].Register.Index;
gds.uav_id = uav_id;
gds.uav_index_mode = uav_index_mode;
gds.src_gpr = ctx->temp_reg;
gds.src_sel_x = 4;
gds.src_sel_y = 4;
gds.src_sel_z = 4;
gds.dst_sel_x = 0;
gds.dst_sel_y = 7;
gds.dst_sel_z = 7;
gds.dst_sel_w = 7;
gds.src_gpr2 = ctx->temp_reg;
gds.alloc_consume = 1;
r = r600_bytecode_add_gds(ctx->bc, &gds);
if (r)
return r;
ctx->bc->cf_last->vpm = 1;
return 0;
}
static int tgsi_load(struct r600_shader_ctx *ctx)
{
struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction;
if (inst->Src[0].Register.File == TGSI_FILE_HW_ATOMIC)
return tgsi_load_gds(ctx);
return 0;
}
static int get_gds_op(int opcode)
{
switch (opcode) {
case TGSI_OPCODE_ATOMUADD:
return FETCH_OP_GDS_ADD_RET;
case TGSI_OPCODE_ATOMAND:
return FETCH_OP_GDS_AND_RET;
case TGSI_OPCODE_ATOMOR:
return FETCH_OP_GDS_OR_RET;
case TGSI_OPCODE_ATOMXOR:
return FETCH_OP_GDS_XOR_RET;
case TGSI_OPCODE_ATOMUMIN:
return FETCH_OP_GDS_MIN_UINT_RET;
case TGSI_OPCODE_ATOMUMAX:
return FETCH_OP_GDS_MAX_UINT_RET;
case TGSI_OPCODE_ATOMXCHG:
return FETCH_OP_GDS_XCHG_RET;
case TGSI_OPCODE_ATOMCAS:
return FETCH_OP_GDS_CMP_XCHG_RET;
default:
return -1;
}
}
static int tgsi_atomic_op_gds(struct r600_shader_ctx *ctx)
{
struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction;
struct r600_bytecode_gds gds;
struct r600_bytecode_alu alu;
int gds_op = get_gds_op(inst->Instruction.Opcode);
int r;
int uav_id = 0;
int uav_index_mode = 0;
if (gds_op == -1) {
fprintf(stderr, "unknown GDS op for opcode %d\n", inst->Instruction.Opcode);
return -1;
}
uav_id = find_hw_atomic_counter(ctx, &inst->Src[0]);
if (inst->Src[0].Register.Indirect)
uav_index_mode = 2;
if (inst->Src[2].Register.File == TGSI_FILE_IMMEDIATE) {
int value = (ctx->literals[4 * inst->Src[2].Register.Index + inst->Src[2].Register.SwizzleX]);
int abs_value = abs(value);
if (abs_value != value && gds_op == FETCH_OP_GDS_ADD_RET)
gds_op = FETCH_OP_GDS_SUB_RET;
memset(&alu, 0, sizeof(struct r600_bytecode_alu));
alu.op = ALU_OP1_MOV;
alu.dst.sel = ctx->temp_reg;
alu.dst.chan = 0;
alu.src[0].sel = V_SQ_ALU_SRC_LITERAL;
alu.src[0].value = abs_value;
alu.last = 1;
alu.dst.write = 1;
r = r600_bytecode_add_alu(ctx->bc, &alu);
if (r)
return r;
} else {
memset(&alu, 0, sizeof(struct r600_bytecode_alu));
alu.op = ALU_OP1_MOV;
alu.dst.sel = ctx->temp_reg;
alu.dst.chan = 0;
r600_bytecode_src(&alu.src[0], &ctx->src[2], 0);
alu.last = 1;
alu.dst.write = 1;
r = r600_bytecode_add_alu(ctx->bc, &alu);
if (r)
return r;
}
memset(&gds, 0, sizeof(struct r600_bytecode_gds));
gds.op = gds_op;
gds.dst_gpr = ctx->file_offset[inst->Dst[0].Register.File] + inst->Dst[0].Register.Index;
gds.uav_id = uav_id;
gds.uav_index_mode = uav_index_mode;
gds.src_gpr = ctx->temp_reg;
gds.src_gpr2 = ctx->temp_reg;
gds.src_sel_x = 4;
gds.src_sel_y = 0;
gds.src_sel_z = 4;
gds.dst_sel_x = 0;
gds.dst_sel_y = 7;
gds.dst_sel_z = 7;
gds.dst_sel_w = 7;
gds.alloc_consume = 1;
r = r600_bytecode_add_gds(ctx->bc, &gds);
if (r)
return r;
ctx->bc->cf_last->vpm = 1;
return 0;
}
static int tgsi_atomic_op(struct r600_shader_ctx *ctx)
{
struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction;
if (inst->Src[0].Register.File == TGSI_FILE_HW_ATOMIC)
return tgsi_atomic_op_gds(ctx);
return 0;
}
static int tgsi_lrp(struct r600_shader_ctx *ctx)
{
struct tgsi_full_instruction *inst = &ctx->parse.FullToken.FullInstruction;
@ -9190,22 +9385,22 @@ static const struct r600_shader_tgsi_instruction eg_shader_tgsi_instruction[] =
[TGSI_OPCODE_UCMP] = { ALU_OP0_NOP, tgsi_ucmp},
[TGSI_OPCODE_IABS] = { 0, tgsi_iabs},
[TGSI_OPCODE_ISSG] = { 0, tgsi_issg},
[TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_load},
[TGSI_OPCODE_STORE] = { ALU_OP0_NOP, tgsi_unsupported},
[163] = { ALU_OP0_NOP, tgsi_unsupported},
[164] = { ALU_OP0_NOP, tgsi_unsupported},
[165] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_BARRIER] = { ALU_OP0_GROUP_BARRIER, tgsi_barrier},
[TGSI_OPCODE_ATOMUADD] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMXCHG] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMCAS] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMAND] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMOR] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMXOR] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMUMIN] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMUMAX] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMIMIN] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMIMAX] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMUADD] = { V_RAT_INST_ADD_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMXCHG] = { V_RAT_INST_XCHG_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMCAS] = { V_RAT_INST_CMPXCHG_INT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMAND] = { V_RAT_INST_AND_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMOR] = { V_RAT_INST_OR_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMXOR] = { V_RAT_INST_XOR_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMUMIN] = { V_RAT_INST_MIN_UINT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMUMAX] = { V_RAT_INST_MAX_UINT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMIMIN] = { V_RAT_INST_MIN_INT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMIMAX] = { V_RAT_INST_MAX_INT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_TEX2] = { FETCH_OP_SAMPLE, tgsi_tex},
[TGSI_OPCODE_TXB2] = { FETCH_OP_SAMPLE_LB, tgsi_tex},
[TGSI_OPCODE_TXL2] = { FETCH_OP_SAMPLE_L, tgsi_tex},
@ -9413,22 +9608,22 @@ static const struct r600_shader_tgsi_instruction cm_shader_tgsi_instruction[] =
[TGSI_OPCODE_UCMP] = { ALU_OP0_NOP, tgsi_ucmp},
[TGSI_OPCODE_IABS] = { 0, tgsi_iabs},
[TGSI_OPCODE_ISSG] = { 0, tgsi_issg},
[TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_LOAD] = { ALU_OP0_NOP, tgsi_load},
[TGSI_OPCODE_STORE] = { ALU_OP0_NOP, tgsi_unsupported},
[163] = { ALU_OP0_NOP, tgsi_unsupported},
[164] = { ALU_OP0_NOP, tgsi_unsupported},
[165] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_BARRIER] = { ALU_OP0_GROUP_BARRIER, tgsi_barrier},
[TGSI_OPCODE_ATOMUADD] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMXCHG] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMCAS] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMAND] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMOR] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMXOR] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMUMIN] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMUMAX] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMIMIN] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMIMAX] = { ALU_OP0_NOP, tgsi_unsupported},
[TGSI_OPCODE_ATOMUADD] = { V_RAT_INST_ADD_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMXCHG] = { V_RAT_INST_XCHG_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMCAS] = { V_RAT_INST_CMPXCHG_INT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMAND] = { V_RAT_INST_AND_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMOR] = { V_RAT_INST_OR_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMXOR] = { V_RAT_INST_XOR_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMUMIN] = { V_RAT_INST_MIN_UINT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMUMAX] = { V_RAT_INST_MAX_UINT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMIMIN] = { V_RAT_INST_MIN_INT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_ATOMIMAX] = { V_RAT_INST_MAX_INT_RTN, tgsi_atomic_op},
[TGSI_OPCODE_TEX2] = { FETCH_OP_SAMPLE, tgsi_tex},
[TGSI_OPCODE_TXB2] = { FETCH_OP_SAMPLE_LB, tgsi_tex},
[TGSI_OPCODE_TXL2] = { FETCH_OP_SAMPLE_L, tgsi_tex},

19
src/gallium/drivers/r600/r600_shader.h
View File

@ -56,15 +56,25 @@ struct r600_shader_io {
int ring_offset;
};
struct r600_shader_atomic {
unsigned start, end;
unsigned buffer_id;
unsigned hw_idx;
unsigned array_id;
};
struct r600_shader {
unsigned processor_type;
struct r600_bytecode bc;
unsigned ninput;
unsigned noutput;
unsigned nhwatomic;
unsigned nlds;
unsigned nsys_inputs;
struct r600_shader_io input[64];
struct r600_shader_io output[64];
struct r600_shader_atomic atomics[8];
unsigned nhwatomic_ranges;
boolean uses_kill;
boolean fs_write_all;
boolean two_side;
@ -105,26 +115,35 @@ struct r600_shader {
struct r600_shader_array * arrays;
boolean uses_doubles;
boolean uses_atomics;
uint8_t atomic_base;
};
union r600_shader_key {
struct {
unsigned nr_cbufs:4;
unsigned first_atomic_counter:4;
unsigned color_two_side:1;
unsigned alpha_to_one:1;
} ps;
struct {
unsigned prim_id_out:8;
unsigned first_atomic_counter:4;
unsigned as_es:1; /* export shader */
unsigned as_ls:1; /* local shader */
unsigned as_gs_a:1;
} vs;
struct {
unsigned first_atomic_counter:4;
unsigned as_es:1;
} tes;
struct {
unsigned first_atomic_counter:4;
unsigned prim_mode:3;
} tcs;
struct {
unsigned first_atomic_counter:4;
} gs;
};
struct r600_shader_array {

46
src/gallium/drivers/r600/r600_state_common.c
View File

@ -698,6 +698,38 @@ static void r600_update_compressed_colortex_mask(struct r600_samplerview_state *
}
}
static int r600_get_hw_atomic_count(const struct pipe_context *ctx,
enum pipe_shader_type shader)
{
const struct r600_context *rctx = (struct r600_context *)ctx;
int value = 0;
switch (shader) {
case PIPE_SHADER_FRAGMENT:
case PIPE_SHADER_COMPUTE:
default:
break;
case PIPE_SHADER_VERTEX:
value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC];
break;
case PIPE_SHADER_GEOMETRY:
value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC] +
rctx->vs_shader->info.file_count[TGSI_FILE_HW_ATOMIC];
break;
case PIPE_SHADER_TESS_EVAL:
value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC] +
rctx->vs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] +
(rctx->gs_shader ? rctx->gs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] : 0);
break;
case PIPE_SHADER_TESS_CTRL:
value = rctx->ps_shader->info.file_count[TGSI_FILE_HW_ATOMIC] +
rctx->vs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] +
(rctx->gs_shader ? rctx->gs_shader->info.file_count[TGSI_FILE_HW_ATOMIC] : 0) +
rctx->tes_shader->info.file_count[TGSI_FILE_HW_ATOMIC];
break;
}
return value;
}
/* Compute the key for the hw shader variant */
static inline void r600_shader_selector_key(const struct pipe_context *ctx,
const struct r600_pipe_shader_selector *sel,
@ -716,11 +748,14 @@ static inline void r600_shader_selector_key(const struct pipe_context *ctx,
key->vs.as_gs_a = true;
key->vs.prim_id_out = rctx->ps_shader->current->shader.input[rctx->ps_shader->current->shader.ps_prim_id_input].spi_sid;
}
key->vs.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_VERTEX);
break;
}
case PIPE_SHADER_GEOMETRY:
key->gs.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_GEOMETRY);
break;
case PIPE_SHADER_FRAGMENT: {
key->ps.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_FRAGMENT);
key->ps.color_two_side = rctx->rasterizer && rctx->rasterizer->two_side;
key->ps.alpha_to_one = rctx->alpha_to_one &&
rctx->rasterizer && rctx->rasterizer->multisample_enable &&
@ -733,9 +768,11 @@ static inline void r600_shader_selector_key(const struct pipe_context *ctx,
}
case PIPE_SHADER_TESS_EVAL:
key->tes.as_es = (rctx->gs_shader != NULL);
key->tes.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_TESS_EVAL);
break;
case PIPE_SHADER_TESS_CTRL:
key->tcs.prim_mode = rctx->tes_shader->info.properties[TGSI_PROPERTY_TES_PRIM_MODE];
key->tcs.first_atomic_counter = r600_get_hw_atomic_count(ctx, PIPE_SHADER_TESS_CTRL);
break;
default:
assert(0);
@ -1700,6 +1737,8 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info
unsigned num_patches, dirty_tex_counter, index_offset = 0;
unsigned index_size = info->index_size;
int index_bias;
struct r600_shader_atomic combined_atomics[8];
uint8_t atomic_used_mask;
if (!info->indirect && !info->count && (index_size || !info->count_from_stream_output)) {
return;
@ -1739,6 +1778,9 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info
: (rctx->tes_shader)? rctx->tes_shader->info.properties[TGSI_PROPERTY_TES_PRIM_MODE]
: info->mode;
if (rctx->b.chip_class >= EVERGREEN)
evergreen_emit_atomic_buffer_setup(rctx, combined_atomics, &atomic_used_mask);
if (index_size) {
index_offset += info->start * index_size;
@ -2019,6 +2061,10 @@ static void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info
radeon_emit(cs, EVENT_TYPE(EVENT_TYPE_SQ_NON_EVENT));
}
if (rctx->b.chip_class >= EVERGREEN)
evergreen_emit_atomic_buffer_save(rctx, combined_atomics, &atomic_used_mask);
if (rctx->trace_buf)
eg_trace_emit(rctx);

2
src/gallium/drivers/r600/r600d_common.h
View File

@ -51,6 +51,8 @@
#define STRMOUT_SELECT_BUFFER(x) (((unsigned)(x) & 0x3) << 8)
#define PKT3_WAIT_REG_MEM 0x3C
#define WAIT_REG_MEM_EQUAL 3
#define WAIT_REG_MEM_GEQUAL 5
#define WAIT_REG_MEM_MEMORY (1 << 4)
#define WAIT_REG_MEM_MEM_SPACE(x) (((unsigned)(x) & 0x3) << 4)
#define PKT3_COPY_DATA 0x40
#define COPY_DATA_SRC_SEL(x) ((x) & 0xf)