mirrors
/
mesa
mirror of https://gitlab.freedesktop.org/mesa/mesa
1
0
Fork 0
Code Issues 9 Packages Projects Releases Wiki Activity
mesa/src/amd/compiler/aco_insert_exec_mask.cpp

802 lines
30 KiB
C++
Raw Permalink Blame History

/*
* Copyright © 2019 Valve Corporation
*
* SPDX-License-Identifier: MIT
*/
#include "aco_builder.h"
#include "aco_ir.h"
#include "util/u_math.h"
#include <set>
#include <vector>
namespace aco {
namespace {
enum WQMState : uint8_t {
Unspecified = 0,
Exact = 1 << 0,
WQM = 1 << 1, /* with control flow applied */
};
enum mask_type : uint8_t {
mask_type_global = 1 << 0,
mask_type_exact = 1 << 1,
mask_type_wqm = 1 << 2,
mask_type_loop = 1 << 3, /* active lanes of a loop */
};
struct loop_info {
Block* loop_header;
uint16_t num_exec_masks;
bool has_divergent_break;
bool has_divergent_continue;
bool has_discard; /* has a discard or demote */
loop_info(Block* b, uint16_t num, bool breaks, bool cont, bool discard)
: loop_header(b), num_exec_masks(num), has_divergent_break(breaks),
has_divergent_continue(cont), has_discard(discard)
{}
};
struct block_info {
std::vector<std::pair<Operand, uint8_t>>
exec; /* Vector of exec masks. Either a temporary or const -1. */
};
struct exec_ctx {
Program* program;
std::vector<block_info> info;
std::vector<loop_info> loop;
bool handle_wqm = false;
exec_ctx(Program* program_) : program(program_), info(program->blocks.size()) {}
};
bool
needs_exact(aco_ptr<Instruction>& instr)
{
if (instr->isMUBUF()) {
return instr->mubuf().disable_wqm;
} else if (instr->isMTBUF()) {
return instr->mtbuf().disable_wqm;
} else if (instr->isMIMG()) {
return instr->mimg().disable_wqm;
} else if (instr->isFlatLike()) {
return instr->flatlike().disable_wqm;
} else {
/* Require Exact for p_jump_to_epilog because if p_exit_early_if is
* emitted inside the same block, the main FS will always jump to the PS
* epilog without considering the exec mask.
*/
return instr->isEXP() || instr->opcode == aco_opcode::p_jump_to_epilog ||
instr->opcode == aco_opcode::p_dual_src_export_gfx11;
}
}
WQMState
get_instr_needs(aco_ptr<Instruction>& instr)
{
if (needs_exact(instr))
return Exact;
bool pred_by_exec = needs_exec_mask(instr.get()) || instr->opcode == aco_opcode::p_logical_end ||
instr->isBranch();
return pred_by_exec ? WQM : Unspecified;
}
Operand
get_exec_op(Operand t)
{
if (t.isUndefined())
return Operand(exec, t.regClass());
else
return t;
}
void
transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
{
if (ctx.info[idx].exec.back().second & mask_type_wqm)
return;
if (ctx.info[idx].exec.back().second & mask_type_global) {
Operand exec_mask = ctx.info[idx].exec.back().first;
if (exec_mask.isUndefined())
ctx.info[idx].exec.back().first = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
exec_mask = bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
get_exec_op(exec_mask));
ctx.info[idx].exec.emplace_back(exec_mask, mask_type_global | mask_type_wqm);
return;
}
/* otherwise, the WQM mask should be one below the current mask */
ctx.info[idx].exec.pop_back();
assert(ctx.info[idx].exec.back().second & mask_type_wqm);
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
assert(ctx.info[idx].exec.back().first.isTemp());
ctx.info[idx].exec.back().first =
bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
}
void
transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
{
if (ctx.info[idx].exec.back().second & mask_type_exact)
return;
/* We can't remove the loop exec mask, because that can cause exec.size() to
* be less than num_exec_masks. The loop exec mask also needs to be kept
* around for various uses. */
if ((ctx.info[idx].exec.back().second & mask_type_global) &&
!(ctx.info[idx].exec.back().second & mask_type_loop)) {
ctx.info[idx].exec.pop_back();
assert(ctx.info[idx].exec.back().second & mask_type_exact);
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
assert(ctx.info[idx].exec.back().first.isTemp());
ctx.info[idx].exec.back().first =
bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
return;
}
/* otherwise, we create an exact mask and push to the stack */
Operand wqm = ctx.info[idx].exec.back().first;
if (wqm.isUndefined()) {
wqm = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
Definition(exec, bld.lm), ctx.info[idx].exec[0].first, Operand(exec, bld.lm));
} else {
bld.sop2(Builder::s_and, Definition(exec, bld.lm), bld.def(s1, scc),
ctx.info[idx].exec[0].first, wqm);
}
ctx.info[idx].exec.back().first = Operand(wqm);
ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type_exact);
}
unsigned
add_coupling_code(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions)
{
unsigned idx = block->index;
Builder bld(ctx.program, &instructions);
Block::edge_vec& preds = block->linear_preds;
bool restore_exec = false;
/* start block */
if (preds.empty()) {
aco_ptr<Instruction>& startpgm = block->instructions[0];
assert(startpgm->opcode == aco_opcode::p_startpgm);
bld.insert(std::move(startpgm));
unsigned count = 1;
if (block->instructions[1]->opcode == aco_opcode::p_init_scratch) {
bld.insert(std::move(block->instructions[1]));
count++;
}
Operand start_exec(bld.lm);
/* exec seems to need to be manually initialized with combined shaders */
if (ctx.program->stage.num_sw_stages() > 1 ||
ctx.program->stage.hw == AC_HW_NEXT_GEN_GEOMETRY_SHADER ||
(ctx.program->stage.sw == SWStage::VS &&
(ctx.program->stage.hw == AC_HW_HULL_SHADER ||
ctx.program->stage.hw == AC_HW_LEGACY_GEOMETRY_SHADER)) ||
(ctx.program->stage.sw == SWStage::TES &&
ctx.program->stage.hw == AC_HW_LEGACY_GEOMETRY_SHADER)) {
start_exec = Operand::c32_or_c64(-1u, bld.lm == s2);
bld.copy(Definition(exec, bld.lm), start_exec);
}
/* EXEC is automatically initialized by the HW for compute shaders.
* We know for sure exec is initially -1 when the shader always has full subgroups.
*/
if (ctx.program->stage == compute_cs && ctx.program->info.cs.uses_full_subgroups)
start_exec = Operand::c32_or_c64(-1u, bld.lm == s2);
if (ctx.handle_wqm) {
ctx.info[idx].exec.emplace_back(start_exec, mask_type_global | mask_type_exact);
/* Initialize WQM already */
transition_to_WQM(ctx, bld, idx);
} else {
uint8_t mask = mask_type_global;
if (ctx.program->needs_wqm) {
bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
Operand(exec, bld.lm));
mask |= mask_type_wqm;
} else {
mask |= mask_type_exact;
}
ctx.info[idx].exec.emplace_back(start_exec, mask);
}
return count;
}
/* loop entry block */
if (block->kind & block_kind_loop_header) {
assert(preds[0] == idx - 1);
ctx.info[idx].exec = ctx.info[idx - 1].exec;
loop_info& info = ctx.loop.back();
assert(ctx.info[idx].exec.size() == info.num_exec_masks);
/* create ssa names for outer exec masks */
if (info.has_discard && preds.size() > 1) {
aco_ptr<Instruction> phi;
for (int i = 0; i < info.num_exec_masks - 1; i++) {
phi.reset(
create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1));
phi->definitions[0] = bld.def(bld.lm);
phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[i].first);
ctx.info[idx].exec[i].first = bld.insert(std::move(phi));
}
}
ctx.info[idx].exec.back().second |= mask_type_loop;
if (info.has_divergent_continue) {
/* create ssa name for loop active mask */
aco_ptr<Instruction> phi{
create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
phi->definitions[0] = bld.def(bld.lm);
phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec.back().first);
ctx.info[idx].exec.back().first = bld.insert(std::move(phi));
restore_exec = true;
uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
ctx.info[idx].exec.emplace_back(ctx.info[idx].exec.back().first, mask_type);
}
} else if (block->kind & block_kind_loop_exit) {
Block* header = ctx.loop.back().loop_header;
loop_info& info = ctx.loop.back();
for (ASSERTED unsigned pred : preds)
assert(ctx.info[pred].exec.size() >= info.num_exec_masks);
/* fill the loop header phis */
Block::edge_vec& header_preds = header->linear_preds;
int instr_idx = 0;
if (info.has_discard && header_preds.size() > 1) {
while (instr_idx < info.num_exec_masks - 1) {
aco_ptr<Instruction>& phi = header->instructions[instr_idx];
assert(phi->opcode == aco_opcode::p_linear_phi);
for (unsigned i = 1; i < phi->operands.size(); i++)
phi->operands[i] = get_exec_op(ctx.info[header_preds[i]].exec[instr_idx].first);
instr_idx++;
}
}
if (info.has_divergent_continue) {
aco_ptr<Instruction>& phi = header->instructions[instr_idx++];
assert(phi->opcode == aco_opcode::p_linear_phi);
for (unsigned i = 1; i < phi->operands.size(); i++)
phi->operands[i] =
get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks - 1].first);
}
if (info.has_divergent_break) {
restore_exec = true;
/* Drop the loop active mask. */
info.num_exec_masks--;
}
assert(!(block->kind & block_kind_top_level) || info.num_exec_masks <= 2);
/* create the loop exit phis if not trivial */
for (unsigned exec_idx = 0; exec_idx < info.num_exec_masks; exec_idx++) {
Operand same = ctx.info[preds[0]].exec[exec_idx].first;
uint8_t type = ctx.info[header_preds[0]].exec[exec_idx].second;
bool trivial = true;
for (unsigned i = 1; i < preds.size() && trivial; i++) {
if (ctx.info[preds[i]].exec[exec_idx].first != same)
trivial = false;
}
if (trivial) {
ctx.info[idx].exec.emplace_back(same, type);
} else {
/* create phi for loop footer */
aco_ptr<Instruction> phi{
create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
phi->definitions[0] = bld.def(bld.lm);
for (unsigned i = 0; i < phi->operands.size(); i++)
phi->operands[i] = get_exec_op(ctx.info[preds[i]].exec[exec_idx].first);
ctx.info[idx].exec.emplace_back(bld.insert(std::move(phi)), type);
}
}
assert(ctx.info[idx].exec.size() == info.num_exec_masks);
ctx.loop.pop_back();
} else if (preds.size() == 1) {
ctx.info[idx].exec = ctx.info[preds[0]].exec;
} else {
assert(preds.size() == 2);
/* if one of the predecessors ends in exact mask, we pop it from stack */
unsigned num_exec_masks =
std::min(ctx.info[preds[0]].exec.size(), ctx.info[preds[1]].exec.size());
if (block->kind & block_kind_merge) {
restore_exec = true;
num_exec_masks--;
}
if (block->kind & block_kind_top_level)
num_exec_masks = std::min(num_exec_masks, 2u);
/* create phis for diverged exec masks */
for (unsigned i = 0; i < num_exec_masks; i++) {
/* skip trivial phis */
if (ctx.info[preds[0]].exec[i].first == ctx.info[preds[1]].exec[i].first) {
Operand t = ctx.info[preds[0]].exec[i].first;
/* discard/demote can change the state of the current exec mask */
assert(!t.isTemp() ||
ctx.info[preds[0]].exec[i].second == ctx.info[preds[1]].exec[i].second);
uint8_t mask = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
ctx.info[idx].exec.emplace_back(t, mask);
continue;
}
Temp phi = bld.pseudo(aco_opcode::p_linear_phi, bld.def(bld.lm),
get_exec_op(ctx.info[preds[0]].exec[i].first),
get_exec_op(ctx.info[preds[1]].exec[i].first));
uint8_t mask_type = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
ctx.info[idx].exec.emplace_back(phi, mask_type);
}
}
unsigned i = 0;
while (block->instructions[i]->opcode == aco_opcode::p_phi ||
block->instructions[i]->opcode == aco_opcode::p_linear_phi) {
bld.insert(std::move(block->instructions[i]));
i++;
}
if (ctx.handle_wqm) {
/* End WQM handling if not needed anymore */
if (block->kind & block_kind_top_level && ctx.info[idx].exec.size() == 2) {
if (block->instructions[i]->opcode == aco_opcode::p_end_wqm) {
ctx.info[idx].exec.back().second |= mask_type_global;
transition_to_Exact(ctx, bld, idx);
ctx.handle_wqm = false;
restore_exec = false;
i++;
}
}
}
/* restore exec mask after divergent control flow */
if (restore_exec) {
Operand restore = get_exec_op(ctx.info[idx].exec.back().first);
assert(restore.size() == bld.lm.size());
bld.copy(Definition(exec, bld.lm), restore);
if (!restore.isConstant())
ctx.info[idx].exec.back().first = Operand(bld.lm);
}
return i;
}
/* Avoid live-range splits in Exact mode:
* Because the data register of atomic VMEM instructions
* is shared between src and dst, it might be necessary
* to create live-range splits during RA.
* Make the live-range splits explicit in WQM mode.
*/
void
handle_atomic_data(exec_ctx& ctx, Builder& bld, unsigned block_idx, aco_ptr<Instruction>& instr)
{
/* check if this is an atomic VMEM instruction */
int idx = -1;
if (!instr->isVMEM() || instr->definitions.empty())
return;
else if (instr->isMIMG())
idx = instr->operands[2].isTemp() ? 2 : -1;
else if (instr->operands.size() == 4)
idx = 3;
if (idx != -1) {
/* insert explicit copy of atomic data in WQM-mode */
transition_to_WQM(ctx, bld, block_idx);
Temp data = instr->operands[idx].getTemp();
data = bld.copy(bld.def(data.regClass()), data);
instr->operands[idx].setTemp(data);
}
}
void
process_instructions(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions,
unsigned idx)
{
block_info& info = ctx.info[block->index];
WQMState state;
if (info.exec.back().second & mask_type_wqm) {
state = WQM;
} else {
assert(!ctx.handle_wqm || info.exec.back().second & mask_type_exact);
state = Exact;
}
Builder bld(ctx.program, &instructions);
for (; idx < block->instructions.size(); idx++) {
aco_ptr<Instruction> instr = std::move(block->instructions[idx]);
WQMState needs = ctx.handle_wqm ? get_instr_needs(instr) : Unspecified;
if (needs == WQM && state != WQM) {
transition_to_WQM(ctx, bld, block->index);
state = WQM;
} else if (needs == Exact) {
if (ctx.handle_wqm)
handle_atomic_data(ctx, bld, block->index, instr);
transition_to_Exact(ctx, bld, block->index);
state = Exact;
}
if (instr->opcode == aco_opcode::p_discard_if) {
Operand current_exec = Operand(exec, bld.lm);
if (block->instructions[idx + 1]->opcode == aco_opcode::p_end_wqm) {
/* Transition to Exact without extra instruction. */
info.exec.resize(1);
assert(info.exec[0].second == (mask_type_exact | mask_type_global));
current_exec = get_exec_op(info.exec[0].first);
info.exec[0].first = Operand(bld.lm);
state = Exact;
} else if (info.exec.size() >= 2 && ctx.handle_wqm) {
/* Preserve the WQM mask */
info.exec[1].second &= ~mask_type_global;
}
Temp cond, exit_cond;
if (instr->operands[0].isConstant()) {
assert(instr->operands[0].constantValue() == -1u);
/* save condition and set exec to zero */
exit_cond = bld.tmp(s1);
cond =
bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
Definition(exec, bld.lm), Operand::zero(), Operand(exec, bld.lm));
} else {
cond = instr->operands[0].getTemp();
/* discard from current exec */
exit_cond = bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc),
current_exec, cond)
.def(1)
.getTemp();
}
/* discard from inner to outer exec mask on stack */
int num = info.exec.size() - 2;
for (int i = num; i >= 0; i--) {
Instruction* andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
info.exec[i].first, cond);
info.exec[i].first = Operand(andn2->definitions[0].getTemp());
exit_cond = andn2->definitions[1].getTemp();
}
instr->opcode = aco_opcode::p_exit_early_if;
instr->operands[0] = bld.scc(exit_cond);
assert(!ctx.handle_wqm || (info.exec[0].second & mask_type_wqm) == 0);
} else if (instr->opcode == aco_opcode::p_is_helper) {
Definition dst = instr->definitions[0];
assert(dst.size() == bld.lm.size());
if (state == Exact) {
instr.reset(create_instruction(bld.w64or32(Builder::s_mov), Format::SOP1, 1, 1));
instr->operands[0] = Operand::zero();
instr->definitions[0] = dst;
} else {
std::pair<Operand, uint8_t>& exact_mask = info.exec[0];
assert(exact_mask.second & mask_type_exact);
instr.reset(create_instruction(bld.w64or32(Builder::s_andn2), Format::SOP2, 2, 2));
instr->operands[0] = Operand(exec, bld.lm); /* current exec */
instr->operands[1] = Operand(exact_mask.first);
instr->definitions[0] = dst;
instr->definitions[1] = bld.def(s1, scc);
}
} else if (instr->opcode == aco_opcode::p_demote_to_helper) {
assert((info.exec[0].second & mask_type_exact) &&
(info.exec[0].second & mask_type_global));
const bool nested_cf = !(info.exec.back().second & mask_type_global);
if (ctx.handle_wqm && state == Exact && nested_cf) {
/* Transition back to WQM without extra instruction. */
info.exec.pop_back();
state = WQM;
} else if (block->instructions[idx + 1]->opcode == aco_opcode::p_end_wqm) {
/* Transition to Exact without extra instruction. */
info.exec.resize(1);
state = Exact;
} else if (nested_cf) {
/* Save curent exec temporarily. */
info.exec.back().first = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
}
/* Remove invocations from global exact mask. */
Definition def = state == Exact ? Definition(exec, bld.lm) : bld.def(bld.lm);
Operand src = instr->operands[0].isConstant() ? Operand(exec, bld.lm) : instr->operands[0];
Definition exit_cond =
bld.sop2(Builder::s_andn2, def, bld.def(s1, scc), get_exec_op(info.exec[0].first), src)
.def(1);
info.exec[0].first = Operand(def.getTemp());
/* Update global WQM mask and store in exec. */
if (state == WQM) {
assert(info.exec.size() > 1);
exit_cond =
bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc), def.getTemp())
.def(1);
}
/* End shader if global mask is zero. */
instr->opcode = aco_opcode::p_exit_early_if;
instr->operands[0] = bld.scc(exit_cond.getTemp());
bld.insert(std::move(instr));
/* Update all other exec masks. */
if (nested_cf) {
const unsigned global_idx = state == WQM ? 1 : 0;
for (unsigned i = global_idx + 1; i < info.exec.size() - 1; i++) {
info.exec[i].first =
bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc),
get_exec_op(info.exec[i].first), Operand(exec, bld.lm));
}
/* Update current exec and save WQM mask. */
info.exec[global_idx].first =
bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
Definition(exec, bld.lm), info.exec.back().first, Operand(exec, bld.lm));
info.exec.back().first = Operand(bld.lm);
}
continue;
} else if (instr->opcode == aco_opcode::p_elect) {
bool all_lanes_enabled = info.exec.back().first.constantEquals(-1u);
Definition dst = instr->definitions[0];
if (all_lanes_enabled) {
bld.copy(Definition(dst), Operand::c32_or_c64(1u, dst.size() == 2));
} else {
Temp first_lane_idx = bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm));
bld.sop2(Builder::s_lshl, Definition(dst), bld.def(s1, scc),
Operand::c32_or_c64(1u, dst.size() == 2), Operand(first_lane_idx));
}
continue;
} else if (instr->opcode == aco_opcode::p_end_wqm) {
assert(block->kind & block_kind_top_level);
assert(info.exec.size() <= 2);
/* This instruction indicates the end of WQM mode. */
info.exec.back().second |= mask_type_global;
transition_to_Exact(ctx, bld, block->index);
state = Exact;
ctx.handle_wqm = false;
continue;
}
bld.insert(std::move(instr));
}
}
void
add_branch_code(exec_ctx& ctx, Block* block)
{
unsigned idx = block->index;
Builder bld(ctx.program, block);
if (block->linear_succs.empty())
return;
if (block->kind & block_kind_loop_preheader) {
/* collect information about the succeeding loop */
bool has_divergent_break = false;
bool has_divergent_continue = false;
bool has_discard = false;
unsigned loop_nest_depth = ctx.program->blocks[idx + 1].loop_nest_depth;
for (unsigned i = idx + 1; ctx.program->blocks[i].loop_nest_depth >= loop_nest_depth; i++) {
Block& loop_block = ctx.program->blocks[i];
if (loop_block.kind & block_kind_uses_discard)
has_discard = true;
if (loop_block.loop_nest_depth != loop_nest_depth)
continue;
if (loop_block.kind & block_kind_uniform)
continue;
else if (loop_block.kind & block_kind_break)
has_divergent_break = true;
else if (loop_block.kind & block_kind_continue)
has_divergent_continue = true;
}
if (has_divergent_break) {
/* save restore exec mask */
uint8_t mask = ctx.info[idx].exec.back().second;
if (ctx.info[idx].exec.back().first.constantEquals(-1u)) {
ctx.info[idx].exec.emplace_back(Operand(exec, bld.lm), mask);
} else {
bld.reset(bld.instructions, std::prev(bld.instructions->end()));
Operand restore = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
ctx.info[idx].exec.emplace(std::prev(ctx.info[idx].exec.end()), restore, mask);
bld.reset(bld.instructions);
}
ctx.info[idx].exec.back().second &= (mask_type_wqm | mask_type_exact);
}
unsigned num_exec_masks = ctx.info[idx].exec.size();
ctx.loop.emplace_back(&ctx.program->blocks[block->linear_succs[0]], num_exec_masks,
has_divergent_break, has_divergent_continue, has_discard);
}
/* For normal breaks, this is the exec mask. For discard+break, it's the
* old exec mask before it was zero'd.
*/
Operand break_cond = Operand(exec, bld.lm);
if (block->kind & block_kind_continue_or_break) {
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind &
block_kind_loop_header);
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind &
block_kind_loop_exit);
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
block->instructions.pop_back();
bool need_parallelcopy = false;
while (!(ctx.info[idx].exec.back().second & mask_type_loop)) {
ctx.info[idx].exec.pop_back();
need_parallelcopy = true;
}
if (need_parallelcopy)
ctx.info[idx].exec.back().first =
bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), Operand(exec, bld.lm),
block->linear_succs[1], block->linear_succs[0]);
return;
}
if (block->kind & block_kind_uniform) {
Pseudo_branch_instruction& branch = block->instructions.back()->branch();
if (branch.opcode == aco_opcode::p_branch) {
branch.target[0] = block->linear_succs[0];
} else {
branch.target[0] = block->linear_succs[1];
branch.target[1] = block->linear_succs[0];
}
return;
}
if (block->kind & block_kind_branch) {
// orig = s_and_saveexec_b64
assert(block->linear_succs.size() == 2);
assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_z);
Temp cond = block->instructions.back()->operands[0].getTemp();
const bool sel_ctrl = block->instructions.back()->branch().selection_control_remove;
block->instructions.pop_back();
uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
if (ctx.info[idx].exec.back().first.constantEquals(-1u)) {
bld.copy(Definition(exec, bld.lm), cond);
} else {
Temp old_exec = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
Definition(exec, bld.lm), cond, Operand(exec, bld.lm));
ctx.info[idx].exec.back().first = Operand(old_exec);
}
/* add next current exec to the stack */
ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type);
Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
block->linear_succs[1], block->linear_succs[0]);
r->branch().selection_control_remove = sel_ctrl;
return;
}
if (block->kind & block_kind_invert) {
// exec = s_andn2_b64 (original_exec, exec)
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
const bool sel_ctrl = block->instructions.back()->branch().selection_control_remove;
block->instructions.pop_back();
assert(ctx.info[idx].exec.size() >= 2);
Operand orig_exec = ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].first;
bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc), orig_exec,
Operand(exec, bld.lm));
Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
block->linear_succs[1], block->linear_succs[0]);
r->branch().selection_control_remove = sel_ctrl;
return;
}
if (block->kind & block_kind_break) {
// loop_mask = s_andn2_b64 (loop_mask, exec)
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
block->instructions.pop_back();
Temp cond = Temp();
for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
cond = bld.tmp(s1);
Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
exec_mask, break_cond);
ctx.info[idx].exec[exec_idx].first = exec_mask;
if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
break;
}
/* check if the successor is the merge block, otherwise set exec to 0 */
// TODO: this could be done better by directly branching to the merge block
unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
Block& succ = ctx.program->blocks[succ_idx];
if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
}
bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
block->linear_succs[0]);
return;
}
if (block->kind & block_kind_continue) {
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
block->instructions.pop_back();
Temp cond = Temp();
for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
break;
cond = bld.tmp(s1);
Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
exec_mask, Operand(exec, bld.lm));
ctx.info[idx].exec[exec_idx].first = exec_mask;
}
assert(cond != Temp());
/* check if the successor is the merge block, otherwise set exec to 0 */
// TODO: this could be done better by directly branching to the merge block
unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
Block& succ = ctx.program->blocks[succ_idx];
if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
}
bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
block->linear_succs[0]);
return;
}
}
void
process_block(exec_ctx& ctx, Block* block)
{
std::vector<aco_ptr<Instruction>> instructions;
instructions.reserve(block->instructions.size());
unsigned idx = add_coupling_code(ctx, block, instructions);
assert(!block->linear_succs.empty() || ctx.info[block->index].exec.size() <= 2);
process_instructions(ctx, block, instructions, idx);
block->instructions = std::move(instructions);
add_branch_code(ctx, block);
}
} /* end namespace */
void
insert_exec_mask(Program* program)
{
exec_ctx ctx(program);
if (program->needs_wqm && program->needs_exact)
ctx.handle_wqm = true;
for (Block& block : program->blocks)
process_block(ctx, &block);
}
} // namespace aco
Reference in New Issue View Git Blame Copy Permalink