2019-09-17 12:22:17 +01:00
|
|
|
/*
|
|
|
|
* Copyright © 2018 Valve Corporation
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "aco_builder.h"
|
2021-06-09 14:40:03 +01:00
|
|
|
#include "aco_ir.h"
|
2021-06-09 09:14:54 +01:00
|
|
|
|
2021-06-10 10:33:15 +01:00
|
|
|
#include "common/sid.h"
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2021-06-09 14:40:03 +01:00
|
|
|
#include <map>
|
|
|
|
#include <vector>
|
2019-09-17 12:22:17 +01:00
|
|
|
|
|
|
|
namespace aco {
|
|
|
|
|
|
|
|
struct lower_context {
|
2021-06-09 09:14:54 +01:00
|
|
|
Program* program;
|
|
|
|
Block* block;
|
2019-09-17 12:22:17 +01:00
|
|
|
std::vector<aco_ptr<Instruction>> instructions;
|
|
|
|
};
|
|
|
|
|
2020-05-15 16:28:03 +01:00
|
|
|
/* used by handle_operands() indirectly through Builder::copy */
|
|
|
|
uint8_t int8_mul_table[512] = {
|
2021-06-09 09:14:54 +01:00
|
|
|
0, 20, 1, 1, 1, 2, 1, 3, 1, 4, 1, 5, 1, 6, 1, 7, 1, 8, 1, 9,
|
|
|
|
1, 10, 1, 11, 1, 12, 1, 13, 1, 14, 1, 15, 1, 16, 1, 17, 1, 18, 1, 19,
|
|
|
|
1, 20, 1, 21, 1, 22, 1, 23, 1, 24, 1, 25, 1, 26, 1, 27, 1, 28, 1, 29,
|
|
|
|
1, 30, 1, 31, 1, 32, 1, 33, 1, 34, 1, 35, 1, 36, 1, 37, 1, 38, 1, 39,
|
|
|
|
1, 40, 1, 41, 1, 42, 1, 43, 1, 44, 1, 45, 1, 46, 1, 47, 1, 48, 1, 49,
|
|
|
|
1, 50, 1, 51, 1, 52, 1, 53, 1, 54, 1, 55, 1, 56, 1, 57, 1, 58, 1, 59,
|
|
|
|
1, 60, 1, 61, 1, 62, 1, 63, 1, 64, 5, 13, 2, 33, 17, 19, 2, 34, 3, 23,
|
|
|
|
2, 35, 11, 53, 2, 36, 7, 47, 2, 37, 3, 25, 2, 38, 7, 11, 2, 39, 53, 243,
|
|
|
|
2, 40, 3, 27, 2, 41, 17, 35, 2, 42, 5, 17, 2, 43, 3, 29, 2, 44, 15, 23,
|
|
|
|
2, 45, 7, 13, 2, 46, 3, 31, 2, 47, 5, 19, 2, 48, 19, 59, 2, 49, 3, 33,
|
|
|
|
2, 50, 7, 51, 2, 51, 15, 41, 2, 52, 3, 35, 2, 53, 11, 33, 2, 54, 23, 27,
|
|
|
|
2, 55, 3, 37, 2, 56, 9, 41, 2, 57, 5, 23, 2, 58, 3, 39, 2, 59, 7, 17,
|
|
|
|
2, 60, 9, 241, 2, 61, 3, 41, 2, 62, 5, 25, 2, 63, 35, 245, 2, 64, 3, 43,
|
|
|
|
5, 26, 9, 43, 3, 44, 7, 19, 10, 39, 3, 45, 4, 34, 11, 59, 3, 46, 9, 243,
|
|
|
|
4, 35, 3, 47, 22, 53, 7, 57, 3, 48, 5, 29, 10, 245, 3, 49, 4, 37, 9, 45,
|
|
|
|
3, 50, 7, 241, 4, 38, 3, 51, 7, 22, 5, 31, 3, 52, 7, 59, 7, 242, 3, 53,
|
|
|
|
4, 40, 7, 23, 3, 54, 15, 45, 4, 41, 3, 55, 6, 241, 9, 47, 3, 56, 13, 13,
|
|
|
|
5, 34, 3, 57, 4, 43, 11, 39, 3, 58, 5, 35, 4, 44, 3, 59, 6, 243, 7, 245,
|
|
|
|
3, 60, 5, 241, 7, 26, 3, 61, 4, 46, 5, 37, 3, 62, 11, 17, 4, 47, 3, 63,
|
|
|
|
5, 38, 5, 243, 3, 64, 7, 247, 9, 50, 5, 39, 4, 241, 33, 37, 6, 33, 13, 35,
|
|
|
|
4, 242, 5, 245, 6, 247, 7, 29, 4, 51, 5, 41, 5, 246, 7, 249, 3, 240, 11, 19,
|
|
|
|
5, 42, 3, 241, 4, 245, 25, 29, 3, 242, 5, 43, 4, 246, 3, 243, 17, 58, 17, 43,
|
|
|
|
3, 244, 5, 249, 6, 37, 3, 245, 2, 240, 5, 45, 2, 241, 21, 23, 2, 242, 3, 247,
|
|
|
|
2, 243, 5, 251, 2, 244, 29, 61, 2, 245, 3, 249, 2, 246, 17, 29, 2, 247, 9, 55,
|
|
|
|
1, 240, 1, 241, 1, 242, 1, 243, 1, 244, 1, 245, 1, 246, 1, 247, 1, 248, 1, 249,
|
|
|
|
1, 250, 1, 251, 1, 252, 1, 253, 1, 254, 1, 255};
|
|
|
|
|
|
|
|
aco_opcode
|
|
|
|
get_reduce_opcode(chip_class chip, ReduceOp op)
|
|
|
|
{
|
2020-05-18 16:38:17 +01:00
|
|
|
/* Because some 16-bit instructions are already VOP3 on GFX10, we use the
|
|
|
|
* 32-bit opcodes (VOP2) which allows to remove the tempory VGPR and to use
|
|
|
|
* DPP with the arithmetic instructions. This requires to sign-extend.
|
|
|
|
*/
|
2019-11-11 17:37:43 +00:00
|
|
|
switch (op) {
|
2020-04-08 07:39:28 +01:00
|
|
|
case iadd8:
|
2020-05-25 18:59:57 +01:00
|
|
|
case iadd16:
|
|
|
|
if (chip >= GFX10) {
|
|
|
|
return aco_opcode::v_add_u32;
|
|
|
|
} else if (chip >= GFX8) {
|
|
|
|
return aco_opcode::v_add_u16;
|
|
|
|
} else {
|
|
|
|
return aco_opcode::v_add_co_u32;
|
|
|
|
}
|
|
|
|
break;
|
2020-04-08 07:39:28 +01:00
|
|
|
case imul8:
|
2020-05-25 18:59:57 +01:00
|
|
|
case imul16:
|
|
|
|
if (chip >= GFX10) {
|
|
|
|
return aco_opcode::v_mul_lo_u16_e64;
|
|
|
|
} else if (chip >= GFX8) {
|
|
|
|
return aco_opcode::v_mul_lo_u16;
|
|
|
|
} else {
|
|
|
|
return aco_opcode::v_mul_u32_u24;
|
|
|
|
}
|
|
|
|
break;
|
2020-04-08 07:39:28 +01:00
|
|
|
case fadd16: return aco_opcode::v_add_f16;
|
|
|
|
case fmul16: return aco_opcode::v_mul_f16;
|
|
|
|
case imax8:
|
2020-05-25 18:59:57 +01:00
|
|
|
case imax16:
|
|
|
|
if (chip >= GFX10) {
|
|
|
|
return aco_opcode::v_max_i32;
|
|
|
|
} else if (chip >= GFX8) {
|
|
|
|
return aco_opcode::v_max_i16;
|
|
|
|
} else {
|
|
|
|
return aco_opcode::v_max_i32;
|
|
|
|
}
|
|
|
|
break;
|
2020-04-08 07:39:28 +01:00
|
|
|
case imin8:
|
2020-05-25 18:59:57 +01:00
|
|
|
case imin16:
|
|
|
|
if (chip >= GFX10) {
|
|
|
|
return aco_opcode::v_min_i32;
|
|
|
|
} else if (chip >= GFX8) {
|
|
|
|
return aco_opcode::v_min_i16;
|
|
|
|
} else {
|
|
|
|
return aco_opcode::v_min_i32;
|
|
|
|
}
|
|
|
|
break;
|
2020-04-08 07:39:28 +01:00
|
|
|
case umin8:
|
2020-05-25 18:59:57 +01:00
|
|
|
case umin16:
|
|
|
|
if (chip >= GFX10) {
|
|
|
|
return aco_opcode::v_min_u32;
|
|
|
|
} else if (chip >= GFX8) {
|
|
|
|
return aco_opcode::v_min_u16;
|
|
|
|
} else {
|
|
|
|
return aco_opcode::v_min_u32;
|
|
|
|
}
|
|
|
|
break;
|
2020-04-08 07:39:28 +01:00
|
|
|
case umax8:
|
2020-05-25 18:59:57 +01:00
|
|
|
case umax16:
|
|
|
|
if (chip >= GFX10) {
|
|
|
|
return aco_opcode::v_max_u32;
|
|
|
|
} else if (chip >= GFX8) {
|
|
|
|
return aco_opcode::v_max_u16;
|
|
|
|
} else {
|
|
|
|
return aco_opcode::v_max_u32;
|
|
|
|
}
|
|
|
|
break;
|
2020-04-08 07:39:28 +01:00
|
|
|
case fmin16: return aco_opcode::v_min_f16;
|
|
|
|
case fmax16: return aco_opcode::v_max_f16;
|
2019-11-11 17:37:43 +00:00
|
|
|
case iadd32: return chip >= GFX9 ? aco_opcode::v_add_u32 : aco_opcode::v_add_co_u32;
|
|
|
|
case imul32: return aco_opcode::v_mul_lo_u32;
|
|
|
|
case fadd32: return aco_opcode::v_add_f32;
|
|
|
|
case fmul32: return aco_opcode::v_mul_f32;
|
|
|
|
case imax32: return aco_opcode::v_max_i32;
|
|
|
|
case imin32: return aco_opcode::v_min_i32;
|
|
|
|
case umin32: return aco_opcode::v_min_u32;
|
|
|
|
case umax32: return aco_opcode::v_max_u32;
|
|
|
|
case fmin32: return aco_opcode::v_min_f32;
|
|
|
|
case fmax32: return aco_opcode::v_max_f32;
|
2020-04-08 07:39:28 +01:00
|
|
|
case iand8:
|
|
|
|
case iand16:
|
2019-11-11 17:37:43 +00:00
|
|
|
case iand32: return aco_opcode::v_and_b32;
|
2020-04-08 07:39:28 +01:00
|
|
|
case ixor8:
|
|
|
|
case ixor16:
|
2019-11-11 17:37:43 +00:00
|
|
|
case ixor32: return aco_opcode::v_xor_b32;
|
2020-04-08 07:39:28 +01:00
|
|
|
case ior8:
|
|
|
|
case ior16:
|
2019-11-11 17:37:43 +00:00
|
|
|
case ior32: return aco_opcode::v_or_b32;
|
|
|
|
case iadd64: return aco_opcode::num_opcodes;
|
|
|
|
case imul64: return aco_opcode::num_opcodes;
|
|
|
|
case fadd64: return aco_opcode::v_add_f64;
|
|
|
|
case fmul64: return aco_opcode::v_mul_f64;
|
|
|
|
case imin64: return aco_opcode::num_opcodes;
|
|
|
|
case imax64: return aco_opcode::num_opcodes;
|
|
|
|
case umin64: return aco_opcode::num_opcodes;
|
|
|
|
case umax64: return aco_opcode::num_opcodes;
|
|
|
|
case fmin64: return aco_opcode::v_min_f64;
|
|
|
|
case fmax64: return aco_opcode::v_max_f64;
|
|
|
|
case iand64: return aco_opcode::num_opcodes;
|
|
|
|
case ior64: return aco_opcode::num_opcodes;
|
|
|
|
case ixor64: return aco_opcode::num_opcodes;
|
|
|
|
default: return aco_opcode::num_opcodes;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
bool
|
|
|
|
is_vop3_reduce_opcode(aco_opcode opcode)
|
2020-05-18 16:38:17 +01:00
|
|
|
{
|
|
|
|
/* 64-bit reductions are VOP3. */
|
|
|
|
if (opcode == aco_opcode::num_opcodes)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return instr_info.format[(int)opcode] == Format::VOP3;
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_vadd32(Builder& bld, Definition def, Operand src0, Operand src1)
|
2019-11-11 19:48:54 +00:00
|
|
|
{
|
2021-06-09 09:14:54 +01:00
|
|
|
Instruction* instr = bld.vadd32(def, src0, src1, false, Operand(s2), true);
|
2019-11-27 10:04:47 +00:00
|
|
|
if (instr->definitions.size() >= 2) {
|
|
|
|
assert(instr->definitions[1].regClass() == bld.lm);
|
2019-11-11 19:48:54 +00:00
|
|
|
instr->definitions[1].setFixed(vcc);
|
2019-11-27 10:04:47 +00:00
|
|
|
}
|
2019-11-11 19:48:54 +00:00
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_int64_dpp_op(lower_context* ctx, PhysReg dst_reg, PhysReg src0_reg, PhysReg src1_reg,
|
|
|
|
PhysReg vtmp_reg, ReduceOp op, unsigned dpp_ctrl, unsigned row_mask,
|
|
|
|
unsigned bank_mask, bool bound_ctrl, Operand* identity = NULL)
|
2019-11-11 19:48:54 +00:00
|
|
|
{
|
|
|
|
Builder bld(ctx->program, &ctx->instructions);
|
2021-06-09 09:14:54 +01:00
|
|
|
Definition dst[] = {Definition(dst_reg, v1), Definition(PhysReg{dst_reg + 1}, v1)};
|
|
|
|
Definition vtmp_def[] = {Definition(vtmp_reg, v1), Definition(PhysReg{vtmp_reg + 1}, v1)};
|
|
|
|
Operand src0[] = {Operand(src0_reg, v1), Operand(PhysReg{src0_reg + 1}, v1)};
|
|
|
|
Operand src1[] = {Operand(src1_reg, v1), Operand(PhysReg{src1_reg + 1}, v1)};
|
2019-11-11 19:48:54 +00:00
|
|
|
Operand src1_64 = Operand(src1_reg, v2);
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand vtmp_op[] = {Operand(vtmp_reg, v1), Operand(PhysReg{vtmp_reg + 1}, v1)};
|
2019-11-11 19:48:54 +00:00
|
|
|
Operand vtmp_op64 = Operand(vtmp_reg, v2);
|
|
|
|
if (op == iadd64) {
|
|
|
|
if (ctx->program->chip_class >= GFX10) {
|
|
|
|
if (identity)
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, vtmp_def[0], identity[0]);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, vtmp_def[0], src0[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-27 10:04:47 +00:00
|
|
|
bld.vop3(aco_opcode::v_add_co_u32_e64, dst[0], bld.def(bld.lm, vcc), vtmp_op[0], src1[0]);
|
2019-11-11 19:48:54 +00:00
|
|
|
} else {
|
2019-11-27 10:04:47 +00:00
|
|
|
bld.vop2_dpp(aco_opcode::v_add_co_u32, dst[0], bld.def(bld.lm, vcc), src0[0], src1[0],
|
2019-11-11 19:48:54 +00:00
|
|
|
dpp_ctrl, row_mask, bank_mask, bound_ctrl);
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop2_dpp(aco_opcode::v_addc_co_u32, dst[1], bld.def(bld.lm, vcc), src0[1], src1[1],
|
|
|
|
Operand(vcc, bld.lm), dpp_ctrl, row_mask, bank_mask, bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
} else if (op == iand64) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop2_dpp(aco_opcode::v_and_b32, dst[0], src0[0], src1[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
|
|
|
bld.vop2_dpp(aco_opcode::v_and_b32, dst[1], src0[1], src1[1], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
} else if (op == ior64) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop2_dpp(aco_opcode::v_or_b32, dst[0], src0[0], src1[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
|
|
|
bld.vop2_dpp(aco_opcode::v_or_b32, dst[1], src0[1], src1[1], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
} else if (op == ixor64) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop2_dpp(aco_opcode::v_xor_b32, dst[0], src0[0], src1[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
|
|
|
bld.vop2_dpp(aco_opcode::v_xor_b32, dst[1], src0[1], src1[1], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
} else if (op == umin64 || op == umax64 || op == imin64 || op == imax64) {
|
|
|
|
aco_opcode cmp = aco_opcode::num_opcodes;
|
|
|
|
switch (op) {
|
2021-06-09 09:14:54 +01:00
|
|
|
case umin64: cmp = aco_opcode::v_cmp_gt_u64; break;
|
|
|
|
case umax64: cmp = aco_opcode::v_cmp_lt_u64; break;
|
|
|
|
case imin64: cmp = aco_opcode::v_cmp_gt_i64; break;
|
|
|
|
case imax64: cmp = aco_opcode::v_cmp_lt_i64; break;
|
|
|
|
default: break;
|
2019-11-11 19:48:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (identity) {
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, vtmp_def[0], identity[0]);
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, vtmp_def[1], identity[1]);
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, vtmp_def[0], src0[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, vtmp_def[1], src0[1], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
|
2019-11-27 10:04:47 +00:00
|
|
|
bld.vopc(cmp, bld.def(bld.lm, vcc), vtmp_op64, src1_64);
|
|
|
|
bld.vop2(aco_opcode::v_cndmask_b32, dst[0], vtmp_op[0], src1[0], Operand(vcc, bld.lm));
|
|
|
|
bld.vop2(aco_opcode::v_cndmask_b32, dst[1], vtmp_op[1], src1[1], Operand(vcc, bld.lm));
|
2019-11-11 19:48:54 +00:00
|
|
|
} else if (op == imul64) {
|
|
|
|
/* t4 = dpp(x_hi)
|
|
|
|
* t1 = umul_lo(t4, y_lo)
|
|
|
|
* t3 = dpp(x_lo)
|
|
|
|
* t0 = umul_lo(t3, y_hi)
|
|
|
|
* t2 = iadd(t0, t1)
|
|
|
|
* t5 = umul_hi(t3, y_lo)
|
|
|
|
* res_hi = iadd(t2, t5)
|
|
|
|
* res_lo = umul_lo(t3, y_lo)
|
|
|
|
* Requires that res_hi != src0[0] and res_hi != src1[0]
|
|
|
|
* and that vtmp[0] != res_hi.
|
|
|
|
*/
|
|
|
|
if (identity)
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, vtmp_def[0], identity[1]);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, vtmp_def[0], src0[1], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
bld.vop3(aco_opcode::v_mul_lo_u32, vtmp_def[1], vtmp_op[0], src1[0]);
|
|
|
|
if (identity)
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, vtmp_def[0], identity[0]);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, vtmp_def[0], src0[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
bld.vop3(aco_opcode::v_mul_lo_u32, vtmp_def[0], vtmp_op[0], src1[1]);
|
|
|
|
emit_vadd32(bld, vtmp_def[1], vtmp_op[0], vtmp_op[1]);
|
|
|
|
if (identity)
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, vtmp_def[0], identity[0]);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, vtmp_def[0], src0[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
bld.vop3(aco_opcode::v_mul_hi_u32, vtmp_def[0], vtmp_op[0], src1[0]);
|
|
|
|
emit_vadd32(bld, dst[1], vtmp_op[1], vtmp_op[0]);
|
|
|
|
if (identity)
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, vtmp_def[0], identity[0]);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, vtmp_def[0], src0[0], dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-11-11 19:48:54 +00:00
|
|
|
bld.vop3(aco_opcode::v_mul_lo_u32, dst[0], vtmp_op[0], src1[0]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_int64_op(lower_context* ctx, PhysReg dst_reg, PhysReg src0_reg, PhysReg src1_reg, PhysReg vtmp,
|
|
|
|
ReduceOp op)
|
2019-11-11 19:48:54 +00:00
|
|
|
{
|
|
|
|
Builder bld(ctx->program, &ctx->instructions);
|
2021-06-09 09:14:54 +01:00
|
|
|
Definition dst[] = {Definition(dst_reg, v1), Definition(PhysReg{dst_reg + 1}, v1)};
|
2020-02-07 11:55:43 +00:00
|
|
|
RegClass src0_rc = src0_reg.reg() >= 256 ? v1 : s1;
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand src0[] = {Operand(src0_reg, src0_rc), Operand(PhysReg{src0_reg + 1}, src0_rc)};
|
|
|
|
Operand src1[] = {Operand(src1_reg, v1), Operand(PhysReg{src1_reg + 1}, v1)};
|
2020-02-07 11:55:43 +00:00
|
|
|
Operand src0_64 = Operand(src0_reg, src0_reg.reg() >= 256 ? v2 : s2);
|
2019-11-11 19:48:54 +00:00
|
|
|
Operand src1_64 = Operand(src1_reg, v2);
|
|
|
|
|
|
|
|
if (src0_rc == s1 &&
|
|
|
|
(op == imul64 || op == umin64 || op == umax64 || op == imin64 || op == imax64)) {
|
2020-02-07 11:55:43 +00:00
|
|
|
assert(vtmp.reg() != 0);
|
2019-11-11 19:48:54 +00:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(vtmp, v1), src0[0]);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{vtmp + 1}, v1), src0[1]);
|
2019-11-11 19:48:54 +00:00
|
|
|
src0_reg = vtmp;
|
|
|
|
src0[0] = Operand(vtmp, v1);
|
2021-06-09 09:14:54 +01:00
|
|
|
src0[1] = Operand(PhysReg{vtmp + 1}, v1);
|
2019-11-11 19:48:54 +00:00
|
|
|
src0_64 = Operand(vtmp, v2);
|
|
|
|
} else if (src0_rc == s1 && op == iadd64) {
|
2020-02-07 11:55:43 +00:00
|
|
|
assert(vtmp.reg() != 0);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{vtmp + 1}, v1), src0[1]);
|
|
|
|
src0[1] = Operand(PhysReg{vtmp + 1}, v1);
|
2019-11-11 19:48:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (op == iadd64) {
|
|
|
|
if (ctx->program->chip_class >= GFX10) {
|
2019-11-27 10:04:47 +00:00
|
|
|
bld.vop3(aco_opcode::v_add_co_u32_e64, dst[0], bld.def(bld.lm, vcc), src0[0], src1[0]);
|
2019-11-11 19:48:54 +00:00
|
|
|
} else {
|
2019-11-27 10:04:47 +00:00
|
|
|
bld.vop2(aco_opcode::v_add_co_u32, dst[0], bld.def(bld.lm, vcc), src0[0], src1[0]);
|
2019-11-11 19:48:54 +00:00
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop2(aco_opcode::v_addc_co_u32, dst[1], bld.def(bld.lm, vcc), src0[1], src1[1],
|
|
|
|
Operand(vcc, bld.lm));
|
2019-11-11 19:48:54 +00:00
|
|
|
} else if (op == iand64) {
|
|
|
|
bld.vop2(aco_opcode::v_and_b32, dst[0], src0[0], src1[0]);
|
|
|
|
bld.vop2(aco_opcode::v_and_b32, dst[1], src0[1], src1[1]);
|
|
|
|
} else if (op == ior64) {
|
|
|
|
bld.vop2(aco_opcode::v_or_b32, dst[0], src0[0], src1[0]);
|
|
|
|
bld.vop2(aco_opcode::v_or_b32, dst[1], src0[1], src1[1]);
|
|
|
|
} else if (op == ixor64) {
|
|
|
|
bld.vop2(aco_opcode::v_xor_b32, dst[0], src0[0], src1[0]);
|
|
|
|
bld.vop2(aco_opcode::v_xor_b32, dst[1], src0[1], src1[1]);
|
|
|
|
} else if (op == umin64 || op == umax64 || op == imin64 || op == imax64) {
|
|
|
|
aco_opcode cmp = aco_opcode::num_opcodes;
|
|
|
|
switch (op) {
|
2021-06-09 09:14:54 +01:00
|
|
|
case umin64: cmp = aco_opcode::v_cmp_gt_u64; break;
|
|
|
|
case umax64: cmp = aco_opcode::v_cmp_lt_u64; break;
|
|
|
|
case imin64: cmp = aco_opcode::v_cmp_gt_i64; break;
|
|
|
|
case imax64: cmp = aco_opcode::v_cmp_lt_i64; break;
|
|
|
|
default: break;
|
2019-11-11 19:48:54 +00:00
|
|
|
}
|
|
|
|
|
2019-11-27 10:04:47 +00:00
|
|
|
bld.vopc(cmp, bld.def(bld.lm, vcc), src0_64, src1_64);
|
|
|
|
bld.vop2(aco_opcode::v_cndmask_b32, dst[0], src0[0], src1[0], Operand(vcc, bld.lm));
|
|
|
|
bld.vop2(aco_opcode::v_cndmask_b32, dst[1], src0[1], src1[1], Operand(vcc, bld.lm));
|
2019-11-11 19:48:54 +00:00
|
|
|
} else if (op == imul64) {
|
|
|
|
if (src1_reg == dst_reg) {
|
|
|
|
/* it's fine if src0==dst but not if src1==dst */
|
|
|
|
std::swap(src0_reg, src1_reg);
|
|
|
|
std::swap(src0[0], src1[0]);
|
|
|
|
std::swap(src0[1], src1[1]);
|
|
|
|
std::swap(src0_64, src1_64);
|
|
|
|
}
|
|
|
|
assert(!(src0_reg == src1_reg));
|
|
|
|
/* t1 = umul_lo(x_hi, y_lo)
|
|
|
|
* t0 = umul_lo(x_lo, y_hi)
|
|
|
|
* t2 = iadd(t0, t1)
|
|
|
|
* t5 = umul_hi(x_lo, y_lo)
|
|
|
|
* res_hi = iadd(t2, t5)
|
|
|
|
* res_lo = umul_lo(x_lo, y_lo)
|
|
|
|
* assumes that it's ok to modify x_hi/y_hi, since we might not have vtmp
|
|
|
|
*/
|
2021-06-09 09:14:54 +01:00
|
|
|
Definition tmp0_def(PhysReg{src0_reg + 1}, v1);
|
|
|
|
Definition tmp1_def(PhysReg{src1_reg + 1}, v1);
|
2019-11-11 19:48:54 +00:00
|
|
|
Operand tmp0_op = src0[1];
|
|
|
|
Operand tmp1_op = src1[1];
|
|
|
|
bld.vop3(aco_opcode::v_mul_lo_u32, tmp0_def, src0[1], src1[0]);
|
|
|
|
bld.vop3(aco_opcode::v_mul_lo_u32, tmp1_def, src0[0], src1[1]);
|
|
|
|
emit_vadd32(bld, tmp0_def, tmp1_op, tmp0_op);
|
|
|
|
bld.vop3(aco_opcode::v_mul_hi_u32, tmp1_def, src0[0], src1[0]);
|
|
|
|
emit_vadd32(bld, dst[1], tmp0_op, tmp1_op);
|
|
|
|
bld.vop3(aco_opcode::v_mul_lo_u32, dst[0], src0[0], src1[0]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_dpp_op(lower_context* ctx, PhysReg dst_reg, PhysReg src0_reg, PhysReg src1_reg, PhysReg vtmp,
|
|
|
|
ReduceOp op, unsigned size, unsigned dpp_ctrl, unsigned row_mask, unsigned bank_mask,
|
|
|
|
bool bound_ctrl, Operand* identity = NULL) /* for VOP3 with sparse writes */
|
2019-09-17 12:22:17 +01:00
|
|
|
{
|
2019-11-11 17:37:43 +00:00
|
|
|
Builder bld(ctx->program, &ctx->instructions);
|
2019-09-17 12:22:17 +01:00
|
|
|
RegClass rc = RegClass(RegType::vgpr, size);
|
2019-11-11 17:37:43 +00:00
|
|
|
Definition dst(dst_reg, rc);
|
|
|
|
Operand src0(src0_reg, rc);
|
|
|
|
Operand src1(src1_reg, rc);
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2019-11-11 17:37:43 +00:00
|
|
|
aco_opcode opcode = get_reduce_opcode(ctx->program->chip_class, op);
|
2020-05-18 16:38:17 +01:00
|
|
|
bool vop3 = is_vop3_reduce_opcode(opcode);
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2019-11-11 17:37:43 +00:00
|
|
|
if (!vop3) {
|
|
|
|
if (opcode == aco_opcode::v_add_co_u32)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop2_dpp(opcode, dst, bld.def(bld.lm, vcc), src0, src1, dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl);
|
2019-09-17 12:22:17 +01:00
|
|
|
else
|
2019-11-11 17:37:43 +00:00
|
|
|
bld.vop2_dpp(opcode, dst, src0, src1, dpp_ctrl, row_mask, bank_mask, bound_ctrl);
|
|
|
|
return;
|
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2019-11-11 19:48:54 +00:00
|
|
|
if (opcode == aco_opcode::num_opcodes) {
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_int64_dpp_op(ctx, dst_reg, src0_reg, src1_reg, vtmp, op, dpp_ctrl, row_mask, bank_mask,
|
|
|
|
bound_ctrl, identity);
|
2019-11-11 19:48:54 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-11-11 17:37:43 +00:00
|
|
|
if (identity)
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(vtmp, v1), identity[0]);
|
|
|
|
if (identity && size >= 2)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{vtmp + 1}, v1), identity[1]);
|
2019-11-11 17:37:43 +00:00
|
|
|
|
|
|
|
for (unsigned i = 0; i < size; i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, Definition(PhysReg{vtmp + i}, v1),
|
|
|
|
Operand(PhysReg{src0_reg + i}, v1), dpp_ctrl, row_mask, bank_mask, bound_ctrl);
|
2019-11-11 17:37:43 +00:00
|
|
|
|
|
|
|
bld.vop3(opcode, dst, Operand(vtmp, rc), src1);
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_op(lower_context* ctx, PhysReg dst_reg, PhysReg src0_reg, PhysReg src1_reg, PhysReg vtmp,
|
|
|
|
ReduceOp op, unsigned size)
|
2019-11-11 17:37:43 +00:00
|
|
|
{
|
|
|
|
Builder bld(ctx->program, &ctx->instructions);
|
|
|
|
RegClass rc = RegClass(RegType::vgpr, size);
|
|
|
|
Definition dst(dst_reg, rc);
|
2020-02-07 11:55:43 +00:00
|
|
|
Operand src0(src0_reg, RegClass(src0_reg.reg() >= 256 ? RegType::vgpr : RegType::sgpr, size));
|
2019-11-11 17:37:43 +00:00
|
|
|
Operand src1(src1_reg, rc);
|
|
|
|
|
|
|
|
aco_opcode opcode = get_reduce_opcode(ctx->program->chip_class, op);
|
2020-05-18 16:38:17 +01:00
|
|
|
bool vop3 = is_vop3_reduce_opcode(opcode);
|
2019-11-11 17:37:43 +00:00
|
|
|
|
2019-11-11 19:48:54 +00:00
|
|
|
if (opcode == aco_opcode::num_opcodes) {
|
|
|
|
emit_int64_op(ctx, dst_reg, src0_reg, src1_reg, vtmp, op);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-11-11 17:37:43 +00:00
|
|
|
if (vop3) {
|
|
|
|
bld.vop3(opcode, dst, src0, src1);
|
|
|
|
} else if (opcode == aco_opcode::v_add_co_u32) {
|
2019-11-27 10:04:47 +00:00
|
|
|
bld.vop2(opcode, dst, bld.def(bld.lm, vcc), src0, src1);
|
2019-11-11 17:37:43 +00:00
|
|
|
} else {
|
|
|
|
bld.vop2(opcode, dst, src0, src1);
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_dpp_mov(lower_context* ctx, PhysReg dst, PhysReg src0, unsigned size, unsigned dpp_ctrl,
|
|
|
|
unsigned row_mask, unsigned bank_mask, bool bound_ctrl)
|
2019-09-12 19:28:52 +01:00
|
|
|
{
|
2019-11-11 17:37:43 +00:00
|
|
|
Builder bld(ctx->program, &ctx->instructions);
|
|
|
|
for (unsigned i = 0; i < size; i++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, Definition(PhysReg{dst + i}, v1),
|
|
|
|
Operand(PhysReg{src0 + i}, v1), dpp_ctrl, row_mask, bank_mask, bound_ctrl);
|
2019-11-11 17:37:43 +00:00
|
|
|
}
|
2019-09-12 19:28:52 +01:00
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_ds_swizzle(Builder bld, PhysReg dst, PhysReg src, unsigned size, unsigned ds_pattern)
|
2019-11-20 15:53:42 +00:00
|
|
|
{
|
|
|
|
for (unsigned i = 0; i < size; i++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.ds(aco_opcode::ds_swizzle_b32, Definition(PhysReg{dst + i}, v1),
|
|
|
|
Operand(PhysReg{src + i}, v1), ds_pattern);
|
2019-11-20 15:53:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_reduction(lower_context* ctx, aco_opcode op, ReduceOp reduce_op, unsigned cluster_size,
|
|
|
|
PhysReg tmp, PhysReg stmp, PhysReg vtmp, PhysReg sitmp, Operand src, Definition dst)
|
2019-09-17 12:22:17 +01:00
|
|
|
{
|
2019-11-27 15:59:11 +00:00
|
|
|
assert(cluster_size == ctx->program->wave_size || op == aco_opcode::p_reduce);
|
|
|
|
assert(cluster_size <= ctx->program->wave_size);
|
2019-09-17 12:22:17 +01:00
|
|
|
|
|
|
|
Builder bld(ctx->program, &ctx->instructions);
|
|
|
|
|
|
|
|
Operand identity[2];
|
2021-07-13 10:22:46 +01:00
|
|
|
identity[0] = Operand::c32(get_reduction_identity(reduce_op, 0));
|
|
|
|
identity[1] = Operand::c32(get_reduction_identity(reduce_op, 1));
|
2019-09-17 12:22:17 +01:00
|
|
|
Operand vcndmask_identity[2] = {identity[0], identity[1]};
|
|
|
|
|
|
|
|
/* First, copy the source to tmp and set inactive lanes to the identity */
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop1(Builder::s_or_saveexec, Definition(stmp, bld.lm), Definition(scc, s1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Definition(exec, bld.lm), Operand::c64(UINT64_MAX), Operand(exec, bld.lm));
|
2019-09-17 12:22:17 +01:00
|
|
|
|
|
|
|
for (unsigned i = 0; i < src.size(); i++) {
|
2019-09-12 19:28:52 +01:00
|
|
|
/* p_exclusive_scan needs it to be a sgpr or inline constant for the v_writelane_b32
|
|
|
|
* except on GFX10, where v_writelane_b32 can take a literal. */
|
2021-06-09 09:14:54 +01:00
|
|
|
if (identity[i].isLiteral() && op == aco_opcode::p_exclusive_scan &&
|
|
|
|
ctx->program->chip_class < GFX10) {
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(PhysReg{sitmp + i}, s1), identity[i]);
|
|
|
|
identity[i] = Operand(PhysReg{sitmp + i}, s1);
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{tmp + i}, v1), identity[i]);
|
|
|
|
vcndmask_identity[i] = Operand(PhysReg{tmp + i}, v1);
|
2019-09-17 12:22:17 +01:00
|
|
|
} else if (identity[i].isLiteral()) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{tmp + i}, v1), identity[i]);
|
|
|
|
vcndmask_identity[i] = Operand(PhysReg{tmp + i}, v1);
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < src.size(); i++) {
|
|
|
|
bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(PhysReg{tmp + i}, v1),
|
|
|
|
vcndmask_identity[i], Operand(PhysReg{src.physReg() + i}, v1),
|
2019-11-27 10:04:47 +00:00
|
|
|
Operand(stmp, bld.lm));
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
2020-05-20 10:32:50 +01:00
|
|
|
if (src.regClass() == v1b) {
|
2020-06-04 09:35:23 +01:00
|
|
|
if (ctx->program->chip_class >= GFX8) {
|
2021-06-09 09:14:54 +01:00
|
|
|
aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(
|
|
|
|
aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
|
2020-06-04 09:35:23 +01:00
|
|
|
sdwa->operands[0] = Operand(PhysReg{tmp}, v1);
|
|
|
|
sdwa->definitions[0] = Definition(PhysReg{tmp}, v1);
|
2021-08-30 16:58:36 +01:00
|
|
|
bool sext = reduce_op == imin8 || reduce_op == imax8;
|
|
|
|
sdwa->sel[0] = SubdwordSel(1, 0, sext);
|
|
|
|
sdwa->dst_sel = SubdwordSel::dword;
|
2020-06-04 09:35:23 +01:00
|
|
|
bld.insert(std::move(sdwa));
|
|
|
|
} else {
|
|
|
|
aco_opcode opcode;
|
|
|
|
|
|
|
|
if (reduce_op == imin8 || reduce_op == imax8)
|
|
|
|
opcode = aco_opcode::v_bfe_i32;
|
|
|
|
else
|
|
|
|
opcode = aco_opcode::v_bfe_u32;
|
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(opcode, Definition(PhysReg{tmp}, v1), Operand(PhysReg{tmp}, v1), Operand::zero(),
|
|
|
|
Operand::c32(8u));
|
2020-06-04 09:35:23 +01:00
|
|
|
}
|
2020-05-28 08:09:49 +01:00
|
|
|
} else if (src.regClass() == v2b) {
|
|
|
|
if (ctx->program->chip_class >= GFX10 &&
|
2021-06-09 09:14:54 +01:00
|
|
|
(reduce_op == iadd16 || reduce_op == imax16 || reduce_op == imin16 ||
|
|
|
|
reduce_op == umin16 || reduce_op == umax16)) {
|
|
|
|
aco_ptr<SDWA_instruction> sdwa{create_instruction<SDWA_instruction>(
|
|
|
|
aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)};
|
2020-05-28 08:09:49 +01:00
|
|
|
sdwa->operands[0] = Operand(PhysReg{tmp}, v1);
|
|
|
|
sdwa->definitions[0] = Definition(PhysReg{tmp}, v1);
|
2021-08-30 16:58:36 +01:00
|
|
|
bool sext = reduce_op == imin16 || reduce_op == imax16 || reduce_op == iadd16;
|
|
|
|
sdwa->sel[0] = SubdwordSel(2, 0, sext);
|
|
|
|
sdwa->dst_sel = SubdwordSel::dword;
|
2020-05-28 08:09:49 +01:00
|
|
|
bld.insert(std::move(sdwa));
|
2020-05-26 15:21:44 +01:00
|
|
|
} else if (ctx->program->chip_class == GFX6 || ctx->program->chip_class == GFX7) {
|
2020-06-05 07:54:52 +01:00
|
|
|
aco_opcode opcode;
|
|
|
|
|
|
|
|
if (reduce_op == imin16 || reduce_op == imax16 || reduce_op == iadd16)
|
|
|
|
opcode = aco_opcode::v_bfe_i32;
|
|
|
|
else
|
|
|
|
opcode = aco_opcode::v_bfe_u32;
|
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(opcode, Definition(PhysReg{tmp}, v1), Operand(PhysReg{tmp}, v1), Operand::zero(),
|
|
|
|
Operand::c32(16u));
|
2020-05-28 08:09:49 +01:00
|
|
|
}
|
2020-05-20 10:32:50 +01:00
|
|
|
}
|
|
|
|
|
2019-11-20 15:53:42 +00:00
|
|
|
bool reduction_needs_last_op = false;
|
2019-09-17 12:22:17 +01:00
|
|
|
switch (op) {
|
|
|
|
case aco_opcode::p_reduce:
|
2021-06-09 09:14:54 +01:00
|
|
|
if (cluster_size == 1)
|
|
|
|
break;
|
2019-11-20 15:53:42 +00:00
|
|
|
|
|
|
|
if (ctx->program->chip_class <= GFX7) {
|
|
|
|
reduction_needs_last_op = true;
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), (1 << 15) | dpp_quad_perm(1, 0, 3, 2));
|
2021-06-09 09:14:54 +01:00
|
|
|
if (cluster_size == 2)
|
|
|
|
break;
|
2019-11-20 15:53:42 +00:00
|
|
|
emit_op(ctx, tmp, vtmp, tmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), (1 << 15) | dpp_quad_perm(2, 3, 0, 1));
|
2021-06-09 09:14:54 +01:00
|
|
|
if (cluster_size == 4)
|
|
|
|
break;
|
2019-11-20 15:53:42 +00:00
|
|
|
emit_op(ctx, tmp, vtmp, tmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x1f, 0, 0x04));
|
2021-06-09 09:14:54 +01:00
|
|
|
if (cluster_size == 8)
|
|
|
|
break;
|
2019-11-20 15:53:42 +00:00
|
|
|
emit_op(ctx, tmp, vtmp, tmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x1f, 0, 0x08));
|
2021-06-09 09:14:54 +01:00
|
|
|
if (cluster_size == 16)
|
|
|
|
break;
|
2019-11-20 15:53:42 +00:00
|
|
|
emit_op(ctx, tmp, vtmp, tmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x1f, 0, 0x10));
|
2021-06-09 09:14:54 +01:00
|
|
|
if (cluster_size == 32)
|
|
|
|
break;
|
2019-11-20 15:53:42 +00:00
|
|
|
emit_op(ctx, tmp, vtmp, tmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.readlane(Definition(PhysReg{dst.physReg() + i}, s1), Operand(PhysReg{tmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::zero());
|
2019-11-20 15:53:42 +00:00
|
|
|
// TODO: it would be more effective to do the last reduction step on SALU
|
|
|
|
emit_op(ctx, tmp, dst.physReg(), tmp, vtmp, reduce_op, src.size());
|
|
|
|
reduction_needs_last_op = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_quad_perm(1, 0, 3, 2), 0xf,
|
|
|
|
0xf, false);
|
|
|
|
if (cluster_size == 2)
|
|
|
|
break;
|
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_quad_perm(2, 3, 0, 1), 0xf,
|
|
|
|
0xf, false);
|
|
|
|
if (cluster_size == 4)
|
|
|
|
break;
|
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_half_mirror, 0xf, 0xf,
|
|
|
|
false);
|
|
|
|
if (cluster_size == 8)
|
|
|
|
break;
|
2019-11-20 15:53:42 +00:00
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_mirror, 0xf, 0xf, false);
|
2021-06-09 09:14:54 +01:00
|
|
|
if (cluster_size == 16)
|
|
|
|
break;
|
2019-11-27 15:59:11 +00:00
|
|
|
|
|
|
|
if (ctx->program->chip_class >= GFX10) {
|
|
|
|
/* GFX10+ doesn't support row_bcast15 and row_bcast31 */
|
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop3(aco_opcode::v_permlanex16_b32, Definition(PhysReg{vtmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand(PhysReg{tmp + i}, v1), Operand::zero(), Operand::zero());
|
2019-11-27 15:59:11 +00:00
|
|
|
|
2019-11-20 15:53:42 +00:00
|
|
|
if (cluster_size == 32) {
|
|
|
|
reduction_needs_last_op = true;
|
|
|
|
break;
|
2019-11-27 15:59:11 +00:00
|
|
|
}
|
|
|
|
|
2019-11-20 15:53:42 +00:00
|
|
|
emit_op(ctx, tmp, tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
2019-09-17 12:22:17 +01:00
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.readlane(Definition(PhysReg{dst.physReg() + i}, s1), Operand(PhysReg{tmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::zero());
|
2019-11-20 15:53:42 +00:00
|
|
|
// TODO: it would be more effective to do the last reduction step on SALU
|
|
|
|
emit_op(ctx, tmp, dst.physReg(), tmp, vtmp, reduce_op, src.size());
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (cluster_size == 32) {
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x1f, 0, 0x10));
|
|
|
|
reduction_needs_last_op = true;
|
|
|
|
break;
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
2019-11-20 15:53:42 +00:00
|
|
|
assert(cluster_size == 64);
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_bcast15, 0xa, 0xf,
|
|
|
|
false);
|
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_bcast31, 0xc, 0xf,
|
|
|
|
false);
|
2019-09-17 12:22:17 +01:00
|
|
|
break;
|
|
|
|
case aco_opcode::p_exclusive_scan:
|
2019-09-12 19:28:52 +01:00
|
|
|
if (ctx->program->chip_class >= GFX10) { /* gfx10 doesn't support wf_sr1, so emulate it */
|
|
|
|
/* shift rows right */
|
2019-11-11 17:37:43 +00:00
|
|
|
emit_dpp_mov(ctx, vtmp, tmp, src.size(), dpp_row_sr(1), 0xf, 0xf, true);
|
2019-09-12 19:28:52 +01:00
|
|
|
|
|
|
|
/* fill in the gaps in rows 1 and 3 */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_lo, s1), Operand::c32(0x10000u));
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_hi, s1), Operand::c32(0x10000u));
|
2019-09-12 19:28:52 +01:00
|
|
|
for (unsigned i = 0; i < src.size(); i++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
Instruction* perm =
|
|
|
|
bld.vop3(aco_opcode::v_permlanex16_b32, Definition(PhysReg{vtmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand(PhysReg{tmp + i}, v1), Operand::c32(0xffffffffu),
|
|
|
|
Operand::c32(0xffffffffu))
|
2021-06-09 09:14:54 +01:00
|
|
|
.instr;
|
2021-01-21 16:13:34 +00:00
|
|
|
perm->vop3().opsel = 1; /* FI (Fetch Inactive) */
|
2019-09-12 19:28:52 +01:00
|
|
|
}
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(Builder::s_mov, Definition(exec, bld.lm), Operand::c64(UINT64_MAX));
|
2019-09-12 19:28:52 +01:00
|
|
|
|
2019-11-27 15:59:11 +00:00
|
|
|
if (ctx->program->wave_size == 64) {
|
|
|
|
/* fill in the gap in row 2 */
|
|
|
|
for (unsigned i = 0; i < src.size(); i++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.readlane(Definition(PhysReg{sitmp + i}, s1), Operand(PhysReg{tmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(31u));
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.writelane(Definition(PhysReg{vtmp + i}, v1), Operand(PhysReg{sitmp + i}, s1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(32u), Operand(PhysReg{vtmp + i}, v1));
|
2019-11-27 15:59:11 +00:00
|
|
|
}
|
2019-09-12 19:28:52 +01:00
|
|
|
}
|
|
|
|
std::swap(tmp, vtmp);
|
2019-11-21 09:23:13 +00:00
|
|
|
} else if (ctx->program->chip_class >= GFX8) {
|
2019-11-11 17:37:43 +00:00
|
|
|
emit_dpp_mov(ctx, tmp, tmp, src.size(), dpp_wf_sr1, 0xf, 0xf, true);
|
2019-11-21 09:23:13 +00:00
|
|
|
} else {
|
|
|
|
// TODO: use LDS on CS with a single write and shifted read
|
|
|
|
/* wavefront shift_right by 1 on SI/CI */
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), (1 << 15) | dpp_quad_perm(0, 0, 1, 2));
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_ds_swizzle(bld, tmp, tmp, src.size(),
|
|
|
|
ds_pattern_bitmode(0x1F, 0x00, 0x07)); /* mirror(8) */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_lo, s1), Operand::c32(0x10101010u));
|
2019-11-21 09:23:13 +00:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_hi, s1), Operand(exec_lo, s1));
|
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{vtmp + i}, v1),
|
|
|
|
Operand(PhysReg{tmp + i}, v1));
|
2019-11-21 09:23:13 +00:00
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand::c64(UINT64_MAX));
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_ds_swizzle(bld, tmp, tmp, src.size(),
|
|
|
|
ds_pattern_bitmode(0x1F, 0x00, 0x08)); /* swap(8) */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_lo, s1), Operand::c32(0x01000100u));
|
2019-11-21 09:23:13 +00:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_hi, s1), Operand(exec_lo, s1));
|
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{vtmp + i}, v1),
|
|
|
|
Operand(PhysReg{tmp + i}, v1));
|
2019-11-21 09:23:13 +00:00
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand::c64(UINT64_MAX));
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_ds_swizzle(bld, tmp, tmp, src.size(),
|
|
|
|
ds_pattern_bitmode(0x1F, 0x00, 0x10)); /* swap(16) */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfm_b32, Definition(exec_lo, s1), Operand::c32(1u),
|
|
|
|
Operand::c32(16u));
|
|
|
|
bld.sop2(aco_opcode::s_bfm_b32, Definition(exec_hi, s1), Operand::c32(1u),
|
|
|
|
Operand::c32(16u));
|
2019-11-21 09:23:13 +00:00
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{vtmp + i}, v1),
|
|
|
|
Operand(PhysReg{tmp + i}, v1));
|
2019-11-21 09:23:13 +00:00
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand::c64(UINT64_MAX));
|
2019-11-21 09:23:13 +00:00
|
|
|
for (unsigned i = 0; i < src.size(); i++) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.writelane(Definition(PhysReg{vtmp + i}, v1), identity[i], Operand::zero(),
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand(PhysReg{vtmp + i}, v1));
|
|
|
|
bld.readlane(Definition(PhysReg{sitmp + i}, s1), Operand(PhysReg{tmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::zero());
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.writelane(Definition(PhysReg{vtmp + i}, v1), Operand(PhysReg{sitmp + i}, s1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(32u), Operand(PhysReg{vtmp + i}, v1));
|
|
|
|
identity[i] = Operand::zero(); /* prevent further uses of identity */
|
2019-11-21 09:23:13 +00:00
|
|
|
}
|
|
|
|
std::swap(tmp, vtmp);
|
2019-09-12 19:28:52 +01:00
|
|
|
}
|
2019-11-21 09:23:13 +00:00
|
|
|
|
2019-09-17 12:22:17 +01:00
|
|
|
for (unsigned i = 0; i < src.size(); i++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
if (!identity[i].isConstant() ||
|
|
|
|
identity[i].constantValue()) { /* bound_ctrl should take care of this overwise */
|
2019-09-12 19:28:52 +01:00
|
|
|
if (ctx->program->chip_class < GFX10)
|
2021-06-09 09:14:54 +01:00
|
|
|
assert((identity[i].isConstant() && !identity[i].isLiteral()) ||
|
|
|
|
identity[i].physReg() == PhysReg{sitmp + i});
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.writelane(Definition(PhysReg{tmp + i}, v1), identity[i], Operand::zero(),
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand(PhysReg{tmp + i}, v1));
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
}
|
2021-04-10 16:11:58 +01:00
|
|
|
FALLTHROUGH;
|
2019-09-17 12:22:17 +01:00
|
|
|
case aco_opcode::p_inclusive_scan:
|
2019-11-27 15:59:11 +00:00
|
|
|
assert(cluster_size == ctx->program->wave_size);
|
2019-11-20 17:51:39 +00:00
|
|
|
if (ctx->program->chip_class <= GFX7) {
|
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x1e, 0x00, 0x00));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_lo, s1), Operand::c32(0xAAAAAAAAu));
|
2019-11-20 17:51:39 +00:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_hi, s1), Operand(exec_lo, s1));
|
|
|
|
emit_op(ctx, tmp, tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand::c64(UINT64_MAX));
|
2019-11-20 17:51:39 +00:00
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x1c, 0x01, 0x00));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_lo, s1), Operand::c32(0xCCCCCCCCu));
|
2019-11-20 17:51:39 +00:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_hi, s1), Operand(exec_lo, s1));
|
|
|
|
emit_op(ctx, tmp, tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand::c64(UINT64_MAX));
|
2019-11-20 17:51:39 +00:00
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x18, 0x03, 0x00));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_lo, s1), Operand::c32(0xF0F0F0F0u));
|
2019-11-20 17:51:39 +00:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_hi, s1), Operand(exec_lo, s1));
|
|
|
|
emit_op(ctx, tmp, tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand::c64(UINT64_MAX));
|
2019-11-20 17:51:39 +00:00
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x10, 0x07, 0x00));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_lo, s1), Operand::c32(0xFF00FF00u));
|
2019-11-20 17:51:39 +00:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(exec_hi, s1), Operand(exec_lo, s1));
|
|
|
|
emit_op(ctx, tmp, tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand::c64(UINT64_MAX));
|
2019-11-20 17:51:39 +00:00
|
|
|
emit_ds_swizzle(bld, vtmp, tmp, src.size(), ds_pattern_bitmode(0x00, 0x0f, 0x00));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfm_b32, Definition(exec_lo, s1), Operand::c32(16u),
|
|
|
|
Operand::c32(16u));
|
|
|
|
bld.sop2(aco_opcode::s_bfm_b32, Definition(exec_hi, s1), Operand::c32(16u),
|
|
|
|
Operand::c32(16u));
|
2019-11-20 17:51:39 +00:00
|
|
|
emit_op(ctx, tmp, tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.readlane(Definition(PhysReg{sitmp + i}, s1), Operand(PhysReg{tmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(31u));
|
|
|
|
bld.sop2(aco_opcode::s_bfm_b64, Definition(exec, s2), Operand::c32(32u),
|
|
|
|
Operand::c32(32u));
|
2019-11-20 17:51:39 +00:00
|
|
|
emit_op(ctx, tmp, sitmp, tmp, vtmp, reduce_op, src.size());
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_sr(1), 0xf, 0xf, false,
|
|
|
|
identity);
|
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_sr(2), 0xf, 0xf, false,
|
|
|
|
identity);
|
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_sr(4), 0xf, 0xf, false,
|
|
|
|
identity);
|
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_sr(8), 0xf, 0xf, false,
|
|
|
|
identity);
|
2019-09-12 19:28:52 +01:00
|
|
|
if (ctx->program->chip_class >= GFX10) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfm_b32, Definition(exec_lo, s1), Operand::c32(16u),
|
|
|
|
Operand::c32(16u));
|
|
|
|
bld.sop2(aco_opcode::s_bfm_b32, Definition(exec_hi, s1), Operand::c32(16u),
|
|
|
|
Operand::c32(16u));
|
2019-09-12 19:28:52 +01:00
|
|
|
for (unsigned i = 0; i < src.size(); i++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
Instruction* perm =
|
|
|
|
bld.vop3(aco_opcode::v_permlanex16_b32, Definition(PhysReg{vtmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand(PhysReg{tmp + i}, v1), Operand::c32(0xffffffffu),
|
|
|
|
Operand::c32(0xffffffffu))
|
2021-06-09 09:14:54 +01:00
|
|
|
.instr;
|
2021-01-21 16:13:34 +00:00
|
|
|
perm->vop3().opsel = 1; /* FI (Fetch Inactive) */
|
2019-09-12 19:28:52 +01:00
|
|
|
}
|
2019-11-11 19:48:54 +00:00
|
|
|
emit_op(ctx, tmp, tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
2019-09-12 19:28:52 +01:00
|
|
|
|
2019-11-27 15:59:11 +00:00
|
|
|
if (ctx->program->wave_size == 64) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfm_b64, Definition(exec, s2), Operand::c32(32u),
|
|
|
|
Operand::c32(32u));
|
2019-11-27 15:59:11 +00:00
|
|
|
for (unsigned i = 0; i < src.size(); i++)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.readlane(Definition(PhysReg{sitmp + i}, s1), Operand(PhysReg{tmp + i}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(31u));
|
2019-11-27 15:59:11 +00:00
|
|
|
emit_op(ctx, tmp, sitmp, tmp, vtmp, reduce_op, src.size());
|
|
|
|
}
|
2019-09-12 19:28:52 +01:00
|
|
|
} else {
|
2021-06-09 09:14:54 +01:00
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_bcast15, 0xa, 0xf,
|
|
|
|
false, identity);
|
|
|
|
emit_dpp_op(ctx, tmp, tmp, tmp, vtmp, reduce_op, src.size(), dpp_row_bcast31, 0xc, 0xf,
|
|
|
|
false, identity);
|
2019-09-12 19:28:52 +01:00
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
break;
|
2021-06-09 09:14:54 +01:00
|
|
|
default: unreachable("Invalid reduction mode");
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
2019-11-20 15:53:42 +00:00
|
|
|
if (op == aco_opcode::p_reduce) {
|
|
|
|
if (reduction_needs_last_op && dst.regClass().type() == RegType::vgpr) {
|
|
|
|
bld.sop1(Builder::s_mov, Definition(exec, bld.lm), Operand(stmp, bld.lm));
|
|
|
|
emit_op(ctx, dst.physReg(), tmp, vtmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (reduction_needs_last_op)
|
|
|
|
emit_op(ctx, tmp, vtmp, tmp, PhysReg{0}, reduce_op, src.size());
|
|
|
|
}
|
|
|
|
|
|
|
|
/* restore exec */
|
|
|
|
bld.sop1(Builder::s_mov, Definition(exec, bld.lm), Operand(stmp, bld.lm));
|
|
|
|
|
|
|
|
if (dst.regClass().type() == RegType::sgpr) {
|
2019-09-17 12:22:17 +01:00
|
|
|
for (unsigned k = 0; k < src.size(); k++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.readlane(Definition(PhysReg{dst.physReg() + k}, s1), Operand(PhysReg{tmp + k}, v1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(ctx->program->wave_size - 1));
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
2019-11-20 15:53:42 +00:00
|
|
|
} else if (dst.physReg() != tmp) {
|
2019-09-17 12:22:17 +01:00
|
|
|
for (unsigned k = 0; k < src.size(); k++) {
|
2019-11-20 17:51:39 +00:00
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(PhysReg{dst.physReg() + k}, v1),
|
2019-09-17 12:22:17 +01:00
|
|
|
Operand(PhysReg{tmp + k}, v1));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_gfx10_wave64_bpermute(Program* program, aco_ptr<Instruction>& instr, Builder& bld)
|
2020-05-27 00:22:28 +01:00
|
|
|
{
|
|
|
|
/* Emulates proper bpermute on GFX10 in wave64 mode.
|
|
|
|
*
|
|
|
|
* This is necessary because on GFX10 the bpermute instruction only works
|
|
|
|
* on half waves (you can think of it as having a cluster size of 32), so we
|
|
|
|
* manually swap the data between the two halves using two shared VGPRs.
|
|
|
|
*/
|
|
|
|
|
|
|
|
assert(program->chip_class >= GFX10);
|
2020-10-08 12:54:18 +01:00
|
|
|
assert(program->wave_size == 64);
|
2020-05-27 00:22:28 +01:00
|
|
|
|
|
|
|
unsigned shared_vgpr_reg_0 = align(program->config->num_vgprs, 4) + 256;
|
|
|
|
Definition dst = instr->definitions[0];
|
|
|
|
Definition tmp_exec = instr->definitions[1];
|
|
|
|
Definition clobber_scc = instr->definitions[2];
|
|
|
|
Operand index_x4 = instr->operands[0];
|
|
|
|
Operand input_data = instr->operands[1];
|
|
|
|
Operand same_half = instr->operands[2];
|
|
|
|
|
|
|
|
assert(dst.regClass() == v1);
|
|
|
|
assert(tmp_exec.regClass() == bld.lm);
|
|
|
|
assert(clobber_scc.isFixed() && clobber_scc.physReg() == scc);
|
|
|
|
assert(same_half.regClass() == bld.lm);
|
|
|
|
assert(index_x4.regClass() == v1);
|
|
|
|
assert(input_data.regClass().type() == RegType::vgpr);
|
|
|
|
assert(input_data.bytes() <= 4);
|
|
|
|
assert(dst.physReg() != index_x4.physReg());
|
|
|
|
assert(dst.physReg() != input_data.physReg());
|
|
|
|
assert(tmp_exec.physReg() != same_half.physReg());
|
|
|
|
|
|
|
|
PhysReg shared_vgpr_lo(shared_vgpr_reg_0);
|
|
|
|
PhysReg shared_vgpr_hi(shared_vgpr_reg_0 + 1);
|
|
|
|
|
|
|
|
/* Permute the input within the same half-wave */
|
|
|
|
bld.ds(aco_opcode::ds_bpermute_b32, dst, index_x4, input_data);
|
|
|
|
|
|
|
|
/* HI: Copy data from high lanes 32-63 to shared vgpr */
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, Definition(shared_vgpr_hi, v1), input_data,
|
|
|
|
dpp_quad_perm(0, 1, 2, 3), 0xc, 0xf, false);
|
2020-05-27 00:22:28 +01:00
|
|
|
/* Save EXEC */
|
|
|
|
bld.sop1(aco_opcode::s_mov_b64, tmp_exec, Operand(exec, s2));
|
|
|
|
/* Set EXEC to enable LO lanes only */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfm_b64, Definition(exec, s2), Operand::c32(32u), Operand::zero());
|
2020-05-27 00:22:28 +01:00
|
|
|
/* LO: Copy data from low lanes 0-31 to shared vgpr */
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(shared_vgpr_lo, v1), input_data);
|
|
|
|
/* LO: bpermute shared vgpr (high lanes' data) */
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.ds(aco_opcode::ds_bpermute_b32, Definition(shared_vgpr_hi, v1), index_x4,
|
|
|
|
Operand(shared_vgpr_hi, v1));
|
2020-05-27 00:22:28 +01:00
|
|
|
/* Set EXEC to enable HI lanes only */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfm_b64, Definition(exec, s2), Operand::c32(32u), Operand::c32(32u));
|
2020-05-27 00:22:28 +01:00
|
|
|
/* HI: bpermute shared vgpr (low lanes' data) */
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.ds(aco_opcode::ds_bpermute_b32, Definition(shared_vgpr_lo, v1), index_x4,
|
|
|
|
Operand(shared_vgpr_lo, v1));
|
2020-05-27 00:22:28 +01:00
|
|
|
|
|
|
|
/* Only enable lanes which use the other half's data */
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop2(aco_opcode::s_andn2_b64, Definition(exec, s2), clobber_scc,
|
|
|
|
Operand(tmp_exec.physReg(), s2), same_half);
|
2020-05-27 00:22:28 +01:00
|
|
|
/* LO: Copy shared vgpr (high lanes' bpermuted data) to output vgpr */
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, dst, Operand(shared_vgpr_hi, v1), dpp_quad_perm(0, 1, 2, 3),
|
|
|
|
0x3, 0xf, false);
|
2020-05-27 00:22:28 +01:00
|
|
|
/* HI: Copy shared vgpr (low lanes' bpermuted data) to output vgpr */
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop1_dpp(aco_opcode::v_mov_b32, dst, Operand(shared_vgpr_lo, v1), dpp_quad_perm(0, 1, 2, 3),
|
|
|
|
0xc, 0xf, false);
|
2020-05-27 00:22:28 +01:00
|
|
|
|
|
|
|
/* Restore saved EXEC */
|
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand(tmp_exec.physReg(), s2));
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
/* RA assumes that the result is always in the low part of the register, so we have to shift, if
|
|
|
|
* it's not there already */
|
2020-05-27 00:22:28 +01:00
|
|
|
if (input_data.physReg().byte()) {
|
|
|
|
unsigned right_shift = input_data.physReg().byte() * 8;
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshrrev_b32, dst, Operand::c32(right_shift),
|
|
|
|
Operand(dst.physReg(), v1));
|
2020-05-27 00:22:28 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_gfx6_bpermute(Program* program, aco_ptr<Instruction>& instr, Builder& bld)
|
2020-05-27 00:28:03 +01:00
|
|
|
{
|
|
|
|
/* Emulates bpermute using readlane instructions */
|
|
|
|
|
|
|
|
Operand index = instr->operands[0];
|
|
|
|
Operand input = instr->operands[1];
|
|
|
|
Definition dst = instr->definitions[0];
|
|
|
|
Definition temp_exec = instr->definitions[1];
|
|
|
|
Definition clobber_vcc = instr->definitions[2];
|
|
|
|
|
|
|
|
assert(dst.regClass() == v1);
|
|
|
|
assert(temp_exec.regClass() == bld.lm);
|
|
|
|
assert(clobber_vcc.regClass() == bld.lm);
|
|
|
|
assert(clobber_vcc.physReg() == vcc);
|
|
|
|
assert(index.regClass() == v1);
|
|
|
|
assert(index.physReg() != dst.physReg());
|
|
|
|
assert(input.regClass().type() == RegType::vgpr);
|
|
|
|
assert(input.bytes() <= 4);
|
|
|
|
assert(input.physReg() != dst.physReg());
|
|
|
|
|
|
|
|
/* Save original EXEC */
|
|
|
|
bld.sop1(aco_opcode::s_mov_b64, temp_exec, Operand(exec, s2));
|
|
|
|
|
|
|
|
/* An "unrolled loop" that is executed per each lane.
|
|
|
|
* This takes only a few instructions per lane, as opposed to a "real" loop
|
|
|
|
* with branching, where the branch instruction alone would take 16+ cycles.
|
|
|
|
*/
|
|
|
|
for (unsigned n = 0; n < program->wave_size; ++n) {
|
|
|
|
/* Activate the lane which has N for its source index */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vopc(aco_opcode::v_cmpx_eq_u32, Definition(exec, bld.lm), clobber_vcc, Operand::c32(n),
|
|
|
|
index);
|
2020-05-27 00:28:03 +01:00
|
|
|
/* Read the data from lane N */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.readlane(Definition(vcc, s1), input, Operand::c32(n));
|
2020-05-27 00:28:03 +01:00
|
|
|
/* On the active lane, move the data we read from lane N to the destination VGPR */
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, dst, Operand(vcc, s1));
|
|
|
|
/* Restore original EXEC */
|
|
|
|
bld.sop1(aco_opcode::s_mov_b64, Definition(exec, s2), Operand(temp_exec.physReg(), s2));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-09-17 12:22:17 +01:00
|
|
|
struct copy_operation {
|
|
|
|
Operand op;
|
|
|
|
Definition def;
|
2020-02-20 10:34:40 +00:00
|
|
|
unsigned bytes;
|
2020-02-27 12:07:21 +00:00
|
|
|
union {
|
|
|
|
uint8_t uses[8];
|
|
|
|
uint64_t is_used = 0;
|
|
|
|
};
|
2019-09-17 12:22:17 +01:00
|
|
|
};
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
split_copy(lower_context* ctx, unsigned offset, Definition* def, Operand* op,
|
|
|
|
const copy_operation& src, bool ignore_uses, unsigned max_size)
|
2020-04-06 17:13:52 +01:00
|
|
|
{
|
|
|
|
PhysReg def_reg = src.def.physReg();
|
|
|
|
PhysReg op_reg = src.op.physReg();
|
|
|
|
def_reg.reg_b += offset;
|
|
|
|
op_reg.reg_b += offset;
|
|
|
|
|
2020-11-26 14:18:18 +00:00
|
|
|
/* 64-bit VGPR copies (implemented with v_lshrrev_b64) are slow before GFX10 */
|
2021-06-09 09:14:54 +01:00
|
|
|
if (ctx->program->chip_class < GFX10 && src.def.regClass().type() == RegType::vgpr)
|
2020-11-26 14:18:18 +00:00
|
|
|
max_size = MIN2(max_size, 4);
|
|
|
|
unsigned max_align = src.def.regClass().type() == RegType::vgpr ? 4 : 16;
|
2020-04-06 17:13:52 +01:00
|
|
|
|
|
|
|
/* make sure the size is a power of two and reg % bytes == 0 */
|
|
|
|
unsigned bytes = 1;
|
|
|
|
for (; bytes <= max_size; bytes *= 2) {
|
|
|
|
unsigned next = bytes * 2u;
|
2020-11-26 14:18:18 +00:00
|
|
|
bool can_increase = def_reg.reg_b % MIN2(next, max_align) == 0 &&
|
2020-04-06 17:13:52 +01:00
|
|
|
offset + next <= src.bytes && next <= max_size;
|
|
|
|
if (!src.op.isConstant() && can_increase)
|
2020-11-26 14:18:18 +00:00
|
|
|
can_increase = op_reg.reg_b % MIN2(next, max_align) == 0;
|
2020-04-06 17:13:52 +01:00
|
|
|
for (unsigned i = 0; !ignore_uses && can_increase && (i < bytes); i++)
|
|
|
|
can_increase = (src.uses[offset + bytes + i] == 0) == (src.uses[offset] == 0);
|
|
|
|
if (!can_increase)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2021-06-11 13:05:18 +01:00
|
|
|
*def = Definition(src.def.tempId(), def_reg, src.def.regClass().resize(bytes));
|
2020-04-06 17:13:52 +01:00
|
|
|
if (src.op.isConstant()) {
|
2020-05-15 16:28:03 +01:00
|
|
|
assert(bytes >= 1 && bytes <= 8);
|
2020-12-03 15:18:30 +00:00
|
|
|
uint64_t val = src.op.constantValue64() >> (offset * 8u);
|
|
|
|
*op = Operand::get_const(ctx->program->chip_class, val, bytes);
|
2020-04-06 17:13:52 +01:00
|
|
|
} else {
|
2021-06-11 13:05:18 +01:00
|
|
|
RegClass op_cls = src.op.regClass().resize(bytes);
|
2020-04-06 17:13:52 +01:00
|
|
|
*op = Operand(op_reg, op_cls);
|
|
|
|
op->setTemp(Temp(src.op.tempId(), op_cls));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
uint32_t
|
|
|
|
get_intersection_mask(int a_start, int a_size, int b_start, int b_size)
|
2020-04-06 17:13:52 +01:00
|
|
|
{
|
|
|
|
int intersection_start = MAX2(b_start - a_start, 0);
|
|
|
|
int intersection_end = MAX2(b_start + b_size - a_start, 0);
|
|
|
|
if (intersection_start >= a_size || intersection_end == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
uint32_t mask = u_bit_consecutive(0, a_size);
|
|
|
|
return u_bit_consecutive(intersection_start, intersection_end - intersection_start) & mask;
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
copy_constant(lower_context* ctx, Builder& bld, Definition dst, Operand op)
|
2020-10-13 13:32:38 +01:00
|
|
|
{
|
2020-10-15 19:39:37 +01:00
|
|
|
assert(op.bytes() == dst.bytes());
|
|
|
|
|
2020-06-05 19:18:32 +01:00
|
|
|
if (dst.bytes() == 4 && op.isLiteral()) {
|
2020-10-15 19:39:37 +01:00
|
|
|
uint32_t imm = op.constantValue();
|
2020-06-05 19:18:32 +01:00
|
|
|
if (dst.regClass() == s1 && (imm >= 0xffff8000 || imm <= 0x7fff)) {
|
2020-10-15 19:39:37 +01:00
|
|
|
bld.sopk(aco_opcode::s_movk_i32, dst, imm & 0xFFFFu);
|
|
|
|
return;
|
|
|
|
} else if (util_bitreverse(imm) <= 64 || util_bitreverse(imm) >= 0xFFFFFFF0) {
|
|
|
|
uint32_t rev = util_bitreverse(imm);
|
2020-06-05 19:18:32 +01:00
|
|
|
if (dst.regClass() == s1)
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::s_brev_b32, dst, Operand::c32(rev));
|
2020-06-05 19:18:32 +01:00
|
|
|
else
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop1(aco_opcode::v_bfrev_b32, dst, Operand::c32(rev));
|
2020-10-15 19:39:37 +01:00
|
|
|
return;
|
2020-06-05 19:18:32 +01:00
|
|
|
} else if (dst.regClass() == s1 && imm != 0) {
|
2020-10-15 19:39:37 +01:00
|
|
|
unsigned start = (ffs(imm) - 1) & 0x1f;
|
|
|
|
unsigned size = util_bitcount(imm) & 0x1f;
|
|
|
|
if ((((1u << size) - 1u) << start) == imm) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfm_b32, dst, Operand::c32(size), Operand::c32(start));
|
2020-10-15 19:39:37 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-10-15 15:09:20 +01:00
|
|
|
if (op.bytes() == 4 && op.constantEquals(0x3e22f983) && ctx->program->chip_class >= GFX8)
|
|
|
|
op.setFixed(PhysReg{248}); /* it can be an inline constant on GFX8+ */
|
|
|
|
|
2020-10-15 19:39:37 +01:00
|
|
|
if (dst.regClass() == s1) {
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, dst, op);
|
|
|
|
} else if (dst.regClass() == s2) {
|
2020-11-26 14:18:18 +00:00
|
|
|
/* s_ashr_i64 writes SCC, so we can't use it */
|
|
|
|
assert(Operand::is_constant_representable(op.constantValue64(), 8, true, false));
|
2020-10-15 19:39:37 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b64, dst, op);
|
2020-11-26 14:18:18 +00:00
|
|
|
} else if (dst.regClass() == v2) {
|
|
|
|
if (Operand::is_constant_representable(op.constantValue64(), 8, true, false)) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_lshrrev_b64, dst, Operand::zero(), op);
|
2020-11-26 14:18:18 +00:00
|
|
|
} else {
|
|
|
|
assert(Operand::is_constant_representable(op.constantValue64(), 8, false, true));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_ashrrev_i64, dst, Operand::zero(), op);
|
2020-11-26 14:18:18 +00:00
|
|
|
}
|
2020-10-15 19:39:37 +01:00
|
|
|
} else if (dst.regClass() == v1) {
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, dst, op);
|
2020-12-30 15:06:04 +00:00
|
|
|
} else {
|
|
|
|
assert(dst.regClass() == v1b || dst.regClass() == v2b);
|
|
|
|
|
|
|
|
if (dst.regClass() == v1b && ctx->program->chip_class >= GFX9) {
|
|
|
|
uint8_t val = op.constantValue();
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand op32 = Operand::c32((uint32_t)val | (val & 0x80u ? 0xffffff00u : 0u));
|
2020-12-30 15:06:04 +00:00
|
|
|
if (op32.isLiteral()) {
|
|
|
|
uint32_t a = (uint32_t)int8_mul_table[val * 2];
|
|
|
|
uint32_t b = (uint32_t)int8_mul_table[val * 2 + 1];
|
|
|
|
bld.vop2_sdwa(aco_opcode::v_mul_u32_u24, dst,
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(a | (a & 0x80u ? 0xffffff00u : 0x0u)),
|
|
|
|
Operand::c32(b | (b & 0x80u ? 0xffffff00u : 0x0u)));
|
2020-12-30 15:06:04 +00:00
|
|
|
} else {
|
|
|
|
bld.vop1_sdwa(aco_opcode::v_mov_b32, dst, op32);
|
|
|
|
}
|
|
|
|
} else if (dst.regClass() == v2b && ctx->program->chip_class >= GFX9 && !op.isLiteral()) {
|
|
|
|
if (op.constantValue() >= 0xfff0 || op.constantValue() <= 64) {
|
|
|
|
/* use v_mov_b32 to avoid possible issues with denormal flushing or
|
|
|
|
* NaN. v_add_f16 is still needed for float constants. */
|
|
|
|
uint32_t val32 = (int32_t)(int16_t)op.constantValue();
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop1_sdwa(aco_opcode::v_mov_b32, dst, Operand::c32(val32));
|
2020-12-30 15:06:04 +00:00
|
|
|
} else {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2_sdwa(aco_opcode::v_add_f16, dst, op, Operand::zero());
|
2020-12-30 15:06:04 +00:00
|
|
|
}
|
|
|
|
} else if (dst.regClass() == v2b && ctx->program->chip_class >= GFX10 &&
|
|
|
|
(ctx->block->fp_mode.denorm16_64 & fp_denorm_keep_in)) {
|
|
|
|
if (dst.physReg().byte() == 2) {
|
|
|
|
Operand def_lo(dst.physReg().advance(-2), v2b);
|
|
|
|
Instruction* instr = bld.vop3(aco_opcode::v_pack_b32_f16, dst, def_lo, op);
|
2021-01-21 16:13:34 +00:00
|
|
|
instr->vop3().opsel = 0;
|
2020-12-30 15:06:04 +00:00
|
|
|
} else {
|
|
|
|
assert(dst.physReg().byte() == 0);
|
|
|
|
Operand def_hi(dst.physReg().advance(2), v2b);
|
|
|
|
Instruction* instr = bld.vop3(aco_opcode::v_pack_b32_f16, dst, op, def_hi);
|
2021-01-21 16:13:34 +00:00
|
|
|
instr->vop3().opsel = 2;
|
2020-12-30 15:06:04 +00:00
|
|
|
}
|
2020-10-16 13:18:08 +01:00
|
|
|
} else {
|
2020-12-30 15:06:04 +00:00
|
|
|
uint32_t offset = dst.physReg().byte() * 8u;
|
|
|
|
uint32_t mask = ((1u << (dst.bytes() * 8)) - 1) << offset;
|
|
|
|
uint32_t val = (op.constantValue() << offset) & mask;
|
2020-10-15 19:39:37 +01:00
|
|
|
dst = Definition(PhysReg(dst.physReg().reg()), v1);
|
|
|
|
Operand def_op(dst.physReg(), v1);
|
2020-12-30 15:06:04 +00:00
|
|
|
if (val != mask)
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_and_b32, dst, Operand::c32(~mask), def_op);
|
2020-12-30 15:06:04 +00:00
|
|
|
if (val != 0)
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_or_b32, dst, Operand::c32(val), def_op);
|
2020-10-15 19:39:37 +01:00
|
|
|
}
|
2020-10-13 13:32:38 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-16 17:19:36 +01:00
|
|
|
void
|
|
|
|
copy_linear_vgpr(Builder& bld, Definition def, Operand op, bool preserve_scc, PhysReg scratch_sgpr)
|
|
|
|
{
|
|
|
|
if (preserve_scc)
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(scratch_sgpr, s1), Operand(scc, s1));
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < 2; i++) {
|
|
|
|
if (def.size() == 2)
|
|
|
|
bld.vop3(aco_opcode::v_lshrrev_b64, def, Operand::zero(), op);
|
|
|
|
else
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, def, op);
|
|
|
|
|
|
|
|
bld.sop1(Builder::s_not, Definition(exec, bld.lm), Definition(scc, s1),
|
|
|
|
Operand(exec, bld.lm));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (preserve_scc)
|
|
|
|
bld.sopc(aco_opcode::s_cmp_lg_i32, Definition(scc, s1), Operand(scratch_sgpr, s1),
|
|
|
|
Operand::zero());
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
swap_linear_vgpr(Builder& bld, Definition def, Operand op, bool preserve_scc, PhysReg scratch_sgpr)
|
|
|
|
{
|
|
|
|
if (preserve_scc)
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(scratch_sgpr, s1), Operand(scc, s1));
|
|
|
|
|
|
|
|
Operand def_as_op = Operand(def.physReg(), def.regClass());
|
|
|
|
Definition op_as_def = Definition(op.physReg(), op.regClass());
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < 2; i++) {
|
|
|
|
if (bld.program->chip_class >= GFX9) {
|
|
|
|
bld.vop1(aco_opcode::v_swap_b32, def, op_as_def, op, def_as_op);
|
|
|
|
} else {
|
|
|
|
bld.vop2(aco_opcode::v_xor_b32, op_as_def, op, def_as_op);
|
|
|
|
bld.vop2(aco_opcode::v_xor_b32, def, op, def_as_op);
|
|
|
|
bld.vop2(aco_opcode::v_xor_b32, op_as_def, op, def_as_op);
|
|
|
|
}
|
|
|
|
|
|
|
|
bld.sop1(Builder::s_not, Definition(exec, bld.lm), Definition(scc, s1),
|
|
|
|
Operand(exec, bld.lm));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (preserve_scc)
|
|
|
|
bld.sopc(aco_opcode::s_cmp_lg_i32, Definition(scc, s1), Operand(scratch_sgpr, s1),
|
|
|
|
Operand::zero());
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
bool
|
|
|
|
do_copy(lower_context* ctx, Builder& bld, const copy_operation& copy, bool* preserve_scc,
|
|
|
|
PhysReg scratch_sgpr)
|
2020-04-06 17:13:52 +01:00
|
|
|
{
|
|
|
|
bool did_copy = false;
|
|
|
|
for (unsigned offset = 0; offset < copy.bytes;) {
|
|
|
|
if (copy.uses[offset]) {
|
|
|
|
offset++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
Definition def;
|
|
|
|
Operand op;
|
2020-12-03 15:18:30 +00:00
|
|
|
split_copy(ctx, offset, &def, &op, copy, false, 8);
|
2020-04-06 17:13:52 +01:00
|
|
|
|
|
|
|
if (def.physReg() == scc) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sopc(aco_opcode::s_cmp_lg_i32, def, op, Operand::zero());
|
2020-04-06 17:13:52 +01:00
|
|
|
*preserve_scc = true;
|
2020-10-15 15:09:20 +01:00
|
|
|
} else if (op.isConstant()) {
|
|
|
|
copy_constant(ctx, bld, def, op);
|
2021-06-16 17:19:36 +01:00
|
|
|
} else if (def.regClass().is_linear_vgpr()) {
|
|
|
|
copy_linear_vgpr(bld, def, op, *preserve_scc, scratch_sgpr);
|
2020-10-15 15:09:20 +01:00
|
|
|
} else if (def.regClass() == v1) {
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, def, op);
|
2020-11-26 14:18:18 +00:00
|
|
|
} else if (def.regClass() == v2) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_lshrrev_b64, def, Operand::zero(), op);
|
2020-10-15 15:09:20 +01:00
|
|
|
} else if (def.regClass() == s1) {
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, def, op);
|
|
|
|
} else if (def.regClass() == s2) {
|
|
|
|
bld.sop1(aco_opcode::s_mov_b64, def, op);
|
2020-05-06 11:58:02 +01:00
|
|
|
} else if (def.regClass().is_subdword() && ctx->program->chip_class < GFX8) {
|
|
|
|
if (op.physReg().byte()) {
|
|
|
|
assert(def.physReg().byte() == 0);
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshrrev_b32, def, Operand::c32(op.physReg().byte() * 8), op);
|
2020-05-06 11:58:02 +01:00
|
|
|
} else if (def.physReg().byte()) {
|
|
|
|
assert(op.physReg().byte() == 0);
|
2020-10-16 14:12:28 +01:00
|
|
|
/* preserve the target's lower half */
|
|
|
|
uint32_t bits = def.physReg().byte() * 8;
|
|
|
|
PhysReg lo_reg = PhysReg(def.physReg().reg());
|
2021-06-09 09:14:54 +01:00
|
|
|
Definition lo_half =
|
|
|
|
Definition(lo_reg, RegClass::get(RegType::vgpr, def.physReg().byte()));
|
|
|
|
Definition dst =
|
|
|
|
Definition(lo_reg, RegClass::get(RegType::vgpr, lo_half.bytes() + op.bytes()));
|
2020-10-16 14:12:28 +01:00
|
|
|
|
2020-05-27 18:31:33 +01:00
|
|
|
if (def.physReg().reg() == op.physReg().reg()) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_and_b32, lo_half, Operand::c32((1 << bits) - 1u),
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand(lo_reg, lo_half.regClass()));
|
2020-10-16 14:12:28 +01:00
|
|
|
if (def.physReg().byte() == 1) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_mul_u32_u24, dst, Operand::c32((1 << bits) + 1u), op);
|
2020-10-16 14:12:28 +01:00
|
|
|
} else if (def.physReg().byte() == 2) {
|
|
|
|
bld.vop2(aco_opcode::v_cvt_pk_u16_u32, dst, Operand(lo_reg, v2b), op);
|
|
|
|
} else if (def.physReg().byte() == 3) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(scratch_sgpr, s1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32((1 << bits) + 1u));
|
2020-10-16 14:12:28 +01:00
|
|
|
bld.vop3(aco_opcode::v_mul_lo_u32, dst, Operand(scratch_sgpr, s1), op);
|
2020-05-27 18:31:33 +01:00
|
|
|
}
|
|
|
|
} else {
|
2020-10-16 14:12:28 +01:00
|
|
|
lo_half.setFixed(lo_half.physReg().advance(4 - def.physReg().byte()));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshlrev_b32, lo_half, Operand::c32(32 - bits),
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand(lo_reg, lo_half.regClass()));
|
|
|
|
bld.vop3(aco_opcode::v_alignbyte_b32, dst, op,
|
|
|
|
Operand(lo_half.physReg(), lo_half.regClass()),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(4 - def.physReg().byte()));
|
2020-05-27 18:31:33 +01:00
|
|
|
}
|
2020-05-06 11:58:02 +01:00
|
|
|
} else {
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, def, op);
|
|
|
|
}
|
2020-10-15 19:39:37 +01:00
|
|
|
} else if (def.regClass().is_subdword()) {
|
|
|
|
bld.vop1_sdwa(aco_opcode::v_mov_b32, def, op);
|
2020-04-06 17:13:52 +01:00
|
|
|
} else {
|
2020-10-15 19:39:37 +01:00
|
|
|
unreachable("unsupported copy");
|
2020-04-06 17:13:52 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
did_copy = true;
|
|
|
|
offset += def.bytes();
|
|
|
|
}
|
|
|
|
return did_copy;
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
do_swap(lower_context* ctx, Builder& bld, const copy_operation& copy, bool preserve_scc,
|
|
|
|
Pseudo_instruction* pi)
|
2020-04-06 17:13:52 +01:00
|
|
|
{
|
|
|
|
unsigned offset = 0;
|
|
|
|
|
|
|
|
if (copy.bytes == 3 && (copy.def.physReg().reg_b % 4 <= 1) &&
|
|
|
|
(copy.def.physReg().reg_b % 4) == (copy.op.physReg().reg_b % 4)) {
|
2021-06-09 09:14:54 +01:00
|
|
|
/* instead of doing a 2-byte and 1-byte swap, do a 4-byte swap and then fixup with a 1-byte
|
|
|
|
* swap */
|
2020-04-06 17:13:52 +01:00
|
|
|
PhysReg op = copy.op.physReg();
|
|
|
|
PhysReg def = copy.def.physReg();
|
|
|
|
op.reg_b &= ~0x3;
|
|
|
|
def.reg_b &= ~0x3;
|
|
|
|
|
|
|
|
copy_operation tmp;
|
|
|
|
tmp.op = Operand(op, v1);
|
|
|
|
tmp.def = Definition(def, v1);
|
|
|
|
tmp.bytes = 4;
|
|
|
|
memset(tmp.uses, 1, 4);
|
|
|
|
do_swap(ctx, bld, tmp, preserve_scc, pi);
|
|
|
|
|
|
|
|
op.reg_b += copy.def.physReg().reg_b % 4 == 0 ? 3 : 0;
|
|
|
|
def.reg_b += copy.def.physReg().reg_b % 4 == 0 ? 3 : 0;
|
|
|
|
tmp.op = Operand(op, v1b);
|
|
|
|
tmp.def = Definition(def, v1b);
|
|
|
|
tmp.bytes = 1;
|
|
|
|
tmp.uses[0] = 1;
|
|
|
|
do_swap(ctx, bld, tmp, preserve_scc, pi);
|
|
|
|
|
|
|
|
offset = copy.bytes;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (; offset < copy.bytes;) {
|
|
|
|
Definition def;
|
|
|
|
Operand op;
|
2020-11-26 14:18:18 +00:00
|
|
|
unsigned max_size = copy.def.regClass().type() == RegType::vgpr ? 4 : 8;
|
|
|
|
split_copy(ctx, offset, &def, &op, copy, true, max_size);
|
2020-04-06 17:13:52 +01:00
|
|
|
|
|
|
|
assert(op.regClass() == def.regClass());
|
|
|
|
Operand def_as_op = Operand(def.physReg(), def.regClass());
|
|
|
|
Definition op_as_def = Definition(op.physReg(), op.regClass());
|
2021-06-16 17:19:36 +01:00
|
|
|
if (def.regClass().is_linear_vgpr()) {
|
|
|
|
swap_linear_vgpr(bld, def, op, preserve_scc, pi->scratch_sgpr);
|
|
|
|
} else if (ctx->program->chip_class >= GFX9 && def.regClass() == v1) {
|
2020-04-06 17:13:52 +01:00
|
|
|
bld.vop1(aco_opcode::v_swap_b32, def, op_as_def, op, def_as_op);
|
2020-06-23 11:55:34 +01:00
|
|
|
} else if (def.regClass() == v1) {
|
2020-05-06 11:58:02 +01:00
|
|
|
assert(def.physReg().byte() == 0 && op.physReg().byte() == 0);
|
2020-04-06 17:13:52 +01:00
|
|
|
bld.vop2(aco_opcode::v_xor_b32, op_as_def, op, def_as_op);
|
|
|
|
bld.vop2(aco_opcode::v_xor_b32, def, op, def_as_op);
|
|
|
|
bld.vop2(aco_opcode::v_xor_b32, op_as_def, op, def_as_op);
|
|
|
|
} else if (op.physReg() == scc || def.physReg() == scc) {
|
|
|
|
/* we need to swap scc and another sgpr */
|
|
|
|
assert(!preserve_scc);
|
|
|
|
|
|
|
|
PhysReg other = op.physReg() == scc ? def.physReg() : op.physReg();
|
|
|
|
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(pi->scratch_sgpr, s1), Operand(scc, s1));
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sopc(aco_opcode::s_cmp_lg_i32, Definition(scc, s1), Operand(other, s1),
|
|
|
|
Operand::zero());
|
2020-04-06 17:13:52 +01:00
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(other, s1), Operand(pi->scratch_sgpr, s1));
|
|
|
|
} else if (def.regClass() == s1) {
|
|
|
|
if (preserve_scc) {
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(pi->scratch_sgpr, s1), op);
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, op_as_def, def_as_op);
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, def, Operand(pi->scratch_sgpr, s1));
|
|
|
|
} else {
|
|
|
|
bld.sop2(aco_opcode::s_xor_b32, op_as_def, Definition(scc, s1), op, def_as_op);
|
|
|
|
bld.sop2(aco_opcode::s_xor_b32, def, Definition(scc, s1), op, def_as_op);
|
|
|
|
bld.sop2(aco_opcode::s_xor_b32, op_as_def, Definition(scc, s1), op, def_as_op);
|
|
|
|
}
|
2020-04-17 14:33:34 +01:00
|
|
|
} else if (def.regClass() == s2) {
|
|
|
|
if (preserve_scc)
|
|
|
|
bld.sop1(aco_opcode::s_mov_b32, Definition(pi->scratch_sgpr, s1), Operand(scc, s1));
|
|
|
|
bld.sop2(aco_opcode::s_xor_b64, op_as_def, Definition(scc, s1), op, def_as_op);
|
|
|
|
bld.sop2(aco_opcode::s_xor_b64, def, Definition(scc, s1), op, def_as_op);
|
|
|
|
bld.sop2(aco_opcode::s_xor_b64, op_as_def, Definition(scc, s1), op, def_as_op);
|
|
|
|
if (preserve_scc)
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sopc(aco_opcode::s_cmp_lg_i32, Definition(scc, s1), Operand(pi->scratch_sgpr, s1),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::zero());
|
2020-10-15 18:47:12 +01:00
|
|
|
} else if (def.bytes() == 2 && def.physReg().reg() == op.physReg().reg()) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop3(aco_opcode::v_alignbyte_b32, Definition(def.physReg(), v1), def_as_op, op,
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(2u));
|
2020-04-06 17:13:52 +01:00
|
|
|
} else {
|
|
|
|
assert(def.regClass().is_subdword());
|
|
|
|
bld.vop2_sdwa(aco_opcode::v_xor_b32, op_as_def, op, def_as_op);
|
|
|
|
bld.vop2_sdwa(aco_opcode::v_xor_b32, def, op, def_as_op);
|
|
|
|
bld.vop2_sdwa(aco_opcode::v_xor_b32, op_as_def, op, def_as_op);
|
|
|
|
}
|
|
|
|
|
|
|
|
offset += def.bytes();
|
|
|
|
}
|
|
|
|
|
2020-05-16 17:30:21 +01:00
|
|
|
if (ctx->program->chip_class <= GFX7)
|
|
|
|
return;
|
|
|
|
|
2020-04-06 17:13:52 +01:00
|
|
|
/* fixup in case we swapped bytes we shouldn't have */
|
|
|
|
copy_operation tmp_copy = copy;
|
|
|
|
tmp_copy.op.setFixed(copy.def.physReg());
|
|
|
|
tmp_copy.def.setFixed(copy.op.physReg());
|
2020-05-27 18:31:33 +01:00
|
|
|
do_copy(ctx, bld, tmp_copy, &preserve_scc, pi->scratch_sgpr);
|
2020-04-06 17:13:52 +01:00
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
do_pack_2x16(lower_context* ctx, Builder& bld, Definition def, Operand lo, Operand hi)
|
2020-05-11 16:42:37 +01:00
|
|
|
{
|
2020-10-13 13:32:38 +01:00
|
|
|
if (lo.isConstant() && hi.isConstant()) {
|
2021-07-13 10:22:46 +01:00
|
|
|
copy_constant(ctx, bld, def, Operand::c32(lo.constantValue() | (hi.constantValue() << 16)));
|
2020-10-13 13:32:38 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-10-15 18:47:12 +01:00
|
|
|
bool can_use_pack = (ctx->block->fp_mode.denorm16_64 & fp_denorm_keep_in) &&
|
|
|
|
(ctx->program->chip_class >= GFX10 ||
|
2021-06-09 09:14:54 +01:00
|
|
|
(ctx->program->chip_class >= GFX9 && !lo.isLiteral() && !hi.isLiteral()));
|
2020-10-15 18:47:12 +01:00
|
|
|
|
|
|
|
if (can_use_pack) {
|
2020-05-11 16:42:37 +01:00
|
|
|
Instruction* instr = bld.vop3(aco_opcode::v_pack_b32_f16, def, lo, hi);
|
|
|
|
/* opsel: 0 = select low half, 1 = select high half. [0] = src0, [1] = src1 */
|
2021-01-21 16:13:34 +00:00
|
|
|
instr->vop3().opsel = hi.physReg().byte() | (lo.physReg().byte() >> 1);
|
2020-10-15 18:47:12 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* a single alignbyte can be sufficient: hi can be a 32-bit integer constant */
|
|
|
|
if (lo.physReg().byte() == 2 && hi.physReg().byte() == 0 &&
|
2021-07-13 10:22:46 +01:00
|
|
|
(!hi.isConstant() || !Operand::c32(hi.constantValue()).isLiteral() ||
|
2020-10-15 18:47:12 +01:00
|
|
|
ctx->program->chip_class >= GFX10)) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_alignbyte_b32, def, hi, lo, Operand::c32(2u));
|
2020-10-15 18:47:12 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
Definition def_lo = Definition(def.physReg(), v2b);
|
|
|
|
Definition def_hi = Definition(def.physReg().advance(2), v2b);
|
|
|
|
|
|
|
|
if (lo.isConstant()) {
|
|
|
|
/* move hi and zero low bits */
|
|
|
|
if (hi.physReg().byte() == 0)
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshlrev_b32, def_hi, Operand::c32(16u), hi);
|
2020-10-15 18:47:12 +01:00
|
|
|
else
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_and_b32, def_hi, Operand::c32(~0xFFFFu), hi);
|
|
|
|
bld.vop2(aco_opcode::v_or_b32, def, Operand::c32(lo.constantValue()),
|
|
|
|
Operand(def.physReg(), v1));
|
2020-10-15 18:47:12 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (hi.isConstant()) {
|
|
|
|
/* move lo and zero high bits */
|
|
|
|
if (lo.physReg().byte() == 2)
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshrrev_b32, def_lo, Operand::c32(16u), lo);
|
2020-10-15 18:47:12 +01:00
|
|
|
else
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_and_b32, def_lo, Operand::c32(0xFFFFu), lo);
|
|
|
|
bld.vop2(aco_opcode::v_or_b32, def, Operand::c32(hi.constantValue() << 16u),
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand(def.physReg(), v1));
|
2020-10-15 18:47:12 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (lo.physReg().reg() == def.physReg().reg()) {
|
|
|
|
/* lo is in the high bits of def */
|
|
|
|
assert(lo.physReg().byte() == 2);
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshrrev_b32, def_lo, Operand::c32(16u), lo);
|
2020-10-15 18:47:12 +01:00
|
|
|
lo.setFixed(def.physReg());
|
|
|
|
} else if (hi.physReg() == def.physReg()) {
|
|
|
|
/* hi is in the low bits of def */
|
|
|
|
assert(hi.physReg().byte() == 0);
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshlrev_b32, def_hi, Operand::c32(16u), hi);
|
2020-10-15 18:47:12 +01:00
|
|
|
hi.setFixed(def.physReg().advance(2));
|
2020-05-11 16:42:37 +01:00
|
|
|
} else if (ctx->program->chip_class >= GFX8) {
|
2020-10-15 18:47:12 +01:00
|
|
|
/* either lo or hi can be placed with just a v_mov */
|
|
|
|
assert(lo.physReg().byte() == 0 || hi.physReg().byte() == 2);
|
|
|
|
Operand& op = lo.physReg().byte() == 0 ? lo : hi;
|
2020-10-27 10:35:27 +00:00
|
|
|
PhysReg reg = def.physReg().advance(op.physReg().byte());
|
|
|
|
bld.vop1(aco_opcode::v_mov_b32, Definition(reg, v2b), op);
|
|
|
|
op.setFixed(reg);
|
2020-10-15 18:47:12 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (ctx->program->chip_class >= GFX8) {
|
|
|
|
/* either hi or lo are already placed correctly */
|
|
|
|
if (lo.physReg().reg() == def.physReg().reg())
|
2020-10-15 19:39:37 +01:00
|
|
|
bld.vop1_sdwa(aco_opcode::v_mov_b32, def_hi, hi);
|
2020-10-15 18:47:12 +01:00
|
|
|
else
|
2020-10-15 19:39:37 +01:00
|
|
|
bld.vop1_sdwa(aco_opcode::v_mov_b32, def_lo, lo);
|
2020-10-15 18:47:12 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* alignbyte needs the operands in the following way:
|
|
|
|
* | xx hi | lo xx | >> 2 byte */
|
|
|
|
if (lo.physReg().byte() != hi.physReg().byte()) {
|
|
|
|
/* | xx lo | hi xx | => | lo hi | lo hi | */
|
|
|
|
assert(lo.physReg().byte() == 0 && hi.physReg().byte() == 2);
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_alignbyte_b32, def, lo, hi, Operand::c32(2u));
|
2020-10-15 18:47:12 +01:00
|
|
|
lo = Operand(def_hi.physReg(), v2b);
|
|
|
|
hi = Operand(def_lo.physReg(), v2b);
|
|
|
|
} else if (lo.physReg().byte() == 0) {
|
|
|
|
/* | xx hi | xx lo | => | xx hi | lo 00 | */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshlrev_b32, def_hi, Operand::c32(16u), lo);
|
2020-10-15 18:47:12 +01:00
|
|
|
lo = Operand(def_hi.physReg(), v2b);
|
2020-05-11 16:42:37 +01:00
|
|
|
} else {
|
2020-10-15 18:47:12 +01:00
|
|
|
/* | hi xx | lo xx | => | 00 hi | lo xx | */
|
|
|
|
assert(hi.physReg().byte() == 2);
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshrrev_b32, def_lo, Operand::c32(16u), hi);
|
2020-10-15 18:47:12 +01:00
|
|
|
hi = Operand(def_lo.physReg(), v2b);
|
2020-05-11 16:42:37 +01:00
|
|
|
}
|
2020-10-15 18:47:12 +01:00
|
|
|
/* perform the alignbyte */
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_alignbyte_b32, def, hi, lo, Operand::c32(2u));
|
2020-05-11 16:42:37 +01:00
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
try_coalesce_copies(lower_context* ctx, std::map<PhysReg, copy_operation>& copy_map,
|
|
|
|
copy_operation& copy)
|
2020-11-26 14:30:32 +00:00
|
|
|
{
|
|
|
|
// TODO try more relaxed alignment for subdword copies
|
|
|
|
unsigned next_def_align = util_next_power_of_two(copy.bytes + 1);
|
|
|
|
unsigned next_op_align = next_def_align;
|
|
|
|
if (copy.def.regClass().type() == RegType::vgpr)
|
|
|
|
next_def_align = MIN2(next_def_align, 4);
|
|
|
|
if (copy.op.regClass().type() == RegType::vgpr)
|
|
|
|
next_op_align = MIN2(next_op_align, 4);
|
|
|
|
|
|
|
|
if (copy.bytes >= 8 || copy.def.physReg().reg_b % next_def_align ||
|
|
|
|
(!copy.op.isConstant() && copy.op.physReg().reg_b % next_op_align))
|
|
|
|
return;
|
|
|
|
|
|
|
|
auto other = copy_map.find(copy.def.physReg().advance(copy.bytes));
|
|
|
|
if (other == copy_map.end() || copy.bytes + other->second.bytes > 8 ||
|
|
|
|
copy.op.isConstant() != other->second.op.isConstant())
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* don't create 64-bit copies before GFX10 */
|
|
|
|
if (copy.bytes >= 4 && copy.def.regClass().type() == RegType::vgpr &&
|
|
|
|
ctx->program->chip_class < GFX10)
|
|
|
|
return;
|
|
|
|
|
|
|
|
unsigned new_size = copy.bytes + other->second.bytes;
|
|
|
|
if (copy.op.isConstant()) {
|
2021-06-09 09:14:54 +01:00
|
|
|
uint64_t val =
|
|
|
|
copy.op.constantValue64() | (other->second.op.constantValue64() << (copy.bytes * 8u));
|
2021-09-02 17:04:29 +01:00
|
|
|
if (!util_is_power_of_two_or_zero(new_size))
|
|
|
|
return;
|
|
|
|
if (!Operand::is_constant_representable(val, new_size, true,
|
2020-11-26 14:30:32 +00:00
|
|
|
copy.def.regClass().type() == RegType::vgpr))
|
|
|
|
return;
|
|
|
|
copy.op = Operand::get_const(ctx->program->chip_class, val, new_size);
|
|
|
|
} else {
|
|
|
|
if (other->second.op.physReg() != copy.op.physReg().advance(copy.bytes))
|
|
|
|
return;
|
2021-06-11 13:05:18 +01:00
|
|
|
copy.op = Operand(copy.op.physReg(), copy.op.regClass().resize(new_size));
|
2020-11-26 14:30:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
copy.bytes = new_size;
|
2021-06-11 13:05:18 +01:00
|
|
|
copy.def = Definition(copy.def.physReg(), copy.def.regClass().resize(copy.bytes));
|
2020-11-26 14:30:32 +00:00
|
|
|
copy_map.erase(other);
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
handle_operands(std::map<PhysReg, copy_operation>& copy_map, lower_context* ctx,
|
|
|
|
chip_class chip_class, Pseudo_instruction* pi)
|
2019-09-17 12:22:17 +01:00
|
|
|
{
|
|
|
|
Builder bld(ctx->program, &ctx->instructions);
|
2020-06-05 21:05:31 +01:00
|
|
|
unsigned num_instructions_before = ctx->instructions.size();
|
2019-09-17 12:22:17 +01:00
|
|
|
aco_ptr<Instruction> mov;
|
|
|
|
bool writes_scc = false;
|
|
|
|
|
|
|
|
/* count the number of uses for each dst reg */
|
2020-11-03 13:40:05 +00:00
|
|
|
for (auto it = copy_map.begin(); it != copy_map.end();) {
|
2019-09-17 12:22:17 +01:00
|
|
|
|
|
|
|
if (it->second.def.physReg() == scc)
|
|
|
|
writes_scc = true;
|
|
|
|
|
|
|
|
assert(!pi->tmp_in_scc || !(it->second.def.physReg() == pi->scratch_sgpr));
|
|
|
|
|
|
|
|
/* if src and dst reg are the same, remove operation */
|
|
|
|
if (it->first == it->second.op.physReg()) {
|
|
|
|
it = copy_map.erase(it);
|
|
|
|
continue;
|
|
|
|
}
|
2020-02-27 12:07:21 +00:00
|
|
|
|
|
|
|
/* split large copies */
|
|
|
|
if (it->second.bytes > 8) {
|
|
|
|
assert(!it->second.op.isConstant());
|
|
|
|
assert(!it->second.def.regClass().is_subdword());
|
|
|
|
RegClass rc = RegClass(it->second.def.regClass().type(), it->second.def.size() - 2);
|
|
|
|
Definition hi_def = Definition(PhysReg{it->first + 2}, rc);
|
|
|
|
rc = RegClass(it->second.op.regClass().type(), it->second.op.size() - 2);
|
|
|
|
Operand hi_op = Operand(PhysReg{it->second.op.physReg() + 2}, rc);
|
|
|
|
copy_operation copy = {hi_op, hi_def, it->second.bytes - 8};
|
|
|
|
copy_map[hi_def.physReg()] = copy;
|
|
|
|
assert(it->second.op.physReg().byte() == 0 && it->second.def.physReg().byte() == 0);
|
2021-06-09 09:14:54 +01:00
|
|
|
it->second.op = Operand(it->second.op.physReg(),
|
|
|
|
it->second.op.regClass().type() == RegType::sgpr ? s2 : v2);
|
|
|
|
it->second.def = Definition(it->second.def.physReg(),
|
|
|
|
it->second.def.regClass().type() == RegType::sgpr ? s2 : v2);
|
2020-02-27 12:07:21 +00:00
|
|
|
it->second.bytes = 8;
|
|
|
|
}
|
2020-05-07 18:15:59 +01:00
|
|
|
|
2020-11-26 14:30:32 +00:00
|
|
|
try_coalesce_copies(ctx, copy_map, it->second);
|
2020-02-27 12:07:21 +00:00
|
|
|
|
|
|
|
/* check if the definition reg is used by another copy operation */
|
|
|
|
for (std::pair<const PhysReg, copy_operation>& copy : copy_map) {
|
|
|
|
if (copy.second.op.isConstant())
|
|
|
|
continue;
|
|
|
|
for (uint16_t i = 0; i < it->second.bytes; i++) {
|
|
|
|
/* distance might underflow */
|
|
|
|
unsigned distance = it->first.reg_b + i - copy.second.op.physReg().reg_b;
|
|
|
|
if (distance < copy.second.bytes)
|
|
|
|
it->second.uses[i] += 1;
|
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
++it;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* first, handle paths in the location transfer graph */
|
|
|
|
bool preserve_scc = pi->tmp_in_scc && !writes_scc;
|
2020-06-05 21:21:02 +01:00
|
|
|
bool skip_partial_copies = true;
|
2020-11-03 13:40:05 +00:00
|
|
|
for (auto it = copy_map.begin();;) {
|
2020-06-05 21:21:02 +01:00
|
|
|
if (copy_map.empty()) {
|
2021-06-09 09:14:54 +01:00
|
|
|
ctx->program->statistics[statistic_copies] +=
|
|
|
|
ctx->instructions.size() - num_instructions_before;
|
2020-06-05 21:21:02 +01:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (it == copy_map.end()) {
|
2020-06-26 12:13:20 +01:00
|
|
|
if (!skip_partial_copies)
|
2020-06-05 21:21:02 +01:00
|
|
|
break;
|
|
|
|
skip_partial_copies = false;
|
|
|
|
it = copy_map.begin();
|
2020-02-27 12:07:21 +00:00
|
|
|
}
|
|
|
|
|
2020-05-11 16:42:37 +01:00
|
|
|
/* check if we can pack one register at once */
|
|
|
|
if (it->first.byte() == 0 && it->second.bytes == 2) {
|
|
|
|
PhysReg reg_hi = it->first.advance(2);
|
|
|
|
std::map<PhysReg, copy_operation>::iterator other = copy_map.find(reg_hi);
|
|
|
|
if (other != copy_map.end() && other->second.bytes == 2) {
|
|
|
|
/* check if the target register is otherwise unused */
|
2021-06-09 09:14:54 +01:00
|
|
|
bool unused_lo = !it->second.is_used || (it->second.is_used == 0x0101 &&
|
|
|
|
other->second.op.physReg() == it->first);
|
2020-10-16 12:29:28 +01:00
|
|
|
bool unused_hi = !other->second.is_used ||
|
2021-06-09 09:14:54 +01:00
|
|
|
(other->second.is_used == 0x0101 && it->second.op.physReg() == reg_hi);
|
2020-05-11 16:42:37 +01:00
|
|
|
if (unused_lo && unused_hi) {
|
|
|
|
Operand lo = it->second.op;
|
|
|
|
Operand hi = other->second.op;
|
|
|
|
do_pack_2x16(ctx, bld, Definition(it->first, v1), lo, hi);
|
|
|
|
copy_map.erase(it);
|
|
|
|
copy_map.erase(other);
|
|
|
|
|
2020-11-03 13:40:05 +00:00
|
|
|
for (std::pair<const PhysReg, copy_operation>& other2 : copy_map) {
|
|
|
|
for (uint16_t i = 0; i < other2.second.bytes; i++) {
|
2020-05-11 16:42:37 +01:00
|
|
|
/* distance might underflow */
|
2020-11-03 13:40:05 +00:00
|
|
|
unsigned distance_lo = other2.first.reg_b + i - lo.physReg().reg_b;
|
|
|
|
unsigned distance_hi = other2.first.reg_b + i - hi.physReg().reg_b;
|
2020-05-11 16:42:37 +01:00
|
|
|
if (distance_lo < 2 || distance_hi < 2)
|
2020-11-03 13:40:05 +00:00
|
|
|
other2.second.uses[i] -= 1;
|
2020-05-11 16:42:37 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
it = copy_map.begin();
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-05-16 17:30:21 +01:00
|
|
|
/* on GFX6/7, we need some small workarounds as there is no
|
|
|
|
* SDWA instruction to do partial register writes */
|
2020-05-27 11:08:31 +01:00
|
|
|
if (ctx->program->chip_class < GFX8 && it->second.bytes < 4) {
|
2021-06-09 09:14:54 +01:00
|
|
|
if (it->first.byte() == 0 && it->second.op.physReg().byte() == 0 && !it->second.is_used &&
|
|
|
|
pi->opcode == aco_opcode::p_split_vector) {
|
2020-05-16 17:30:21 +01:00
|
|
|
/* Other operations might overwrite the high bits, so change all users
|
|
|
|
* of the high bits to the new target where they are still available.
|
|
|
|
* This mechanism depends on also emitting dead definitions. */
|
2020-05-27 11:08:31 +01:00
|
|
|
PhysReg reg_hi = it->second.op.physReg().advance(it->second.bytes);
|
|
|
|
while (reg_hi != PhysReg(it->second.op.physReg().reg() + 1)) {
|
|
|
|
std::map<PhysReg, copy_operation>::iterator other = copy_map.begin();
|
|
|
|
for (other = copy_map.begin(); other != copy_map.end(); other++) {
|
|
|
|
/* on GFX6/7, if the high bits are used as operand, they cannot be a target */
|
|
|
|
if (other->second.op.physReg() == reg_hi) {
|
|
|
|
other->second.op.setFixed(it->first.advance(reg_hi.byte()));
|
|
|
|
break; /* break because an operand can only be used once */
|
|
|
|
}
|
2020-05-16 17:30:21 +01:00
|
|
|
}
|
2020-05-27 11:08:31 +01:00
|
|
|
reg_hi = reg_hi.advance(it->second.bytes);
|
2020-05-16 17:30:21 +01:00
|
|
|
}
|
2020-05-27 11:08:31 +01:00
|
|
|
} else if (it->first.byte()) {
|
|
|
|
assert(pi->opcode == aco_opcode::p_create_vector);
|
2020-05-16 17:30:21 +01:00
|
|
|
/* on GFX6/7, if we target an upper half where the lower half hasn't yet been handled,
|
2021-06-09 09:14:54 +01:00
|
|
|
* move to the target operand's high bits. This is save to do as it cannot be an operand
|
|
|
|
*/
|
2020-05-16 17:30:21 +01:00
|
|
|
PhysReg lo = PhysReg(it->first.reg());
|
|
|
|
std::map<PhysReg, copy_operation>::iterator other = copy_map.find(lo);
|
|
|
|
if (other != copy_map.end()) {
|
2020-05-27 11:08:31 +01:00
|
|
|
assert(other->second.bytes == it->first.byte());
|
|
|
|
PhysReg new_reg_hi = other->second.op.physReg().advance(it->first.byte());
|
|
|
|
it->second.def = Definition(new_reg_hi, it->second.def.regClass());
|
2020-05-16 17:30:21 +01:00
|
|
|
it->second.is_used = 0;
|
2020-05-27 11:08:31 +01:00
|
|
|
other->second.bytes += it->second.bytes;
|
2021-06-09 09:14:54 +01:00
|
|
|
other->second.def.setTemp(Temp(other->second.def.tempId(),
|
|
|
|
RegClass::get(RegType::vgpr, other->second.bytes)));
|
|
|
|
other->second.op.setTemp(Temp(other->second.op.tempId(),
|
|
|
|
RegClass::get(RegType::vgpr, other->second.bytes)));
|
2020-05-16 17:30:21 +01:00
|
|
|
/* if the new target's high bits are also a target, change uses */
|
|
|
|
std::map<PhysReg, copy_operation>::iterator target = copy_map.find(new_reg_hi);
|
|
|
|
if (target != copy_map.end()) {
|
2020-05-27 11:08:31 +01:00
|
|
|
for (unsigned i = 0; i < it->second.bytes; i++)
|
|
|
|
target->second.uses[i]++;
|
2020-05-16 17:30:21 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-04-06 17:13:52 +01:00
|
|
|
/* find portions where the target reg is not used as operand for any other copy */
|
|
|
|
if (it->second.is_used) {
|
2020-06-05 21:21:02 +01:00
|
|
|
if (it->second.op.isConstant() || skip_partial_copies) {
|
|
|
|
/* we have to skip constants until is_used=0.
|
|
|
|
* we also skip partial copies at the beginning to help coalescing */
|
2020-04-06 17:13:52 +01:00
|
|
|
++it;
|
|
|
|
continue;
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
2020-04-06 17:13:52 +01:00
|
|
|
unsigned has_zero_use_bytes = 0;
|
|
|
|
for (unsigned i = 0; i < it->second.bytes; i++)
|
|
|
|
has_zero_use_bytes |= (it->second.uses[i] == 0) << i;
|
|
|
|
|
|
|
|
if (has_zero_use_bytes) {
|
|
|
|
/* Skipping partial copying and doing a v_swap_b32 and then fixup
|
|
|
|
* copies is usually beneficial for sub-dword copies, but if doing
|
|
|
|
* a partial copy allows further copies, it should be done instead. */
|
|
|
|
bool partial_copy = (has_zero_use_bytes == 0xf) || (has_zero_use_bytes == 0xf0);
|
2020-02-27 12:07:21 +00:00
|
|
|
for (std::pair<const PhysReg, copy_operation>& copy : copy_map) {
|
2020-06-26 12:13:20 +01:00
|
|
|
/* on GFX6/7, we can only do copies with full registers */
|
|
|
|
if (partial_copy || ctx->program->chip_class <= GFX7)
|
2020-04-06 17:13:52 +01:00
|
|
|
break;
|
2020-02-27 12:07:21 +00:00
|
|
|
for (uint16_t i = 0; i < copy.second.bytes; i++) {
|
|
|
|
/* distance might underflow */
|
|
|
|
unsigned distance = copy.first.reg_b + i - it->second.op.physReg().reg_b;
|
2020-04-06 17:13:52 +01:00
|
|
|
if (distance < it->second.bytes && copy.second.uses[i] == 1 &&
|
|
|
|
!it->second.uses[distance])
|
|
|
|
partial_copy = true;
|
2020-02-27 12:07:21 +00:00
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
2020-04-06 17:13:52 +01:00
|
|
|
|
|
|
|
if (!partial_copy) {
|
|
|
|
++it;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* full target reg is used: register swapping needed */
|
|
|
|
++it;
|
|
|
|
continue;
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
2020-04-06 17:13:52 +01:00
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2020-05-27 18:31:33 +01:00
|
|
|
bool did_copy = do_copy(ctx, bld, it->second, &preserve_scc, pi->scratch_sgpr);
|
2020-06-05 21:21:02 +01:00
|
|
|
skip_partial_copies = did_copy;
|
2020-04-22 21:56:02 +01:00
|
|
|
std::pair<PhysReg, copy_operation> copy = *it;
|
2020-04-06 17:13:52 +01:00
|
|
|
|
|
|
|
if (it->second.is_used == 0) {
|
|
|
|
/* the target reg is not used as operand for any other copy, so we
|
|
|
|
* copied to all of it */
|
2019-09-17 12:22:17 +01:00
|
|
|
copy_map.erase(it);
|
|
|
|
it = copy_map.begin();
|
|
|
|
} else {
|
2020-04-06 17:13:52 +01:00
|
|
|
/* we only performed some portions of this copy, so split it to only
|
|
|
|
* leave the portions that still need to be done */
|
|
|
|
copy_operation original = it->second; /* the map insertion below can overwrite this */
|
|
|
|
copy_map.erase(it);
|
|
|
|
for (unsigned offset = 0; offset < original.bytes;) {
|
|
|
|
if (original.uses[offset] == 0) {
|
|
|
|
offset++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
Definition def;
|
|
|
|
Operand op;
|
2020-12-03 15:18:30 +00:00
|
|
|
split_copy(ctx, offset, &def, &op, original, false, 8);
|
2020-04-06 17:13:52 +01:00
|
|
|
|
2020-11-03 13:40:05 +00:00
|
|
|
copy_operation new_copy = {op, def, def.bytes()};
|
|
|
|
for (unsigned i = 0; i < new_copy.bytes; i++)
|
|
|
|
new_copy.uses[i] = original.uses[i + offset];
|
|
|
|
copy_map[def.physReg()] = new_copy;
|
2020-04-06 17:13:52 +01:00
|
|
|
|
|
|
|
offset += def.bytes();
|
|
|
|
}
|
|
|
|
|
|
|
|
it = copy_map.begin();
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
2020-04-22 21:56:02 +01:00
|
|
|
|
|
|
|
/* Reduce the number of uses of the operand reg by one. Do this after
|
|
|
|
* splitting the copy or removing it in case the copy writes to it's own
|
|
|
|
* operand (for example, v[7:8] = v[8:9]) */
|
|
|
|
if (did_copy && !copy.second.op.isConstant()) {
|
|
|
|
for (std::pair<const PhysReg, copy_operation>& other : copy_map) {
|
2021-06-09 09:14:54 +01:00
|
|
|
for (uint16_t i = 0; i < other.second.bytes; i++) {
|
2020-04-22 21:56:02 +01:00
|
|
|
/* distance might underflow */
|
|
|
|
unsigned distance = other.first.reg_b + i - copy.second.op.physReg().reg_b;
|
|
|
|
if (distance < copy.second.bytes && !copy.second.uses[distance])
|
|
|
|
other.second.uses[i] -= 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/* all target regs are needed as operand somewhere which means, all entries are part of a cycle */
|
2020-04-06 17:13:52 +01:00
|
|
|
unsigned largest = 0;
|
2020-05-27 11:00:19 +01:00
|
|
|
for (const std::pair<const PhysReg, copy_operation>& op : copy_map)
|
2020-04-06 17:13:52 +01:00
|
|
|
largest = MAX2(largest, op.second.bytes);
|
|
|
|
|
|
|
|
while (!copy_map.empty()) {
|
|
|
|
|
2020-05-07 16:26:42 +01:00
|
|
|
/* Perform larger swaps first, because larger swaps swaps can make other
|
|
|
|
* swaps unnecessary. */
|
2020-04-06 17:13:52 +01:00
|
|
|
auto it = copy_map.begin();
|
|
|
|
for (auto it2 = copy_map.begin(); it2 != copy_map.end(); ++it2) {
|
|
|
|
if (it2->second.bytes > it->second.bytes) {
|
|
|
|
it = it2;
|
|
|
|
if (it->second.bytes == largest)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2020-03-23 20:44:27 +00:00
|
|
|
|
|
|
|
/* should already be done */
|
|
|
|
assert(!it->second.op.isConstant());
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2020-04-06 17:13:52 +01:00
|
|
|
assert(it->second.op.isFixed());
|
|
|
|
assert(it->second.def.regClass() == it->second.op.regClass());
|
|
|
|
|
|
|
|
if (it->first == it->second.op.physReg()) {
|
|
|
|
copy_map.erase(it);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2019-09-17 12:22:17 +01:00
|
|
|
if (preserve_scc && it->second.def.getTemp().type() == RegType::sgpr)
|
|
|
|
assert(!(it->second.def.physReg() == pi->scratch_sgpr));
|
|
|
|
|
|
|
|
/* to resolve the cycle, we have to swap the src reg with the dst reg */
|
|
|
|
copy_operation swap = it->second;
|
2020-04-27 20:13:53 +01:00
|
|
|
|
|
|
|
/* if this is self-intersecting, we have to split it because
|
|
|
|
* self-intersecting swaps don't make sense */
|
2020-12-03 19:55:10 +00:00
|
|
|
PhysReg src = swap.op.physReg(), dst = swap.def.physReg();
|
|
|
|
if (abs((int)src.reg_b - (int)dst.reg_b) < (int)swap.bytes) {
|
|
|
|
unsigned offset = abs((int)src.reg_b - (int)dst.reg_b);
|
2020-04-27 20:13:53 +01:00
|
|
|
|
2021-07-07 16:41:56 +01:00
|
|
|
copy_operation remaining;
|
2020-12-03 19:55:10 +00:00
|
|
|
src.reg_b += offset;
|
|
|
|
dst.reg_b += offset;
|
2021-07-07 16:41:56 +01:00
|
|
|
remaining.bytes = swap.bytes - offset;
|
|
|
|
memcpy(remaining.uses, swap.uses + offset, remaining.bytes);
|
2021-06-11 13:05:18 +01:00
|
|
|
remaining.op = Operand(src, swap.def.regClass().resize(remaining.bytes));
|
|
|
|
remaining.def = Definition(dst, swap.def.regClass().resize(remaining.bytes));
|
2021-07-07 16:41:56 +01:00
|
|
|
copy_map[dst] = remaining;
|
2020-04-27 20:13:53 +01:00
|
|
|
|
|
|
|
memset(swap.uses + offset, 0, swap.bytes - offset);
|
|
|
|
swap.bytes = offset;
|
|
|
|
}
|
|
|
|
|
2020-06-23 11:55:34 +01:00
|
|
|
/* GFX6-7 can only swap full registers */
|
|
|
|
if (ctx->program->chip_class <= GFX7)
|
|
|
|
swap.bytes = align(swap.bytes, 4);
|
|
|
|
|
2020-04-06 17:13:52 +01:00
|
|
|
do_swap(ctx, bld, swap, preserve_scc, pi);
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2020-04-06 17:13:52 +01:00
|
|
|
/* remove from map */
|
|
|
|
copy_map.erase(it);
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2020-05-07 16:26:42 +01:00
|
|
|
/* change the operand reg of the target's uses and split uses if needed */
|
2020-04-06 17:13:52 +01:00
|
|
|
uint32_t bytes_left = u_bit_consecutive(0, swap.bytes);
|
2020-11-03 13:40:05 +00:00
|
|
|
for (auto target = copy_map.begin(); target != copy_map.end(); ++target) {
|
2021-06-09 09:14:54 +01:00
|
|
|
if (target->second.op.physReg() == swap.def.physReg() &&
|
|
|
|
swap.bytes == target->second.bytes) {
|
2019-09-17 12:22:17 +01:00
|
|
|
target->second.op.setFixed(swap.op.physReg());
|
|
|
|
break;
|
|
|
|
}
|
2020-04-06 17:13:52 +01:00
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
uint32_t imask =
|
|
|
|
get_intersection_mask(swap.def.physReg().reg_b, swap.bytes,
|
|
|
|
target->second.op.physReg().reg_b, target->second.bytes);
|
2020-04-06 17:13:52 +01:00
|
|
|
|
|
|
|
if (!imask)
|
|
|
|
continue;
|
|
|
|
|
2020-05-07 16:26:42 +01:00
|
|
|
int offset = (int)target->second.op.physReg().reg_b - (int)swap.def.physReg().reg_b;
|
|
|
|
|
|
|
|
/* split and update the middle (the portion that reads the swap's
|
|
|
|
* definition) to read the swap's operand instead */
|
|
|
|
int target_op_end = target->second.op.physReg().reg_b + target->second.bytes;
|
|
|
|
int swap_def_end = swap.def.physReg().reg_b + swap.bytes;
|
|
|
|
int before_bytes = MAX2(-offset, 0);
|
|
|
|
int after_bytes = MAX2(target_op_end - swap_def_end, 0);
|
|
|
|
int middle_bytes = target->second.bytes - before_bytes - after_bytes;
|
|
|
|
|
|
|
|
if (after_bytes) {
|
|
|
|
unsigned after_offset = before_bytes + middle_bytes;
|
|
|
|
assert(after_offset > 0);
|
|
|
|
copy_operation copy;
|
|
|
|
copy.bytes = after_bytes;
|
|
|
|
memcpy(copy.uses, target->second.uses + after_offset, copy.bytes);
|
2021-06-11 13:05:18 +01:00
|
|
|
RegClass rc = target->second.op.regClass().resize(after_bytes);
|
2020-05-07 16:26:42 +01:00
|
|
|
copy.op = Operand(target->second.op.physReg().advance(after_offset), rc);
|
|
|
|
copy.def = Definition(target->second.def.physReg().advance(after_offset), rc);
|
|
|
|
copy_map[copy.def.physReg()] = copy;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (middle_bytes) {
|
|
|
|
copy_operation copy;
|
|
|
|
copy.bytes = middle_bytes;
|
|
|
|
memcpy(copy.uses, target->second.uses + before_bytes, copy.bytes);
|
2021-06-11 13:05:18 +01:00
|
|
|
RegClass rc = target->second.op.regClass().resize(middle_bytes);
|
2020-05-07 16:26:42 +01:00
|
|
|
copy.op = Operand(swap.op.physReg().advance(MAX2(offset, 0)), rc);
|
|
|
|
copy.def = Definition(target->second.def.physReg().advance(before_bytes), rc);
|
|
|
|
copy_map[copy.def.physReg()] = copy;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (before_bytes) {
|
|
|
|
copy_operation copy;
|
|
|
|
target->second.bytes = before_bytes;
|
2021-06-11 13:05:18 +01:00
|
|
|
RegClass rc = target->second.op.regClass().resize(before_bytes);
|
2020-05-07 16:26:42 +01:00
|
|
|
target->second.op = Operand(target->second.op.physReg(), rc);
|
|
|
|
target->second.def = Definition(target->second.def.physReg(), rc);
|
|
|
|
memset(target->second.uses + target->second.bytes, 0, 8 - target->second.bytes);
|
|
|
|
}
|
2020-04-06 17:13:52 +01:00
|
|
|
|
2020-05-07 16:26:42 +01:00
|
|
|
/* break early since we know each byte of the swap's definition is used
|
|
|
|
* at most once */
|
2020-04-06 17:13:52 +01:00
|
|
|
bytes_left &= ~imask;
|
|
|
|
if (!bytes_left)
|
|
|
|
break;
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
}
|
2020-06-05 21:05:31 +01:00
|
|
|
ctx->program->statistics[statistic_copies] += ctx->instructions.size() - num_instructions_before;
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_set_mode(Builder& bld, float_mode new_mode, bool set_round, bool set_denorm)
|
2020-07-02 13:33:55 +01:00
|
|
|
{
|
|
|
|
if (bld.program->chip_class >= GFX10) {
|
|
|
|
if (set_round)
|
|
|
|
bld.sopp(aco_opcode::s_round_mode, -1, new_mode.round);
|
|
|
|
if (set_denorm)
|
|
|
|
bld.sopp(aco_opcode::s_denorm_mode, -1, new_mode.denorm);
|
|
|
|
} else if (set_round || set_denorm) {
|
|
|
|
/* "((size - 1) << 11) | register" (MODE is encoded as register 1) */
|
2021-06-09 09:14:54 +01:00
|
|
|
Instruction* instr =
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sopk(aco_opcode::s_setreg_imm32_b32, Operand::c8(new_mode.val), (7 << 11) | 1).instr;
|
2020-07-02 13:33:55 +01:00
|
|
|
/* has to be a literal */
|
|
|
|
instr->operands[0].setFixed(PhysReg{255});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
emit_set_mode_from_block(Builder& bld, Program& program, Block* block, bool always_set)
|
2020-11-03 13:40:05 +00:00
|
|
|
{
|
|
|
|
float_mode config_mode;
|
|
|
|
config_mode.val = program.config->float_mode;
|
|
|
|
|
|
|
|
bool set_round = always_set && block->fp_mode.round != config_mode.round;
|
|
|
|
bool set_denorm = always_set && block->fp_mode.denorm != config_mode.denorm;
|
|
|
|
if (block->kind & block_kind_top_level) {
|
|
|
|
for (unsigned pred : block->linear_preds) {
|
|
|
|
if (program.blocks[pred].fp_mode.round != block->fp_mode.round)
|
|
|
|
set_round = true;
|
|
|
|
if (program.blocks[pred].fp_mode.denorm != block->fp_mode.denorm)
|
|
|
|
set_denorm = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/* only allow changing modes at top-level blocks so this doesn't break
|
|
|
|
* the "jump over empty blocks" optimization */
|
|
|
|
assert((!set_round && !set_denorm) || (block->kind & block_kind_top_level));
|
|
|
|
emit_set_mode(bld, block->fp_mode, set_round, set_denorm);
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
void
|
|
|
|
lower_to_hw_instr(Program* program)
|
2019-09-17 12:22:17 +01:00
|
|
|
{
|
2021-06-09 09:14:54 +01:00
|
|
|
Block* discard_block = NULL;
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
for (int block_idx = program->blocks.size() - 1; block_idx >= 0; block_idx--) {
|
|
|
|
Block* block = &program->blocks[block_idx];
|
2019-09-17 12:22:17 +01:00
|
|
|
lower_context ctx;
|
|
|
|
ctx.program = program;
|
2020-10-13 13:38:16 +01:00
|
|
|
ctx.block = block;
|
2019-09-17 12:22:17 +01:00
|
|
|
Builder bld(program, &ctx.instructions);
|
|
|
|
|
2020-11-03 13:40:05 +00:00
|
|
|
emit_set_mode_from_block(bld, *program, block, (block_idx == 0));
|
2019-11-09 20:51:45 +00:00
|
|
|
|
2020-11-03 13:40:05 +00:00
|
|
|
for (size_t instr_idx = 0; instr_idx < block->instructions.size(); instr_idx++) {
|
|
|
|
aco_ptr<Instruction>& instr = block->instructions[instr_idx];
|
2019-09-17 12:22:17 +01:00
|
|
|
aco_ptr<Instruction> mov;
|
2021-01-20 15:27:16 +00:00
|
|
|
if (instr->isPseudo() && instr->opcode != aco_opcode::p_unit_test) {
|
2021-06-09 09:14:54 +01:00
|
|
|
Pseudo_instruction* pi = &instr->pseudo();
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
switch (instr->opcode) {
|
|
|
|
case aco_opcode::p_extract_vector: {
|
2020-02-20 10:34:40 +00:00
|
|
|
PhysReg reg = instr->operands[0].physReg();
|
2020-02-27 12:07:21 +00:00
|
|
|
Definition& def = instr->definitions[0];
|
|
|
|
reg.reg_b += instr->operands[1].constantValue() * def.bytes();
|
2020-02-20 10:34:40 +00:00
|
|
|
|
2020-02-27 12:07:21 +00:00
|
|
|
if (reg == def.physReg())
|
2019-09-17 12:22:17 +01:00
|
|
|
break;
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
RegClass op_rc = def.regClass().is_subdword()
|
|
|
|
? def.regClass()
|
|
|
|
: RegClass(instr->operands[0].getTemp().type(), def.size());
|
2019-09-17 12:22:17 +01:00
|
|
|
std::map<PhysReg, copy_operation> copy_operations;
|
2020-02-27 12:07:21 +00:00
|
|
|
copy_operations[def.physReg()] = {Operand(reg, op_rc), def, def.bytes()};
|
2019-09-17 12:22:17 +01:00
|
|
|
handle_operands(copy_operations, &ctx, program->chip_class, pi);
|
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_create_vector: {
|
2019-09-17 12:22:17 +01:00
|
|
|
std::map<PhysReg, copy_operation> copy_operations;
|
2020-02-20 10:34:40 +00:00
|
|
|
PhysReg reg = instr->definitions[0].physReg();
|
2020-02-27 12:07:21 +00:00
|
|
|
|
2019-09-17 12:22:17 +01:00
|
|
|
for (const Operand& op : instr->operands) {
|
|
|
|
if (op.isConstant()) {
|
2021-06-09 09:14:54 +01:00
|
|
|
const Definition def = Definition(
|
2021-06-11 13:05:18 +01:00
|
|
|
reg, instr->definitions[0].getTemp().regClass().resize(op.bytes()));
|
2020-02-27 12:07:21 +00:00
|
|
|
copy_operations[reg] = {op, def, op.bytes()};
|
2020-02-20 10:34:40 +00:00
|
|
|
reg.reg_b += op.bytes();
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
if (op.isUndefined()) {
|
2020-02-27 12:07:21 +00:00
|
|
|
// TODO: coalesce subdword copies if dst byte is 0
|
2020-02-20 10:34:40 +00:00
|
|
|
reg.reg_b += op.bytes();
|
2019-09-17 12:22:17 +01:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
RegClass rc_def =
|
|
|
|
op.regClass().is_subdword()
|
|
|
|
? op.regClass()
|
2021-06-11 13:05:18 +01:00
|
|
|
: instr->definitions[0].getTemp().regClass().resize(op.bytes());
|
2020-02-27 12:07:21 +00:00
|
|
|
const Definition def = Definition(reg, rc_def);
|
|
|
|
copy_operations[def.physReg()] = {op, def, op.bytes()};
|
2020-02-20 10:34:40 +00:00
|
|
|
reg.reg_b += op.bytes();
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
handle_operands(copy_operations, &ctx, program->chip_class, pi);
|
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_split_vector: {
|
2019-09-17 12:22:17 +01:00
|
|
|
std::map<PhysReg, copy_operation> copy_operations;
|
2020-02-20 10:34:40 +00:00
|
|
|
PhysReg reg = instr->operands[0].physReg();
|
|
|
|
|
|
|
|
for (const Definition& def : instr->definitions) {
|
2021-06-09 09:14:54 +01:00
|
|
|
RegClass rc_op = def.regClass().is_subdword()
|
|
|
|
? def.regClass()
|
2021-06-11 13:05:18 +01:00
|
|
|
: instr->operands[0].getTemp().regClass().resize(def.bytes());
|
2020-02-27 12:07:21 +00:00
|
|
|
const Operand op = Operand(reg, rc_op);
|
|
|
|
copy_operations[def.physReg()] = {op, def, def.bytes()};
|
2020-02-20 10:34:40 +00:00
|
|
|
reg.reg_b += def.bytes();
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
handle_operands(copy_operations, &ctx, program->chip_class, pi);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case aco_opcode::p_parallelcopy:
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_wqm: {
|
2019-09-17 12:22:17 +01:00
|
|
|
std::map<PhysReg, copy_operation> copy_operations;
|
2020-11-03 13:40:05 +00:00
|
|
|
for (unsigned j = 0; j < instr->operands.size(); j++) {
|
|
|
|
assert(instr->definitions[j].bytes() == instr->operands[j].bytes());
|
2021-06-09 09:14:54 +01:00
|
|
|
copy_operations[instr->definitions[j].physReg()] = {
|
|
|
|
instr->operands[j], instr->definitions[j], instr->operands[j].bytes()};
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
handle_operands(copy_operations, &ctx, program->chip_class, pi);
|
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_exit_early_if: {
|
2020-03-12 16:20:16 +00:00
|
|
|
/* don't bother with an early exit near the end of the program */
|
2020-11-03 13:40:05 +00:00
|
|
|
if ((block->instructions.size() - 1 - instr_idx) <= 4 &&
|
2021-06-09 09:14:54 +01:00
|
|
|
block->instructions.back()->opcode == aco_opcode::s_endpgm) {
|
|
|
|
unsigned null_exp_dest =
|
|
|
|
(ctx.program->stage.hw == HWStage::FS) ? 9 /* NULL */ : V_008DFC_SQ_EXP_POS;
|
2020-03-12 16:20:16 +00:00
|
|
|
bool ignore_early_exit = true;
|
|
|
|
|
2020-11-03 13:40:05 +00:00
|
|
|
for (unsigned k = instr_idx + 1; k < block->instructions.size(); ++k) {
|
2021-06-09 09:14:54 +01:00
|
|
|
const aco_ptr<Instruction>& instr2 = block->instructions[k];
|
2020-11-03 13:40:05 +00:00
|
|
|
if (instr2->opcode == aco_opcode::s_endpgm ||
|
|
|
|
instr2->opcode == aco_opcode::p_logical_end)
|
2020-03-12 16:20:16 +00:00
|
|
|
continue;
|
2020-11-03 13:40:05 +00:00
|
|
|
else if (instr2->opcode == aco_opcode::exp &&
|
2021-01-21 16:13:34 +00:00
|
|
|
instr2->exp().dest == null_exp_dest)
|
2020-03-12 16:20:16 +00:00
|
|
|
continue;
|
2020-11-03 13:40:05 +00:00
|
|
|
else if (instr2->opcode == aco_opcode::p_parallelcopy &&
|
2021-06-09 09:14:54 +01:00
|
|
|
instr2->definitions[0].isFixed() &&
|
|
|
|
instr2->definitions[0].physReg() == exec)
|
2020-03-12 16:20:16 +00:00
|
|
|
continue;
|
|
|
|
|
|
|
|
ignore_early_exit = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ignore_early_exit)
|
|
|
|
break;
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
2019-10-08 13:40:17 +01:00
|
|
|
if (!discard_block) {
|
2019-09-17 12:22:17 +01:00
|
|
|
discard_block = program->create_and_insert_block();
|
2020-11-03 13:40:05 +00:00
|
|
|
block = &program->blocks[block_idx];
|
2019-09-17 12:22:17 +01:00
|
|
|
|
|
|
|
bld.reset(discard_block);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1), 0,
|
|
|
|
V_008DFC_SQ_EXP_NULL, false, true, true);
|
2019-09-17 12:22:17 +01:00
|
|
|
bld.sopp(aco_opcode::s_endpgm);
|
|
|
|
|
|
|
|
bld.reset(&ctx.instructions);
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
// TODO: exec can be zero here with block_kind_discard
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2019-10-08 13:40:17 +01:00
|
|
|
assert(instr->operands[0].physReg() == scc);
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sopp(aco_opcode::s_cbranch_scc0, Definition(exec, s2), instr->operands[0],
|
|
|
|
discard_block->index);
|
2019-09-17 12:22:17 +01:00
|
|
|
|
2019-10-08 13:40:17 +01:00
|
|
|
discard_block->linear_preds.push_back(block->index);
|
|
|
|
block->linear_succs.push_back(discard_block->index);
|
2019-09-17 12:22:17 +01:00
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_spill: {
|
2019-09-17 12:22:17 +01:00
|
|
|
assert(instr->operands[0].regClass() == v1.as_linear());
|
2021-02-22 13:55:45 +00:00
|
|
|
for (unsigned i = 0; i < instr->operands[2].size(); i++) {
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand src =
|
|
|
|
instr->operands[2].isConstant()
|
2021-07-13 10:22:46 +01:00
|
|
|
? Operand::c32(uint32_t(instr->operands[2].constantValue64() >> (32 * i)))
|
2021-06-09 09:14:54 +01:00
|
|
|
: Operand(PhysReg{instr->operands[2].physReg() + i}, s1);
|
|
|
|
bld.writelane(bld.def(v1, instr->operands[0].physReg()), src,
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(instr->operands[1].constantValue() + i),
|
2019-11-07 17:02:33 +00:00
|
|
|
instr->operands[0]);
|
2021-02-22 13:55:45 +00:00
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_reload: {
|
2019-09-17 12:22:17 +01:00
|
|
|
assert(instr->operands[0].regClass() == v1.as_linear());
|
2019-11-07 17:02:33 +00:00
|
|
|
for (unsigned i = 0; i < instr->definitions[0].size(); i++)
|
|
|
|
bld.readlane(bld.def(s1, PhysReg{instr->definitions[0].physReg() + i}),
|
2021-07-13 10:22:46 +01:00
|
|
|
instr->operands[0],
|
|
|
|
Operand::c32(instr->operands[1].constantValue() + i));
|
2019-09-17 12:22:17 +01:00
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_as_uniform: {
|
|
|
|
if (instr->operands[0].isConstant() ||
|
|
|
|
instr->operands[0].regClass().type() == RegType::sgpr) {
|
2019-09-17 12:22:17 +01:00
|
|
|
std::map<PhysReg, copy_operation> copy_operations;
|
2021-06-09 09:14:54 +01:00
|
|
|
copy_operations[instr->definitions[0].physReg()] = {
|
|
|
|
instr->operands[0], instr->definitions[0], instr->definitions[0].bytes()};
|
2019-09-17 12:22:17 +01:00
|
|
|
handle_operands(copy_operations, &ctx, program->chip_class, pi);
|
|
|
|
} else {
|
|
|
|
assert(instr->operands[0].regClass().type() == RegType::vgpr);
|
|
|
|
assert(instr->definitions[0].regClass().type() == RegType::sgpr);
|
|
|
|
assert(instr->operands[0].size() == instr->definitions[0].size());
|
|
|
|
for (unsigned i = 0; i < instr->definitions[0].size(); i++) {
|
|
|
|
bld.vop1(aco_opcode::v_readfirstlane_b32,
|
|
|
|
bld.def(s1, PhysReg{instr->definitions[0].physReg() + i}),
|
|
|
|
Operand(PhysReg{instr->operands[0].physReg() + i}, v1));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_bpermute: {
|
2020-05-27 00:22:28 +01:00
|
|
|
if (ctx.program->chip_class <= GFX7)
|
2020-05-27 00:28:03 +01:00
|
|
|
emit_gfx6_bpermute(program, instr, bld);
|
2020-06-08 17:45:35 +01:00
|
|
|
else if (ctx.program->chip_class >= GFX10 && ctx.program->wave_size == 64)
|
2020-05-27 00:22:28 +01:00
|
|
|
emit_gfx10_wave64_bpermute(program, instr, bld);
|
|
|
|
else
|
|
|
|
unreachable("Current hardware supports ds_bpermute, don't emit p_bpermute.");
|
2021-02-01 12:42:38 +00:00
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_constaddr: {
|
2021-02-01 12:42:38 +00:00
|
|
|
unsigned id = instr->definitions[0].tempId();
|
|
|
|
PhysReg reg = instr->definitions[0].physReg();
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop1(aco_opcode::p_constaddr_getpc, instr->definitions[0], Operand::c32(id));
|
2021-02-01 12:42:38 +00:00
|
|
|
bld.sop2(aco_opcode::p_constaddr_addlo, Definition(reg, s1), bld.def(s1, scc),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand(reg, s1), Operand::c32(id));
|
2021-02-01 12:42:38 +00:00
|
|
|
bld.sop2(aco_opcode::s_addc_u32, Definition(reg.advance(4), s1), bld.def(s1, scc),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand(reg.advance(4), s1), Operand::zero(), Operand(scc, s1));
|
2020-08-12 14:35:15 +01:00
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_extract: {
|
2020-08-12 14:35:15 +01:00
|
|
|
assert(instr->operands[1].isConstant());
|
|
|
|
assert(instr->operands[2].isConstant());
|
|
|
|
assert(instr->operands[3].isConstant());
|
|
|
|
if (instr->definitions[0].regClass() == s1)
|
|
|
|
assert(instr->definitions.size() >= 2 && instr->definitions[1].physReg() == scc);
|
|
|
|
Definition dst = instr->definitions[0];
|
|
|
|
Operand op = instr->operands[0];
|
|
|
|
unsigned bits = instr->operands[2].constantValue();
|
|
|
|
unsigned index = instr->operands[1].constantValue();
|
|
|
|
unsigned offset = index * bits;
|
|
|
|
bool signext = !instr->operands[3].constantEquals(0);
|
|
|
|
|
|
|
|
if (dst.regClass() == s1) {
|
|
|
|
if (offset == (32 - bits)) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop2(signext ? aco_opcode::s_ashr_i32 : aco_opcode::s_lshr_b32, dst,
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.def(s1, scc), op, Operand::c32(offset));
|
2020-08-12 14:35:15 +01:00
|
|
|
} else if (offset == 0 && signext && (bits == 8 || bits == 16)) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop1(bits == 8 ? aco_opcode::s_sext_i32_i8 : aco_opcode::s_sext_i32_i16,
|
|
|
|
dst, op);
|
2020-08-12 14:35:15 +01:00
|
|
|
} else {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop2(signext ? aco_opcode::s_bfe_i32 : aco_opcode::s_bfe_u32, dst,
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.def(s1, scc), op, Operand::c32((bits << 16) | offset));
|
2020-08-12 14:35:15 +01:00
|
|
|
}
|
2021-06-23 17:27:19 +01:00
|
|
|
} else if ((dst.regClass() == v1 && op.regClass() == v1) ||
|
|
|
|
ctx.program->chip_class <= GFX7) {
|
2020-08-12 14:35:15 +01:00
|
|
|
assert(op.physReg().byte() == 0 && dst.physReg().byte() == 0);
|
|
|
|
if (offset == (32 - bits) && op.regClass() != s1) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop2(signext ? aco_opcode::v_ashrrev_i32 : aco_opcode::v_lshrrev_b32, dst,
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(offset), op);
|
2020-08-12 14:35:15 +01:00
|
|
|
} else {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.vop3(signext ? aco_opcode::v_bfe_i32 : aco_opcode::v_bfe_u32, dst, op,
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(offset), Operand::c32(bits));
|
2020-08-12 14:35:15 +01:00
|
|
|
}
|
2021-06-23 17:27:19 +01:00
|
|
|
} else {
|
|
|
|
assert(dst.regClass() == v2b || dst.regClass() == v1b || op.regClass() == v2b ||
|
|
|
|
op.regClass() == v1b);
|
|
|
|
SDWA_instruction& sdwa =
|
|
|
|
bld.vop1_sdwa(aco_opcode::v_mov_b32, dst, op).instr->sdwa();
|
|
|
|
sdwa.sel[0] = SubdwordSel(bits / 8, offset / 8, signext);
|
2020-08-12 14:35:15 +01:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_insert: {
|
2020-08-12 14:35:15 +01:00
|
|
|
assert(instr->operands[1].isConstant());
|
|
|
|
assert(instr->operands[2].isConstant());
|
|
|
|
if (instr->definitions[0].regClass() == s1)
|
|
|
|
assert(instr->definitions.size() >= 2 && instr->definitions[1].physReg() == scc);
|
|
|
|
Definition dst = instr->definitions[0];
|
|
|
|
Operand op = instr->operands[0];
|
|
|
|
unsigned bits = instr->operands[2].constantValue();
|
|
|
|
unsigned index = instr->operands[1].constantValue();
|
|
|
|
unsigned offset = index * bits;
|
|
|
|
|
|
|
|
if (dst.regClass() == s1) {
|
|
|
|
if (offset == (32 - bits)) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.sop2(aco_opcode::s_lshl_b32, dst, bld.def(s1, scc), op,
|
|
|
|
Operand::c32(offset));
|
2020-08-12 14:35:15 +01:00
|
|
|
} else if (offset == 0) {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfe_u32, dst, bld.def(s1, scc), op,
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(bits << 16));
|
2020-08-12 14:35:15 +01:00
|
|
|
} else {
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop2(aco_opcode::s_bfe_u32, dst, bld.def(s1, scc), op,
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand::c32(bits << 16));
|
2021-06-09 09:14:54 +01:00
|
|
|
bld.sop2(aco_opcode::s_lshl_b32, dst, bld.def(s1, scc),
|
2021-07-13 10:22:46 +01:00
|
|
|
Operand(dst.physReg(), s1), Operand::c32(offset));
|
2020-08-12 14:35:15 +01:00
|
|
|
}
|
|
|
|
} else if (dst.regClass() == v1 || ctx.program->chip_class <= GFX7) {
|
|
|
|
if (offset == (dst.bytes() * 8u - bits)) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop2(aco_opcode::v_lshlrev_b32, dst, Operand::c32(offset), op);
|
2020-08-12 14:35:15 +01:00
|
|
|
} else if (offset == 0) {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_bfe_u32, dst, op, Operand::zero(), Operand::c32(bits));
|
2021-06-09 09:14:54 +01:00
|
|
|
} else if (program->chip_class >= GFX9 ||
|
|
|
|
(op.regClass() != s1 && program->chip_class >= GFX8)) {
|
2021-09-01 14:54:35 +01:00
|
|
|
bld.vop1_sdwa(aco_opcode::v_mov_b32, dst, op).instr->sdwa().dst_sel =
|
|
|
|
SubdwordSel(bits / 8, offset / 8, false);
|
2020-08-12 14:35:15 +01:00
|
|
|
} else {
|
2021-07-13 10:22:46 +01:00
|
|
|
bld.vop3(aco_opcode::v_bfe_u32, dst, op, Operand::zero(), Operand::c32(bits));
|
|
|
|
bld.vop2(aco_opcode::v_lshlrev_b32, dst, Operand::c32(offset),
|
2021-06-09 09:14:54 +01:00
|
|
|
Operand(dst.physReg(), v1));
|
2020-08-12 14:35:15 +01:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
assert(dst.regClass() == v2b);
|
2021-09-01 14:54:35 +01:00
|
|
|
bld.vop2_sdwa(aco_opcode::v_lshlrev_b32, dst, Operand::c32(offset), op)
|
2021-09-01 14:22:12 +01:00
|
|
|
.instr->sdwa()
|
2021-09-01 14:54:35 +01:00
|
|
|
.sel[1] = SubdwordSel::ubyte;
|
2020-08-12 14:35:15 +01:00
|
|
|
}
|
2020-06-26 15:54:22 +01:00
|
|
|
break;
|
2020-05-27 00:22:28 +01:00
|
|
|
}
|
2021-06-09 09:14:54 +01:00
|
|
|
default: break;
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
2021-01-20 15:27:16 +00:00
|
|
|
} else if (instr->isBranch()) {
|
2021-01-21 16:13:34 +00:00
|
|
|
Pseudo_branch_instruction* branch = &instr->branch();
|
2020-11-26 16:36:47 +00:00
|
|
|
uint32_t target = branch->target[0];
|
|
|
|
|
2019-09-17 12:22:17 +01:00
|
|
|
/* check if all blocks from current to target are empty */
|
2020-11-26 16:36:47 +00:00
|
|
|
/* In case there are <= 4 SALU or <= 2 VALU instructions, remove the branch */
|
|
|
|
bool can_remove = block->index < target;
|
|
|
|
unsigned num_scalar = 0;
|
|
|
|
unsigned num_vector = 0;
|
2019-09-17 12:22:17 +01:00
|
|
|
for (unsigned i = block->index + 1; can_remove && i < branch->target[0]; i++) {
|
2020-11-26 16:36:47 +00:00
|
|
|
/* uniform branches must not be ignored if they
|
|
|
|
* are about to jump over actual instructions */
|
|
|
|
if (!program->blocks[i].instructions.empty() &&
|
|
|
|
(branch->opcode != aco_opcode::p_cbranch_z ||
|
|
|
|
branch->operands[0].physReg() != exec)) {
|
2019-09-17 12:22:17 +01:00
|
|
|
can_remove = false;
|
2020-11-26 16:36:47 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (aco_ptr<Instruction>& inst : program->blocks[i].instructions) {
|
2021-01-20 15:27:16 +00:00
|
|
|
if (inst->isSOPP()) {
|
2020-11-26 16:36:47 +00:00
|
|
|
can_remove = false;
|
|
|
|
} else if (inst->isSALU()) {
|
|
|
|
num_scalar++;
|
|
|
|
} else if (inst->isVALU()) {
|
|
|
|
num_vector++;
|
|
|
|
} else {
|
|
|
|
can_remove = false;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (num_scalar + num_vector * 2 > 4)
|
|
|
|
can_remove = false;
|
|
|
|
|
|
|
|
if (!can_remove)
|
|
|
|
break;
|
|
|
|
}
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
2020-11-26 16:36:47 +00:00
|
|
|
|
2019-09-17 12:22:17 +01:00
|
|
|
if (can_remove)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
switch (instr->opcode) {
|
2021-06-09 09:14:54 +01:00
|
|
|
case aco_opcode::p_branch:
|
|
|
|
assert(block->linear_succs[0] == target);
|
|
|
|
bld.sopp(aco_opcode::s_branch, branch->definitions[0], target);
|
|
|
|
break;
|
|
|
|
case aco_opcode::p_cbranch_nz:
|
|
|
|
assert(block->linear_succs[1] == target);
|
|
|
|
if (branch->operands[0].physReg() == exec)
|
|
|
|
bld.sopp(aco_opcode::s_cbranch_execnz, branch->definitions[0], target);
|
|
|
|
else if (branch->operands[0].physReg() == vcc)
|
|
|
|
bld.sopp(aco_opcode::s_cbranch_vccnz, branch->definitions[0], target);
|
|
|
|
else {
|
|
|
|
assert(branch->operands[0].physReg() == scc);
|
|
|
|
bld.sopp(aco_opcode::s_cbranch_scc1, branch->definitions[0], target);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
case aco_opcode::p_cbranch_z:
|
|
|
|
assert(block->linear_succs[1] == target);
|
|
|
|
if (branch->operands[0].physReg() == exec)
|
|
|
|
bld.sopp(aco_opcode::s_cbranch_execz, branch->definitions[0], target);
|
|
|
|
else if (branch->operands[0].physReg() == vcc)
|
|
|
|
bld.sopp(aco_opcode::s_cbranch_vccz, branch->definitions[0], target);
|
|
|
|
else {
|
|
|
|
assert(branch->operands[0].physReg() == scc);
|
|
|
|
bld.sopp(aco_opcode::s_cbranch_scc0, branch->definitions[0], target);
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default: unreachable("Unknown Pseudo branch instruction!");
|
2019-09-17 12:22:17 +01:00
|
|
|
}
|
|
|
|
|
2021-01-20 15:27:16 +00:00
|
|
|
} else if (instr->isReduction()) {
|
2021-01-21 16:13:34 +00:00
|
|
|
Pseudo_reduction_instruction& reduce = instr->reduction();
|
|
|
|
emit_reduction(&ctx, reduce.opcode, reduce.reduce_op, reduce.cluster_size,
|
2021-06-09 09:14:54 +01:00
|
|
|
reduce.operands[1].physReg(), // tmp
|
2021-01-21 16:13:34 +00:00
|
|
|
reduce.definitions[1].physReg(), // stmp
|
2021-06-09 09:14:54 +01:00
|
|
|
reduce.operands[2].physReg(), // vtmp
|
2021-01-21 16:13:34 +00:00
|
|
|
reduce.definitions[2].physReg(), // sitmp
|
|
|
|
reduce.operands[0], reduce.definitions[0]);
|
2021-01-20 15:27:16 +00:00
|
|
|
} else if (instr->isBarrier()) {
|
2021-01-21 16:13:34 +00:00
|
|
|
Pseudo_barrier_instruction& barrier = instr->barrier();
|
2020-06-26 15:54:22 +01:00
|
|
|
|
|
|
|
/* Anything larger than a workgroup isn't possible. Anything
|
|
|
|
* smaller requires no instructions and this pseudo instruction
|
|
|
|
* would only be included to control optimizations. */
|
2021-01-21 16:13:34 +00:00
|
|
|
bool emit_s_barrier = barrier.exec_scope == scope_workgroup &&
|
2020-06-26 15:54:22 +01:00
|
|
|
program->workgroup_size > program->wave_size;
|
|
|
|
|
|
|
|
bld.insert(std::move(instr));
|
|
|
|
if (emit_s_barrier)
|
|
|
|
bld.sopp(aco_opcode::s_barrier);
|
2020-07-02 13:35:41 +01:00
|
|
|
} else if (instr->opcode == aco_opcode::p_cvt_f16_f32_rtne) {
|
|
|
|
float_mode new_mode = block->fp_mode;
|
|
|
|
new_mode.round16_64 = fp_round_ne;
|
|
|
|
bool set_round = new_mode.round != block->fp_mode.round;
|
|
|
|
|
|
|
|
emit_set_mode(bld, new_mode, set_round, false);
|
|
|
|
|
|
|
|
instr->opcode = aco_opcode::v_cvt_f16_f32;
|
|
|
|
ctx.instructions.emplace_back(std::move(instr));
|
|
|
|
|
|
|
|
emit_set_mode(bld, block->fp_mode, set_round, false);
|
2019-09-17 12:22:17 +01:00
|
|
|
} else {
|
|
|
|
ctx.instructions.emplace_back(std::move(instr));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
block->instructions.swap(ctx.instructions);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-09 09:14:54 +01:00
|
|
|
} // namespace aco
|