2020-02-27 14:38:21 +00:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2019 Collabora, Ltd.
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
|
|
* SOFTWARE.
|
|
|
|
*
|
|
|
|
* Authors (Collabora):
|
|
|
|
* Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "compiler.h"
|
|
|
|
|
|
|
|
/* Midgard's generic load/store instructions, particularly to implement SSBOs
|
|
|
|
* and globals, have support for address arithmetic natively. In particularly,
|
|
|
|
* they take two indirect arguments A, B and two immediates #s, #c, calculating
|
|
|
|
* the address:
|
|
|
|
*
|
|
|
|
* A + (zext?(B) << #s) + #c
|
|
|
|
*
|
|
|
|
* This allows for fast indexing into arrays. This file tries to pattern match the offset in NIR with this form to reduce pressure on the ALU pipe.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct mir_address {
|
|
|
|
nir_ssa_scalar A;
|
|
|
|
nir_ssa_scalar B;
|
|
|
|
|
|
|
|
bool zext;
|
|
|
|
unsigned shift;
|
|
|
|
unsigned bias;
|
|
|
|
};
|
|
|
|
|
2020-07-31 15:43:55 +01:00
|
|
|
static bool
|
|
|
|
mir_args_ssa(nir_ssa_scalar s, unsigned count)
|
|
|
|
{
|
|
|
|
nir_alu_instr *alu = nir_instr_as_alu(s.def->parent_instr);
|
|
|
|
assert(count <= nir_op_infos[alu->op].num_inputs);
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < count; ++i) {
|
|
|
|
if (!alu->src[i].src.is_ssa)
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2020-02-27 14:38:21 +00:00
|
|
|
/* Matches a constant in either slot and moves it to the bias */
|
|
|
|
|
|
|
|
static void
|
|
|
|
mir_match_constant(struct mir_address *address)
|
|
|
|
{
|
|
|
|
if (address->A.def && nir_ssa_scalar_is_const(address->A)) {
|
|
|
|
address->bias += nir_ssa_scalar_as_uint(address->A);
|
|
|
|
address->A.def = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (address->B.def && nir_ssa_scalar_is_const(address->B)) {
|
|
|
|
address->bias += nir_ssa_scalar_as_uint(address->B);
|
|
|
|
address->B.def = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Matches an iadd when there is a free slot or constant */
|
|
|
|
|
|
|
|
static void
|
|
|
|
mir_match_iadd(struct mir_address *address, bool first_free)
|
|
|
|
{
|
|
|
|
if (!address->B.def || !nir_ssa_scalar_is_alu(address->B))
|
|
|
|
return;
|
|
|
|
|
2020-07-31 15:43:55 +01:00
|
|
|
if (!mir_args_ssa(address->B, 2))
|
|
|
|
return;
|
|
|
|
|
2020-02-27 14:38:21 +00:00
|
|
|
nir_op op = nir_ssa_scalar_alu_op(address->B);
|
|
|
|
|
|
|
|
if (op != nir_op_iadd) return;
|
|
|
|
|
|
|
|
nir_ssa_scalar op1 = nir_ssa_scalar_chase_alu_src(address->B, 0);
|
|
|
|
nir_ssa_scalar op2 = nir_ssa_scalar_chase_alu_src(address->B, 1);
|
|
|
|
|
|
|
|
if (nir_ssa_scalar_is_const(op1)) {
|
|
|
|
address->bias += nir_ssa_scalar_as_uint(op1);
|
|
|
|
address->B = op2;
|
|
|
|
} else if (nir_ssa_scalar_is_const(op2)) {
|
|
|
|
address->bias += nir_ssa_scalar_as_uint(op2);
|
|
|
|
address->B = op1;
|
|
|
|
} else if (first_free && !address->A.def) {
|
|
|
|
address->A = op1;
|
|
|
|
address->B = op2;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Matches u2u64 and sets zext */
|
|
|
|
|
|
|
|
static void
|
|
|
|
mir_match_u2u64(struct mir_address *address)
|
|
|
|
{
|
|
|
|
if (!address->B.def || !nir_ssa_scalar_is_alu(address->B))
|
|
|
|
return;
|
|
|
|
|
2020-07-31 15:43:55 +01:00
|
|
|
if (!mir_args_ssa(address->B, 1))
|
|
|
|
return;
|
|
|
|
|
2020-02-27 14:38:21 +00:00
|
|
|
nir_op op = nir_ssa_scalar_alu_op(address->B);
|
|
|
|
if (op != nir_op_u2u64) return;
|
|
|
|
nir_ssa_scalar arg = nir_ssa_scalar_chase_alu_src(address->B, 0);
|
|
|
|
|
|
|
|
address->B = arg;
|
|
|
|
address->zext = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Matches ishl to shift */
|
|
|
|
|
|
|
|
static void
|
|
|
|
mir_match_ishl(struct mir_address *address)
|
|
|
|
{
|
|
|
|
if (!address->B.def || !nir_ssa_scalar_is_alu(address->B))
|
|
|
|
return;
|
|
|
|
|
2020-07-31 15:43:55 +01:00
|
|
|
if (!mir_args_ssa(address->B, 2))
|
|
|
|
return;
|
|
|
|
|
2020-02-27 14:38:21 +00:00
|
|
|
nir_op op = nir_ssa_scalar_alu_op(address->B);
|
|
|
|
if (op != nir_op_ishl) return;
|
|
|
|
nir_ssa_scalar op1 = nir_ssa_scalar_chase_alu_src(address->B, 0);
|
|
|
|
nir_ssa_scalar op2 = nir_ssa_scalar_chase_alu_src(address->B, 1);
|
|
|
|
|
|
|
|
if (!nir_ssa_scalar_is_const(op2)) return;
|
|
|
|
|
|
|
|
unsigned shift = nir_ssa_scalar_as_uint(op2);
|
|
|
|
if (shift > 0x7) return;
|
|
|
|
|
|
|
|
address->B = op1;
|
|
|
|
address->shift = shift;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Strings through mov which can happen from NIR vectorization */
|
|
|
|
|
|
|
|
static void
|
|
|
|
mir_match_mov(struct mir_address *address)
|
|
|
|
{
|
|
|
|
if (address->A.def && nir_ssa_scalar_is_alu(address->A)) {
|
|
|
|
nir_op op = nir_ssa_scalar_alu_op(address->A);
|
|
|
|
|
2020-07-31 15:43:55 +01:00
|
|
|
if (op == nir_op_mov && mir_args_ssa(address->A, 1))
|
2020-02-27 14:38:21 +00:00
|
|
|
address->A = nir_ssa_scalar_chase_alu_src(address->A, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (address->B.def && nir_ssa_scalar_is_alu(address->B)) {
|
|
|
|
nir_op op = nir_ssa_scalar_alu_op(address->B);
|
|
|
|
|
2020-07-31 15:43:55 +01:00
|
|
|
if (op == nir_op_mov && mir_args_ssa(address->B, 1))
|
2020-02-27 14:38:21 +00:00
|
|
|
address->B = nir_ssa_scalar_chase_alu_src(address->B, 0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Tries to pattern match into mir_address */
|
|
|
|
|
|
|
|
static struct mir_address
|
|
|
|
mir_match_offset(nir_ssa_def *offset, bool first_free)
|
|
|
|
{
|
|
|
|
struct mir_address address = {
|
|
|
|
.B = { .def = offset }
|
|
|
|
};
|
|
|
|
|
|
|
|
mir_match_mov(&address);
|
|
|
|
mir_match_constant(&address);
|
|
|
|
mir_match_mov(&address);
|
|
|
|
mir_match_iadd(&address, first_free);
|
|
|
|
mir_match_mov(&address);
|
|
|
|
mir_match_u2u64(&address);
|
|
|
|
mir_match_mov(&address);
|
|
|
|
mir_match_ishl(&address);
|
|
|
|
|
|
|
|
return address;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
mir_set_offset(compiler_context *ctx, midgard_instruction *ins, nir_src *offset, bool is_shared)
|
|
|
|
{
|
|
|
|
for(unsigned i = 0; i < 16; ++i) {
|
|
|
|
ins->swizzle[1][i] = 0;
|
|
|
|
ins->swizzle[2][i] = 0;
|
|
|
|
}
|
|
|
|
|
2020-08-13 23:56:40 +01:00
|
|
|
bool force_zext = (nir_src_bit_size(*offset) < 64);
|
|
|
|
|
2020-02-27 14:38:21 +00:00
|
|
|
if (!offset->is_ssa) {
|
|
|
|
ins->load_store.arg_1 |= is_shared ? 0x6E : 0x7E;
|
|
|
|
ins->src[2] = nir_src_index(ctx, offset);
|
2020-05-13 16:05:34 +01:00
|
|
|
ins->src_types[2] = nir_type_uint | nir_src_bit_size(*offset);
|
2020-08-13 23:56:40 +01:00
|
|
|
|
|
|
|
if (force_zext)
|
|
|
|
ins->load_store.arg_1 |= 0x80;
|
|
|
|
|
2020-02-27 14:38:21 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct mir_address match = mir_match_offset(offset->ssa, !is_shared);
|
|
|
|
|
|
|
|
if (match.A.def) {
|
|
|
|
ins->src[1] = nir_ssa_index(match.A.def);
|
|
|
|
ins->swizzle[1][0] = match.A.comp;
|
2020-05-13 16:05:34 +01:00
|
|
|
ins->src_types[1] = nir_type_uint | match.A.def->bit_size;
|
2020-02-27 14:38:21 +00:00
|
|
|
} else
|
|
|
|
ins->load_store.arg_1 |= is_shared ? 0x6E : 0x7E;
|
|
|
|
|
|
|
|
if (match.B.def) {
|
|
|
|
ins->src[2] = nir_ssa_index(match.B.def);
|
|
|
|
ins->swizzle[2][0] = match.B.comp;
|
2020-05-13 16:05:34 +01:00
|
|
|
ins->src_types[2] = nir_type_uint | match.B.def->bit_size;
|
2020-02-27 14:38:21 +00:00
|
|
|
} else
|
|
|
|
ins->load_store.arg_2 = 0x1E;
|
|
|
|
|
2020-08-13 23:56:40 +01:00
|
|
|
if (match.zext || force_zext)
|
2020-02-27 14:38:21 +00:00
|
|
|
ins->load_store.arg_1 |= 0x80;
|
|
|
|
|
|
|
|
assert(match.shift <= 7);
|
|
|
|
ins->load_store.arg_2 |= (match.shift) << 5;
|
|
|
|
|
|
|
|
ins->constants.u32[0] = match.bias;
|
|
|
|
}
|