2014-02-12 19:00:46 +00:00
|
|
|
/*
|
|
|
|
* Copyright © 2014 Intel Corporation
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/** @file brw_fs_combine_constants.cpp
|
|
|
|
*
|
|
|
|
* This file contains the opt_combine_constants() pass that runs after the
|
|
|
|
* regular optimization loop. It passes over the instruction list and
|
|
|
|
* selectively promotes immediate values to registers by emitting a mov(1)
|
|
|
|
* instruction.
|
|
|
|
*
|
|
|
|
* This is useful on Gen 7 particularly, because a few instructions can be
|
|
|
|
* coissued (i.e., issued in the same cycle as another thread on the same EU
|
|
|
|
* issues an instruction) under some circumstances, one of which is that they
|
|
|
|
* cannot use immediate values.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "brw_fs.h"
|
|
|
|
#include "brw_cfg.h"
|
2019-01-29 09:58:49 +00:00
|
|
|
#include "util/half_float.h"
|
2014-02-12 19:00:46 +00:00
|
|
|
|
2015-06-03 16:20:50 +01:00
|
|
|
using namespace brw;
|
|
|
|
|
2015-02-27 00:06:45 +00:00
|
|
|
static const bool debug = false;
|
|
|
|
|
2014-02-12 19:00:46 +00:00
|
|
|
/* Returns whether an instruction could co-issue if its immediate source were
|
|
|
|
* replaced with a GRF source.
|
|
|
|
*/
|
|
|
|
static bool
|
2016-08-22 23:01:08 +01:00
|
|
|
could_coissue(const struct gen_device_info *devinfo, const fs_inst *inst)
|
2014-02-12 19:00:46 +00:00
|
|
|
{
|
2021-03-29 22:41:58 +01:00
|
|
|
if (devinfo->ver != 7)
|
2014-02-12 19:00:46 +00:00
|
|
|
return false;
|
|
|
|
|
|
|
|
switch (inst->opcode) {
|
|
|
|
case BRW_OPCODE_MOV:
|
|
|
|
case BRW_OPCODE_CMP:
|
|
|
|
case BRW_OPCODE_ADD:
|
|
|
|
case BRW_OPCODE_MUL:
|
intel/fs: Don't count integer instructions as being possibly coissue
Integer instructions don't coissue. Before e64be391dd0
("intel/compiler: generalize the combine constants pass"), this pass
only looked at float sources. There's no shader-db data in that commit,
so I collected some. The results are not good:
Haswell
total instructions in shared programs: 11898805 -> 11908127 (0.08%)
instructions in affected programs: 1218680 -> 1228002 (0.76%)
helped: 2
HURT: 5171
helped stats (abs) min: 12 max: 111 x̄: 61.50 x̃: 61
helped stats (rel) min: 1.59% max: 9.20% x̄: 5.40% x̃: 5.40%
HURT stats (abs) min: 1 max: 311 x̄: 1.83 x̃: 1
HURT stats (rel) min: 0.02% max: 9.91% x̄: 1.05% x̃: 0.70%
95% mean confidence interval for instructions value: 1.55 2.05
95% mean confidence interval for instructions %-change: 1.02% 1.08%
Instructions are HURT.
total cycles in shared programs: 221664974 -> 221404750 (-0.12%)
cycles in affected programs: 120012620 -> 119752396 (-0.22%)
helped: 3464
HURT: 3159
helped stats (abs) min: 1 max: 428160 x̄: 314.55 x̃: 16
helped stats (rel) min: <.01% max: 57.33% x̄: 3.40% x̃: 1.28%
HURT stats (abs) min: 1 max: 87846 x̄: 262.54 x̃: 14
HURT stats (rel) min: <.01% max: 85.57% x̄: 3.01% x̃: 0.77%
95% mean confidence interval for cycles value: -224.23 145.65
95% mean confidence interval for cycles %-change: -0.50% -0.19%
Inconclusive result (value mean confidence interval includes 0).
total spills in shared programs: 9804 -> 10047 (2.48%)
spills in affected programs: 6869 -> 7112 (3.54%)
helped: 2
HURT: 41
total fills in shared programs: 19863 -> 20319 (2.30%)
fills in affected programs: 17428 -> 17884 (2.62%)
helped: 2
HURT: 41
LOST: 20
GAINED: 13
This also prevents regressions in "intel/fs: Promote integer constants
after lowering integer multiplication" (note: that patch will probably
not be committed). When the passes are reorderd, code like
mul(8) acc0<1>D g9<8,8,1>D -2078209981D { align1 1Q };
gets turned into
mov(1) g23<1>D 2078209981D { align1 WE_all 1N };
...
mul(8) acc0<1>D g13<8,8,1>D -g23<0,1,0>D { align1 1Q compacted };
It's not 100% clear why, but these produce different results. Note that
-2078209981 & 0x0ffff = 0x0843, and -(2078209981 & 0x0ffff) =
0xffff0843. It seems like the upper 16-bits of the negation should be
ignored.
Fixes: e64be391dd0 ("intel/compiler: generalize the combine constants pass")
Cc: Iago Toral Quiroga <itoral@igalia.com>
Suggested-by: Matt Turner <mattst88@gmail.com>
Reviewed-by: Matt Turner <mattst88@gmail.com>
The shaders with spills or fills hurt are the usual suspects. A couple
compute shaders in Dirt Showdown and a compute shader in Bioshock
Infinite. On Haswell, a compute shader (that appears twice in
shader-db) from Aztec Ruins was also hurt for spill and fills.
Haswell
total instructions in shared programs: 11573934 -> 11568335 (-0.05%)
instructions in affected programs: 828623 -> 823024 (-0.68%)
helped: 2825
HURT: 6
helped stats (abs) min: 1 max: 134 x̄: 2.16 x̃: 1
helped stats (rel) min: 0.02% max: 9.05% x̄: 0.84% x̃: 0.61%
HURT stats (abs) min: 1 max: 216 x̄: 81.83 x̃: 56
HURT stats (rel) min: 0.16% max: 8.65% x̄: 4.21% x̃: 4.68%
95% mean confidence interval for instructions value: -2.31 -1.64
95% mean confidence interval for instructions %-change: -0.85% -0.80%
Instructions are helped.
total cycles in shared programs: 187573593 -> 187004633 (-0.30%)
cycles in affected programs: 82816107 -> 82247147 (-0.69%)
helped: 2186
HURT: 1741
helped stats (abs) min: 1 max: 35230 x̄: 326.96 x̃: 16
helped stats (rel) min: <.01% max: 46.11% x̄: 3.11% x̃: 0.90%
HURT stats (abs) min: 1 max: 6138 x̄: 83.73 x̃: 16
HURT stats (rel) min: <.01% max: 104.11% x̄: 2.73% x̃: 0.75%
95% mean confidence interval for cycles value: -197.13 -92.64
95% mean confidence interval for cycles %-change: -0.72% -0.33%
Cycles are helped.
total spills in shared programs: 7870 -> 7743 (-1.61%)
spills in affected programs: 2260 -> 2133 (-5.62%)
helped: 31
HURT: 5
total fills in shared programs: 6320 -> 6263 (-0.90%)
fills in affected programs: 3547 -> 3490 (-1.61%)
helped: 31
HURT: 6
LOST: 9
GAINED: 9
Ivybridge
total instructions in shared programs: 11863372 -> 11859793 (-0.03%)
instructions in affected programs: 757183 -> 753604 (-0.47%)
helped: 2236
HURT: 3
helped stats (abs) min: 1 max: 81 x̄: 1.86 x̃: 1
helped stats (rel) min: 0.03% max: 5.26% x̄: 0.74% x̃: 0.48%
HURT stats (abs) min: 11 max: 301 x̄: 192.33 x̃: 265
HURT stats (rel) min: 1.55% max: 10.51% x̄: 6.89% x̃: 8.62%
95% mean confidence interval for instructions value: -2.01 -1.18
95% mean confidence interval for instructions %-change: -0.77% -0.70%
Instructions are helped.
total cycles in shared programs: 178377378 -> 177946087 (-0.24%)
cycles in affected programs: 76261390 -> 75830099 (-0.57%)
helped: 1635
HURT: 1395
helped stats (abs) min: 1 max: 34796 x̄: 333.53 x̃: 16
helped stats (rel) min: <.01% max: 47.15% x̄: 2.82% x̃: 0.64%
HURT stats (abs) min: 1 max: 4315 x̄: 81.74 x̃: 18
HURT stats (rel) min: <.01% max: 49.98% x̄: 1.99% x̃: 0.53%
95% mean confidence interval for cycles value: -197.06 -87.62
95% mean confidence interval for cycles %-change: -0.78% -0.43%
Cycles are helped.
total spills in shared programs: 4188 -> 4182 (-0.14%)
spills in affected programs: 1557 -> 1551 (-0.39%)
helped: 30
HURT: 3
total fills in shared programs: 5056 -> 5245 (3.74%)
fills in affected programs: 2708 -> 2897 (6.98%)
helped: 30
HURT: 3
LOST: 5
GAINED: 1
No shader-db changes on any other Intel platform.
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/3544>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/3544>
2020-01-23 00:23:14 +00:00
|
|
|
/* Only float instructions can coissue. We don't have a great
|
|
|
|
* understanding of whether or not something like float(int(a) + int(b))
|
|
|
|
* would be considered float (based on the destination type) or integer
|
|
|
|
* (based on the source types), so we take the conservative choice of
|
|
|
|
* only promoting when both destination and source are float.
|
|
|
|
*/
|
|
|
|
return inst->dst.type == BRW_REGISTER_TYPE_F &&
|
|
|
|
inst->src[0].type == BRW_REGISTER_TYPE_F;
|
2014-02-12 19:00:46 +00:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-27 05:07:06 +00:00
|
|
|
/**
|
|
|
|
* Returns true for instructions that don't support immediate sources.
|
|
|
|
*/
|
|
|
|
static bool
|
2016-08-22 23:01:08 +01:00
|
|
|
must_promote_imm(const struct gen_device_info *devinfo, const fs_inst *inst)
|
2014-10-27 05:07:06 +00:00
|
|
|
{
|
|
|
|
switch (inst->opcode) {
|
2015-03-17 00:53:34 +00:00
|
|
|
case SHADER_OPCODE_POW:
|
2021-03-29 22:41:58 +01:00
|
|
|
return devinfo->ver < 8;
|
2014-10-27 05:07:06 +00:00
|
|
|
case BRW_OPCODE_MAD:
|
|
|
|
case BRW_OPCODE_LRP:
|
|
|
|
return true;
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-02-12 19:00:46 +00:00
|
|
|
/** A box for putting fs_regs in a linked list. */
|
|
|
|
struct reg_link {
|
|
|
|
DECLARE_RALLOC_CXX_OPERATORS(reg_link)
|
|
|
|
|
|
|
|
reg_link(fs_reg *reg) : reg(reg) {}
|
|
|
|
|
|
|
|
struct exec_node link;
|
|
|
|
fs_reg *reg;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct exec_node *
|
|
|
|
link(void *mem_ctx, fs_reg *reg)
|
|
|
|
{
|
|
|
|
reg_link *l = new(mem_ctx) reg_link(reg);
|
|
|
|
return &l->link;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Information about an immediate value.
|
|
|
|
*/
|
|
|
|
struct imm {
|
|
|
|
/** The common ancestor of all blocks using this immediate value. */
|
|
|
|
bblock_t *block;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* The instruction generating the immediate value, if all uses are contained
|
|
|
|
* within a single basic block. Otherwise, NULL.
|
|
|
|
*/
|
|
|
|
fs_inst *inst;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* A list of fs_regs that refer to this immediate. If we promote it, we'll
|
|
|
|
* have to patch these up to refer to the new GRF.
|
|
|
|
*/
|
|
|
|
exec_list *uses;
|
|
|
|
|
2019-01-29 09:58:49 +00:00
|
|
|
/** The immediate value */
|
|
|
|
union {
|
|
|
|
char bytes[8];
|
|
|
|
double df;
|
|
|
|
int64_t d64;
|
|
|
|
float f;
|
|
|
|
int32_t d;
|
|
|
|
int16_t w;
|
|
|
|
};
|
|
|
|
uint8_t size;
|
|
|
|
|
|
|
|
/** When promoting half-float we need to account for certain restrictions */
|
|
|
|
bool is_half_float;
|
2014-02-12 19:00:46 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* The GRF register and subregister number where we've decided to store the
|
|
|
|
* constant value.
|
|
|
|
*/
|
|
|
|
uint8_t subreg_offset;
|
2015-10-26 11:35:14 +00:00
|
|
|
uint16_t nr;
|
2014-02-12 19:00:46 +00:00
|
|
|
|
|
|
|
/** The number of coissuable instructions using this immediate. */
|
|
|
|
uint16_t uses_by_coissue;
|
|
|
|
|
2014-10-27 05:07:06 +00:00
|
|
|
/**
|
|
|
|
* Whether this constant is used by an instruction that can't handle an
|
|
|
|
* immediate source (and already has to be promoted to a GRF).
|
|
|
|
*/
|
|
|
|
bool must_promote;
|
|
|
|
|
2014-02-12 19:00:46 +00:00
|
|
|
uint16_t first_use_ip;
|
|
|
|
uint16_t last_use_ip;
|
|
|
|
};
|
|
|
|
|
|
|
|
/** The working set of information about immediates. */
|
|
|
|
struct table {
|
|
|
|
struct imm *imm;
|
|
|
|
int size;
|
|
|
|
int len;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct imm *
|
2019-01-29 09:58:49 +00:00
|
|
|
find_imm(struct table *table, void *data, uint8_t size)
|
2014-02-12 19:00:46 +00:00
|
|
|
{
|
|
|
|
for (int i = 0; i < table->len; i++) {
|
2019-01-29 09:58:49 +00:00
|
|
|
if (table->imm[i].size == size &&
|
|
|
|
!memcmp(table->imm[i].bytes, data, size)) {
|
2014-02-12 19:00:46 +00:00
|
|
|
return &table->imm[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct imm *
|
|
|
|
new_imm(struct table *table, void *mem_ctx)
|
|
|
|
{
|
|
|
|
if (table->len == table->size) {
|
|
|
|
table->size *= 2;
|
|
|
|
table->imm = reralloc(mem_ctx, table->imm, struct imm, table->size);
|
|
|
|
}
|
|
|
|
return &table->imm[table->len++];
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Comparator used for sorting an array of imm structures.
|
|
|
|
*
|
|
|
|
* We sort by basic block number, then last use IP, then first use IP (least
|
|
|
|
* to greatest). This sorting causes immediates live in the same area to be
|
|
|
|
* allocated to the same register in the hopes that all values will be dead
|
|
|
|
* about the same time and the register can be reused.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
compare(const void *_a, const void *_b)
|
|
|
|
{
|
|
|
|
const struct imm *a = (const struct imm *)_a,
|
|
|
|
*b = (const struct imm *)_b;
|
|
|
|
|
|
|
|
int block_diff = a->block->num - b->block->num;
|
|
|
|
if (block_diff)
|
|
|
|
return block_diff;
|
|
|
|
|
|
|
|
int end_diff = a->last_use_ip - b->last_use_ip;
|
|
|
|
if (end_diff)
|
|
|
|
return end_diff;
|
|
|
|
|
|
|
|
return a->first_use_ip - b->first_use_ip;
|
|
|
|
}
|
|
|
|
|
2019-01-29 09:58:49 +00:00
|
|
|
static bool
|
|
|
|
get_constant_value(const struct gen_device_info *devinfo,
|
|
|
|
const fs_inst *inst, uint32_t src_idx,
|
|
|
|
void *out, brw_reg_type *out_type)
|
|
|
|
{
|
|
|
|
const bool can_do_source_mods = inst->can_do_source_mods(devinfo);
|
|
|
|
const fs_reg *src = &inst->src[src_idx];
|
|
|
|
|
|
|
|
*out_type = src->type;
|
|
|
|
|
|
|
|
switch (*out_type) {
|
|
|
|
case BRW_REGISTER_TYPE_DF: {
|
|
|
|
double val = !can_do_source_mods ? src->df : fabs(src->df);
|
|
|
|
memcpy(out, &val, 8);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case BRW_REGISTER_TYPE_F: {
|
|
|
|
float val = !can_do_source_mods ? src->f : fabsf(src->f);
|
|
|
|
memcpy(out, &val, 4);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case BRW_REGISTER_TYPE_HF: {
|
|
|
|
uint16_t val = src->d & 0xffffu;
|
|
|
|
if (can_do_source_mods)
|
|
|
|
val = _mesa_float_to_half(fabsf(_mesa_half_to_float(val)));
|
|
|
|
memcpy(out, &val, 2);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case BRW_REGISTER_TYPE_Q: {
|
2019-09-26 15:14:34 +01:00
|
|
|
int64_t val = !can_do_source_mods ? src->d64 : llabs(src->d64);
|
2019-01-29 09:58:49 +00:00
|
|
|
memcpy(out, &val, 8);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case BRW_REGISTER_TYPE_UQ:
|
|
|
|
memcpy(out, &src->u64, 8);
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_D: {
|
|
|
|
int32_t val = !can_do_source_mods ? src->d : abs(src->d);
|
|
|
|
memcpy(out, &val, 4);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case BRW_REGISTER_TYPE_UD:
|
|
|
|
memcpy(out, &src->ud, 4);
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_W: {
|
|
|
|
int16_t val = src->d & 0xffffu;
|
|
|
|
if (can_do_source_mods)
|
|
|
|
val = abs(val);
|
|
|
|
memcpy(out, &val, 2);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case BRW_REGISTER_TYPE_UW:
|
|
|
|
memcpy(out, &src->ud, 2);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
};
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct brw_reg
|
|
|
|
build_imm_reg_for_copy(struct imm *imm)
|
|
|
|
{
|
|
|
|
switch (imm->size) {
|
|
|
|
case 8:
|
|
|
|
return brw_imm_d(imm->d64);
|
|
|
|
case 4:
|
|
|
|
return brw_imm_d(imm->d);
|
|
|
|
case 2:
|
|
|
|
return brw_imm_w(imm->w);
|
|
|
|
default:
|
|
|
|
unreachable("not implemented");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline uint32_t
|
|
|
|
get_alignment_for_imm(const struct imm *imm)
|
|
|
|
{
|
|
|
|
if (imm->is_half_float)
|
|
|
|
return 4; /* At least MAD seems to require this */
|
|
|
|
else
|
|
|
|
return imm->size;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool
|
2019-07-23 12:48:58 +01:00
|
|
|
needs_negate(const fs_reg *reg, const struct imm *imm)
|
2019-01-29 09:58:49 +00:00
|
|
|
{
|
|
|
|
switch (reg->type) {
|
|
|
|
case BRW_REGISTER_TYPE_DF:
|
|
|
|
return signbit(reg->df) != signbit(imm->df);
|
|
|
|
case BRW_REGISTER_TYPE_F:
|
|
|
|
return signbit(reg->f) != signbit(imm->f);
|
|
|
|
case BRW_REGISTER_TYPE_Q:
|
|
|
|
return (reg->d64 < 0) != (imm->d64 < 0);
|
|
|
|
case BRW_REGISTER_TYPE_D:
|
|
|
|
return (reg->d < 0) != (imm->d < 0);
|
|
|
|
case BRW_REGISTER_TYPE_HF:
|
|
|
|
return (reg->d & 0x8000u) != (imm->w & 0x8000u);
|
|
|
|
case BRW_REGISTER_TYPE_W:
|
2019-04-19 19:32:59 +01:00
|
|
|
return ((int16_t)reg->d < 0) != (imm->w < 0);
|
2019-01-29 09:58:49 +00:00
|
|
|
case BRW_REGISTER_TYPE_UQ:
|
|
|
|
case BRW_REGISTER_TYPE_UD:
|
|
|
|
case BRW_REGISTER_TYPE_UW:
|
|
|
|
return false;
|
|
|
|
default:
|
|
|
|
unreachable("not implemented");
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-05-22 00:15:16 +01:00
|
|
|
static bool
|
|
|
|
representable_as_hf(float f, uint16_t *hf)
|
|
|
|
{
|
|
|
|
union fi u;
|
|
|
|
uint16_t h = _mesa_float_to_half(f);
|
|
|
|
u.f = _mesa_half_to_float(h);
|
|
|
|
|
|
|
|
if (u.f == f) {
|
|
|
|
*hf = h;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool
|
|
|
|
represent_src_as_imm(const struct gen_device_info *devinfo,
|
|
|
|
fs_reg *src)
|
|
|
|
{
|
2021-04-06 21:11:09 +01:00
|
|
|
/* TODO - Fix the codepath below to use a bfloat16 immediate on XeHP,
|
|
|
|
* since HF/F mixed mode has been removed from the hardware.
|
|
|
|
*/
|
|
|
|
if (devinfo->ver == 12 && devinfo->verx10 < 125) {
|
2019-05-22 00:15:16 +01:00
|
|
|
uint16_t hf;
|
|
|
|
if (representable_as_hf(src->f, &hf)) {
|
|
|
|
*src = retype(brw_imm_uw(hf), BRW_REGISTER_TYPE_HF);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2014-02-12 19:00:46 +00:00
|
|
|
bool
|
|
|
|
fs_visitor::opt_combine_constants()
|
|
|
|
{
|
|
|
|
void *const_ctx = ralloc_context(NULL);
|
|
|
|
|
|
|
|
struct table table;
|
|
|
|
table.size = 8;
|
|
|
|
table.len = 0;
|
|
|
|
table.imm = ralloc_array(const_ctx, struct imm, table.size);
|
|
|
|
|
2016-03-10 07:31:05 +00:00
|
|
|
const brw::idom_tree &idom = idom_analysis.require();
|
2014-02-12 19:00:46 +00:00
|
|
|
unsigned ip = -1;
|
|
|
|
|
|
|
|
/* Make a pass through all instructions and count the number of times each
|
2014-10-27 05:07:06 +00:00
|
|
|
* constant is used by coissueable instructions or instructions that cannot
|
|
|
|
* take immediate arguments.
|
2014-02-12 19:00:46 +00:00
|
|
|
*/
|
|
|
|
foreach_block_and_inst(block, fs_inst, inst, cfg) {
|
|
|
|
ip++;
|
|
|
|
|
2015-03-17 00:53:34 +00:00
|
|
|
if (!could_coissue(devinfo, inst) && !must_promote_imm(devinfo, inst))
|
2014-02-12 19:00:46 +00:00
|
|
|
continue;
|
|
|
|
|
2019-05-22 00:15:16 +01:00
|
|
|
bool represented_as_imm = false;
|
2014-02-12 19:00:46 +00:00
|
|
|
for (int i = 0; i < inst->sources; i++) {
|
2019-01-29 09:58:49 +00:00
|
|
|
if (inst->src[i].file != IMM)
|
|
|
|
continue;
|
|
|
|
|
2019-05-22 00:15:16 +01:00
|
|
|
if (!represented_as_imm && i == 0 &&
|
|
|
|
inst->opcode == BRW_OPCODE_MAD &&
|
|
|
|
represent_src_as_imm(devinfo, &inst->src[i])) {
|
|
|
|
represented_as_imm = true;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2019-01-29 09:58:49 +00:00
|
|
|
char data[8];
|
|
|
|
brw_reg_type type;
|
|
|
|
if (!get_constant_value(devinfo, inst, i, data, &type))
|
2014-02-12 19:00:46 +00:00
|
|
|
continue;
|
|
|
|
|
2019-01-29 09:58:49 +00:00
|
|
|
uint8_t size = type_sz(type);
|
|
|
|
|
|
|
|
struct imm *imm = find_imm(&table, data, size);
|
2014-02-12 19:00:46 +00:00
|
|
|
|
|
|
|
if (imm) {
|
2016-03-10 07:31:05 +00:00
|
|
|
bblock_t *intersection = idom.intersect(block, imm->block);
|
2014-02-12 19:00:46 +00:00
|
|
|
if (intersection != imm->block)
|
|
|
|
imm->inst = NULL;
|
|
|
|
imm->block = intersection;
|
|
|
|
imm->uses->push_tail(link(const_ctx, &inst->src[i]));
|
2015-04-16 02:00:05 +01:00
|
|
|
imm->uses_by_coissue += could_coissue(devinfo, inst);
|
2015-03-17 00:53:34 +00:00
|
|
|
imm->must_promote = imm->must_promote || must_promote_imm(devinfo, inst);
|
2014-02-12 19:00:46 +00:00
|
|
|
imm->last_use_ip = ip;
|
2019-01-29 09:58:49 +00:00
|
|
|
if (type == BRW_REGISTER_TYPE_HF)
|
|
|
|
imm->is_half_float = true;
|
2014-02-12 19:00:46 +00:00
|
|
|
} else {
|
|
|
|
imm = new_imm(&table, const_ctx);
|
|
|
|
imm->block = block;
|
|
|
|
imm->inst = inst;
|
|
|
|
imm->uses = new(const_ctx) exec_list();
|
|
|
|
imm->uses->push_tail(link(const_ctx, &inst->src[i]));
|
2019-01-29 09:58:49 +00:00
|
|
|
memcpy(imm->bytes, data, size);
|
|
|
|
imm->size = size;
|
|
|
|
imm->is_half_float = type == BRW_REGISTER_TYPE_HF;
|
2015-04-16 02:00:05 +01:00
|
|
|
imm->uses_by_coissue = could_coissue(devinfo, inst);
|
2015-03-17 00:53:34 +00:00
|
|
|
imm->must_promote = must_promote_imm(devinfo, inst);
|
2014-02-12 19:00:46 +00:00
|
|
|
imm->first_use_ip = ip;
|
|
|
|
imm->last_use_ip = ip;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Remove constants from the table that don't have enough uses to make them
|
|
|
|
* profitable to store in a register.
|
|
|
|
*/
|
|
|
|
for (int i = 0; i < table.len;) {
|
|
|
|
struct imm *imm = &table.imm[i];
|
|
|
|
|
2014-10-27 05:07:06 +00:00
|
|
|
if (!imm->must_promote && imm->uses_by_coissue < 4) {
|
2014-02-12 19:00:46 +00:00
|
|
|
table.imm[i] = table.imm[table.len - 1];
|
|
|
|
table.len--;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
i++;
|
|
|
|
}
|
|
|
|
if (table.len == 0) {
|
|
|
|
ralloc_free(const_ctx);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (cfg->num_blocks != 1)
|
|
|
|
qsort(table.imm, table.len, sizeof(struct imm), compare);
|
|
|
|
|
|
|
|
/* Insert MOVs to load the constant values into GRFs. */
|
2016-02-01 19:14:01 +00:00
|
|
|
fs_reg reg(VGRF, alloc.allocate(1));
|
2014-02-12 19:00:46 +00:00
|
|
|
reg.stride = 0;
|
|
|
|
for (int i = 0; i < table.len; i++) {
|
|
|
|
struct imm *imm = &table.imm[i];
|
2015-06-03 16:20:50 +01:00
|
|
|
/* Insert it either before the instruction that generated the immediate
|
|
|
|
* or after the last non-control flow instruction of the common ancestor.
|
|
|
|
*/
|
|
|
|
exec_node *n = (imm->inst ? imm->inst :
|
|
|
|
imm->block->last_non_control_flow_inst()->next);
|
|
|
|
|
2019-01-29 09:58:49 +00:00
|
|
|
/* From the BDW and CHV PRM, 3D Media GPGPU, Special Restrictions:
|
|
|
|
*
|
|
|
|
* "In Align16 mode, the channel selects and channel enables apply to a
|
|
|
|
* pair of half-floats, because these parameters are defined for DWord
|
|
|
|
* elements ONLY. This is applicable when both source and destination
|
|
|
|
* are half-floats."
|
|
|
|
*
|
|
|
|
* This means that Align16 instructions that use promoted HF immediates
|
|
|
|
* and use a <0,1,0>:HF region would read 2 HF slots instead of
|
|
|
|
* replicating the single one we want. To avoid this, we always populate
|
|
|
|
* both HF slots within a DWord with the constant.
|
|
|
|
*/
|
2021-03-29 22:41:58 +01:00
|
|
|
const uint32_t width = devinfo->ver == 8 && imm->is_half_float ? 2 : 1;
|
2019-01-29 09:58:49 +00:00
|
|
|
const fs_builder ibld = bld.at(imm->block, n).exec_all().group(width, 0);
|
|
|
|
|
|
|
|
/* Put the immediate in an offset aligned to its size. Some instructions
|
|
|
|
* seem to have additional alignment requirements, so account for that
|
|
|
|
* too.
|
|
|
|
*/
|
|
|
|
reg.offset = ALIGN(reg.offset, get_alignment_for_imm(imm));
|
2014-02-12 19:00:46 +00:00
|
|
|
|
2019-01-29 09:58:49 +00:00
|
|
|
/* Ensure we have enough space in the register to copy the immediate */
|
|
|
|
struct brw_reg imm_reg = build_imm_reg_for_copy(imm);
|
|
|
|
if (reg.offset + type_sz(imm_reg.type) * width > REG_SIZE) {
|
2016-02-01 19:14:01 +00:00
|
|
|
reg.nr = alloc.allocate(1);
|
2016-09-01 23:11:21 +01:00
|
|
|
reg.offset = 0;
|
2014-02-12 19:00:46 +00:00
|
|
|
}
|
2019-01-29 09:58:49 +00:00
|
|
|
|
|
|
|
ibld.MOV(retype(reg, imm_reg.type), imm_reg);
|
|
|
|
imm->nr = reg.nr;
|
|
|
|
imm->subreg_offset = reg.offset;
|
|
|
|
|
|
|
|
reg.offset += imm->size * width;
|
2014-02-12 19:00:46 +00:00
|
|
|
}
|
2016-10-17 22:10:26 +01:00
|
|
|
shader_stats.promoted_constants = table.len;
|
2014-02-12 19:00:46 +00:00
|
|
|
|
|
|
|
/* Rewrite the immediate sources to refer to the new GRFs. */
|
|
|
|
for (int i = 0; i < table.len; i++) {
|
|
|
|
foreach_list_typed(reg_link, link, link, table.imm[i].uses) {
|
|
|
|
fs_reg *reg = link->reg;
|
2019-01-29 09:58:49 +00:00
|
|
|
#ifdef DEBUG
|
|
|
|
switch (reg->type) {
|
|
|
|
case BRW_REGISTER_TYPE_DF:
|
|
|
|
assert((isnan(reg->df) && isnan(table.imm[i].df)) ||
|
|
|
|
(fabs(reg->df) == fabs(table.imm[i].df)));
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_F:
|
|
|
|
assert((isnan(reg->f) && isnan(table.imm[i].f)) ||
|
|
|
|
(fabsf(reg->f) == fabsf(table.imm[i].f)));
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_HF:
|
|
|
|
assert((isnan(_mesa_half_to_float(reg->d & 0xffffu)) &&
|
|
|
|
isnan(_mesa_half_to_float(table.imm[i].w))) ||
|
|
|
|
(fabsf(_mesa_half_to_float(reg->d & 0xffffu)) ==
|
|
|
|
fabsf(_mesa_half_to_float(table.imm[i].w))));
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_Q:
|
|
|
|
assert(abs(reg->d64) == abs(table.imm[i].d64));
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_UQ:
|
|
|
|
assert(reg->d64 == table.imm[i].d64);
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_D:
|
|
|
|
assert(abs(reg->d) == abs(table.imm[i].d));
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_UD:
|
|
|
|
assert(reg->d == table.imm[i].d);
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_W:
|
|
|
|
assert(abs((int16_t) (reg->d & 0xffff)) == table.imm[i].w);
|
|
|
|
break;
|
|
|
|
case BRW_REGISTER_TYPE_UW:
|
|
|
|
assert((reg->ud & 0xffffu) == (uint16_t) table.imm[i].w);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
#endif
|
2018-12-12 06:04:42 +00:00
|
|
|
|
2015-10-27 00:09:25 +00:00
|
|
|
reg->file = VGRF;
|
2016-09-01 23:11:21 +01:00
|
|
|
reg->offset = table.imm[i].subreg_offset;
|
2014-02-12 19:00:46 +00:00
|
|
|
reg->stride = 0;
|
2019-01-29 09:58:49 +00:00
|
|
|
reg->negate = needs_negate(reg, &table.imm[i]);
|
2018-12-12 06:04:42 +00:00
|
|
|
reg->nr = table.imm[i].nr;
|
2014-02-12 19:00:46 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-27 00:06:45 +00:00
|
|
|
if (debug) {
|
|
|
|
for (int i = 0; i < table.len; i++) {
|
|
|
|
struct imm *imm = &table.imm[i];
|
|
|
|
|
2019-01-29 09:58:49 +00:00
|
|
|
printf("0x%016" PRIx64 " - block %3d, reg %3d sub %2d, "
|
|
|
|
"Uses: (%2d, %2d), IP: %4d to %4d, length %4d\n",
|
|
|
|
(uint64_t)(imm->d & BITFIELD64_MASK(imm->size * 8)),
|
2015-02-27 00:06:45 +00:00
|
|
|
imm->block->num,
|
|
|
|
imm->nr,
|
|
|
|
imm->subreg_offset,
|
|
|
|
imm->must_promote,
|
|
|
|
imm->uses_by_coissue,
|
|
|
|
imm->first_use_ip,
|
|
|
|
imm->last_use_ip,
|
|
|
|
imm->last_use_ip - imm->first_use_ip);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-02-12 19:00:46 +00:00
|
|
|
ralloc_free(const_ctx);
|
2016-03-14 02:26:37 +00:00
|
|
|
invalidate_analysis(DEPENDENCY_INSTRUCTIONS | DEPENDENCY_VARIABLES);
|
2014-02-12 19:00:46 +00:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|