389 lines
14 KiB
C
389 lines
14 KiB
C
/*
|
||
* Copyright © Microsoft Corporation
|
||
*
|
||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||
* copy of this software and associated documentation files (the "Software"),
|
||
* to deal in the Software without restriction, including without limitation
|
||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||
* and/or sell copies of the Software, and to permit persons to whom the
|
||
* Software is furnished to do so, subject to the following conditions:
|
||
*
|
||
* The above copyright notice and this permission notice (including the next
|
||
* paragraph) shall be included in all copies or substantial portions of the
|
||
* Software.
|
||
*
|
||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||
* IN THE SOFTWARE.
|
||
*/
|
||
|
||
#include "u_math.h"
|
||
#include "nir.h"
|
||
#include "glsl_types.h"
|
||
#include "nir_types.h"
|
||
#include "nir_builder.h"
|
||
|
||
#include "clc_nir.h"
|
||
#include "clc_compiler.h"
|
||
#include "../compiler/dxil_nir.h"
|
||
|
||
static bool
|
||
lower_load_base_global_invocation_id(nir_builder *b, nir_intrinsic_instr *intr,
|
||
nir_variable *var)
|
||
{
|
||
b->cursor = nir_after_instr(&intr->instr);
|
||
|
||
nir_ssa_def *offset =
|
||
build_load_ubo_dxil(b, nir_imm_int(b, var->data.binding),
|
||
nir_imm_int(b,
|
||
offsetof(struct clc_work_properties_data,
|
||
global_offset_x)),
|
||
nir_dest_num_components(intr->dest),
|
||
nir_dest_bit_size(intr->dest));
|
||
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(offset));
|
||
nir_instr_remove(&intr->instr);
|
||
return true;
|
||
}
|
||
|
||
static bool
|
||
lower_load_work_dim(nir_builder *b, nir_intrinsic_instr *intr,
|
||
nir_variable *var)
|
||
{
|
||
b->cursor = nir_after_instr(&intr->instr);
|
||
|
||
nir_ssa_def *dim =
|
||
build_load_ubo_dxil(b, nir_imm_int(b, var->data.binding),
|
||
nir_imm_int(b,
|
||
offsetof(struct clc_work_properties_data,
|
||
work_dim)),
|
||
nir_dest_num_components(intr->dest),
|
||
nir_dest_bit_size(intr->dest));
|
||
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(dim));
|
||
nir_instr_remove(&intr->instr);
|
||
return true;
|
||
}
|
||
|
||
static bool
|
||
lower_load_local_group_size(nir_builder *b, nir_intrinsic_instr *intr)
|
||
{
|
||
b->cursor = nir_after_instr(&intr->instr);
|
||
|
||
nir_const_value v[3] = {
|
||
nir_const_value_for_int(b->shader->info.cs.local_size[0], 32),
|
||
nir_const_value_for_int(b->shader->info.cs.local_size[1], 32),
|
||
nir_const_value_for_int(b->shader->info.cs.local_size[2], 32)
|
||
};
|
||
nir_ssa_def *size = nir_build_imm(b, 3, 32, v);
|
||
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(size));
|
||
nir_instr_remove(&intr->instr);
|
||
return true;
|
||
}
|
||
|
||
static bool
|
||
lower_load_num_work_groups(nir_builder *b, nir_intrinsic_instr *intr,
|
||
nir_variable *var)
|
||
{
|
||
b->cursor = nir_after_instr(&intr->instr);
|
||
|
||
nir_ssa_def *count =
|
||
build_load_ubo_dxil(b, nir_imm_int(b, var->data.binding),
|
||
nir_imm_int(b,
|
||
offsetof(struct clc_work_properties_data,
|
||
group_count_total_x)),
|
||
nir_dest_num_components(intr->dest),
|
||
nir_dest_bit_size(intr->dest));
|
||
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(count));
|
||
nir_instr_remove(&intr->instr);
|
||
return true;
|
||
}
|
||
|
||
static bool
|
||
lower_load_base_work_group_id(nir_builder *b, nir_intrinsic_instr *intr,
|
||
nir_variable *var)
|
||
{
|
||
b->cursor = nir_after_instr(&intr->instr);
|
||
|
||
nir_ssa_def *offset =
|
||
build_load_ubo_dxil(b, nir_imm_int(b, var->data.binding),
|
||
nir_imm_int(b,
|
||
offsetof(struct clc_work_properties_data,
|
||
group_id_offset_x)),
|
||
nir_dest_num_components(intr->dest),
|
||
nir_dest_bit_size(intr->dest));
|
||
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(offset));
|
||
nir_instr_remove(&intr->instr);
|
||
return true;
|
||
}
|
||
|
||
bool
|
||
clc_nir_lower_system_values(nir_shader *nir, nir_variable *var)
|
||
{
|
||
bool progress = false;
|
||
|
||
foreach_list_typed(nir_function, func, node, &nir->functions) {
|
||
if (!func->is_entrypoint)
|
||
continue;
|
||
assert(func->impl);
|
||
|
||
nir_builder b;
|
||
nir_builder_init(&b, func->impl);
|
||
|
||
nir_foreach_block(block, func->impl) {
|
||
nir_foreach_instr_safe(instr, block) {
|
||
if (instr->type != nir_instr_type_intrinsic)
|
||
continue;
|
||
|
||
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
|
||
|
||
switch (intr->intrinsic) {
|
||
case nir_intrinsic_load_base_global_invocation_id:
|
||
progress |= lower_load_base_global_invocation_id(&b, intr, var);
|
||
break;
|
||
case nir_intrinsic_load_work_dim:
|
||
progress |= lower_load_work_dim(&b, intr, var);
|
||
break;
|
||
case nir_intrinsic_load_local_group_size:
|
||
lower_load_local_group_size(&b, intr);
|
||
break;
|
||
case nir_intrinsic_load_num_work_groups:
|
||
lower_load_num_work_groups(&b, intr, var);
|
||
break;
|
||
case nir_intrinsic_load_base_work_group_id:
|
||
lower_load_base_work_group_id(&b, intr, var);
|
||
break;
|
||
default: break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return progress;
|
||
}
|
||
|
||
static bool
|
||
lower_load_kernel_input(nir_builder *b, nir_intrinsic_instr *intr,
|
||
nir_variable *var)
|
||
{
|
||
nir_intrinsic_instr *load;
|
||
|
||
b->cursor = nir_before_instr(&intr->instr);
|
||
|
||
unsigned bit_size = nir_dest_bit_size(intr->dest);
|
||
enum glsl_base_type base_type;
|
||
|
||
switch (bit_size) {
|
||
case 64:
|
||
base_type = GLSL_TYPE_UINT64;
|
||
break;
|
||
case 32:
|
||
base_type = GLSL_TYPE_UINT;
|
||
break;
|
||
case 16:
|
||
base_type = GLSL_TYPE_UINT16;
|
||
break;
|
||
case 8:
|
||
base_type = GLSL_TYPE_UINT8;
|
||
break;
|
||
}
|
||
|
||
const struct glsl_type *type =
|
||
glsl_vector_type(base_type, nir_dest_num_components(intr->dest));
|
||
nir_ssa_def *ptr = nir_vec2(b, nir_imm_int(b, var->data.binding),
|
||
nir_u2u(b, intr->src[0].ssa, 32));
|
||
nir_deref_instr *deref = nir_build_deref_cast(b, ptr, nir_var_mem_ubo, type,
|
||
bit_size / 8);
|
||
deref->cast.align_mul = nir_intrinsic_align_mul(intr);
|
||
deref->cast.align_offset = nir_intrinsic_align_offset(intr);
|
||
|
||
nir_ssa_def *result =
|
||
nir_load_deref(b, deref);
|
||
nir_ssa_def_rewrite_uses(&intr->dest.ssa, nir_src_for_ssa(result));
|
||
nir_instr_remove(&intr->instr);
|
||
return true;
|
||
}
|
||
|
||
bool
|
||
clc_nir_lower_kernel_input_loads(nir_shader *nir, nir_variable *var)
|
||
{
|
||
bool progress = false;
|
||
|
||
foreach_list_typed(nir_function, func, node, &nir->functions) {
|
||
if (!func->is_entrypoint)
|
||
continue;
|
||
assert(func->impl);
|
||
|
||
nir_builder b;
|
||
nir_builder_init(&b, func->impl);
|
||
|
||
nir_foreach_block(block, func->impl) {
|
||
nir_foreach_instr_safe(instr, block) {
|
||
if (instr->type != nir_instr_type_intrinsic)
|
||
continue;
|
||
|
||
nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
|
||
|
||
if (intr->intrinsic == nir_intrinsic_load_kernel_input)
|
||
progress |= lower_load_kernel_input(&b, intr, var);
|
||
}
|
||
}
|
||
}
|
||
|
||
return progress;
|
||
}
|
||
|
||
|
||
static nir_variable *
|
||
add_printf_var(struct nir_shader *nir, unsigned uav_id)
|
||
{
|
||
/* This size is arbitrary. Minimum required per spec is 1MB */
|
||
const unsigned max_printf_size = 1 * 1024 * 1024;
|
||
const unsigned printf_array_size = max_printf_size / sizeof(unsigned);
|
||
nir_variable *var =
|
||
nir_variable_create(nir, nir_var_mem_ssbo,
|
||
glsl_array_type(glsl_uint_type(), printf_array_size, sizeof(unsigned)),
|
||
"kernel_work_properies");
|
||
var->data.binding = uav_id;
|
||
return var;
|
||
}
|
||
|
||
static void
|
||
lower_printf_impl(nir_builder *b, nir_intrinsic_instr *instr, nir_variable *var)
|
||
{
|
||
/* Atomic add a buffer size counter to determine where to write.
|
||
* If overflowed, return -1, otherwise, store the arguments and return 0.
|
||
*/
|
||
b->cursor = nir_before_instr(&instr->instr);
|
||
nir_deref_instr *ssbo_deref = nir_build_deref_var(b, var);
|
||
nir_deref_instr *counter_deref = nir_build_deref_array_imm(b, ssbo_deref, 0);
|
||
nir_deref_instr *struct_deref = nir_instr_as_deref(instr->src[1].ssa->parent_instr);
|
||
nir_variable *struct_var = nir_deref_instr_get_variable(struct_deref);
|
||
const struct glsl_type *struct_type = struct_var->type;
|
||
/* Align the struct size to 4 for natural SSBO alignment */
|
||
int struct_size = align(glsl_get_cl_size(struct_type), 4);
|
||
|
||
/* Hardcoding 64bit pointers to simplify some code below */
|
||
assert(instr->src[0].ssa->num_components == 1 && instr->src[0].ssa->bit_size == 64);
|
||
|
||
nir_intrinsic_instr *atomic = nir_intrinsic_instr_create(b->shader, nir_intrinsic_deref_atomic_add);
|
||
nir_ssa_dest_init(&atomic->instr, &atomic->dest, 1, 32, NULL);
|
||
atomic->src[0] = nir_src_for_ssa(&counter_deref->dest.ssa);
|
||
atomic->src[1] = nir_src_for_ssa(nir_imm_int(b, struct_size + sizeof(uint64_t)));
|
||
nir_builder_instr_insert(b, &atomic->instr);
|
||
|
||
int max_valid_offset =
|
||
glsl_get_cl_size(var->type) - /* buffer size */
|
||
struct_size - /* printf args size */
|
||
sizeof(uint64_t) - /* format string */
|
||
sizeof(int); /* the first int in the buffer is for the counter */
|
||
nir_push_if(b, nir_ilt(b, &atomic->dest.ssa, nir_imm_int(b, max_valid_offset)));
|
||
nir_ssa_def *printf_succ_val = nir_imm_int(b, 0);
|
||
|
||
nir_ssa_def *start_offset = nir_u2u64(b, nir_iadd(b, &atomic->dest.ssa, nir_imm_int(b, sizeof(int))));
|
||
nir_deref_instr *as_byte_array = nir_build_deref_cast(b, &ssbo_deref->dest.ssa, nir_var_mem_ssbo, glsl_uint8_t_type(), 1);
|
||
nir_deref_instr *as_offset_byte_array = nir_build_deref_ptr_as_array(b, as_byte_array, start_offset);
|
||
nir_deref_instr *format_string_write_deref =
|
||
nir_build_deref_cast(b, &as_offset_byte_array->dest.ssa, nir_var_mem_ssbo, glsl_uint64_t_type(), 8);
|
||
nir_store_deref(b, format_string_write_deref, instr->src[0].ssa, ~0);
|
||
|
||
for (unsigned i = 0; i < glsl_get_length(struct_type); ++i) {
|
||
nir_ssa_def *field_offset_from_start = nir_imm_int64(b, glsl_get_struct_field_offset(struct_type, i) + sizeof(uint64_t));
|
||
nir_ssa_def *field_offset = nir_iadd(b, start_offset, field_offset_from_start);
|
||
|
||
const struct glsl_type *field_type = glsl_get_struct_field(struct_type, i);
|
||
nir_deref_instr *field_read_deref = nir_build_deref_struct(b, struct_deref, i);
|
||
nir_ssa_def *field_value = nir_load_deref(b, field_read_deref);
|
||
|
||
/* Clang does promotion of arguments to their "native" size. That means that any floats
|
||
* have been converted to doubles for the call to printf. Since we don't support doubles,
|
||
* convert them back here; copy-prop and other optimizations should remove all hint of doubles.
|
||
*/
|
||
if (glsl_get_base_type(field_type) == GLSL_TYPE_DOUBLE) {
|
||
field_value = nir_f2f32(b, field_value);
|
||
field_type = glsl_float_type();
|
||
}
|
||
|
||
as_offset_byte_array = nir_build_deref_ptr_as_array(b, as_byte_array, field_offset);
|
||
nir_deref_instr *field_write_deref =
|
||
nir_build_deref_cast(b, &as_offset_byte_array->dest.ssa, nir_var_mem_ssbo, field_type, glsl_get_cl_size(field_type));
|
||
|
||
nir_store_deref(b, field_write_deref, field_value, ~0);
|
||
}
|
||
|
||
nir_push_else(b, NULL);
|
||
nir_ssa_def *printf_fail_val = nir_imm_int(b, -1);
|
||
nir_pop_if(b, NULL);
|
||
|
||
nir_ssa_def *return_value = nir_if_phi(b, printf_succ_val, printf_fail_val);
|
||
nir_ssa_def_rewrite_uses(&instr->dest.ssa, nir_src_for_ssa(return_value));
|
||
nir_instr_remove(&instr->instr);
|
||
}
|
||
|
||
static nir_variable *
|
||
find_identical_const_sampler(nir_shader *nir, nir_variable *sampler)
|
||
{
|
||
nir_foreach_variable_with_modes(uniform, nir, nir_var_uniform) {
|
||
if (!glsl_type_is_sampler(uniform->type) || !uniform->data.sampler.is_inline_sampler)
|
||
continue;
|
||
if (uniform->data.sampler.addressing_mode == sampler->data.sampler.addressing_mode &&
|
||
uniform->data.sampler.normalized_coordinates == sampler->data.sampler.normalized_coordinates &&
|
||
uniform->data.sampler.filter_mode == sampler->data.sampler.filter_mode)
|
||
return uniform;
|
||
}
|
||
unreachable("Should have at least found the input sampler");
|
||
}
|
||
|
||
bool
|
||
clc_nir_dedupe_const_samplers(nir_shader *nir)
|
||
{
|
||
bool progress = false;
|
||
nir_foreach_function(func, nir) {
|
||
if (!func->impl)
|
||
continue;
|
||
|
||
nir_builder b;
|
||
nir_builder_init(&b, func->impl);
|
||
|
||
nir_foreach_block(block, func->impl) {
|
||
nir_foreach_instr_safe(instr, block) {
|
||
if (instr->type != nir_instr_type_tex)
|
||
continue;
|
||
|
||
nir_tex_instr *tex = nir_instr_as_tex(instr);
|
||
int sampler_idx = nir_tex_instr_src_index(tex, nir_tex_src_sampler_deref);
|
||
if (sampler_idx == -1)
|
||
continue;
|
||
|
||
nir_deref_instr *deref = nir_src_as_deref(tex->src[sampler_idx].src);
|
||
nir_variable *sampler = nir_deref_instr_get_variable(deref);
|
||
if (!sampler)
|
||
continue;
|
||
|
||
assert(sampler->data.mode == nir_var_uniform);
|
||
|
||
if (!sampler->data.sampler.is_inline_sampler)
|
||
continue;
|
||
|
||
nir_variable *replacement = find_identical_const_sampler(nir, sampler);
|
||
if (replacement == sampler)
|
||
continue;
|
||
|
||
b.cursor = nir_before_instr(&tex->instr);
|
||
nir_deref_instr *replacement_deref = nir_build_deref_var(&b, replacement);
|
||
nir_instr_rewrite_src(&tex->instr, &tex->src[sampler_idx].src,
|
||
nir_src_for_ssa(&replacement_deref->dest.ssa));
|
||
nir_deref_instr_remove_if_unused(deref);
|
||
progress = true;
|
||
}
|
||
}
|
||
|
||
if (progress) {
|
||
nir_metadata_preserve(func->impl, nir_metadata_block_index | nir_metadata_dominance);
|
||
}
|
||
}
|
||
return progress;
|
||
}
|