diff --git a/src/compiler/Makefile.sources b/src/compiler/Makefile.sources index f5975896c66..aaea6a08d6b 100644 --- a/src/compiler/Makefile.sources +++ b/src/compiler/Makefile.sources @@ -329,6 +329,7 @@ NIR_FILES = \ nir/nir_opt_loop_unroll.c \ nir/nir_opt_large_constants.c \ nir/nir_opt_load_store_vectorize.c \ + nir/nir_opt_memcpy.c \ nir/nir_opt_move.c \ nir/nir_opt_peephole_select.c \ nir/nir_opt_rematerialize_compares.c \ diff --git a/src/compiler/nir/meson.build b/src/compiler/nir/meson.build index f6030b093f9..eff000ee066 100644 --- a/src/compiler/nir/meson.build +++ b/src/compiler/nir/meson.build @@ -210,6 +210,7 @@ files_libnir = files( 'nir_opt_large_constants.c', 'nir_opt_load_store_vectorize.c', 'nir_opt_loop_unroll.c', + 'nir_opt_memcpy.c', 'nir_opt_move.c', 'nir_opt_peephole_select.c', 'nir_opt_rematerialize_compares.c', diff --git a/src/compiler/nir/nir.h b/src/compiler/nir/nir.h index 0402c1c87d0..99421534880 100644 --- a/src/compiler/nir/nir.h +++ b/src/compiler/nir/nir.h @@ -4126,6 +4126,7 @@ void nir_lower_deref_copy_instr(struct nir_builder *b, nir_intrinsic_instr *copy); bool nir_lower_var_copies(nir_shader *shader); +bool nir_opt_memcpy(nir_shader *shader); bool nir_lower_memcpy(nir_shader *shader); void nir_fixup_deref_modes(nir_shader *shader); diff --git a/src/compiler/nir/nir_opt_memcpy.c b/src/compiler/nir/nir_opt_memcpy.c new file mode 100644 index 00000000000..fe2a3039be1 --- /dev/null +++ b/src/compiler/nir/nir_opt_memcpy.c @@ -0,0 +1,224 @@ +/* + * Copyright © 2020 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "nir_builder.h" + +static bool +opt_memcpy_deref_cast(nir_intrinsic_instr *cpy, nir_src *deref_src) +{ + assert(cpy->intrinsic == nir_intrinsic_memcpy_deref); + + nir_deref_instr *cast = nir_src_as_deref(*deref_src); + if (cast == NULL || cast->deref_type != nir_deref_type_cast) + return false; + + /* We always have to replace the source with a deref, not a bare uint + * pointer. If it's the first deref in the chain, bail. + */ + nir_deref_instr *parent = nir_src_as_deref(cast->parent); + if (parent == NULL) + return false; + + /* If it has useful alignment information, we want to keep that */ + if (cast->cast.align_mul > 0) + return false; + + /* Casts to uint8 or int8 never do us any good; get rid of them */ + if (cast->type == glsl_int8_t_type() || + cast->type == glsl_uint8_t_type()) { + nir_instr_rewrite_src(&cpy->instr, deref_src, + nir_src_for_ssa(&parent->dest.ssa)); + return true; + } + + int64_t parent_type_size = glsl_get_explicit_size(parent->type, false); + if (parent_type_size < 0) + return false; + + if (!nir_src_is_const(cpy->src[2])) + return false; + + /* We don't want to get rid of the cast if the resulting type would be + * smaller than the amount of data we're copying. + */ + if (nir_src_as_uint(cpy->src[2]) < (uint64_t)parent_type_size) + return false; + + nir_instr_rewrite_src(&cpy->instr, deref_src, + nir_src_for_ssa(&parent->dest.ssa)); + return true; +} + +static bool +type_is_tightly_packed(const struct glsl_type *type, unsigned *size_out) +{ + unsigned size = 0; + if (glsl_type_is_struct_or_ifc(type)) { + unsigned num_fields = glsl_get_length(type); + for (unsigned i = 0; i < num_fields; i++) { + const struct glsl_struct_field *field = + glsl_get_struct_field_data(type, i); + + if (field->offset < 0 || field->offset != size) + return false; + + unsigned field_size; + if (!type_is_tightly_packed(field->type, &field_size)) + return false; + + size = field->offset + field_size; + } + } else if (glsl_type_is_array_or_matrix(type)) { + if (glsl_type_is_unsized_array(type)) + return false; + + unsigned stride = glsl_get_explicit_stride(type); + if (stride == 0) + return false; + + const struct glsl_type *elem_type = glsl_get_array_element(type); + + unsigned elem_size; + if (!type_is_tightly_packed(elem_type, &elem_size)) + return false; + + if (elem_size != stride) + return false; + + size = stride * glsl_get_length(type); + } else { + assert(glsl_type_is_vector_or_scalar(type)); + if (glsl_get_explicit_stride(type) > 0) + return false; + + if (glsl_type_is_boolean(type)) + return false; + + size = glsl_get_explicit_size(type, false); + } + + if (size_out) + *size_out = size; + return true; +} + +static bool +try_lower_memcpy(nir_builder *b, nir_intrinsic_instr *cpy) +{ + nir_deref_instr *dst = nir_src_as_deref(cpy->src[0]); + nir_deref_instr *src = nir_src_as_deref(cpy->src[1]); + + /* A self-copy can always be eliminated */ + if (dst == src) { + nir_instr_remove(&cpy->instr); + return true; + } + + if (!nir_src_is_const(cpy->src[2])) + return false; + + uint64_t size = nir_src_as_uint(cpy->src[2]); + if (size == 0) { + nir_instr_remove(&cpy->instr); + return true; + } + + if (glsl_type_is_vector_or_scalar(src->type) && + glsl_type_is_vector_or_scalar(dst->type) && + glsl_get_explicit_size(dst->type, false) == size && + glsl_get_explicit_size(src->type, false) == size) { + b->cursor = nir_instr_remove(&cpy->instr); + nir_ssa_def *data = + nir_load_deref_with_access(b, src, nir_intrinsic_src_access(cpy)); + data = nir_bitcast_vector(b, data, glsl_get_bit_size(dst->type)); + assert(data->num_components == glsl_get_vector_elements(dst->type)); + nir_store_deref_with_access(b, src, data, ~0 /* write mask */, + nir_intrinsic_dst_access(cpy)); + return true; + } + + unsigned type_size; + if (dst->type == src->type && + type_is_tightly_packed(dst->type, &type_size) && + type_size == size) { + b->cursor = nir_instr_remove(&cpy->instr); + nir_copy_deref_with_access(b, dst, src, + nir_intrinsic_dst_access(cpy), + nir_intrinsic_src_access(cpy)); + return true; + } + + return false; +} + +static bool +opt_memcpy_impl(nir_function_impl *impl) +{ + bool progress = false; + + nir_builder b; + nir_builder_init(&b, impl); + + nir_foreach_block(block, impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + + nir_intrinsic_instr *cpy = nir_instr_as_intrinsic(instr); + if (cpy->intrinsic != nir_intrinsic_memcpy_deref) + continue; + + while (opt_memcpy_deref_cast(cpy, &cpy->src[0])) + progress = true; + while (opt_memcpy_deref_cast(cpy, &cpy->src[1])) + progress = true; + + if (try_lower_memcpy(&b, cpy)) { + progress = true; + continue; + } + } + } + + if (progress) { + nir_metadata_preserve(impl, nir_metadata_block_index | + nir_metadata_dominance); + } else { + nir_metadata_preserve(impl, nir_metadata_all); + } + + return progress; +} + +bool +nir_opt_memcpy(nir_shader *shader) +{ + bool progress = false; + + nir_foreach_function(function, shader) { + if (function->impl && opt_memcpy_impl(function->impl)) + progress = true; + } + + return progress; +}