|
|
|
@ -0,0 +1,317 @@
|
|
|
|
|
/*
|
|
|
|
|
* Copyright © 2021 Google
|
|
|
|
|
*
|
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
|
*
|
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
|
* Software.
|
|
|
|
|
*
|
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include "radv_private.h"
|
|
|
|
|
#include "radv_meta.h"
|
|
|
|
|
|
|
|
|
|
static nir_shader *
|
|
|
|
|
build_dcc_retile_compute_shader(struct radv_device *dev)
|
|
|
|
|
{
|
|
|
|
|
const struct glsl_type *buf_type = glsl_image_type(GLSL_SAMPLER_DIM_BUF,
|
|
|
|
|
false,
|
|
|
|
|
GLSL_TYPE_UINT);
|
|
|
|
|
nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, NULL, "dcc_retile_compute");
|
|
|
|
|
|
|
|
|
|
b.shader->info.cs.local_size[0] = 256;
|
|
|
|
|
b.shader->info.cs.local_size[1] = 1;
|
|
|
|
|
b.shader->info.cs.local_size[2] = 1;
|
|
|
|
|
|
|
|
|
|
nir_variable *indices = nir_variable_create(b.shader, nir_var_uniform,
|
|
|
|
|
buf_type, "indices_in");
|
|
|
|
|
indices->data.descriptor_set = 0;
|
|
|
|
|
indices->data.binding = 0;
|
|
|
|
|
nir_variable *input_dcc = nir_variable_create(b.shader, nir_var_uniform,
|
|
|
|
|
buf_type, "dcc_in");
|
|
|
|
|
input_dcc->data.descriptor_set = 0;
|
|
|
|
|
input_dcc->data.binding = 1;
|
|
|
|
|
nir_variable *output_dcc = nir_variable_create(b.shader, nir_var_uniform,
|
|
|
|
|
buf_type, "dcc_out");
|
|
|
|
|
output_dcc->data.descriptor_set = 0;
|
|
|
|
|
output_dcc->data.binding = 2;
|
|
|
|
|
|
|
|
|
|
nir_ssa_def *indices_ref = &nir_build_deref_var(&b, indices)->dest.ssa;
|
|
|
|
|
nir_ssa_def *input_dcc_ref = &nir_build_deref_var(&b, input_dcc)->dest.ssa;
|
|
|
|
|
nir_ssa_def *output_dcc_ref = &nir_build_deref_var(&b, output_dcc)->dest.ssa;
|
|
|
|
|
|
|
|
|
|
nir_ssa_def *invoc_id = nir_load_local_invocation_id(&b);
|
|
|
|
|
nir_ssa_def *wg_id = nir_load_work_group_id(&b, 32);
|
|
|
|
|
nir_ssa_def *block_size = nir_imm_ivec4(&b,
|
|
|
|
|
b.shader->info.cs.local_size[0],
|
|
|
|
|
0, 0, 0);
|
|
|
|
|
|
|
|
|
|
nir_ssa_def *global_id = nir_iadd(&b, nir_imul(&b, wg_id, block_size), invoc_id);
|
|
|
|
|
|
|
|
|
|
nir_intrinsic_instr *index_vals = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_load);
|
|
|
|
|
index_vals->num_components = 2;
|
|
|
|
|
index_vals->src[0] = nir_src_for_ssa(indices_ref);
|
|
|
|
|
index_vals->src[1] = nir_src_for_ssa(global_id);
|
|
|
|
|
index_vals->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
|
|
|
|
|
index_vals->src[3] = nir_src_for_ssa(nir_imm_int(&b, 0));
|
|
|
|
|
nir_ssa_dest_init(&index_vals->instr, &index_vals->dest, 2, 32, "indices");
|
|
|
|
|
nir_builder_instr_insert(&b, &index_vals->instr);
|
|
|
|
|
|
|
|
|
|
nir_ssa_def *src = nir_channels(&b, &index_vals->dest.ssa, 1);
|
|
|
|
|
nir_ssa_def *dst = nir_channels(&b, &index_vals->dest.ssa, 2);
|
|
|
|
|
|
|
|
|
|
nir_intrinsic_instr *dcc_val = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_load);
|
|
|
|
|
dcc_val->num_components = 1;
|
|
|
|
|
dcc_val->src[0] = nir_src_for_ssa(input_dcc_ref);
|
|
|
|
|
dcc_val->src[1] = nir_src_for_ssa(nir_vec4(&b, src, src, src, src));
|
|
|
|
|
dcc_val->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
|
|
|
|
|
dcc_val->src[3] = nir_src_for_ssa(nir_imm_int(&b, 0));
|
|
|
|
|
nir_ssa_dest_init(&dcc_val->instr, &dcc_val->dest, 1, 32, "dcc_val");
|
|
|
|
|
nir_builder_instr_insert(&b, &dcc_val->instr);
|
|
|
|
|
|
|
|
|
|
nir_intrinsic_instr *store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_image_deref_store);
|
|
|
|
|
store->num_components = 1;
|
|
|
|
|
store->src[0] = nir_src_for_ssa(output_dcc_ref);
|
|
|
|
|
store->src[1] = nir_src_for_ssa(nir_vec4(&b, dst, dst, dst, dst));
|
|
|
|
|
store->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
|
|
|
|
|
store->src[3] = nir_src_for_ssa(&dcc_val->dest.ssa);
|
|
|
|
|
store->src[4] = nir_src_for_ssa(nir_imm_int(&b, 0));
|
|
|
|
|
|
|
|
|
|
nir_builder_instr_insert(&b, &store->instr);
|
|
|
|
|
return b.shader;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
radv_device_finish_meta_dcc_retile_state(struct radv_device *device)
|
|
|
|
|
{
|
|
|
|
|
struct radv_meta_state *state = &device->meta_state;
|
|
|
|
|
|
|
|
|
|
radv_DestroyPipeline(radv_device_to_handle(device),
|
|
|
|
|
state->dcc_retile.pipeline,
|
|
|
|
|
&state->alloc);
|
|
|
|
|
radv_DestroyPipelineLayout(radv_device_to_handle(device),
|
|
|
|
|
state->dcc_retile.p_layout,
|
|
|
|
|
&state->alloc);
|
|
|
|
|
radv_DestroyDescriptorSetLayout(radv_device_to_handle(device),
|
|
|
|
|
state->dcc_retile.ds_layout,
|
|
|
|
|
&state->alloc);
|
|
|
|
|
|
|
|
|
|
/* Reset for next finish. */
|
|
|
|
|
memset(&state->dcc_retile, 0, sizeof(state->dcc_retile));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
VkResult
|
|
|
|
|
radv_device_init_meta_dcc_retile_state(struct radv_device *device)
|
|
|
|
|
{
|
|
|
|
|
VkResult result = VK_SUCCESS;
|
|
|
|
|
struct radv_shader_module cs = { .nir = NULL };
|
|
|
|
|
|
|
|
|
|
cs.nir = build_dcc_retile_compute_shader(device);
|
|
|
|
|
|
|
|
|
|
VkDescriptorSetLayoutCreateInfo ds_create_info = {
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
|
|
|
|
|
.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR,
|
|
|
|
|
.bindingCount = 3,
|
|
|
|
|
.pBindings = (VkDescriptorSetLayoutBinding[]) {
|
|
|
|
|
{
|
|
|
|
|
.binding = 0,
|
|
|
|
|
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
|
|
|
|
|
.descriptorCount = 1,
|
|
|
|
|
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
|
|
|
|
|
.pImmutableSamplers = NULL
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
.binding = 1,
|
|
|
|
|
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
|
|
|
|
|
.descriptorCount = 1,
|
|
|
|
|
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
|
|
|
|
|
.pImmutableSamplers = NULL
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
.binding = 2,
|
|
|
|
|
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
|
|
|
|
|
.descriptorCount = 1,
|
|
|
|
|
.stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
|
|
|
|
|
.pImmutableSamplers = NULL
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
result = radv_CreateDescriptorSetLayout(radv_device_to_handle(device),
|
|
|
|
|
&ds_create_info,
|
|
|
|
|
&device->meta_state.alloc,
|
|
|
|
|
&device->meta_state.dcc_retile.ds_layout);
|
|
|
|
|
if (result != VK_SUCCESS)
|
|
|
|
|
goto cleanup;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
VkPipelineLayoutCreateInfo pl_create_info = {
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
|
|
|
|
|
.setLayoutCount = 1,
|
|
|
|
|
.pSetLayouts = &device->meta_state.dcc_retile.ds_layout,
|
|
|
|
|
.pushConstantRangeCount = 0,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
result = radv_CreatePipelineLayout(radv_device_to_handle(device),
|
|
|
|
|
&pl_create_info,
|
|
|
|
|
&device->meta_state.alloc,
|
|
|
|
|
&device->meta_state.dcc_retile.p_layout);
|
|
|
|
|
if (result != VK_SUCCESS)
|
|
|
|
|
goto cleanup;
|
|
|
|
|
|
|
|
|
|
/* compute shader */
|
|
|
|
|
|
|
|
|
|
VkPipelineShaderStageCreateInfo pipeline_shader_stage = {
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
|
|
|
|
|
.stage = VK_SHADER_STAGE_COMPUTE_BIT,
|
|
|
|
|
.module = radv_shader_module_to_handle(&cs),
|
|
|
|
|
.pName = "main",
|
|
|
|
|
.pSpecializationInfo = NULL,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
VkComputePipelineCreateInfo vk_pipeline_info = {
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,
|
|
|
|
|
.stage = pipeline_shader_stage,
|
|
|
|
|
.flags = 0,
|
|
|
|
|
.layout = device->meta_state.dcc_retile.p_layout,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
result = radv_CreateComputePipelines(radv_device_to_handle(device),
|
|
|
|
|
radv_pipeline_cache_to_handle(&device->meta_state.cache),
|
|
|
|
|
1, &vk_pipeline_info, NULL,
|
|
|
|
|
&device->meta_state.dcc_retile.pipeline);
|
|
|
|
|
if (result != VK_SUCCESS)
|
|
|
|
|
goto cleanup;
|
|
|
|
|
|
|
|
|
|
cleanup:
|
|
|
|
|
if (result != VK_SUCCESS)
|
|
|
|
|
radv_device_finish_meta_dcc_retile_state(device);
|
|
|
|
|
ralloc_free(cs.nir);
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
radv_retile_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image)
|
|
|
|
|
{
|
|
|
|
|
struct radv_meta_saved_state saved_state;
|
|
|
|
|
struct radv_device *device = cmd_buffer->device;
|
|
|
|
|
uint32_t retile_map_size = ac_surface_get_retile_map_size(&image->planes[0].surface);
|
|
|
|
|
|
|
|
|
|
assert(image->type == VK_IMAGE_TYPE_2D);
|
|
|
|
|
assert(image->info.array_size == 1 && image->info.levels == 1);
|
|
|
|
|
|
|
|
|
|
struct radv_cmd_state *state = &cmd_buffer->state;
|
|
|
|
|
|
|
|
|
|
state->flush_bits |= radv_dst_access_flush(cmd_buffer, VK_ACCESS_SHADER_READ_BIT, image) |
|
|
|
|
|
radv_dst_access_flush(cmd_buffer, VK_ACCESS_SHADER_WRITE_BIT, image);
|
|
|
|
|
|
|
|
|
|
/* Compile pipelines if not already done so. */
|
|
|
|
|
if (!cmd_buffer->device->meta_state.dcc_retile.pipeline) {
|
|
|
|
|
VkResult ret = radv_device_init_meta_dcc_retile_state(cmd_buffer->device);
|
|
|
|
|
if (ret != VK_SUCCESS) {
|
|
|
|
|
cmd_buffer->record_result = ret;
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_DESCRIPTORS |
|
|
|
|
|
RADV_META_SAVE_COMPUTE_PIPELINE);
|
|
|
|
|
|
|
|
|
|
radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer),
|
|
|
|
|
VK_PIPELINE_BIND_POINT_COMPUTE,
|
|
|
|
|
device->meta_state.dcc_retile.pipeline);
|
|
|
|
|
|
|
|
|
|
struct radv_buffer buffer = {
|
|
|
|
|
.size = image->size,
|
|
|
|
|
.bo = image->bo,
|
|
|
|
|
.offset = image->offset
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct radv_buffer retile_buffer = {
|
|
|
|
|
.size = retile_map_size,
|
|
|
|
|
.bo = image->retile_map,
|
|
|
|
|
.offset = 0
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct radv_buffer_view views[3];
|
|
|
|
|
VkBufferView view_handles[3];
|
|
|
|
|
radv_buffer_view_init(views + 0, cmd_buffer->device, &(VkBufferViewCreateInfo) {
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
|
|
|
|
|
.buffer = radv_buffer_to_handle(&retile_buffer),
|
|
|
|
|
.offset = 0,
|
|
|
|
|
.range = retile_map_size,
|
|
|
|
|
.format = image->planes[0].surface.u.gfx9.dcc_retile_use_uint16 ?
|
|
|
|
|
VK_FORMAT_R16G16_UINT : VK_FORMAT_R32G32_UINT,
|
|
|
|
|
});
|
|
|
|
|
radv_buffer_view_init(views + 1, cmd_buffer->device, &(VkBufferViewCreateInfo) {
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
|
|
|
|
|
.buffer = radv_buffer_to_handle(&buffer),
|
|
|
|
|
.offset = image->planes[0].surface.dcc_offset,
|
|
|
|
|
.range = image->planes[0].surface.dcc_size,
|
|
|
|
|
.format = VK_FORMAT_R8_UINT,
|
|
|
|
|
});
|
|
|
|
|
radv_buffer_view_init(views + 2, cmd_buffer->device, &(VkBufferViewCreateInfo) {
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO,
|
|
|
|
|
.buffer = radv_buffer_to_handle(&buffer),
|
|
|
|
|
.offset = image->planes[0].surface.display_dcc_offset,
|
|
|
|
|
.range = image->planes[0].surface.u.gfx9.display_dcc_size,
|
|
|
|
|
.format = VK_FORMAT_R8_UINT,
|
|
|
|
|
});
|
|
|
|
|
for (unsigned i = 0; i < 3; ++i)
|
|
|
|
|
view_handles[i] = radv_buffer_view_to_handle(&views[i]);
|
|
|
|
|
|
|
|
|
|
radv_meta_push_descriptor_set(cmd_buffer,
|
|
|
|
|
VK_PIPELINE_BIND_POINT_COMPUTE,
|
|
|
|
|
device->meta_state.dcc_retile.p_layout,
|
|
|
|
|
0, /* set */
|
|
|
|
|
3, /* descriptorWriteCount */
|
|
|
|
|
(VkWriteDescriptorSet[]) {
|
|
|
|
|
{
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
|
|
|
|
|
.dstBinding = 0,
|
|
|
|
|
.dstArrayElement = 0,
|
|
|
|
|
.descriptorCount = 1,
|
|
|
|
|
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
|
|
|
|
|
.pTexelBufferView = &view_handles[0],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
|
|
|
|
|
.dstBinding = 1,
|
|
|
|
|
.dstArrayElement = 0,
|
|
|
|
|
.descriptorCount = 1,
|
|
|
|
|
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
|
|
|
|
|
.pTexelBufferView = &view_handles[1],
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
|
|
|
|
|
.dstBinding = 2,
|
|
|
|
|
.dstArrayElement = 0,
|
|
|
|
|
.descriptorCount = 1,
|
|
|
|
|
.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER,
|
|
|
|
|
.pTexelBufferView = &view_handles[2],
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
/* src+dst pairs count double, so the number of DCC bytes we move is
|
|
|
|
|
* actually half of dcc_retile_num_elements. */
|
|
|
|
|
radv_unaligned_dispatch(cmd_buffer, image->planes[0].surface.u.gfx9.dcc_retile_num_elements / 2, 1, 1);
|
|
|
|
|
|
|
|
|
|
radv_meta_restore(&saved_state, cmd_buffer);
|
|
|
|
|
|
|
|
|
|
state->flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH |
|
|
|
|
|
radv_src_access_flush(cmd_buffer, VK_ACCESS_SHADER_WRITE_BIT, image);
|
|
|
|
|
}
|
|
|
|
|
|