197 lines
8.3 KiB
C
197 lines
8.3 KiB
C
/*
|
|
* Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
|
|
* Copyright 2015-2021 Advanced Micro Devices, Inc.
|
|
* All Rights Reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* on the rights to use, copy, modify, merge, publish, distribute, sub
|
|
* license, and/or sell copies of the Software, and to permit persons to whom
|
|
* the Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including the next
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
* Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
|
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
*/
|
|
#include "util/u_memory.h"
|
|
#include "radv_cs.h"
|
|
#include "radv_private.h"
|
|
#include "sid.h"
|
|
|
|
static bool
|
|
radv_translate_format_to_hw(struct radeon_info *info, VkFormat format, unsigned *hw_fmt,
|
|
unsigned *hw_type)
|
|
{
|
|
const struct util_format_description *desc = vk_format_description(format);
|
|
*hw_fmt = radv_translate_colorformat(format);
|
|
|
|
int firstchan;
|
|
for (firstchan = 0; firstchan < 4; firstchan++) {
|
|
if (desc->channel[firstchan].type != UTIL_FORMAT_TYPE_VOID) {
|
|
break;
|
|
}
|
|
}
|
|
if (firstchan == 4 || desc->channel[firstchan].type == UTIL_FORMAT_TYPE_FLOAT) {
|
|
*hw_type = V_028C70_NUMBER_FLOAT;
|
|
} else {
|
|
*hw_type = V_028C70_NUMBER_UNORM;
|
|
if (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB)
|
|
*hw_type = V_028C70_NUMBER_SRGB;
|
|
else if (desc->channel[firstchan].type == UTIL_FORMAT_TYPE_SIGNED) {
|
|
if (desc->channel[firstchan].pure_integer) {
|
|
*hw_type = V_028C70_NUMBER_SINT;
|
|
} else {
|
|
assert(desc->channel[firstchan].normalized);
|
|
*hw_type = V_028C70_NUMBER_SNORM;
|
|
}
|
|
} else if (desc->channel[firstchan].type == UTIL_FORMAT_TYPE_UNSIGNED) {
|
|
if (desc->channel[firstchan].pure_integer) {
|
|
*hw_type = V_028C70_NUMBER_UINT;
|
|
} else {
|
|
assert(desc->channel[firstchan].normalized);
|
|
*hw_type = V_028C70_NUMBER_UNORM;
|
|
}
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
static bool
|
|
radv_sdma_v4_v5_copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
|
|
struct radv_buffer *buffer,
|
|
const VkBufferImageCopy2 *region)
|
|
{
|
|
assert(image->plane_count == 1);
|
|
struct radv_device *device = cmd_buffer->device;
|
|
unsigned bpp = image->planes[0].surface.bpe;
|
|
uint64_t dst_address = buffer->bo->va;
|
|
uint64_t src_address = image->bindings[0].bo->va + image->planes[0].surface.u.gfx9.surf_offset;
|
|
unsigned src_pitch = image->planes[0].surface.u.gfx9.surf_pitch;
|
|
unsigned copy_width = DIV_ROUND_UP(image->info.width, image->planes[0].surface.blk_w);
|
|
unsigned copy_height = DIV_ROUND_UP(image->info.height, image->planes[0].surface.blk_h);
|
|
bool tmz = false;
|
|
|
|
uint32_t ib_pad_dw_mask = cmd_buffer->device->physical_device->rad_info.ib_pad_dw_mask[AMD_IP_SDMA];
|
|
|
|
/* Linear -> linear sub-window copy. */
|
|
if (image->planes[0].surface.is_linear) {
|
|
ASSERTED unsigned cdw_max =
|
|
radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs, align(8, ib_pad_dw_mask + 1));
|
|
unsigned bytes = src_pitch * copy_height * bpp;
|
|
|
|
if (!(bytes < (1u << 22)))
|
|
return false;
|
|
|
|
radeon_emit(cmd_buffer->cs, 0x00000000);
|
|
|
|
src_address += image->planes[0].surface.u.gfx9.offset[0];
|
|
|
|
radeon_emit(cmd_buffer->cs, CIK_SDMA_PACKET(CIK_SDMA_OPCODE_COPY,
|
|
CIK_SDMA_COPY_SUB_OPCODE_LINEAR, (tmz ? 4 : 0)));
|
|
radeon_emit(cmd_buffer->cs, bytes);
|
|
radeon_emit(cmd_buffer->cs, 0);
|
|
radeon_emit(cmd_buffer->cs, src_address);
|
|
radeon_emit(cmd_buffer->cs, src_address >> 32);
|
|
radeon_emit(cmd_buffer->cs, dst_address);
|
|
radeon_emit(cmd_buffer->cs, dst_address >> 32);
|
|
|
|
while (cmd_buffer->cs->cdw & ib_pad_dw_mask)
|
|
radeon_emit(cmd_buffer->cs, SDMA_NOP_PAD);
|
|
|
|
assert(cmd_buffer->cs->cdw <= cdw_max);
|
|
|
|
return true;
|
|
}
|
|
/* Tiled sub-window copy -> Linear */
|
|
else {
|
|
unsigned tiled_width = copy_width;
|
|
unsigned tiled_height = copy_height;
|
|
unsigned linear_pitch = region->bufferRowLength;
|
|
unsigned linear_slice_pitch = region->bufferRowLength * copy_height;
|
|
uint64_t tiled_address = src_address;
|
|
uint64_t linear_address = dst_address;
|
|
bool is_v5 = device->physical_device->rad_info.gfx_level >= GFX10;
|
|
/* Only SDMA 5 supports DCC with SDMA */
|
|
bool dcc = radv_dcc_enabled(image, 0) && is_v5;
|
|
|
|
/* Check if everything fits into the bitfields */
|
|
if (!(tiled_width < (1 << 14) && tiled_height < (1 << 14) && linear_pitch < (1 << 14) &&
|
|
linear_slice_pitch < (1 << 28) && copy_width < (1 << 14) && copy_height < (1 << 14)))
|
|
return false;
|
|
|
|
ASSERTED unsigned cdw_max = radeon_check_space(cmd_buffer->device->ws, cmd_buffer->cs,
|
|
align(15 + dcc * 3, ib_pad_dw_mask + 1));
|
|
|
|
radeon_emit(cmd_buffer->cs, 0x00000000);
|
|
radeon_emit(cmd_buffer->cs,
|
|
CIK_SDMA_PACKET(CIK_SDMA_OPCODE_COPY, CIK_SDMA_COPY_SUB_OPCODE_TILED_SUB_WINDOW,
|
|
(tmz ? 4 : 0)) |
|
|
dcc << 19 | (is_v5 ? 0 : 0 /* tiled->buffer.b.b.last_level */) << 20 |
|
|
1u << 31);
|
|
radeon_emit(cmd_buffer->cs,
|
|
(uint32_t)tiled_address | (image->planes[0].surface.tile_swizzle << 8));
|
|
radeon_emit(cmd_buffer->cs, (uint32_t)(tiled_address >> 32));
|
|
radeon_emit(cmd_buffer->cs, 0);
|
|
radeon_emit(cmd_buffer->cs, ((tiled_width - 1) << 16));
|
|
radeon_emit(cmd_buffer->cs, (tiled_height - 1));
|
|
radeon_emit(
|
|
cmd_buffer->cs,
|
|
util_logbase2(bpp) | image->planes[0].surface.u.gfx9.swizzle_mode << 3 |
|
|
image->planes[0].surface.u.gfx9.resource_type << 9 |
|
|
(is_v5 ? 0 /* tiled->buffer.b.b.last_level */ : image->planes[0].surface.u.gfx9.epitch)
|
|
<< 16);
|
|
radeon_emit(cmd_buffer->cs, (uint32_t)linear_address);
|
|
radeon_emit(cmd_buffer->cs, (uint32_t)(linear_address >> 32));
|
|
radeon_emit(cmd_buffer->cs, 0);
|
|
radeon_emit(cmd_buffer->cs, ((linear_pitch - 1) << 16));
|
|
radeon_emit(cmd_buffer->cs, linear_slice_pitch - 1);
|
|
radeon_emit(cmd_buffer->cs, (copy_width - 1) | ((copy_height - 1) << 16));
|
|
radeon_emit(cmd_buffer->cs, 0);
|
|
|
|
if (dcc) {
|
|
unsigned hw_fmt, hw_type;
|
|
uint64_t md_address = tiled_address + image->planes[0].surface.meta_offset;
|
|
|
|
radv_translate_format_to_hw(&device->physical_device->rad_info, image->vk.format, &hw_fmt,
|
|
&hw_type);
|
|
|
|
/* Add metadata */
|
|
radeon_emit(cmd_buffer->cs, (uint32_t)md_address);
|
|
radeon_emit(cmd_buffer->cs, (uint32_t)(md_address >> 32));
|
|
radeon_emit(cmd_buffer->cs,
|
|
hw_fmt | vi_alpha_is_on_msb(device, image->vk.format) << 8 | hw_type << 9 |
|
|
image->planes[0].surface.u.gfx9.color.dcc.max_compressed_block_size << 24 |
|
|
V_028C78_MAX_BLOCK_SIZE_256B << 26 | tmz << 29 |
|
|
image->planes[0].surface.u.gfx9.color.dcc.pipe_aligned << 31);
|
|
}
|
|
|
|
while (cmd_buffer->cs->cdw & ib_pad_dw_mask)
|
|
radeon_emit(cmd_buffer->cs, SDMA_NOP_PAD);
|
|
|
|
assert(cmd_buffer->cs->cdw <= cdw_max);
|
|
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool
|
|
radv_sdma_copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
|
|
struct radv_buffer *buffer, const VkBufferImageCopy2 *region)
|
|
{
|
|
assert(cmd_buffer->device->physical_device->rad_info.gfx_level >= GFX9);
|
|
return radv_sdma_v4_v5_copy_image_to_buffer(cmd_buffer, image, buffer, region);
|
|
}
|