From 69d74a46a0f3e9c9b77a93636c0a33a85078d25e Mon Sep 17 00:00:00 2001 From: Philip Rebohle Date: Fri, 16 Feb 2024 19:49:38 +0100 Subject: [PATCH] [dxbc] Remove emitRawBuffer{Load,Store} And factor these into the callers. We need to preserve the raw index and offset parameters to use raw access chains. --- src/dxbc/dxbc_compiler.cpp | 439 +++++++++++++++++-------------------- src/dxbc/dxbc_compiler.h | 13 -- 2 files changed, 206 insertions(+), 246 deletions(-) diff --git a/src/dxbc/dxbc_compiler.cpp b/src/dxbc/dxbc_compiler.cpp index 31a0b0a9..26d0d66e 100644 --- a/src/dxbc/dxbc_compiler.cpp +++ b/src/dxbc/dxbc_compiler.cpp @@ -2824,11 +2824,55 @@ namespace dxvk { // the data depends on the register type. const DxbcRegister& dstReg = ins.dst[0]; const DxbcRegister& srcReg = isStructured ? ins.src[2] : ins.src[1]; - + // Retrieve common info about the buffer const DxbcBufferInfo bufferInfo = getBufferInfo(srcReg); - - // Compute element index + + // Shared memory is the only type of buffer that + // is not accessed through a texel buffer view + bool isTgsm = srcReg.type == DxbcOperandType::ThreadGroupSharedMemory; + bool isSsbo = bufferInfo.isSsbo; + + // Common types and IDs used while loading the data + uint32_t bufferId = isTgsm || isSsbo ? 0 : m_module.opLoad(bufferInfo.typeId, bufferInfo.varId); + + uint32_t vectorTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 4 }); + uint32_t scalarTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 1 }); + + // Since all data is represented as a sequence of 32-bit + // integers, we have to load each component individually. + std::array ccomps = { 0, 0, 0, 0 }; + std::array scomps = { 0, 0, 0, 0 }; + uint32_t scount = 0; + + // The sparse feedback ID will be non-zero for sparse + // instructions on input. We need to reset it to 0. + SpirvMemoryOperands memoryOperands; + SpirvImageOperands imageOperands; + imageOperands.sparse = ins.dstCount == 2; + + uint32_t coherence = bufferInfo.coherence; + + if (isTgsm && m_moduleInfo.options.forceVolatileTgsmAccess) { + memoryOperands.flags |= spv::MemoryAccessVolatileMask; + coherence = spv::ScopeWorkgroup; + } + + if (coherence) { + memoryOperands.flags |= spv::MemoryAccessNonPrivatePointerMask; + + if (coherence != spv::ScopeInvocation) { + memoryOperands.flags |= spv::MemoryAccessMakePointerVisibleMask; + memoryOperands.makeVisible = m_module.constu32(coherence); + + imageOperands.flags = spv::ImageOperandsNonPrivateTexelMask + | spv::ImageOperandsMakeTexelVisibleMask; + imageOperands.makeVisible = m_module.constu32(coherence); + } + } + + uint32_t sparseFeedbackId = 0; + const DxbcRegisterValue elementIndex = isStructured ? emitCalcBufferIndexStructured( emitRegisterLoad(ins.src[0], DxbcRegMask(true, false, false, false)), @@ -2837,10 +2881,81 @@ namespace dxvk { : emitCalcBufferIndexRaw( emitRegisterLoad(ins.src[0], DxbcRegMask(true, false, false, false))); - uint32_t sparseFeedbackId = uint32_t(ins.dstCount == 2); + for (uint32_t i = 0; i < 4; i++) { + uint32_t sindex = srcReg.swizzle[i]; - emitRegisterStore(dstReg, emitRawBufferLoad(srcReg, - elementIndex, dstReg.mask, sparseFeedbackId)); + if (!dstReg.mask[i]) + continue; + + if (ccomps[sindex] == 0) { + uint32_t elementIndexAdjusted = m_module.opIAdd( + getVectorTypeId(elementIndex.type), elementIndex.id, + m_module.consti32(sindex)); + + // Load requested component from the buffer + uint32_t zero = 0; + + if (isTgsm) { + ccomps[sindex] = m_module.opLoad(scalarTypeId, + m_module.opAccessChain(bufferInfo.typeId, + bufferInfo.varId, 1, &elementIndexAdjusted), + memoryOperands); + } else if (isSsbo) { + uint32_t indices[2] = { m_module.constu32(0), elementIndexAdjusted }; + ccomps[sindex] = m_module.opLoad(scalarTypeId, + m_module.opAccessChain(bufferInfo.typeId, + bufferInfo.varId, 2, indices), + memoryOperands); + } else { + uint32_t resultTypeId = vectorTypeId; + uint32_t resultId = 0; + + if (imageOperands.sparse) + resultTypeId = getSparseResultTypeId(vectorTypeId); + + if (srcReg.type == DxbcOperandType::Resource) { + resultId = m_module.opImageFetch(resultTypeId, + bufferId, elementIndexAdjusted, imageOperands); + } else if (srcReg.type == DxbcOperandType::UnorderedAccessView) { + resultId = m_module.opImageRead(resultTypeId, + bufferId, elementIndexAdjusted, imageOperands); + } else { + throw DxvkError("DxbcCompiler: Invalid operand type for strucured/raw load"); + } + + // Only read sparse feedback once. This may be somewhat inaccurate + // for reads that straddle pages, but we can't easily emulate this. + if (imageOperands.sparse) { + imageOperands.sparse = false; + sparseFeedbackId = resultId; + + resultId = emitExtractSparseTexel(vectorTypeId, resultId); + } + + ccomps[sindex] = m_module.opCompositeExtract(scalarTypeId, resultId, 1, &zero); + } + } + } + + for (uint32_t i = 0; i < 4; i++) { + uint32_t sindex = srcReg.swizzle[i]; + + if (dstReg.mask[i]) + scomps[scount++] = ccomps[sindex]; + } + + DxbcRegisterValue result = { }; + result.type.ctype = DxbcScalarType::Uint32; + result.type.ccount = scount; + result.id = scomps[0]; + + if (scount > 1) { + result.id = m_module.opCompositeConstruct( + getVectorTypeId(result.type), + scount, scomps.data()); + } + + emitRegisterStore(dstReg, result); if (sparseFeedbackId) emitStoreSparseFeedback(ins.dst[1], sparseFeedbackId); @@ -2863,11 +2978,49 @@ namespace dxvk { // the data depends on the register type. const DxbcRegister& dstReg = ins.dst[0]; const DxbcRegister& srcReg = isStructured ? ins.src[2] : ins.src[1]; - + + DxbcRegisterValue value = emitRegisterLoad(srcReg, dstReg.mask); + value = emitRegisterBitcast(value, DxbcScalarType::Uint32); + // Retrieve common info about the buffer const DxbcBufferInfo bufferInfo = getBufferInfo(dstReg); - - // Compute element index + + // Thread Group Shared Memory is not accessed through a texel buffer view + bool isTgsm = dstReg.type == DxbcOperandType::ThreadGroupSharedMemory; + bool isSsbo = bufferInfo.isSsbo; + + uint32_t bufferId = isTgsm || isSsbo ? 0 : m_module.opLoad(bufferInfo.typeId, bufferInfo.varId); + + uint32_t scalarTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 1 }); + uint32_t vectorTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 4 }); + + uint32_t srcComponentIndex = 0; + + // Set memory operands according to resource properties + SpirvMemoryOperands memoryOperands; + SpirvImageOperands imageOperands; + + uint32_t coherence = bufferInfo.coherence; + + if (isTgsm && m_moduleInfo.options.forceVolatileTgsmAccess) { + memoryOperands.flags |= spv::MemoryAccessVolatileMask; + coherence = spv::ScopeWorkgroup; + } + + if (coherence) { + memoryOperands.flags |= spv::MemoryAccessNonPrivatePointerMask; + + if (coherence != spv::ScopeInvocation) { + memoryOperands.flags |= spv::MemoryAccessMakePointerAvailableMask; + memoryOperands.makeAvailable = m_module.constu32(coherence); + + imageOperands.flags = spv::ImageOperandsNonPrivateTexelMask + | spv::ImageOperandsMakeTexelAvailableMask; + imageOperands.makeAvailable = m_module.constu32(coherence); + } + } + + // Compute flat element index const DxbcRegisterValue elementIndex = isStructured ? emitCalcBufferIndexStructured( emitRegisterLoad(ins.src[0], DxbcRegMask(true, false, false, false)), @@ -2875,9 +3028,50 @@ namespace dxvk { bufferInfo.stride) : emitCalcBufferIndexRaw( emitRegisterLoad(ins.src[0], DxbcRegMask(true, false, false, false))); - - emitRawBufferStore(dstReg, elementIndex, - emitRegisterLoad(srcReg, dstReg.mask)); + + for (uint32_t i = 0; i < 4; i++) { + if (dstReg.mask[i]) { + uint32_t srcComponentId = value.type.ccount > 1 + ? m_module.opCompositeExtract(scalarTypeId, + value.id, 1, &srcComponentIndex) + : value.id; + + // Add the component offset to the element index + uint32_t elementIndexAdjusted = i != 0 + ? m_module.opIAdd(getVectorTypeId(elementIndex.type), + elementIndex.id, m_module.consti32(i)) + : elementIndex.id; + + if (isTgsm) { + m_module.opStore( + m_module.opAccessChain(bufferInfo.typeId, + bufferInfo.varId, 1, &elementIndexAdjusted), + srcComponentId, memoryOperands); + } else if (isSsbo) { + uint32_t indices[2] = { m_module.constu32(0), elementIndexAdjusted }; + m_module.opStore( + m_module.opAccessChain(bufferInfo.typeId, + bufferInfo.varId, 2, indices), + srcComponentId, memoryOperands); + } else if (dstReg.type == DxbcOperandType::UnorderedAccessView) { + const std::array srcVectorIds = { + srcComponentId, srcComponentId, + srcComponentId, srcComponentId, + }; + + m_module.opImageWrite( + bufferId, elementIndexAdjusted, + m_module.opCompositeConstruct(vectorTypeId, + 4, srcVectorIds.data()), + imageOperands); + } else { + throw DxvkError("DxbcCompiler: Invalid operand type for strucured/raw store"); + } + + // Write next component + srcComponentIndex += 1; + } + } } @@ -5205,227 +5399,6 @@ namespace dxvk { } - DxbcRegisterValue DxbcCompiler::emitRawBufferLoad( - const DxbcRegister& operand, - DxbcRegisterValue elementIndex, - DxbcRegMask writeMask, - uint32_t& sparseFeedbackId) { - const DxbcBufferInfo bufferInfo = getBufferInfo(operand); - - // Shared memory is the only type of buffer that - // is not accessed through a texel buffer view - bool isTgsm = operand.type == DxbcOperandType::ThreadGroupSharedMemory; - bool isSsbo = bufferInfo.isSsbo; - - // Common types and IDs used while loading the data - uint32_t bufferId = isTgsm || isSsbo ? 0 : m_module.opLoad(bufferInfo.typeId, bufferInfo.varId); - - uint32_t vectorTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 4 }); - uint32_t scalarTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 1 }); - - // Since all data is represented as a sequence of 32-bit - // integers, we have to load each component individually. - std::array ccomps = { 0, 0, 0, 0 }; - std::array scomps = { 0, 0, 0, 0 }; - uint32_t scount = 0; - - // The sparse feedback ID will be non-zero for sparse - // instructions on input. We need to reset it to 0. - SpirvMemoryOperands memoryOperands; - SpirvImageOperands imageOperands; - imageOperands.sparse = sparseFeedbackId != 0; - - uint32_t coherence = bufferInfo.coherence; - - if (isTgsm && m_moduleInfo.options.forceVolatileTgsmAccess) { - memoryOperands.flags |= spv::MemoryAccessVolatileMask; - coherence = spv::ScopeWorkgroup; - } - - if (coherence) { - memoryOperands.flags |= spv::MemoryAccessNonPrivatePointerMask; - - if (coherence != spv::ScopeInvocation) { - memoryOperands.flags |= spv::MemoryAccessMakePointerVisibleMask; - memoryOperands.makeVisible = m_module.constu32(coherence); - - imageOperands.flags = spv::ImageOperandsNonPrivateTexelMask - | spv::ImageOperandsMakeTexelVisibleMask; - imageOperands.makeVisible = m_module.constu32(coherence); - } - } - - sparseFeedbackId = 0; - - for (uint32_t i = 0; i < 4; i++) { - uint32_t sindex = operand.swizzle[i]; - - if (!writeMask[i]) - continue; - - if (ccomps[sindex] == 0) { - uint32_t elementIndexAdjusted = m_module.opIAdd( - getVectorTypeId(elementIndex.type), elementIndex.id, - m_module.consti32(sindex)); - - // Load requested component from the buffer - uint32_t zero = 0; - - if (isTgsm) { - ccomps[sindex] = m_module.opLoad(scalarTypeId, - m_module.opAccessChain(bufferInfo.typeId, - bufferInfo.varId, 1, &elementIndexAdjusted), - memoryOperands); - } else if (isSsbo) { - uint32_t indices[2] = { m_module.constu32(0), elementIndexAdjusted }; - ccomps[sindex] = m_module.opLoad(scalarTypeId, - m_module.opAccessChain(bufferInfo.typeId, - bufferInfo.varId, 2, indices), - memoryOperands); - } else { - uint32_t resultTypeId = vectorTypeId; - uint32_t resultId = 0; - - if (imageOperands.sparse) - resultTypeId = getSparseResultTypeId(vectorTypeId); - - if (operand.type == DxbcOperandType::Resource) { - resultId = m_module.opImageFetch(resultTypeId, - bufferId, elementIndexAdjusted, imageOperands); - } else if (operand.type == DxbcOperandType::UnorderedAccessView) { - resultId = m_module.opImageRead(resultTypeId, - bufferId, elementIndexAdjusted, imageOperands); - } else { - throw DxvkError("DxbcCompiler: Invalid operand type for strucured/raw load"); - } - - // Only read sparse feedback once. This may be somewhat inaccurate - // for reads that straddle pages, but we can't easily emulate this. - if (imageOperands.sparse) { - imageOperands.sparse = false; - sparseFeedbackId = resultId; - - resultId = emitExtractSparseTexel(vectorTypeId, resultId); - } - - ccomps[sindex] = m_module.opCompositeExtract(scalarTypeId, resultId, 1, &zero); - } - } - } - - for (uint32_t i = 0; i < 4; i++) { - uint32_t sindex = operand.swizzle[i]; - - if (writeMask[i]) - scomps[scount++] = ccomps[sindex]; - } - - DxbcRegisterValue result; - result.type.ctype = DxbcScalarType::Uint32; - result.type.ccount = scount; - result.id = scomps[0]; - - if (scount > 1) { - result.id = m_module.opCompositeConstruct( - getVectorTypeId(result.type), - scount, scomps.data()); - } - - return result; - } - - - void DxbcCompiler::emitRawBufferStore( - const DxbcRegister& operand, - DxbcRegisterValue elementIndex, - DxbcRegisterValue value) { - const DxbcBufferInfo bufferInfo = getBufferInfo(operand); - - // Cast source value to the expected data type - value = emitRegisterBitcast(value, DxbcScalarType::Uint32); - - // Thread Group Shared Memory is not accessed through a texel buffer view - bool isTgsm = operand.type == DxbcOperandType::ThreadGroupSharedMemory; - bool isSsbo = bufferInfo.isSsbo; - - // Perform the actual write operation - uint32_t bufferId = isTgsm || isSsbo ? 0 : m_module.opLoad(bufferInfo.typeId, bufferInfo.varId); - - uint32_t scalarTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 1 }); - uint32_t vectorTypeId = getVectorTypeId({ DxbcScalarType::Uint32, 4 }); - - uint32_t srcComponentIndex = 0; - - // Set memory operands according to resource properties - SpirvMemoryOperands memoryOperands; - SpirvImageOperands imageOperands; - - uint32_t coherence = bufferInfo.coherence; - - if (isTgsm && m_moduleInfo.options.forceVolatileTgsmAccess) { - memoryOperands.flags |= spv::MemoryAccessVolatileMask; - coherence = spv::ScopeWorkgroup; - } - - if (coherence) { - memoryOperands.flags |= spv::MemoryAccessNonPrivatePointerMask; - - if (coherence != spv::ScopeInvocation) { - memoryOperands.flags |= spv::MemoryAccessMakePointerAvailableMask; - memoryOperands.makeAvailable = m_module.constu32(coherence); - - imageOperands.flags = spv::ImageOperandsNonPrivateTexelMask - | spv::ImageOperandsMakeTexelAvailableMask; - imageOperands.makeAvailable = m_module.constu32(coherence); - } - } - - for (uint32_t i = 0; i < 4; i++) { - if (operand.mask[i]) { - uint32_t srcComponentId = value.type.ccount > 1 - ? m_module.opCompositeExtract(scalarTypeId, - value.id, 1, &srcComponentIndex) - : value.id; - - // Add the component offset to the element index - uint32_t elementIndexAdjusted = i != 0 - ? m_module.opIAdd(getVectorTypeId(elementIndex.type), - elementIndex.id, m_module.consti32(i)) - : elementIndex.id; - - if (isTgsm) { - m_module.opStore( - m_module.opAccessChain(bufferInfo.typeId, - bufferInfo.varId, 1, &elementIndexAdjusted), - srcComponentId, memoryOperands); - } else if (isSsbo) { - uint32_t indices[2] = { m_module.constu32(0), elementIndexAdjusted }; - m_module.opStore( - m_module.opAccessChain(bufferInfo.typeId, - bufferInfo.varId, 2, indices), - srcComponentId, memoryOperands); - } else if (operand.type == DxbcOperandType::UnorderedAccessView) { - const std::array srcVectorIds = { - srcComponentId, srcComponentId, - srcComponentId, srcComponentId, - }; - - m_module.opImageWrite( - bufferId, elementIndexAdjusted, - m_module.opCompositeConstruct(vectorTypeId, - 4, srcVectorIds.data()), - imageOperands); - } else { - throw DxvkError("DxbcCompiler: Invalid operand type for strucured/raw store"); - } - - // Write next component - srcComponentIndex += 1; - } - } - } - - DxbcRegisterValue DxbcCompiler::emitQueryBufferSize( const DxbcRegister& resource) { const DxbcBufferInfo bufferInfo = getBufferInfo(resource); diff --git a/src/dxbc/dxbc_compiler.h b/src/dxbc/dxbc_compiler.h index d7fba34f..d23bc59f 100644 --- a/src/dxbc/dxbc_compiler.h +++ b/src/dxbc/dxbc_compiler.h @@ -945,19 +945,6 @@ namespace dxvk { const DxbcRegister& operand, const DxbcRegister& address); - /////////////////////////////// - // Resource load/store methods - DxbcRegisterValue emitRawBufferLoad( - const DxbcRegister& operand, - DxbcRegisterValue elementIndex, - DxbcRegMask writeMask, - uint32_t& sparseFeedbackId); - - void emitRawBufferStore( - const DxbcRegister& operand, - DxbcRegisterValue elementIndex, - DxbcRegisterValue value); - ////////////////////////// // Resource query methods DxbcRegisterValue emitQueryBufferSize(