369 lines
15 KiB
C
369 lines
15 KiB
C
/*
|
|
* Copyright © 2017 Intel Corporation
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included
|
|
* in all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
* DEALINGS IN THE SOFTWARE.
|
|
*/
|
|
|
|
/**
|
|
* @file crocus_pipe_control.c
|
|
*
|
|
* PIPE_CONTROL is the main flushing and synchronization primitive on Intel
|
|
* GPUs. It can invalidate caches, stall until rendering reaches various
|
|
* stages of completion, write to memory, and other things. In a way, it's
|
|
* a swiss army knife command - it has all kinds of capabilities, but some
|
|
* significant limitations as well.
|
|
*
|
|
* Unfortunately, it's notoriously complicated and difficult to use. Many
|
|
* sub-commands can't be used together. Some are meant to be used at the
|
|
* top of the pipeline (invalidating caches before drawing), while some are
|
|
* meant to be used at the end (stalling or flushing after drawing).
|
|
*
|
|
* Also, there's a list of restrictions a mile long, which vary by generation.
|
|
* Do this before doing that, or suffer the consequences (usually a GPU hang).
|
|
*
|
|
* This file contains helpers for emitting them safely. You can simply call
|
|
* crocus_emit_pipe_control_flush() with the desired operations (as logical
|
|
* PIPE_CONTROL_* bits), and it will take care of splitting it into multiple
|
|
* PIPE_CONTROL commands as necessary. The per-generation workarounds are
|
|
* applied in crocus_emit_raw_pipe_control() in crocus_state.c.
|
|
*/
|
|
|
|
#include "crocus_context.h"
|
|
#include "util/hash_table.h"
|
|
#include "util/set.h"
|
|
|
|
/**
|
|
* Emit a PIPE_CONTROL with various flushing flags.
|
|
*
|
|
* The caller is responsible for deciding what flags are appropriate for the
|
|
* given generation.
|
|
*/
|
|
void
|
|
crocus_emit_pipe_control_flush(struct crocus_batch *batch,
|
|
const char *reason,
|
|
uint32_t flags)
|
|
{
|
|
const struct intel_device_info *devinfo = &batch->screen->devinfo;
|
|
|
|
if (devinfo->ver >= 6 &&
|
|
(flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
|
|
(flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
|
|
/* A pipe control command with flush and invalidate bits set
|
|
* simultaneously is an inherently racy operation on Gen6+ if the
|
|
* contents of the flushed caches were intended to become visible from
|
|
* any of the invalidated caches. Split it in two PIPE_CONTROLs, the
|
|
* first one should stall the pipeline to make sure that the flushed R/W
|
|
* caches are coherent with memory once the specified R/O caches are
|
|
* invalidated. On pre-Gen6 hardware the (implicit) R/O cache
|
|
* invalidation seems to happen at the bottom of the pipeline together
|
|
* with any write cache flush, so this shouldn't be a concern. In order
|
|
* to ensure a full stall, we do an end-of-pipe sync.
|
|
*/
|
|
crocus_emit_end_of_pipe_sync(batch, reason,
|
|
flags & PIPE_CONTROL_CACHE_FLUSH_BITS);
|
|
flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
|
|
}
|
|
|
|
batch->screen->vtbl.emit_raw_pipe_control(batch, reason, flags, NULL, 0, 0);
|
|
}
|
|
|
|
/**
|
|
* Emit a PIPE_CONTROL that writes to a buffer object.
|
|
*
|
|
* \p flags should contain one of the following items:
|
|
* - PIPE_CONTROL_WRITE_IMMEDIATE
|
|
* - PIPE_CONTROL_WRITE_TIMESTAMP
|
|
* - PIPE_CONTROL_WRITE_DEPTH_COUNT
|
|
*/
|
|
void
|
|
crocus_emit_pipe_control_write(struct crocus_batch *batch,
|
|
const char *reason, uint32_t flags,
|
|
struct crocus_bo *bo, uint32_t offset,
|
|
uint64_t imm)
|
|
{
|
|
batch->screen->vtbl.emit_raw_pipe_control(batch, reason, flags, bo, offset, imm);
|
|
}
|
|
|
|
/**
|
|
* Restriction [DevSNB, DevIVB]:
|
|
*
|
|
* Prior to changing Depth/Stencil Buffer state (i.e. any combination of
|
|
* 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
|
|
* 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
|
|
* (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
|
|
* cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
|
|
* another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
|
|
* unless SW can otherwise guarantee that the pipeline from WM onwards is
|
|
* already flushed (e.g., via a preceding MI_FLUSH).
|
|
*/
|
|
void
|
|
crocus_emit_depth_stall_flushes(struct crocus_batch *batch)
|
|
{
|
|
UNUSED const struct intel_device_info *devinfo = &batch->screen->devinfo;
|
|
|
|
assert(devinfo->ver >= 6);
|
|
|
|
crocus_emit_pipe_control_flush(batch, "depth stall", PIPE_CONTROL_DEPTH_STALL);
|
|
crocus_emit_pipe_control_flush(batch, "depth stall", PIPE_CONTROL_DEPTH_CACHE_FLUSH);
|
|
crocus_emit_pipe_control_flush(batch, "depth stall", PIPE_CONTROL_DEPTH_STALL);
|
|
}
|
|
|
|
/*
|
|
* From Sandybridge PRM, volume 2, "1.7.2 End-of-Pipe Synchronization":
|
|
*
|
|
* Write synchronization is a special case of end-of-pipe
|
|
* synchronization that requires that the render cache and/or depth
|
|
* related caches are flushed to memory, where the data will become
|
|
* globally visible. This type of synchronization is required prior to
|
|
* SW (CPU) actually reading the result data from memory, or initiating
|
|
* an operation that will use as a read surface (such as a texture
|
|
* surface) a previous render target and/or depth/stencil buffer
|
|
*
|
|
* From Haswell PRM, volume 2, part 1, "End-of-Pipe Synchronization":
|
|
*
|
|
* Exercising the write cache flush bits (Render Target Cache Flush
|
|
* Enable, Depth Cache Flush Enable, DC Flush) in PIPE_CONTROL only
|
|
* ensures the write caches are flushed and doesn't guarantee the data
|
|
* is globally visible.
|
|
*
|
|
* SW can track the completion of the end-of-pipe-synchronization by
|
|
* using "Notify Enable" and "PostSync Operation - Write Immediate
|
|
* Data" in the PIPE_CONTROL command.
|
|
*/
|
|
void
|
|
crocus_emit_end_of_pipe_sync(struct crocus_batch *batch,
|
|
const char *reason, uint32_t flags)
|
|
{
|
|
const struct intel_device_info *devinfo = &batch->screen->devinfo;
|
|
|
|
if (devinfo->ver >= 6) {
|
|
/* From Sandybridge PRM, volume 2, "1.7.3.1 Writing a Value to Memory":
|
|
*
|
|
* "The most common action to perform upon reaching a synchronization
|
|
* point is to write a value out to memory. An immediate value
|
|
* (included with the synchronization command) may be written."
|
|
*
|
|
* From Broadwell PRM, volume 7, "End-of-Pipe Synchronization":
|
|
*
|
|
* "In case the data flushed out by the render engine is to be read
|
|
* back in to the render engine in coherent manner, then the render
|
|
* engine has to wait for the fence completion before accessing the
|
|
* flushed data. This can be achieved by following means on various
|
|
* products: PIPE_CONTROL command with CS Stall and the required
|
|
* write caches flushed with Post-Sync-Operation as Write Immediate
|
|
* Data.
|
|
*
|
|
* Example:
|
|
* - Workload-1 (3D/GPGPU/MEDIA)
|
|
* - PIPE_CONTROL (CS Stall, Post-Sync-Operation Write Immediate
|
|
* Data, Required Write Cache Flush bits set)
|
|
* - Workload-2 (Can use the data produce or output by Workload-1)
|
|
*/
|
|
crocus_emit_pipe_control_write(batch, reason,
|
|
flags | PIPE_CONTROL_CS_STALL |
|
|
PIPE_CONTROL_WRITE_IMMEDIATE,
|
|
batch->ice->workaround_bo,
|
|
batch->ice->workaround_offset, 0);
|
|
|
|
if (batch->screen->devinfo.is_haswell) {
|
|
#define GEN7_3DPRIM_START_INSTANCE 0x243C
|
|
batch->screen->vtbl.load_register_mem32(batch, GEN7_3DPRIM_START_INSTANCE,
|
|
batch->ice->workaround_bo,
|
|
batch->ice->workaround_offset);
|
|
}
|
|
} else {
|
|
/* On gen4-5, a regular pipe control seems to suffice. */
|
|
crocus_emit_pipe_control_flush(batch, reason, flags);
|
|
}
|
|
}
|
|
|
|
/* Emit a pipelined flush to either flush render and texture cache for
|
|
* reading from a FBO-drawn texture, or flush so that frontbuffer
|
|
* render appears on the screen in DRI1.
|
|
*
|
|
* This is also used for the always_flush_cache driconf debug option.
|
|
*/
|
|
void
|
|
crocus_emit_mi_flush(struct crocus_batch *batch)
|
|
{
|
|
const struct intel_device_info *devinfo = &batch->screen->devinfo;
|
|
int flags = PIPE_CONTROL_RENDER_TARGET_FLUSH;
|
|
if (devinfo->ver >= 6) {
|
|
flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE |
|
|
PIPE_CONTROL_CONST_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_DATA_CACHE_FLUSH |
|
|
PIPE_CONTROL_DEPTH_CACHE_FLUSH |
|
|
PIPE_CONTROL_VF_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_CS_STALL;
|
|
}
|
|
crocus_emit_pipe_control_flush(batch, "mi flush", flags);
|
|
}
|
|
|
|
/**
|
|
* Emits a PIPE_CONTROL with a non-zero post-sync operation, for
|
|
* implementing two workarounds on gen6. From section 1.4.7.1
|
|
* "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
|
|
*
|
|
* [DevSNB-C+{W/A}] Before any depth stall flush (including those
|
|
* produced by non-pipelined state commands), software needs to first
|
|
* send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
|
|
* 0.
|
|
*
|
|
* [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
|
|
* =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
|
|
*
|
|
* And the workaround for these two requires this workaround first:
|
|
*
|
|
* [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
|
|
* BEFORE the pipe-control with a post-sync op and no write-cache
|
|
* flushes.
|
|
*
|
|
* And this last workaround is tricky because of the requirements on
|
|
* that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
|
|
* volume 2 part 1:
|
|
*
|
|
* "1 of the following must also be set:
|
|
* - Render Target Cache Flush Enable ([12] of DW1)
|
|
* - Depth Cache Flush Enable ([0] of DW1)
|
|
* - Stall at Pixel Scoreboard ([1] of DW1)
|
|
* - Depth Stall ([13] of DW1)
|
|
* - Post-Sync Operation ([13] of DW1)
|
|
* - Notify Enable ([8] of DW1)"
|
|
*
|
|
* The cache flushes require the workaround flush that triggered this
|
|
* one, so we can't use it. Depth stall would trigger the same.
|
|
* Post-sync nonzero is what triggered this second workaround, so we
|
|
* can't use that one either. Notify enable is IRQs, which aren't
|
|
* really our business. That leaves only stall at scoreboard.
|
|
*/
|
|
void
|
|
crocus_emit_post_sync_nonzero_flush(struct crocus_batch *batch)
|
|
{
|
|
crocus_emit_pipe_control_flush(batch, "nonzero",
|
|
PIPE_CONTROL_CS_STALL |
|
|
PIPE_CONTROL_STALL_AT_SCOREBOARD);
|
|
|
|
crocus_emit_pipe_control_write(batch, "nonzero",
|
|
PIPE_CONTROL_WRITE_IMMEDIATE,
|
|
batch->ice->workaround_bo,
|
|
batch->ice->workaround_offset, 0);
|
|
}
|
|
|
|
/**
|
|
* Flush and invalidate all caches (for debugging purposes).
|
|
*/
|
|
void
|
|
crocus_flush_all_caches(struct crocus_batch *batch)
|
|
{
|
|
crocus_emit_pipe_control_flush(batch, "debug: flush all caches",
|
|
PIPE_CONTROL_CS_STALL |
|
|
PIPE_CONTROL_DATA_CACHE_FLUSH |
|
|
PIPE_CONTROL_DEPTH_CACHE_FLUSH |
|
|
PIPE_CONTROL_RENDER_TARGET_FLUSH |
|
|
PIPE_CONTROL_VF_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_INSTRUCTION_INVALIDATE |
|
|
PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_CONST_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_STATE_CACHE_INVALIDATE);
|
|
}
|
|
|
|
static void
|
|
crocus_texture_barrier(struct pipe_context *ctx, unsigned flags)
|
|
{
|
|
struct crocus_context *ice = (void *) ctx;
|
|
struct crocus_batch *render_batch = &ice->batches[CROCUS_BATCH_RENDER];
|
|
struct crocus_batch *compute_batch = &ice->batches[CROCUS_BATCH_COMPUTE];
|
|
const struct intel_device_info *devinfo = &render_batch->screen->devinfo;
|
|
|
|
if (devinfo->ver < 6) {
|
|
crocus_emit_mi_flush(render_batch);
|
|
return;
|
|
}
|
|
|
|
if (render_batch->contains_draw) {
|
|
crocus_batch_maybe_flush(render_batch, 48);
|
|
crocus_emit_pipe_control_flush(render_batch,
|
|
"API: texture barrier (1/2)",
|
|
(flags == 1 ? PIPE_CONTROL_DEPTH_CACHE_FLUSH : 0) |
|
|
PIPE_CONTROL_RENDER_TARGET_FLUSH |
|
|
PIPE_CONTROL_CS_STALL);
|
|
crocus_emit_pipe_control_flush(render_batch,
|
|
"API: texture barrier (2/2)",
|
|
PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
|
|
}
|
|
|
|
if (compute_batch->contains_draw) {
|
|
crocus_batch_maybe_flush(compute_batch, 48);
|
|
crocus_emit_pipe_control_flush(compute_batch,
|
|
"API: texture barrier (1/2)",
|
|
PIPE_CONTROL_CS_STALL);
|
|
crocus_emit_pipe_control_flush(compute_batch,
|
|
"API: texture barrier (2/2)",
|
|
PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
|
|
}
|
|
}
|
|
|
|
static void
|
|
crocus_memory_barrier(struct pipe_context *ctx, unsigned flags)
|
|
{
|
|
struct crocus_context *ice = (void *) ctx;
|
|
unsigned bits = PIPE_CONTROL_DATA_CACHE_FLUSH | PIPE_CONTROL_CS_STALL;
|
|
const struct intel_device_info *devinfo = &ice->batches[0].screen->devinfo;
|
|
|
|
assert(devinfo->ver == 7);
|
|
|
|
if (flags & (PIPE_BARRIER_VERTEX_BUFFER |
|
|
PIPE_BARRIER_INDEX_BUFFER |
|
|
PIPE_BARRIER_INDIRECT_BUFFER)) {
|
|
bits |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
|
|
}
|
|
|
|
if (flags & PIPE_BARRIER_CONSTANT_BUFFER) {
|
|
bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_CONST_CACHE_INVALIDATE;
|
|
}
|
|
|
|
if (flags & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_FRAMEBUFFER)) {
|
|
bits |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
|
|
PIPE_CONTROL_RENDER_TARGET_FLUSH;
|
|
}
|
|
|
|
/* Typed surface messages are handled by the render cache on IVB, so we
|
|
* need to flush it too.
|
|
*/
|
|
if (!devinfo->is_haswell)
|
|
bits |= PIPE_CONTROL_RENDER_TARGET_FLUSH;
|
|
|
|
for (int i = 0; i < ice->batch_count; i++) {
|
|
if (ice->batches[i].contains_draw) {
|
|
crocus_batch_maybe_flush(&ice->batches[i], 24);
|
|
crocus_emit_pipe_control_flush(&ice->batches[i], "API: memory barrier",
|
|
bits);
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
crocus_init_flush_functions(struct pipe_context *ctx)
|
|
{
|
|
ctx->memory_barrier = crocus_memory_barrier;
|
|
ctx->texture_barrier = crocus_texture_barrier;
|
|
}
|