3273 lines
103 KiB
C
3273 lines
103 KiB
C
/**************************************************************************
|
|
*
|
|
* Copyright 2017 Advanced Micro Devices, Inc.
|
|
* All Rights Reserved.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* on the rights to use, copy, modify, merge, publish, distribute, sub
|
|
* license, and/or sell copies of the Software, and to permit persons to whom
|
|
* the Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including the next
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
* Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
|
|
* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
|
|
* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
*
|
|
**************************************************************************/
|
|
|
|
#include "util/u_threaded_context.h"
|
|
#include "util/u_cpu_detect.h"
|
|
#include "util/format/u_format.h"
|
|
#include "util/u_inlines.h"
|
|
#include "util/u_memory.h"
|
|
#include "util/u_upload_mgr.h"
|
|
#include "driver_trace/tr_context.h"
|
|
#include "util/log.h"
|
|
#include "compiler/shader_info.h"
|
|
|
|
/* 0 = disabled, 1 = assertions, 2 = printfs */
|
|
#define TC_DEBUG 0
|
|
|
|
#if TC_DEBUG >= 1
|
|
#define tc_assert assert
|
|
#else
|
|
#define tc_assert(x)
|
|
#endif
|
|
|
|
#if TC_DEBUG >= 2
|
|
#define tc_printf mesa_logi
|
|
#define tc_asprintf asprintf
|
|
#define tc_strcmp strcmp
|
|
#else
|
|
#define tc_printf(...)
|
|
#define tc_asprintf(...) 0
|
|
#define tc_strcmp(...) 0
|
|
#endif
|
|
|
|
#define TC_SENTINEL 0x5ca1ab1e
|
|
|
|
enum tc_call_id {
|
|
#define CALL(name) TC_CALL_##name,
|
|
#include "u_threaded_context_calls.h"
|
|
#undef CALL
|
|
TC_NUM_CALLS,
|
|
};
|
|
|
|
/* This is actually variable-sized, because indirect isn't allocated if it's
|
|
* not needed. */
|
|
struct tc_draw_single {
|
|
struct pipe_draw_info info;
|
|
};
|
|
|
|
typedef void (*tc_execute)(struct pipe_context *pipe, union tc_payload *payload);
|
|
|
|
static const tc_execute execute_func[TC_NUM_CALLS];
|
|
|
|
static void
|
|
tc_batch_check(UNUSED struct tc_batch *batch)
|
|
{
|
|
tc_assert(batch->sentinel == TC_SENTINEL);
|
|
tc_assert(batch->num_total_call_slots <= TC_CALLS_PER_BATCH);
|
|
}
|
|
|
|
static void
|
|
tc_debug_check(struct threaded_context *tc)
|
|
{
|
|
for (unsigned i = 0; i < TC_MAX_BATCHES; i++) {
|
|
tc_batch_check(&tc->batch_slots[i]);
|
|
tc_assert(tc->batch_slots[i].tc == tc);
|
|
}
|
|
}
|
|
|
|
static void
|
|
tc_set_driver_thread(struct threaded_context *tc)
|
|
{
|
|
#ifndef NDEBUG
|
|
tc->driver_thread = util_get_thread_id();
|
|
#endif
|
|
}
|
|
|
|
static void
|
|
tc_clear_driver_thread(struct threaded_context *tc)
|
|
{
|
|
#ifndef NDEBUG
|
|
memset(&tc->driver_thread, 0, sizeof(tc->driver_thread));
|
|
#endif
|
|
}
|
|
|
|
/* We don't want to read or write min_index and max_index, because
|
|
* it shouldn't be needed by drivers at this point.
|
|
*/
|
|
#define DRAW_INFO_SIZE_WITHOUT_MIN_MAX_INDEX \
|
|
offsetof(struct pipe_draw_info, min_index)
|
|
|
|
static void
|
|
simplify_draw_info(struct pipe_draw_info *info)
|
|
{
|
|
/* Clear these fields to facilitate draw merging.
|
|
* Drivers shouldn't use them.
|
|
*/
|
|
info->has_user_indices = false;
|
|
info->index_bounds_valid = false;
|
|
info->take_index_buffer_ownership = false;
|
|
info->_pad = 0;
|
|
|
|
/* This shouldn't be set when merging single draws. */
|
|
info->increment_draw_id = false;
|
|
|
|
if (info->mode != PIPE_PRIM_PATCHES)
|
|
info->vertices_per_patch = 0;
|
|
|
|
if (info->index_size) {
|
|
if (!info->primitive_restart)
|
|
info->restart_index = 0;
|
|
} else {
|
|
assert(!info->primitive_restart);
|
|
info->index_bias = 0;
|
|
info->primitive_restart = false;
|
|
info->restart_index = 0;
|
|
info->index.resource = NULL;
|
|
}
|
|
}
|
|
|
|
static bool
|
|
is_next_call_a_mergeable_draw(struct tc_draw_single *first_info,
|
|
struct tc_call *next,
|
|
struct tc_draw_single **next_info)
|
|
{
|
|
if (next->call_id != TC_CALL_draw_single)
|
|
return false;
|
|
|
|
*next_info = (struct tc_draw_single*)&next->payload;
|
|
simplify_draw_info(&(*next_info)->info);
|
|
|
|
STATIC_ASSERT(offsetof(struct pipe_draw_info, min_index) ==
|
|
sizeof(struct pipe_draw_info) - 8);
|
|
STATIC_ASSERT(offsetof(struct pipe_draw_info, max_index) ==
|
|
sizeof(struct pipe_draw_info) - 4);
|
|
|
|
/* All fields must be the same except start and count. */
|
|
/* u_threaded_context stores start/count in min/max_index for single draws. */
|
|
return memcmp((uint32_t*)&first_info->info,
|
|
(uint32_t*)&(*next_info)->info,
|
|
DRAW_INFO_SIZE_WITHOUT_MIN_MAX_INDEX) == 0;
|
|
}
|
|
|
|
static void
|
|
tc_batch_execute(void *job, UNUSED int thread_index)
|
|
{
|
|
struct tc_batch *batch = job;
|
|
struct pipe_context *pipe = batch->tc->pipe;
|
|
struct tc_call *last = &batch->call[batch->num_total_call_slots];
|
|
|
|
tc_batch_check(batch);
|
|
tc_set_driver_thread(batch->tc);
|
|
|
|
assert(!batch->token);
|
|
|
|
for (struct tc_call *iter = batch->call; iter != last;) {
|
|
tc_assert(iter->sentinel == TC_SENTINEL);
|
|
|
|
/* Draw call merging. */
|
|
if (iter->call_id == TC_CALL_draw_single) {
|
|
struct tc_call *first = iter;
|
|
struct tc_call *next = first + first->num_call_slots;
|
|
struct tc_draw_single *first_info =
|
|
(struct tc_draw_single*)&first->payload;
|
|
struct tc_draw_single *next_info;
|
|
|
|
simplify_draw_info(&first_info->info);
|
|
|
|
/* If at least 2 consecutive draw calls can be merged... */
|
|
if (next != last && next->call_id == TC_CALL_draw_single &&
|
|
first_info->info.drawid == 0 &&
|
|
is_next_call_a_mergeable_draw(first_info, next, &next_info)) {
|
|
/* Merge up to 256 draw calls. */
|
|
struct pipe_draw_start_count multi[256];
|
|
unsigned num_draws = 2;
|
|
|
|
/* u_threaded_context stores start/count in min/max_index for single draws. */
|
|
multi[0].start = first_info->info.min_index;
|
|
multi[0].count = first_info->info.max_index;
|
|
multi[1].start = next_info->info.min_index;
|
|
multi[1].count = next_info->info.max_index;
|
|
|
|
if (next_info->info.index_size)
|
|
pipe_resource_reference(&next_info->info.index.resource, NULL);
|
|
|
|
/* Find how many other draws can be merged. */
|
|
next = next + next->num_call_slots;
|
|
for (; next != last && num_draws < ARRAY_SIZE(multi) &&
|
|
is_next_call_a_mergeable_draw(first_info, next, &next_info);
|
|
next += next->num_call_slots, num_draws++) {
|
|
/* u_threaded_context stores start/count in min/max_index for single draws. */
|
|
multi[num_draws].start = next_info->info.min_index;
|
|
multi[num_draws].count = next_info->info.max_index;
|
|
|
|
if (next_info->info.index_size)
|
|
pipe_resource_reference(&next_info->info.index.resource, NULL);
|
|
}
|
|
|
|
pipe->draw_vbo(pipe, &first_info->info, NULL, multi, num_draws);
|
|
if (first_info->info.index_size)
|
|
pipe_resource_reference(&first_info->info.index.resource, NULL);
|
|
iter = next;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
execute_func[iter->call_id](pipe, &iter->payload);
|
|
iter += iter->num_call_slots;
|
|
}
|
|
|
|
tc_clear_driver_thread(batch->tc);
|
|
tc_batch_check(batch);
|
|
batch->num_total_call_slots = 0;
|
|
}
|
|
|
|
static void
|
|
tc_batch_flush(struct threaded_context *tc)
|
|
{
|
|
struct tc_batch *next = &tc->batch_slots[tc->next];
|
|
|
|
tc_assert(next->num_total_call_slots != 0);
|
|
tc_batch_check(next);
|
|
tc_debug_check(tc);
|
|
tc->bytes_mapped_estimate = 0;
|
|
p_atomic_add(&tc->num_offloaded_slots, next->num_total_call_slots);
|
|
|
|
if (next->token) {
|
|
next->token->tc = NULL;
|
|
tc_unflushed_batch_token_reference(&next->token, NULL);
|
|
}
|
|
|
|
util_queue_add_job(&tc->queue, next, &next->fence, tc_batch_execute,
|
|
NULL, 0);
|
|
tc->last = tc->next;
|
|
tc->next = (tc->next + 1) % TC_MAX_BATCHES;
|
|
}
|
|
|
|
/* This is the function that adds variable-sized calls into the current
|
|
* batch. It also flushes the batch if there is not enough space there.
|
|
* All other higher-level "add" functions use it.
|
|
*/
|
|
static union tc_payload *
|
|
tc_add_sized_call(struct threaded_context *tc, enum tc_call_id id,
|
|
unsigned num_call_slots)
|
|
{
|
|
struct tc_batch *next = &tc->batch_slots[tc->next];
|
|
assert(num_call_slots <= TC_CALLS_PER_BATCH);
|
|
tc_debug_check(tc);
|
|
|
|
if (unlikely(next->num_total_call_slots + num_call_slots > TC_CALLS_PER_BATCH)) {
|
|
tc_batch_flush(tc);
|
|
next = &tc->batch_slots[tc->next];
|
|
tc_assert(next->num_total_call_slots == 0);
|
|
}
|
|
|
|
tc_assert(util_queue_fence_is_signalled(&next->fence));
|
|
|
|
struct tc_call *call = &next->call[next->num_total_call_slots];
|
|
next->num_total_call_slots += num_call_slots;
|
|
|
|
call->sentinel = TC_SENTINEL;
|
|
call->call_id = id;
|
|
call->num_call_slots = num_call_slots;
|
|
|
|
tc_debug_check(tc);
|
|
return &call->payload;
|
|
}
|
|
|
|
#define tc_payload_size_to_call_slots(size) \
|
|
DIV_ROUND_UP(offsetof(struct tc_call, payload) + (size), sizeof(struct tc_call))
|
|
|
|
#define tc_add_struct_typed_call(tc, execute, type) \
|
|
((struct type*)tc_add_sized_call(tc, execute, \
|
|
tc_payload_size_to_call_slots(sizeof(struct type))))
|
|
|
|
#define tc_add_slot_based_call(tc, execute, type, num_slots) \
|
|
((struct type*)tc_add_sized_call(tc, execute, tc_payload_size_to_call_slots( \
|
|
sizeof(struct type) + \
|
|
sizeof(((struct type*)NULL)->slot[0]) * \
|
|
(num_slots))))
|
|
|
|
static union tc_payload *
|
|
tc_add_small_call(struct threaded_context *tc, enum tc_call_id id)
|
|
{
|
|
return tc_add_sized_call(tc, id, tc_payload_size_to_call_slots(0));
|
|
}
|
|
|
|
static bool
|
|
tc_is_sync(struct threaded_context *tc)
|
|
{
|
|
struct tc_batch *last = &tc->batch_slots[tc->last];
|
|
struct tc_batch *next = &tc->batch_slots[tc->next];
|
|
|
|
return util_queue_fence_is_signalled(&last->fence) &&
|
|
!next->num_total_call_slots;
|
|
}
|
|
|
|
static void
|
|
_tc_sync(struct threaded_context *tc, UNUSED const char *info, UNUSED const char *func)
|
|
{
|
|
struct tc_batch *last = &tc->batch_slots[tc->last];
|
|
struct tc_batch *next = &tc->batch_slots[tc->next];
|
|
bool synced = false;
|
|
|
|
tc_debug_check(tc);
|
|
|
|
/* Only wait for queued calls... */
|
|
if (!util_queue_fence_is_signalled(&last->fence)) {
|
|
util_queue_fence_wait(&last->fence);
|
|
synced = true;
|
|
}
|
|
|
|
tc_debug_check(tc);
|
|
|
|
if (next->token) {
|
|
next->token->tc = NULL;
|
|
tc_unflushed_batch_token_reference(&next->token, NULL);
|
|
}
|
|
|
|
/* .. and execute unflushed calls directly. */
|
|
if (next->num_total_call_slots) {
|
|
p_atomic_add(&tc->num_direct_slots, next->num_total_call_slots);
|
|
tc->bytes_mapped_estimate = 0;
|
|
tc_batch_execute(next, 0);
|
|
synced = true;
|
|
}
|
|
|
|
if (synced) {
|
|
p_atomic_inc(&tc->num_syncs);
|
|
|
|
if (tc_strcmp(func, "tc_destroy") != 0) {
|
|
tc_printf("sync %s %s", func, info);
|
|
}
|
|
}
|
|
|
|
tc_debug_check(tc);
|
|
}
|
|
|
|
#define tc_sync(tc) _tc_sync(tc, "", __func__)
|
|
#define tc_sync_msg(tc, info) _tc_sync(tc, info, __func__)
|
|
|
|
/**
|
|
* Call this from fence_finish for same-context fence waits of deferred fences
|
|
* that haven't been flushed yet.
|
|
*
|
|
* The passed pipe_context must be the one passed to pipe_screen::fence_finish,
|
|
* i.e., the wrapped one.
|
|
*/
|
|
void
|
|
threaded_context_flush(struct pipe_context *_pipe,
|
|
struct tc_unflushed_batch_token *token,
|
|
bool prefer_async)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
/* This is called from the gallium frontend / application thread. */
|
|
if (token->tc && token->tc == tc) {
|
|
struct tc_batch *last = &tc->batch_slots[tc->last];
|
|
|
|
/* Prefer to do the flush in the driver thread if it is already
|
|
* running. That should be better for cache locality.
|
|
*/
|
|
if (prefer_async || !util_queue_fence_is_signalled(&last->fence))
|
|
tc_batch_flush(tc);
|
|
else
|
|
tc_sync(token->tc);
|
|
}
|
|
}
|
|
|
|
static void
|
|
tc_set_resource_reference(struct pipe_resource **dst, struct pipe_resource *src)
|
|
{
|
|
*dst = NULL;
|
|
pipe_resource_reference(dst, src);
|
|
}
|
|
|
|
void
|
|
threaded_resource_init(struct pipe_resource *res)
|
|
{
|
|
struct threaded_resource *tres = threaded_resource(res);
|
|
|
|
tres->latest = &tres->b;
|
|
util_range_init(&tres->valid_buffer_range);
|
|
tres->base_valid_buffer_range = &tres->valid_buffer_range;
|
|
tres->is_shared = false;
|
|
tres->is_user_ptr = false;
|
|
tres->pending_staging_uploads = 0;
|
|
util_range_init(&tres->pending_staging_uploads_range);
|
|
}
|
|
|
|
void
|
|
threaded_resource_deinit(struct pipe_resource *res)
|
|
{
|
|
struct threaded_resource *tres = threaded_resource(res);
|
|
|
|
if (tres->latest != &tres->b)
|
|
pipe_resource_reference(&tres->latest, NULL);
|
|
util_range_destroy(&tres->valid_buffer_range);
|
|
util_range_destroy(&tres->pending_staging_uploads_range);
|
|
}
|
|
|
|
struct pipe_context *
|
|
threaded_context_unwrap_sync(struct pipe_context *pipe)
|
|
{
|
|
if (!pipe || !pipe->priv)
|
|
return pipe;
|
|
|
|
tc_sync(threaded_context(pipe));
|
|
return (struct pipe_context*)pipe->priv;
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* simple functions
|
|
*/
|
|
|
|
#define TC_FUNC1(func, m_payload, qualifier, type, deref, deref2) \
|
|
static void \
|
|
tc_call_##func(struct pipe_context *pipe, union tc_payload *payload) \
|
|
{ \
|
|
pipe->func(pipe, deref2((type*)payload)); \
|
|
} \
|
|
\
|
|
static void \
|
|
tc_##func(struct pipe_context *_pipe, qualifier type deref param) \
|
|
{ \
|
|
struct threaded_context *tc = threaded_context(_pipe); \
|
|
type *p = (type*)tc_add_sized_call(tc, TC_CALL_##func, \
|
|
tc_payload_size_to_call_slots(sizeof(type))); \
|
|
*p = deref(param); \
|
|
}
|
|
|
|
TC_FUNC1(set_active_query_state, flags, , bool, , *)
|
|
|
|
TC_FUNC1(set_blend_color, blend_color, const, struct pipe_blend_color, *, )
|
|
TC_FUNC1(set_stencil_ref, stencil_ref, const, struct pipe_stencil_ref, , *)
|
|
TC_FUNC1(set_clip_state, clip_state, const, struct pipe_clip_state, *, )
|
|
TC_FUNC1(set_sample_mask, sample_mask, , unsigned, , *)
|
|
TC_FUNC1(set_min_samples, min_samples, , unsigned, , *)
|
|
TC_FUNC1(set_polygon_stipple, polygon_stipple, const, struct pipe_poly_stipple, *, )
|
|
|
|
TC_FUNC1(texture_barrier, flags, , unsigned, , *)
|
|
TC_FUNC1(memory_barrier, flags, , unsigned, , *)
|
|
|
|
|
|
/********************************************************************
|
|
* queries
|
|
*/
|
|
|
|
static struct pipe_query *
|
|
tc_create_query(struct pipe_context *_pipe, unsigned query_type,
|
|
unsigned index)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
return pipe->create_query(pipe, query_type, index);
|
|
}
|
|
|
|
static struct pipe_query *
|
|
tc_create_batch_query(struct pipe_context *_pipe, unsigned num_queries,
|
|
unsigned *query_types)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
return pipe->create_batch_query(pipe, num_queries, query_types);
|
|
}
|
|
|
|
static void
|
|
tc_call_destroy_query(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct threaded_query *tq = threaded_query(payload->query);
|
|
|
|
if (list_is_linked(&tq->head_unflushed))
|
|
list_del(&tq->head_unflushed);
|
|
|
|
pipe->destroy_query(pipe, payload->query);
|
|
}
|
|
|
|
static void
|
|
tc_destroy_query(struct pipe_context *_pipe, struct pipe_query *query)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
tc_add_small_call(tc, TC_CALL_destroy_query)->query = query;
|
|
}
|
|
|
|
static void
|
|
tc_call_begin_query(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
pipe->begin_query(pipe, payload->query);
|
|
}
|
|
|
|
static bool
|
|
tc_begin_query(struct pipe_context *_pipe, struct pipe_query *query)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
union tc_payload *payload = tc_add_small_call(tc, TC_CALL_begin_query);
|
|
|
|
payload->query = query;
|
|
return true; /* we don't care about the return value for this call */
|
|
}
|
|
|
|
struct tc_end_query_payload {
|
|
struct threaded_context *tc;
|
|
struct pipe_query *query;
|
|
};
|
|
|
|
static void
|
|
tc_call_end_query(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_end_query_payload *p = (struct tc_end_query_payload *)payload;
|
|
struct threaded_query *tq = threaded_query(p->query);
|
|
|
|
if (!list_is_linked(&tq->head_unflushed))
|
|
list_add(&tq->head_unflushed, &p->tc->unflushed_queries);
|
|
|
|
pipe->end_query(pipe, p->query);
|
|
}
|
|
|
|
static bool
|
|
tc_end_query(struct pipe_context *_pipe, struct pipe_query *query)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_query *tq = threaded_query(query);
|
|
struct tc_end_query_payload *payload =
|
|
tc_add_struct_typed_call(tc, TC_CALL_end_query, tc_end_query_payload);
|
|
|
|
payload->tc = tc;
|
|
payload->query = query;
|
|
|
|
tq->flushed = false;
|
|
|
|
return true; /* we don't care about the return value for this call */
|
|
}
|
|
|
|
static bool
|
|
tc_get_query_result(struct pipe_context *_pipe,
|
|
struct pipe_query *query, bool wait,
|
|
union pipe_query_result *result)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_query *tq = threaded_query(query);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
bool flushed = tq->flushed;
|
|
|
|
if (!flushed) {
|
|
tc_sync_msg(tc, wait ? "wait" : "nowait");
|
|
tc_set_driver_thread(tc);
|
|
}
|
|
|
|
bool success = pipe->get_query_result(pipe, query, wait, result);
|
|
|
|
if (!flushed)
|
|
tc_clear_driver_thread(tc);
|
|
|
|
if (success) {
|
|
tq->flushed = true;
|
|
if (list_is_linked(&tq->head_unflushed)) {
|
|
/* This is safe because it can only happen after we sync'd. */
|
|
list_del(&tq->head_unflushed);
|
|
}
|
|
}
|
|
return success;
|
|
}
|
|
|
|
struct tc_query_result_resource {
|
|
struct pipe_query *query;
|
|
bool wait;
|
|
enum pipe_query_value_type result_type;
|
|
int index;
|
|
struct pipe_resource *resource;
|
|
unsigned offset;
|
|
};
|
|
|
|
static void
|
|
tc_call_get_query_result_resource(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
struct tc_query_result_resource *p = (struct tc_query_result_resource *)payload;
|
|
|
|
pipe->get_query_result_resource(pipe, p->query, p->wait, p->result_type,
|
|
p->index, p->resource, p->offset);
|
|
pipe_resource_reference(&p->resource, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_get_query_result_resource(struct pipe_context *_pipe,
|
|
struct pipe_query *query, bool wait,
|
|
enum pipe_query_value_type result_type, int index,
|
|
struct pipe_resource *resource, unsigned offset)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_query_result_resource *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_get_query_result_resource,
|
|
tc_query_result_resource);
|
|
|
|
p->query = query;
|
|
p->wait = wait;
|
|
p->result_type = result_type;
|
|
p->index = index;
|
|
tc_set_resource_reference(&p->resource, resource);
|
|
p->offset = offset;
|
|
}
|
|
|
|
struct tc_render_condition {
|
|
struct pipe_query *query;
|
|
bool condition;
|
|
unsigned mode;
|
|
};
|
|
|
|
static void
|
|
tc_call_render_condition(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_render_condition *p = (struct tc_render_condition *)payload;
|
|
pipe->render_condition(pipe, p->query, p->condition, p->mode);
|
|
}
|
|
|
|
static void
|
|
tc_render_condition(struct pipe_context *_pipe,
|
|
struct pipe_query *query, bool condition,
|
|
enum pipe_render_cond_flag mode)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_render_condition *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_render_condition, tc_render_condition);
|
|
|
|
p->query = query;
|
|
p->condition = condition;
|
|
p->mode = mode;
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* constant (immutable) states
|
|
*/
|
|
|
|
#define TC_CSO_CREATE(name, sname) \
|
|
static void * \
|
|
tc_create_##name##_state(struct pipe_context *_pipe, \
|
|
const struct pipe_##sname##_state *state) \
|
|
{ \
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe; \
|
|
return pipe->create_##name##_state(pipe, state); \
|
|
}
|
|
|
|
#define TC_CSO_BIND(name) TC_FUNC1(bind_##name##_state, cso, , void *, , *)
|
|
#define TC_CSO_DELETE(name) TC_FUNC1(delete_##name##_state, cso, , void *, , *)
|
|
|
|
#define TC_CSO_WHOLE2(name, sname) \
|
|
TC_CSO_CREATE(name, sname) \
|
|
TC_CSO_BIND(name) \
|
|
TC_CSO_DELETE(name)
|
|
|
|
#define TC_CSO_WHOLE(name) TC_CSO_WHOLE2(name, name)
|
|
|
|
TC_CSO_WHOLE(blend)
|
|
TC_CSO_WHOLE(rasterizer)
|
|
TC_CSO_WHOLE(depth_stencil_alpha)
|
|
TC_CSO_WHOLE(compute)
|
|
TC_CSO_WHOLE2(fs, shader)
|
|
TC_CSO_WHOLE2(vs, shader)
|
|
TC_CSO_WHOLE2(gs, shader)
|
|
TC_CSO_WHOLE2(tcs, shader)
|
|
TC_CSO_WHOLE2(tes, shader)
|
|
TC_CSO_CREATE(sampler, sampler)
|
|
TC_CSO_DELETE(sampler)
|
|
TC_CSO_BIND(vertex_elements)
|
|
TC_CSO_DELETE(vertex_elements)
|
|
|
|
static void *
|
|
tc_create_vertex_elements_state(struct pipe_context *_pipe, unsigned count,
|
|
const struct pipe_vertex_element *elems)
|
|
{
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe;
|
|
|
|
return pipe->create_vertex_elements_state(pipe, count, elems);
|
|
}
|
|
|
|
struct tc_sampler_states {
|
|
ubyte shader, start, count;
|
|
void *slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_bind_sampler_states(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_sampler_states *p = (struct tc_sampler_states *)payload;
|
|
pipe->bind_sampler_states(pipe, p->shader, p->start, p->count, p->slot);
|
|
}
|
|
|
|
static void
|
|
tc_bind_sampler_states(struct pipe_context *_pipe,
|
|
enum pipe_shader_type shader,
|
|
unsigned start, unsigned count, void **states)
|
|
{
|
|
if (!count)
|
|
return;
|
|
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_sampler_states *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_bind_sampler_states, tc_sampler_states, count);
|
|
|
|
p->shader = shader;
|
|
p->start = start;
|
|
p->count = count;
|
|
memcpy(p->slot, states, count * sizeof(states[0]));
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* immediate states
|
|
*/
|
|
|
|
static void
|
|
tc_call_set_framebuffer_state(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct pipe_framebuffer_state *p = (struct pipe_framebuffer_state *)payload;
|
|
|
|
pipe->set_framebuffer_state(pipe, p);
|
|
|
|
unsigned nr_cbufs = p->nr_cbufs;
|
|
for (unsigned i = 0; i < nr_cbufs; i++)
|
|
pipe_surface_reference(&p->cbufs[i], NULL);
|
|
pipe_surface_reference(&p->zsbuf, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_set_framebuffer_state(struct pipe_context *_pipe,
|
|
const struct pipe_framebuffer_state *fb)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_framebuffer_state *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_set_framebuffer_state,
|
|
pipe_framebuffer_state);
|
|
unsigned nr_cbufs = fb->nr_cbufs;
|
|
|
|
p->width = fb->width;
|
|
p->height = fb->height;
|
|
p->samples = fb->samples;
|
|
p->layers = fb->layers;
|
|
p->nr_cbufs = nr_cbufs;
|
|
|
|
for (unsigned i = 0; i < nr_cbufs; i++) {
|
|
p->cbufs[i] = NULL;
|
|
pipe_surface_reference(&p->cbufs[i], fb->cbufs[i]);
|
|
}
|
|
p->zsbuf = NULL;
|
|
pipe_surface_reference(&p->zsbuf, fb->zsbuf);
|
|
}
|
|
|
|
static void
|
|
tc_call_set_tess_state(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
float *p = (float*)payload;
|
|
pipe->set_tess_state(pipe, p, p + 4);
|
|
}
|
|
|
|
static void
|
|
tc_set_tess_state(struct pipe_context *_pipe,
|
|
const float default_outer_level[4],
|
|
const float default_inner_level[2])
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
float *p = (float*)tc_add_sized_call(tc, TC_CALL_set_tess_state,
|
|
tc_payload_size_to_call_slots(sizeof(float) * 6));
|
|
|
|
memcpy(p, default_outer_level, 4 * sizeof(float));
|
|
memcpy(p + 4, default_inner_level, 2 * sizeof(float));
|
|
}
|
|
|
|
struct tc_constant_buffer_info {
|
|
ubyte shader, index;
|
|
bool is_null;
|
|
};
|
|
|
|
struct tc_constant_buffer {
|
|
struct tc_constant_buffer_info info;
|
|
struct pipe_constant_buffer cb;
|
|
};
|
|
|
|
static void
|
|
tc_call_set_constant_buffer(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_constant_buffer *p = (struct tc_constant_buffer *)payload;
|
|
|
|
if (unlikely(p->info.is_null)) {
|
|
pipe->set_constant_buffer(pipe, p->info.shader, p->info.index, false, NULL);
|
|
return;
|
|
}
|
|
|
|
pipe->set_constant_buffer(pipe, p->info.shader, p->info.index, true, &p->cb);
|
|
}
|
|
|
|
static void
|
|
tc_set_constant_buffer(struct pipe_context *_pipe,
|
|
enum pipe_shader_type shader, uint index,
|
|
bool take_ownership,
|
|
const struct pipe_constant_buffer *cb)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
if (unlikely(!cb || (!cb->buffer && !cb->user_buffer))) {
|
|
struct tc_constant_buffer_info *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_set_constant_buffer,
|
|
tc_constant_buffer_info);
|
|
p->shader = shader;
|
|
p->index = index;
|
|
p->is_null = true;
|
|
return;
|
|
}
|
|
|
|
struct pipe_resource *buffer;
|
|
unsigned offset;
|
|
|
|
if (cb->user_buffer) {
|
|
/* This must be done before adding set_constant_buffer, because it could
|
|
* generate e.g. transfer_unmap and flush partially-uninitialized
|
|
* set_constant_buffer to the driver if it was done afterwards.
|
|
*/
|
|
buffer = NULL;
|
|
u_upload_data(tc->base.const_uploader, 0, cb->buffer_size,
|
|
tc->ubo_alignment, cb->user_buffer, &offset, &buffer);
|
|
u_upload_unmap(tc->base.const_uploader);
|
|
take_ownership = true;
|
|
} else {
|
|
buffer = cb->buffer;
|
|
offset = cb->buffer_offset;
|
|
}
|
|
|
|
struct tc_constant_buffer *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_set_constant_buffer,
|
|
tc_constant_buffer);
|
|
p->info.shader = shader;
|
|
p->info.index = index;
|
|
p->info.is_null = false;
|
|
p->cb.user_buffer = NULL;
|
|
p->cb.buffer_offset = offset;
|
|
p->cb.buffer_size = cb->buffer_size;
|
|
|
|
if (take_ownership)
|
|
p->cb.buffer = buffer;
|
|
else
|
|
tc_set_resource_reference(&p->cb.buffer, buffer);
|
|
}
|
|
|
|
struct tc_inlinable_constants {
|
|
ubyte shader;
|
|
ubyte num_values;
|
|
uint32_t values[MAX_INLINABLE_UNIFORMS];
|
|
};
|
|
|
|
static void
|
|
tc_call_set_inlinable_constants(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_inlinable_constants *p = (struct tc_inlinable_constants *)payload;
|
|
|
|
pipe->set_inlinable_constants(pipe, p->shader, p->num_values, p->values);
|
|
}
|
|
|
|
static void
|
|
tc_set_inlinable_constants(struct pipe_context *_pipe,
|
|
enum pipe_shader_type shader,
|
|
uint num_values, uint32_t *values)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_inlinable_constants *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_set_inlinable_constants,
|
|
tc_inlinable_constants);
|
|
p->shader = shader;
|
|
p->num_values = num_values;
|
|
memcpy(p->values, values, num_values * 4);
|
|
}
|
|
|
|
struct tc_sample_locations {
|
|
uint16_t size;
|
|
uint8_t locations[0];
|
|
};
|
|
|
|
|
|
static void
|
|
tc_call_set_sample_locations(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_sample_locations *p = (struct tc_sample_locations *)payload;
|
|
pipe->set_sample_locations(pipe, p->size, &p->locations[0]);
|
|
}
|
|
|
|
static void
|
|
tc_set_sample_locations(struct pipe_context *_pipe, size_t size, const uint8_t *locations)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_sample_locations *p = (struct tc_sample_locations *)tc_add_sized_call(tc,
|
|
TC_CALL_set_sample_locations,
|
|
tc_payload_size_to_call_slots(sizeof(struct tc_sample_locations) + size));
|
|
|
|
p->size = size;
|
|
memcpy(&p->locations, locations, size);
|
|
}
|
|
|
|
struct tc_scissors {
|
|
ubyte start, count;
|
|
struct pipe_scissor_state slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_set_scissor_states(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_scissors *p = (struct tc_scissors *)payload;
|
|
pipe->set_scissor_states(pipe, p->start, p->count, p->slot);
|
|
}
|
|
|
|
static void
|
|
tc_set_scissor_states(struct pipe_context *_pipe,
|
|
unsigned start, unsigned count,
|
|
const struct pipe_scissor_state *states)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_scissors *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_scissor_states, tc_scissors, count);
|
|
|
|
p->start = start;
|
|
p->count = count;
|
|
memcpy(&p->slot, states, count * sizeof(states[0]));
|
|
}
|
|
|
|
struct tc_viewports {
|
|
ubyte start, count;
|
|
struct pipe_viewport_state slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_set_viewport_states(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_viewports *p = (struct tc_viewports *)payload;
|
|
pipe->set_viewport_states(pipe, p->start, p->count, p->slot);
|
|
}
|
|
|
|
static void
|
|
tc_set_viewport_states(struct pipe_context *_pipe,
|
|
unsigned start, unsigned count,
|
|
const struct pipe_viewport_state *states)
|
|
{
|
|
if (!count)
|
|
return;
|
|
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_viewports *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_viewport_states, tc_viewports, count);
|
|
|
|
p->start = start;
|
|
p->count = count;
|
|
memcpy(&p->slot, states, count * sizeof(states[0]));
|
|
}
|
|
|
|
struct tc_window_rects {
|
|
bool include;
|
|
ubyte count;
|
|
struct pipe_scissor_state slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_set_window_rectangles(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
struct tc_window_rects *p = (struct tc_window_rects *)payload;
|
|
pipe->set_window_rectangles(pipe, p->include, p->count, p->slot);
|
|
}
|
|
|
|
static void
|
|
tc_set_window_rectangles(struct pipe_context *_pipe, bool include,
|
|
unsigned count,
|
|
const struct pipe_scissor_state *rects)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_window_rects *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_window_rectangles, tc_window_rects, count);
|
|
|
|
p->include = include;
|
|
p->count = count;
|
|
memcpy(p->slot, rects, count * sizeof(rects[0]));
|
|
}
|
|
|
|
struct tc_sampler_views {
|
|
ubyte shader, start, count, unbind_num_trailing_slots;
|
|
struct pipe_sampler_view *slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_set_sampler_views(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_sampler_views *p = (struct tc_sampler_views *)payload;
|
|
unsigned count = p->count;
|
|
|
|
pipe->set_sampler_views(pipe, p->shader, p->start, p->count,
|
|
p->unbind_num_trailing_slots, p->slot);
|
|
for (unsigned i = 0; i < count; i++)
|
|
pipe_sampler_view_reference(&p->slot[i], NULL);
|
|
}
|
|
|
|
static void
|
|
tc_set_sampler_views(struct pipe_context *_pipe,
|
|
enum pipe_shader_type shader,
|
|
unsigned start, unsigned count,
|
|
unsigned unbind_num_trailing_slots,
|
|
struct pipe_sampler_view **views)
|
|
{
|
|
if (!count && !unbind_num_trailing_slots)
|
|
return;
|
|
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_sampler_views *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_sampler_views, tc_sampler_views, count);
|
|
|
|
p->shader = shader;
|
|
p->start = start;
|
|
p->count = count;
|
|
p->unbind_num_trailing_slots = unbind_num_trailing_slots;
|
|
|
|
if (views) {
|
|
for (unsigned i = 0; i < count; i++) {
|
|
p->slot[i] = NULL;
|
|
pipe_sampler_view_reference(&p->slot[i], views[i]);
|
|
}
|
|
} else {
|
|
memset(p->slot, 0, count * sizeof(views[0]));
|
|
}
|
|
}
|
|
|
|
struct tc_shader_images {
|
|
ubyte shader, start, count;
|
|
ubyte unbind_num_trailing_slots;
|
|
struct pipe_image_view slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_set_shader_images(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_shader_images *p = (struct tc_shader_images *)payload;
|
|
unsigned count = p->count;
|
|
|
|
if (!p->count) {
|
|
pipe->set_shader_images(pipe, p->shader, p->start, 0,
|
|
p->unbind_num_trailing_slots, NULL);
|
|
return;
|
|
}
|
|
|
|
pipe->set_shader_images(pipe, p->shader, p->start, p->count,
|
|
p->unbind_num_trailing_slots, p->slot);
|
|
|
|
for (unsigned i = 0; i < count; i++)
|
|
pipe_resource_reference(&p->slot[i].resource, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_set_shader_images(struct pipe_context *_pipe,
|
|
enum pipe_shader_type shader,
|
|
unsigned start, unsigned count,
|
|
unsigned unbind_num_trailing_slots,
|
|
const struct pipe_image_view *images)
|
|
{
|
|
if (!count && !unbind_num_trailing_slots)
|
|
return;
|
|
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_shader_images *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_shader_images, tc_shader_images,
|
|
images ? count : 0);
|
|
|
|
p->shader = shader;
|
|
p->start = start;
|
|
|
|
if (images) {
|
|
p->count = count;
|
|
p->unbind_num_trailing_slots = unbind_num_trailing_slots;
|
|
|
|
for (unsigned i = 0; i < count; i++) {
|
|
tc_set_resource_reference(&p->slot[i].resource, images[i].resource);
|
|
|
|
if (images[i].access & PIPE_IMAGE_ACCESS_WRITE &&
|
|
images[i].resource &&
|
|
images[i].resource->target == PIPE_BUFFER) {
|
|
struct threaded_resource *tres =
|
|
threaded_resource(images[i].resource);
|
|
|
|
util_range_add(&tres->b, &tres->valid_buffer_range,
|
|
images[i].u.buf.offset,
|
|
images[i].u.buf.offset + images[i].u.buf.size);
|
|
}
|
|
}
|
|
memcpy(p->slot, images, count * sizeof(images[0]));
|
|
} else {
|
|
p->count = 0;
|
|
p->unbind_num_trailing_slots = count + unbind_num_trailing_slots;
|
|
}
|
|
}
|
|
|
|
struct tc_shader_buffers {
|
|
ubyte shader, start, count;
|
|
bool unbind;
|
|
unsigned writable_bitmask;
|
|
struct pipe_shader_buffer slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_set_shader_buffers(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_shader_buffers *p = (struct tc_shader_buffers *)payload;
|
|
unsigned count = p->count;
|
|
|
|
if (p->unbind) {
|
|
pipe->set_shader_buffers(pipe, p->shader, p->start, p->count, NULL, 0);
|
|
return;
|
|
}
|
|
|
|
pipe->set_shader_buffers(pipe, p->shader, p->start, p->count, p->slot,
|
|
p->writable_bitmask);
|
|
|
|
for (unsigned i = 0; i < count; i++)
|
|
pipe_resource_reference(&p->slot[i].buffer, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_set_shader_buffers(struct pipe_context *_pipe,
|
|
enum pipe_shader_type shader,
|
|
unsigned start, unsigned count,
|
|
const struct pipe_shader_buffer *buffers,
|
|
unsigned writable_bitmask)
|
|
{
|
|
if (!count)
|
|
return;
|
|
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_shader_buffers *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_shader_buffers, tc_shader_buffers,
|
|
buffers ? count : 0);
|
|
|
|
p->shader = shader;
|
|
p->start = start;
|
|
p->count = count;
|
|
p->unbind = buffers == NULL;
|
|
p->writable_bitmask = writable_bitmask;
|
|
|
|
if (buffers) {
|
|
for (unsigned i = 0; i < count; i++) {
|
|
struct pipe_shader_buffer *dst = &p->slot[i];
|
|
const struct pipe_shader_buffer *src = buffers + i;
|
|
|
|
tc_set_resource_reference(&dst->buffer, src->buffer);
|
|
dst->buffer_offset = src->buffer_offset;
|
|
dst->buffer_size = src->buffer_size;
|
|
|
|
if (src->buffer) {
|
|
struct threaded_resource *tres = threaded_resource(src->buffer);
|
|
|
|
util_range_add(&tres->b, &tres->valid_buffer_range,
|
|
src->buffer_offset,
|
|
src->buffer_offset + src->buffer_size);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
struct tc_vertex_buffers {
|
|
ubyte start, count;
|
|
ubyte unbind_num_trailing_slots;
|
|
struct pipe_vertex_buffer slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_set_vertex_buffers(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_vertex_buffers *p = (struct tc_vertex_buffers *)payload;
|
|
unsigned count = p->count;
|
|
|
|
if (!count) {
|
|
pipe->set_vertex_buffers(pipe, p->start, 0,
|
|
p->unbind_num_trailing_slots, false, NULL);
|
|
return;
|
|
}
|
|
|
|
for (unsigned i = 0; i < count; i++)
|
|
tc_assert(!p->slot[i].is_user_buffer);
|
|
|
|
pipe->set_vertex_buffers(pipe, p->start, count,
|
|
p->unbind_num_trailing_slots, true, p->slot);
|
|
}
|
|
|
|
static void
|
|
tc_set_vertex_buffers(struct pipe_context *_pipe,
|
|
unsigned start, unsigned count,
|
|
unsigned unbind_num_trailing_slots,
|
|
bool take_ownership,
|
|
const struct pipe_vertex_buffer *buffers)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
if (!count && !unbind_num_trailing_slots)
|
|
return;
|
|
|
|
if (count && buffers) {
|
|
struct tc_vertex_buffers *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_vertex_buffers, tc_vertex_buffers, count);
|
|
p->start = start;
|
|
p->count = count;
|
|
p->unbind_num_trailing_slots = unbind_num_trailing_slots;
|
|
|
|
if (take_ownership) {
|
|
memcpy(p->slot, buffers, count * sizeof(struct pipe_vertex_buffer));
|
|
} else {
|
|
for (unsigned i = 0; i < count; i++) {
|
|
struct pipe_vertex_buffer *dst = &p->slot[i];
|
|
const struct pipe_vertex_buffer *src = buffers + i;
|
|
|
|
tc_assert(!src->is_user_buffer);
|
|
dst->stride = src->stride;
|
|
dst->is_user_buffer = false;
|
|
tc_set_resource_reference(&dst->buffer.resource,
|
|
src->buffer.resource);
|
|
dst->buffer_offset = src->buffer_offset;
|
|
}
|
|
}
|
|
} else {
|
|
struct tc_vertex_buffers *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_set_vertex_buffers, tc_vertex_buffers, 0);
|
|
p->start = start;
|
|
p->count = 0;
|
|
p->unbind_num_trailing_slots = count + unbind_num_trailing_slots;
|
|
}
|
|
}
|
|
|
|
struct tc_stream_outputs {
|
|
unsigned count;
|
|
struct pipe_stream_output_target *targets[PIPE_MAX_SO_BUFFERS];
|
|
unsigned offsets[PIPE_MAX_SO_BUFFERS];
|
|
};
|
|
|
|
static void
|
|
tc_call_set_stream_output_targets(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_stream_outputs *p = (struct tc_stream_outputs *)payload;
|
|
unsigned count = p->count;
|
|
|
|
pipe->set_stream_output_targets(pipe, count, p->targets, p->offsets);
|
|
for (unsigned i = 0; i < count; i++)
|
|
pipe_so_target_reference(&p->targets[i], NULL);
|
|
}
|
|
|
|
static void
|
|
tc_set_stream_output_targets(struct pipe_context *_pipe,
|
|
unsigned count,
|
|
struct pipe_stream_output_target **tgs,
|
|
const unsigned *offsets)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_stream_outputs *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_set_stream_output_targets,
|
|
tc_stream_outputs);
|
|
|
|
for (unsigned i = 0; i < count; i++) {
|
|
p->targets[i] = NULL;
|
|
pipe_so_target_reference(&p->targets[i], tgs[i]);
|
|
}
|
|
p->count = count;
|
|
memcpy(p->offsets, offsets, count * sizeof(unsigned));
|
|
}
|
|
|
|
static void
|
|
tc_set_compute_resources(struct pipe_context *_pipe, unsigned start,
|
|
unsigned count, struct pipe_surface **resources)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->set_compute_resources(pipe, start, count, resources);
|
|
}
|
|
|
|
static void
|
|
tc_set_global_binding(struct pipe_context *_pipe, unsigned first,
|
|
unsigned count, struct pipe_resource **resources,
|
|
uint32_t **handles)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->set_global_binding(pipe, first, count, resources, handles);
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* views
|
|
*/
|
|
|
|
static struct pipe_surface *
|
|
tc_create_surface(struct pipe_context *_pipe,
|
|
struct pipe_resource *resource,
|
|
const struct pipe_surface *surf_tmpl)
|
|
{
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe;
|
|
struct pipe_surface *view =
|
|
pipe->create_surface(pipe, resource, surf_tmpl);
|
|
|
|
if (view)
|
|
view->context = _pipe;
|
|
return view;
|
|
}
|
|
|
|
static void
|
|
tc_surface_destroy(struct pipe_context *_pipe,
|
|
struct pipe_surface *surf)
|
|
{
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe;
|
|
|
|
pipe->surface_destroy(pipe, surf);
|
|
}
|
|
|
|
static struct pipe_sampler_view *
|
|
tc_create_sampler_view(struct pipe_context *_pipe,
|
|
struct pipe_resource *resource,
|
|
const struct pipe_sampler_view *templ)
|
|
{
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe;
|
|
struct pipe_sampler_view *view =
|
|
pipe->create_sampler_view(pipe, resource, templ);
|
|
|
|
if (view)
|
|
view->context = _pipe;
|
|
return view;
|
|
}
|
|
|
|
static void
|
|
tc_sampler_view_destroy(struct pipe_context *_pipe,
|
|
struct pipe_sampler_view *view)
|
|
{
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe;
|
|
|
|
pipe->sampler_view_destroy(pipe, view);
|
|
}
|
|
|
|
static struct pipe_stream_output_target *
|
|
tc_create_stream_output_target(struct pipe_context *_pipe,
|
|
struct pipe_resource *res,
|
|
unsigned buffer_offset,
|
|
unsigned buffer_size)
|
|
{
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe;
|
|
struct threaded_resource *tres = threaded_resource(res);
|
|
struct pipe_stream_output_target *view;
|
|
|
|
util_range_add(&tres->b, &tres->valid_buffer_range, buffer_offset,
|
|
buffer_offset + buffer_size);
|
|
|
|
view = pipe->create_stream_output_target(pipe, res, buffer_offset,
|
|
buffer_size);
|
|
if (view)
|
|
view->context = _pipe;
|
|
return view;
|
|
}
|
|
|
|
static void
|
|
tc_stream_output_target_destroy(struct pipe_context *_pipe,
|
|
struct pipe_stream_output_target *target)
|
|
{
|
|
struct pipe_context *pipe = threaded_context(_pipe)->pipe;
|
|
|
|
pipe->stream_output_target_destroy(pipe, target);
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* bindless
|
|
*/
|
|
|
|
static uint64_t
|
|
tc_create_texture_handle(struct pipe_context *_pipe,
|
|
struct pipe_sampler_view *view,
|
|
const struct pipe_sampler_state *state)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
return pipe->create_texture_handle(pipe, view, state);
|
|
}
|
|
|
|
static void
|
|
tc_call_delete_texture_handle(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
pipe->delete_texture_handle(pipe, payload->handle);
|
|
}
|
|
|
|
static void
|
|
tc_delete_texture_handle(struct pipe_context *_pipe, uint64_t handle)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
union tc_payload *payload =
|
|
tc_add_small_call(tc, TC_CALL_delete_texture_handle);
|
|
|
|
payload->handle = handle;
|
|
}
|
|
|
|
struct tc_make_texture_handle_resident
|
|
{
|
|
uint64_t handle;
|
|
bool resident;
|
|
};
|
|
|
|
static void
|
|
tc_call_make_texture_handle_resident(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
struct tc_make_texture_handle_resident *p =
|
|
(struct tc_make_texture_handle_resident *)payload;
|
|
|
|
pipe->make_texture_handle_resident(pipe, p->handle, p->resident);
|
|
}
|
|
|
|
static void
|
|
tc_make_texture_handle_resident(struct pipe_context *_pipe, uint64_t handle,
|
|
bool resident)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_make_texture_handle_resident *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_make_texture_handle_resident,
|
|
tc_make_texture_handle_resident);
|
|
|
|
p->handle = handle;
|
|
p->resident = resident;
|
|
}
|
|
|
|
static uint64_t
|
|
tc_create_image_handle(struct pipe_context *_pipe,
|
|
const struct pipe_image_view *image)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
return pipe->create_image_handle(pipe, image);
|
|
}
|
|
|
|
static void
|
|
tc_call_delete_image_handle(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
pipe->delete_image_handle(pipe, payload->handle);
|
|
}
|
|
|
|
static void
|
|
tc_delete_image_handle(struct pipe_context *_pipe, uint64_t handle)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
union tc_payload *payload =
|
|
tc_add_small_call(tc, TC_CALL_delete_image_handle);
|
|
|
|
payload->handle = handle;
|
|
}
|
|
|
|
struct tc_make_image_handle_resident
|
|
{
|
|
uint64_t handle;
|
|
unsigned access;
|
|
bool resident;
|
|
};
|
|
|
|
static void
|
|
tc_call_make_image_handle_resident(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
struct tc_make_image_handle_resident *p =
|
|
(struct tc_make_image_handle_resident *)payload;
|
|
|
|
pipe->make_image_handle_resident(pipe, p->handle, p->access, p->resident);
|
|
}
|
|
|
|
static void
|
|
tc_make_image_handle_resident(struct pipe_context *_pipe, uint64_t handle,
|
|
unsigned access, bool resident)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_make_image_handle_resident *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_make_image_handle_resident,
|
|
tc_make_image_handle_resident);
|
|
|
|
p->handle = handle;
|
|
p->access = access;
|
|
p->resident = resident;
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* transfer
|
|
*/
|
|
|
|
struct tc_replace_buffer_storage {
|
|
struct pipe_resource *dst;
|
|
struct pipe_resource *src;
|
|
tc_replace_buffer_storage_func func;
|
|
};
|
|
|
|
static void
|
|
tc_call_replace_buffer_storage(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
struct tc_replace_buffer_storage *p =
|
|
(struct tc_replace_buffer_storage *)payload;
|
|
|
|
p->func(pipe, p->dst, p->src);
|
|
pipe_resource_reference(&p->dst, NULL);
|
|
pipe_resource_reference(&p->src, NULL);
|
|
}
|
|
|
|
static bool
|
|
tc_invalidate_buffer(struct threaded_context *tc,
|
|
struct threaded_resource *tbuf)
|
|
{
|
|
/* We can't check if the buffer is idle, so we invalidate it
|
|
* unconditionally. */
|
|
struct pipe_screen *screen = tc->base.screen;
|
|
struct pipe_resource *new_buf;
|
|
|
|
/* Shared, pinned, and sparse buffers can't be reallocated. */
|
|
if (tbuf->is_shared ||
|
|
tbuf->is_user_ptr ||
|
|
tbuf->b.flags & PIPE_RESOURCE_FLAG_SPARSE)
|
|
return false;
|
|
|
|
/* Allocate a new one. */
|
|
new_buf = screen->resource_create(screen, &tbuf->b);
|
|
if (!new_buf)
|
|
return false;
|
|
|
|
/* Replace the "latest" pointer. */
|
|
if (tbuf->latest != &tbuf->b)
|
|
pipe_resource_reference(&tbuf->latest, NULL);
|
|
|
|
tbuf->latest = new_buf;
|
|
util_range_set_empty(&tbuf->valid_buffer_range);
|
|
|
|
/* The valid range should point to the original buffer. */
|
|
threaded_resource(new_buf)->base_valid_buffer_range =
|
|
&tbuf->valid_buffer_range;
|
|
|
|
/* Enqueue storage replacement of the original buffer. */
|
|
struct tc_replace_buffer_storage *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_replace_buffer_storage,
|
|
tc_replace_buffer_storage);
|
|
|
|
p->func = tc->replace_buffer_storage;
|
|
tc_set_resource_reference(&p->dst, &tbuf->b);
|
|
tc_set_resource_reference(&p->src, new_buf);
|
|
return true;
|
|
}
|
|
|
|
static unsigned
|
|
tc_improve_map_buffer_flags(struct threaded_context *tc,
|
|
struct threaded_resource *tres, unsigned usage,
|
|
unsigned offset, unsigned size)
|
|
{
|
|
/* Never invalidate inside the driver and never infer "unsynchronized". */
|
|
unsigned tc_flags = TC_TRANSFER_MAP_NO_INVALIDATE |
|
|
TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED;
|
|
|
|
/* Prevent a reentry. */
|
|
if (usage & tc_flags)
|
|
return usage;
|
|
|
|
/* Use the staging upload if it's preferred. */
|
|
if (usage & (PIPE_MAP_DISCARD_RANGE |
|
|
PIPE_MAP_DISCARD_WHOLE_RESOURCE) &&
|
|
!(usage & PIPE_MAP_PERSISTENT) &&
|
|
/* Try not to decrement the counter if it's not positive. Still racy,
|
|
* but it makes it harder to wrap the counter from INT_MIN to INT_MAX. */
|
|
tres->max_forced_staging_uploads > 0 &&
|
|
tc->use_forced_staging_uploads &&
|
|
p_atomic_dec_return(&tres->max_forced_staging_uploads) >= 0) {
|
|
usage &= ~(PIPE_MAP_DISCARD_WHOLE_RESOURCE |
|
|
PIPE_MAP_UNSYNCHRONIZED);
|
|
|
|
return usage | tc_flags | PIPE_MAP_DISCARD_RANGE;
|
|
}
|
|
|
|
/* Sparse buffers can't be mapped directly and can't be reallocated
|
|
* (fully invalidated). That may just be a radeonsi limitation, but
|
|
* the threaded context must obey it with radeonsi.
|
|
*/
|
|
if (tres->b.flags & PIPE_RESOURCE_FLAG_SPARSE) {
|
|
/* We can use DISCARD_RANGE instead of full discard. This is the only
|
|
* fast path for sparse buffers that doesn't need thread synchronization.
|
|
*/
|
|
if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE)
|
|
usage |= PIPE_MAP_DISCARD_RANGE;
|
|
|
|
/* Allow DISCARD_WHOLE_RESOURCE and infering UNSYNCHRONIZED in drivers.
|
|
* The threaded context doesn't do unsychronized mappings and invalida-
|
|
* tions of sparse buffers, therefore a correct driver behavior won't
|
|
* result in an incorrect behavior with the threaded context.
|
|
*/
|
|
return usage;
|
|
}
|
|
|
|
usage |= tc_flags;
|
|
|
|
/* Handle CPU reads trivially. */
|
|
if (usage & PIPE_MAP_READ) {
|
|
if (usage & PIPE_MAP_UNSYNCHRONIZED)
|
|
usage |= TC_TRANSFER_MAP_THREADED_UNSYNC; /* don't sync */
|
|
|
|
/* Drivers aren't allowed to do buffer invalidations. */
|
|
return usage & ~PIPE_MAP_DISCARD_WHOLE_RESOURCE;
|
|
}
|
|
|
|
/* See if the buffer range being mapped has never been initialized,
|
|
* in which case it can be mapped unsynchronized. */
|
|
if (!(usage & PIPE_MAP_UNSYNCHRONIZED) &&
|
|
!tres->is_shared &&
|
|
!util_ranges_intersect(&tres->valid_buffer_range, offset, offset + size))
|
|
usage |= PIPE_MAP_UNSYNCHRONIZED;
|
|
|
|
if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
|
|
/* If discarding the entire range, discard the whole resource instead. */
|
|
if (usage & PIPE_MAP_DISCARD_RANGE &&
|
|
offset == 0 && size == tres->b.width0)
|
|
usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
|
|
|
|
/* Discard the whole resource if needed. */
|
|
if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
|
|
if (tc_invalidate_buffer(tc, tres))
|
|
usage |= PIPE_MAP_UNSYNCHRONIZED;
|
|
else
|
|
usage |= PIPE_MAP_DISCARD_RANGE; /* fallback */
|
|
}
|
|
}
|
|
|
|
/* We won't need this flag anymore. */
|
|
/* TODO: We might not need TC_TRANSFER_MAP_NO_INVALIDATE with this. */
|
|
usage &= ~PIPE_MAP_DISCARD_WHOLE_RESOURCE;
|
|
|
|
/* GL_AMD_pinned_memory and persistent mappings can't use staging
|
|
* buffers. */
|
|
if (usage & (PIPE_MAP_UNSYNCHRONIZED |
|
|
PIPE_MAP_PERSISTENT) ||
|
|
tres->is_user_ptr)
|
|
usage &= ~PIPE_MAP_DISCARD_RANGE;
|
|
|
|
/* Unsychronized buffer mappings don't have to synchronize the thread. */
|
|
if (usage & PIPE_MAP_UNSYNCHRONIZED) {
|
|
usage &= ~PIPE_MAP_DISCARD_RANGE;
|
|
usage |= TC_TRANSFER_MAP_THREADED_UNSYNC; /* notify the driver */
|
|
}
|
|
|
|
return usage;
|
|
}
|
|
|
|
static void *
|
|
tc_transfer_map(struct pipe_context *_pipe,
|
|
struct pipe_resource *resource, unsigned level,
|
|
unsigned usage, const struct pipe_box *box,
|
|
struct pipe_transfer **transfer)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_resource *tres = threaded_resource(resource);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
if (resource->target == PIPE_BUFFER) {
|
|
usage = tc_improve_map_buffer_flags(tc, tres, usage, box->x, box->width);
|
|
|
|
/* Do a staging transfer within the threaded context. The driver should
|
|
* only get resource_copy_region.
|
|
*/
|
|
if (usage & PIPE_MAP_DISCARD_RANGE) {
|
|
struct threaded_transfer *ttrans = slab_alloc(&tc->pool_transfers);
|
|
uint8_t *map;
|
|
|
|
ttrans->staging = NULL;
|
|
|
|
u_upload_alloc(tc->base.stream_uploader, 0,
|
|
box->width + (box->x % tc->map_buffer_alignment),
|
|
tc->map_buffer_alignment, &ttrans->offset,
|
|
&ttrans->staging, (void**)&map);
|
|
if (!map) {
|
|
slab_free(&tc->pool_transfers, ttrans);
|
|
return NULL;
|
|
}
|
|
|
|
tc_set_resource_reference(&ttrans->b.resource, resource);
|
|
ttrans->b.level = 0;
|
|
ttrans->b.usage = usage;
|
|
ttrans->b.box = *box;
|
|
ttrans->b.stride = 0;
|
|
ttrans->b.layer_stride = 0;
|
|
*transfer = &ttrans->b;
|
|
|
|
p_atomic_inc(&tres->pending_staging_uploads);
|
|
util_range_add(resource, &tres->pending_staging_uploads_range,
|
|
box->x, box->x + box->width);
|
|
|
|
return map + (box->x % tc->map_buffer_alignment);
|
|
}
|
|
|
|
if (usage & PIPE_MAP_UNSYNCHRONIZED &&
|
|
p_atomic_read(&tres->pending_staging_uploads) &&
|
|
util_ranges_intersect(&tres->pending_staging_uploads_range, box->x, box->x + box->width)) {
|
|
/* Write conflict detected between a staging transfer and the direct mapping we're
|
|
* going to do. Resolve the conflict by ignoring UNSYNCHRONIZED so the direct mapping
|
|
* will have to wait for the staging transfer completion.
|
|
* Note: The conflict detection is only based on the mapped range, not on the actual
|
|
* written range(s).
|
|
*/
|
|
usage &= ~PIPE_MAP_UNSYNCHRONIZED & ~TC_TRANSFER_MAP_THREADED_UNSYNC;
|
|
tc->use_forced_staging_uploads = false;
|
|
}
|
|
}
|
|
|
|
/* Unsychronized buffer mappings don't have to synchronize the thread. */
|
|
if (!(usage & TC_TRANSFER_MAP_THREADED_UNSYNC)) {
|
|
tc_sync_msg(tc, resource->target != PIPE_BUFFER ? " texture" :
|
|
usage & PIPE_MAP_DISCARD_RANGE ? " discard_range" :
|
|
usage & PIPE_MAP_READ ? " read" : " staging conflict");
|
|
tc_set_driver_thread(tc);
|
|
}
|
|
|
|
tc->bytes_mapped_estimate += box->width;
|
|
|
|
void *ret = pipe->transfer_map(pipe, tres->latest ? tres->latest : resource,
|
|
level, usage, box, transfer);
|
|
|
|
if (!(usage & TC_TRANSFER_MAP_THREADED_UNSYNC))
|
|
tc_clear_driver_thread(tc);
|
|
|
|
return ret;
|
|
}
|
|
|
|
struct tc_transfer_flush_region {
|
|
struct pipe_transfer *transfer;
|
|
struct pipe_box box;
|
|
};
|
|
|
|
static void
|
|
tc_call_transfer_flush_region(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
struct tc_transfer_flush_region *p =
|
|
(struct tc_transfer_flush_region *)payload;
|
|
|
|
pipe->transfer_flush_region(pipe, p->transfer, &p->box);
|
|
}
|
|
|
|
struct tc_resource_copy_region {
|
|
struct pipe_resource *dst;
|
|
unsigned dst_level;
|
|
unsigned dstx, dsty, dstz;
|
|
struct pipe_resource *src;
|
|
unsigned src_level;
|
|
struct pipe_box src_box;
|
|
};
|
|
|
|
static void
|
|
tc_resource_copy_region(struct pipe_context *_pipe,
|
|
struct pipe_resource *dst, unsigned dst_level,
|
|
unsigned dstx, unsigned dsty, unsigned dstz,
|
|
struct pipe_resource *src, unsigned src_level,
|
|
const struct pipe_box *src_box);
|
|
|
|
static void
|
|
tc_buffer_do_flush_region(struct threaded_context *tc,
|
|
struct threaded_transfer *ttrans,
|
|
const struct pipe_box *box)
|
|
{
|
|
struct threaded_resource *tres = threaded_resource(ttrans->b.resource);
|
|
|
|
if (ttrans->staging) {
|
|
struct pipe_box src_box;
|
|
|
|
u_box_1d(ttrans->offset + ttrans->b.box.x % tc->map_buffer_alignment +
|
|
(box->x - ttrans->b.box.x),
|
|
box->width, &src_box);
|
|
|
|
/* Copy the staging buffer into the original one. */
|
|
tc_resource_copy_region(&tc->base, ttrans->b.resource, 0, box->x, 0, 0,
|
|
ttrans->staging, 0, &src_box);
|
|
}
|
|
|
|
util_range_add(&tres->b, tres->base_valid_buffer_range,
|
|
box->x, box->x + box->width);
|
|
}
|
|
|
|
static void
|
|
tc_transfer_flush_region(struct pipe_context *_pipe,
|
|
struct pipe_transfer *transfer,
|
|
const struct pipe_box *rel_box)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_transfer *ttrans = threaded_transfer(transfer);
|
|
struct threaded_resource *tres = threaded_resource(transfer->resource);
|
|
unsigned required_usage = PIPE_MAP_WRITE |
|
|
PIPE_MAP_FLUSH_EXPLICIT;
|
|
|
|
if (tres->b.target == PIPE_BUFFER) {
|
|
if ((transfer->usage & required_usage) == required_usage) {
|
|
struct pipe_box box;
|
|
|
|
u_box_1d(transfer->box.x + rel_box->x, rel_box->width, &box);
|
|
tc_buffer_do_flush_region(tc, ttrans, &box);
|
|
}
|
|
|
|
/* Staging transfers don't send the call to the driver. */
|
|
if (ttrans->staging)
|
|
return;
|
|
}
|
|
|
|
struct tc_transfer_flush_region *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_transfer_flush_region,
|
|
tc_transfer_flush_region);
|
|
p->transfer = transfer;
|
|
p->box = *rel_box;
|
|
}
|
|
|
|
struct tc_transfer_unmap {
|
|
union {
|
|
struct pipe_transfer *transfer;
|
|
struct pipe_resource *resource;
|
|
};
|
|
bool was_staging_transfer;
|
|
};
|
|
|
|
static void
|
|
tc_call_transfer_unmap(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_transfer_unmap *p = (struct tc_transfer_unmap *) payload;
|
|
if (p->was_staging_transfer) {
|
|
struct threaded_resource *tres = threaded_resource(payload->resource);
|
|
/* Nothing to do except keeping track of staging uploads */
|
|
assert(tres->pending_staging_uploads > 0);
|
|
p_atomic_dec(&tres->pending_staging_uploads);
|
|
pipe_resource_reference(&p->resource, NULL);
|
|
return;
|
|
}
|
|
pipe->transfer_unmap(pipe, p->transfer);
|
|
}
|
|
|
|
static void
|
|
tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence,
|
|
unsigned flags);
|
|
|
|
static void
|
|
tc_transfer_unmap(struct pipe_context *_pipe, struct pipe_transfer *transfer)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_transfer *ttrans = threaded_transfer(transfer);
|
|
struct threaded_resource *tres = threaded_resource(transfer->resource);
|
|
|
|
/* PIPE_MAP_THREAD_SAFE is only valid with UNSYNCHRONIZED. It can be
|
|
* called from any thread and bypasses all multithreaded queues.
|
|
*/
|
|
if (transfer->usage & PIPE_MAP_THREAD_SAFE) {
|
|
assert(transfer->usage & PIPE_MAP_UNSYNCHRONIZED);
|
|
assert(!(transfer->usage & (PIPE_MAP_FLUSH_EXPLICIT |
|
|
PIPE_MAP_DISCARD_RANGE)));
|
|
|
|
struct pipe_context *pipe = tc->pipe;
|
|
util_range_add(&tres->b, tres->base_valid_buffer_range,
|
|
transfer->box.x, transfer->box.x + transfer->box.width);
|
|
pipe->transfer_unmap(pipe, transfer);
|
|
return;
|
|
}
|
|
|
|
bool was_staging_transfer = false;
|
|
|
|
if (tres->b.target == PIPE_BUFFER) {
|
|
if (transfer->usage & PIPE_MAP_WRITE &&
|
|
!(transfer->usage & PIPE_MAP_FLUSH_EXPLICIT))
|
|
tc_buffer_do_flush_region(tc, ttrans, &transfer->box);
|
|
|
|
if (ttrans->staging) {
|
|
was_staging_transfer = true;
|
|
|
|
pipe_resource_reference(&ttrans->staging, NULL);
|
|
pipe_resource_reference(&ttrans->b.resource, NULL);
|
|
slab_free(&tc->pool_transfers, ttrans);
|
|
}
|
|
}
|
|
struct tc_transfer_unmap *p = tc_add_struct_typed_call(tc, TC_CALL_transfer_unmap,
|
|
tc_transfer_unmap);
|
|
if (was_staging_transfer) {
|
|
tc_set_resource_reference(&p->resource, &tres->b);
|
|
p->was_staging_transfer = true;
|
|
} else {
|
|
p->transfer = transfer;
|
|
p->was_staging_transfer = false;
|
|
}
|
|
|
|
/* tc_transfer_map directly maps the buffers, but tc_transfer_unmap
|
|
* defers the unmap operation to the batch execution.
|
|
* bytes_mapped_estimate is an estimation of the map/unmap bytes delta
|
|
* and if it goes over an optional limit the current batch is flushed,
|
|
* to reclaim some RAM. */
|
|
if (!ttrans->staging && tc->bytes_mapped_limit &&
|
|
tc->bytes_mapped_estimate > tc->bytes_mapped_limit) {
|
|
tc_flush(_pipe, NULL, PIPE_FLUSH_ASYNC);
|
|
}
|
|
}
|
|
|
|
struct tc_buffer_subdata {
|
|
struct pipe_resource *resource;
|
|
unsigned usage, offset, size;
|
|
char slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_buffer_subdata(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_buffer_subdata *p = (struct tc_buffer_subdata *)payload;
|
|
|
|
pipe->buffer_subdata(pipe, p->resource, p->usage, p->offset, p->size,
|
|
p->slot);
|
|
pipe_resource_reference(&p->resource, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_buffer_subdata(struct pipe_context *_pipe,
|
|
struct pipe_resource *resource,
|
|
unsigned usage, unsigned offset,
|
|
unsigned size, const void *data)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_resource *tres = threaded_resource(resource);
|
|
|
|
if (!size)
|
|
return;
|
|
|
|
usage |= PIPE_MAP_WRITE;
|
|
|
|
/* PIPE_MAP_DIRECTLY supresses implicit DISCARD_RANGE. */
|
|
if (!(usage & PIPE_MAP_DIRECTLY))
|
|
usage |= PIPE_MAP_DISCARD_RANGE;
|
|
|
|
usage = tc_improve_map_buffer_flags(tc, tres, usage, offset, size);
|
|
|
|
/* Unsychronized and big transfers should use transfer_map. Also handle
|
|
* full invalidations, because drivers aren't allowed to do them.
|
|
*/
|
|
if (usage & (PIPE_MAP_UNSYNCHRONIZED |
|
|
PIPE_MAP_DISCARD_WHOLE_RESOURCE) ||
|
|
size > TC_MAX_SUBDATA_BYTES) {
|
|
struct pipe_transfer *transfer;
|
|
struct pipe_box box;
|
|
uint8_t *map = NULL;
|
|
|
|
u_box_1d(offset, size, &box);
|
|
|
|
map = tc_transfer_map(_pipe, resource, 0, usage, &box, &transfer);
|
|
if (map) {
|
|
memcpy(map, data, size);
|
|
tc_transfer_unmap(_pipe, transfer);
|
|
}
|
|
return;
|
|
}
|
|
|
|
util_range_add(&tres->b, &tres->valid_buffer_range, offset, offset + size);
|
|
|
|
/* The upload is small. Enqueue it. */
|
|
struct tc_buffer_subdata *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_buffer_subdata, tc_buffer_subdata, size);
|
|
|
|
tc_set_resource_reference(&p->resource, resource);
|
|
p->usage = usage;
|
|
p->offset = offset;
|
|
p->size = size;
|
|
memcpy(p->slot, data, size);
|
|
}
|
|
|
|
struct tc_texture_subdata {
|
|
struct pipe_resource *resource;
|
|
unsigned level, usage, stride, layer_stride;
|
|
struct pipe_box box;
|
|
char slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_texture_subdata(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_texture_subdata *p = (struct tc_texture_subdata *)payload;
|
|
|
|
pipe->texture_subdata(pipe, p->resource, p->level, p->usage, &p->box,
|
|
p->slot, p->stride, p->layer_stride);
|
|
pipe_resource_reference(&p->resource, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_texture_subdata(struct pipe_context *_pipe,
|
|
struct pipe_resource *resource,
|
|
unsigned level, unsigned usage,
|
|
const struct pipe_box *box,
|
|
const void *data, unsigned stride,
|
|
unsigned layer_stride)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
unsigned size;
|
|
|
|
assert(box->height >= 1);
|
|
assert(box->depth >= 1);
|
|
|
|
size = (box->depth - 1) * layer_stride +
|
|
(box->height - 1) * stride +
|
|
box->width * util_format_get_blocksize(resource->format);
|
|
if (!size)
|
|
return;
|
|
|
|
/* Small uploads can be enqueued, big uploads must sync. */
|
|
if (size <= TC_MAX_SUBDATA_BYTES) {
|
|
struct tc_texture_subdata *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_texture_subdata, tc_texture_subdata, size);
|
|
|
|
tc_set_resource_reference(&p->resource, resource);
|
|
p->level = level;
|
|
p->usage = usage;
|
|
p->box = *box;
|
|
p->stride = stride;
|
|
p->layer_stride = layer_stride;
|
|
memcpy(p->slot, data, size);
|
|
} else {
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
tc_set_driver_thread(tc);
|
|
pipe->texture_subdata(pipe, resource, level, usage, box, data,
|
|
stride, layer_stride);
|
|
tc_clear_driver_thread(tc);
|
|
}
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* miscellaneous
|
|
*/
|
|
|
|
#define TC_FUNC_SYNC_RET0(ret_type, func) \
|
|
static ret_type \
|
|
tc_##func(struct pipe_context *_pipe) \
|
|
{ \
|
|
struct threaded_context *tc = threaded_context(_pipe); \
|
|
struct pipe_context *pipe = tc->pipe; \
|
|
tc_sync(tc); \
|
|
return pipe->func(pipe); \
|
|
}
|
|
|
|
TC_FUNC_SYNC_RET0(enum pipe_reset_status, get_device_reset_status)
|
|
TC_FUNC_SYNC_RET0(uint64_t, get_timestamp)
|
|
|
|
static void
|
|
tc_get_sample_position(struct pipe_context *_pipe,
|
|
unsigned sample_count, unsigned sample_index,
|
|
float *out_value)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->get_sample_position(pipe, sample_count, sample_index,
|
|
out_value);
|
|
}
|
|
|
|
static void
|
|
tc_set_device_reset_callback(struct pipe_context *_pipe,
|
|
const struct pipe_device_reset_callback *cb)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->set_device_reset_callback(pipe, cb);
|
|
}
|
|
|
|
struct tc_string_marker {
|
|
int len;
|
|
char slot[0]; /* more will be allocated if needed */
|
|
};
|
|
|
|
static void
|
|
tc_call_emit_string_marker(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_string_marker *p = (struct tc_string_marker *)payload;
|
|
pipe->emit_string_marker(pipe, p->slot, p->len);
|
|
}
|
|
|
|
static void
|
|
tc_emit_string_marker(struct pipe_context *_pipe,
|
|
const char *string, int len)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
if (len <= TC_MAX_STRING_MARKER_BYTES) {
|
|
struct tc_string_marker *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_emit_string_marker, tc_string_marker, len);
|
|
|
|
memcpy(p->slot, string, len);
|
|
p->len = len;
|
|
} else {
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->emit_string_marker(pipe, string, len);
|
|
}
|
|
}
|
|
|
|
static void
|
|
tc_dump_debug_state(struct pipe_context *_pipe, FILE *stream,
|
|
unsigned flags)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->dump_debug_state(pipe, stream, flags);
|
|
}
|
|
|
|
static void
|
|
tc_set_debug_callback(struct pipe_context *_pipe,
|
|
const struct pipe_debug_callback *cb)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
/* Drop all synchronous debug callbacks. Drivers are expected to be OK
|
|
* with this. shader-db will use an environment variable to disable
|
|
* the threaded context.
|
|
*/
|
|
if (cb && cb->debug_message && !cb->async)
|
|
return;
|
|
|
|
tc_sync(tc);
|
|
pipe->set_debug_callback(pipe, cb);
|
|
}
|
|
|
|
static void
|
|
tc_set_log_context(struct pipe_context *_pipe, struct u_log_context *log)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->set_log_context(pipe, log);
|
|
}
|
|
|
|
static void
|
|
tc_create_fence_fd(struct pipe_context *_pipe,
|
|
struct pipe_fence_handle **fence, int fd,
|
|
enum pipe_fd_type type)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->create_fence_fd(pipe, fence, fd, type);
|
|
}
|
|
|
|
static void
|
|
tc_call_fence_server_sync(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
pipe->fence_server_sync(pipe, payload->fence);
|
|
pipe->screen->fence_reference(pipe->screen, &payload->fence, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_fence_server_sync(struct pipe_context *_pipe,
|
|
struct pipe_fence_handle *fence)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_screen *screen = tc->pipe->screen;
|
|
union tc_payload *payload = tc_add_small_call(tc, TC_CALL_fence_server_sync);
|
|
|
|
payload->fence = NULL;
|
|
screen->fence_reference(screen, &payload->fence, fence);
|
|
}
|
|
|
|
static void
|
|
tc_call_fence_server_signal(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
pipe->fence_server_signal(pipe, payload->fence);
|
|
pipe->screen->fence_reference(pipe->screen, &payload->fence, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_fence_server_signal(struct pipe_context *_pipe,
|
|
struct pipe_fence_handle *fence)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_screen *screen = tc->pipe->screen;
|
|
union tc_payload *payload = tc_add_small_call(tc, TC_CALL_fence_server_signal);
|
|
|
|
payload->fence = NULL;
|
|
screen->fence_reference(screen, &payload->fence, fence);
|
|
}
|
|
|
|
static struct pipe_video_codec *
|
|
tc_create_video_codec(UNUSED struct pipe_context *_pipe,
|
|
UNUSED const struct pipe_video_codec *templ)
|
|
{
|
|
unreachable("Threaded context should not be enabled for video APIs");
|
|
return NULL;
|
|
}
|
|
|
|
static struct pipe_video_buffer *
|
|
tc_create_video_buffer(UNUSED struct pipe_context *_pipe,
|
|
UNUSED const struct pipe_video_buffer *templ)
|
|
{
|
|
unreachable("Threaded context should not be enabled for video APIs");
|
|
return NULL;
|
|
}
|
|
|
|
struct tc_context_param {
|
|
enum pipe_context_param param;
|
|
unsigned value;
|
|
};
|
|
|
|
static void
|
|
tc_call_set_context_param(struct pipe_context *pipe,
|
|
union tc_payload *payload)
|
|
{
|
|
struct tc_context_param *p = (struct tc_context_param*)payload;
|
|
|
|
if (pipe->set_context_param)
|
|
pipe->set_context_param(pipe, p->param, p->value);
|
|
}
|
|
|
|
static void
|
|
tc_set_context_param(struct pipe_context *_pipe,
|
|
enum pipe_context_param param,
|
|
unsigned value)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
if (param == PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE) {
|
|
/* Pin the gallium thread as requested. */
|
|
util_set_thread_affinity(tc->queue.threads[0],
|
|
util_get_cpu_caps()->L3_affinity_mask[value],
|
|
NULL, util_get_cpu_caps()->num_cpu_mask_bits);
|
|
|
|
/* Execute this immediately (without enqueuing).
|
|
* It's required to be thread-safe.
|
|
*/
|
|
struct pipe_context *pipe = tc->pipe;
|
|
if (pipe->set_context_param)
|
|
pipe->set_context_param(pipe, param, value);
|
|
return;
|
|
}
|
|
|
|
if (tc->pipe->set_context_param) {
|
|
struct tc_context_param *payload =
|
|
tc_add_struct_typed_call(tc, TC_CALL_set_context_param,
|
|
tc_context_param);
|
|
|
|
payload->param = param;
|
|
payload->value = value;
|
|
}
|
|
}
|
|
|
|
static void
|
|
tc_call_set_frontend_noop(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
pipe->set_frontend_noop(pipe, payload->boolean);
|
|
}
|
|
|
|
static void
|
|
tc_set_frontend_noop(struct pipe_context *_pipe, bool enable)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
tc_add_small_call(tc, TC_CALL_set_frontend_noop)->boolean = enable;
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* draw, launch, clear, blit, copy, flush
|
|
*/
|
|
|
|
struct tc_flush_payload {
|
|
struct threaded_context *tc;
|
|
struct pipe_fence_handle *fence;
|
|
unsigned flags;
|
|
};
|
|
|
|
static void
|
|
tc_flush_queries(struct threaded_context *tc)
|
|
{
|
|
struct threaded_query *tq, *tmp;
|
|
LIST_FOR_EACH_ENTRY_SAFE(tq, tmp, &tc->unflushed_queries, head_unflushed) {
|
|
list_del(&tq->head_unflushed);
|
|
|
|
/* Memory release semantics: due to a possible race with
|
|
* tc_get_query_result, we must ensure that the linked list changes
|
|
* are visible before setting tq->flushed.
|
|
*/
|
|
p_atomic_set(&tq->flushed, true);
|
|
}
|
|
}
|
|
|
|
static void
|
|
tc_call_flush(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_flush_payload *p = (struct tc_flush_payload *)payload;
|
|
struct pipe_screen *screen = pipe->screen;
|
|
|
|
pipe->flush(pipe, p->fence ? &p->fence : NULL, p->flags);
|
|
screen->fence_reference(screen, &p->fence, NULL);
|
|
|
|
if (!(p->flags & PIPE_FLUSH_DEFERRED))
|
|
tc_flush_queries(p->tc);
|
|
}
|
|
|
|
static void
|
|
tc_flush(struct pipe_context *_pipe, struct pipe_fence_handle **fence,
|
|
unsigned flags)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
struct pipe_screen *screen = pipe->screen;
|
|
bool async = flags & (PIPE_FLUSH_DEFERRED | PIPE_FLUSH_ASYNC);
|
|
|
|
if (async && tc->create_fence) {
|
|
if (fence) {
|
|
struct tc_batch *next = &tc->batch_slots[tc->next];
|
|
|
|
if (!next->token) {
|
|
next->token = malloc(sizeof(*next->token));
|
|
if (!next->token)
|
|
goto out_of_memory;
|
|
|
|
pipe_reference_init(&next->token->ref, 1);
|
|
next->token->tc = tc;
|
|
}
|
|
|
|
screen->fence_reference(screen, fence, tc->create_fence(pipe, next->token));
|
|
if (!*fence)
|
|
goto out_of_memory;
|
|
}
|
|
|
|
struct tc_flush_payload *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_flush, tc_flush_payload);
|
|
p->tc = tc;
|
|
p->fence = fence ? *fence : NULL;
|
|
p->flags = flags | TC_FLUSH_ASYNC;
|
|
|
|
if (!(flags & PIPE_FLUSH_DEFERRED))
|
|
tc_batch_flush(tc);
|
|
return;
|
|
}
|
|
|
|
out_of_memory:
|
|
tc_sync_msg(tc, flags & PIPE_FLUSH_END_OF_FRAME ? "end of frame" :
|
|
flags & PIPE_FLUSH_DEFERRED ? "deferred fence" : "normal");
|
|
|
|
if (!(flags & PIPE_FLUSH_DEFERRED))
|
|
tc_flush_queries(tc);
|
|
tc_set_driver_thread(tc);
|
|
pipe->flush(pipe, fence, flags);
|
|
tc_clear_driver_thread(tc);
|
|
}
|
|
|
|
static void
|
|
tc_call_draw_single(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_draw_single *info = (struct tc_draw_single*)payload;
|
|
|
|
/* u_threaded_context stores start/count in min/max_index for single draws. */
|
|
/* Drivers using u_threaded_context shouldn't use min/max_index. */
|
|
struct pipe_draw_start_count *draw =
|
|
(struct pipe_draw_start_count *)&info->info.min_index;
|
|
STATIC_ASSERT(offsetof(struct pipe_draw_start_count, start) == 0);
|
|
STATIC_ASSERT(offsetof(struct pipe_draw_start_count, count) == 4);
|
|
|
|
info->info.index_bounds_valid = false;
|
|
info->info.has_user_indices = false;
|
|
info->info.take_index_buffer_ownership = false;
|
|
|
|
pipe->draw_vbo(pipe, &info->info, NULL, draw, 1);
|
|
if (info->info.index_size)
|
|
pipe_resource_reference(&info->info.index.resource, NULL);
|
|
}
|
|
|
|
struct tc_draw_indirect {
|
|
struct pipe_draw_info info;
|
|
struct pipe_draw_indirect_info indirect;
|
|
struct pipe_draw_start_count draw;
|
|
};
|
|
|
|
static void
|
|
tc_call_draw_indirect(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_draw_indirect *info = (struct tc_draw_indirect*)payload;
|
|
|
|
info->info.index_bounds_valid = false;
|
|
info->info.take_index_buffer_ownership = false;
|
|
|
|
pipe->draw_vbo(pipe, &info->info, &info->indirect, &info->draw, 1);
|
|
if (info->info.index_size)
|
|
pipe_resource_reference(&info->info.index.resource, NULL);
|
|
|
|
pipe_resource_reference(&info->indirect.buffer, NULL);
|
|
pipe_resource_reference(&info->indirect.indirect_draw_count, NULL);
|
|
pipe_so_target_reference(&info->indirect.count_from_stream_output, NULL);
|
|
}
|
|
|
|
struct tc_draw_multi {
|
|
struct pipe_draw_info info;
|
|
unsigned num_draws;
|
|
struct pipe_draw_start_count slot[]; /* variable-sized array */
|
|
};
|
|
|
|
static void
|
|
tc_call_draw_multi(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_draw_multi *info = (struct tc_draw_multi*)payload;
|
|
|
|
info->info.has_user_indices = false;
|
|
info->info.index_bounds_valid = false;
|
|
info->info.take_index_buffer_ownership = false;
|
|
|
|
pipe->draw_vbo(pipe, &info->info, NULL, info->slot, info->num_draws);
|
|
if (info->info.index_size)
|
|
pipe_resource_reference(&info->info.index.resource, NULL);
|
|
}
|
|
|
|
#define DRAW_INFO_SIZE_WITHOUT_INDEXBUF_AND_MIN_MAX_INDEX \
|
|
offsetof(struct pipe_draw_info, index)
|
|
|
|
void
|
|
tc_draw_vbo(struct pipe_context *_pipe, const struct pipe_draw_info *info,
|
|
const struct pipe_draw_indirect_info *indirect,
|
|
const struct pipe_draw_start_count *draws,
|
|
unsigned num_draws)
|
|
{
|
|
STATIC_ASSERT(DRAW_INFO_SIZE_WITHOUT_INDEXBUF_AND_MIN_MAX_INDEX +
|
|
sizeof(intptr_t) == offsetof(struct pipe_draw_info, min_index));
|
|
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
unsigned index_size = info->index_size;
|
|
bool has_user_indices = info->has_user_indices;
|
|
|
|
if (unlikely(indirect)) {
|
|
assert(!has_user_indices);
|
|
assert(num_draws == 1);
|
|
|
|
struct tc_draw_indirect *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_draw_indirect, tc_draw_indirect);
|
|
if (index_size && !info->take_index_buffer_ownership) {
|
|
tc_set_resource_reference(&p->info.index.resource,
|
|
info->index.resource);
|
|
}
|
|
memcpy(&p->info, info, DRAW_INFO_SIZE_WITHOUT_MIN_MAX_INDEX);
|
|
|
|
tc_set_resource_reference(&p->indirect.buffer, indirect->buffer);
|
|
tc_set_resource_reference(&p->indirect.indirect_draw_count,
|
|
indirect->indirect_draw_count);
|
|
p->indirect.count_from_stream_output = NULL;
|
|
pipe_so_target_reference(&p->indirect.count_from_stream_output,
|
|
indirect->count_from_stream_output);
|
|
memcpy(&p->indirect, indirect, sizeof(*indirect));
|
|
p->draw.start = draws[0].start;
|
|
return;
|
|
}
|
|
|
|
if (num_draws == 1) {
|
|
/* Single draw. */
|
|
if (index_size && has_user_indices) {
|
|
unsigned size = draws[0].count * index_size;
|
|
struct pipe_resource *buffer = NULL;
|
|
unsigned offset;
|
|
|
|
if (!size)
|
|
return;
|
|
|
|
/* This must be done before adding draw_vbo, because it could generate
|
|
* e.g. transfer_unmap and flush partially-uninitialized draw_vbo
|
|
* to the driver if it was done afterwards.
|
|
*/
|
|
u_upload_data(tc->base.stream_uploader, 0, size, 4,
|
|
(uint8_t*)info->index.user + draws[0].start * index_size,
|
|
&offset, &buffer);
|
|
if (unlikely(!buffer))
|
|
return;
|
|
|
|
struct tc_draw_single *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_draw_single, tc_draw_single);
|
|
memcpy(&p->info, info, DRAW_INFO_SIZE_WITHOUT_INDEXBUF_AND_MIN_MAX_INDEX);
|
|
p->info.index.resource = buffer;
|
|
/* u_threaded_context stores start/count in min/max_index for single draws. */
|
|
p->info.min_index = offset >> util_logbase2(index_size);
|
|
p->info.max_index = draws[0].count;
|
|
} else {
|
|
/* Non-indexed call or indexed with a real index buffer. */
|
|
struct tc_draw_single *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_draw_single, tc_draw_single);
|
|
if (index_size && !info->take_index_buffer_ownership) {
|
|
tc_set_resource_reference(&p->info.index.resource,
|
|
info->index.resource);
|
|
}
|
|
memcpy(&p->info, info, DRAW_INFO_SIZE_WITHOUT_MIN_MAX_INDEX);
|
|
/* u_threaded_context stores start/count in min/max_index for single draws. */
|
|
p->info.min_index = draws[0].start;
|
|
p->info.max_index = draws[0].count;
|
|
}
|
|
return;
|
|
}
|
|
|
|
const int draw_overhead_bytes = offsetof(struct tc_call, payload) + sizeof(struct tc_draw_multi);
|
|
const int one_draw_payload_bytes = sizeof(((struct tc_draw_multi*)NULL)->slot[0]);
|
|
const int slots_for_one_draw = DIV_ROUND_UP(draw_overhead_bytes + one_draw_payload_bytes,
|
|
sizeof(struct tc_call));
|
|
/* Multi draw. */
|
|
if (index_size && has_user_indices) {
|
|
struct pipe_resource *buffer = NULL;
|
|
unsigned buffer_offset, total_count = 0;
|
|
unsigned index_size_shift = util_logbase2(index_size);
|
|
uint8_t *ptr = NULL;
|
|
|
|
/* Get the total count. */
|
|
for (unsigned i = 0; i < num_draws; i++)
|
|
total_count += draws[i].count;
|
|
|
|
if (!total_count)
|
|
return;
|
|
|
|
/* Allocate space for all index buffers.
|
|
*
|
|
* This must be done before adding draw_vbo, because it could generate
|
|
* e.g. transfer_unmap and flush partially-uninitialized draw_vbo
|
|
* to the driver if it was done afterwards.
|
|
*/
|
|
u_upload_alloc(tc->base.stream_uploader, 0,
|
|
total_count << index_size_shift, 4,
|
|
&buffer_offset, &buffer, (void**)&ptr);
|
|
if (unlikely(!buffer))
|
|
return;
|
|
|
|
int total_offset = 0;
|
|
while (num_draws) {
|
|
struct tc_batch *next = &tc->batch_slots[tc->next];
|
|
|
|
int nb_slots_left = TC_CALLS_PER_BATCH - next->num_total_call_slots;
|
|
/* If there isn't enough place for one draw, try to fill the next one */
|
|
if (nb_slots_left < slots_for_one_draw)
|
|
nb_slots_left = TC_CALLS_PER_BATCH;
|
|
const int size_left_bytes = nb_slots_left * sizeof(struct tc_call);
|
|
|
|
/* How many draws can we fit in the current batch */
|
|
const int dr = MIN2(num_draws, (size_left_bytes - draw_overhead_bytes) / one_draw_payload_bytes);
|
|
|
|
struct tc_draw_multi *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_draw_multi, tc_draw_multi,
|
|
dr);
|
|
memcpy(&p->info, info, DRAW_INFO_SIZE_WITHOUT_INDEXBUF_AND_MIN_MAX_INDEX);
|
|
p->info.index.resource = buffer;
|
|
p->num_draws = dr;
|
|
|
|
/* Upload index buffers. */
|
|
for (unsigned i = 0, offset = 0; i < dr; i++) {
|
|
unsigned count = draws[i + total_offset].count;
|
|
|
|
if (!count) {
|
|
p->slot[i].start = 0;
|
|
p->slot[i].count = 0;
|
|
continue;
|
|
}
|
|
|
|
unsigned size = count << index_size_shift;
|
|
memcpy(ptr + offset,
|
|
(uint8_t*)info->index.user +
|
|
(draws[i + total_offset].start << index_size_shift), size);
|
|
p->slot[i].start = (buffer_offset + offset) >> index_size_shift;
|
|
p->slot[i].count = count;
|
|
offset += size;
|
|
}
|
|
|
|
total_offset += dr;
|
|
num_draws -= dr;
|
|
}
|
|
} else {
|
|
int total_offset = 0;
|
|
bool take_index_buffer_ownership = info->take_index_buffer_ownership;
|
|
while (num_draws) {
|
|
struct tc_batch *next = &tc->batch_slots[tc->next];
|
|
|
|
int nb_slots_left = TC_CALLS_PER_BATCH - next->num_total_call_slots;
|
|
/* If there isn't enough place for one draw, try to fill the next one */
|
|
if (nb_slots_left < slots_for_one_draw)
|
|
nb_slots_left = TC_CALLS_PER_BATCH;
|
|
const int size_left_bytes = nb_slots_left * sizeof(struct tc_call);
|
|
|
|
/* How many draws can we fit in the current batch */
|
|
const int dr = MIN2(num_draws, (size_left_bytes - draw_overhead_bytes) / one_draw_payload_bytes);
|
|
|
|
/* Non-indexed call or indexed with a real index buffer. */
|
|
struct tc_draw_multi *p =
|
|
tc_add_slot_based_call(tc, TC_CALL_draw_multi, tc_draw_multi,
|
|
dr);
|
|
if (index_size && !take_index_buffer_ownership) {
|
|
tc_set_resource_reference(&p->info.index.resource,
|
|
info->index.resource);
|
|
}
|
|
take_index_buffer_ownership = false;
|
|
memcpy(&p->info, info, DRAW_INFO_SIZE_WITHOUT_MIN_MAX_INDEX);
|
|
p->num_draws = dr;
|
|
memcpy(p->slot, &draws[total_offset], sizeof(draws[0]) * dr);
|
|
num_draws -= dr;
|
|
|
|
total_offset += dr;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
tc_call_launch_grid(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct pipe_grid_info *p = (struct pipe_grid_info *)payload;
|
|
|
|
pipe->launch_grid(pipe, p);
|
|
pipe_resource_reference(&p->indirect, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_launch_grid(struct pipe_context *_pipe,
|
|
const struct pipe_grid_info *info)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_grid_info *p = tc_add_struct_typed_call(tc, TC_CALL_launch_grid,
|
|
pipe_grid_info);
|
|
assert(info->input == NULL);
|
|
|
|
tc_set_resource_reference(&p->indirect, info->indirect);
|
|
memcpy(p, info, sizeof(*info));
|
|
}
|
|
|
|
static void
|
|
tc_call_resource_copy_region(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_resource_copy_region *p = (struct tc_resource_copy_region *)payload;
|
|
|
|
pipe->resource_copy_region(pipe, p->dst, p->dst_level, p->dstx, p->dsty,
|
|
p->dstz, p->src, p->src_level, &p->src_box);
|
|
pipe_resource_reference(&p->dst, NULL);
|
|
pipe_resource_reference(&p->src, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_resource_copy_region(struct pipe_context *_pipe,
|
|
struct pipe_resource *dst, unsigned dst_level,
|
|
unsigned dstx, unsigned dsty, unsigned dstz,
|
|
struct pipe_resource *src, unsigned src_level,
|
|
const struct pipe_box *src_box)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_resource *tdst = threaded_resource(dst);
|
|
struct tc_resource_copy_region *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_resource_copy_region,
|
|
tc_resource_copy_region);
|
|
|
|
tc_set_resource_reference(&p->dst, dst);
|
|
p->dst_level = dst_level;
|
|
p->dstx = dstx;
|
|
p->dsty = dsty;
|
|
p->dstz = dstz;
|
|
tc_set_resource_reference(&p->src, src);
|
|
p->src_level = src_level;
|
|
p->src_box = *src_box;
|
|
|
|
if (dst->target == PIPE_BUFFER)
|
|
util_range_add(&tdst->b, &tdst->valid_buffer_range,
|
|
dstx, dstx + src_box->width);
|
|
}
|
|
|
|
static void
|
|
tc_call_blit(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct pipe_blit_info *blit = (struct pipe_blit_info*)payload;
|
|
|
|
pipe->blit(pipe, blit);
|
|
pipe_resource_reference(&blit->dst.resource, NULL);
|
|
pipe_resource_reference(&blit->src.resource, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_blit(struct pipe_context *_pipe, const struct pipe_blit_info *info)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_blit_info *blit =
|
|
tc_add_struct_typed_call(tc, TC_CALL_blit, pipe_blit_info);
|
|
|
|
tc_set_resource_reference(&blit->dst.resource, info->dst.resource);
|
|
tc_set_resource_reference(&blit->src.resource, info->src.resource);
|
|
memcpy(blit, info, sizeof(*info));
|
|
}
|
|
|
|
struct tc_generate_mipmap {
|
|
struct pipe_resource *res;
|
|
enum pipe_format format;
|
|
unsigned base_level;
|
|
unsigned last_level;
|
|
unsigned first_layer;
|
|
unsigned last_layer;
|
|
};
|
|
|
|
static void
|
|
tc_call_generate_mipmap(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_generate_mipmap *p = (struct tc_generate_mipmap *)payload;
|
|
ASSERTED bool result = pipe->generate_mipmap(pipe, p->res, p->format,
|
|
p->base_level,
|
|
p->last_level,
|
|
p->first_layer,
|
|
p->last_layer);
|
|
assert(result);
|
|
pipe_resource_reference(&p->res, NULL);
|
|
}
|
|
|
|
static bool
|
|
tc_generate_mipmap(struct pipe_context *_pipe,
|
|
struct pipe_resource *res,
|
|
enum pipe_format format,
|
|
unsigned base_level,
|
|
unsigned last_level,
|
|
unsigned first_layer,
|
|
unsigned last_layer)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
struct pipe_screen *screen = pipe->screen;
|
|
unsigned bind = PIPE_BIND_SAMPLER_VIEW;
|
|
|
|
if (util_format_is_depth_or_stencil(format))
|
|
bind = PIPE_BIND_DEPTH_STENCIL;
|
|
else
|
|
bind = PIPE_BIND_RENDER_TARGET;
|
|
|
|
if (!screen->is_format_supported(screen, format, res->target,
|
|
res->nr_samples, res->nr_storage_samples,
|
|
bind))
|
|
return false;
|
|
|
|
struct tc_generate_mipmap *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_generate_mipmap, tc_generate_mipmap);
|
|
|
|
tc_set_resource_reference(&p->res, res);
|
|
p->format = format;
|
|
p->base_level = base_level;
|
|
p->last_level = last_level;
|
|
p->first_layer = first_layer;
|
|
p->last_layer = last_layer;
|
|
return true;
|
|
}
|
|
|
|
static void
|
|
tc_call_flush_resource(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
pipe->flush_resource(pipe, payload->resource);
|
|
pipe_resource_reference(&payload->resource, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_flush_resource(struct pipe_context *_pipe,
|
|
struct pipe_resource *resource)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
union tc_payload *payload = tc_add_small_call(tc, TC_CALL_flush_resource);
|
|
|
|
tc_set_resource_reference(&payload->resource, resource);
|
|
}
|
|
|
|
static void
|
|
tc_call_invalidate_resource(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
pipe->invalidate_resource(pipe, payload->resource);
|
|
pipe_resource_reference(&payload->resource, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_invalidate_resource(struct pipe_context *_pipe,
|
|
struct pipe_resource *resource)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
if (resource->target == PIPE_BUFFER) {
|
|
tc_invalidate_buffer(tc, threaded_resource(resource));
|
|
return;
|
|
}
|
|
|
|
union tc_payload *payload = tc_add_small_call(tc, TC_CALL_invalidate_resource);
|
|
tc_set_resource_reference(&payload->resource, resource);
|
|
}
|
|
|
|
struct tc_clear {
|
|
unsigned buffers;
|
|
struct pipe_scissor_state scissor_state;
|
|
union pipe_color_union color;
|
|
double depth;
|
|
unsigned stencil;
|
|
bool scissor_state_set;
|
|
};
|
|
|
|
static void
|
|
tc_call_clear(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_clear *p = (struct tc_clear *)payload;
|
|
pipe->clear(pipe, p->buffers, p->scissor_state_set ? &p->scissor_state : NULL, &p->color, p->depth, p->stencil);
|
|
}
|
|
|
|
static void
|
|
tc_clear(struct pipe_context *_pipe, unsigned buffers, const struct pipe_scissor_state *scissor_state,
|
|
const union pipe_color_union *color, double depth,
|
|
unsigned stencil)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_clear *p = tc_add_struct_typed_call(tc, TC_CALL_clear, tc_clear);
|
|
|
|
p->buffers = buffers;
|
|
if (scissor_state)
|
|
p->scissor_state = *scissor_state;
|
|
p->scissor_state_set = !!scissor_state;
|
|
p->color = *color;
|
|
p->depth = depth;
|
|
p->stencil = stencil;
|
|
}
|
|
|
|
static void
|
|
tc_clear_render_target(struct pipe_context *_pipe,
|
|
struct pipe_surface *dst,
|
|
const union pipe_color_union *color,
|
|
unsigned dstx, unsigned dsty,
|
|
unsigned width, unsigned height,
|
|
bool render_condition_enabled)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->clear_render_target(pipe, dst, color, dstx, dsty, width, height,
|
|
render_condition_enabled);
|
|
}
|
|
|
|
static void
|
|
tc_clear_depth_stencil(struct pipe_context *_pipe,
|
|
struct pipe_surface *dst, unsigned clear_flags,
|
|
double depth, unsigned stencil, unsigned dstx,
|
|
unsigned dsty, unsigned width, unsigned height,
|
|
bool render_condition_enabled)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
tc_sync(tc);
|
|
pipe->clear_depth_stencil(pipe, dst, clear_flags, depth, stencil,
|
|
dstx, dsty, width, height,
|
|
render_condition_enabled);
|
|
}
|
|
|
|
struct tc_clear_buffer {
|
|
struct pipe_resource *res;
|
|
unsigned offset;
|
|
unsigned size;
|
|
char clear_value[16];
|
|
int clear_value_size;
|
|
};
|
|
|
|
static void
|
|
tc_call_clear_buffer(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_clear_buffer *p = (struct tc_clear_buffer *)payload;
|
|
|
|
pipe->clear_buffer(pipe, p->res, p->offset, p->size, p->clear_value,
|
|
p->clear_value_size);
|
|
pipe_resource_reference(&p->res, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_clear_buffer(struct pipe_context *_pipe, struct pipe_resource *res,
|
|
unsigned offset, unsigned size,
|
|
const void *clear_value, int clear_value_size)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct threaded_resource *tres = threaded_resource(res);
|
|
struct tc_clear_buffer *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_clear_buffer, tc_clear_buffer);
|
|
|
|
tc_set_resource_reference(&p->res, res);
|
|
p->offset = offset;
|
|
p->size = size;
|
|
memcpy(p->clear_value, clear_value, clear_value_size);
|
|
p->clear_value_size = clear_value_size;
|
|
|
|
util_range_add(&tres->b, &tres->valid_buffer_range, offset, offset + size);
|
|
}
|
|
|
|
struct tc_clear_texture {
|
|
struct pipe_resource *res;
|
|
unsigned level;
|
|
struct pipe_box box;
|
|
char data[16];
|
|
};
|
|
|
|
static void
|
|
tc_call_clear_texture(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_clear_texture *p = (struct tc_clear_texture *)payload;
|
|
|
|
pipe->clear_texture(pipe, p->res, p->level, &p->box, p->data);
|
|
pipe_resource_reference(&p->res, NULL);
|
|
}
|
|
|
|
static void
|
|
tc_clear_texture(struct pipe_context *_pipe, struct pipe_resource *res,
|
|
unsigned level, const struct pipe_box *box, const void *data)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_clear_texture *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_clear_texture, tc_clear_texture);
|
|
|
|
tc_set_resource_reference(&p->res, res);
|
|
p->level = level;
|
|
p->box = *box;
|
|
memcpy(p->data, data,
|
|
util_format_get_blocksize(res->format));
|
|
}
|
|
|
|
struct tc_resource_commit {
|
|
struct pipe_resource *res;
|
|
unsigned level;
|
|
struct pipe_box box;
|
|
bool commit;
|
|
};
|
|
|
|
static void
|
|
tc_call_resource_commit(struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_resource_commit *p = (struct tc_resource_commit *)payload;
|
|
|
|
pipe->resource_commit(pipe, p->res, p->level, &p->box, p->commit);
|
|
pipe_resource_reference(&p->res, NULL);
|
|
}
|
|
|
|
static bool
|
|
tc_resource_commit(struct pipe_context *_pipe, struct pipe_resource *res,
|
|
unsigned level, struct pipe_box *box, bool commit)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct tc_resource_commit *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_resource_commit, tc_resource_commit);
|
|
|
|
tc_set_resource_reference(&p->res, res);
|
|
p->level = level;
|
|
p->box = *box;
|
|
p->commit = commit;
|
|
return true; /* we don't care about the return value for this call */
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* callback
|
|
*/
|
|
|
|
struct tc_callback_payload {
|
|
void (*fn)(void *data);
|
|
void *data;
|
|
};
|
|
|
|
static void
|
|
tc_call_callback(UNUSED struct pipe_context *pipe, union tc_payload *payload)
|
|
{
|
|
struct tc_callback_payload *p = (struct tc_callback_payload *)payload;
|
|
|
|
p->fn(p->data);
|
|
}
|
|
|
|
static void
|
|
tc_callback(struct pipe_context *_pipe, void (*fn)(void *), void *data,
|
|
bool asap)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
|
|
if (asap && tc_is_sync(tc)) {
|
|
fn(data);
|
|
return;
|
|
}
|
|
|
|
struct tc_callback_payload *p =
|
|
tc_add_struct_typed_call(tc, TC_CALL_callback, tc_callback_payload);
|
|
p->fn = fn;
|
|
p->data = data;
|
|
}
|
|
|
|
|
|
/********************************************************************
|
|
* create & destroy
|
|
*/
|
|
|
|
static void
|
|
tc_destroy(struct pipe_context *_pipe)
|
|
{
|
|
struct threaded_context *tc = threaded_context(_pipe);
|
|
struct pipe_context *pipe = tc->pipe;
|
|
|
|
if (tc->base.const_uploader &&
|
|
tc->base.stream_uploader != tc->base.const_uploader)
|
|
u_upload_destroy(tc->base.const_uploader);
|
|
|
|
if (tc->base.stream_uploader)
|
|
u_upload_destroy(tc->base.stream_uploader);
|
|
|
|
tc_sync(tc);
|
|
|
|
if (util_queue_is_initialized(&tc->queue)) {
|
|
util_queue_destroy(&tc->queue);
|
|
|
|
for (unsigned i = 0; i < TC_MAX_BATCHES; i++) {
|
|
util_queue_fence_destroy(&tc->batch_slots[i].fence);
|
|
assert(!tc->batch_slots[i].token);
|
|
}
|
|
}
|
|
|
|
slab_destroy_child(&tc->pool_transfers);
|
|
assert(tc->batch_slots[tc->next].num_total_call_slots == 0);
|
|
pipe->destroy(pipe);
|
|
os_free_aligned(tc);
|
|
}
|
|
|
|
static const tc_execute execute_func[TC_NUM_CALLS] = {
|
|
#define CALL(name) tc_call_##name,
|
|
#include "u_threaded_context_calls.h"
|
|
#undef CALL
|
|
};
|
|
|
|
/**
|
|
* Wrap an existing pipe_context into a threaded_context.
|
|
*
|
|
* \param pipe pipe_context to wrap
|
|
* \param parent_transfer_pool parent slab pool set up for creating pipe_-
|
|
* transfer objects; the driver should have one
|
|
* in pipe_screen.
|
|
* \param replace_buffer callback for replacing a pipe_resource's storage
|
|
* with another pipe_resource's storage.
|
|
* \param out if successful, the threaded_context will be returned here in
|
|
* addition to the return value if "out" != NULL
|
|
*/
|
|
struct pipe_context *
|
|
threaded_context_create(struct pipe_context *pipe,
|
|
struct slab_parent_pool *parent_transfer_pool,
|
|
tc_replace_buffer_storage_func replace_buffer,
|
|
tc_create_fence_func create_fence,
|
|
struct threaded_context **out)
|
|
{
|
|
struct threaded_context *tc;
|
|
|
|
STATIC_ASSERT(sizeof(union tc_payload) <= 8);
|
|
STATIC_ASSERT(sizeof(struct tc_call) <= 16);
|
|
|
|
if (!pipe)
|
|
return NULL;
|
|
|
|
util_cpu_detect();
|
|
|
|
if (!debug_get_bool_option("GALLIUM_THREAD", util_get_cpu_caps()->nr_cpus > 1))
|
|
return pipe;
|
|
|
|
tc = os_malloc_aligned(sizeof(struct threaded_context), 16);
|
|
if (!tc) {
|
|
pipe->destroy(pipe);
|
|
return NULL;
|
|
}
|
|
memset(tc, 0, sizeof(*tc));
|
|
|
|
pipe = trace_context_create_threaded(pipe->screen, pipe);
|
|
|
|
assert((uintptr_t)tc % 16 == 0);
|
|
/* These should be static asserts, but they don't work with MSVC */
|
|
assert(offsetof(struct threaded_context, batch_slots) % 16 == 0);
|
|
assert(offsetof(struct threaded_context, batch_slots[0].call) % 16 == 0);
|
|
assert(offsetof(struct threaded_context, batch_slots[0].call[1]) % 16 == 0);
|
|
assert(offsetof(struct threaded_context, batch_slots[1].call) % 16 == 0);
|
|
|
|
/* The driver context isn't wrapped, so set its "priv" to NULL. */
|
|
pipe->priv = NULL;
|
|
|
|
tc->pipe = pipe;
|
|
tc->replace_buffer_storage = replace_buffer;
|
|
tc->create_fence = create_fence;
|
|
tc->map_buffer_alignment =
|
|
pipe->screen->get_param(pipe->screen, PIPE_CAP_MIN_MAP_BUFFER_ALIGNMENT);
|
|
tc->ubo_alignment =
|
|
MAX2(pipe->screen->get_param(pipe->screen, PIPE_CAP_CONSTANT_BUFFER_OFFSET_ALIGNMENT), 64);
|
|
tc->base.priv = pipe; /* priv points to the wrapped driver context */
|
|
tc->base.screen = pipe->screen;
|
|
tc->base.destroy = tc_destroy;
|
|
tc->base.callback = tc_callback;
|
|
|
|
tc->base.stream_uploader = u_upload_clone(&tc->base, pipe->stream_uploader);
|
|
if (pipe->stream_uploader == pipe->const_uploader)
|
|
tc->base.const_uploader = tc->base.stream_uploader;
|
|
else
|
|
tc->base.const_uploader = u_upload_clone(&tc->base, pipe->const_uploader);
|
|
|
|
if (!tc->base.stream_uploader || !tc->base.const_uploader)
|
|
goto fail;
|
|
|
|
tc->use_forced_staging_uploads = true;
|
|
|
|
/* The queue size is the number of batches "waiting". Batches are removed
|
|
* from the queue before being executed, so keep one tc_batch slot for that
|
|
* execution. Also, keep one unused slot for an unflushed batch.
|
|
*/
|
|
if (!util_queue_init(&tc->queue, "gdrv", TC_MAX_BATCHES - 2, 1, 0))
|
|
goto fail;
|
|
|
|
for (unsigned i = 0; i < TC_MAX_BATCHES; i++) {
|
|
tc->batch_slots[i].sentinel = TC_SENTINEL;
|
|
tc->batch_slots[i].tc = tc;
|
|
util_queue_fence_init(&tc->batch_slots[i].fence);
|
|
}
|
|
|
|
list_inithead(&tc->unflushed_queries);
|
|
|
|
slab_create_child(&tc->pool_transfers, parent_transfer_pool);
|
|
|
|
tc->base.set_context_param = tc_set_context_param; /* always set this */
|
|
|
|
#define CTX_INIT(_member) \
|
|
tc->base._member = tc->pipe->_member ? tc_##_member : NULL
|
|
|
|
CTX_INIT(flush);
|
|
CTX_INIT(draw_vbo);
|
|
CTX_INIT(launch_grid);
|
|
CTX_INIT(resource_copy_region);
|
|
CTX_INIT(blit);
|
|
CTX_INIT(clear);
|
|
CTX_INIT(clear_render_target);
|
|
CTX_INIT(clear_depth_stencil);
|
|
CTX_INIT(clear_buffer);
|
|
CTX_INIT(clear_texture);
|
|
CTX_INIT(flush_resource);
|
|
CTX_INIT(generate_mipmap);
|
|
CTX_INIT(render_condition);
|
|
CTX_INIT(create_query);
|
|
CTX_INIT(create_batch_query);
|
|
CTX_INIT(destroy_query);
|
|
CTX_INIT(begin_query);
|
|
CTX_INIT(end_query);
|
|
CTX_INIT(get_query_result);
|
|
CTX_INIT(get_query_result_resource);
|
|
CTX_INIT(set_active_query_state);
|
|
CTX_INIT(create_blend_state);
|
|
CTX_INIT(bind_blend_state);
|
|
CTX_INIT(delete_blend_state);
|
|
CTX_INIT(create_sampler_state);
|
|
CTX_INIT(bind_sampler_states);
|
|
CTX_INIT(delete_sampler_state);
|
|
CTX_INIT(create_rasterizer_state);
|
|
CTX_INIT(bind_rasterizer_state);
|
|
CTX_INIT(delete_rasterizer_state);
|
|
CTX_INIT(create_depth_stencil_alpha_state);
|
|
CTX_INIT(bind_depth_stencil_alpha_state);
|
|
CTX_INIT(delete_depth_stencil_alpha_state);
|
|
CTX_INIT(create_fs_state);
|
|
CTX_INIT(bind_fs_state);
|
|
CTX_INIT(delete_fs_state);
|
|
CTX_INIT(create_vs_state);
|
|
CTX_INIT(bind_vs_state);
|
|
CTX_INIT(delete_vs_state);
|
|
CTX_INIT(create_gs_state);
|
|
CTX_INIT(bind_gs_state);
|
|
CTX_INIT(delete_gs_state);
|
|
CTX_INIT(create_tcs_state);
|
|
CTX_INIT(bind_tcs_state);
|
|
CTX_INIT(delete_tcs_state);
|
|
CTX_INIT(create_tes_state);
|
|
CTX_INIT(bind_tes_state);
|
|
CTX_INIT(delete_tes_state);
|
|
CTX_INIT(create_compute_state);
|
|
CTX_INIT(bind_compute_state);
|
|
CTX_INIT(delete_compute_state);
|
|
CTX_INIT(create_vertex_elements_state);
|
|
CTX_INIT(bind_vertex_elements_state);
|
|
CTX_INIT(delete_vertex_elements_state);
|
|
CTX_INIT(set_blend_color);
|
|
CTX_INIT(set_stencil_ref);
|
|
CTX_INIT(set_sample_mask);
|
|
CTX_INIT(set_min_samples);
|
|
CTX_INIT(set_clip_state);
|
|
CTX_INIT(set_constant_buffer);
|
|
CTX_INIT(set_inlinable_constants);
|
|
CTX_INIT(set_framebuffer_state);
|
|
CTX_INIT(set_polygon_stipple);
|
|
CTX_INIT(set_sample_locations);
|
|
CTX_INIT(set_scissor_states);
|
|
CTX_INIT(set_viewport_states);
|
|
CTX_INIT(set_window_rectangles);
|
|
CTX_INIT(set_sampler_views);
|
|
CTX_INIT(set_tess_state);
|
|
CTX_INIT(set_shader_buffers);
|
|
CTX_INIT(set_shader_images);
|
|
CTX_INIT(set_vertex_buffers);
|
|
CTX_INIT(create_stream_output_target);
|
|
CTX_INIT(stream_output_target_destroy);
|
|
CTX_INIT(set_stream_output_targets);
|
|
CTX_INIT(create_sampler_view);
|
|
CTX_INIT(sampler_view_destroy);
|
|
CTX_INIT(create_surface);
|
|
CTX_INIT(surface_destroy);
|
|
CTX_INIT(transfer_map);
|
|
CTX_INIT(transfer_flush_region);
|
|
CTX_INIT(transfer_unmap);
|
|
CTX_INIT(buffer_subdata);
|
|
CTX_INIT(texture_subdata);
|
|
CTX_INIT(texture_barrier);
|
|
CTX_INIT(memory_barrier);
|
|
CTX_INIT(resource_commit);
|
|
CTX_INIT(create_video_codec);
|
|
CTX_INIT(create_video_buffer);
|
|
CTX_INIT(set_compute_resources);
|
|
CTX_INIT(set_global_binding);
|
|
CTX_INIT(get_sample_position);
|
|
CTX_INIT(invalidate_resource);
|
|
CTX_INIT(get_device_reset_status);
|
|
CTX_INIT(set_device_reset_callback);
|
|
CTX_INIT(dump_debug_state);
|
|
CTX_INIT(set_log_context);
|
|
CTX_INIT(emit_string_marker);
|
|
CTX_INIT(set_debug_callback);
|
|
CTX_INIT(create_fence_fd);
|
|
CTX_INIT(fence_server_sync);
|
|
CTX_INIT(fence_server_signal);
|
|
CTX_INIT(get_timestamp);
|
|
CTX_INIT(create_texture_handle);
|
|
CTX_INIT(delete_texture_handle);
|
|
CTX_INIT(make_texture_handle_resident);
|
|
CTX_INIT(create_image_handle);
|
|
CTX_INIT(delete_image_handle);
|
|
CTX_INIT(make_image_handle_resident);
|
|
CTX_INIT(set_frontend_noop);
|
|
#undef CTX_INIT
|
|
|
|
if (out)
|
|
*out = tc;
|
|
|
|
return &tc->base;
|
|
|
|
fail:
|
|
tc_destroy(&tc->base);
|
|
return NULL;
|
|
}
|