From c4e5beef076d42b66b7fefd4b74a759d69bd9f6a Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Fri, 5 Mar 2021 11:36:32 -0800 Subject: [PATCH] freedreno: threaded_context async flush support Signed-off-by: Rob Clark Part-of: --- .../drivers/freedreno/freedreno_context.c | 45 +++++++-- .../drivers/freedreno/freedreno_fence.c | 99 +++++++++++++++++-- .../drivers/freedreno/freedreno_fence.h | 7 ++ 3 files changed, 137 insertions(+), 14 deletions(-) diff --git a/src/gallium/drivers/freedreno/freedreno_context.c b/src/gallium/drivers/freedreno/freedreno_context.c index 49520e969d3..7e6f6109700 100644 --- a/src/gallium/drivers/freedreno/freedreno_context.c +++ b/src/gallium/drivers/freedreno/freedreno_context.c @@ -54,6 +54,42 @@ fd_context_flush(struct pipe_context *pctx, struct pipe_fence_handle **fencep, DBG("%p: flush: flags=%x", batch, flags); + if (fencep && !batch) { + batch = fd_context_batch(ctx); + } else if (!batch) { + fd_bc_dump(ctx->screen, "%p: NULL batch, remaining:\n", ctx); + return; + } + + /* With TC_FLUSH_ASYNC, the fence will have been pre-created from + * the front-end thread. But not yet associated with a batch, + * because we cannot safely access ctx->batch outside of the driver + * thread. So instead, replace the existing batch->fence with the + * one created earlier + */ + if ((flags & TC_FLUSH_ASYNC) && fencep) { + /* We don't currently expect async+flush in the fence-fd + * case.. for that to work properly we'd need TC to tell + * us in the create_fence callback that it needs an fd. + */ + assert(!(flags & PIPE_FLUSH_FENCE_FD)); + + fd_fence_set_batch(*fencep, batch); + fd_fence_ref(&batch->fence, *fencep); + + /* We (a) cannot substitute the provided fence with last_fence, + * and (b) need fd_fence_populate() to be eventually called on + * the fence that was pre-created in frontend-thread: + */ + fd_fence_ref(&ctx->last_fence, NULL); + + /* async flush is not compatible with deferred flush, since + * nothing triggers the batch flush which fence_flush() would + * be waiting for + */ + flags &= ~PIPE_FLUSH_DEFERRED; + } + /* In some sequence of events, we can end up with a last_fence that is * not an "fd" fence, which results in eglDupNativeFenceFDANDROID() * errors. @@ -71,13 +107,6 @@ fd_context_flush(struct pipe_context *pctx, struct pipe_fence_handle **fencep, goto out; } - if (fencep && !batch) { - batch = fd_context_batch(ctx); - } else if (!batch) { - fd_bc_dump(ctx->screen, "%p: NULL batch, remaining:\n", ctx); - return; - } - /* Take a ref to the batch's fence (batch can be unref'd when flushed: */ fd_fence_ref(&fence, batch->fence); @@ -631,7 +660,7 @@ fd_context_init_tc(struct pipe_context *pctx, unsigned flags) struct pipe_context *tc = threaded_context_create(pctx, &ctx->screen->transfer_pool, fd_replace_buffer_storage, - NULL, // TODO fd_create_fence for async flush + fd_fence_create_unflushed, &ctx->tc); uint64_t total_ram; diff --git a/src/gallium/drivers/freedreno/freedreno_fence.c b/src/gallium/drivers/freedreno/freedreno_fence.c index 8aaebca627e..4a1ec5bbf49 100644 --- a/src/gallium/drivers/freedreno/freedreno_fence.c +++ b/src/gallium/drivers/freedreno/freedreno_fence.c @@ -36,11 +36,36 @@ struct pipe_fence_handle { struct pipe_reference reference; + /* fence holds a weak reference to the batch until the batch is flushed, * at which point fd_fence_populate() is called and timestamp and possibly * fence_fd become valid and the week reference is dropped. + * + * Note that with u_threaded_context async flushes, if a fence is requested + * by the frontend, the fence is initially created without a weak reference + * to the batch, which is filled in later when fd_context_flush() is called + * from the driver thread. In this case tc_token will be non-null, in + * which case threaded_context_flush() should be called in fd_fence_finish() */ struct fd_batch *batch; + + struct tc_unflushed_batch_token *tc_token; + bool needs_signal; + + /* For threaded_context async flushes, we must wait on the fence, signalled + * in fd_fence_populate(), to know that the rendering has been actually + * flushed from the driver thread. + * + * The ready fence is created signaled for non-async-flush fences, and only + * transitions once from unsignalled->signalled for async-flush fences + */ + struct util_queue_fence ready; + + /* Note that a fence can outlive the ctx, so we can only assume this is a + * valid ptr for unflushed fences. However we hold a reference to the + * fence->pipe so that is safe to use after flushing. + */ + struct fd_context *ctx; struct fd_pipe *pipe; struct fd_screen *screen; int fence_fd; @@ -48,13 +73,44 @@ struct pipe_fence_handle { uint32_t syncobj; }; -static void fence_flush(struct pipe_fence_handle *fence) - /* TODO this will change w/ threaded-ctx where we need to use threaded_context_flush().. */ +static bool +fence_flush(struct pipe_fence_handle *fence, uint64_t timeout) + /* NOTE: in the !fence_is_signalled() case we may be called from non-driver + * thread, but we don't call fd_batch_flush() in that case + */ in_dt { + if (!util_queue_fence_is_signalled(&fence->ready)) { + if (fence->tc_token) { + threaded_context_flush(&fence->ctx->base, fence->tc_token, + timeout == 0); + } + + if (!timeout) + return false; + + if (timeout == PIPE_TIMEOUT_INFINITE) { + util_queue_fence_wait(&fence->ready); + } else { + int64_t abs_timeout = os_time_get_absolute_timeout(timeout); + if (!util_queue_fence_wait_timeout(&fence->ready, abs_timeout)) { + return false; + } + } + + /* We've already waited for batch to be flushed and fd_fence_populate() + * called: + */ + assert(!fence->batch); + return true; + } + if (fence->batch) fd_batch_flush(fence->batch); + debug_assert(!fence->batch); + + return true; } void fd_fence_populate(struct pipe_fence_handle *fence, @@ -65,10 +121,16 @@ void fd_fence_populate(struct pipe_fence_handle *fence, fence->timestamp = timestamp; fence->fence_fd = fence_fd; fence->batch = NULL; + + if (fence->needs_signal) { + util_queue_fence_signal(&fence->ready); + fence->needs_signal = false; + } } static void fd_fence_destroy(struct pipe_fence_handle *fence) { + tc_unflushed_batch_token_reference(&fence->tc_token, NULL); if (fence->fence_fd != -1) close(fence->fence_fd); if (fence->syncobj) @@ -87,11 +149,12 @@ void fd_fence_ref(struct pipe_fence_handle **ptr, } bool fd_fence_finish(struct pipe_screen *pscreen, - struct pipe_context *ctx, + struct pipe_context *pctx, struct pipe_fence_handle *fence, uint64_t timeout) { - fence_flush(fence); + if (!fence_flush(fence, timeout)) + return false; if (fence->fence_fd != -1) { int ret = sync_wait(fence->fence_fd, timeout / 1000000); @@ -114,7 +177,9 @@ static struct pipe_fence_handle * fence_create(struct fd_context *ctx, return NULL; pipe_reference_init(&fence->reference, 1); + util_queue_fence_init(&fence->ready); + fence->ctx = ctx; fence->batch = batch; fence->pipe = fd_pipe_ref(ctx->pipe); fence->screen = ctx->screen; @@ -157,7 +222,10 @@ void fd_fence_server_sync(struct pipe_context *pctx, { struct fd_context *ctx = fd_context(pctx); - fence_flush(fence); + /* NOTE: we don't expect the combination of fence-fd + async-flush-fence, + * so timeout==0 is ok here: + */ + fence_flush(fence, 0); /* if not an external fence, then nothing more to do without preemption: */ if (fence->fence_fd == -1) @@ -181,7 +249,7 @@ void fd_fence_server_signal(struct pipe_context *pctx, int fd_fence_get_fd(struct pipe_screen *pscreen, struct pipe_fence_handle *fence) { - fence_flush(fence); + fence_flush(fence, PIPE_TIMEOUT_INFINITE); return os_dupfd_cloexec(fence->fence_fd); } @@ -194,3 +262,22 @@ struct pipe_fence_handle * fd_fence_create(struct fd_batch *batch) { return fence_create(batch->ctx, batch, 0, -1, 0); } + +void +fd_fence_set_batch(struct pipe_fence_handle *fence, struct fd_batch *batch) +{ + assert(!fence->batch); + fence->batch = batch; +} + +struct pipe_fence_handle * +fd_fence_create_unflushed(struct pipe_context *pctx, + struct tc_unflushed_batch_token *tc_token) +{ + struct pipe_fence_handle *fence = + fence_create(fd_context(pctx), NULL, 0, -1, 0); + fence->needs_signal = true; + util_queue_fence_reset(&fence->ready); + tc_unflushed_batch_token_reference(&fence->tc_token, tc_token); + return fence; +} diff --git a/src/gallium/drivers/freedreno/freedreno_fence.h b/src/gallium/drivers/freedreno/freedreno_fence.h index 0d17e1edebe..a37678a3d03 100644 --- a/src/gallium/drivers/freedreno/freedreno_fence.h +++ b/src/gallium/drivers/freedreno/freedreno_fence.h @@ -51,4 +51,11 @@ bool fd_fence_is_fd(struct pipe_fence_handle *fence); struct fd_batch; struct pipe_fence_handle * fd_fence_create(struct fd_batch *batch); + +void fd_fence_set_batch(struct pipe_fence_handle *fence, struct fd_batch *batch); + +struct tc_unflushed_batch_token; +struct pipe_fence_handle *fd_fence_create_unflushed(struct pipe_context *pctx, + struct tc_unflushed_batch_token *tc_token); + #endif /* FREEDRENO_FENCE_H_ */