diff --git a/src/gallium/drivers/zink/zink_batch.c b/src/gallium/drivers/zink/zink_batch.c index 406ec8945cb..1964aa318ad 100644 --- a/src/gallium/drivers/zink/zink_batch.c +++ b/src/gallium/drivers/zink/zink_batch.c @@ -108,6 +108,7 @@ zink_reset_batch_state(struct zink_context *ctx, struct zink_batch_state *bs) bs->submit_count++; bs->fence.batch_id = 0; bs->usage.usage = 0; + bs->next = NULL; } static void @@ -127,15 +128,25 @@ zink_clear_batch_state(struct zink_context *ctx, struct zink_batch_state *bs) unref_resources(zink_screen(ctx->base.screen), bs); } +static void +pop_batch_state(struct zink_context *ctx) +{ + const struct zink_batch_state *bs = ctx->batch_states; + ctx->batch_states = bs->next; + ctx->batch_states_count--; + if (ctx->last_fence == &bs->fence) + ctx->last_fence = NULL; +} + void zink_batch_reset_all(struct zink_context *ctx) { simple_mtx_lock(&ctx->batch_mtx); - hash_table_foreach(&ctx->batch_states, entry) { - struct zink_batch_state *bs = entry->data; + while (ctx->batch_states) { + struct zink_batch_state *bs = ctx->batch_states; bs->fence.completed = true; + pop_batch_state(ctx); zink_reset_batch_state(ctx, bs); - _mesa_hash_table_remove(&ctx->batch_states, entry); util_dynarray_append(&ctx->free_batch_states, struct zink_batch_state *, bs); } simple_mtx_unlock(&ctx->batch_mtx); @@ -239,9 +250,9 @@ fail: } static inline bool -find_unused_state(struct hash_entry *entry) +find_unused_state(struct zink_batch_state *bs) { - struct zink_fence *fence = entry->data; + struct zink_fence *fence = &bs->fence; /* we can't reset these from fence_finish because threads */ bool completed = p_atomic_read(&fence->completed); bool submitted = p_atomic_read(&fence->submitted); @@ -251,26 +262,25 @@ find_unused_state(struct hash_entry *entry) static struct zink_batch_state * get_batch_state(struct zink_context *ctx, struct zink_batch *batch) { + struct zink_screen *screen = zink_screen(ctx->base.screen); struct zink_batch_state *bs = NULL; simple_mtx_lock(&ctx->batch_mtx); if (util_dynarray_num_elements(&ctx->free_batch_states, struct zink_batch_state*)) bs = util_dynarray_pop(&ctx->free_batch_states, struct zink_batch_state*); - if (!bs) { - hash_table_foreach(&ctx->batch_states, he) { - struct zink_fence *fence = he->data; - if (zink_screen_check_last_finished(zink_screen(ctx->base.screen), fence->batch_id) || find_unused_state(he)) { - bs = he->data; - _mesa_hash_table_remove(&ctx->batch_states, he); - break; - } + if (!bs && ctx->batch_states) { + /* states are stored sequentially, so if the first one doesn't work, none of them will */ + if (zink_screen_check_last_finished(screen, ctx->batch_states->fence.batch_id) || + find_unused_state(ctx->batch_states)) { + bs = ctx->batch_states; + pop_batch_state(ctx); } } simple_mtx_unlock(&ctx->batch_mtx); if (bs) { if (bs->fence.submitted && !bs->fence.completed) /* this fence is already done, so we need vulkan to release the cmdbuf */ - zink_vkfence_wait(zink_screen(ctx->base.screen), &bs->fence, PIPE_TIMEOUT_INFINITE); + zink_vkfence_wait(screen, &bs->fence, PIPE_TIMEOUT_INFINITE); zink_reset_batch_state(ctx, bs); } else { if (!batch->state) { @@ -329,7 +339,7 @@ post_submit(void *data, void *gdata, int thread_index) if (bs->ctx->reset.reset) bs->ctx->reset.reset(bs->ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET); screen->device_lost = true; - } else if (_mesa_hash_table_num_entries(&bs->ctx->batch_states) > 5000) { + } else if (bs->ctx->batch_states_count > 5000) { zink_screen_batch_id_wait(screen, bs->fence.batch_id - 2500, PIPE_TIMEOUT_INFINITE); } } @@ -342,13 +352,10 @@ submit_queue(void *data, void *gdata, int thread_index) struct zink_screen *screen = zink_screen(ctx->base.screen); VkSubmitInfo si = {0}; - simple_mtx_lock(&ctx->batch_mtx); while (!bs->fence.batch_id) bs->fence.batch_id = p_atomic_inc_return(&screen->curr_batch); - _mesa_hash_table_insert_pre_hashed(&ctx->batch_states, bs->fence.batch_id, (void*)(uintptr_t)bs->fence.batch_id, bs); bs->usage.usage = bs->fence.batch_id; bs->usage.unflushed = false; - simple_mtx_unlock(&ctx->batch_mtx); if (ctx->have_timelines && screen->last_finished > bs->fence.batch_id && bs->fence.batch_id == 1) { if (!zink_screen_init_semaphore(screen)) { @@ -571,39 +578,52 @@ zink_end_batch(struct zink_context *ctx, struct zink_batch *batch) tc_driver_internal_flush_notify(ctx->tc); struct zink_screen *screen = zink_screen(ctx->base.screen); + struct zink_batch_state *bs; - ctx->last_fence = &batch->state->fence; - if (ctx->oom_flush || _mesa_hash_table_num_entries(&ctx->batch_states) > 10) { - simple_mtx_lock(&ctx->batch_mtx); - hash_table_foreach(&ctx->batch_states, he) { - struct zink_fence *fence = he->data; - struct zink_batch_state *bs = he->data; - if (zink_check_batch_completion(ctx, fence->batch_id, true)) { - if (bs->fence.submitted && !bs->fence.completed) - /* this fence is already done, so we need vulkan to release the cmdbuf */ - zink_vkfence_wait(screen, &bs->fence, PIPE_TIMEOUT_INFINITE); - zink_reset_batch_state(ctx, he->data); - _mesa_hash_table_remove(&ctx->batch_states, he); - util_dynarray_append(&ctx->free_batch_states, struct zink_batch_state *, bs); - } + simple_mtx_lock(&ctx->batch_mtx); + if (ctx->oom_flush || ctx->batch_states_count > 10) { + assert(!ctx->batch_states_count || ctx->batch_states); + while (ctx->batch_states) { + bs = ctx->batch_states; + struct zink_fence *fence = &bs->fence; + /* once an incomplete state is reached, no more will be complete */ + if (!zink_check_batch_completion(ctx, fence->batch_id, true)) + break; + + if (bs->fence.submitted && !bs->fence.completed) + /* this fence is already done, so we need vulkan to release the cmdbuf */ + zink_vkfence_wait(screen, &bs->fence, PIPE_TIMEOUT_INFINITE); + pop_batch_state(ctx); + zink_reset_batch_state(ctx, bs); + util_dynarray_append(&ctx->free_batch_states, struct zink_batch_state *, bs); } - simple_mtx_unlock(&ctx->batch_mtx); - if (_mesa_hash_table_num_entries(&ctx->batch_states) > 50) + if (ctx->batch_states_count > 50) ctx->oom_flush = true; } + + bs = batch->state; + if (ctx->last_fence) + zink_batch_state(ctx->last_fence)->next = bs; + else { + assert(!ctx->batch_states); + ctx->batch_states = bs; + } + ctx->last_fence = &bs->fence; + ctx->batch_states_count++; + simple_mtx_unlock(&ctx->batch_mtx); batch->work_count = 0; if (screen->device_lost) return; if (screen->threaded) { - batch->state->queue = screen->thread_queue; - util_queue_add_job(&screen->flush_queue, batch->state, &batch->state->flush_completed, + bs->queue = screen->thread_queue; + util_queue_add_job(&screen->flush_queue, bs, &bs->flush_completed, submit_queue, post_submit, 0); } else { - batch->state->queue = screen->queue; - submit_queue(batch->state, NULL, 0); - post_submit(batch->state, NULL, 0); + bs->queue = screen->queue; + submit_queue(bs, NULL, 0); + post_submit(bs, NULL, 0); } } diff --git a/src/gallium/drivers/zink/zink_batch.h b/src/gallium/drivers/zink/zink_batch.h index 91e12e737f0..ddf668f3d69 100644 --- a/src/gallium/drivers/zink/zink_batch.h +++ b/src/gallium/drivers/zink/zink_batch.h @@ -61,6 +61,7 @@ batch_ptr_add_usage(struct zink_batch *batch, struct set *s, void *ptr); struct zink_batch_state { struct zink_fence fence; + struct zink_batch_state *next; struct zink_batch_usage usage; struct zink_context *ctx; diff --git a/src/gallium/drivers/zink/zink_context.c b/src/gallium/drivers/zink/zink_context.c index 88fb40d208e..536268255b7 100644 --- a/src/gallium/drivers/zink/zink_context.c +++ b/src/gallium/drivers/zink/zink_context.c @@ -105,8 +105,7 @@ zink_context_destroy(struct pipe_context *pctx) simple_mtx_destroy(&ctx->batch_mtx); zink_clear_batch_state(ctx, ctx->batch.state); zink_batch_state_destroy(screen, ctx->batch.state); - hash_table_foreach(&ctx->batch_states, entry) { - struct zink_batch_state *bs = entry->data; + for (struct zink_batch_state *bs = ctx->batch_states; bs; bs = bs->next) { zink_clear_batch_state(ctx, bs); zink_batch_state_destroy(screen, bs); } @@ -3128,8 +3127,13 @@ zink_wait_on_batch(struct zink_context *ctx, uint32_t batch_id) if (ctx->last_fence && (!batch_id || batch_id == zink_batch_state(ctx->last_fence)->fence.batch_id)) fence = ctx->last_fence; else { - struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&ctx->batch_states, batch_id, (void*)(uintptr_t)batch_id); - if (!he) { + for (bs = ctx->batch_states; bs; bs = bs->next) { + if (bs->fence.batch_id < batch_id) + continue; + if (!bs->fence.batch_id || bs->fence.batch_id > batch_id) + break; + } + if (!bs || bs->fence.batch_id != batch_id) { simple_mtx_unlock(&ctx->batch_mtx); /* if we can't find it, it either must have finished already or is on a different context */ if (!zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id)) { @@ -3139,7 +3143,7 @@ zink_wait_on_batch(struct zink_context *ctx, uint32_t batch_id) } return; } - fence = he->data; + fence = &bs->fence; } simple_mtx_unlock(&ctx->batch_mtx); assert(fence); @@ -3172,15 +3176,20 @@ zink_check_batch_completion(struct zink_context *ctx, uint32_t batch_id, bool ha if (ctx->last_fence && batch_id == zink_batch_state(ctx->last_fence)->fence.batch_id) fence = ctx->last_fence; else { - struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&ctx->batch_states, batch_id, (void*)(uintptr_t)batch_id); - /* if we can't find it, it either must have finished already or is on a different context */ - if (!he) { + struct zink_batch_state *bs; + for (bs = ctx->batch_states; bs; bs = bs->next) { + if (bs->fence.batch_id < batch_id) + continue; + if (!bs->fence.batch_id || bs->fence.batch_id > batch_id) + break; + } + if (!bs || bs->fence.batch_id != batch_id) { if (!have_lock) simple_mtx_unlock(&ctx->batch_mtx); /* return compare against last_finished, since this has info from all contexts */ return zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id); } - fence = he->data; + fence = &bs->fence; } if (!have_lock) simple_mtx_unlock(&ctx->batch_mtx); @@ -4106,7 +4115,6 @@ zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) ctx->need_barriers[1] = &ctx->update_barriers[1][0]; util_dynarray_init(&ctx->free_batch_states, ctx); - _mesa_hash_table_init(&ctx->batch_states, ctx, NULL, _mesa_key_pointer_equal); ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state = screen->info.have_EXT_extended_dynamic_state; ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state2 = screen->info.have_EXT_extended_dynamic_state2; diff --git a/src/gallium/drivers/zink/zink_context.h b/src/gallium/drivers/zink/zink_context.h index caab5407de2..2a83c6f1335 100644 --- a/src/gallium/drivers/zink/zink_context.h +++ b/src/gallium/drivers/zink/zink_context.h @@ -198,7 +198,8 @@ struct zink_context { simple_mtx_t batch_mtx; struct zink_fence *deferred_fence; struct zink_fence *last_fence; //the last command buffer submitted - struct hash_table batch_states; //submitted batch states + struct zink_batch_state *batch_states; //list of submitted batch states: ordered by increasing timeline id + unsigned batch_states_count; //number of states in `batch_states` struct util_dynarray free_batch_states; //unused batch states bool oom_flush; bool oom_stall;