diff --git a/src/amd/vulkan/layers/radv_metro_exodus.c b/src/amd/vulkan/layers/radv_metro_exodus.c index d5cc5225027..b67c74d1619 100644 --- a/src/amd/vulkan/layers/radv_metro_exodus.c +++ b/src/amd/vulkan/layers/radv_metro_exodus.c @@ -22,17 +22,16 @@ */ #include "radv_private.h" +#include "vk_common_entrypoints.h" VKAPI_ATTR VkResult VKAPI_CALL metro_exodus_GetSemaphoreCounterValue(VkDevice _device, VkSemaphore _semaphore, uint64_t *pValue) { - RADV_FROM_HANDLE(radv_semaphore, semaphore, _semaphore); - /* See https://gitlab.freedesktop.org/mesa/mesa/-/issues/5119. */ - if (semaphore == NULL) { + if (_semaphore == VK_NULL_HANDLE) { fprintf(stderr, "RADV: Ignoring vkGetSemaphoreCounterValue() with NULL semaphore (game bug)!\n"); return VK_SUCCESS; } - return radv_GetSemaphoreCounterValue(_device, _semaphore, pValue); + return vk_common_GetSemaphoreCounterValue(_device, _semaphore, pValue); } diff --git a/src/amd/vulkan/radv_android.c b/src/amd/vulkan/radv_android.c index d26095de714..1857d72ae2b 100644 --- a/src/amd/vulkan/radv_android.c +++ b/src/amd/vulkan/radv_android.c @@ -37,6 +37,7 @@ #include "util/os_file.h" #include "radv_private.h" +#include "vk_common_entrypoints.h" #include "vk_util.h" #ifdef ANDROID @@ -426,7 +427,7 @@ radv_AcquireImageANDROID(VkDevice device_h, VkImage image_h, int nativeFenceFd, .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, .fd = semaphore_fd, }; - result = radv_ImportSemaphoreFdKHR(device_h, &info); + result = vk_common_ImportSemaphoreFdKHR(device_h, &info); if (result == VK_SUCCESS) semaphore_fd = -1; /* RADV took ownership */ } @@ -439,7 +440,7 @@ radv_AcquireImageANDROID(VkDevice device_h, VkImage image_h, int nativeFenceFd, .handleType = VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT, .fd = fence_fd, }; - result = radv_ImportFenceFdKHR(device_h, &info); + result = vk_common_ImportFenceFdKHR(device_h, &info); if (result == VK_SUCCESS) fence_fd = -1; /* RADV took ownership */ } @@ -471,13 +472,13 @@ radv_QueueSignalReleaseImageANDROID(VkQueue _queue, uint32_t waitSemaphoreCount, for (uint32_t i = 0; i < waitSemaphoreCount; ++i) { int tmp_fd; result = - radv_GetSemaphoreFdKHR(radv_device_to_handle(queue->device), - &(VkSemaphoreGetFdInfoKHR){ - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR, - .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, - .semaphore = pWaitSemaphores[i], - }, - &tmp_fd); + vk_common_GetSemaphoreFdKHR(radv_device_to_handle(queue->device), + &(VkSemaphoreGetFdInfoKHR){ + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR, + .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, + .semaphore = pWaitSemaphores[i], + }, + &tmp_fd); if (result != VK_SUCCESS) { if (fd >= 0) close(fd); diff --git a/src/amd/vulkan/radv_device.c b/src/amd/vulkan/radv_device.c index b82735dc60c..a1ef5466b91 100644 --- a/src/amd/vulkan/radv_device.c +++ b/src/amd/vulkan/radv_device.c @@ -83,40 +83,12 @@ typedef void *drmDevicePtr; static VkResult radv_queue_submit2(struct vk_queue *vqueue, struct vk_queue_submit *submission); -static struct radv_timeline_point * -radv_timeline_find_point_at_least_locked(struct radv_device *device, struct radv_timeline *timeline, - uint64_t p); - -static struct radv_timeline_point *radv_timeline_add_point_locked(struct radv_device *device, - struct radv_timeline *timeline, - uint64_t p); - -static void radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline, - struct list_head *processing_list); - -static void radv_destroy_semaphore_part(struct radv_device *device, - struct radv_semaphore_part *part); - uint64_t radv_get_current_time(void) { return os_time_get_nano(); } -static uint64_t -radv_get_absolute_timeout(uint64_t timeout) -{ - if (timeout == UINT64_MAX) { - return timeout; - } else { - uint64_t current_time = radv_get_current_time(); - - timeout = MIN2(UINT64_MAX - current_time, timeout); - - return current_time + timeout; - } -} - static int radv_device_get_cache_uuid(enum radeon_family family, void *uuid) { @@ -2677,37 +2649,12 @@ radv_queue_init(struct radv_device *device, struct radv_queue *queue, queue->vk.driver_submit = radv_queue_submit2; - list_inithead(&queue->pending_submissions); - mtx_init(&queue->pending_mutex, mtx_plain); - - mtx_init(&queue->thread_mutex, mtx_plain); - if (u_cnd_monotonic_init(&queue->thread_cond)) { - vk_queue_finish(&queue->vk); - return vk_error(device, VK_ERROR_INITIALIZATION_FAILED); - } - queue->cond_created = true; - return VK_SUCCESS; } static void radv_queue_finish(struct radv_queue *queue) { - if (queue->hw_ctx) { - if (queue->cond_created) { - if (queue->thread_running) { - p_atomic_set(&queue->thread_exit, true); - u_cnd_monotonic_broadcast(&queue->thread_cond); - thrd_join(queue->submission_thread, NULL); - } - - u_cnd_monotonic_destroy(&queue->thread_cond); - } - - mtx_destroy(&queue->pending_mutex); - mtx_destroy(&queue->thread_mutex); - } - if (queue->initial_full_flush_preamble_cs) queue->device->ws->cs_destroy(queue->initial_full_flush_preamble_cs); if (queue->initial_preamble_cs) @@ -4186,169 +4133,6 @@ fail: return vk_error(queue, result); } -static VkResult -radv_alloc_sem_counts(struct radv_device *device, struct radv_winsys_sem_counts *counts, - int num_sems, struct radv_semaphore_part **sems, - const uint64_t *timeline_values, VkFence _fence, bool is_signal) -{ - int syncobj_idx = 0, non_reset_idx = 0, timeline_idx = 0; - - if (num_sems == 0 && _fence == VK_NULL_HANDLE) - return VK_SUCCESS; - - for (uint32_t i = 0; i < num_sems; i++) { - switch (sems[i]->kind) { - case RADV_SEMAPHORE_SYNCOBJ: - counts->syncobj_count++; - counts->syncobj_reset_count++; - break; - case RADV_SEMAPHORE_NONE: - break; - case RADV_SEMAPHORE_TIMELINE: - counts->syncobj_count++; - break; - case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: - counts->timeline_syncobj_count++; - break; - } - } - - if (_fence != VK_NULL_HANDLE) - counts->syncobj_count++; - - if (counts->syncobj_count || counts->timeline_syncobj_count) { - counts->points = (uint64_t *)malloc(sizeof(*counts->syncobj) * counts->syncobj_count + - (sizeof(*counts->syncobj) + sizeof(*counts->points)) * - counts->timeline_syncobj_count); - if (!counts->points) - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - counts->syncobj = (uint32_t *)(counts->points + counts->timeline_syncobj_count); - } - - non_reset_idx = counts->syncobj_reset_count; - - for (uint32_t i = 0; i < num_sems; i++) { - switch (sems[i]->kind) { - case RADV_SEMAPHORE_NONE: - unreachable("Empty semaphore"); - break; - case RADV_SEMAPHORE_SYNCOBJ: - counts->syncobj[syncobj_idx++] = sems[i]->syncobj; - break; - case RADV_SEMAPHORE_TIMELINE: { - mtx_lock(&sems[i]->timeline.mutex); - struct radv_timeline_point *point = NULL; - if (is_signal) { - point = radv_timeline_add_point_locked(device, &sems[i]->timeline, timeline_values[i]); - } else { - point = radv_timeline_find_point_at_least_locked(device, &sems[i]->timeline, - timeline_values[i]); - } - - mtx_unlock(&sems[i]->timeline.mutex); - - if (point) { - counts->syncobj[non_reset_idx++] = point->syncobj; - } else { - /* Explicitly remove the semaphore so we might not find - * a point later post-submit. */ - sems[i] = NULL; - } - break; - } - case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: - counts->syncobj[counts->syncobj_count + timeline_idx] = sems[i]->syncobj; - counts->points[timeline_idx] = timeline_values[i]; - ++timeline_idx; - break; - } - } - - if (_fence != VK_NULL_HANDLE) { - RADV_FROM_HANDLE(radv_fence, fence, _fence); - - struct radv_fence_part *part = - fence->temporary.kind != RADV_FENCE_NONE ? &fence->temporary : &fence->permanent; - counts->syncobj[non_reset_idx++] = part->syncobj; - } - - assert(MAX2(syncobj_idx, non_reset_idx) <= counts->syncobj_count); - counts->syncobj_count = MAX2(syncobj_idx, non_reset_idx); - - return VK_SUCCESS; -} - -static void -radv_free_sem_info(struct radv_winsys_sem_info *sem_info) -{ - free(sem_info->wait.points); - free(sem_info->signal.points); -} - -static void -radv_free_temp_syncobjs(struct radv_device *device, int num_sems, struct radv_semaphore_part *sems) -{ - for (uint32_t i = 0; i < num_sems; i++) { - radv_destroy_semaphore_part(device, sems + i); - } -} - -static VkResult -radv_alloc_sem_info(struct radv_device *device, struct radv_winsys_sem_info *sem_info, - int num_wait_sems, struct radv_semaphore_part **wait_sems, - const uint64_t *wait_values, int num_signal_sems, - struct radv_semaphore_part **signal_sems, const uint64_t *signal_values, - VkFence fence) -{ - VkResult ret; - - ret = radv_alloc_sem_counts(device, &sem_info->wait, num_wait_sems, wait_sems, wait_values, - VK_NULL_HANDLE, false); - if (ret) - return ret; - ret = radv_alloc_sem_counts(device, &sem_info->signal, num_signal_sems, signal_sems, - signal_values, fence, true); - if (ret) - radv_free_sem_info(sem_info); - - /* caller can override these */ - sem_info->cs_emit_wait = true; - sem_info->cs_emit_signal = true; - return ret; -} - -static void -radv_finalize_timelines(struct radv_device *device, uint32_t num_wait_sems, - struct radv_semaphore_part **wait_sems, const uint64_t *wait_values, - uint32_t num_signal_sems, struct radv_semaphore_part **signal_sems, - const uint64_t *signal_values, struct list_head *processing_list) -{ - for (uint32_t i = 0; i < num_wait_sems; ++i) { - if (wait_sems[i] && wait_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) { - mtx_lock(&wait_sems[i]->timeline.mutex); - struct radv_timeline_point *point = radv_timeline_find_point_at_least_locked( - device, &wait_sems[i]->timeline, wait_values[i]); - point->wait_count -= 2; - mtx_unlock(&wait_sems[i]->timeline.mutex); - } - } - for (uint32_t i = 0; i < num_signal_sems; ++i) { - if (signal_sems[i] && signal_sems[i]->kind == RADV_SEMAPHORE_TIMELINE) { - mtx_lock(&signal_sems[i]->timeline.mutex); - struct radv_timeline_point *point = radv_timeline_find_point_at_least_locked( - device, &signal_sems[i]->timeline, signal_values[i]); - signal_sems[i]->timeline.highest_submitted = - MAX2(signal_sems[i]->timeline.highest_submitted, point->value); - point->wait_count -= 2; - radv_timeline_trigger_waiters_locked(&signal_sems[i]->timeline, processing_list); - mtx_unlock(&signal_sems[i]->timeline.mutex); - } else if (signal_sems[i] && signal_sems[i]->kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ) { - signal_sems[i]->timeline_syncobj.max_point = - MAX2(signal_sems[i]->timeline_syncobj.max_point, signal_values[i]); - } - } -} - static VkResult radv_sparse_buffer_bind_memory(struct radv_device *device, const VkSparseBufferMemoryBindInfo *bind) { @@ -4533,216 +4317,6 @@ struct radv_deferred_queue_submission { struct list_head processing_list; }; -struct radv_queue_submission { - const VkCommandBufferSubmitInfoKHR *cmd_buffer_infos; - uint32_t cmd_buffer_info_count; - - /* Sparse bindings that happen on a queue. */ - const VkSparseBufferMemoryBindInfo *buffer_binds; - uint32_t buffer_bind_count; - const VkSparseImageOpaqueMemoryBindInfo *image_opaque_binds; - uint32_t image_opaque_bind_count; - const VkSparseImageMemoryBindInfo *image_binds; - uint32_t image_bind_count; - - bool flush_caches; - VkPipelineStageFlags2KHR wait_dst_stage_mask; - - const VkSemaphoreSubmitInfoKHR *wait_semaphore_infos; - uint32_t wait_semaphore_info_count; - - const VkSemaphoreSubmitInfoKHR *signal_semaphore_infos; - uint32_t signal_semaphore_info_count; - - VkFence fence; -}; - -static VkResult radv_queue_trigger_submission(struct radv_deferred_queue_submission *submission, - uint32_t decrement, - struct list_head *processing_list); - -static VkResult -radv_create_deferred_submission(struct radv_queue *queue, - const struct radv_queue_submission *submission, - struct radv_deferred_queue_submission **out) -{ - struct radv_deferred_queue_submission *deferred = NULL; - size_t size = sizeof(struct radv_deferred_queue_submission); - - uint32_t temporary_count = 0; - for (uint32_t i = 0; i < submission->wait_semaphore_info_count; ++i) { - RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphore_infos[i].semaphore); - if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) - ++temporary_count; - } - - size += submission->cmd_buffer_info_count * sizeof(VkCommandBuffer); - size += submission->buffer_bind_count * sizeof(VkSparseBufferMemoryBindInfo); - size += submission->image_opaque_bind_count * sizeof(VkSparseImageOpaqueMemoryBindInfo); - size += submission->image_bind_count * sizeof(VkSparseImageMemoryBindInfo); - - for (uint32_t i = 0; i < submission->image_bind_count; ++i) - size += submission->image_binds[i].bindCount * sizeof(VkSparseImageMemoryBind); - - size += submission->wait_semaphore_info_count * sizeof(struct radv_semaphore_part *); - size += submission->wait_semaphore_info_count * sizeof(uint64_t); - size += temporary_count * sizeof(struct radv_semaphore_part); - size += submission->signal_semaphore_info_count * sizeof(struct radv_semaphore_part *); - size += submission->signal_semaphore_info_count * sizeof(uint64_t); - size += submission->wait_semaphore_info_count * sizeof(struct radv_timeline_waiter); /* legacy */ - - deferred = calloc(1, size); - if (!deferred) - return VK_ERROR_OUT_OF_HOST_MEMORY; - - deferred->queue = queue; - - deferred->cmd_buffers = (void *)(deferred + 1); - deferred->cmd_buffer_count = submission->cmd_buffer_info_count; - for (uint32_t i = 0; i < submission->cmd_buffer_info_count; i++) { - deferred->cmd_buffers[i] = submission->cmd_buffer_infos[i].commandBuffer; - } - - deferred->buffer_binds = (void *)(deferred->cmd_buffers + submission->cmd_buffer_info_count); - deferred->buffer_bind_count = submission->buffer_bind_count; - if (submission->buffer_bind_count) { - memcpy(deferred->buffer_binds, submission->buffer_binds, - submission->buffer_bind_count * sizeof(*deferred->buffer_binds)); - } - - deferred->image_opaque_binds = (void *)(deferred->buffer_binds + submission->buffer_bind_count); - deferred->image_opaque_bind_count = submission->image_opaque_bind_count; - if (submission->image_opaque_bind_count) { - memcpy(deferred->image_opaque_binds, submission->image_opaque_binds, - submission->image_opaque_bind_count * sizeof(*deferred->image_opaque_binds)); - } - - deferred->image_binds = - (void *)(deferred->image_opaque_binds + deferred->image_opaque_bind_count); - deferred->image_bind_count = submission->image_bind_count; - - VkSparseImageMemoryBind *sparse_image_binds = - (void *)(deferred->image_binds + deferred->image_bind_count); - for (uint32_t i = 0; i < deferred->image_bind_count; ++i) { - deferred->image_binds[i] = submission->image_binds[i]; - deferred->image_binds[i].pBinds = sparse_image_binds; - - for (uint32_t j = 0; j < deferred->image_binds[i].bindCount; ++j) - *sparse_image_binds++ = submission->image_binds[i].pBinds[j]; - } - - deferred->flush_caches = submission->flush_caches; - deferred->wait_dst_stage_mask = submission->wait_dst_stage_mask; - - deferred->wait_semaphores = (void *)sparse_image_binds; - deferred->wait_semaphore_count = submission->wait_semaphore_info_count; - - deferred->signal_semaphores = - (void *)(deferred->wait_semaphores + deferred->wait_semaphore_count); - deferred->signal_semaphore_count = submission->signal_semaphore_info_count; - - deferred->fence = submission->fence; - - deferred->temporary_semaphore_parts = - (void *)(deferred->signal_semaphores + deferred->signal_semaphore_count); - deferred->temporary_semaphore_part_count = temporary_count; - - deferred->wait_values = (void *)(deferred->temporary_semaphore_parts + temporary_count); - deferred->signal_values = deferred->wait_values + submission->wait_semaphore_info_count; - - uint32_t temporary_idx = 0; - for (uint32_t i = 0; i < submission->wait_semaphore_info_count; ++i) { - RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->wait_semaphore_infos[i].semaphore); - if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) { - deferred->wait_semaphores[i] = &deferred->temporary_semaphore_parts[temporary_idx]; - deferred->temporary_semaphore_parts[temporary_idx] = semaphore->temporary; - semaphore->temporary.kind = RADV_SEMAPHORE_NONE; - ++temporary_idx; - } else { - deferred->wait_semaphores[i] = &semaphore->permanent; - } - deferred->wait_values[i] = submission->wait_semaphore_infos[i].value; - } - - for (uint32_t i = 0; i < submission->signal_semaphore_info_count; ++i) { - RADV_FROM_HANDLE(radv_semaphore, semaphore, submission->signal_semaphore_infos[i].semaphore); - if (semaphore->temporary.kind != RADV_SEMAPHORE_NONE) { - deferred->signal_semaphores[i] = &semaphore->temporary; - } else { - deferred->signal_semaphores[i] = &semaphore->permanent; - } - deferred->signal_values[i] = submission->signal_semaphore_infos[i].value; - } - - deferred->wait_nodes = (void *)(deferred->signal_values + submission->signal_semaphore_info_count); - /* This is worst-case. radv_queue_enqueue_submission will fill in further, but this - * ensure the submission is not accidentally triggered early when adding wait timelines. */ - deferred->submission_wait_count = 1 + submission->wait_semaphore_info_count; - - *out = deferred; - return VK_SUCCESS; -} - -static VkResult -radv_queue_enqueue_submission(struct radv_deferred_queue_submission *submission, - struct list_head *processing_list) -{ - uint32_t wait_cnt = 0; - struct radv_timeline_waiter *waiter = submission->wait_nodes; - for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) { - if (submission->wait_semaphores[i]->kind == RADV_SEMAPHORE_TIMELINE) { - mtx_lock(&submission->wait_semaphores[i]->timeline.mutex); - if (submission->wait_semaphores[i]->timeline.highest_submitted < - submission->wait_values[i]) { - ++wait_cnt; - waiter->value = submission->wait_values[i]; - waiter->submission = submission; - list_addtail(&waiter->list, &submission->wait_semaphores[i]->timeline.waiters); - ++waiter; - } - mtx_unlock(&submission->wait_semaphores[i]->timeline.mutex); - } - } - - mtx_lock(&submission->queue->pending_mutex); - - bool is_first = list_is_empty(&submission->queue->pending_submissions); - list_addtail(&submission->queue_pending_list, &submission->queue->pending_submissions); - - mtx_unlock(&submission->queue->pending_mutex); - - /* If there is already a submission in the queue, that will decrement the counter by 1 when - * submitted, but if the queue was empty, we decrement ourselves as there is no previous - * submission. */ - uint32_t decrement = submission->wait_semaphore_count - wait_cnt + (is_first ? 1 : 0); - - /* if decrement is zero, then we don't have a refcounted reference to the - * submission anymore, so it is not safe to access the submission. */ - if (!decrement) - return VK_SUCCESS; - - return radv_queue_trigger_submission(submission, decrement, processing_list); -} - -static void -radv_queue_submission_update_queue(struct radv_deferred_queue_submission *submission, - struct list_head *processing_list) -{ - mtx_lock(&submission->queue->pending_mutex); - list_del(&submission->queue_pending_list); - - /* trigger the next submission in the queue. */ - if (!list_is_empty(&submission->queue->pending_submissions)) { - struct radv_deferred_queue_submission *next_submission = - list_first_entry(&submission->queue->pending_submissions, - struct radv_deferred_queue_submission, queue_pending_list); - radv_queue_trigger_submission(next_submission, 1, processing_list); - } - mtx_unlock(&submission->queue->pending_mutex); - - u_cnd_monotonic_broadcast(&submission->queue->device->timeline_cond); -} - static VkResult radv_queue_submit2(struct vk_queue *vqueue, struct vk_queue_submit *submission) { @@ -4852,299 +4426,6 @@ fail: return result; } -static VkResult -radv_queue_submit_deferred(struct radv_deferred_queue_submission *submission, - struct list_head *processing_list) -{ - struct radv_queue *queue = submission->queue; - struct radeon_winsys_ctx *ctx = queue->hw_ctx; - uint32_t max_cs_submission = queue->device->trace_bo ? 1 : RADV_MAX_IBS_PER_SUBMIT; - bool do_flush = submission->flush_caches || submission->wait_dst_stage_mask; - bool can_patch = true; - uint32_t advance; - struct radv_winsys_sem_info sem_info = {0}; - VkResult result; - struct radeon_cmdbuf *initial_preamble_cs = NULL; - struct radeon_cmdbuf *initial_flush_preamble_cs = NULL; - struct radeon_cmdbuf *continue_preamble_cs = NULL; - - result = radv_get_preambles(queue, (struct vk_command_buffer *const *)submission->cmd_buffers, - submission->cmd_buffer_count, &initial_preamble_cs, - &initial_flush_preamble_cs, &continue_preamble_cs); - if (result != VK_SUCCESS) - goto fail; - - result = radv_alloc_sem_info(queue->device, &sem_info, submission->wait_semaphore_count, - submission->wait_semaphores, submission->wait_values, - submission->signal_semaphore_count, submission->signal_semaphores, - submission->signal_values, submission->fence); - if (result != VK_SUCCESS) - goto fail; - - for (uint32_t i = 0; i < submission->buffer_bind_count; ++i) { - result = radv_sparse_buffer_bind_memory(queue->device, submission->buffer_binds + i); - if (result != VK_SUCCESS) - goto fail; - } - - for (uint32_t i = 0; i < submission->image_opaque_bind_count; ++i) { - result = - radv_sparse_image_opaque_bind_memory(queue->device, submission->image_opaque_binds + i); - if (result != VK_SUCCESS) - goto fail; - } - - for (uint32_t i = 0; i < submission->image_bind_count; ++i) { - result = radv_sparse_image_bind_memory(queue->device, submission->image_binds + i); - if (result != VK_SUCCESS) - goto fail; - } - - if (!submission->cmd_buffer_count) { - result = - queue->device->ws->cs_submit(ctx, queue->vk.queue_family_index, queue->vk.index_in_family, - NULL, 0, NULL, NULL, &sem_info, false); - if (result != VK_SUCCESS) - goto fail; - } else { - struct radeon_cmdbuf **cs_array = - malloc(sizeof(struct radeon_cmdbuf *) * (submission->cmd_buffer_count)); - - for (uint32_t j = 0; j < submission->cmd_buffer_count; j++) { - RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, submission->cmd_buffers[j]); - assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY); - - cs_array[j] = cmd_buffer->cs; - if ((cmd_buffer->usage_flags & VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT)) - can_patch = false; - - cmd_buffer->status = RADV_CMD_BUFFER_STATUS_PENDING; - } - - for (uint32_t j = 0; j < submission->cmd_buffer_count; j += advance) { - struct radeon_cmdbuf *initial_preamble = - (do_flush && !j) ? initial_flush_preamble_cs : initial_preamble_cs; - advance = MIN2(max_cs_submission, submission->cmd_buffer_count - j); - - if (queue->device->trace_bo) - *queue->device->trace_id_ptr = 0; - - sem_info.cs_emit_wait = j == 0; - sem_info.cs_emit_signal = j + advance == submission->cmd_buffer_count; - - result = queue->device->ws->cs_submit( - ctx, queue->vk.queue_family_index, queue->vk.index_in_family, cs_array + j, advance, - initial_preamble, continue_preamble_cs, &sem_info, can_patch); - if (result != VK_SUCCESS) { - free(cs_array); - goto fail; - } - - if (queue->device->trace_bo) { - radv_check_gpu_hangs(queue, cs_array[j]); - } - - if (queue->device->tma_bo) { - radv_check_trap_handler(queue); - } - } - - free(cs_array); - } - - radv_finalize_timelines(queue->device, submission->wait_semaphore_count, - submission->wait_semaphores, submission->wait_values, - submission->signal_semaphore_count, submission->signal_semaphores, - submission->signal_values, processing_list); - /* Has to happen after timeline finalization to make sure the - * condition variable is only triggered when timelines and queue have - * been updated. */ - radv_queue_submission_update_queue(submission, processing_list); - -fail: - if (result != VK_SUCCESS && result != VK_ERROR_DEVICE_LOST) { - /* When something bad happened during the submission, such as - * an out of memory issue, it might be hard to recover from - * this inconsistent state. To avoid this sort of problem, we - * assume that we are in a really bad situation and return - * VK_ERROR_DEVICE_LOST to ensure the clients do not attempt - * to submit the same job again to this device. - */ - result = vk_device_set_lost(&queue->device->vk, "vkQueueSubmit() failed"); - } - - radv_free_temp_syncobjs(queue->device, submission->temporary_semaphore_part_count, - submission->temporary_semaphore_parts); - radv_free_sem_info(&sem_info); - free(submission); - return result; -} - -static VkResult -radv_process_submissions(struct list_head *processing_list) -{ - while (!list_is_empty(processing_list)) { - struct radv_deferred_queue_submission *submission = - list_first_entry(processing_list, struct radv_deferred_queue_submission, processing_list); - list_del(&submission->processing_list); - - VkResult result = radv_queue_submit_deferred(submission, processing_list); - if (result != VK_SUCCESS) - return result; - } - return VK_SUCCESS; -} - -static VkResult -wait_for_submission_timelines_available(struct radv_deferred_queue_submission *submission, - uint64_t timeout) -{ - struct radv_device *device = submission->queue->device; - uint32_t syncobj_count = 0; - uint32_t syncobj_idx = 0; - - for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) { - if (submission->wait_semaphores[i]->kind != RADV_SEMAPHORE_TIMELINE_SYNCOBJ) - continue; - - if (submission->wait_semaphores[i]->timeline_syncobj.max_point >= submission->wait_values[i]) - continue; - ++syncobj_count; - } - - if (!syncobj_count) - return VK_SUCCESS; - - uint64_t *points = malloc((sizeof(uint64_t) + sizeof(uint32_t)) * syncobj_count); - if (!points) - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - - uint32_t *syncobj = (uint32_t *)(points + syncobj_count); - - for (uint32_t i = 0; i < submission->wait_semaphore_count; ++i) { - if (submission->wait_semaphores[i]->kind != RADV_SEMAPHORE_TIMELINE_SYNCOBJ) - continue; - - if (submission->wait_semaphores[i]->timeline_syncobj.max_point >= submission->wait_values[i]) - continue; - - syncobj[syncobj_idx] = submission->wait_semaphores[i]->syncobj; - points[syncobj_idx] = submission->wait_values[i]; - ++syncobj_idx; - } - - VkResult result = VK_SUCCESS; - if (syncobj_idx > 0) { - result = device->ws->wait_timeline_syncobj(device->ws, syncobj, points, syncobj_idx, true, - true, timeout); - } - - free(points); - return result; -} - -static int -radv_queue_submission_thread_run(void *q) -{ - struct radv_queue *queue = q; - - mtx_lock(&queue->thread_mutex); - while (!p_atomic_read(&queue->thread_exit)) { - struct radv_deferred_queue_submission *submission = queue->thread_submission; - struct list_head processing_list; - VkResult result = VK_SUCCESS; - if (!submission) { - u_cnd_monotonic_wait(&queue->thread_cond, &queue->thread_mutex); - continue; - } - mtx_unlock(&queue->thread_mutex); - - /* Wait at most 5 seconds so we have a chance to notice shutdown when - * a semaphore never gets signaled. If it takes longer we just retry - * the wait next iteration. */ - result = - wait_for_submission_timelines_available(submission, radv_get_absolute_timeout(5000000000)); - if (result != VK_SUCCESS) { - mtx_lock(&queue->thread_mutex); - continue; - } - - /* The lock isn't held but nobody will add one until we finish - * the current submission. */ - p_atomic_set(&queue->thread_submission, NULL); - - list_inithead(&processing_list); - list_addtail(&submission->processing_list, &processing_list); - result = radv_process_submissions(&processing_list); - - mtx_lock(&queue->thread_mutex); - } - mtx_unlock(&queue->thread_mutex); - return 0; -} - -static VkResult -radv_queue_trigger_submission(struct radv_deferred_queue_submission *submission, uint32_t decrement, - struct list_head *processing_list) -{ - struct radv_queue *queue = submission->queue; - int ret; - if (p_atomic_add_return(&submission->submission_wait_count, -decrement)) - return VK_SUCCESS; - - if (wait_for_submission_timelines_available(submission, radv_get_absolute_timeout(0)) == - VK_SUCCESS) { - list_addtail(&submission->processing_list, processing_list); - return VK_SUCCESS; - } - - mtx_lock(&queue->thread_mutex); - - /* A submission can only be ready for the thread if it doesn't have - * any predecessors in the same queue, so there can only be one such - * submission at a time. */ - assert(queue->thread_submission == NULL); - - /* Only start the thread on demand to save resources for the many games - * which only use binary semaphores. */ - if (!queue->thread_running) { - ret = thrd_create(&queue->submission_thread, radv_queue_submission_thread_run, queue); - if (ret) { - mtx_unlock(&queue->thread_mutex); - return vk_errorf(queue, VK_ERROR_DEVICE_LOST, - "Failed to start submission thread"); - } - queue->thread_running = true; - } - - queue->thread_submission = submission; - mtx_unlock(&queue->thread_mutex); - - u_cnd_monotonic_signal(&queue->thread_cond); - return VK_SUCCESS; -} - -static VkResult -radv_queue_submit(struct radv_queue *queue, const struct radv_queue_submission *submission) -{ - struct radv_deferred_queue_submission *deferred = NULL; - - VkResult result = radv_create_deferred_submission(queue, submission, &deferred); - if (result != VK_SUCCESS) - return result; - - struct list_head processing_list; - list_inithead(&processing_list); - - result = radv_queue_enqueue_submission(deferred, &processing_list); - if (result != VK_SUCCESS) { - /* If anything is in the list we leak. */ - assert(list_is_empty(&processing_list)); - return result; - } - return radv_process_submissions(&processing_list); -} - bool radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs) { @@ -5159,123 +4440,6 @@ radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs) return true; } -/* Signals fence as soon as all the work currently put on queue is done. */ -static VkResult -radv_signal_fence(struct radv_queue *queue, VkFence fence) -{ - return radv_queue_submit(queue, &(struct radv_queue_submission){.fence = fence}); -} - -static bool -radv_submit_has_effects(const VkSubmitInfo2KHR *info) -{ - return info->commandBufferInfoCount || info->waitSemaphoreInfoCount || - info->signalSemaphoreInfoCount; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_QueueSubmit2KHR(VkQueue _queue, uint32_t submitCount, const VkSubmitInfo2KHR* pSubmits, - VkFence fence) -{ - RADV_FROM_HANDLE(radv_queue, queue, _queue); - VkResult result; - uint32_t fence_idx = 0; - bool flushed_caches = false; - - if (vk_device_is_lost(&queue->device->vk)) - return VK_ERROR_DEVICE_LOST; - - if (fence != VK_NULL_HANDLE) { - for (uint32_t i = 0; i < submitCount; ++i) - if (radv_submit_has_effects(pSubmits + i)) - fence_idx = i; - } else - fence_idx = UINT32_MAX; - - for (uint32_t i = 0; i < submitCount; i++) { - if (!radv_submit_has_effects(pSubmits + i) && fence_idx != i) - continue; - - VkPipelineStageFlags2KHR wait_dst_stage_mask = 0; - - /* Wait semaphores. */ - for (unsigned j = 0; j < pSubmits[i].waitSemaphoreInfoCount; ++j) { - wait_dst_stage_mask |= pSubmits[i].pWaitSemaphoreInfos[j].stageMask; - } - - /* Signal semaphores. */ - for (unsigned j = 0; j < pSubmits[i].signalSemaphoreInfoCount; ++j) { - wait_dst_stage_mask |= pSubmits[i].pSignalSemaphoreInfos[j].stageMask; - } - - result = radv_queue_submit( - queue, &(struct radv_queue_submission){ - .cmd_buffer_infos = pSubmits[i].pCommandBufferInfos, - .cmd_buffer_info_count = pSubmits[i].commandBufferInfoCount, - .wait_dst_stage_mask = wait_dst_stage_mask, - .flush_caches = !flushed_caches, - .wait_semaphore_infos = pSubmits[i].pWaitSemaphoreInfos, - .wait_semaphore_info_count = pSubmits[i].waitSemaphoreInfoCount, - .signal_semaphore_infos = pSubmits[i].pSignalSemaphoreInfos, - .signal_semaphore_info_count = pSubmits[i].signalSemaphoreInfoCount, - .fence = i == fence_idx ? fence : VK_NULL_HANDLE, - }); - if (result != VK_SUCCESS) - return result; - - flushed_caches = true; - } - - if (fence != VK_NULL_HANDLE && !submitCount) { - result = radv_signal_fence(queue, fence); - if (result != VK_SUCCESS) - return result; - } - - return VK_SUCCESS; -} - -static const char * -radv_get_queue_family_name(struct radv_queue *queue) -{ - switch (queue->vk.queue_family_index) { - case RADV_QUEUE_GENERAL: - return "graphics"; - case RADV_QUEUE_COMPUTE: - return "compute"; - case RADV_QUEUE_TRANSFER: - return "transfer"; - default: - unreachable("Unknown queue family"); - } -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_QueueWaitIdle(VkQueue _queue) -{ - RADV_FROM_HANDLE(radv_queue, queue, _queue); - - if (vk_device_is_lost(&queue->device->vk)) - return VK_ERROR_DEVICE_LOST; - - mtx_lock(&queue->pending_mutex); - while (!list_is_empty(&queue->pending_submissions)) { - u_cnd_monotonic_wait(&queue->device->timeline_cond, &queue->pending_mutex); - } - mtx_unlock(&queue->pending_mutex); - - if (!queue->device->ws->ctx_wait_idle( - queue->hw_ctx, radv_queue_family_to_ring(queue->vk.queue_family_index), - queue->vk.index_in_family)) { - return vk_device_set_lost(&queue->device->vk, - "Failed to wait for a '%s' queue " - "to be idle. GPU hang ?", - radv_get_queue_family_name(queue)); - } - - return VK_SUCCESS; -} - VKAPI_ATTR VkResult VKAPI_CALL radv_EnumerateInstanceExtensionProperties(const char *pLayerName, uint32_t *pPropertyCount, VkExtensionProperties *pProperties) @@ -5708,7 +4872,7 @@ radv_GetBufferMemoryRequirements2(VkDevice _device, const VkBufferMemoryRequirem VKAPI_ATTR void VKAPI_CALL radv_GetDeviceBufferMemoryRequirementsKHR(VkDevice _device, - const VkDeviceBufferMemoryRequirementsKHR* pInfo, + const VkDeviceBufferMemoryRequirementsKHR *pInfo, VkMemoryRequirements2 *pMemoryRequirements) { RADV_FROM_HANDLE(radv_device, device, _device); @@ -5853,631 +5017,6 @@ radv_BindImageMemory2(VkDevice _device, uint32_t bindInfoCount, return VK_SUCCESS; } -static bool -radv_sparse_bind_has_effects(const VkBindSparseInfo *info) -{ - return info->bufferBindCount || info->imageOpaqueBindCount || info->imageBindCount || - info->waitSemaphoreCount || info->signalSemaphoreCount; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_QueueBindSparse(VkQueue _queue, uint32_t bindInfoCount, const VkBindSparseInfo *pBindInfo, - VkFence fence) -{ - RADV_FROM_HANDLE(radv_queue, queue, _queue); - uint32_t fence_idx = 0; - - if (vk_device_is_lost(&queue->device->vk)) - return VK_ERROR_DEVICE_LOST; - - if (fence != VK_NULL_HANDLE) { - for (uint32_t i = 0; i < bindInfoCount; ++i) - if (radv_sparse_bind_has_effects(pBindInfo + i)) - fence_idx = i; - } else - fence_idx = UINT32_MAX; - - for (uint32_t i = 0; i < bindInfoCount; ++i) { - if (i != fence_idx && !radv_sparse_bind_has_effects(pBindInfo + i)) - continue; - - const VkTimelineSemaphoreSubmitInfo *timeline_info = - vk_find_struct_const(pBindInfo[i].pNext, TIMELINE_SEMAPHORE_SUBMIT_INFO); - const uint64_t *wait_values = timeline_info && timeline_info->waitSemaphoreValueCount ? - timeline_info->pWaitSemaphoreValues : NULL; - const uint64_t *signal_values = timeline_info && timeline_info->signalSemaphoreValueCount ? - timeline_info->pSignalSemaphoreValues : NULL; - - VkSemaphoreSubmitInfoKHR *wait_semaphore_infos = - malloc(sizeof(*wait_semaphore_infos) * pBindInfo[i].waitSemaphoreCount); - VkSemaphoreSubmitInfoKHR *signal_semaphore_infos = - malloc(sizeof(*signal_semaphore_infos) * pBindInfo[i].signalSemaphoreCount); - - if (!wait_semaphore_infos || !signal_semaphore_infos) { - free(wait_semaphore_infos); - free(signal_semaphore_infos); - return VK_ERROR_OUT_OF_HOST_MEMORY; - } - - for (uint32_t j = 0; j < pBindInfo[i].waitSemaphoreCount; j++) { - wait_semaphore_infos[j] = (VkSemaphoreSubmitInfoKHR) { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO_KHR, - .semaphore = pBindInfo[i].pWaitSemaphores[j], - .value = wait_values ? wait_values[j] : 0, - }; - } - - for (uint32_t j = 0; j < pBindInfo[i].signalSemaphoreCount; j++) { - signal_semaphore_infos[j] = (VkSemaphoreSubmitInfoKHR) { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO_KHR, - .semaphore = pBindInfo[i].pSignalSemaphores[j], - .value = signal_values ? signal_values[j] : 0, - }; - } - - VkResult result = radv_queue_submit( - queue, &(struct radv_queue_submission){ - .buffer_binds = pBindInfo[i].pBufferBinds, - .buffer_bind_count = pBindInfo[i].bufferBindCount, - .image_opaque_binds = pBindInfo[i].pImageOpaqueBinds, - .image_opaque_bind_count = pBindInfo[i].imageOpaqueBindCount, - .image_binds = pBindInfo[i].pImageBinds, - .image_bind_count = pBindInfo[i].imageBindCount, - .wait_semaphore_infos = wait_semaphore_infos, - .wait_semaphore_info_count = pBindInfo[i].waitSemaphoreCount, - .signal_semaphore_infos = signal_semaphore_infos, - .signal_semaphore_info_count = pBindInfo[i].signalSemaphoreCount, - .fence = i == fence_idx ? fence : VK_NULL_HANDLE, - }); - - free(wait_semaphore_infos); - free(signal_semaphore_infos); - - if (result != VK_SUCCESS) - return result; - } - - if (fence != VK_NULL_HANDLE && !bindInfoCount) { - VkResult result = radv_signal_fence(queue, fence); - if (result != VK_SUCCESS) - return result; - } - - return VK_SUCCESS; -} - -static void -radv_destroy_fence_part(struct radv_device *device, struct radv_fence_part *part) -{ - if (part->kind != RADV_FENCE_NONE) - device->ws->destroy_syncobj(device->ws, part->syncobj); - part->kind = RADV_FENCE_NONE; -} - -static void -radv_destroy_fence(struct radv_device *device, const VkAllocationCallbacks *pAllocator, - struct radv_fence *fence) -{ - radv_destroy_fence_part(device, &fence->temporary); - radv_destroy_fence_part(device, &fence->permanent); - - vk_object_base_finish(&fence->base); - vk_free2(&device->vk.alloc, pAllocator, fence); -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_CreateFence(VkDevice _device, const VkFenceCreateInfo *pCreateInfo, - const VkAllocationCallbacks *pAllocator, VkFence *pFence) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - bool create_signaled = false; - struct radv_fence *fence; - int ret; - - fence = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*fence), 8, - VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); - if (!fence) - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - - vk_object_base_init(&device->vk, &fence->base, VK_OBJECT_TYPE_FENCE); - - fence->permanent.kind = RADV_FENCE_SYNCOBJ; - - if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) - create_signaled = true; - - ret = device->ws->create_syncobj(device->ws, create_signaled, &fence->permanent.syncobj); - if (ret) { - radv_destroy_fence(device, pAllocator, fence); - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - } - - *pFence = radv_fence_to_handle(fence); - - return VK_SUCCESS; -} - -VKAPI_ATTR void VKAPI_CALL -radv_DestroyFence(VkDevice _device, VkFence _fence, const VkAllocationCallbacks *pAllocator) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_fence, fence, _fence); - - if (!fence) - return; - - radv_destroy_fence(device, pAllocator, fence); -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_WaitForFences(VkDevice _device, uint32_t fenceCount, const VkFence *pFences, VkBool32 waitAll, - uint64_t timeout) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - uint32_t *handles; - - if (vk_device_is_lost(&device->vk)) - return VK_ERROR_DEVICE_LOST; - - timeout = radv_get_absolute_timeout(timeout); - - handles = malloc(sizeof(uint32_t) * fenceCount); - if (!handles) - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - - for (uint32_t i = 0; i < fenceCount; ++i) { - RADV_FROM_HANDLE(radv_fence, fence, pFences[i]); - - struct radv_fence_part *part = - fence->temporary.kind != RADV_FENCE_NONE ? &fence->temporary : &fence->permanent; - - assert(part->kind == RADV_FENCE_SYNCOBJ); - handles[i] = part->syncobj; - } - - bool success = device->ws->wait_syncobj(device->ws, handles, fenceCount, waitAll, timeout); - free(handles); - return success ? VK_SUCCESS : VK_TIMEOUT; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_ResetFences(VkDevice _device, uint32_t fenceCount, const VkFence *pFences) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - - for (unsigned i = 0; i < fenceCount; ++i) { - RADV_FROM_HANDLE(radv_fence, fence, pFences[i]); - - /* From the Vulkan 1.0.53 spec: - * - * "If any member of pFences currently has its payload - * imported with temporary permanence, that fence’s prior - * permanent payload is irst restored. The remaining - * operations described therefore operate on the restored - * payload." - */ - if (fence->temporary.kind != RADV_FENCE_NONE) - radv_destroy_fence_part(device, &fence->temporary); - - device->ws->reset_syncobj(device->ws, fence->permanent.syncobj); - } - - return VK_SUCCESS; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_GetFenceStatus(VkDevice _device, VkFence _fence) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_fence, fence, _fence); - - struct radv_fence_part *part = - fence->temporary.kind != RADV_FENCE_NONE ? &fence->temporary : &fence->permanent; - - if (vk_device_is_lost(&device->vk)) - return VK_ERROR_DEVICE_LOST; - - bool success = device->ws->wait_syncobj(device->ws, &part->syncobj, 1, true, 0); - return success ? VK_SUCCESS : VK_NOT_READY; -} - -// Queue semaphore functions - -static void -radv_create_timeline(struct radv_timeline *timeline, uint64_t value) -{ - timeline->highest_signaled = value; - timeline->highest_submitted = value; - list_inithead(&timeline->points); - list_inithead(&timeline->free_points); - list_inithead(&timeline->waiters); - mtx_init(&timeline->mutex, mtx_plain); -} - -static void -radv_destroy_timeline(struct radv_device *device, struct radv_timeline *timeline) -{ - list_for_each_entry_safe(struct radv_timeline_point, point, &timeline->free_points, list) - { - list_del(&point->list); - device->ws->destroy_syncobj(device->ws, point->syncobj); - free(point); - } - list_for_each_entry_safe(struct radv_timeline_point, point, &timeline->points, list) - { - list_del(&point->list); - device->ws->destroy_syncobj(device->ws, point->syncobj); - free(point); - } - mtx_destroy(&timeline->mutex); -} - -static void -radv_timeline_gc_locked(struct radv_device *device, struct radv_timeline *timeline) -{ - list_for_each_entry_safe(struct radv_timeline_point, point, &timeline->points, list) - { - if (point->wait_count || point->value > timeline->highest_submitted) - return; - - if (device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, 0)) { - timeline->highest_signaled = point->value; - list_del(&point->list); - list_add(&point->list, &timeline->free_points); - } - } -} - -static struct radv_timeline_point * -radv_timeline_find_point_at_least_locked(struct radv_device *device, struct radv_timeline *timeline, - uint64_t p) -{ - radv_timeline_gc_locked(device, timeline); - - if (p <= timeline->highest_signaled) - return NULL; - - list_for_each_entry(struct radv_timeline_point, point, &timeline->points, list) - { - if (point->value >= p) { - ++point->wait_count; - return point; - } - } - return NULL; -} - -static struct radv_timeline_point * -radv_timeline_add_point_locked(struct radv_device *device, struct radv_timeline *timeline, - uint64_t p) -{ - radv_timeline_gc_locked(device, timeline); - - struct radv_timeline_point *ret = NULL; - struct radv_timeline_point *prev = NULL; - int r; - - if (p <= timeline->highest_signaled) - return NULL; - - list_for_each_entry(struct radv_timeline_point, point, &timeline->points, list) - { - if (point->value == p) { - return NULL; - } - - if (point->value < p) - prev = point; - } - - if (list_is_empty(&timeline->free_points)) { - ret = malloc(sizeof(struct radv_timeline_point)); - r = device->ws->create_syncobj(device->ws, false, &ret->syncobj); - if (r) { - free(ret); - return NULL; - } - } else { - ret = list_first_entry(&timeline->free_points, struct radv_timeline_point, list); - list_del(&ret->list); - - device->ws->reset_syncobj(device->ws, ret->syncobj); - } - - ret->value = p; - ret->wait_count = 1; - - if (prev) { - list_add(&ret->list, &prev->list); - } else { - list_addtail(&ret->list, &timeline->points); - } - return ret; -} - -static VkResult -radv_timeline_wait(struct radv_device *device, struct radv_timeline *timeline, uint64_t value, - uint64_t abs_timeout) -{ - mtx_lock(&timeline->mutex); - - while (timeline->highest_submitted < value) { - struct timespec abstime; - timespec_from_nsec(&abstime, abs_timeout); - - u_cnd_monotonic_timedwait(&device->timeline_cond, &timeline->mutex, &abstime); - - if (radv_get_current_time() >= abs_timeout && timeline->highest_submitted < value) { - mtx_unlock(&timeline->mutex); - return VK_TIMEOUT; - } - } - - struct radv_timeline_point *point = - radv_timeline_find_point_at_least_locked(device, timeline, value); - mtx_unlock(&timeline->mutex); - if (!point) - return VK_SUCCESS; - - bool success = device->ws->wait_syncobj(device->ws, &point->syncobj, 1, true, abs_timeout); - - mtx_lock(&timeline->mutex); - point->wait_count--; - mtx_unlock(&timeline->mutex); - return success ? VK_SUCCESS : VK_TIMEOUT; -} - -static void -radv_timeline_trigger_waiters_locked(struct radv_timeline *timeline, - struct list_head *processing_list) -{ - list_for_each_entry_safe(struct radv_timeline_waiter, waiter, &timeline->waiters, list) - { - if (waiter->value > timeline->highest_submitted) - continue; - - radv_queue_trigger_submission(waiter->submission, 1, processing_list); - list_del(&waiter->list); - } -} - -static void -radv_destroy_semaphore_part(struct radv_device *device, struct radv_semaphore_part *part) -{ - switch (part->kind) { - case RADV_SEMAPHORE_NONE: - break; - case RADV_SEMAPHORE_TIMELINE: - radv_destroy_timeline(device, &part->timeline); - break; - case RADV_SEMAPHORE_SYNCOBJ: - case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: - device->ws->destroy_syncobj(device->ws, part->syncobj); - break; - } - part->kind = RADV_SEMAPHORE_NONE; -} - -static VkSemaphoreType -radv_get_semaphore_type(const void *pNext, uint64_t *initial_value) -{ - const VkSemaphoreTypeCreateInfo *type_info = - vk_find_struct_const(pNext, SEMAPHORE_TYPE_CREATE_INFO); - - if (!type_info) - return VK_SEMAPHORE_TYPE_BINARY; - - if (initial_value) - *initial_value = type_info->initialValue; - return type_info->semaphoreType; -} - -static void -radv_destroy_semaphore(struct radv_device *device, const VkAllocationCallbacks *pAllocator, - struct radv_semaphore *sem) -{ - radv_destroy_semaphore_part(device, &sem->temporary); - radv_destroy_semaphore_part(device, &sem->permanent); - vk_object_base_finish(&sem->base); - vk_free2(&device->vk.alloc, pAllocator, sem); -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_CreateSemaphore(VkDevice _device, const VkSemaphoreCreateInfo *pCreateInfo, - const VkAllocationCallbacks *pAllocator, VkSemaphore *pSemaphore) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - uint64_t initial_value = 0; - VkSemaphoreType type = radv_get_semaphore_type(pCreateInfo->pNext, &initial_value); - - struct radv_semaphore *sem = - vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*sem), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); - if (!sem) - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - - vk_object_base_init(&device->vk, &sem->base, VK_OBJECT_TYPE_SEMAPHORE); - - sem->temporary.kind = RADV_SEMAPHORE_NONE; - sem->permanent.kind = RADV_SEMAPHORE_NONE; - - if (type == VK_SEMAPHORE_TYPE_TIMELINE && - device->physical_device->rad_info.has_timeline_syncobj) { - int ret = device->ws->create_syncobj(device->ws, false, &sem->permanent.syncobj); - if (ret) { - radv_destroy_semaphore(device, pAllocator, sem); - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - } - device->ws->signal_syncobj(device->ws, sem->permanent.syncobj, initial_value); - sem->permanent.timeline_syncobj.max_point = initial_value; - sem->permanent.kind = RADV_SEMAPHORE_TIMELINE_SYNCOBJ; - } else if (type == VK_SEMAPHORE_TYPE_TIMELINE) { - radv_create_timeline(&sem->permanent.timeline, initial_value); - sem->permanent.kind = RADV_SEMAPHORE_TIMELINE; - } else { - int ret = device->ws->create_syncobj(device->ws, false, &sem->permanent.syncobj); - if (ret) { - radv_destroy_semaphore(device, pAllocator, sem); - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - } - sem->permanent.kind = RADV_SEMAPHORE_SYNCOBJ; - } - - *pSemaphore = radv_semaphore_to_handle(sem); - return VK_SUCCESS; -} - -VKAPI_ATTR void VKAPI_CALL -radv_DestroySemaphore(VkDevice _device, VkSemaphore _semaphore, - const VkAllocationCallbacks *pAllocator) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_semaphore, sem, _semaphore); - if (!_semaphore) - return; - - radv_destroy_semaphore(device, pAllocator, sem); -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_GetSemaphoreCounterValue(VkDevice _device, VkSemaphore _semaphore, uint64_t *pValue) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_semaphore, semaphore, _semaphore); - - if (vk_device_is_lost(&device->vk)) - return VK_ERROR_DEVICE_LOST; - - struct radv_semaphore_part *part = semaphore->temporary.kind != RADV_SEMAPHORE_NONE - ? &semaphore->temporary - : &semaphore->permanent; - - switch (part->kind) { - case RADV_SEMAPHORE_TIMELINE: { - mtx_lock(&part->timeline.mutex); - radv_timeline_gc_locked(device, &part->timeline); - *pValue = part->timeline.highest_signaled; - mtx_unlock(&part->timeline.mutex); - return VK_SUCCESS; - } - case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: { - return device->ws->query_syncobj(device->ws, part->syncobj, pValue); - } - case RADV_SEMAPHORE_NONE: - case RADV_SEMAPHORE_SYNCOBJ: - unreachable("Invalid semaphore type"); - } - unreachable("Unhandled semaphore type"); -} - -static VkResult -radv_wait_timelines(struct radv_device *device, const VkSemaphoreWaitInfo *pWaitInfo, - uint64_t abs_timeout) -{ - if ((pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT_KHR) && pWaitInfo->semaphoreCount > 1) { - for (;;) { - for (uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) { - RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]); - VkResult result = - radv_timeline_wait(device, &semaphore->permanent.timeline, pWaitInfo->pValues[i], 0); - - if (result == VK_SUCCESS) - return VK_SUCCESS; - } - if (radv_get_current_time() > abs_timeout) - return VK_TIMEOUT; - } - } - - for (uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) { - RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]); - VkResult result = radv_timeline_wait(device, &semaphore->permanent.timeline, - pWaitInfo->pValues[i], abs_timeout); - - if (result != VK_SUCCESS) - return result; - } - return VK_SUCCESS; -} -VKAPI_ATTR VkResult VKAPI_CALL -radv_WaitSemaphores(VkDevice _device, const VkSemaphoreWaitInfo *pWaitInfo, uint64_t timeout) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - - if (vk_device_is_lost(&device->vk)) - return VK_ERROR_DEVICE_LOST; - - uint64_t abs_timeout = radv_get_absolute_timeout(timeout); - - if (radv_semaphore_from_handle(pWaitInfo->pSemaphores[0])->permanent.kind == - RADV_SEMAPHORE_TIMELINE) - return radv_wait_timelines(device, pWaitInfo, abs_timeout); - - if (pWaitInfo->semaphoreCount > UINT32_MAX / sizeof(uint32_t)) - return vk_errorf(device, VK_ERROR_OUT_OF_HOST_MEMORY, - "semaphoreCount integer overflow"); - - bool wait_all = !(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT_KHR); - uint32_t *handles = malloc(sizeof(*handles) * pWaitInfo->semaphoreCount); - if (!handles) - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - - for (uint32_t i = 0; i < pWaitInfo->semaphoreCount; ++i) { - RADV_FROM_HANDLE(radv_semaphore, semaphore, pWaitInfo->pSemaphores[i]); - handles[i] = semaphore->permanent.syncobj; - } - - VkResult result = - device->ws->wait_timeline_syncobj(device->ws, handles, pWaitInfo->pValues, - pWaitInfo->semaphoreCount, wait_all, false, abs_timeout); - free(handles); - return result; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_SignalSemaphore(VkDevice _device, const VkSemaphoreSignalInfo *pSignalInfo) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_semaphore, semaphore, pSignalInfo->semaphore); - - struct radv_semaphore_part *part = semaphore->temporary.kind != RADV_SEMAPHORE_NONE - ? &semaphore->temporary - : &semaphore->permanent; - - switch (part->kind) { - case RADV_SEMAPHORE_TIMELINE: { - mtx_lock(&part->timeline.mutex); - radv_timeline_gc_locked(device, &part->timeline); - part->timeline.highest_submitted = MAX2(part->timeline.highest_submitted, pSignalInfo->value); - part->timeline.highest_signaled = MAX2(part->timeline.highest_signaled, pSignalInfo->value); - - struct list_head processing_list; - list_inithead(&processing_list); - radv_timeline_trigger_waiters_locked(&part->timeline, &processing_list); - mtx_unlock(&part->timeline.mutex); - - VkResult result = radv_process_submissions(&processing_list); - - /* This needs to happen after radv_process_submissions, so - * that any submitted submissions that are now unblocked get - * processed before we wake the application. This way we - * ensure that any binary semaphores that are now unblocked - * are usable by the application. */ - u_cnd_monotonic_broadcast(&device->timeline_cond); - - return result; - } - case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: { - part->timeline_syncobj.max_point = MAX2(part->timeline_syncobj.max_point, pSignalInfo->value); - device->ws->signal_syncobj(device->ws, part->syncobj, pSignalInfo->value); - break; - } - case RADV_SEMAPHORE_NONE: - case RADV_SEMAPHORE_SYNCOBJ: - unreachable("Invalid semaphore type"); - } - return VK_SUCCESS; -} - static void radv_destroy_event(struct radv_device *device, const VkAllocationCallbacks *pAllocator, struct radv_event *event) @@ -7728,280 +6267,6 @@ radv_GetMemoryFdPropertiesKHR(VkDevice _device, VkExternalMemoryHandleTypeFlagBi } } -static VkResult -radv_import_opaque_fd(struct radv_device *device, int fd, uint32_t *syncobj) -{ - uint32_t syncobj_handle = 0; - int ret = device->ws->import_syncobj(device->ws, fd, &syncobj_handle); - if (ret != 0) - return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE); - - if (*syncobj) - device->ws->destroy_syncobj(device->ws, *syncobj); - - *syncobj = syncobj_handle; - close(fd); - - return VK_SUCCESS; -} - -static VkResult -radv_import_sync_fd(struct radv_device *device, int fd, uint32_t *syncobj) -{ - /* If we create a syncobj we do it locally so that if we have an error, we don't - * leave a syncobj in an undetermined state in the fence. */ - uint32_t syncobj_handle = *syncobj; - if (!syncobj_handle) { - bool create_signaled = fd == -1 ? true : false; - - int ret = device->ws->create_syncobj(device->ws, create_signaled, &syncobj_handle); - if (ret) { - return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); - } - } else { - if (fd == -1) - device->ws->signal_syncobj(device->ws, syncobj_handle, 0); - } - - if (fd != -1) { - int ret = device->ws->import_syncobj_from_sync_file(device->ws, syncobj_handle, fd); - if (ret) - return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE); - close(fd); - } - - *syncobj = syncobj_handle; - - return VK_SUCCESS; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_ImportSemaphoreFdKHR(VkDevice _device, - const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_semaphore, sem, pImportSemaphoreFdInfo->semaphore); - VkResult result; - struct radv_semaphore_part *dst = NULL; - bool timeline = sem->permanent.kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ; - - if (pImportSemaphoreFdInfo->flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) { - assert(!timeline); - dst = &sem->temporary; - } else { - dst = &sem->permanent; - } - - uint32_t syncobj = - (dst->kind == RADV_SEMAPHORE_SYNCOBJ || dst->kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ) - ? dst->syncobj - : 0; - - switch (pImportSemaphoreFdInfo->handleType) { - case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: - result = radv_import_opaque_fd(device, pImportSemaphoreFdInfo->fd, &syncobj); - break; - case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT: - assert(!timeline); - result = radv_import_sync_fd(device, pImportSemaphoreFdInfo->fd, &syncobj); - break; - default: - unreachable("Unhandled semaphore handle type"); - } - - if (result == VK_SUCCESS) { - dst->syncobj = syncobj; - dst->kind = RADV_SEMAPHORE_SYNCOBJ; - if (timeline) { - dst->kind = RADV_SEMAPHORE_TIMELINE_SYNCOBJ; - dst->timeline_syncobj.max_point = 0; - } - } - - return result; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_GetSemaphoreFdKHR(VkDevice _device, const VkSemaphoreGetFdInfoKHR *pGetFdInfo, int *pFd) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_semaphore, sem, pGetFdInfo->semaphore); - int ret; - uint32_t syncobj_handle; - - if (sem->temporary.kind != RADV_SEMAPHORE_NONE) { - assert(sem->temporary.kind == RADV_SEMAPHORE_SYNCOBJ || - sem->temporary.kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ); - syncobj_handle = sem->temporary.syncobj; - } else { - assert(sem->permanent.kind == RADV_SEMAPHORE_SYNCOBJ || - sem->permanent.kind == RADV_SEMAPHORE_TIMELINE_SYNCOBJ); - syncobj_handle = sem->permanent.syncobj; - } - - switch (pGetFdInfo->handleType) { - case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: - ret = device->ws->export_syncobj(device->ws, syncobj_handle, pFd); - if (ret) - return vk_error(device, VK_ERROR_TOO_MANY_OBJECTS); - break; - case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT: - ret = device->ws->export_syncobj_to_sync_file(device->ws, syncobj_handle, pFd); - if (ret) - return vk_error(device, VK_ERROR_TOO_MANY_OBJECTS); - - if (sem->temporary.kind != RADV_SEMAPHORE_NONE) { - radv_destroy_semaphore_part(device, &sem->temporary); - } else { - device->ws->reset_syncobj(device->ws, syncobj_handle); - } - break; - default: - unreachable("Unhandled semaphore handle type"); - } - - return VK_SUCCESS; -} - -VKAPI_ATTR void VKAPI_CALL -radv_GetPhysicalDeviceExternalSemaphoreProperties( - VkPhysicalDevice physicalDevice, - const VkPhysicalDeviceExternalSemaphoreInfo *pExternalSemaphoreInfo, - VkExternalSemaphoreProperties *pExternalSemaphoreProperties) -{ - RADV_FROM_HANDLE(radv_physical_device, pdevice, physicalDevice); - VkSemaphoreType type = radv_get_semaphore_type(pExternalSemaphoreInfo->pNext, NULL); - - if (type == VK_SEMAPHORE_TYPE_TIMELINE && pdevice->rad_info.has_timeline_syncobj && - pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) { - pExternalSemaphoreProperties->exportFromImportedHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - pExternalSemaphoreProperties->compatibleHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - pExternalSemaphoreProperties->externalSemaphoreFeatures = - VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | - VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; - } else if (type == VK_SEMAPHORE_TYPE_TIMELINE) { - pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0; - pExternalSemaphoreProperties->compatibleHandleTypes = 0; - pExternalSemaphoreProperties->externalSemaphoreFeatures = 0; - } else if (pExternalSemaphoreInfo->handleType == - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT || - pExternalSemaphoreInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT) { - pExternalSemaphoreProperties->exportFromImportedHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; - pExternalSemaphoreProperties->compatibleHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT | - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; - pExternalSemaphoreProperties->externalSemaphoreFeatures = - VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | - VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; - } else if (pExternalSemaphoreInfo->handleType == - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) { - pExternalSemaphoreProperties->exportFromImportedHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - pExternalSemaphoreProperties->compatibleHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - pExternalSemaphoreProperties->externalSemaphoreFeatures = - VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | - VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; - } else { - pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0; - pExternalSemaphoreProperties->compatibleHandleTypes = 0; - pExternalSemaphoreProperties->externalSemaphoreFeatures = 0; - } -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_ImportFenceFdKHR(VkDevice _device, const VkImportFenceFdInfoKHR *pImportFenceFdInfo) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_fence, fence, pImportFenceFdInfo->fence); - struct radv_fence_part *dst = NULL; - VkResult result; - - if (pImportFenceFdInfo->flags & VK_FENCE_IMPORT_TEMPORARY_BIT) { - dst = &fence->temporary; - } else { - dst = &fence->permanent; - } - - uint32_t syncobj = dst->kind == RADV_FENCE_SYNCOBJ ? dst->syncobj : 0; - - switch (pImportFenceFdInfo->handleType) { - case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: - result = radv_import_opaque_fd(device, pImportFenceFdInfo->fd, &syncobj); - break; - case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: - result = radv_import_sync_fd(device, pImportFenceFdInfo->fd, &syncobj); - break; - default: - unreachable("Unhandled fence handle type"); - } - - if (result == VK_SUCCESS) { - dst->syncobj = syncobj; - dst->kind = RADV_FENCE_SYNCOBJ; - } - - return result; -} - -VKAPI_ATTR VkResult VKAPI_CALL -radv_GetFenceFdKHR(VkDevice _device, const VkFenceGetFdInfoKHR *pGetFdInfo, int *pFd) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - RADV_FROM_HANDLE(radv_fence, fence, pGetFdInfo->fence); - int ret; - - struct radv_fence_part *part = - fence->temporary.kind != RADV_FENCE_NONE ? &fence->temporary : &fence->permanent; - - switch (pGetFdInfo->handleType) { - case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: - ret = device->ws->export_syncobj(device->ws, part->syncobj, pFd); - if (ret) - return vk_error(device, VK_ERROR_TOO_MANY_OBJECTS); - break; - case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: - ret = device->ws->export_syncobj_to_sync_file(device->ws, part->syncobj, pFd); - if (ret) - return vk_error(device, VK_ERROR_TOO_MANY_OBJECTS); - - if (part == &fence->temporary) { - radv_destroy_fence_part(device, part); - } else { - device->ws->reset_syncobj(device->ws, part->syncobj); - } - break; - default: - unreachable("Unhandled fence handle type"); - } - - return VK_SUCCESS; -} - -VKAPI_ATTR void VKAPI_CALL -radv_GetPhysicalDeviceExternalFenceProperties( - VkPhysicalDevice physicalDevice, const VkPhysicalDeviceExternalFenceInfo *pExternalFenceInfo, - VkExternalFenceProperties *pExternalFenceProperties) -{ - if (pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT || - pExternalFenceInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT) { - pExternalFenceProperties->exportFromImportedHandleTypes = - VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; - pExternalFenceProperties->compatibleHandleTypes = - VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; - pExternalFenceProperties->externalFenceFeatures = - VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT | VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; - } else { - pExternalFenceProperties->exportFromImportedHandleTypes = 0; - pExternalFenceProperties->compatibleHandleTypes = 0; - pExternalFenceProperties->externalFenceFeatures = 0; - } -} - VKAPI_ATTR void VKAPI_CALL radv_GetDeviceGroupPeerMemoryFeatures(VkDevice device, uint32_t heapIndex, uint32_t localDeviceIndex, uint32_t remoteDeviceIndex, diff --git a/src/amd/vulkan/radv_private.h b/src/amd/vulkan/radv_private.h index 523224cf88f..8aa4ce2c347 100644 --- a/src/amd/vulkan/radv_private.h +++ b/src/amd/vulkan/radv_private.h @@ -722,17 +722,6 @@ struct radv_queue { struct radeon_cmdbuf *initial_preamble_cs; struct radeon_cmdbuf *initial_full_flush_preamble_cs; struct radeon_cmdbuf *continue_preamble_cs; - - struct list_head pending_submissions; - mtx_t pending_mutex; - - mtx_t thread_mutex; - struct u_cnd_monotonic thread_cond; - struct radv_deferred_queue_submission *thread_submission; - thrd_t submission_thread; - bool thread_exit; - bool thread_running; - bool cond_created; }; #define RADV_BORDER_COLOR_COUNT 4096 @@ -2554,71 +2543,6 @@ struct radv_query_pool { uint32_t pipeline_stats_mask; }; -typedef enum { - RADV_SEMAPHORE_NONE, - RADV_SEMAPHORE_SYNCOBJ, - RADV_SEMAPHORE_TIMELINE_SYNCOBJ, - RADV_SEMAPHORE_TIMELINE, -} radv_semaphore_kind; - -struct radv_deferred_queue_submission; - -struct radv_timeline_waiter { - struct list_head list; - struct radv_deferred_queue_submission *submission; - uint64_t value; -}; - -struct radv_timeline_point { - struct list_head list; - - uint64_t value; - uint32_t syncobj; - - /* Separate from the list to accomodate CPU wait being async, as well - * as prevent point deletion during submission. */ - unsigned wait_count; -}; - -struct radv_timeline { - mtx_t mutex; - - uint64_t highest_signaled; - uint64_t highest_submitted; - - struct list_head points; - - /* Keep free points on hand so we do not have to recreate syncobjs all - * the time. */ - struct list_head free_points; - - /* Submissions that are deferred waiting for a specific value to be - * submitted. */ - struct list_head waiters; -}; - -struct radv_timeline_syncobj { - /* Keep syncobj first, so common-code can just handle this as - * non-timeline syncobj. */ - uint32_t syncobj; - uint64_t max_point; /* max submitted point. */ -}; - -struct radv_semaphore_part { - radv_semaphore_kind kind; - union { - uint32_t syncobj; - struct radv_timeline timeline; - struct radv_timeline_syncobj timeline_syncobj; - }; -}; - -struct radv_semaphore { - struct vk_object_base base; - struct radv_semaphore_part permanent; - struct radv_semaphore_part temporary; -}; - bool radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs); void radv_set_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point, @@ -2647,24 +2571,6 @@ uint32_t radv_init_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *im uint32_t radv_init_fmask(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image, const VkImageSubresourceRange *range); -typedef enum { - RADV_FENCE_NONE, - RADV_FENCE_SYNCOBJ, -} radv_fence_kind; - -struct radv_fence_part { - radv_fence_kind kind; - - /* DRM syncobj handle for syncobj-based fences. */ - uint32_t syncobj; -}; - -struct radv_fence { - struct vk_object_base base; - struct radv_fence_part permanent; - struct radv_fence_part temporary; -}; - /* radv_nir_to_llvm.c */ struct radv_shader_args; struct radv_nir_compiler_options; @@ -2974,7 +2880,6 @@ VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_update_template, base, VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE) VK_DEFINE_NONDISP_HANDLE_CASTS(radv_device_memory, base, VkDeviceMemory, VK_OBJECT_TYPE_DEVICE_MEMORY) -VK_DEFINE_NONDISP_HANDLE_CASTS(radv_fence, base, VkFence, VK_OBJECT_TYPE_FENCE) VK_DEFINE_NONDISP_HANDLE_CASTS(radv_event, base, VkEvent, VK_OBJECT_TYPE_EVENT) VK_DEFINE_NONDISP_HANDLE_CASTS(radv_framebuffer, base, VkFramebuffer, VK_OBJECT_TYPE_FRAMEBUFFER) @@ -2996,8 +2901,6 @@ VK_DEFINE_NONDISP_HANDLE_CASTS(radv_sampler, base, VkSampler, VK_DEFINE_NONDISP_HANDLE_CASTS(radv_sampler_ycbcr_conversion, base, VkSamplerYcbcrConversion, VK_OBJECT_TYPE_SAMPLER_YCBCR_CONVERSION) -VK_DEFINE_NONDISP_HANDLE_CASTS(radv_semaphore, base, VkSemaphore, - VK_OBJECT_TYPE_SEMAPHORE) #ifdef __cplusplus } diff --git a/src/amd/vulkan/radv_radeon_winsys.h b/src/amd/vulkan/radv_radeon_winsys.h index 222e7a365e0..b7db453e61b 100644 --- a/src/amd/vulkan/radv_radeon_winsys.h +++ b/src/amd/vulkan/radv_radeon_winsys.h @@ -172,20 +172,6 @@ struct radeon_winsys_bo { bool use_global_list; enum radeon_bo_domain initial_domain; }; -struct radv_winsys_sem_counts { - uint32_t syncobj_count; - uint32_t syncobj_reset_count; /* for wait only, whether to reset the syncobj */ - uint32_t timeline_syncobj_count; - uint32_t *syncobj; - uint64_t *points; -}; - -struct radv_winsys_sem_info { - bool cs_emit_signal; - bool cs_emit_wait; - struct radv_winsys_sem_counts wait; - struct radv_winsys_sem_counts signal; -}; struct radv_winsys_bo_list { struct radeon_winsys_bo **bos; @@ -276,12 +262,6 @@ struct radeon_winsys { void (*cs_grow)(struct radeon_cmdbuf *cs, size_t min_size); - VkResult (*cs_submit)(struct radeon_winsys_ctx *ctx, enum ring_type ring_type, int queue_index, - struct radeon_cmdbuf **cs_array, unsigned cs_count, - struct radeon_cmdbuf *initial_preamble_cs, - struct radeon_cmdbuf *continue_preamble_cs, - struct radv_winsys_sem_info *sem_info, bool can_patch); - VkResult (*cs_submit2)(struct radeon_winsys_ctx *ctx, enum ring_type ring_type, int queue_index, struct radeon_cmdbuf **cs_array, unsigned cs_count, struct radeon_cmdbuf *initial_preamble_cs, diff --git a/src/amd/vulkan/radv_wsi.c b/src/amd/vulkan/radv_wsi.c index a8e79585969..d8bdde9faa7 100644 --- a/src/amd/vulkan/radv_wsi.c +++ b/src/amd/vulkan/radv_wsi.c @@ -26,6 +26,8 @@ #include "util/macros.h" #include "radv_meta.h" #include "radv_private.h" +#include "vk_fence.h" +#include "vk_semaphore.h" #include "vk_util.h" #include "wsi_common.h" @@ -59,6 +61,8 @@ radv_init_wsi(struct radv_physical_device *physical_device) physical_device->wsi_device.supports_modifiers = physical_device->rad_info.chip_class >= GFX9; physical_device->wsi_device.set_memory_ownership = radv_wsi_set_memory_ownership; + physical_device->wsi_device.signal_semaphore_with_memory = true; + physical_device->wsi_device.signal_fence_with_memory = true; wsi_device_setup_syncobj_fd(&physical_device->wsi_device, physical_device->local_fd); @@ -74,46 +78,6 @@ radv_finish_wsi(struct radv_physical_device *physical_device) wsi_device_finish(&physical_device->wsi_device, &physical_device->instance->vk.alloc); } -VKAPI_ATTR VkResult VKAPI_CALL -radv_AcquireNextImage2KHR(VkDevice _device, const VkAcquireNextImageInfoKHR *pAcquireInfo, - uint32_t *pImageIndex) -{ - RADV_FROM_HANDLE(radv_device, device, _device); - struct radv_physical_device *pdevice = device->physical_device; - RADV_FROM_HANDLE(radv_fence, fence, pAcquireInfo->fence); - RADV_FROM_HANDLE(radv_semaphore, semaphore, pAcquireInfo->semaphore); - - VkResult result = - wsi_common_acquire_next_image2(&pdevice->wsi_device, _device, pAcquireInfo, pImageIndex); - - if (result == VK_SUCCESS || result == VK_SUBOPTIMAL_KHR) { - if (fence) { - struct radv_fence_part *part = - fence->temporary.kind != RADV_FENCE_NONE ? &fence->temporary : &fence->permanent; - - device->ws->signal_syncobj(device->ws, part->syncobj, 0); - } - if (semaphore) { - struct radv_semaphore_part *part = semaphore->temporary.kind != RADV_SEMAPHORE_NONE - ? &semaphore->temporary - : &semaphore->permanent; - - switch (part->kind) { - case RADV_SEMAPHORE_NONE: - /* Do not need to do anything. */ - break; - case RADV_SEMAPHORE_TIMELINE: - case RADV_SEMAPHORE_TIMELINE_SYNCOBJ: - unreachable("WSI only allows binary semaphores."); - case RADV_SEMAPHORE_SYNCOBJ: - device->ws->signal_syncobj(device->ws, part->syncobj, 0); - break; - } - } - } - return result; -} - VKAPI_ATTR VkResult VKAPI_CALL radv_QueuePresentKHR(VkQueue _queue, const VkPresentInfoKHR *pPresentInfo) { diff --git a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c index 2b01cb716c1..ac14343d106 100644 --- a/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c +++ b/src/amd/vulkan/winsys/amdgpu/radv_amdgpu_cs.c @@ -83,6 +83,21 @@ struct radv_amdgpu_cs { unsigned num_old_cs_buffers; }; +struct radv_winsys_sem_counts { + uint32_t syncobj_count; + uint32_t syncobj_reset_count; /* for wait only, whether to reset the syncobj */ + uint32_t timeline_syncobj_count; + uint32_t *syncobj; + uint64_t *points; +}; + +struct radv_winsys_sem_info { + bool cs_emit_signal; + bool cs_emit_wait; + struct radv_winsys_sem_counts wait; + struct radv_winsys_sem_counts signal; +}; + static uint32_t radv_amdgpu_ctx_queue_syncobj(struct radv_amdgpu_ctx *ctx, unsigned ip, unsigned ring); @@ -1992,7 +2007,6 @@ radv_amdgpu_cs_init_functions(struct radv_amdgpu_winsys *ws) ws->base.cs_reset = radv_amdgpu_cs_reset; ws->base.cs_add_buffer = radv_amdgpu_cs_add_buffer; ws->base.cs_execute_secondary = radv_amdgpu_cs_execute_secondary; - ws->base.cs_submit = radv_amdgpu_winsys_cs_submit; ws->base.cs_submit2 = radv_amdgpu_winsys_cs_submit2; ws->base.cs_dump = radv_amdgpu_winsys_cs_dump; ws->base.create_syncobj = radv_amdgpu_create_syncobj;