diff --git a/src/gallium/drivers/d3d12/d3d12_batch.cpp b/src/gallium/drivers/d3d12/d3d12_batch.cpp index de85b858c49..93fa816e363 100644 --- a/src/gallium/drivers/d3d12/d3d12_batch.cpp +++ b/src/gallium/drivers/d3d12/d3d12_batch.cpp @@ -25,6 +25,7 @@ #include "d3d12_context.h" #include "d3d12_fence.h" #include "d3d12_query.h" +#include "d3d12_residency.h" #include "d3d12_resource.h" #include "d3d12_screen.h" #include "d3d12_surface.h" @@ -209,6 +210,8 @@ d3d12_end_batch(struct d3d12_context *ctx, struct d3d12_batch *batch) mtx_lock(&screen->submit_mutex); + d3d12_process_batch_residency(screen, batch); + ID3D12CommandList* cmdlists[] = { ctx->cmdlist }; screen->cmdqueue->ExecuteCommandLists(1, cmdlists); batch->fence = d3d12_create_fence(screen); diff --git a/src/gallium/drivers/d3d12/d3d12_bufmgr.h b/src/gallium/drivers/d3d12/d3d12_bufmgr.h index cdbc1a3970a..37a51d7363b 100644 --- a/src/gallium/drivers/d3d12/d3d12_bufmgr.h +++ b/src/gallium/drivers/d3d12/d3d12_bufmgr.h @@ -53,7 +53,7 @@ struct d3d12_bo { struct list_head residency_list_entry; uint64_t estimated_size; - uint64_t last_used_timestamp; + int64_t last_used_timestamp; uint64_t last_used_fence; enum d3d12_residency_status residency_status; }; diff --git a/src/gallium/drivers/d3d12/d3d12_residency.cpp b/src/gallium/drivers/d3d12/d3d12_residency.cpp new file mode 100644 index 00000000000..75654c5b610 --- /dev/null +++ b/src/gallium/drivers/d3d12/d3d12_residency.cpp @@ -0,0 +1,251 @@ +/* + * Copyright © Microsoft Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "d3d12_batch.h" +#include "d3d12_bufmgr.h" +#include "d3d12_residency.h" +#include "d3d12_screen.h" + +#include "util/os_time.h" + +#include + +static constexpr unsigned residency_batch_size = 128; + +static void +evict_aged_allocations(struct d3d12_screen *screen, uint64_t completed_fence, int64_t time, int64_t grace_period) +{ + ID3D12Pageable *to_evict[residency_batch_size]; + unsigned num_pending_evictions = 0; + + list_for_each_entry_safe(struct d3d12_bo, bo, &screen->residency_list, residency_list_entry) { + /* This residency list should all be base bos, not suballocated ones */ + assert(bo->res); + + if (bo->last_used_fence > completed_fence || + time - bo->last_used_timestamp <= grace_period) { + /* List is LRU-sorted, this bo is still in use, so we're done */ + break; + } + + if (bo->residency_status == d3d12_permanently_resident) + continue; + + to_evict[num_pending_evictions++] = bo->res; + bo->residency_status = d3d12_evicted; + list_del(&bo->residency_list_entry); + + if (num_pending_evictions == residency_batch_size) { + screen->dev->Evict(num_pending_evictions, to_evict); + num_pending_evictions = 0; + } + } + + if (num_pending_evictions) + screen->dev->Evict(num_pending_evictions, to_evict); +} + +static void +evict_to_fence_or_budget(struct d3d12_screen *screen, uint64_t target_fence, uint64_t current_usage, uint64_t target_budget) +{ + screen->fence->SetEventOnCompletion(target_fence, nullptr); + + ID3D12Pageable *to_evict[residency_batch_size]; + unsigned num_pending_evictions = 0; + + list_for_each_entry_safe(struct d3d12_bo, bo, &screen->residency_list, residency_list_entry) { + /* This residency list should all be base bos, not suballocated ones */ + assert(bo->res); + + if (bo->last_used_fence > target_fence || current_usage < target_budget) { + break; + } + + if (bo->residency_status == d3d12_permanently_resident) + continue; + + to_evict[num_pending_evictions++] = bo->res; + bo->residency_status = d3d12_evicted; + list_del(&bo->residency_list_entry); + + current_usage -= bo->estimated_size; + + if (num_pending_evictions == residency_batch_size) { + screen->dev->Evict(num_pending_evictions, to_evict); + num_pending_evictions = 0; + } + } + + if (num_pending_evictions) + screen->dev->Evict(num_pending_evictions, to_evict); +} + +static constexpr int64_t eviction_grace_period_seconds_min = 1; +static constexpr int64_t eviction_grace_period_seconds_max = 60; +static constexpr int64_t microseconds_per_second = 1000000; +static constexpr int64_t eviction_grace_period_microseconds_min = + eviction_grace_period_seconds_min * microseconds_per_second; +static constexpr int64_t eviction_grace_period_microseconds_max = + eviction_grace_period_seconds_max * microseconds_per_second; +static constexpr double trim_percentage_usage_threshold = 0.7; + +static int64_t +get_eviction_grace_period(struct d3d12_memory_info *mem_info) +{ + double pressure = double(mem_info->usage) / double(mem_info->budget); + pressure = MIN2(pressure, 1.0); + + if (pressure > trim_percentage_usage_threshold) { + /* Normalize pressure for the range [0, threshold] */ + pressure = (pressure - trim_percentage_usage_threshold) / (1.0 - trim_percentage_usage_threshold); + /* Linearly interpolate between min and max period based on pressure */ + return (int64_t)((eviction_grace_period_microseconds_max - eviction_grace_period_microseconds_min) * + (1.0 - pressure)) + eviction_grace_period_microseconds_min; + } + + /* Unlimited grace period, essentially don't trim at all */ + return INT64_MAX; +} + +void +d3d12_process_batch_residency(struct d3d12_screen *screen, struct d3d12_batch *batch) +{ + d3d12_memory_info mem_info; + screen->get_memory_info(screen, &mem_info); + + uint64_t completed_fence_value = screen->fence->GetCompletedValue(); + uint64_t pending_fence_value = screen->fence_value + 1; + int64_t current_time = os_time_get(); + int64_t grace_period = get_eviction_grace_period(&mem_info); + + /* Gather base bos for the batch */ + uint64_t size_to_make_resident = 0; + set *base_bo_set = _mesa_pointer_set_create(nullptr); + hash_table_foreach(batch->bos, entry) { + struct d3d12_bo *bo = (struct d3d12_bo *)entry->key; + uint64_t offset; + struct d3d12_bo *base_bo = d3d12_bo_get_base(bo, &offset); + + if (base_bo->residency_status == d3d12_evicted) { + bool added = false; + _mesa_set_search_or_add(base_bo_set, base_bo, &added); + assert(!added); + + base_bo->residency_status = d3d12_resident; + size_to_make_resident += base_bo->estimated_size; + list_addtail(&base_bo->residency_list_entry, &screen->residency_list); + } else if (base_bo->last_used_fence != pending_fence_value) { + /* First time seeing this already-resident base bo in this batch */ + list_del(&base_bo->residency_list_entry); + list_addtail(&base_bo->residency_list_entry, &screen->residency_list); + } + + base_bo->last_used_fence = pending_fence_value; + base_bo->last_used_timestamp = current_time; + } + + /* Now that bos referenced by this batch are moved to the end of the LRU, trim it */ + evict_aged_allocations(screen, completed_fence_value, current_time, grace_period); + + /* If there's nothing needing to be made newly resident, we're done once we've trimmed */ + if (base_bo_set->entries == 0) + return; + + uint64_t residency_fence_value_snapshot = screen->residency_fence_value; + + struct set_entry *entry = _mesa_set_next_entry(base_bo_set, nullptr); + uint64_t batch_memory_size = 0; + unsigned batch_count = 0; + ID3D12Pageable *to_make_resident[residency_batch_size]; + while (true) { + /* Refresh memory stats */ + screen->get_memory_info(screen, &mem_info); + + int64_t available_memory = (int64_t)mem_info.budget - (int64_t)mem_info.usage; + + assert(!list_is_empty(&screen->residency_list)); + struct d3d12_bo *oldest_resident_bo = + list_first_entry(&screen->residency_list, struct d3d12_bo, residency_list_entry); + bool anything_to_wait_for = oldest_resident_bo->last_used_fence < pending_fence_value; + + /* We've got some room, or we can't free up any more room, make some resources resident */ + HRESULT hr = S_OK; + if ((available_memory || !anything_to_wait_for) && batch_count < residency_batch_size) { + for (; entry; entry = _mesa_set_next_entry(base_bo_set, entry)) { + struct d3d12_bo *bo = (struct d3d12_bo *)entry->key; + if (anything_to_wait_for && + (int64_t)(batch_memory_size + bo->estimated_size) > available_memory) + break; + + batch_memory_size += bo->estimated_size; + to_make_resident[batch_count++] = bo->res; + if (batch_count == residency_batch_size) + break; + } + + if (batch_count) { + hr = screen->dev->EnqueueMakeResident(D3D12_RESIDENCY_FLAG_NONE, batch_count, to_make_resident, + screen->residency_fence, screen->residency_fence_value + 1); + if (SUCCEEDED(hr)) + ++screen->residency_fence_value; + } + + if (SUCCEEDED(hr) && batch_count == residency_batch_size) { + batch_count = 0; + size_to_make_resident -= batch_memory_size; + continue; + } + } + + /* We need to free up some space, either we broke early from the resource loop, + * or the MakeResident call itself failed. + */ + if (FAILED(hr) || entry) { + if (!anything_to_wait_for) { + assert(false); + break; + } + + evict_to_fence_or_budget(screen, oldest_resident_bo->last_used_fence, mem_info.usage + size_to_make_resident, mem_info.budget); + continue; + } + + /* Made it to the end without explicitly needing to loop, so we're done */ + break; + } + _mesa_set_destroy(base_bo_set, nullptr); + + /* The GPU needs to wait for these resources to be made resident */ + if (residency_fence_value_snapshot != screen->residency_fence_value) + screen->cmdqueue->Wait(screen->residency_fence, screen->residency_fence_value); +} + +bool +d3d12_init_residency(struct d3d12_screen *screen) +{ + list_inithead(&screen->residency_list); + if (FAILED(screen->dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&screen->residency_fence)))) + return false; + + return true; +} diff --git a/src/gallium/drivers/d3d12/d3d12_residency.h b/src/gallium/drivers/d3d12/d3d12_residency.h new file mode 100644 index 00000000000..af84e65e682 --- /dev/null +++ b/src/gallium/drivers/d3d12/d3d12_residency.h @@ -0,0 +1,33 @@ +/* + * Copyright © Microsoft Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#pragma once + +struct d3d12_screen; +struct d3d12_batch; + +void +d3d12_process_batch_residency(struct d3d12_screen *screen, struct d3d12_batch *batch); + +bool +d3d12_init_residency(struct d3d12_screen *screen); diff --git a/src/gallium/drivers/d3d12/d3d12_screen.cpp b/src/gallium/drivers/d3d12/d3d12_screen.cpp index 1a476121350..ea4c430afc9 100644 --- a/src/gallium/drivers/d3d12/d3d12_screen.cpp +++ b/src/gallium/drivers/d3d12/d3d12_screen.cpp @@ -29,6 +29,7 @@ #include "d3d12_debug.h" #include "d3d12_fence.h" #include "d3d12_format.h" +#include "d3d12_residency.h" #include "d3d12_resource.h" #include "d3d12_nir_passes.h" @@ -797,7 +798,7 @@ enable_gpu_validation() debug3->SetEnableGPUBasedValidation(true); } -static ID3D12Device * +static ID3D12Device3 * create_device(IUnknown *adapter) { typedef HRESULT(WINAPI *PFN_D3D12CREATEDEVICE)(IUnknown*, D3D_FEATURE_LEVEL, REFIID, void**); @@ -835,7 +836,7 @@ create_device(IUnknown *adapter) return NULL; } - ID3D12Device *dev; + ID3D12Device3 *dev; if (SUCCEEDED(D3D12CreateDevice(adapter, D3D_FEATURE_LEVEL_11_0, IID_PPV_ARGS(&dev)))) return dev; @@ -1188,7 +1189,8 @@ d3d12_init_screen(struct d3d12_screen *screen, struct sw_winsys *winsys, IUnknow if (FAILED(screen->dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&screen->fence)))) goto failed; - list_inithead(&screen->residency_list); + if (!d3d12_init_residency(screen)) + goto failed; UINT64 timestamp_freq; if (FAILED(screen->cmdqueue->GetTimestampFrequency(×tamp_freq))) diff --git a/src/gallium/drivers/d3d12/d3d12_screen.h b/src/gallium/drivers/d3d12/d3d12_screen.h index 2e52ea941f9..c4b137141aa 100644 --- a/src/gallium/drivers/d3d12/d3d12_screen.h +++ b/src/gallium/drivers/d3d12/d3d12_screen.h @@ -65,14 +65,17 @@ struct d3d12_screen { struct pipe_screen base; struct sw_winsys *winsys; - ID3D12Device *dev; + ID3D12Device3 *dev; ID3D12CommandQueue *cmdqueue; void (*get_memory_info)(struct d3d12_screen *screen, struct d3d12_memory_info *output); mtx_t submit_mutex; ID3D12Fence *fence; uint64_t fence_value; + struct list_head residency_list; + ID3D12Fence *residency_fence; + uint64_t residency_fence_value; struct slab_parent_pool transfer_pool; struct pb_manager *bufmgr; diff --git a/src/gallium/drivers/d3d12/meson.build b/src/gallium/drivers/d3d12/meson.build index 41fe141d68b..0bca92d4042 100644 --- a/src/gallium/drivers/d3d12/meson.build +++ b/src/gallium/drivers/d3d12/meson.build @@ -40,6 +40,7 @@ files_libd3d12 = files( 'd3d12_nir_passes.c', 'd3d12_pipeline_state.cpp', 'd3d12_query.cpp', + 'd3d12_residency.cpp', 'd3d12_resource.cpp', 'd3d12_root_signature.cpp', 'd3d12_screen.cpp',