d3d12: Implement residency management algorithm
Reviewed-by: Bill Kristiansen <billkris@microsoft.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/14959>
This commit is contained in:
parent
40dafd0094
commit
e3a2cb4b67
|
@ -25,6 +25,7 @@
|
||||||
#include "d3d12_context.h"
|
#include "d3d12_context.h"
|
||||||
#include "d3d12_fence.h"
|
#include "d3d12_fence.h"
|
||||||
#include "d3d12_query.h"
|
#include "d3d12_query.h"
|
||||||
|
#include "d3d12_residency.h"
|
||||||
#include "d3d12_resource.h"
|
#include "d3d12_resource.h"
|
||||||
#include "d3d12_screen.h"
|
#include "d3d12_screen.h"
|
||||||
#include "d3d12_surface.h"
|
#include "d3d12_surface.h"
|
||||||
|
@ -209,6 +210,8 @@ d3d12_end_batch(struct d3d12_context *ctx, struct d3d12_batch *batch)
|
||||||
|
|
||||||
mtx_lock(&screen->submit_mutex);
|
mtx_lock(&screen->submit_mutex);
|
||||||
|
|
||||||
|
d3d12_process_batch_residency(screen, batch);
|
||||||
|
|
||||||
ID3D12CommandList* cmdlists[] = { ctx->cmdlist };
|
ID3D12CommandList* cmdlists[] = { ctx->cmdlist };
|
||||||
screen->cmdqueue->ExecuteCommandLists(1, cmdlists);
|
screen->cmdqueue->ExecuteCommandLists(1, cmdlists);
|
||||||
batch->fence = d3d12_create_fence(screen);
|
batch->fence = d3d12_create_fence(screen);
|
||||||
|
|
|
@ -53,7 +53,7 @@ struct d3d12_bo {
|
||||||
|
|
||||||
struct list_head residency_list_entry;
|
struct list_head residency_list_entry;
|
||||||
uint64_t estimated_size;
|
uint64_t estimated_size;
|
||||||
uint64_t last_used_timestamp;
|
int64_t last_used_timestamp;
|
||||||
uint64_t last_used_fence;
|
uint64_t last_used_fence;
|
||||||
enum d3d12_residency_status residency_status;
|
enum d3d12_residency_status residency_status;
|
||||||
};
|
};
|
||||||
|
|
|
@ -0,0 +1,251 @@
|
||||||
|
/*
|
||||||
|
* Copyright © Microsoft Corporation
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
* copy of this software and associated documentation files (the "Software"),
|
||||||
|
* to deal in the Software without restriction, including without limitation
|
||||||
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
* and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice (including the next
|
||||||
|
* paragraph) shall be included in all copies or substantial portions of the
|
||||||
|
* Software.
|
||||||
|
*
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||||
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||||
|
* IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "d3d12_batch.h"
|
||||||
|
#include "d3d12_bufmgr.h"
|
||||||
|
#include "d3d12_residency.h"
|
||||||
|
#include "d3d12_screen.h"
|
||||||
|
|
||||||
|
#include "util/os_time.h"
|
||||||
|
|
||||||
|
#include <dxguids/dxguids.h>
|
||||||
|
|
||||||
|
static constexpr unsigned residency_batch_size = 128;
|
||||||
|
|
||||||
|
static void
|
||||||
|
evict_aged_allocations(struct d3d12_screen *screen, uint64_t completed_fence, int64_t time, int64_t grace_period)
|
||||||
|
{
|
||||||
|
ID3D12Pageable *to_evict[residency_batch_size];
|
||||||
|
unsigned num_pending_evictions = 0;
|
||||||
|
|
||||||
|
list_for_each_entry_safe(struct d3d12_bo, bo, &screen->residency_list, residency_list_entry) {
|
||||||
|
/* This residency list should all be base bos, not suballocated ones */
|
||||||
|
assert(bo->res);
|
||||||
|
|
||||||
|
if (bo->last_used_fence > completed_fence ||
|
||||||
|
time - bo->last_used_timestamp <= grace_period) {
|
||||||
|
/* List is LRU-sorted, this bo is still in use, so we're done */
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bo->residency_status == d3d12_permanently_resident)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
to_evict[num_pending_evictions++] = bo->res;
|
||||||
|
bo->residency_status = d3d12_evicted;
|
||||||
|
list_del(&bo->residency_list_entry);
|
||||||
|
|
||||||
|
if (num_pending_evictions == residency_batch_size) {
|
||||||
|
screen->dev->Evict(num_pending_evictions, to_evict);
|
||||||
|
num_pending_evictions = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (num_pending_evictions)
|
||||||
|
screen->dev->Evict(num_pending_evictions, to_evict);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
evict_to_fence_or_budget(struct d3d12_screen *screen, uint64_t target_fence, uint64_t current_usage, uint64_t target_budget)
|
||||||
|
{
|
||||||
|
screen->fence->SetEventOnCompletion(target_fence, nullptr);
|
||||||
|
|
||||||
|
ID3D12Pageable *to_evict[residency_batch_size];
|
||||||
|
unsigned num_pending_evictions = 0;
|
||||||
|
|
||||||
|
list_for_each_entry_safe(struct d3d12_bo, bo, &screen->residency_list, residency_list_entry) {
|
||||||
|
/* This residency list should all be base bos, not suballocated ones */
|
||||||
|
assert(bo->res);
|
||||||
|
|
||||||
|
if (bo->last_used_fence > target_fence || current_usage < target_budget) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bo->residency_status == d3d12_permanently_resident)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
to_evict[num_pending_evictions++] = bo->res;
|
||||||
|
bo->residency_status = d3d12_evicted;
|
||||||
|
list_del(&bo->residency_list_entry);
|
||||||
|
|
||||||
|
current_usage -= bo->estimated_size;
|
||||||
|
|
||||||
|
if (num_pending_evictions == residency_batch_size) {
|
||||||
|
screen->dev->Evict(num_pending_evictions, to_evict);
|
||||||
|
num_pending_evictions = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (num_pending_evictions)
|
||||||
|
screen->dev->Evict(num_pending_evictions, to_evict);
|
||||||
|
}
|
||||||
|
|
||||||
|
static constexpr int64_t eviction_grace_period_seconds_min = 1;
|
||||||
|
static constexpr int64_t eviction_grace_period_seconds_max = 60;
|
||||||
|
static constexpr int64_t microseconds_per_second = 1000000;
|
||||||
|
static constexpr int64_t eviction_grace_period_microseconds_min =
|
||||||
|
eviction_grace_period_seconds_min * microseconds_per_second;
|
||||||
|
static constexpr int64_t eviction_grace_period_microseconds_max =
|
||||||
|
eviction_grace_period_seconds_max * microseconds_per_second;
|
||||||
|
static constexpr double trim_percentage_usage_threshold = 0.7;
|
||||||
|
|
||||||
|
static int64_t
|
||||||
|
get_eviction_grace_period(struct d3d12_memory_info *mem_info)
|
||||||
|
{
|
||||||
|
double pressure = double(mem_info->usage) / double(mem_info->budget);
|
||||||
|
pressure = MIN2(pressure, 1.0);
|
||||||
|
|
||||||
|
if (pressure > trim_percentage_usage_threshold) {
|
||||||
|
/* Normalize pressure for the range [0, threshold] */
|
||||||
|
pressure = (pressure - trim_percentage_usage_threshold) / (1.0 - trim_percentage_usage_threshold);
|
||||||
|
/* Linearly interpolate between min and max period based on pressure */
|
||||||
|
return (int64_t)((eviction_grace_period_microseconds_max - eviction_grace_period_microseconds_min) *
|
||||||
|
(1.0 - pressure)) + eviction_grace_period_microseconds_min;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Unlimited grace period, essentially don't trim at all */
|
||||||
|
return INT64_MAX;
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
d3d12_process_batch_residency(struct d3d12_screen *screen, struct d3d12_batch *batch)
|
||||||
|
{
|
||||||
|
d3d12_memory_info mem_info;
|
||||||
|
screen->get_memory_info(screen, &mem_info);
|
||||||
|
|
||||||
|
uint64_t completed_fence_value = screen->fence->GetCompletedValue();
|
||||||
|
uint64_t pending_fence_value = screen->fence_value + 1;
|
||||||
|
int64_t current_time = os_time_get();
|
||||||
|
int64_t grace_period = get_eviction_grace_period(&mem_info);
|
||||||
|
|
||||||
|
/* Gather base bos for the batch */
|
||||||
|
uint64_t size_to_make_resident = 0;
|
||||||
|
set *base_bo_set = _mesa_pointer_set_create(nullptr);
|
||||||
|
hash_table_foreach(batch->bos, entry) {
|
||||||
|
struct d3d12_bo *bo = (struct d3d12_bo *)entry->key;
|
||||||
|
uint64_t offset;
|
||||||
|
struct d3d12_bo *base_bo = d3d12_bo_get_base(bo, &offset);
|
||||||
|
|
||||||
|
if (base_bo->residency_status == d3d12_evicted) {
|
||||||
|
bool added = false;
|
||||||
|
_mesa_set_search_or_add(base_bo_set, base_bo, &added);
|
||||||
|
assert(!added);
|
||||||
|
|
||||||
|
base_bo->residency_status = d3d12_resident;
|
||||||
|
size_to_make_resident += base_bo->estimated_size;
|
||||||
|
list_addtail(&base_bo->residency_list_entry, &screen->residency_list);
|
||||||
|
} else if (base_bo->last_used_fence != pending_fence_value) {
|
||||||
|
/* First time seeing this already-resident base bo in this batch */
|
||||||
|
list_del(&base_bo->residency_list_entry);
|
||||||
|
list_addtail(&base_bo->residency_list_entry, &screen->residency_list);
|
||||||
|
}
|
||||||
|
|
||||||
|
base_bo->last_used_fence = pending_fence_value;
|
||||||
|
base_bo->last_used_timestamp = current_time;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Now that bos referenced by this batch are moved to the end of the LRU, trim it */
|
||||||
|
evict_aged_allocations(screen, completed_fence_value, current_time, grace_period);
|
||||||
|
|
||||||
|
/* If there's nothing needing to be made newly resident, we're done once we've trimmed */
|
||||||
|
if (base_bo_set->entries == 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
uint64_t residency_fence_value_snapshot = screen->residency_fence_value;
|
||||||
|
|
||||||
|
struct set_entry *entry = _mesa_set_next_entry(base_bo_set, nullptr);
|
||||||
|
uint64_t batch_memory_size = 0;
|
||||||
|
unsigned batch_count = 0;
|
||||||
|
ID3D12Pageable *to_make_resident[residency_batch_size];
|
||||||
|
while (true) {
|
||||||
|
/* Refresh memory stats */
|
||||||
|
screen->get_memory_info(screen, &mem_info);
|
||||||
|
|
||||||
|
int64_t available_memory = (int64_t)mem_info.budget - (int64_t)mem_info.usage;
|
||||||
|
|
||||||
|
assert(!list_is_empty(&screen->residency_list));
|
||||||
|
struct d3d12_bo *oldest_resident_bo =
|
||||||
|
list_first_entry(&screen->residency_list, struct d3d12_bo, residency_list_entry);
|
||||||
|
bool anything_to_wait_for = oldest_resident_bo->last_used_fence < pending_fence_value;
|
||||||
|
|
||||||
|
/* We've got some room, or we can't free up any more room, make some resources resident */
|
||||||
|
HRESULT hr = S_OK;
|
||||||
|
if ((available_memory || !anything_to_wait_for) && batch_count < residency_batch_size) {
|
||||||
|
for (; entry; entry = _mesa_set_next_entry(base_bo_set, entry)) {
|
||||||
|
struct d3d12_bo *bo = (struct d3d12_bo *)entry->key;
|
||||||
|
if (anything_to_wait_for &&
|
||||||
|
(int64_t)(batch_memory_size + bo->estimated_size) > available_memory)
|
||||||
|
break;
|
||||||
|
|
||||||
|
batch_memory_size += bo->estimated_size;
|
||||||
|
to_make_resident[batch_count++] = bo->res;
|
||||||
|
if (batch_count == residency_batch_size)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (batch_count) {
|
||||||
|
hr = screen->dev->EnqueueMakeResident(D3D12_RESIDENCY_FLAG_NONE, batch_count, to_make_resident,
|
||||||
|
screen->residency_fence, screen->residency_fence_value + 1);
|
||||||
|
if (SUCCEEDED(hr))
|
||||||
|
++screen->residency_fence_value;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (SUCCEEDED(hr) && batch_count == residency_batch_size) {
|
||||||
|
batch_count = 0;
|
||||||
|
size_to_make_resident -= batch_memory_size;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* We need to free up some space, either we broke early from the resource loop,
|
||||||
|
* or the MakeResident call itself failed.
|
||||||
|
*/
|
||||||
|
if (FAILED(hr) || entry) {
|
||||||
|
if (!anything_to_wait_for) {
|
||||||
|
assert(false);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
evict_to_fence_or_budget(screen, oldest_resident_bo->last_used_fence, mem_info.usage + size_to_make_resident, mem_info.budget);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Made it to the end without explicitly needing to loop, so we're done */
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_mesa_set_destroy(base_bo_set, nullptr);
|
||||||
|
|
||||||
|
/* The GPU needs to wait for these resources to be made resident */
|
||||||
|
if (residency_fence_value_snapshot != screen->residency_fence_value)
|
||||||
|
screen->cmdqueue->Wait(screen->residency_fence, screen->residency_fence_value);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool
|
||||||
|
d3d12_init_residency(struct d3d12_screen *screen)
|
||||||
|
{
|
||||||
|
list_inithead(&screen->residency_list);
|
||||||
|
if (FAILED(screen->dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&screen->residency_fence))))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
|
@ -0,0 +1,33 @@
|
||||||
|
/*
|
||||||
|
* Copyright © Microsoft Corporation
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||||
|
* copy of this software and associated documentation files (the "Software"),
|
||||||
|
* to deal in the Software without restriction, including without limitation
|
||||||
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||||
|
* and/or sell copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice (including the next
|
||||||
|
* paragraph) shall be included in all copies or substantial portions of the
|
||||||
|
* Software.
|
||||||
|
*
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||||
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||||
|
* IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
struct d3d12_screen;
|
||||||
|
struct d3d12_batch;
|
||||||
|
|
||||||
|
void
|
||||||
|
d3d12_process_batch_residency(struct d3d12_screen *screen, struct d3d12_batch *batch);
|
||||||
|
|
||||||
|
bool
|
||||||
|
d3d12_init_residency(struct d3d12_screen *screen);
|
|
@ -29,6 +29,7 @@
|
||||||
#include "d3d12_debug.h"
|
#include "d3d12_debug.h"
|
||||||
#include "d3d12_fence.h"
|
#include "d3d12_fence.h"
|
||||||
#include "d3d12_format.h"
|
#include "d3d12_format.h"
|
||||||
|
#include "d3d12_residency.h"
|
||||||
#include "d3d12_resource.h"
|
#include "d3d12_resource.h"
|
||||||
#include "d3d12_nir_passes.h"
|
#include "d3d12_nir_passes.h"
|
||||||
|
|
||||||
|
@ -797,7 +798,7 @@ enable_gpu_validation()
|
||||||
debug3->SetEnableGPUBasedValidation(true);
|
debug3->SetEnableGPUBasedValidation(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
static ID3D12Device *
|
static ID3D12Device3 *
|
||||||
create_device(IUnknown *adapter)
|
create_device(IUnknown *adapter)
|
||||||
{
|
{
|
||||||
typedef HRESULT(WINAPI *PFN_D3D12CREATEDEVICE)(IUnknown*, D3D_FEATURE_LEVEL, REFIID, void**);
|
typedef HRESULT(WINAPI *PFN_D3D12CREATEDEVICE)(IUnknown*, D3D_FEATURE_LEVEL, REFIID, void**);
|
||||||
|
@ -835,7 +836,7 @@ create_device(IUnknown *adapter)
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
ID3D12Device *dev;
|
ID3D12Device3 *dev;
|
||||||
if (SUCCEEDED(D3D12CreateDevice(adapter, D3D_FEATURE_LEVEL_11_0,
|
if (SUCCEEDED(D3D12CreateDevice(adapter, D3D_FEATURE_LEVEL_11_0,
|
||||||
IID_PPV_ARGS(&dev))))
|
IID_PPV_ARGS(&dev))))
|
||||||
return dev;
|
return dev;
|
||||||
|
@ -1188,7 +1189,8 @@ d3d12_init_screen(struct d3d12_screen *screen, struct sw_winsys *winsys, IUnknow
|
||||||
if (FAILED(screen->dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&screen->fence))))
|
if (FAILED(screen->dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(&screen->fence))))
|
||||||
goto failed;
|
goto failed;
|
||||||
|
|
||||||
list_inithead(&screen->residency_list);
|
if (!d3d12_init_residency(screen))
|
||||||
|
goto failed;
|
||||||
|
|
||||||
UINT64 timestamp_freq;
|
UINT64 timestamp_freq;
|
||||||
if (FAILED(screen->cmdqueue->GetTimestampFrequency(×tamp_freq)))
|
if (FAILED(screen->cmdqueue->GetTimestampFrequency(×tamp_freq)))
|
||||||
|
|
|
@ -65,14 +65,17 @@ struct d3d12_screen {
|
||||||
struct pipe_screen base;
|
struct pipe_screen base;
|
||||||
struct sw_winsys *winsys;
|
struct sw_winsys *winsys;
|
||||||
|
|
||||||
ID3D12Device *dev;
|
ID3D12Device3 *dev;
|
||||||
ID3D12CommandQueue *cmdqueue;
|
ID3D12CommandQueue *cmdqueue;
|
||||||
void (*get_memory_info)(struct d3d12_screen *screen, struct d3d12_memory_info *output);
|
void (*get_memory_info)(struct d3d12_screen *screen, struct d3d12_memory_info *output);
|
||||||
|
|
||||||
mtx_t submit_mutex;
|
mtx_t submit_mutex;
|
||||||
ID3D12Fence *fence;
|
ID3D12Fence *fence;
|
||||||
uint64_t fence_value;
|
uint64_t fence_value;
|
||||||
|
|
||||||
struct list_head residency_list;
|
struct list_head residency_list;
|
||||||
|
ID3D12Fence *residency_fence;
|
||||||
|
uint64_t residency_fence_value;
|
||||||
|
|
||||||
struct slab_parent_pool transfer_pool;
|
struct slab_parent_pool transfer_pool;
|
||||||
struct pb_manager *bufmgr;
|
struct pb_manager *bufmgr;
|
||||||
|
|
|
@ -40,6 +40,7 @@ files_libd3d12 = files(
|
||||||
'd3d12_nir_passes.c',
|
'd3d12_nir_passes.c',
|
||||||
'd3d12_pipeline_state.cpp',
|
'd3d12_pipeline_state.cpp',
|
||||||
'd3d12_query.cpp',
|
'd3d12_query.cpp',
|
||||||
|
'd3d12_residency.cpp',
|
||||||
'd3d12_resource.cpp',
|
'd3d12_resource.cpp',
|
||||||
'd3d12_root_signature.cpp',
|
'd3d12_root_signature.cpp',
|
||||||
'd3d12_screen.cpp',
|
'd3d12_screen.cpp',
|
||||||
|
|
Loading…
Reference in New Issue