2015-04-16 21:43:23 +01:00
|
|
|
/*
|
|
|
|
* Copyright © 2011 Marek Olšák <maraeo@gmail.com>
|
|
|
|
* Copyright © 2015 Advanced Micro Devices, Inc.
|
|
|
|
* All Rights Reserved.
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining
|
|
|
|
* a copy of this software and associated documentation files (the
|
|
|
|
* "Software"), to deal in the Software without restriction, including
|
|
|
|
* without limitation the rights to use, copy, modify, merge, publish,
|
|
|
|
* distribute, sub license, and/or sell copies of the Software, and to
|
|
|
|
* permit persons to whom the Software is furnished to do so, subject to
|
|
|
|
* the following conditions:
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
|
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
|
|
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
|
|
* NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
|
|
|
|
* AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
|
|
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
|
|
|
|
* USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the
|
|
|
|
* next paragraph) shall be included in all copies or substantial portions
|
|
|
|
* of the Software.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "amdgpu_cs.h"
|
|
|
|
|
2017-10-22 16:38:44 +01:00
|
|
|
#include "util/os_time.h"
|
2015-04-16 21:43:23 +01:00
|
|
|
#include "state_tracker/drm_driver.h"
|
|
|
|
#include <amdgpu_drm.h>
|
|
|
|
#include <xf86drm.h>
|
|
|
|
#include <stdio.h>
|
2016-04-10 15:48:55 +01:00
|
|
|
#include <inttypes.h>
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2017-08-29 15:45:46 +01:00
|
|
|
#ifndef AMDGPU_GEM_CREATE_VM_ALWAYS_VALID
|
|
|
|
#define AMDGPU_GEM_CREATE_VM_ALWAYS_VALID (1 << 6)
|
|
|
|
#endif
|
|
|
|
|
2017-02-13 11:51:12 +00:00
|
|
|
/* Set to 1 for verbose output showing committed sparse buffer ranges. */
|
|
|
|
#define DEBUG_SPARSE_COMMITS 0
|
2017-02-07 16:03:59 +00:00
|
|
|
|
|
|
|
struct amdgpu_sparse_backing_chunk {
|
|
|
|
uint32_t begin, end;
|
|
|
|
};
|
|
|
|
|
2016-09-07 09:50:59 +01:00
|
|
|
static struct pb_buffer *
|
|
|
|
amdgpu_bo_create(struct radeon_winsys *rws,
|
|
|
|
uint64_t size,
|
|
|
|
unsigned alignment,
|
|
|
|
enum radeon_bo_domain domain,
|
|
|
|
enum radeon_bo_flag flags);
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
static bool amdgpu_bo_wait(struct pb_buffer *_buf, uint64_t timeout,
|
|
|
|
enum radeon_bo_usage usage)
|
|
|
|
{
|
2015-12-06 19:57:05 +00:00
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
2015-12-06 21:19:38 +00:00
|
|
|
struct amdgpu_winsys *ws = bo->ws;
|
2016-03-08 00:19:31 +00:00
|
|
|
int64_t abs_timeout;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-03-08 00:19:31 +00:00
|
|
|
if (timeout == 0) {
|
|
|
|
if (p_atomic_read(&bo->num_active_ioctls))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
} else {
|
|
|
|
abs_timeout = os_time_get_absolute_timeout(timeout);
|
|
|
|
|
|
|
|
/* Wait if any ioctl is being submitted with this buffer. */
|
|
|
|
if (!os_wait_until_zero_abs_timeout(&bo->num_active_ioctls, abs_timeout))
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
if (bo->is_shared) {
|
|
|
|
/* We can't use user fences for shared buffers, because user fences
|
|
|
|
* are local to this process only. If we want to wait for all buffer
|
|
|
|
* uses in all processes, we have to use amdgpu_bo_wait_for_idle.
|
|
|
|
*/
|
|
|
|
bool buffer_busy = true;
|
|
|
|
int r;
|
|
|
|
|
|
|
|
r = amdgpu_bo_wait_for_idle(bo->bo, timeout, &buffer_busy);
|
|
|
|
if (r)
|
|
|
|
fprintf(stderr, "%s: amdgpu_bo_wait_for_idle failed %i\n", __func__,
|
|
|
|
r);
|
|
|
|
return !buffer_busy;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (timeout == 0) {
|
2016-09-07 10:01:17 +01:00
|
|
|
unsigned idle_fences;
|
|
|
|
bool buffer_idle;
|
|
|
|
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_lock(&ws->bo_fence_lock);
|
2016-09-07 10:01:17 +01:00
|
|
|
|
|
|
|
for (idle_fences = 0; idle_fences < bo->num_fences; ++idle_fences) {
|
|
|
|
if (!amdgpu_fence_wait(bo->fences[idle_fences], 0, false))
|
|
|
|
break;
|
2016-09-07 09:50:14 +01:00
|
|
|
}
|
2016-09-07 10:01:17 +01:00
|
|
|
|
|
|
|
/* Release the idle fences to avoid checking them again later. */
|
|
|
|
for (unsigned i = 0; i < idle_fences; ++i)
|
|
|
|
amdgpu_fence_reference(&bo->fences[i], NULL);
|
|
|
|
|
|
|
|
memmove(&bo->fences[0], &bo->fences[idle_fences],
|
|
|
|
(bo->num_fences - idle_fences) * sizeof(*bo->fences));
|
|
|
|
bo->num_fences -= idle_fences;
|
|
|
|
|
|
|
|
buffer_idle = !bo->num_fences;
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_unlock(&ws->bo_fence_lock);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-09-07 10:01:17 +01:00
|
|
|
return buffer_idle;
|
2015-04-16 21:43:23 +01:00
|
|
|
} else {
|
|
|
|
bool buffer_idle = true;
|
|
|
|
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_lock(&ws->bo_fence_lock);
|
2016-09-07 10:01:17 +01:00
|
|
|
while (bo->num_fences && buffer_idle) {
|
|
|
|
struct pipe_fence_handle *fence = NULL;
|
|
|
|
bool fence_idle = false;
|
|
|
|
|
|
|
|
amdgpu_fence_reference(&fence, bo->fences[0]);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-09-07 10:01:17 +01:00
|
|
|
/* Wait for the fence. */
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_unlock(&ws->bo_fence_lock);
|
2016-09-07 09:50:14 +01:00
|
|
|
if (amdgpu_fence_wait(fence, abs_timeout, true))
|
|
|
|
fence_idle = true;
|
|
|
|
else
|
|
|
|
buffer_idle = false;
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_lock(&ws->bo_fence_lock);
|
2016-09-07 10:01:17 +01:00
|
|
|
|
|
|
|
/* Release an idle fence to avoid checking it again later, keeping in
|
|
|
|
* mind that the fence array may have been modified by other threads.
|
|
|
|
*/
|
|
|
|
if (fence_idle && bo->num_fences && bo->fences[0] == fence) {
|
|
|
|
amdgpu_fence_reference(&bo->fences[0], NULL);
|
|
|
|
memmove(&bo->fences[0], &bo->fences[1],
|
|
|
|
(bo->num_fences - 1) * sizeof(*bo->fences));
|
|
|
|
bo->num_fences--;
|
|
|
|
}
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-09-07 10:01:17 +01:00
|
|
|
amdgpu_fence_reference(&fence, NULL);
|
|
|
|
}
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_unlock(&ws->bo_fence_lock);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
return buffer_idle;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static enum radeon_bo_domain amdgpu_bo_get_initial_domain(
|
2015-12-06 23:00:59 +00:00
|
|
|
struct pb_buffer *buf)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
return ((struct amdgpu_winsys_bo*)buf)->initial_domain;
|
|
|
|
}
|
|
|
|
|
2016-09-07 10:01:17 +01:00
|
|
|
static void amdgpu_bo_remove_fences(struct amdgpu_winsys_bo *bo)
|
|
|
|
{
|
|
|
|
for (unsigned i = 0; i < bo->num_fences; ++i)
|
|
|
|
amdgpu_fence_reference(&bo->fences[i], NULL);
|
|
|
|
|
|
|
|
FREE(bo->fences);
|
|
|
|
bo->num_fences = 0;
|
|
|
|
bo->max_fences = 0;
|
|
|
|
}
|
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
void amdgpu_bo_destroy(struct pb_buffer *_buf)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
assert(bo->bo && "must not be called for slab entries");
|
|
|
|
|
2017-08-29 15:24:45 +01:00
|
|
|
if (bo->ws->debug_all_bos) {
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_lock(&bo->ws->global_bo_list_lock);
|
2017-08-29 15:24:45 +01:00
|
|
|
LIST_DEL(&bo->u.real.global_list_item);
|
|
|
|
bo->ws->num_buffers--;
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_unlock(&bo->ws->global_bo_list_lock);
|
2017-08-29 15:24:45 +01:00
|
|
|
}
|
2016-01-14 18:31:18 +00:00
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
amdgpu_bo_va_op(bo->bo, 0, bo->base.size, bo->va, 0, AMDGPU_VA_OP_UNMAP);
|
2016-09-07 09:37:42 +01:00
|
|
|
amdgpu_va_range_free(bo->u.real.va_handle);
|
2015-04-16 21:43:23 +01:00
|
|
|
amdgpu_bo_free(bo->bo);
|
|
|
|
|
2016-09-07 10:01:17 +01:00
|
|
|
amdgpu_bo_remove_fences(bo);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
if (bo->initial_domain & RADEON_DOMAIN_VRAM)
|
2016-05-08 11:27:14 +01:00
|
|
|
bo->ws->allocated_vram -= align64(bo->base.size, bo->ws->info.gart_page_size);
|
2015-04-16 21:43:23 +01:00
|
|
|
else if (bo->initial_domain & RADEON_DOMAIN_GTT)
|
2016-05-08 11:27:14 +01:00
|
|
|
bo->ws->allocated_gtt -= align64(bo->base.size, bo->ws->info.gart_page_size);
|
2016-09-04 13:21:41 +01:00
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
if (bo->u.real.map_count >= 1) {
|
2016-09-04 13:21:41 +01:00
|
|
|
if (bo->initial_domain & RADEON_DOMAIN_VRAM)
|
|
|
|
bo->ws->mapped_vram -= bo->base.size;
|
2016-09-11 20:49:24 +01:00
|
|
|
else if (bo->initial_domain & RADEON_DOMAIN_GTT)
|
2016-09-04 13:21:41 +01:00
|
|
|
bo->ws->mapped_gtt -= bo->base.size;
|
2017-01-23 20:44:45 +00:00
|
|
|
bo->ws->num_mapped_buffers--;
|
2016-09-04 13:21:41 +01:00
|
|
|
}
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
FREE(bo);
|
|
|
|
}
|
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
static void amdgpu_bo_destroy_or_cache(struct pb_buffer *_buf)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
assert(bo->bo); /* slab buffers have a separate vtbl */
|
|
|
|
|
|
|
|
if (bo->u.real.use_reusable_pool)
|
|
|
|
pb_cache_add_buffer(&bo->u.real.cache_entry);
|
2015-12-06 19:57:05 +00:00
|
|
|
else
|
|
|
|
amdgpu_bo_destroy(_buf);
|
|
|
|
}
|
|
|
|
|
2015-12-06 23:00:59 +00:00
|
|
|
static void *amdgpu_bo_map(struct pb_buffer *buf,
|
2015-04-16 21:43:23 +01:00
|
|
|
struct radeon_winsys_cs *rcs,
|
|
|
|
enum pipe_transfer_usage usage)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
|
2016-09-07 09:37:42 +01:00
|
|
|
struct amdgpu_winsys_bo *real;
|
2015-04-16 21:43:23 +01:00
|
|
|
struct amdgpu_cs *cs = (struct amdgpu_cs*)rcs;
|
|
|
|
int r;
|
|
|
|
void *cpu = NULL;
|
2016-09-07 09:37:42 +01:00
|
|
|
uint64_t offset = 0;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2017-02-07 16:03:59 +00:00
|
|
|
assert(!bo->sparse);
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
/* If it's not unsynchronized bo_map, flush CS if needed and then wait. */
|
|
|
|
if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
|
|
|
|
/* DONTBLOCK doesn't make sense with UNSYNCHRONIZED. */
|
|
|
|
if (usage & PIPE_TRANSFER_DONTBLOCK) {
|
|
|
|
if (!(usage & PIPE_TRANSFER_WRITE)) {
|
|
|
|
/* Mapping for read.
|
|
|
|
*
|
|
|
|
* Since we are mapping for read, we don't need to wait
|
|
|
|
* if the GPU is using the buffer for read too
|
|
|
|
* (neither one is changing it).
|
|
|
|
*
|
|
|
|
* Only check whether the buffer is being used for write. */
|
|
|
|
if (cs && amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
|
|
|
|
RADEON_USAGE_WRITE)) {
|
2017-11-28 16:54:55 +00:00
|
|
|
cs->flush_cs(cs->flush_data, PIPE_FLUSH_ASYNC, NULL);
|
2015-04-16 21:43:23 +01:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
|
|
|
|
RADEON_USAGE_WRITE)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (cs && amdgpu_bo_is_referenced_by_cs(cs, bo)) {
|
2017-11-28 16:54:55 +00:00
|
|
|
cs->flush_cs(cs->flush_data, PIPE_FLUSH_ASYNC, NULL);
|
2015-04-16 21:43:23 +01:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!amdgpu_bo_wait((struct pb_buffer*)bo, 0,
|
|
|
|
RADEON_USAGE_READWRITE)) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
uint64_t time = os_time_get_nano();
|
|
|
|
|
|
|
|
if (!(usage & PIPE_TRANSFER_WRITE)) {
|
|
|
|
/* Mapping for read.
|
|
|
|
*
|
|
|
|
* Since we are mapping for read, we don't need to wait
|
|
|
|
* if the GPU is using the buffer for read too
|
|
|
|
* (neither one is changing it).
|
|
|
|
*
|
|
|
|
* Only check whether the buffer is being used for write. */
|
2017-02-02 17:40:18 +00:00
|
|
|
if (cs) {
|
|
|
|
if (amdgpu_bo_is_referenced_by_cs_with_usage(cs, bo,
|
|
|
|
RADEON_USAGE_WRITE)) {
|
|
|
|
cs->flush_cs(cs->flush_data, 0, NULL);
|
|
|
|
} else {
|
|
|
|
/* Try to avoid busy-waiting in amdgpu_bo_wait. */
|
|
|
|
if (p_atomic_read(&bo->num_active_ioctls))
|
|
|
|
amdgpu_cs_sync_flush(rcs);
|
|
|
|
}
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
2017-02-02 17:40:18 +00:00
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
|
|
|
|
RADEON_USAGE_WRITE);
|
|
|
|
} else {
|
|
|
|
/* Mapping for write. */
|
2016-03-08 00:19:31 +00:00
|
|
|
if (cs) {
|
|
|
|
if (amdgpu_bo_is_referenced_by_cs(cs, bo)) {
|
|
|
|
cs->flush_cs(cs->flush_data, 0, NULL);
|
|
|
|
} else {
|
|
|
|
/* Try to avoid busy-waiting in amdgpu_bo_wait. */
|
|
|
|
if (p_atomic_read(&bo->num_active_ioctls))
|
|
|
|
amdgpu_cs_sync_flush(rcs);
|
|
|
|
}
|
|
|
|
}
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
amdgpu_bo_wait((struct pb_buffer*)bo, PIPE_TIMEOUT_INFINITE,
|
|
|
|
RADEON_USAGE_READWRITE);
|
|
|
|
}
|
|
|
|
|
2015-12-06 21:19:38 +00:00
|
|
|
bo->ws->buffer_wait_time += os_time_get_nano() - time;
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If the buffer is created from user memory, return the user pointer. */
|
|
|
|
if (bo->user_ptr)
|
2016-09-07 09:37:42 +01:00
|
|
|
return bo->user_ptr;
|
|
|
|
|
|
|
|
if (bo->bo) {
|
|
|
|
real = bo;
|
|
|
|
} else {
|
|
|
|
real = bo->u.slab.real;
|
|
|
|
offset = bo->va - real->va;
|
|
|
|
}
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
r = amdgpu_bo_cpu_map(real->bo, &cpu);
|
2015-12-09 21:45:56 +00:00
|
|
|
if (r) {
|
|
|
|
/* Clear the cache and try again. */
|
2016-09-07 09:37:42 +01:00
|
|
|
pb_cache_release_all_buffers(&real->ws->bo_cache);
|
|
|
|
r = amdgpu_bo_cpu_map(real->bo, &cpu);
|
2016-08-05 00:28:17 +01:00
|
|
|
if (r)
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
if (p_atomic_inc_return(&real->u.real.map_count) == 1) {
|
|
|
|
if (real->initial_domain & RADEON_DOMAIN_VRAM)
|
|
|
|
real->ws->mapped_vram += real->base.size;
|
|
|
|
else if (real->initial_domain & RADEON_DOMAIN_GTT)
|
|
|
|
real->ws->mapped_gtt += real->base.size;
|
2017-01-23 20:44:45 +00:00
|
|
|
real->ws->num_mapped_buffers++;
|
2015-12-09 21:45:56 +00:00
|
|
|
}
|
2016-09-07 09:37:42 +01:00
|
|
|
return (uint8_t*)cpu + offset;
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
2015-12-06 23:00:59 +00:00
|
|
|
static void amdgpu_bo_unmap(struct pb_buffer *buf)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
|
2016-09-07 09:37:42 +01:00
|
|
|
struct amdgpu_winsys_bo *real;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2017-02-07 16:03:59 +00:00
|
|
|
assert(!bo->sparse);
|
|
|
|
|
2016-08-05 00:18:08 +01:00
|
|
|
if (bo->user_ptr)
|
|
|
|
return;
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
real = bo->bo ? bo : bo->u.slab.real;
|
|
|
|
|
|
|
|
if (p_atomic_dec_zero(&real->u.real.map_count)) {
|
|
|
|
if (real->initial_domain & RADEON_DOMAIN_VRAM)
|
|
|
|
real->ws->mapped_vram -= real->base.size;
|
|
|
|
else if (real->initial_domain & RADEON_DOMAIN_GTT)
|
|
|
|
real->ws->mapped_gtt -= real->base.size;
|
2017-01-23 20:44:45 +00:00
|
|
|
real->ws->num_mapped_buffers--;
|
2016-08-05 00:28:17 +01:00
|
|
|
}
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
amdgpu_bo_cpu_unmap(real->bo);
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct pb_vtbl amdgpu_winsys_bo_vtbl = {
|
2015-12-06 19:57:05 +00:00
|
|
|
amdgpu_bo_destroy_or_cache
|
|
|
|
/* other functions are never called */
|
2015-04-16 21:43:23 +01:00
|
|
|
};
|
|
|
|
|
2016-01-14 18:31:18 +00:00
|
|
|
static void amdgpu_add_buffer_to_global_list(struct amdgpu_winsys_bo *bo)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys *ws = bo->ws;
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
assert(bo->bo);
|
|
|
|
|
2017-08-29 15:24:45 +01:00
|
|
|
if (ws->debug_all_bos) {
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_lock(&ws->global_bo_list_lock);
|
2017-08-29 15:24:45 +01:00
|
|
|
LIST_ADDTAIL(&bo->u.real.global_list_item, &ws->global_bo_list);
|
|
|
|
ws->num_buffers++;
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_unlock(&ws->global_bo_list_lock);
|
2017-08-29 15:24:45 +01:00
|
|
|
}
|
2016-01-14 18:31:18 +00:00
|
|
|
}
|
|
|
|
|
2015-12-06 21:19:38 +00:00
|
|
|
static struct amdgpu_winsys_bo *amdgpu_create_bo(struct amdgpu_winsys *ws,
|
2016-04-10 15:48:55 +01:00
|
|
|
uint64_t size,
|
2015-12-06 19:57:05 +00:00
|
|
|
unsigned alignment,
|
|
|
|
unsigned usage,
|
|
|
|
enum radeon_bo_domain initial_domain,
|
2016-07-17 12:23:53 +01:00
|
|
|
unsigned flags,
|
|
|
|
unsigned pb_cache_bucket)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
struct amdgpu_bo_alloc_request request = {0};
|
|
|
|
amdgpu_bo_handle buf_handle;
|
|
|
|
uint64_t va = 0;
|
|
|
|
struct amdgpu_winsys_bo *bo;
|
|
|
|
amdgpu_va_handle va_handle;
|
2016-06-22 10:22:33 +01:00
|
|
|
unsigned va_gap_size;
|
2015-04-16 21:43:23 +01:00
|
|
|
int r;
|
|
|
|
|
2017-11-30 23:06:34 +00:00
|
|
|
/* VRAM or GTT must be specified, but not both at the same time. */
|
|
|
|
assert(util_bitcount(initial_domain & RADEON_DOMAIN_VRAM_GTT) == 1);
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
bo = CALLOC_STRUCT(amdgpu_winsys_bo);
|
|
|
|
if (!bo) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
pb_cache_init_entry(&ws->bo_cache, &bo->u.real.cache_entry, &bo->base,
|
2016-07-17 12:23:53 +01:00
|
|
|
pb_cache_bucket);
|
2015-04-16 21:43:23 +01:00
|
|
|
request.alloc_size = size;
|
2015-12-06 19:57:05 +00:00
|
|
|
request.phys_alignment = alignment;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-01-26 07:38:55 +00:00
|
|
|
if (initial_domain & RADEON_DOMAIN_VRAM)
|
2015-04-16 21:43:23 +01:00
|
|
|
request.preferred_heap |= AMDGPU_GEM_DOMAIN_VRAM;
|
2016-01-26 07:38:55 +00:00
|
|
|
if (initial_domain & RADEON_DOMAIN_GTT)
|
2015-04-16 21:43:23 +01:00
|
|
|
request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
|
2016-01-26 07:38:55 +00:00
|
|
|
|
2017-11-30 23:06:34 +00:00
|
|
|
/* If VRAM is just stolen system memory, allow both VRAM and
|
|
|
|
* GTT, whichever has free space. If a buffer is evicted from
|
|
|
|
* VRAM to GTT, it will stay there.
|
|
|
|
*
|
|
|
|
* DRM 3.6.0 has good BO move throttling, so we can allow VRAM-only
|
|
|
|
* placements even with a low amount of stolen VRAM.
|
|
|
|
*/
|
|
|
|
if (!ws->info.has_dedicated_vram && ws->info.drm_minor < 6)
|
|
|
|
request.preferred_heap |= AMDGPU_GEM_DOMAIN_GTT;
|
|
|
|
|
2016-01-26 07:38:55 +00:00
|
|
|
if (flags & RADEON_FLAG_NO_CPU_ACCESS)
|
|
|
|
request.flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
|
|
|
|
if (flags & RADEON_FLAG_GTT_WC)
|
|
|
|
request.flags |= AMDGPU_GEM_CREATE_CPU_GTT_USWC;
|
2017-12-11 15:29:40 +00:00
|
|
|
/* TODO: Enable this once the kernel handles it efficiently. */
|
|
|
|
/*if (flags & RADEON_FLAG_NO_INTERPROCESS_SHARING &&
|
2017-09-08 16:34:59 +01:00
|
|
|
ws->info.drm_minor >= 20)
|
2017-12-11 15:29:40 +00:00
|
|
|
request.flags |= AMDGPU_GEM_CREATE_VM_ALWAYS_VALID;*/
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2015-12-06 21:19:38 +00:00
|
|
|
r = amdgpu_bo_alloc(ws->dev, &request, &buf_handle);
|
2015-04-16 21:43:23 +01:00
|
|
|
if (r) {
|
|
|
|
fprintf(stderr, "amdgpu: Failed to allocate a buffer:\n");
|
2016-04-10 15:48:55 +01:00
|
|
|
fprintf(stderr, "amdgpu: size : %"PRIu64" bytes\n", size);
|
|
|
|
fprintf(stderr, "amdgpu: alignment : %u bytes\n", alignment);
|
|
|
|
fprintf(stderr, "amdgpu: domains : %u\n", initial_domain);
|
2015-04-16 21:43:23 +01:00
|
|
|
goto error_bo_alloc;
|
|
|
|
}
|
|
|
|
|
2016-06-22 10:22:33 +01:00
|
|
|
va_gap_size = ws->check_vm ? MAX2(4 * alignment, 64 * 1024) : 0;
|
2017-05-23 17:40:18 +01:00
|
|
|
if (size > ws->info.pte_fragment_size)
|
|
|
|
alignment = MAX2(alignment, ws->info.pte_fragment_size);
|
2015-12-06 21:19:38 +00:00
|
|
|
r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
|
2016-06-22 10:22:33 +01:00
|
|
|
size + va_gap_size, alignment, 0, &va, &va_handle, 0);
|
2015-04-16 21:43:23 +01:00
|
|
|
if (r)
|
|
|
|
goto error_va_alloc;
|
|
|
|
|
2017-12-01 01:07:05 +00:00
|
|
|
unsigned vm_flags = AMDGPU_VM_PAGE_READABLE |
|
|
|
|
AMDGPU_VM_PAGE_EXECUTABLE;
|
|
|
|
|
|
|
|
if (!(flags & RADEON_FLAG_READ_ONLY))
|
|
|
|
vm_flags |= AMDGPU_VM_PAGE_WRITEABLE;
|
|
|
|
|
|
|
|
r = amdgpu_bo_va_op_raw(ws->dev, buf_handle, 0, size, va, vm_flags,
|
|
|
|
AMDGPU_VA_OP_MAP);
|
2015-04-16 21:43:23 +01:00
|
|
|
if (r)
|
|
|
|
goto error_va_map;
|
|
|
|
|
|
|
|
pipe_reference_init(&bo->base.reference, 1);
|
2015-12-06 19:57:05 +00:00
|
|
|
bo->base.alignment = alignment;
|
|
|
|
bo->base.usage = usage;
|
2015-04-16 21:43:23 +01:00
|
|
|
bo->base.size = size;
|
|
|
|
bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
|
2015-12-06 21:19:38 +00:00
|
|
|
bo->ws = ws;
|
2015-04-16 21:43:23 +01:00
|
|
|
bo->bo = buf_handle;
|
|
|
|
bo->va = va;
|
2016-09-07 09:37:42 +01:00
|
|
|
bo->u.real.va_handle = va_handle;
|
2015-12-06 19:57:05 +00:00
|
|
|
bo->initial_domain = initial_domain;
|
2015-12-06 21:19:38 +00:00
|
|
|
bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
|
2017-08-29 15:45:46 +01:00
|
|
|
bo->is_local = !!(request.flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
if (initial_domain & RADEON_DOMAIN_VRAM)
|
2016-05-08 11:27:14 +01:00
|
|
|
ws->allocated_vram += align64(size, ws->info.gart_page_size);
|
2015-12-06 19:57:05 +00:00
|
|
|
else if (initial_domain & RADEON_DOMAIN_GTT)
|
2016-05-08 11:27:14 +01:00
|
|
|
ws->allocated_gtt += align64(size, ws->info.gart_page_size);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-01-14 18:31:18 +00:00
|
|
|
amdgpu_add_buffer_to_global_list(bo);
|
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
return bo;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
error_va_map:
|
|
|
|
amdgpu_va_range_free(va_handle);
|
|
|
|
|
|
|
|
error_va_alloc:
|
|
|
|
amdgpu_bo_free(buf_handle);
|
|
|
|
|
|
|
|
error_bo_alloc:
|
|
|
|
FREE(bo);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
bool amdgpu_bo_can_reclaim(struct pb_buffer *_buf)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
|
|
|
|
|
|
|
if (amdgpu_bo_is_referenced_by_any_cs(bo)) {
|
2015-12-06 19:57:05 +00:00
|
|
|
return false;
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
return amdgpu_bo_wait(_buf, 0, RADEON_USAGE_READWRITE);
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
2016-09-07 09:50:59 +01:00
|
|
|
bool amdgpu_bo_can_reclaim_slab(void *priv, struct pb_slab_entry *entry)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = NULL; /* fix container_of */
|
|
|
|
bo = container_of(entry, bo, u.slab.entry);
|
|
|
|
|
|
|
|
return amdgpu_bo_can_reclaim(&bo->base);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void amdgpu_bo_slab_destroy(struct pb_buffer *_buf)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
|
|
|
|
|
|
|
assert(!bo->bo);
|
|
|
|
|
|
|
|
pb_slab_free(&bo->ws->bo_slabs, &bo->u.slab.entry);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct pb_vtbl amdgpu_winsys_bo_slab_vtbl = {
|
|
|
|
amdgpu_bo_slab_destroy
|
|
|
|
/* other functions are never called */
|
|
|
|
};
|
|
|
|
|
|
|
|
struct pb_slab *amdgpu_bo_slab_alloc(void *priv, unsigned heap,
|
|
|
|
unsigned entry_size,
|
|
|
|
unsigned group_index)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys *ws = priv;
|
|
|
|
struct amdgpu_slab *slab = CALLOC_STRUCT(amdgpu_slab);
|
2017-06-29 17:01:16 +01:00
|
|
|
enum radeon_bo_domain domains = radeon_domain_from_heap(heap);
|
|
|
|
enum radeon_bo_flag flags = radeon_flags_from_heap(heap);
|
2016-09-07 09:50:59 +01:00
|
|
|
uint32_t base_id;
|
|
|
|
|
|
|
|
if (!slab)
|
|
|
|
return NULL;
|
|
|
|
|
2017-06-27 20:54:20 +01:00
|
|
|
unsigned slab_size = 1 << AMDGPU_SLAB_BO_SIZE_LOG2;
|
2016-09-07 09:50:59 +01:00
|
|
|
slab->buffer = amdgpu_winsys_bo(amdgpu_bo_create(&ws->base,
|
2017-06-27 20:54:20 +01:00
|
|
|
slab_size, slab_size,
|
2016-09-07 09:50:59 +01:00
|
|
|
domains, flags));
|
|
|
|
if (!slab->buffer)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
assert(slab->buffer->bo);
|
|
|
|
|
|
|
|
slab->base.num_entries = slab->buffer->base.size / entry_size;
|
|
|
|
slab->base.num_free = slab->base.num_entries;
|
|
|
|
slab->entries = CALLOC(slab->base.num_entries, sizeof(*slab->entries));
|
|
|
|
if (!slab->entries)
|
|
|
|
goto fail_buffer;
|
|
|
|
|
|
|
|
LIST_INITHEAD(&slab->base.free);
|
|
|
|
|
|
|
|
base_id = __sync_fetch_and_add(&ws->next_bo_unique_id, slab->base.num_entries);
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < slab->base.num_entries; ++i) {
|
|
|
|
struct amdgpu_winsys_bo *bo = &slab->entries[i];
|
|
|
|
|
|
|
|
bo->base.alignment = entry_size;
|
|
|
|
bo->base.usage = slab->buffer->base.usage;
|
|
|
|
bo->base.size = entry_size;
|
|
|
|
bo->base.vtbl = &amdgpu_winsys_bo_slab_vtbl;
|
|
|
|
bo->ws = ws;
|
|
|
|
bo->va = slab->buffer->va + i * entry_size;
|
|
|
|
bo->initial_domain = domains;
|
|
|
|
bo->unique_id = base_id + i;
|
|
|
|
bo->u.slab.entry.slab = &slab->base;
|
|
|
|
bo->u.slab.entry.group_index = group_index;
|
|
|
|
bo->u.slab.real = slab->buffer;
|
|
|
|
|
|
|
|
LIST_ADDTAIL(&bo->u.slab.entry.head, &slab->base.free);
|
|
|
|
}
|
|
|
|
|
|
|
|
return &slab->base;
|
|
|
|
|
|
|
|
fail_buffer:
|
|
|
|
amdgpu_winsys_bo_reference(&slab->buffer, NULL);
|
|
|
|
fail:
|
|
|
|
FREE(slab);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
void amdgpu_bo_slab_free(void *priv, struct pb_slab *pslab)
|
|
|
|
{
|
|
|
|
struct amdgpu_slab *slab = amdgpu_slab(pslab);
|
|
|
|
|
|
|
|
for (unsigned i = 0; i < slab->base.num_entries; ++i)
|
|
|
|
amdgpu_bo_remove_fences(&slab->entries[i]);
|
|
|
|
|
|
|
|
FREE(slab->entries);
|
|
|
|
amdgpu_winsys_bo_reference(&slab->buffer, NULL);
|
|
|
|
FREE(slab);
|
|
|
|
}
|
|
|
|
|
2017-02-13 11:51:12 +00:00
|
|
|
#if DEBUG_SPARSE_COMMITS
|
|
|
|
static void
|
|
|
|
sparse_dump(struct amdgpu_winsys_bo *bo, const char *func)
|
|
|
|
{
|
|
|
|
fprintf(stderr, "%s: %p (size=%"PRIu64", num_va_pages=%u) @ %s\n"
|
|
|
|
"Commitments:\n",
|
|
|
|
__func__, bo, bo->base.size, bo->u.sparse.num_va_pages, func);
|
|
|
|
|
|
|
|
struct amdgpu_sparse_backing *span_backing = NULL;
|
|
|
|
uint32_t span_first_backing_page = 0;
|
|
|
|
uint32_t span_first_va_page = 0;
|
|
|
|
uint32_t va_page = 0;
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
struct amdgpu_sparse_backing *backing = 0;
|
|
|
|
uint32_t backing_page = 0;
|
|
|
|
|
|
|
|
if (va_page < bo->u.sparse.num_va_pages) {
|
|
|
|
backing = bo->u.sparse.commitments[va_page].backing;
|
|
|
|
backing_page = bo->u.sparse.commitments[va_page].page;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (span_backing &&
|
|
|
|
(backing != span_backing ||
|
|
|
|
backing_page != span_first_backing_page + (va_page - span_first_va_page))) {
|
|
|
|
fprintf(stderr, " %u..%u: backing=%p:%u..%u\n",
|
|
|
|
span_first_va_page, va_page - 1, span_backing,
|
|
|
|
span_first_backing_page,
|
|
|
|
span_first_backing_page + (va_page - span_first_va_page) - 1);
|
|
|
|
|
|
|
|
span_backing = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (va_page >= bo->u.sparse.num_va_pages)
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (backing && !span_backing) {
|
|
|
|
span_backing = backing;
|
|
|
|
span_first_backing_page = backing_page;
|
|
|
|
span_first_va_page = va_page;
|
|
|
|
}
|
|
|
|
|
|
|
|
va_page++;
|
|
|
|
}
|
|
|
|
|
|
|
|
fprintf(stderr, "Backing:\n");
|
|
|
|
|
|
|
|
list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
|
|
|
|
fprintf(stderr, " %p (size=%"PRIu64")\n", backing, backing->bo->base.size);
|
|
|
|
for (unsigned i = 0; i < backing->num_chunks; ++i)
|
|
|
|
fprintf(stderr, " %u..%u\n", backing->chunks[i].begin, backing->chunks[i].end);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2017-02-07 16:04:49 +00:00
|
|
|
/*
|
|
|
|
* Attempt to allocate the given number of backing pages. Fewer pages may be
|
|
|
|
* allocated (depending on the fragmentation of existing backing buffers),
|
|
|
|
* which will be reflected by a change to *pnum_pages.
|
|
|
|
*/
|
|
|
|
static struct amdgpu_sparse_backing *
|
|
|
|
sparse_backing_alloc(struct amdgpu_winsys_bo *bo, uint32_t *pstart_page, uint32_t *pnum_pages)
|
|
|
|
{
|
|
|
|
struct amdgpu_sparse_backing *best_backing;
|
|
|
|
unsigned best_idx;
|
|
|
|
uint32_t best_num_pages;
|
|
|
|
|
|
|
|
best_backing = NULL;
|
|
|
|
best_idx = 0;
|
|
|
|
best_num_pages = 0;
|
|
|
|
|
|
|
|
/* This is a very simple and inefficient best-fit algorithm. */
|
|
|
|
list_for_each_entry(struct amdgpu_sparse_backing, backing, &bo->u.sparse.backing, list) {
|
|
|
|
for (unsigned idx = 0; idx < backing->num_chunks; ++idx) {
|
|
|
|
uint32_t cur_num_pages = backing->chunks[idx].end - backing->chunks[idx].begin;
|
|
|
|
if ((best_num_pages < *pnum_pages && cur_num_pages > best_num_pages) ||
|
|
|
|
(best_num_pages > *pnum_pages && cur_num_pages < best_num_pages)) {
|
|
|
|
best_backing = backing;
|
|
|
|
best_idx = idx;
|
|
|
|
best_num_pages = cur_num_pages;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Allocate a new backing buffer if necessary. */
|
|
|
|
if (!best_backing) {
|
|
|
|
struct pb_buffer *buf;
|
|
|
|
uint64_t size;
|
|
|
|
uint32_t pages;
|
|
|
|
|
|
|
|
best_backing = CALLOC_STRUCT(amdgpu_sparse_backing);
|
|
|
|
if (!best_backing)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
best_backing->max_chunks = 4;
|
|
|
|
best_backing->chunks = CALLOC(best_backing->max_chunks,
|
|
|
|
sizeof(*best_backing->chunks));
|
|
|
|
if (!best_backing->chunks) {
|
|
|
|
FREE(best_backing);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
assert(bo->u.sparse.num_backing_pages < DIV_ROUND_UP(bo->base.size, RADEON_SPARSE_PAGE_SIZE));
|
|
|
|
|
|
|
|
size = MIN3(bo->base.size / 16,
|
|
|
|
8 * 1024 * 1024,
|
|
|
|
bo->base.size - (uint64_t)bo->u.sparse.num_backing_pages * RADEON_SPARSE_PAGE_SIZE);
|
|
|
|
size = MAX2(size, RADEON_SPARSE_PAGE_SIZE);
|
|
|
|
|
|
|
|
buf = amdgpu_bo_create(&bo->ws->base, size, RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
bo->initial_domain,
|
2017-06-27 18:37:43 +01:00
|
|
|
bo->u.sparse.flags | RADEON_FLAG_NO_SUBALLOC);
|
2017-02-07 16:04:49 +00:00
|
|
|
if (!buf) {
|
|
|
|
FREE(best_backing->chunks);
|
|
|
|
FREE(best_backing);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* We might have gotten a bigger buffer than requested via caching. */
|
|
|
|
pages = buf->size / RADEON_SPARSE_PAGE_SIZE;
|
|
|
|
|
|
|
|
best_backing->bo = amdgpu_winsys_bo(buf);
|
|
|
|
best_backing->num_chunks = 1;
|
|
|
|
best_backing->chunks[0].begin = 0;
|
|
|
|
best_backing->chunks[0].end = pages;
|
|
|
|
|
|
|
|
list_add(&best_backing->list, &bo->u.sparse.backing);
|
|
|
|
bo->u.sparse.num_backing_pages += pages;
|
|
|
|
|
|
|
|
best_idx = 0;
|
|
|
|
best_num_pages = pages;
|
|
|
|
}
|
|
|
|
|
|
|
|
*pnum_pages = MIN2(*pnum_pages, best_num_pages);
|
|
|
|
*pstart_page = best_backing->chunks[best_idx].begin;
|
|
|
|
best_backing->chunks[best_idx].begin += *pnum_pages;
|
|
|
|
|
|
|
|
if (best_backing->chunks[best_idx].begin >= best_backing->chunks[best_idx].end) {
|
|
|
|
memmove(&best_backing->chunks[best_idx], &best_backing->chunks[best_idx + 1],
|
|
|
|
sizeof(*best_backing->chunks) * (best_backing->num_chunks - best_idx - 1));
|
|
|
|
best_backing->num_chunks--;
|
|
|
|
}
|
|
|
|
|
|
|
|
return best_backing;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2017-02-07 16:58:39 +00:00
|
|
|
sparse_free_backing_buffer(struct amdgpu_winsys_bo *bo,
|
|
|
|
struct amdgpu_sparse_backing *backing)
|
2017-02-07 16:04:49 +00:00
|
|
|
{
|
2017-02-07 16:58:39 +00:00
|
|
|
struct amdgpu_winsys *ws = backing->bo->ws;
|
|
|
|
|
2017-02-07 16:04:49 +00:00
|
|
|
bo->u.sparse.num_backing_pages -= backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE;
|
|
|
|
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_lock(&ws->bo_fence_lock);
|
2017-02-07 16:58:39 +00:00
|
|
|
amdgpu_add_fences(backing->bo, bo->num_fences, bo->fences);
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_unlock(&ws->bo_fence_lock);
|
2017-02-07 16:58:39 +00:00
|
|
|
|
2017-02-07 16:04:49 +00:00
|
|
|
list_del(&backing->list);
|
|
|
|
amdgpu_winsys_bo_reference(&backing->bo, NULL);
|
|
|
|
FREE(backing->chunks);
|
|
|
|
FREE(backing);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Return a range of pages from the given backing buffer back into the
|
|
|
|
* free structure.
|
|
|
|
*/
|
|
|
|
static bool
|
|
|
|
sparse_backing_free(struct amdgpu_winsys_bo *bo,
|
|
|
|
struct amdgpu_sparse_backing *backing,
|
|
|
|
uint32_t start_page, uint32_t num_pages)
|
|
|
|
{
|
|
|
|
uint32_t end_page = start_page + num_pages;
|
|
|
|
unsigned low = 0;
|
|
|
|
unsigned high = backing->num_chunks;
|
|
|
|
|
|
|
|
/* Find the first chunk with begin >= start_page. */
|
|
|
|
while (low < high) {
|
|
|
|
unsigned mid = low + (high - low) / 2;
|
|
|
|
|
|
|
|
if (backing->chunks[mid].begin >= start_page)
|
|
|
|
high = mid;
|
|
|
|
else
|
|
|
|
low = mid + 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
assert(low >= backing->num_chunks || end_page <= backing->chunks[low].begin);
|
|
|
|
assert(low == 0 || backing->chunks[low - 1].end <= start_page);
|
|
|
|
|
|
|
|
if (low > 0 && backing->chunks[low - 1].end == start_page) {
|
|
|
|
backing->chunks[low - 1].end = end_page;
|
|
|
|
|
|
|
|
if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
|
|
|
|
backing->chunks[low - 1].end = backing->chunks[low].end;
|
|
|
|
memmove(&backing->chunks[low], &backing->chunks[low + 1],
|
|
|
|
sizeof(*backing->chunks) * (backing->num_chunks - low - 1));
|
|
|
|
backing->num_chunks--;
|
|
|
|
}
|
|
|
|
} else if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
|
|
|
|
backing->chunks[low].begin = start_page;
|
|
|
|
} else {
|
|
|
|
if (backing->num_chunks >= backing->max_chunks) {
|
|
|
|
unsigned new_max_chunks = 2 * backing->max_chunks;
|
|
|
|
struct amdgpu_sparse_backing_chunk *new_chunks =
|
|
|
|
REALLOC(backing->chunks,
|
|
|
|
sizeof(*backing->chunks) * backing->max_chunks,
|
|
|
|
sizeof(*backing->chunks) * new_max_chunks);
|
|
|
|
if (!new_chunks)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
backing->max_chunks = new_max_chunks;
|
|
|
|
backing->chunks = new_chunks;
|
|
|
|
}
|
|
|
|
|
|
|
|
memmove(&backing->chunks[low + 1], &backing->chunks[low],
|
|
|
|
sizeof(*backing->chunks) * (backing->num_chunks - low));
|
|
|
|
backing->chunks[low].begin = start_page;
|
|
|
|
backing->chunks[low].end = end_page;
|
|
|
|
backing->num_chunks++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (backing->num_chunks == 1 && backing->chunks[0].begin == 0 &&
|
|
|
|
backing->chunks[0].end == backing->bo->base.size / RADEON_SPARSE_PAGE_SIZE)
|
2017-02-07 16:58:39 +00:00
|
|
|
sparse_free_backing_buffer(bo, backing);
|
2017-02-07 16:04:49 +00:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void amdgpu_bo_sparse_destroy(struct pb_buffer *_buf)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
|
|
|
int r;
|
|
|
|
|
|
|
|
assert(!bo->bo && bo->sparse);
|
|
|
|
|
|
|
|
r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
|
|
|
|
(uint64_t)bo->u.sparse.num_va_pages * RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
bo->va, 0, AMDGPU_VA_OP_CLEAR);
|
|
|
|
if (r) {
|
|
|
|
fprintf(stderr, "amdgpu: clearing PRT VA region on destroy failed (%d)\n", r);
|
|
|
|
}
|
|
|
|
|
|
|
|
while (!list_empty(&bo->u.sparse.backing)) {
|
|
|
|
struct amdgpu_sparse_backing *dummy = NULL;
|
2017-02-07 16:58:39 +00:00
|
|
|
sparse_free_backing_buffer(bo,
|
|
|
|
container_of(bo->u.sparse.backing.next,
|
2017-02-07 16:04:49 +00:00
|
|
|
dummy, list));
|
|
|
|
}
|
|
|
|
|
|
|
|
amdgpu_va_range_free(bo->u.sparse.va_handle);
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_destroy(&bo->u.sparse.commit_lock);
|
2017-02-07 16:04:49 +00:00
|
|
|
FREE(bo->u.sparse.commitments);
|
|
|
|
FREE(bo);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct pb_vtbl amdgpu_winsys_bo_sparse_vtbl = {
|
|
|
|
amdgpu_bo_sparse_destroy
|
|
|
|
/* other functions are never called */
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct pb_buffer *
|
|
|
|
amdgpu_bo_sparse_create(struct amdgpu_winsys *ws, uint64_t size,
|
|
|
|
enum radeon_bo_domain domain,
|
|
|
|
enum radeon_bo_flag flags)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo;
|
|
|
|
uint64_t map_size;
|
|
|
|
uint64_t va_gap_size;
|
|
|
|
int r;
|
|
|
|
|
|
|
|
/* We use 32-bit page numbers; refuse to attempt allocating sparse buffers
|
|
|
|
* that exceed this limit. This is not really a restriction: we don't have
|
|
|
|
* that much virtual address space anyway.
|
|
|
|
*/
|
|
|
|
if (size > (uint64_t)INT32_MAX * RADEON_SPARSE_PAGE_SIZE)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
bo = CALLOC_STRUCT(amdgpu_winsys_bo);
|
|
|
|
if (!bo)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
pipe_reference_init(&bo->base.reference, 1);
|
|
|
|
bo->base.alignment = RADEON_SPARSE_PAGE_SIZE;
|
|
|
|
bo->base.size = size;
|
|
|
|
bo->base.vtbl = &amdgpu_winsys_bo_sparse_vtbl;
|
|
|
|
bo->ws = ws;
|
|
|
|
bo->initial_domain = domain;
|
|
|
|
bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
|
|
|
|
bo->sparse = true;
|
|
|
|
bo->u.sparse.flags = flags & ~RADEON_FLAG_SPARSE;
|
|
|
|
|
|
|
|
bo->u.sparse.num_va_pages = DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
|
|
|
|
bo->u.sparse.commitments = CALLOC(bo->u.sparse.num_va_pages,
|
|
|
|
sizeof(*bo->u.sparse.commitments));
|
|
|
|
if (!bo->u.sparse.commitments)
|
|
|
|
goto error_alloc_commitments;
|
|
|
|
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_init(&bo->u.sparse.commit_lock, mtx_plain);
|
2017-02-07 16:04:49 +00:00
|
|
|
LIST_INITHEAD(&bo->u.sparse.backing);
|
|
|
|
|
|
|
|
/* For simplicity, we always map a multiple of the page size. */
|
|
|
|
map_size = align64(size, RADEON_SPARSE_PAGE_SIZE);
|
|
|
|
va_gap_size = ws->check_vm ? 4 * RADEON_SPARSE_PAGE_SIZE : 0;
|
|
|
|
r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
|
|
|
|
map_size + va_gap_size, RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
0, &bo->va, &bo->u.sparse.va_handle, 0);
|
|
|
|
if (r)
|
|
|
|
goto error_va_alloc;
|
|
|
|
|
|
|
|
r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0, size, bo->va,
|
|
|
|
AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_MAP);
|
|
|
|
if (r)
|
|
|
|
goto error_va_map;
|
|
|
|
|
|
|
|
return &bo->base;
|
|
|
|
|
|
|
|
error_va_map:
|
|
|
|
amdgpu_va_range_free(bo->u.sparse.va_handle);
|
|
|
|
error_va_alloc:
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_destroy(&bo->u.sparse.commit_lock);
|
2017-02-07 16:04:49 +00:00
|
|
|
FREE(bo->u.sparse.commitments);
|
|
|
|
error_alloc_commitments:
|
|
|
|
FREE(bo);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool
|
|
|
|
amdgpu_bo_sparse_commit(struct pb_buffer *buf, uint64_t offset, uint64_t size,
|
|
|
|
bool commit)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buf);
|
|
|
|
struct amdgpu_sparse_commitment *comm;
|
|
|
|
uint32_t va_page, end_va_page;
|
|
|
|
bool ok = true;
|
|
|
|
int r;
|
|
|
|
|
|
|
|
assert(bo->sparse);
|
|
|
|
assert(offset % RADEON_SPARSE_PAGE_SIZE == 0);
|
|
|
|
assert(offset <= bo->base.size);
|
|
|
|
assert(size <= bo->base.size - offset);
|
|
|
|
assert(size % RADEON_SPARSE_PAGE_SIZE == 0 || offset + size == bo->base.size);
|
|
|
|
|
|
|
|
comm = bo->u.sparse.commitments;
|
|
|
|
va_page = offset / RADEON_SPARSE_PAGE_SIZE;
|
|
|
|
end_va_page = va_page + DIV_ROUND_UP(size, RADEON_SPARSE_PAGE_SIZE);
|
|
|
|
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_lock(&bo->u.sparse.commit_lock);
|
2017-02-07 16:04:49 +00:00
|
|
|
|
2017-02-13 11:51:12 +00:00
|
|
|
#if DEBUG_SPARSE_COMMITS
|
|
|
|
sparse_dump(bo, __func__);
|
|
|
|
#endif
|
|
|
|
|
2017-02-07 16:04:49 +00:00
|
|
|
if (commit) {
|
|
|
|
while (va_page < end_va_page) {
|
|
|
|
uint32_t span_va_page;
|
|
|
|
|
|
|
|
/* Skip pages that are already committed. */
|
|
|
|
if (comm[va_page].backing) {
|
|
|
|
va_page++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Determine length of uncommitted span. */
|
|
|
|
span_va_page = va_page;
|
|
|
|
while (va_page < end_va_page && !comm[va_page].backing)
|
|
|
|
va_page++;
|
|
|
|
|
|
|
|
/* Fill the uncommitted span with chunks of backing memory. */
|
|
|
|
while (span_va_page < va_page) {
|
|
|
|
struct amdgpu_sparse_backing *backing;
|
|
|
|
uint32_t backing_start, backing_size;
|
|
|
|
|
|
|
|
backing_size = va_page - span_va_page;
|
|
|
|
backing = sparse_backing_alloc(bo, &backing_start, &backing_size);
|
|
|
|
if (!backing) {
|
|
|
|
ok = false;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
r = amdgpu_bo_va_op_raw(bo->ws->dev, backing->bo->bo,
|
|
|
|
(uint64_t)backing_start * RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
(uint64_t)backing_size * RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
bo->va + (uint64_t)span_va_page * RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
AMDGPU_VM_PAGE_READABLE |
|
|
|
|
AMDGPU_VM_PAGE_WRITEABLE |
|
|
|
|
AMDGPU_VM_PAGE_EXECUTABLE,
|
|
|
|
AMDGPU_VA_OP_REPLACE);
|
|
|
|
if (r) {
|
|
|
|
ok = sparse_backing_free(bo, backing, backing_start, backing_size);
|
|
|
|
assert(ok && "sufficient memory should already be allocated");
|
|
|
|
|
|
|
|
ok = false;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
while (backing_size) {
|
|
|
|
comm[span_va_page].backing = backing;
|
|
|
|
comm[span_va_page].page = backing_start;
|
|
|
|
span_va_page++;
|
|
|
|
backing_start++;
|
|
|
|
backing_size--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
r = amdgpu_bo_va_op_raw(bo->ws->dev, NULL, 0,
|
|
|
|
(uint64_t)(end_va_page - va_page) * RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
bo->va + (uint64_t)va_page * RADEON_SPARSE_PAGE_SIZE,
|
|
|
|
AMDGPU_VM_PAGE_PRT, AMDGPU_VA_OP_REPLACE);
|
|
|
|
if (r) {
|
|
|
|
ok = false;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
while (va_page < end_va_page) {
|
|
|
|
struct amdgpu_sparse_backing *backing;
|
|
|
|
uint32_t backing_start;
|
|
|
|
uint32_t span_pages;
|
|
|
|
|
|
|
|
/* Skip pages that are already uncommitted. */
|
|
|
|
if (!comm[va_page].backing) {
|
|
|
|
va_page++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Group contiguous spans of pages. */
|
|
|
|
backing = comm[va_page].backing;
|
|
|
|
backing_start = comm[va_page].page;
|
|
|
|
comm[va_page].backing = NULL;
|
|
|
|
|
|
|
|
span_pages = 1;
|
|
|
|
va_page++;
|
|
|
|
|
|
|
|
while (va_page < end_va_page &&
|
|
|
|
comm[va_page].backing == backing &&
|
|
|
|
comm[va_page].page == backing_start + span_pages) {
|
|
|
|
comm[va_page].backing = NULL;
|
|
|
|
va_page++;
|
|
|
|
span_pages++;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!sparse_backing_free(bo, backing, backing_start, span_pages)) {
|
|
|
|
/* Couldn't allocate tracking data structures, so we have to leak */
|
|
|
|
fprintf(stderr, "amdgpu: leaking PRT backing memory\n");
|
|
|
|
ok = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
out:
|
|
|
|
|
2017-10-16 01:59:32 +01:00
|
|
|
simple_mtx_unlock(&bo->u.sparse.commit_lock);
|
2017-02-07 16:04:49 +00:00
|
|
|
|
|
|
|
return ok;
|
|
|
|
}
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
static unsigned eg_tile_split(unsigned tile_split)
|
|
|
|
{
|
|
|
|
switch (tile_split) {
|
|
|
|
case 0: tile_split = 64; break;
|
|
|
|
case 1: tile_split = 128; break;
|
|
|
|
case 2: tile_split = 256; break;
|
|
|
|
case 3: tile_split = 512; break;
|
|
|
|
default:
|
|
|
|
case 4: tile_split = 1024; break;
|
|
|
|
case 5: tile_split = 2048; break;
|
|
|
|
case 6: tile_split = 4096; break;
|
|
|
|
}
|
|
|
|
return tile_split;
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned eg_tile_split_rev(unsigned eg_tile_split)
|
|
|
|
{
|
|
|
|
switch (eg_tile_split) {
|
|
|
|
case 64: return 0;
|
|
|
|
case 128: return 1;
|
|
|
|
case 256: return 2;
|
|
|
|
case 512: return 3;
|
|
|
|
default:
|
|
|
|
case 1024: return 4;
|
|
|
|
case 2048: return 5;
|
|
|
|
case 4096: return 6;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-02-24 00:13:22 +00:00
|
|
|
static void amdgpu_buffer_get_metadata(struct pb_buffer *_buf,
|
|
|
|
struct radeon_bo_metadata *md)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
2015-12-06 19:57:05 +00:00
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
2015-04-16 21:43:23 +01:00
|
|
|
struct amdgpu_bo_info info = {0};
|
2016-11-23 17:42:53 +00:00
|
|
|
uint64_t tiling_flags;
|
2015-04-16 21:43:23 +01:00
|
|
|
int r;
|
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
assert(bo->bo && "must not be called for slab entries");
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
r = amdgpu_bo_query_info(bo->bo, &info);
|
|
|
|
if (r)
|
|
|
|
return;
|
|
|
|
|
|
|
|
tiling_flags = info.metadata.tiling_info;
|
|
|
|
|
2016-11-06 13:51:57 +00:00
|
|
|
if (bo->ws->info.chip_class >= GFX9) {
|
2016-11-23 17:42:53 +00:00
|
|
|
md->u.gfx9.swizzle_mode = AMDGPU_TILING_GET(tiling_flags, SWIZZLE_MODE);
|
2016-11-06 13:51:57 +00:00
|
|
|
} else {
|
|
|
|
md->u.legacy.microtile = RADEON_LAYOUT_LINEAR;
|
|
|
|
md->u.legacy.macrotile = RADEON_LAYOUT_LINEAR;
|
|
|
|
|
|
|
|
if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 4) /* 2D_TILED_THIN1 */
|
|
|
|
md->u.legacy.macrotile = RADEON_LAYOUT_TILED;
|
|
|
|
else if (AMDGPU_TILING_GET(tiling_flags, ARRAY_MODE) == 2) /* 1D_TILED_THIN1 */
|
|
|
|
md->u.legacy.microtile = RADEON_LAYOUT_TILED;
|
|
|
|
|
|
|
|
md->u.legacy.pipe_config = AMDGPU_TILING_GET(tiling_flags, PIPE_CONFIG);
|
|
|
|
md->u.legacy.bankw = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_WIDTH);
|
|
|
|
md->u.legacy.bankh = 1 << AMDGPU_TILING_GET(tiling_flags, BANK_HEIGHT);
|
|
|
|
md->u.legacy.tile_split = eg_tile_split(AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT));
|
|
|
|
md->u.legacy.mtilea = 1 << AMDGPU_TILING_GET(tiling_flags, MACRO_TILE_ASPECT);
|
|
|
|
md->u.legacy.num_banks = 2 << AMDGPU_TILING_GET(tiling_flags, NUM_BANKS);
|
|
|
|
md->u.legacy.scanout = AMDGPU_TILING_GET(tiling_flags, MICRO_TILE_MODE) == 0; /* DISPLAY */
|
|
|
|
}
|
2016-02-24 00:24:06 +00:00
|
|
|
|
|
|
|
md->size_metadata = info.metadata.size_metadata;
|
|
|
|
memcpy(md->metadata, info.metadata.umd_metadata, sizeof(md->metadata));
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
2016-02-24 00:13:22 +00:00
|
|
|
static void amdgpu_buffer_set_metadata(struct pb_buffer *_buf,
|
|
|
|
struct radeon_bo_metadata *md)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
2015-12-06 19:57:05 +00:00
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(_buf);
|
2015-04-16 21:43:23 +01:00
|
|
|
struct amdgpu_bo_metadata metadata = {0};
|
2016-11-23 17:42:53 +00:00
|
|
|
uint64_t tiling_flags = 0;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-09-07 09:37:42 +01:00
|
|
|
assert(bo->bo && "must not be called for slab entries");
|
|
|
|
|
2016-11-06 13:51:57 +00:00
|
|
|
if (bo->ws->info.chip_class >= GFX9) {
|
2016-11-23 17:42:53 +00:00
|
|
|
tiling_flags |= AMDGPU_TILING_SET(SWIZZLE_MODE, md->u.gfx9.swizzle_mode);
|
2016-11-06 13:51:57 +00:00
|
|
|
} else {
|
|
|
|
if (md->u.legacy.macrotile == RADEON_LAYOUT_TILED)
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 4); /* 2D_TILED_THIN1 */
|
|
|
|
else if (md->u.legacy.microtile == RADEON_LAYOUT_TILED)
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 2); /* 1D_TILED_THIN1 */
|
|
|
|
else
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(ARRAY_MODE, 1); /* LINEAR_ALIGNED */
|
|
|
|
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(PIPE_CONFIG, md->u.legacy.pipe_config);
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(BANK_WIDTH, util_logbase2(md->u.legacy.bankw));
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(BANK_HEIGHT, util_logbase2(md->u.legacy.bankh));
|
|
|
|
if (md->u.legacy.tile_split)
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(TILE_SPLIT, eg_tile_split_rev(md->u.legacy.tile_split));
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(MACRO_TILE_ASPECT, util_logbase2(md->u.legacy.mtilea));
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(NUM_BANKS, util_logbase2(md->u.legacy.num_banks)-1);
|
|
|
|
|
|
|
|
if (md->u.legacy.scanout)
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 0); /* DISPLAY_MICRO_TILING */
|
|
|
|
else
|
|
|
|
tiling_flags |= AMDGPU_TILING_SET(MICRO_TILE_MODE, 1); /* THIN_MICRO_TILING */
|
|
|
|
}
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
metadata.tiling_info = tiling_flags;
|
2016-02-24 00:24:06 +00:00
|
|
|
metadata.size_metadata = md->size_metadata;
|
|
|
|
memcpy(metadata.umd_metadata, md->metadata, sizeof(md->metadata));
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
amdgpu_bo_set_metadata(bo->bo, &metadata);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct pb_buffer *
|
|
|
|
amdgpu_bo_create(struct radeon_winsys *rws,
|
2016-04-10 15:48:55 +01:00
|
|
|
uint64_t size,
|
2015-04-16 21:43:23 +01:00
|
|
|
unsigned alignment,
|
|
|
|
enum radeon_bo_domain domain,
|
|
|
|
enum radeon_bo_flag flags)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys *ws = amdgpu_winsys(rws);
|
2015-12-06 19:57:05 +00:00
|
|
|
struct amdgpu_winsys_bo *bo;
|
2017-07-18 21:08:44 +01:00
|
|
|
unsigned usage = 0, pb_cache_bucket = 0;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2017-06-29 17:01:16 +01:00
|
|
|
/* VRAM implies WC. This is not optional. */
|
|
|
|
assert(!(domain & RADEON_DOMAIN_VRAM) || flags & RADEON_FLAG_GTT_WC);
|
|
|
|
|
2017-06-29 17:33:05 +01:00
|
|
|
/* NO_CPU_ACCESS is valid with VRAM only. */
|
|
|
|
assert(domain == RADEON_DOMAIN_VRAM || !(flags & RADEON_FLAG_NO_CPU_ACCESS));
|
|
|
|
|
2017-11-23 09:29:49 +00:00
|
|
|
/* Sparse buffers must have NO_CPU_ACCESS set. */
|
|
|
|
assert(!(flags & RADEON_FLAG_SPARSE) || flags & RADEON_FLAG_NO_CPU_ACCESS);
|
|
|
|
|
2016-09-07 09:50:59 +01:00
|
|
|
/* Sub-allocate small buffers from slabs. */
|
2017-06-27 18:37:43 +01:00
|
|
|
if (!(flags & (RADEON_FLAG_NO_SUBALLOC | RADEON_FLAG_SPARSE)) &&
|
2016-09-07 09:50:59 +01:00
|
|
|
size <= (1 << AMDGPU_SLAB_MAX_SIZE_LOG2) &&
|
|
|
|
alignment <= MAX2(1 << AMDGPU_SLAB_MIN_SIZE_LOG2, util_next_power_of_two(size))) {
|
|
|
|
struct pb_slab_entry *entry;
|
2017-06-29 17:01:16 +01:00
|
|
|
int heap = radeon_get_heap_index(domain, flags);
|
2016-09-07 09:50:59 +01:00
|
|
|
|
2017-06-29 17:01:16 +01:00
|
|
|
if (heap < 0 || heap >= RADEON_MAX_SLAB_HEAPS)
|
2016-09-07 09:50:59 +01:00
|
|
|
goto no_slab;
|
|
|
|
|
|
|
|
entry = pb_slab_alloc(&ws->bo_slabs, size, heap);
|
|
|
|
if (!entry) {
|
|
|
|
/* Clear the cache and try again. */
|
|
|
|
pb_cache_release_all_buffers(&ws->bo_cache);
|
|
|
|
|
|
|
|
entry = pb_slab_alloc(&ws->bo_slabs, size, heap);
|
|
|
|
}
|
|
|
|
if (!entry)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
bo = NULL;
|
|
|
|
bo = container_of(entry, bo, u.slab.entry);
|
|
|
|
|
|
|
|
pipe_reference_init(&bo->base.reference, 1);
|
|
|
|
|
|
|
|
return &bo->base;
|
|
|
|
}
|
|
|
|
no_slab:
|
|
|
|
|
2017-02-07 16:04:49 +00:00
|
|
|
if (flags & RADEON_FLAG_SPARSE) {
|
|
|
|
assert(RADEON_SPARSE_PAGE_SIZE % alignment == 0);
|
|
|
|
|
|
|
|
return amdgpu_bo_sparse_create(ws, size, domain, flags);
|
|
|
|
}
|
|
|
|
|
2016-09-09 10:49:18 +01:00
|
|
|
/* This flag is irrelevant for the cache. */
|
2017-06-27 18:37:43 +01:00
|
|
|
flags &= ~RADEON_FLAG_NO_SUBALLOC;
|
2016-09-09 10:49:18 +01:00
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
/* Align size to page size. This is the minimum alignment for normal
|
|
|
|
* BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
|
|
|
|
* like constant/uniform buffers, can benefit from better and more reuse.
|
|
|
|
*/
|
2016-05-08 11:27:14 +01:00
|
|
|
size = align64(size, ws->info.gart_page_size);
|
2016-05-08 12:40:40 +01:00
|
|
|
alignment = align(alignment, ws->info.gart_page_size);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2017-07-18 21:08:44 +01:00
|
|
|
bool use_reusable_pool = flags & RADEON_FLAG_NO_INTERPROCESS_SHARING;
|
2017-06-29 17:49:48 +01:00
|
|
|
|
2017-07-18 21:08:44 +01:00
|
|
|
if (use_reusable_pool) {
|
|
|
|
int heap = radeon_get_heap_index(domain, flags);
|
|
|
|
assert(heap >= 0 && heap < RADEON_MAX_CACHED_HEAPS);
|
|
|
|
usage = 1 << heap; /* Only set one usage bit for each heap. */
|
2016-07-17 12:23:53 +01:00
|
|
|
|
2017-07-18 21:08:44 +01:00
|
|
|
pb_cache_bucket = radeon_get_pb_cache_bucket_index(heap);
|
|
|
|
|
|
|
|
/* Get a buffer from the cache. */
|
|
|
|
bo = (struct amdgpu_winsys_bo*)
|
|
|
|
pb_cache_reclaim_buffer(&ws->bo_cache, size, alignment, usage,
|
|
|
|
pb_cache_bucket);
|
|
|
|
if (bo)
|
|
|
|
return &bo->base;
|
|
|
|
}
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
/* Create a new one. */
|
2016-07-17 12:23:53 +01:00
|
|
|
bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags,
|
|
|
|
pb_cache_bucket);
|
2015-12-09 21:36:26 +00:00
|
|
|
if (!bo) {
|
|
|
|
/* Clear the cache and try again. */
|
2016-09-07 09:50:59 +01:00
|
|
|
pb_slabs_reclaim(&ws->bo_slabs);
|
2015-12-09 21:36:26 +00:00
|
|
|
pb_cache_release_all_buffers(&ws->bo_cache);
|
2016-07-17 12:23:53 +01:00
|
|
|
bo = amdgpu_create_bo(ws, size, alignment, usage, domain, flags,
|
|
|
|
pb_cache_bucket);
|
2015-12-09 21:36:26 +00:00
|
|
|
if (!bo)
|
|
|
|
return NULL;
|
|
|
|
}
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2017-07-18 21:08:44 +01:00
|
|
|
bo->u.real.use_reusable_pool = use_reusable_pool;
|
2015-12-06 19:57:05 +00:00
|
|
|
return &bo->base;
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pb_buffer *amdgpu_bo_from_handle(struct radeon_winsys *rws,
|
|
|
|
struct winsys_handle *whandle,
|
2016-01-12 14:59:11 +00:00
|
|
|
unsigned *stride,
|
|
|
|
unsigned *offset)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
struct amdgpu_winsys *ws = amdgpu_winsys(rws);
|
|
|
|
struct amdgpu_winsys_bo *bo;
|
|
|
|
enum amdgpu_bo_handle_type type;
|
|
|
|
struct amdgpu_bo_import_result result = {0};
|
|
|
|
uint64_t va;
|
|
|
|
amdgpu_va_handle va_handle;
|
|
|
|
struct amdgpu_bo_info info = {0};
|
|
|
|
enum radeon_bo_domain initial = 0;
|
|
|
|
int r;
|
|
|
|
|
|
|
|
/* Initialize the structure. */
|
|
|
|
bo = CALLOC_STRUCT(amdgpu_winsys_bo);
|
|
|
|
if (!bo) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
switch (whandle->type) {
|
|
|
|
case DRM_API_HANDLE_TYPE_SHARED:
|
|
|
|
type = amdgpu_bo_handle_type_gem_flink_name;
|
|
|
|
break;
|
|
|
|
case DRM_API_HANDLE_TYPE_FD:
|
|
|
|
type = amdgpu_bo_handle_type_dma_buf_fd;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
r = amdgpu_bo_import(ws->dev, type, whandle->handle, &result);
|
|
|
|
if (r)
|
|
|
|
goto error;
|
|
|
|
|
|
|
|
/* Get initial domains. */
|
|
|
|
r = amdgpu_bo_query_info(result.buf_handle, &info);
|
|
|
|
if (r)
|
|
|
|
goto error_query;
|
|
|
|
|
|
|
|
r = amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
|
|
|
|
result.alloc_size, 1 << 20, 0, &va, &va_handle, 0);
|
|
|
|
if (r)
|
|
|
|
goto error_query;
|
|
|
|
|
|
|
|
r = amdgpu_bo_va_op(result.buf_handle, 0, result.alloc_size, va, 0, AMDGPU_VA_OP_MAP);
|
|
|
|
if (r)
|
|
|
|
goto error_va_map;
|
|
|
|
|
|
|
|
if (info.preferred_heap & AMDGPU_GEM_DOMAIN_VRAM)
|
|
|
|
initial |= RADEON_DOMAIN_VRAM;
|
|
|
|
if (info.preferred_heap & AMDGPU_GEM_DOMAIN_GTT)
|
|
|
|
initial |= RADEON_DOMAIN_GTT;
|
|
|
|
|
|
|
|
|
|
|
|
pipe_reference_init(&bo->base.reference, 1);
|
|
|
|
bo->base.alignment = info.phys_alignment;
|
|
|
|
bo->bo = result.buf_handle;
|
|
|
|
bo->base.size = result.alloc_size;
|
|
|
|
bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
|
2015-12-06 21:19:38 +00:00
|
|
|
bo->ws = ws;
|
2015-04-16 21:43:23 +01:00
|
|
|
bo->va = va;
|
2016-09-07 09:37:42 +01:00
|
|
|
bo->u.real.va_handle = va_handle;
|
2015-04-16 21:43:23 +01:00
|
|
|
bo->initial_domain = initial;
|
|
|
|
bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
|
|
|
|
bo->is_shared = true;
|
|
|
|
|
|
|
|
if (stride)
|
|
|
|
*stride = whandle->stride;
|
2016-01-12 14:59:11 +00:00
|
|
|
if (offset)
|
|
|
|
*offset = whandle->offset;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
if (bo->initial_domain & RADEON_DOMAIN_VRAM)
|
2016-05-08 11:27:14 +01:00
|
|
|
ws->allocated_vram += align64(bo->base.size, ws->info.gart_page_size);
|
2015-04-16 21:43:23 +01:00
|
|
|
else if (bo->initial_domain & RADEON_DOMAIN_GTT)
|
2016-05-08 11:27:14 +01:00
|
|
|
ws->allocated_gtt += align64(bo->base.size, ws->info.gart_page_size);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-01-14 18:31:18 +00:00
|
|
|
amdgpu_add_buffer_to_global_list(bo);
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
return &bo->base;
|
|
|
|
|
|
|
|
error_va_map:
|
|
|
|
amdgpu_va_range_free(va_handle);
|
|
|
|
|
|
|
|
error_query:
|
|
|
|
amdgpu_bo_free(result.buf_handle);
|
|
|
|
|
|
|
|
error:
|
|
|
|
FREE(bo);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-06-21 20:29:39 +01:00
|
|
|
static bool amdgpu_bo_get_handle(struct pb_buffer *buffer,
|
|
|
|
unsigned stride, unsigned offset,
|
|
|
|
unsigned slice_size,
|
|
|
|
struct winsys_handle *whandle)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
2015-12-06 19:57:05 +00:00
|
|
|
struct amdgpu_winsys_bo *bo = amdgpu_winsys_bo(buffer);
|
2015-04-16 21:43:23 +01:00
|
|
|
enum amdgpu_bo_handle_type type;
|
|
|
|
int r;
|
|
|
|
|
2017-06-27 18:32:48 +01:00
|
|
|
/* Don't allow exports of slab entries and sparse buffers. */
|
|
|
|
if (!bo->bo)
|
|
|
|
return false;
|
2016-09-07 09:37:42 +01:00
|
|
|
|
|
|
|
bo->u.real.use_reusable_pool = false;
|
2015-09-01 03:14:33 +01:00
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
switch (whandle->type) {
|
|
|
|
case DRM_API_HANDLE_TYPE_SHARED:
|
|
|
|
type = amdgpu_bo_handle_type_gem_flink_name;
|
|
|
|
break;
|
|
|
|
case DRM_API_HANDLE_TYPE_FD:
|
|
|
|
type = amdgpu_bo_handle_type_dma_buf_fd;
|
|
|
|
break;
|
|
|
|
case DRM_API_HANDLE_TYPE_KMS:
|
|
|
|
type = amdgpu_bo_handle_type_kms;
|
|
|
|
break;
|
|
|
|
default:
|
2016-06-21 20:29:39 +01:00
|
|
|
return false;
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
r = amdgpu_bo_export(bo->bo, type, &whandle->handle);
|
|
|
|
if (r)
|
2016-06-21 20:29:39 +01:00
|
|
|
return false;
|
2015-04-16 21:43:23 +01:00
|
|
|
|
|
|
|
whandle->stride = stride;
|
2016-01-12 14:59:11 +00:00
|
|
|
whandle->offset = offset;
|
2016-01-14 14:01:39 +00:00
|
|
|
whandle->offset += slice_size * whandle->layer;
|
2015-04-16 21:43:23 +01:00
|
|
|
bo->is_shared = true;
|
2016-06-21 20:29:39 +01:00
|
|
|
return true;
|
2015-04-16 21:43:23 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pb_buffer *amdgpu_bo_from_ptr(struct radeon_winsys *rws,
|
2016-04-10 15:48:55 +01:00
|
|
|
void *pointer, uint64_t size)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
struct amdgpu_winsys *ws = amdgpu_winsys(rws);
|
|
|
|
amdgpu_bo_handle buf_handle;
|
|
|
|
struct amdgpu_winsys_bo *bo;
|
|
|
|
uint64_t va;
|
|
|
|
amdgpu_va_handle va_handle;
|
|
|
|
|
|
|
|
bo = CALLOC_STRUCT(amdgpu_winsys_bo);
|
|
|
|
if (!bo)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (amdgpu_create_bo_from_user_mem(ws->dev, pointer, size, &buf_handle))
|
|
|
|
goto error;
|
|
|
|
|
|
|
|
if (amdgpu_va_range_alloc(ws->dev, amdgpu_gpu_va_range_general,
|
|
|
|
size, 1 << 12, 0, &va, &va_handle, 0))
|
|
|
|
goto error_va_alloc;
|
|
|
|
|
|
|
|
if (amdgpu_bo_va_op(buf_handle, 0, size, va, 0, AMDGPU_VA_OP_MAP))
|
|
|
|
goto error_va_map;
|
|
|
|
|
|
|
|
/* Initialize it. */
|
|
|
|
pipe_reference_init(&bo->base.reference, 1);
|
|
|
|
bo->bo = buf_handle;
|
|
|
|
bo->base.alignment = 0;
|
|
|
|
bo->base.size = size;
|
|
|
|
bo->base.vtbl = &amdgpu_winsys_bo_vtbl;
|
2015-12-06 21:19:38 +00:00
|
|
|
bo->ws = ws;
|
2015-04-16 21:43:23 +01:00
|
|
|
bo->user_ptr = pointer;
|
|
|
|
bo->va = va;
|
2016-09-07 09:37:42 +01:00
|
|
|
bo->u.real.va_handle = va_handle;
|
2015-04-16 21:43:23 +01:00
|
|
|
bo->initial_domain = RADEON_DOMAIN_GTT;
|
|
|
|
bo->unique_id = __sync_fetch_and_add(&ws->next_bo_unique_id, 1);
|
|
|
|
|
2016-05-08 11:27:14 +01:00
|
|
|
ws->allocated_gtt += align64(bo->base.size, ws->info.gart_page_size);
|
2015-04-16 21:43:23 +01:00
|
|
|
|
2016-01-14 18:31:18 +00:00
|
|
|
amdgpu_add_buffer_to_global_list(bo);
|
|
|
|
|
2015-04-16 21:43:23 +01:00
|
|
|
return (struct pb_buffer*)bo;
|
|
|
|
|
|
|
|
error_va_map:
|
|
|
|
amdgpu_va_range_free(va_handle);
|
|
|
|
|
|
|
|
error_va_alloc:
|
|
|
|
amdgpu_bo_free(buf_handle);
|
|
|
|
|
|
|
|
error:
|
|
|
|
FREE(bo);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-01-12 14:29:18 +00:00
|
|
|
static bool amdgpu_bo_is_user_ptr(struct pb_buffer *buf)
|
|
|
|
{
|
|
|
|
return ((struct amdgpu_winsys_bo*)buf)->user_ptr != NULL;
|
|
|
|
}
|
|
|
|
|
2017-06-27 19:27:29 +01:00
|
|
|
static bool amdgpu_bo_is_suballocated(struct pb_buffer *buf)
|
|
|
|
{
|
|
|
|
struct amdgpu_winsys_bo *bo = (struct amdgpu_winsys_bo*)buf;
|
|
|
|
|
|
|
|
return !bo->bo && !bo->sparse;
|
|
|
|
}
|
|
|
|
|
2015-12-06 23:00:59 +00:00
|
|
|
static uint64_t amdgpu_bo_get_va(struct pb_buffer *buf)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
|
|
|
return ((struct amdgpu_winsys_bo*)buf)->va;
|
|
|
|
}
|
|
|
|
|
2015-12-06 19:57:05 +00:00
|
|
|
void amdgpu_bo_init_functions(struct amdgpu_winsys *ws)
|
2015-04-16 21:43:23 +01:00
|
|
|
{
|
2016-02-24 00:13:22 +00:00
|
|
|
ws->base.buffer_set_metadata = amdgpu_buffer_set_metadata;
|
|
|
|
ws->base.buffer_get_metadata = amdgpu_buffer_get_metadata;
|
2015-04-16 21:43:23 +01:00
|
|
|
ws->base.buffer_map = amdgpu_bo_map;
|
|
|
|
ws->base.buffer_unmap = amdgpu_bo_unmap;
|
|
|
|
ws->base.buffer_wait = amdgpu_bo_wait;
|
|
|
|
ws->base.buffer_create = amdgpu_bo_create;
|
|
|
|
ws->base.buffer_from_handle = amdgpu_bo_from_handle;
|
|
|
|
ws->base.buffer_from_ptr = amdgpu_bo_from_ptr;
|
2016-01-12 14:29:18 +00:00
|
|
|
ws->base.buffer_is_user_ptr = amdgpu_bo_is_user_ptr;
|
2017-06-27 19:27:29 +01:00
|
|
|
ws->base.buffer_is_suballocated = amdgpu_bo_is_suballocated;
|
2015-04-16 21:43:23 +01:00
|
|
|
ws->base.buffer_get_handle = amdgpu_bo_get_handle;
|
2017-02-07 16:04:49 +00:00
|
|
|
ws->base.buffer_commit = amdgpu_bo_sparse_commit;
|
2015-04-16 21:43:23 +01:00
|
|
|
ws->base.buffer_get_virtual_address = amdgpu_bo_get_va;
|
|
|
|
ws->base.buffer_get_initial_domain = amdgpu_bo_get_initial_domain;
|
|
|
|
}
|