294 lines
9.6 KiB
C
294 lines
9.6 KiB
C
/*
|
|
* Copyright © 2017 Intel Corporation
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included
|
|
* in all copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
* DEALINGS IN THE SOFTWARE.
|
|
*/
|
|
|
|
/**
|
|
* @file iris_program_cache.c
|
|
*
|
|
* The in-memory program cache. This is basically a hash table mapping
|
|
* API-specified shaders and a state key to a compiled variant. It also
|
|
* takes care of uploading shader assembly into a BO for use on the GPU.
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <errno.h>
|
|
#include "pipe/p_defines.h"
|
|
#include "pipe/p_state.h"
|
|
#include "pipe/p_context.h"
|
|
#include "pipe/p_screen.h"
|
|
#include "util/u_atomic.h"
|
|
#include "util/u_upload_mgr.h"
|
|
#include "compiler/nir/nir.h"
|
|
#include "compiler/nir/nir_builder.h"
|
|
#include "intel/common/intel_disasm.h"
|
|
#include "intel/compiler/brw_compiler.h"
|
|
#include "intel/compiler/brw_eu.h"
|
|
#include "intel/compiler/brw_nir.h"
|
|
#include "iris_context.h"
|
|
#include "iris_resource.h"
|
|
|
|
struct keybox {
|
|
uint16_t size;
|
|
enum iris_program_cache_id cache_id;
|
|
uint8_t data[0];
|
|
};
|
|
|
|
static struct keybox *
|
|
make_keybox(void *mem_ctx,
|
|
enum iris_program_cache_id cache_id,
|
|
const void *key,
|
|
uint32_t key_size)
|
|
{
|
|
struct keybox *keybox =
|
|
ralloc_size(mem_ctx, sizeof(struct keybox) + key_size);
|
|
|
|
keybox->cache_id = cache_id;
|
|
keybox->size = key_size;
|
|
memcpy(keybox->data, key, key_size);
|
|
|
|
return keybox;
|
|
}
|
|
|
|
static uint32_t
|
|
keybox_hash(const void *void_key)
|
|
{
|
|
const struct keybox *key = void_key;
|
|
return _mesa_hash_data(&key->cache_id, key->size + sizeof(key->cache_id));
|
|
}
|
|
|
|
static bool
|
|
keybox_equals(const void *void_a, const void *void_b)
|
|
{
|
|
const struct keybox *a = void_a, *b = void_b;
|
|
if (a->size != b->size)
|
|
return false;
|
|
|
|
return memcmp(a->data, b->data, a->size) == 0;
|
|
}
|
|
|
|
struct iris_compiled_shader *
|
|
iris_find_cached_shader(struct iris_context *ice,
|
|
enum iris_program_cache_id cache_id,
|
|
uint32_t key_size,
|
|
const void *key)
|
|
{
|
|
struct keybox *keybox = make_keybox(NULL, cache_id, key, key_size);
|
|
struct hash_entry *entry =
|
|
_mesa_hash_table_search(ice->shaders.cache, keybox);
|
|
|
|
ralloc_free(keybox);
|
|
|
|
return entry ? entry->data : NULL;
|
|
}
|
|
|
|
void
|
|
iris_delete_shader_variant(struct iris_compiled_shader *shader)
|
|
{
|
|
pipe_resource_reference(&shader->assembly.res, NULL);
|
|
util_queue_fence_destroy(&shader->ready);
|
|
ralloc_free(shader);
|
|
}
|
|
|
|
struct iris_compiled_shader *
|
|
iris_create_shader_variant(const struct iris_screen *screen,
|
|
void *mem_ctx,
|
|
enum iris_program_cache_id cache_id,
|
|
uint32_t key_size,
|
|
const void *key)
|
|
{
|
|
#ifndef NDEBUG
|
|
if (cache_id == IRIS_CACHE_BLORP) {
|
|
/* Blorp shader must have a mem_ctx. */
|
|
assert(mem_ctx != NULL);
|
|
} else if (cache_id == IRIS_CACHE_TCS) {
|
|
/* Pass-through tessellation control shaders (generated by the driver)
|
|
* will have a mem_ctx, and other tessellation control shaders will not.
|
|
*/
|
|
} else {
|
|
/* Shaders that are neither blorp nor tessellation control must not have
|
|
* a mem_ctx.
|
|
*/
|
|
assert(mem_ctx == NULL);
|
|
}
|
|
#endif
|
|
|
|
struct iris_compiled_shader *shader =
|
|
rzalloc_size(mem_ctx, sizeof(struct iris_compiled_shader) +
|
|
screen->vtbl.derived_program_state_size(cache_id));
|
|
|
|
pipe_reference_init(&shader->ref, 1);
|
|
util_queue_fence_init(&shader->ready);
|
|
util_queue_fence_reset(&shader->ready);
|
|
|
|
if (cache_id != IRIS_CACHE_BLORP) {
|
|
assert(key_size <= sizeof(union iris_any_prog_key));
|
|
memcpy(&shader->key, key, key_size);
|
|
}
|
|
|
|
return shader;
|
|
}
|
|
|
|
void
|
|
iris_upload_shader(struct iris_screen *screen,
|
|
struct iris_uncompiled_shader *ish,
|
|
struct iris_compiled_shader *shader,
|
|
struct hash_table *driver_shaders,
|
|
struct u_upload_mgr *uploader,
|
|
enum iris_program_cache_id cache_id,
|
|
uint32_t key_size,
|
|
const void *key,
|
|
const void *assembly)
|
|
{
|
|
const struct intel_device_info *devinfo = &screen->devinfo;
|
|
|
|
u_upload_alloc(uploader, 0, shader->prog_data->program_size, 64,
|
|
&shader->assembly.offset, &shader->assembly.res,
|
|
&shader->map);
|
|
memcpy(shader->map, assembly, shader->prog_data->program_size);
|
|
|
|
struct iris_resource *res = (void *) shader->assembly.res;
|
|
uint64_t shader_data_addr = res->bo->address +
|
|
shader->assembly.offset +
|
|
shader->prog_data->const_data_offset;
|
|
|
|
struct brw_shader_reloc_value reloc_values[] = {
|
|
{
|
|
.id = BRW_SHADER_RELOC_CONST_DATA_ADDR_LOW,
|
|
.value = shader_data_addr,
|
|
},
|
|
{
|
|
.id = BRW_SHADER_RELOC_CONST_DATA_ADDR_HIGH,
|
|
.value = shader_data_addr >> 32,
|
|
},
|
|
};
|
|
brw_write_shader_relocs(&screen->compiler->isa, shader->map,
|
|
shader->prog_data, reloc_values,
|
|
ARRAY_SIZE(reloc_values));
|
|
|
|
/* Store the 3DSTATE shader packets and other derived state. */
|
|
screen->vtbl.store_derived_program_state(devinfo, cache_id, shader);
|
|
|
|
util_queue_fence_signal(&shader->ready);
|
|
|
|
if (!ish) {
|
|
struct keybox *keybox = make_keybox(shader, cache_id, key, key_size);
|
|
_mesa_hash_table_insert(driver_shaders, keybox, shader);
|
|
}
|
|
}
|
|
|
|
bool
|
|
iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
|
|
const void *key, uint32_t key_size,
|
|
uint32_t *kernel_out, void *prog_data_out)
|
|
{
|
|
struct blorp_context *blorp = blorp_batch->blorp;
|
|
struct iris_context *ice = blorp->driver_ctx;
|
|
struct iris_batch *batch = blorp_batch->driver_batch;
|
|
struct iris_compiled_shader *shader =
|
|
iris_find_cached_shader(ice, IRIS_CACHE_BLORP, key_size, key);
|
|
|
|
if (!shader)
|
|
return false;
|
|
|
|
struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
|
|
*kernel_out =
|
|
iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
|
|
*((void **) prog_data_out) = shader->prog_data;
|
|
|
|
iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
|
|
|
|
return true;
|
|
}
|
|
|
|
bool
|
|
iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
|
|
const void *key, uint32_t key_size,
|
|
const void *kernel, UNUSED uint32_t kernel_size,
|
|
const struct brw_stage_prog_data *prog_data_templ,
|
|
UNUSED uint32_t prog_data_size,
|
|
uint32_t *kernel_out, void *prog_data_out)
|
|
{
|
|
struct blorp_context *blorp = blorp_batch->blorp;
|
|
struct iris_context *ice = blorp->driver_ctx;
|
|
struct iris_batch *batch = blorp_batch->driver_batch;
|
|
struct iris_screen *screen = batch->screen;
|
|
|
|
void *prog_data = ralloc_size(NULL, prog_data_size);
|
|
memcpy(prog_data, prog_data_templ, prog_data_size);
|
|
|
|
struct iris_binding_table bt;
|
|
memset(&bt, 0, sizeof(bt));
|
|
|
|
struct iris_compiled_shader *shader =
|
|
iris_create_shader_variant(screen, ice->shaders.cache, IRIS_CACHE_BLORP,
|
|
key_size, key);
|
|
|
|
iris_finalize_program(shader, prog_data, NULL, NULL, 0, 0, 0, &bt);
|
|
|
|
iris_upload_shader(screen, NULL, shader, ice->shaders.cache,
|
|
ice->shaders.uploader_driver,
|
|
IRIS_CACHE_BLORP, key_size, key, kernel);
|
|
|
|
struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
|
|
*kernel_out =
|
|
iris_bo_offset_from_base_address(bo) + shader->assembly.offset;
|
|
*((void **) prog_data_out) = shader->prog_data;
|
|
|
|
iris_use_pinned_bo(batch, bo, false, IRIS_DOMAIN_NONE);
|
|
|
|
return true;
|
|
}
|
|
|
|
void
|
|
iris_init_program_cache(struct iris_context *ice)
|
|
{
|
|
ice->shaders.cache =
|
|
_mesa_hash_table_create(ice, keybox_hash, keybox_equals);
|
|
|
|
ice->shaders.uploader_driver =
|
|
u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
|
|
IRIS_RESOURCE_FLAG_SHADER_MEMZONE |
|
|
IRIS_RESOURCE_FLAG_DEVICE_MEM);
|
|
ice->shaders.uploader_unsync =
|
|
u_upload_create(&ice->ctx, 16384, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
|
|
IRIS_RESOURCE_FLAG_SHADER_MEMZONE |
|
|
IRIS_RESOURCE_FLAG_DEVICE_MEM);
|
|
}
|
|
|
|
void
|
|
iris_destroy_program_cache(struct iris_context *ice)
|
|
{
|
|
for (int i = 0; i < MESA_SHADER_STAGES; i++) {
|
|
iris_shader_variant_reference(&ice->shaders.prog[i], NULL);
|
|
}
|
|
iris_shader_variant_reference(&ice->shaders.last_vue_shader, NULL);
|
|
|
|
hash_table_foreach(ice->shaders.cache, entry) {
|
|
struct iris_compiled_shader *shader = entry->data;
|
|
iris_delete_shader_variant(shader);
|
|
}
|
|
|
|
u_upload_destroy(ice->shaders.uploader_driver);
|
|
u_upload_destroy(ice->shaders.uploader_unsync);
|
|
|
|
ralloc_free(ice->shaders.cache);
|
|
}
|