diff --git a/src/gallium/drivers/iris/iris_context.h b/src/gallium/drivers/iris/iris_context.h index a3e4da792e9..add6a9a9e34 100644 --- a/src/gallium/drivers/iris/iris_context.h +++ b/src/gallium/drivers/iris/iris_context.h @@ -781,6 +781,19 @@ struct iris_bo *iris_get_scratch_space(struct iris_context *ice, unsigned per_thread_scratch, gl_shader_stage stage); +/* iris_disk_cache.c */ + +void iris_disk_cache_store(struct disk_cache *cache, + const struct iris_uncompiled_shader *ish, + const struct iris_compiled_shader *shader, + const void *prog_key, + uint32_t prog_key_size); +struct iris_compiled_shader * +iris_disk_cache_retrieve(struct iris_context *ice, + const struct iris_uncompiled_shader *ish, + const void *prog_key, + uint32_t prog_key_size); + /* iris_program_cache.c */ void iris_init_program_cache(struct iris_context *ice); diff --git a/src/gallium/drivers/iris/iris_disk_cache.c b/src/gallium/drivers/iris/iris_disk_cache.c index c0dc46e7e25..2b022aff66a 100644 --- a/src/gallium/drivers/iris/iris_disk_cache.c +++ b/src/gallium/drivers/iris/iris_disk_cache.c @@ -39,6 +39,186 @@ #include "iris_context.h" +static bool debug = false; + +/** + * Compute a disk cache key for the given uncompiled shader and NOS key. + */ +static void +iris_disk_cache_compute_key(struct disk_cache *cache, + const struct iris_uncompiled_shader *ish, + const void *orig_prog_key, + uint32_t prog_key_size, + cache_key cache_key) +{ + gl_shader_stage stage = ish->nir->info.stage; + + /* Create a copy of the program key with program_string_id zeroed out. + * It's essentially random data which we don't want to include in our + * hashing and comparisons. We'll set a proper value on a cache hit. + */ + union brw_any_prog_key prog_key; + memcpy(&prog_key, orig_prog_key, prog_key_size); + brw_prog_key_set_id(&prog_key, stage, 0); + + uint32_t data_size = prog_key_size + ish->ir_cache_binary_size; + + void *data = malloc(data_size); + memcpy(data, &prog_key, prog_key_size); + memcpy(data + prog_key_size, ish->ir_cache_binary, + ish->ir_cache_binary_size); + + disk_cache_compute_key(cache, data, data_size, cache_key); + + free(data); +} + +/** + * Store the given compiled shader in the disk cache. + * + * This should only be called on newly compiled shaders. No checking is + * done to prevent repeated stores of the same shader. + */ +void +iris_disk_cache_store(struct disk_cache *cache, + const struct iris_uncompiled_shader *ish, + const struct iris_compiled_shader *shader, + const void *prog_key, + uint32_t prog_key_size) +{ +#ifdef ENABLE_SHADER_CACHE + if (!cache) + return; + + gl_shader_stage stage = ish->nir->info.stage; + const struct brw_stage_prog_data *prog_data = shader->prog_data; + + cache_key cache_key; + iris_disk_cache_compute_key(cache, ish, prog_key, prog_key_size, cache_key); + + if (debug) { + char sha1[41]; + _mesa_sha1_format(sha1, cache_key); + fprintf(stderr, "[mesa disk cache] storing %s\n", sha1); + } + + struct blob blob; + blob_init(&blob); + + /* We write the following data to the cache blob: + * + * 1. Prog data (must come first because it has the assembly size) + * 2. Assembly code + * 3. Number of entries in the system value array + * 4. System value array + * 5. Legacy param array (only used for compute workgroup ID) + */ + blob_write_bytes(&blob, shader->prog_data, brw_prog_data_size(stage)); + blob_write_bytes(&blob, shader->map, shader->prog_data->program_size); + blob_write_bytes(&blob, &shader->num_system_values, sizeof(unsigned)); + blob_write_bytes(&blob, shader->system_values, + shader->num_system_values * sizeof(enum brw_param_builtin)); + blob_write_bytes(&blob, prog_data->param, + prog_data->nr_params * sizeof(uint32_t)); + + disk_cache_put(cache, cache_key, blob.data, blob.size, NULL); + blob_finish(&blob); +#endif +} + +/** + * Search for a compiled shader in the disk cache. If found, upload it + * to the in-memory program cache so we can use it. + */ +struct iris_compiled_shader * +iris_disk_cache_retrieve(struct iris_context *ice, + const struct iris_uncompiled_shader *ish, + const void *prog_key, + uint32_t key_size) +{ +#ifdef ENABLE_SHADER_CACHE + struct iris_screen *screen = (void *) ice->ctx.screen; + struct disk_cache *cache = screen->disk_cache; + gl_shader_stage stage = ish->nir->info.stage; + + if (!cache) + return NULL; + + cache_key cache_key; + iris_disk_cache_compute_key(cache, ish, prog_key, key_size, cache_key); + + if (debug) { + char sha1[41]; + _mesa_sha1_format(sha1, cache_key); + fprintf(stderr, "[mesa disk cache] retrieving %s: ", sha1); + } + + size_t size; + void *buffer = disk_cache_get(screen->disk_cache, cache_key, &size); + + if (debug) + fprintf(stderr, "%s\n", buffer ? "found" : "missing"); + + if (!buffer) + return NULL; + + const uint32_t prog_data_size = brw_prog_data_size(stage); + + struct brw_stage_prog_data *prog_data = ralloc_size(NULL, prog_data_size); + const void *assembly; + uint32_t num_system_values; + uint32_t *system_values = NULL; + uint32_t *so_decls = NULL; + + struct blob_reader blob; + blob_reader_init(&blob, buffer, size); + blob_copy_bytes(&blob, prog_data, prog_data_size); + assembly = blob_read_bytes(&blob, prog_data->program_size); + num_system_values = blob_read_uint32(&blob); + if (num_system_values) { + system_values = + ralloc_array(NULL, enum brw_param_builtin, num_system_values); + blob_copy_bytes(&blob, system_values, + num_system_values * sizeof(enum brw_param_builtin)); + } + + prog_data->param = NULL; + prog_data->pull_param = NULL; + assert(prog_data->nr_pull_params == 0); + + if (prog_data->nr_params) { + prog_data->param = ralloc_array(NULL, uint32_t, prog_data->nr_params); + blob_copy_bytes(&blob, prog_data->param, + prog_data->nr_params * sizeof(uint32_t)); + } + + if (stage == MESA_SHADER_VERTEX || + stage == MESA_SHADER_TESS_EVAL || + stage == MESA_SHADER_GEOMETRY) { + struct brw_vue_prog_data *vue_prog_data = (void *) prog_data; + so_decls = ice->vtbl.create_so_decl_list(&ish->stream_output, + &vue_prog_data->vue_map); + } + + /* System values and uniforms are stored in constant buffer 0, the + * user-facing UBOs are indexed by one. So if any constant buffer is + * needed, the constant buffer 0 will be needed, so account for it. + */ + unsigned num_cbufs = ish->nir->info.num_ubos; + if (num_cbufs || num_system_values || ish->nir->num_uniforms) + num_cbufs++; + + /* Upload our newly read shader to the in-memory program cache and + * return it to the caller. + */ + return iris_upload_shader(ice, stage, key_size, prog_key, assembly, + prog_data, so_decls, system_values, + num_system_values, num_cbufs); +#else + return NULL; +#endif +} + /** * Initialize the on-disk shader cache. */ diff --git a/src/gallium/drivers/iris/iris_program.c b/src/gallium/drivers/iris/iris_program.c index 5c1af3c8a4b..21651bfd2c9 100644 --- a/src/gallium/drivers/iris/iris_program.c +++ b/src/gallium/drivers/iris/iris_program.c @@ -587,6 +587,8 @@ iris_compile_vs(struct iris_context *ice, prog_data, so_decls, system_values, num_system_values, num_cbufs); + iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key)); + ralloc_free(mem_ctx); return shader; } @@ -611,6 +613,9 @@ iris_update_compiled_vs(struct iris_context *ice) struct iris_compiled_shader *shader = iris_find_cached_shader(ice, IRIS_CACHE_VS, sizeof(key), &key); + if (!shader) + shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)); + if (!shader) shader = iris_compile_vs(ice, ish, &key); @@ -778,6 +783,9 @@ iris_compile_tcs(struct iris_context *ice, prog_data, NULL, system_values, num_system_values, num_cbufs); + if (ish) + iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key)); + ralloc_free(mem_ctx); return shader; } @@ -811,6 +819,9 @@ iris_update_compiled_tcs(struct iris_context *ice) struct iris_compiled_shader *shader = iris_find_cached_shader(ice, IRIS_CACHE_TCS, sizeof(key), &key); + if (tcs && !shader) + shader = iris_disk_cache_retrieve(ice, tcs, &key, sizeof(key)); + if (!shader) shader = iris_compile_tcs(ice, tcs, &key); @@ -880,6 +891,8 @@ iris_compile_tes(struct iris_context *ice, prog_data, so_decls, system_values, num_system_values, num_cbufs); + iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key)); + ralloc_free(mem_ctx); return shader; } @@ -905,6 +918,9 @@ iris_update_compiled_tes(struct iris_context *ice) struct iris_compiled_shader *shader = iris_find_cached_shader(ice, IRIS_CACHE_TES, sizeof(key), &key); + if (!shader) + shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)); + if (!shader) shader = iris_compile_tes(ice, ish, &key); @@ -980,6 +996,8 @@ iris_compile_gs(struct iris_context *ice, prog_data, so_decls, system_values, num_system_values, num_cbufs); + iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key)); + ralloc_free(mem_ctx); return shader; } @@ -1006,6 +1024,9 @@ iris_update_compiled_gs(struct iris_context *ice) shader = iris_find_cached_shader(ice, IRIS_CACHE_GS, sizeof(key), &key); + if (!shader) + shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)); + if (!shader) shader = iris_compile_gs(ice, ish, &key); } @@ -1070,6 +1091,8 @@ iris_compile_fs(struct iris_context *ice, prog_data, NULL, system_values, num_system_values, num_cbufs); + iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key)); + ralloc_free(mem_ctx); return shader; } @@ -1096,6 +1119,9 @@ iris_update_compiled_fs(struct iris_context *ice) struct iris_compiled_shader *shader = iris_find_cached_shader(ice, IRIS_CACHE_FS, sizeof(key), &key); + if (!shader) + shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)); + if (!shader) shader = iris_compile_fs(ice, ish, &key, ice->shaders.last_vue_map); @@ -1330,6 +1356,8 @@ iris_compile_cs(struct iris_context *ice, prog_data, NULL, system_values, num_system_values, num_cbufs); + iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key)); + ralloc_free(mem_ctx); return shader; } @@ -1349,6 +1377,9 @@ iris_update_compiled_compute_shader(struct iris_context *ice) struct iris_compiled_shader *shader = iris_find_cached_shader(ice, IRIS_CACHE_CS, sizeof(key), &key); + if (!shader) + shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)); + if (!shader) shader = iris_compile_cs(ice, ish, &key); @@ -1512,7 +1543,8 @@ iris_create_vs_state(struct pipe_context *ctx, const struct gen_device_info *devinfo = &screen->devinfo; struct brw_vs_prog_key key = { KEY_INIT(devinfo->gen) }; - iris_compile_vs(ice, ish, &key); + if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key))) + iris_compile_vs(ice, ish, &key); } return ish; @@ -1551,7 +1583,8 @@ iris_create_tcs_state(struct pipe_context *ctx, if (compiler->use_tcs_8_patch) key.input_vertices = info->tess.tcs_vertices_out; - iris_compile_tcs(ice, ish, &key); + if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key))) + iris_compile_tcs(ice, ish, &key); } return ish; @@ -1577,7 +1610,8 @@ iris_create_tes_state(struct pipe_context *ctx, .patch_inputs_read = info->patch_inputs_read, }; - iris_compile_tes(ice, ish, &key); + if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key))) + iris_compile_tes(ice, ish, &key); } return ish; @@ -1597,7 +1631,8 @@ iris_create_gs_state(struct pipe_context *ctx, const struct gen_device_info *devinfo = &screen->devinfo; struct brw_gs_prog_key key = { KEY_INIT(devinfo->gen) }; - iris_compile_gs(ice, ish, &key); + if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key))) + iris_compile_gs(ice, ish, &key); } return ish; @@ -1641,7 +1676,8 @@ iris_create_fs_state(struct pipe_context *ctx, can_rearrange_varyings ? 0 : info->inputs_read | VARYING_BIT_POS, }; - iris_compile_fs(ice, ish, &key, NULL); + if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key))) + iris_compile_fs(ice, ish, &key, NULL); } return ish; @@ -1664,7 +1700,8 @@ iris_create_compute_state(struct pipe_context *ctx, const struct gen_device_info *devinfo = &screen->devinfo; struct brw_cs_prog_key key = { KEY_INIT(devinfo->gen) }; - iris_compile_cs(ice, ish, &key); + if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key))) + iris_compile_cs(ice, ish, &key); } return ish;