817 lines
29 KiB
C
817 lines
29 KiB
C
/*
|
|
* Copyright 2014-2019 Advanced Micro Devices, Inc.
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including the next
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
* Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*/
|
|
|
|
#include "ac_rtld.h"
|
|
|
|
#include "ac_binary.h"
|
|
#include "ac_gpu_info.h"
|
|
#include "util/compiler.h"
|
|
#include "util/u_dynarray.h"
|
|
#include "util/u_math.h"
|
|
|
|
#include <gelf.h>
|
|
#include <libelf.h>
|
|
#include <stdarg.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#ifndef EM_AMDGPU
|
|
// Old distributions may not have this enum constant
|
|
#define EM_AMDGPU 224
|
|
#endif
|
|
|
|
#ifndef STT_AMDGPU_LDS
|
|
#define STT_AMDGPU_LDS 13 // this is deprecated -- remove
|
|
#endif
|
|
|
|
#ifndef SHN_AMDGPU_LDS
|
|
#define SHN_AMDGPU_LDS 0xff00
|
|
#endif
|
|
|
|
#ifndef R_AMDGPU_NONE
|
|
#define R_AMDGPU_NONE 0
|
|
#define R_AMDGPU_ABS32_LO 1
|
|
#define R_AMDGPU_ABS32_HI 2
|
|
#define R_AMDGPU_ABS64 3
|
|
#define R_AMDGPU_REL32 4
|
|
#define R_AMDGPU_REL64 5
|
|
#define R_AMDGPU_ABS32 6
|
|
#define R_AMDGPU_GOTPCREL 7
|
|
#define R_AMDGPU_GOTPCREL32_LO 8
|
|
#define R_AMDGPU_GOTPCREL32_HI 9
|
|
#define R_AMDGPU_REL32_LO 10
|
|
#define R_AMDGPU_REL32_HI 11
|
|
#define R_AMDGPU_RELATIVE64 13
|
|
#endif
|
|
|
|
/* For the UMR disassembler. */
|
|
#define DEBUGGER_END_OF_CODE_MARKER 0xbf9f0000 /* invalid instruction */
|
|
#define DEBUGGER_NUM_MARKERS 5
|
|
|
|
struct ac_rtld_section {
|
|
bool is_rx : 1;
|
|
bool is_pasted_text : 1;
|
|
uint64_t offset;
|
|
const char *name;
|
|
};
|
|
|
|
struct ac_rtld_part {
|
|
Elf *elf;
|
|
struct ac_rtld_section *sections;
|
|
unsigned num_sections;
|
|
};
|
|
|
|
static void report_errorvf(const char *fmt, va_list va)
|
|
{
|
|
fprintf(stderr, "ac_rtld error: ");
|
|
|
|
vfprintf(stderr, fmt, va);
|
|
|
|
fprintf(stderr, "\n");
|
|
}
|
|
|
|
static void report_errorf(const char *fmt, ...) PRINTFLIKE(1, 2);
|
|
|
|
static void report_errorf(const char *fmt, ...)
|
|
{
|
|
va_list va;
|
|
va_start(va, fmt);
|
|
report_errorvf(fmt, va);
|
|
va_end(va);
|
|
}
|
|
|
|
static void report_elf_errorf(const char *fmt, ...) PRINTFLIKE(1, 2);
|
|
|
|
static void report_elf_errorf(const char *fmt, ...)
|
|
{
|
|
va_list va;
|
|
va_start(va, fmt);
|
|
report_errorvf(fmt, va);
|
|
va_end(va);
|
|
|
|
fprintf(stderr, "ELF error: %s\n", elf_errmsg(elf_errno()));
|
|
}
|
|
|
|
/**
|
|
* Find a symbol in a dynarray of struct ac_rtld_symbol by \p name and shader
|
|
* \p part_idx.
|
|
*/
|
|
static const struct ac_rtld_symbol *find_symbol(const struct util_dynarray *symbols,
|
|
const char *name, unsigned part_idx)
|
|
{
|
|
util_dynarray_foreach (symbols, struct ac_rtld_symbol, symbol) {
|
|
if ((symbol->part_idx == ~0u || symbol->part_idx == part_idx) && !strcmp(name, symbol->name))
|
|
return symbol;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
static int compare_symbol_by_align(const void *lhsp, const void *rhsp)
|
|
{
|
|
const struct ac_rtld_symbol *lhs = lhsp;
|
|
const struct ac_rtld_symbol *rhs = rhsp;
|
|
if (rhs->align > lhs->align)
|
|
return 1;
|
|
if (rhs->align < lhs->align)
|
|
return -1;
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* Sort the given symbol list by decreasing alignment and assign offsets.
|
|
*/
|
|
static bool layout_symbols(struct ac_rtld_symbol *symbols, unsigned num_symbols,
|
|
uint64_t *ptotal_size)
|
|
{
|
|
qsort(symbols, num_symbols, sizeof(*symbols), compare_symbol_by_align);
|
|
|
|
uint64_t total_size = *ptotal_size;
|
|
|
|
for (unsigned i = 0; i < num_symbols; ++i) {
|
|
struct ac_rtld_symbol *s = &symbols[i];
|
|
assert(util_is_power_of_two_nonzero(s->align));
|
|
|
|
total_size = align64(total_size, s->align);
|
|
s->offset = total_size;
|
|
|
|
if (total_size + s->size < total_size) {
|
|
report_errorf("%s: size overflow", __FUNCTION__);
|
|
return false;
|
|
}
|
|
|
|
total_size += s->size;
|
|
}
|
|
|
|
*ptotal_size = total_size;
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Read LDS symbols from the given \p section of the ELF of \p part and append
|
|
* them to the LDS symbols list.
|
|
*
|
|
* Shared LDS symbols are filtered out.
|
|
*/
|
|
static bool read_private_lds_symbols(struct ac_rtld_binary *binary, unsigned part_idx,
|
|
Elf_Scn *section, uint32_t *lds_end_align)
|
|
{
|
|
#define report_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_errorf(#cond); \
|
|
return false; \
|
|
} \
|
|
} while (false)
|
|
#define report_elf_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_elf_errorf(#cond); \
|
|
return false; \
|
|
} \
|
|
} while (false)
|
|
|
|
struct ac_rtld_part *part = &binary->parts[part_idx];
|
|
Elf64_Shdr *shdr = elf64_getshdr(section);
|
|
uint32_t strtabidx = shdr->sh_link;
|
|
Elf_Data *symbols_data = elf_getdata(section, NULL);
|
|
report_elf_if(!symbols_data);
|
|
|
|
const Elf64_Sym *symbol = symbols_data->d_buf;
|
|
size_t num_symbols = symbols_data->d_size / sizeof(Elf64_Sym);
|
|
|
|
for (size_t j = 0; j < num_symbols; ++j, ++symbol) {
|
|
struct ac_rtld_symbol s = {0};
|
|
|
|
if (ELF64_ST_TYPE(symbol->st_info) == STT_AMDGPU_LDS) {
|
|
/* old-style LDS symbols from initial prototype -- remove eventually */
|
|
s.align = MIN2(1u << (symbol->st_other >> 3), 1u << 16);
|
|
} else if (symbol->st_shndx == SHN_AMDGPU_LDS) {
|
|
s.align = MIN2(symbol->st_value, 1u << 16);
|
|
report_if(!util_is_power_of_two_nonzero(s.align));
|
|
} else
|
|
continue;
|
|
|
|
report_if(symbol->st_size > 1u << 29);
|
|
|
|
s.name = elf_strptr(part->elf, strtabidx, symbol->st_name);
|
|
s.size = symbol->st_size;
|
|
s.part_idx = part_idx;
|
|
|
|
if (!strcmp(s.name, "__lds_end")) {
|
|
report_elf_if(s.size != 0);
|
|
*lds_end_align = MAX2(*lds_end_align, s.align);
|
|
continue;
|
|
}
|
|
|
|
const struct ac_rtld_symbol *shared = find_symbol(&binary->lds_symbols, s.name, part_idx);
|
|
if (shared) {
|
|
report_elf_if(s.align > shared->align);
|
|
report_elf_if(s.size > shared->size);
|
|
continue;
|
|
}
|
|
|
|
util_dynarray_append(&binary->lds_symbols, struct ac_rtld_symbol, s);
|
|
}
|
|
|
|
return true;
|
|
|
|
#undef report_if
|
|
#undef report_elf_if
|
|
}
|
|
|
|
/**
|
|
* Open a binary consisting of one or more shader parts.
|
|
*
|
|
* \param binary the uninitialized struct
|
|
* \param i binary opening parameters
|
|
*/
|
|
bool ac_rtld_open(struct ac_rtld_binary *binary, struct ac_rtld_open_info i)
|
|
{
|
|
/* One of the libelf implementations
|
|
* (http://www.mr511.de/software/english.htm) requires calling
|
|
* elf_version() before elf_memory().
|
|
*/
|
|
elf_version(EV_CURRENT);
|
|
|
|
memset(binary, 0, sizeof(*binary));
|
|
memcpy(&binary->options, &i.options, sizeof(binary->options));
|
|
binary->wave_size = i.wave_size;
|
|
binary->gfx_level = i.info->gfx_level;
|
|
binary->num_parts = i.num_parts;
|
|
binary->parts = calloc(sizeof(*binary->parts), i.num_parts);
|
|
if (!binary->parts)
|
|
return false;
|
|
|
|
uint64_t pasted_text_size = 0;
|
|
uint64_t rx_align = 1;
|
|
uint64_t rx_size = 0;
|
|
uint64_t exec_size = 0;
|
|
|
|
#define report_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_errorf(#cond); \
|
|
goto fail; \
|
|
} \
|
|
} while (false)
|
|
#define report_elf_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_elf_errorf(#cond); \
|
|
goto fail; \
|
|
} \
|
|
} while (false)
|
|
|
|
/* Copy and layout shared LDS symbols. */
|
|
if (i.num_shared_lds_symbols) {
|
|
if (!util_dynarray_resize(&binary->lds_symbols, struct ac_rtld_symbol,
|
|
i.num_shared_lds_symbols))
|
|
goto fail;
|
|
|
|
memcpy(binary->lds_symbols.data, i.shared_lds_symbols, binary->lds_symbols.size);
|
|
}
|
|
|
|
util_dynarray_foreach (&binary->lds_symbols, struct ac_rtld_symbol, symbol)
|
|
symbol->part_idx = ~0u;
|
|
|
|
unsigned max_lds_size = 64 * 1024;
|
|
|
|
if (i.info->gfx_level == GFX6 ||
|
|
(i.shader_type != MESA_SHADER_COMPUTE && i.shader_type != MESA_SHADER_FRAGMENT))
|
|
max_lds_size = 32 * 1024;
|
|
|
|
uint64_t shared_lds_size = 0;
|
|
if (!layout_symbols(binary->lds_symbols.data, i.num_shared_lds_symbols, &shared_lds_size))
|
|
goto fail;
|
|
|
|
if (shared_lds_size > max_lds_size) {
|
|
fprintf(stderr, "ac_rtld error(1): too much LDS (used = %u, max = %u)\n",
|
|
(unsigned)shared_lds_size, max_lds_size);
|
|
goto fail;
|
|
}
|
|
binary->lds_size = shared_lds_size;
|
|
|
|
/* First pass over all parts: open ELFs, pre-determine the placement of
|
|
* sections in the memory image, and collect and layout private LDS symbols. */
|
|
uint32_t lds_end_align = 0;
|
|
|
|
if (binary->options.halt_at_entry)
|
|
pasted_text_size += 4;
|
|
|
|
for (unsigned part_idx = 0; part_idx < i.num_parts; ++part_idx) {
|
|
struct ac_rtld_part *part = &binary->parts[part_idx];
|
|
unsigned part_lds_symbols_begin =
|
|
util_dynarray_num_elements(&binary->lds_symbols, struct ac_rtld_symbol);
|
|
|
|
part->elf = elf_memory((char *)i.elf_ptrs[part_idx], i.elf_sizes[part_idx]);
|
|
report_elf_if(!part->elf);
|
|
|
|
const Elf64_Ehdr *ehdr = elf64_getehdr(part->elf);
|
|
report_elf_if(!ehdr);
|
|
report_if(ehdr->e_machine != EM_AMDGPU);
|
|
|
|
size_t section_str_index;
|
|
size_t num_shdrs;
|
|
report_elf_if(elf_getshdrstrndx(part->elf, §ion_str_index) < 0);
|
|
report_elf_if(elf_getshdrnum(part->elf, &num_shdrs) < 0);
|
|
|
|
part->num_sections = num_shdrs;
|
|
part->sections = calloc(sizeof(*part->sections), num_shdrs);
|
|
report_if(!part->sections);
|
|
|
|
Elf_Scn *section = NULL;
|
|
while ((section = elf_nextscn(part->elf, section))) {
|
|
Elf64_Shdr *shdr = elf64_getshdr(section);
|
|
struct ac_rtld_section *s = &part->sections[elf_ndxscn(section)];
|
|
s->name = elf_strptr(part->elf, section_str_index, shdr->sh_name);
|
|
report_elf_if(!s->name);
|
|
|
|
/* Cannot actually handle linked objects yet */
|
|
report_elf_if(shdr->sh_addr != 0);
|
|
|
|
/* Alignment must be 0 or a power of two */
|
|
report_elf_if(shdr->sh_addralign & (shdr->sh_addralign - 1));
|
|
uint64_t sh_align = MAX2(shdr->sh_addralign, 1);
|
|
|
|
if (shdr->sh_flags & SHF_ALLOC && shdr->sh_type != SHT_NOTE) {
|
|
report_if(shdr->sh_flags & SHF_WRITE);
|
|
|
|
s->is_rx = true;
|
|
|
|
if (shdr->sh_flags & SHF_EXECINSTR) {
|
|
report_elf_if(shdr->sh_size & 3);
|
|
|
|
if (!strcmp(s->name, ".text"))
|
|
s->is_pasted_text = true;
|
|
|
|
exec_size += shdr->sh_size;
|
|
}
|
|
|
|
if (s->is_pasted_text) {
|
|
s->offset = pasted_text_size;
|
|
pasted_text_size += shdr->sh_size;
|
|
} else {
|
|
rx_align = align(rx_align, sh_align);
|
|
rx_size = align(rx_size, sh_align);
|
|
s->offset = rx_size;
|
|
rx_size += shdr->sh_size;
|
|
}
|
|
} else if (shdr->sh_type == SHT_SYMTAB) {
|
|
if (!read_private_lds_symbols(binary, part_idx, section, &lds_end_align))
|
|
goto fail;
|
|
}
|
|
}
|
|
|
|
uint64_t part_lds_size = shared_lds_size;
|
|
if (!layout_symbols(util_dynarray_element(&binary->lds_symbols, struct ac_rtld_symbol,
|
|
part_lds_symbols_begin),
|
|
util_dynarray_num_elements(&binary->lds_symbols, struct ac_rtld_symbol) -
|
|
part_lds_symbols_begin,
|
|
&part_lds_size))
|
|
goto fail;
|
|
binary->lds_size = MAX2(binary->lds_size, part_lds_size);
|
|
}
|
|
|
|
binary->rx_end_markers = pasted_text_size;
|
|
pasted_text_size += 4 * DEBUGGER_NUM_MARKERS;
|
|
|
|
/* __lds_end is a special symbol that points at the end of the memory
|
|
* occupied by other LDS symbols. Its alignment is taken as the
|
|
* maximum of its alignment over all shader parts where it occurs.
|
|
*/
|
|
if (lds_end_align) {
|
|
binary->lds_size = align(binary->lds_size, lds_end_align);
|
|
|
|
struct ac_rtld_symbol *lds_end =
|
|
util_dynarray_grow(&binary->lds_symbols, struct ac_rtld_symbol, 1);
|
|
lds_end->name = "__lds_end";
|
|
lds_end->size = 0;
|
|
lds_end->align = lds_end_align;
|
|
lds_end->offset = binary->lds_size;
|
|
lds_end->part_idx = ~0u;
|
|
}
|
|
|
|
if (binary->lds_size > max_lds_size) {
|
|
fprintf(stderr, "ac_rtld error(2): too much LDS (used = %u, max = %u)\n",
|
|
(unsigned)binary->lds_size, max_lds_size);
|
|
goto fail;
|
|
}
|
|
|
|
/* Second pass: Adjust offsets of non-pasted text sections. */
|
|
binary->rx_size = pasted_text_size;
|
|
binary->rx_size = align(binary->rx_size, rx_align);
|
|
|
|
for (unsigned part_idx = 0; part_idx < i.num_parts; ++part_idx) {
|
|
struct ac_rtld_part *part = &binary->parts[part_idx];
|
|
size_t num_shdrs;
|
|
elf_getshdrnum(part->elf, &num_shdrs);
|
|
|
|
for (unsigned j = 0; j < num_shdrs; ++j) {
|
|
struct ac_rtld_section *s = &part->sections[j];
|
|
if (s->is_rx && !s->is_pasted_text)
|
|
s->offset += binary->rx_size;
|
|
}
|
|
}
|
|
|
|
binary->rx_size += rx_size;
|
|
binary->exec_size = exec_size;
|
|
|
|
/* The SQ fetches up to N cache lines of 16 dwords
|
|
* ahead of the PC, configurable by SH_MEM_CONFIG and
|
|
* S_INST_PREFETCH. This can cause two issues:
|
|
*
|
|
* (1) Crossing a page boundary to an unmapped page. The logic
|
|
* does not distinguish between a required fetch and a "mere"
|
|
* prefetch and will fault.
|
|
*
|
|
* (2) Prefetching instructions that will be changed for a
|
|
* different shader.
|
|
*
|
|
* (2) is not currently an issue because we flush the I$ at IB
|
|
* boundaries, but (1) needs to be addressed. Due to buffer
|
|
* suballocation, we just play it safe.
|
|
*/
|
|
unsigned prefetch_distance = 0;
|
|
|
|
if (!i.info->has_graphics && i.info->family >= CHIP_ALDEBARAN)
|
|
prefetch_distance = 16;
|
|
else if (i.info->gfx_level >= GFX10)
|
|
prefetch_distance = 3;
|
|
|
|
if (prefetch_distance) {
|
|
if (i.info->gfx_level >= GFX11)
|
|
binary->rx_size = align(binary->rx_size + prefetch_distance * 64, 128);
|
|
else
|
|
binary->rx_size = align(binary->rx_size + prefetch_distance * 64, 64);
|
|
}
|
|
|
|
return true;
|
|
|
|
#undef report_if
|
|
#undef report_elf_if
|
|
|
|
fail:
|
|
ac_rtld_close(binary);
|
|
return false;
|
|
}
|
|
|
|
void ac_rtld_close(struct ac_rtld_binary *binary)
|
|
{
|
|
for (unsigned i = 0; i < binary->num_parts; ++i) {
|
|
struct ac_rtld_part *part = &binary->parts[i];
|
|
free(part->sections);
|
|
elf_end(part->elf);
|
|
}
|
|
|
|
util_dynarray_fini(&binary->lds_symbols);
|
|
free(binary->parts);
|
|
binary->parts = NULL;
|
|
binary->num_parts = 0;
|
|
}
|
|
|
|
static bool get_section_by_name(struct ac_rtld_part *part, const char *name, const char **data,
|
|
size_t *nbytes)
|
|
{
|
|
for (unsigned i = 0; i < part->num_sections; ++i) {
|
|
struct ac_rtld_section *s = &part->sections[i];
|
|
if (s->name && !strcmp(name, s->name)) {
|
|
Elf_Scn *target_scn = elf_getscn(part->elf, i);
|
|
Elf_Data *target_data = elf_getdata(target_scn, NULL);
|
|
if (!target_data) {
|
|
report_elf_errorf("ac_rtld: get_section_by_name: elf_getdata");
|
|
return false;
|
|
}
|
|
|
|
*data = target_data->d_buf;
|
|
*nbytes = target_data->d_size;
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool ac_rtld_get_section_by_name(struct ac_rtld_binary *binary, const char *name, const char **data,
|
|
size_t *nbytes)
|
|
{
|
|
assert(binary->num_parts == 1);
|
|
return get_section_by_name(&binary->parts[0], name, data, nbytes);
|
|
}
|
|
|
|
bool ac_rtld_read_config(const struct radeon_info *info, struct ac_rtld_binary *binary,
|
|
struct ac_shader_config *config)
|
|
{
|
|
for (unsigned i = 0; i < binary->num_parts; ++i) {
|
|
struct ac_rtld_part *part = &binary->parts[i];
|
|
const char *config_data;
|
|
size_t config_nbytes;
|
|
|
|
if (!get_section_by_name(part, ".AMDGPU.config", &config_data, &config_nbytes))
|
|
return false;
|
|
|
|
/* TODO: be precise about scratch use? */
|
|
struct ac_shader_config c = {0};
|
|
ac_parse_shader_binary_config(config_data, config_nbytes, binary->wave_size, info, &c);
|
|
|
|
config->num_sgprs = MAX2(config->num_sgprs, c.num_sgprs);
|
|
config->num_vgprs = MAX2(config->num_vgprs, c.num_vgprs);
|
|
config->spilled_sgprs = MAX2(config->spilled_sgprs, c.spilled_sgprs);
|
|
config->spilled_vgprs = MAX2(config->spilled_vgprs, c.spilled_vgprs);
|
|
config->scratch_bytes_per_wave =
|
|
MAX2(config->scratch_bytes_per_wave, c.scratch_bytes_per_wave);
|
|
|
|
assert(i == 0 || config->float_mode == c.float_mode);
|
|
config->float_mode = c.float_mode;
|
|
|
|
/* SPI_PS_INPUT_ENA/ADDR can't be combined. Only the value from
|
|
* the main shader part is used. */
|
|
assert(config->spi_ps_input_ena == 0 && config->spi_ps_input_addr == 0);
|
|
config->spi_ps_input_ena = c.spi_ps_input_ena;
|
|
config->spi_ps_input_addr = c.spi_ps_input_addr;
|
|
|
|
/* TODO: consistently use LDS symbols for this */
|
|
config->lds_size = MAX2(config->lds_size, c.lds_size);
|
|
|
|
/* TODO: Should we combine these somehow? It's currently only
|
|
* used for radeonsi's compute, where multiple parts aren't used. */
|
|
assert(config->rsrc1 == 0 && config->rsrc2 == 0);
|
|
config->rsrc1 = c.rsrc1;
|
|
config->rsrc2 = c.rsrc2;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
static bool resolve_symbol(const struct ac_rtld_upload_info *u, unsigned part_idx,
|
|
const Elf64_Sym *sym, const char *name, uint64_t *value)
|
|
{
|
|
/* TODO: properly disentangle the undef and the LDS cases once
|
|
* STT_AMDGPU_LDS is retired. */
|
|
if (sym->st_shndx == SHN_UNDEF || sym->st_shndx == SHN_AMDGPU_LDS) {
|
|
const struct ac_rtld_symbol *lds_sym = find_symbol(&u->binary->lds_symbols, name, part_idx);
|
|
|
|
if (lds_sym) {
|
|
*value = lds_sym->offset;
|
|
return true;
|
|
}
|
|
|
|
/* TODO: resolve from other parts */
|
|
|
|
if (u->get_external_symbol(u->binary->gfx_level, u->cb_data, name, value))
|
|
return true;
|
|
|
|
report_errorf("symbol %s: unknown", name);
|
|
return false;
|
|
}
|
|
|
|
struct ac_rtld_part *part = &u->binary->parts[part_idx];
|
|
if (sym->st_shndx >= part->num_sections) {
|
|
report_errorf("symbol %s: section out of bounds", name);
|
|
return false;
|
|
}
|
|
|
|
struct ac_rtld_section *s = &part->sections[sym->st_shndx];
|
|
if (!s->is_rx) {
|
|
report_errorf("symbol %s: bad section", name);
|
|
return false;
|
|
}
|
|
|
|
uint64_t section_base = u->rx_va + s->offset;
|
|
|
|
*value = section_base + sym->st_value;
|
|
return true;
|
|
}
|
|
|
|
static bool apply_relocs(const struct ac_rtld_upload_info *u, unsigned part_idx,
|
|
const Elf64_Shdr *reloc_shdr, const Elf_Data *reloc_data)
|
|
{
|
|
#define report_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_errorf(#cond); \
|
|
return false; \
|
|
} \
|
|
} while (false)
|
|
#define report_elf_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_elf_errorf(#cond); \
|
|
return false; \
|
|
} \
|
|
} while (false)
|
|
|
|
struct ac_rtld_part *part = &u->binary->parts[part_idx];
|
|
Elf_Scn *target_scn = elf_getscn(part->elf, reloc_shdr->sh_info);
|
|
report_elf_if(!target_scn);
|
|
|
|
Elf_Data *target_data = elf_getdata(target_scn, NULL);
|
|
report_elf_if(!target_data);
|
|
|
|
Elf_Scn *symbols_scn = elf_getscn(part->elf, reloc_shdr->sh_link);
|
|
report_elf_if(!symbols_scn);
|
|
|
|
Elf64_Shdr *symbols_shdr = elf64_getshdr(symbols_scn);
|
|
report_elf_if(!symbols_shdr);
|
|
uint32_t strtabidx = symbols_shdr->sh_link;
|
|
|
|
Elf_Data *symbols_data = elf_getdata(symbols_scn, NULL);
|
|
report_elf_if(!symbols_data);
|
|
|
|
const Elf64_Sym *symbols = symbols_data->d_buf;
|
|
size_t num_symbols = symbols_data->d_size / sizeof(Elf64_Sym);
|
|
|
|
struct ac_rtld_section *s = &part->sections[reloc_shdr->sh_info];
|
|
report_if(!s->is_rx);
|
|
|
|
const char *orig_base = target_data->d_buf;
|
|
char *dst_base = u->rx_ptr + s->offset;
|
|
uint64_t va_base = u->rx_va + s->offset;
|
|
|
|
Elf64_Rel *rel = reloc_data->d_buf;
|
|
size_t num_relocs = reloc_data->d_size / sizeof(*rel);
|
|
for (size_t i = 0; i < num_relocs; ++i, ++rel) {
|
|
size_t r_sym = ELF64_R_SYM(rel->r_info);
|
|
unsigned r_type = ELF64_R_TYPE(rel->r_info);
|
|
|
|
const char *orig_ptr = orig_base + rel->r_offset;
|
|
char *dst_ptr = dst_base + rel->r_offset;
|
|
uint64_t va = va_base + rel->r_offset;
|
|
|
|
uint64_t symbol;
|
|
uint64_t addend;
|
|
|
|
if (r_sym == STN_UNDEF) {
|
|
symbol = 0;
|
|
} else {
|
|
report_elf_if(r_sym >= num_symbols);
|
|
|
|
const Elf64_Sym *sym = &symbols[r_sym];
|
|
const char *symbol_name = elf_strptr(part->elf, strtabidx, sym->st_name);
|
|
report_elf_if(!symbol_name);
|
|
|
|
if (!resolve_symbol(u, part_idx, sym, symbol_name, &symbol))
|
|
return false;
|
|
}
|
|
|
|
/* TODO: Should we also support .rela sections, where the
|
|
* addend is part of the relocation record? */
|
|
|
|
/* Load the addend from the ELF instead of the destination,
|
|
* because the destination may be in VRAM. */
|
|
switch (r_type) {
|
|
case R_AMDGPU_ABS32:
|
|
case R_AMDGPU_ABS32_LO:
|
|
case R_AMDGPU_ABS32_HI:
|
|
case R_AMDGPU_REL32:
|
|
case R_AMDGPU_REL32_LO:
|
|
case R_AMDGPU_REL32_HI:
|
|
addend = *(const uint32_t *)orig_ptr;
|
|
break;
|
|
case R_AMDGPU_ABS64:
|
|
case R_AMDGPU_REL64:
|
|
addend = *(const uint64_t *)orig_ptr;
|
|
break;
|
|
default:
|
|
report_errorf("unsupported r_type == %u", r_type);
|
|
return false;
|
|
}
|
|
|
|
uint64_t abs = symbol + addend;
|
|
|
|
switch (r_type) {
|
|
case R_AMDGPU_ABS32:
|
|
assert((uint32_t)abs == abs);
|
|
FALLTHROUGH;
|
|
case R_AMDGPU_ABS32_LO:
|
|
*(uint32_t *)dst_ptr = util_cpu_to_le32(abs);
|
|
break;
|
|
case R_AMDGPU_ABS32_HI:
|
|
*(uint32_t *)dst_ptr = util_cpu_to_le32(abs >> 32);
|
|
break;
|
|
case R_AMDGPU_ABS64:
|
|
*(uint64_t *)dst_ptr = util_cpu_to_le64(abs);
|
|
break;
|
|
case R_AMDGPU_REL32:
|
|
assert((int64_t)(int32_t)(abs - va) == (int64_t)(abs - va));
|
|
FALLTHROUGH;
|
|
case R_AMDGPU_REL32_LO:
|
|
*(uint32_t *)dst_ptr = util_cpu_to_le32(abs - va);
|
|
break;
|
|
case R_AMDGPU_REL32_HI:
|
|
*(uint32_t *)dst_ptr = util_cpu_to_le32((abs - va) >> 32);
|
|
break;
|
|
case R_AMDGPU_REL64:
|
|
*(uint64_t *)dst_ptr = util_cpu_to_le64(abs - va);
|
|
break;
|
|
default:
|
|
unreachable("bad r_type");
|
|
}
|
|
}
|
|
|
|
return true;
|
|
|
|
#undef report_if
|
|
#undef report_elf_if
|
|
}
|
|
|
|
/**
|
|
* Upload the binary or binaries to the provided GPU buffers, including
|
|
* relocations.
|
|
*/
|
|
int ac_rtld_upload(struct ac_rtld_upload_info *u)
|
|
{
|
|
#define report_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_errorf(#cond); \
|
|
return -1; \
|
|
} \
|
|
} while (false)
|
|
#define report_elf_if(cond) \
|
|
do { \
|
|
if ((cond)) { \
|
|
report_errorf(#cond); \
|
|
return -1; \
|
|
} \
|
|
} while (false)
|
|
|
|
int size = 0;
|
|
if (u->binary->options.halt_at_entry) {
|
|
/* s_sethalt 1 */
|
|
*(uint32_t *)u->rx_ptr = util_cpu_to_le32(0xbf8d0001);
|
|
}
|
|
|
|
/* First pass: upload raw section data and lay out private LDS symbols. */
|
|
for (unsigned i = 0; i < u->binary->num_parts; ++i) {
|
|
struct ac_rtld_part *part = &u->binary->parts[i];
|
|
|
|
Elf_Scn *section = NULL;
|
|
while ((section = elf_nextscn(part->elf, section))) {
|
|
Elf64_Shdr *shdr = elf64_getshdr(section);
|
|
struct ac_rtld_section *s = &part->sections[elf_ndxscn(section)];
|
|
|
|
if (!s->is_rx)
|
|
continue;
|
|
|
|
report_if(shdr->sh_type != SHT_PROGBITS);
|
|
|
|
Elf_Data *data = elf_getdata(section, NULL);
|
|
report_elf_if(!data || data->d_size != shdr->sh_size);
|
|
memcpy(u->rx_ptr + s->offset, data->d_buf, shdr->sh_size);
|
|
|
|
size = MAX2(size, s->offset + shdr->sh_size);
|
|
}
|
|
}
|
|
|
|
if (u->binary->rx_end_markers) {
|
|
uint32_t *dst = (uint32_t *)(u->rx_ptr + u->binary->rx_end_markers);
|
|
for (unsigned i = 0; i < DEBUGGER_NUM_MARKERS; ++i)
|
|
*dst++ = util_cpu_to_le32(DEBUGGER_END_OF_CODE_MARKER);
|
|
size += 4 * DEBUGGER_NUM_MARKERS;
|
|
}
|
|
|
|
/* Second pass: handle relocations, overwriting uploaded data where
|
|
* appropriate. */
|
|
for (unsigned i = 0; i < u->binary->num_parts; ++i) {
|
|
struct ac_rtld_part *part = &u->binary->parts[i];
|
|
Elf_Scn *section = NULL;
|
|
while ((section = elf_nextscn(part->elf, section))) {
|
|
Elf64_Shdr *shdr = elf64_getshdr(section);
|
|
if (shdr->sh_type == SHT_REL) {
|
|
Elf_Data *relocs = elf_getdata(section, NULL);
|
|
report_elf_if(!relocs || relocs->d_size != shdr->sh_size);
|
|
if (!apply_relocs(u, i, shdr, relocs))
|
|
return -1;
|
|
} else if (shdr->sh_type == SHT_RELA) {
|
|
report_errorf("SHT_RELA not supported");
|
|
return -1;
|
|
}
|
|
}
|
|
}
|
|
|
|
return size;
|
|
|
|
#undef report_if
|
|
#undef report_elf_if
|
|
}
|