diff --git a/RecompModTool/main.cpp b/RecompModTool/main.cpp index 2265bb3..74c2a30 100644 --- a/RecompModTool/main.cpp +++ b/RecompModTool/main.cpp @@ -12,6 +12,7 @@ struct ModConfig { std::filesystem::path elf_path; std::filesystem::path func_reference_syms_file_path; std::vector data_reference_syms_file_paths; + std::vector dependency_paths; }; static std::filesystem::path concat_if_not_empty(const std::filesystem::path& parent, const std::filesystem::path& child) { @@ -21,14 +22,14 @@ static std::filesystem::path concat_if_not_empty(const std::filesystem::path& pa return child; } -static std::vector get_data_syms_paths(const toml::array* data_syms_paths_array, const std::filesystem::path& basedir) { +static std::vector get_toml_path_array(const toml::array* toml_array, const std::filesystem::path& basedir) { std::vector ret; // Reserve room for all the funcs in the map. - ret.reserve(data_syms_paths_array->size()); - data_syms_paths_array->for_each([&ret, &basedir](auto&& el) { + ret.reserve(toml_array->size()); + toml_array->for_each([&ret, &basedir](auto&& el) { if constexpr (toml::is_string) { - ret.emplace_back(concat_if_not_empty(basedir, el.template value_exact().value())); + ret.emplace_back(concat_if_not_empty(basedir, el.ref())); } else { throw toml::parse_error("Invalid type for data reference symbol file entry", el.source()); @@ -38,6 +39,80 @@ static std::vector get_data_syms_paths(const toml::array* return ret; } +static bool read_dependency_file(const std::filesystem::path& dependency_path, N64Recomp::Context& context, std::vector& import_symbol_mod_ids) { + toml::table toml_data{}; + + try { + toml_data = toml::parse_file(dependency_path.native()); + + const auto dependency_data = toml_data["dependency"]; + if (!dependency_data.is_array()) { + if (dependency_data) { + throw toml::parse_error("No dependency array found", dependency_data.node()->source()); + } + else { + throw toml::parse_error("Invalid dependency array", dependency_data.node()->source()); + } + } + + toml::array* dependency_array = dependency_data.as_array(); + for (const auto& dependency_node : *dependency_array) { + if (!dependency_node.is_table()) { + throw toml::parse_error("Invalid dependency entry", dependency_node.source()); + } + + // Mod ID + toml::node_view mod_id_node = dependency_node[toml::path{"mod_id"}]; + if (!mod_id_node.is_string()) { + if (mod_id_node) { + throw toml::parse_error("Invalid mod id", mod_id_node.node()->source()); + } + else { + throw toml::parse_error("Dependency entry is missing mod id", dependency_node.source()); + } + } + const std::string& mod_id = mod_id_node.ref(); + + // Symbol list + toml::node_view functions_data = dependency_node[toml::path{"functions"}]; + if (functions_data.is_array()) { + const toml::array* functions_array = functions_data.as_array(); + for (const auto& function_node : *functions_array) { + if (!function_node.is_string()) { + throw toml::parse_error("Invalid dependency function", function_node.source()); + } + const std::string& function_name = function_node.ref(); + context.reference_symbol_names.emplace_back(function_name); + context.reference_symbols_by_name[function_name] = context.reference_symbols.size(); + context.reference_symbols.emplace_back( + N64Recomp::ReferenceSymbol { + .section_index = N64Recomp::SectionImport, + .section_offset = 0, + .is_function = true + } + ); + import_symbol_mod_ids.emplace_back(mod_id); + } + } + else { + if (functions_data) { + throw toml::parse_error("Mod toml is missing data reference symbol file list", functions_data.node()->source()); + } + else { + throw toml::parse_error("Invalid data reference symbol file list", functions_data.node()->source()); + } + } + } + + } + catch (const toml::parse_error& err) { + std::cerr << "Syntax error parsing symbol import file: " << *err.source().path << " (" << err.source().begin << "):\n" << err.description() << std::endl; + return false; + } + + return true; +} + ModConfig parse_mod_config(const std::filesystem::path& config_path, bool& good) { ModConfig ret{}; good = false; @@ -90,7 +165,7 @@ ModConfig parse_mod_config(const std::filesystem::path& config_path, bool& good) toml::node_view data_reference_syms_file_data = config_data["data_reference_syms_files"]; if (data_reference_syms_file_data.is_array()) { const toml::array* array = data_reference_syms_file_data.as_array(); - ret.data_reference_syms_file_paths = get_data_syms_paths(array, basedir); + ret.data_reference_syms_file_paths = get_toml_path_array(array, basedir); } else { if (data_reference_syms_file_data) { @@ -100,6 +175,16 @@ ModConfig parse_mod_config(const std::filesystem::path& config_path, bool& good) throw toml::parse_error("Invalid data reference symbol file list", data_reference_syms_file_data.node()->source()); } } + + // Imported symbols files (optional) + toml::node_view dependency_data = config_data["dependencies"]; + if (dependency_data.is_array()) { + const toml::array* array = dependency_data.as_array(); + ret.dependency_paths = get_toml_path_array(array, basedir); + } + else if (dependency_data) { + throw toml::parse_error("Invalid imported symbols file list", dependency_data.node()->source()); + } } catch (const toml::parse_error& err) { std::cerr << "Syntax error parsing toml: " << *err.source().path << " (" << err.source().begin << "):\n" << err.description() << std::endl; @@ -110,6 +195,10 @@ ModConfig parse_mod_config(const std::filesystem::path& config_path, bool& good) return ret; } +static inline uint32_t round_up_16(uint32_t value) { + return (value + 15) & (~15); +} + N64Recomp::ModContext build_mod_context(const N64Recomp::Context& input_context, bool& good) { N64Recomp::ModContext ret{}; good = false; @@ -122,10 +211,20 @@ N64Recomp::ModContext build_mod_context(const N64Recomp::Context& input_context, // Sort the vector based on the rom address of the corresponding section. std::sort(section_order.begin(), section_order.end(), [&](uint16_t a, uint16_t b) { - return input_context.sections[a].rom_addr < input_context.sections[b].rom_addr; + const auto& section_a = input_context.sections[a]; + const auto& section_b = input_context.sections[b]; + // Sort primarily by ROM address. + if (section_a.rom_addr != section_b.rom_addr) { + return section_a.rom_addr < section_b.rom_addr; + } + // Sort secondarily by RAM address. + return section_a.ram_addr < section_b.ram_addr; } ); + // TODO avoid a copy here. + ret.base_context.rom = input_context.rom; + uint32_t rom_to_ram = (uint32_t)-1; size_t output_section_index = (size_t)-1; ret.base_context.sections.resize(1); @@ -135,9 +234,21 @@ N64Recomp::ModContext build_mod_context(const N64Recomp::Context& input_context, const auto& cur_section = input_context.sections[section_index]; uint32_t cur_rom_to_ram = cur_section.ram_addr - cur_section.rom_addr; - // Stop checking sections once a non-allocated section has been reached. + // Check if this is a non-allocated section. if (cur_section.rom_addr == (uint32_t)-1) { - break; + // If so, check if it has a vram address directly after the current output section. If it does, then add this + // section's size to the output section's bss size. + if (output_section_index != -1 && cur_section.size != 0) { + auto& section_out = ret.base_context.sections[output_section_index]; + uint32_t output_section_bss_start = section_out.ram_addr + section_out.size; + uint32_t output_section_bss_end = output_section_bss_start + section_out.bss_size; + // Check if the current section starts at the end of the output section, allowing for a range of matches to account for 16 byte section alignment. + if (cur_section.ram_addr >= output_section_bss_end && cur_section.ram_addr <= round_up_16(output_section_bss_end)) { + // Calculate the output section's bss size by using its non-bss end address and the current section's end address. + section_out.bss_size = cur_section.ram_addr + cur_section.size - output_section_bss_start; + } + } + continue; } // Check if this section matches up with the previous section to merge them together. @@ -178,8 +289,23 @@ N64Recomp::ModContext build_mod_context(const N64Recomp::Context& input_context, // If this is the patch section, create a replacement for this function. if (patch_section || force_patch_section) { // Find the corresponding symbol in the reference symbols. + bool original_func_exists = false; auto find_sym_it = input_context.reference_symbols_by_name.find(cur_func.name); + + // Check if the function was found. if (find_sym_it == input_context.reference_symbols_by_name.end()) { + original_func_exists = false; + } + // Ignore reference symbols in the import section, as those are imports and not original symbols. + else if (input_context.reference_symbols[find_sym_it->second].section_index == N64Recomp::SectionImport) { + original_func_exists = false; + } + else { + original_func_exists = true; + } + + // Check that the function being patched exists in the original reference symbols. + if (!original_func_exists) { fmt::print("Function {} is marked as a patch but doesn't exist in the original ROM!\n", cur_func.name); return {}; } @@ -222,10 +348,89 @@ N64Recomp::ModContext build_mod_context(const N64Recomp::Context& input_context, ret.base_context.functions[output_func_index].words.resize(cur_func.words.size()); } - // TODO relocs (including reference symbols and HI16 and LO16 patching for non-relocatable reference symbols) - + // Copy relocs and patch HI16/LO16/26 relocs for non-relocatable reference symbols + section_out.relocs.reserve(cur_section.relocs.size()); + for (const auto& cur_reloc : cur_section.relocs) { + // Skip null relocs. + if (cur_reloc.type == N64Recomp::RelocType::R_MIPS_NONE) { + continue; + } + // Reloc to an imported symbol. + if (cur_reloc.reference_symbol && cur_reloc.target_section == N64Recomp::SectionImport) { + // Copy the reloc as-is. + section_out.relocs.emplace_back(cur_reloc); + } + // Reloc to a reference symbol. + else if (cur_reloc.reference_symbol) { + const auto& reloc_section = input_context.reference_sections[cur_reloc.target_section]; + // Patch relocations to non-relocatable reference sections. + if (!reloc_section.relocatable) { + uint32_t reloc_target_address = reloc_section.ram_addr + cur_reloc.target_section_offset; + uint32_t reloc_rom_address = cur_reloc.address - cur_section.ram_addr + cur_section.rom_addr; + + uint32_t* reloc_word_ptr = reinterpret_cast(ret.base_context.rom.data() + reloc_rom_address); + uint32_t reloc_word = byteswap(*reloc_word_ptr); + switch (cur_reloc.type) { + case N64Recomp::RelocType::R_MIPS_32: + // Don't patch MIPS32 relocations, as they've already been patched during elf parsing. + break; + case N64Recomp::RelocType::R_MIPS_NONE: + // Nothing to do. + break; + case N64Recomp::RelocType::R_MIPS_HI16: + reloc_word &= 0xFFFF0000; + reloc_word |= (reloc_target_address - (int16_t)(reloc_target_address & 0xFFFF)) >> 16 & 0xFFFF; + break; + case N64Recomp::RelocType::R_MIPS_LO16: + reloc_word &= 0xFFFF0000; + reloc_word |= reloc_target_address & 0xFFFF; + break; + case N64Recomp::RelocType::R_MIPS_26: + if (reloc_target_address & 0x3) { + fmt::print("R_MIPS_26 reloc at address 0x{:08X} in section {} has a target address not divisible by 4!\n", + cur_reloc.address, cur_section.name); + return {}; + } + reloc_word &= 0xFC000000; + reloc_word |= (reloc_target_address >> 2) & 0x3FFFFFF; + break; + default: + fmt::print("Unsupported or unknown relocation type {} in reloc at address 0x{:08X} in section {}!\n", + (int)cur_reloc.type, cur_reloc.address, cur_section.name); + return {}; + } + *reloc_word_ptr = byteswap(reloc_word); + } + // Copy relocations to relocatable reference sections as-is. + else { + section_out.relocs.emplace_back(cur_reloc); + } + } + // Reloc to an internal symbol. + else { + const N64Recomp::Section& target_section = input_context.sections[cur_reloc.target_section]; + uint32_t target_rom_to_ram = target_section.ram_addr - target_section.rom_addr; + bool is_noload = target_section.rom_addr == (uint32_t)-1; + if (!is_noload && target_rom_to_ram != cur_rom_to_ram) { + fmt::print("Reloc at address 0x{:08X} in section {} points to a different section!\n", + cur_reloc.address, cur_section.name); + return {}; + } + uint32_t output_section_offset = cur_reloc.target_section_offset + target_section.ram_addr - cur_section.ram_addr; + section_out.relocs.emplace_back(N64Recomp::Reloc{ + .address = cur_reloc.address, + .target_section_offset = output_section_offset, + .symbol_index = 0, + .target_section = N64Recomp::SectionSelf, + .type = cur_reloc.type, + .reference_symbol = false, + }); + } + } // TODO exports + + // TODO imports } good = true; @@ -262,6 +467,16 @@ int main(int argc, const char** argv) { context.import_reference_context(reference_context); } + size_t import_section_symbol_start = context.reference_symbols.size(); + std::vector import_symbol_mod_ids{}; + + for (const std::filesystem::path& dependency_path : config.dependency_paths) { + if (!read_dependency_file(dependency_path, context, import_symbol_mod_ids)) { + fmt::print(stderr, "Failed to read dependency file: {}\n", dependency_path.string()); + return EXIT_FAILURE; + } + } + for (const std::filesystem::path& cur_data_sym_path : config.data_reference_syms_file_paths) { if (!context.read_data_reference_syms(cur_data_sym_path)) { fmt::print(stderr, "Failed to load provided data reference symbol file: {}\n", cur_data_sym_path.string()); @@ -301,7 +516,7 @@ int main(int argc, const char** argv) { output_syms_file.write(reinterpret_cast(symbols_bin.data()), symbols_bin.size()); std::ofstream output_binary_file{ config.output_binary_path, std::ios::binary }; - output_binary_file.write(reinterpret_cast(context.rom.data()), context.rom.size()); + output_binary_file.write(reinterpret_cast(mod_context.base_context.rom.data()), mod_context.base_context.rom.size()); return EXIT_SUCCESS; } diff --git a/include/n64recomp.h b/include/n64recomp.h index 925c8ae..bf88e63 100644 --- a/include/n64recomp.h +++ b/include/n64recomp.h @@ -59,6 +59,7 @@ namespace N64Recomp { constexpr uint16_t SectionSelf = (uint16_t)-1; constexpr uint16_t SectionAbsolute = (uint16_t)-2; + constexpr uint16_t SectionImport = (uint16_t)-3; // Imported symbols for mods struct Section { uint32_t rom_addr = 0; uint32_t ram_addr = 0; diff --git a/src/config.cpp b/src/config.cpp index 9648840..56772ed 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -612,6 +612,7 @@ void N64Recomp::Context::import_reference_context(const N64Recomp::Context& refe for (const N64Recomp::Function& func_in: reference_context.functions) { const N64Recomp::Section& func_section = reference_context.sections[func_in.section_index]; + // TODO Check if reference_symbols_by_name already contains the name and show a conflict error if so. reference_symbols_by_name.emplace(func_in.name, reference_symbols.size()); reference_symbols.emplace_back(N64Recomp::ReferenceSymbol{ @@ -708,6 +709,7 @@ bool N64Recomp::Context::read_data_reference_syms(const std::filesystem::path& d throw toml::parse_error("Reference data symbol entry is missing required field(s)", data_sym_el.source()); } + // TODO Check if reference_symbols_by_name already contains the name and show a conflict error if so. this->reference_symbols_by_name.emplace(name.value(), reference_symbols.size()); this->reference_symbols.emplace_back( diff --git a/src/elf.cpp b/src/elf.cpp index 59d24ef..06e194d 100644 --- a/src/elf.cpp +++ b/src/elf.cpp @@ -390,7 +390,7 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP bool found_rel_symbol = symbol_accessor.get_symbol( rel_symbol, rel_symbol_name, rel_symbol_value, rel_symbol_size, rel_symbol_bind, rel_symbol_type, rel_symbol_section_index, rel_symbol_other); - uint32_t rel_section_vram = section_out.ram_addr; + uint32_t rel_section_vram = 0; uint32_t rel_symbol_offset = 0; // Check if the symbol is undefined and to know whether to look for it in the reference symbols. @@ -412,7 +412,13 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP bool target_section_relocatable = false; - if (reloc_out.target_section != N64Recomp::SectionAbsolute && context.reference_sections[reloc_out.target_section].relocatable) { + if (reloc_out.target_section == N64Recomp::SectionImport) { + target_section_relocatable = true; + } + else if (reloc_out.target_section == N64Recomp::SectionAbsolute) { + target_section_relocatable = false; + } + else if (context.reference_sections[reloc_out.target_section].relocatable) { target_section_relocatable = true; } @@ -425,6 +431,7 @@ ELFIO::section* read_sections(N64Recomp::Context& context, const N64Recomp::ElfP else { reloc_out.reference_symbol = false; reloc_out.target_section = rel_symbol_section_index; + rel_section_vram = context.sections[rel_symbol_section_index].ram_addr; } // Reloc pairing, see MIPS System V ABI documentation page 4-18 (https://refspecs.linuxfoundation.org/elf/mipsabi.pdf) diff --git a/src/mod_symbols.cpp b/src/mod_symbols.cpp index 53e61af..feab9d3 100644 --- a/src/mod_symbols.cpp +++ b/src/mod_symbols.cpp @@ -8,6 +8,8 @@ struct FileHeader { struct FileSubHeaderV1 { uint32_t num_sections; uint32_t num_replacements; + uint32_t num_exports; + uint32_t num_imports; }; struct SectionHeaderV1 { @@ -38,6 +40,9 @@ struct ReplacementV1 { uint32_t flags; // force }; +constexpr uint32_t SectionSelfVromV1 = 0xFFFFFFFF; +constexpr uint32_t SectionImportVromV1 = 0xFFFFFFFE; + template const T* reinterpret_data(std::span data, size_t& offset, size_t count = 1) { if (offset + (sizeof(T) * count) > data.size()) { @@ -127,9 +132,12 @@ bool parse_v1(std::span data, const std::unordered_map(relocs[reloc_index].type); cur_reloc.target_section_offset = relocs[reloc_index].target_section_offset; uint32_t target_section_vrom = relocs[reloc_index].target_section_vrom; - if (target_section_vrom == 0) { + if (target_section_vrom == SectionSelfVromV1) { cur_reloc.target_section = N64Recomp::SectionSelf; } + else if (target_section_vrom == SectionImportVromV1) { + cur_reloc.target_section = N64Recomp::SectionImport; + } else { // TODO lookup by section index by original vrom auto find_section_it = sections_by_vrom.find(target_section_vrom); @@ -259,7 +267,10 @@ std::vector N64Recomp::symbols_to_bin_v1(const N64Recomp::ModContext& m for (const Reloc& cur_reloc : cur_section.relocs) { uint32_t target_section_vrom; if (cur_reloc.target_section == SectionSelf) { - target_section_vrom = 0; + target_section_vrom = SectionSelfVromV1; + } + else if (cur_reloc.target_section == SectionImport) { + target_section_vrom = SectionImportVromV1; } else if (cur_reloc.reference_symbol) { target_section_vrom = context.reference_sections[cur_reloc.target_section].rom_addr;