diff --git a/N64Recomp b/N64Recomp index 58b2f86..b822023 160000 --- a/N64Recomp +++ b/N64Recomp @@ -1 +1 @@ -Subproject commit 58b2f8698fb7c9adef4b8cd18ef50154c3ac416b +Subproject commit b822023d300eedc2428558d557cad1008ee73654 diff --git a/librecomp/include/librecomp/game.hpp b/librecomp/include/librecomp/game.hpp index 37bab29..57625ae 100644 --- a/librecomp/include/librecomp/game.hpp +++ b/librecomp/include/librecomp/game.hpp @@ -23,9 +23,11 @@ namespace recomp { std::string internal_name; std::u8string game_id; std::string mod_game_id; - std::span cache_data; SaveType save_type = SaveType::None; bool is_enabled; + // Only needed for mod function hooking support, not needed if `has_compressed_code` is false. + std::vector (*decompression_routine)(std::span compressed_rom) = nullptr; + bool has_compressed_code = false; gpr entrypoint_address; void (*entrypoint)(uint8_t* rdram, recomp_context* context); @@ -73,6 +75,7 @@ namespace recomp { bool is_rom_valid(std::u8string& game_id); bool is_rom_loaded(); void set_rom_contents(std::vector&& new_rom); + std::span get_rom(); void do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr, size_t num_bytes); void do_rom_pio(uint8_t* rdram, gpr ram_address, uint32_t physical_addr); const Version& get_project_version(); diff --git a/librecomp/include/librecomp/mods.hpp b/librecomp/include/librecomp/mods.hpp index 65e5c85..cf54b4b 100644 --- a/librecomp/include/librecomp/mods.hpp +++ b/librecomp/include/librecomp/mods.hpp @@ -101,6 +101,9 @@ namespace recomp { InvalidImport, InvalidCallbackEvent, InvalidFunctionReplacement, + HooksUnavailable, + InvalidHook, + CannotBeHooked, FailedToFindReplacement, BaseRecompConflict, ModConflict, @@ -235,6 +238,7 @@ namespace recomp { bool requires_manifest; }; + class LiveRecompilerCodeHandle; class ModContext { public: ModContext(); @@ -254,11 +258,11 @@ namespace recomp { bool is_content_runtime_toggleable(ModContentTypeId content_type) const; private: ModOpenError open_mod(const std::filesystem::path& mod_path, std::string& error_param, const std::vector& supported_content_types, bool requires_manifest); - ModLoadError load_mod(recomp::mods::ModHandle& mod, std::string& error_param); - void check_dependencies(recomp::mods::ModHandle& mod, std::vector>& errors); - CodeModLoadError init_mod_code(uint8_t* rdram, const std::unordered_map& section_vrom_map, recomp::mods::ModHandle& mod, int32_t load_address, uint32_t& ram_used, std::string& error_param); - CodeModLoadError load_mod_code(uint8_t* rdram, recomp::mods::ModHandle& mod, uint32_t base_event_index, std::string& error_param); - CodeModLoadError resolve_code_dependencies(recomp::mods::ModHandle& mod, const std::unordered_set base_patched_funcs, std::string& error_param); + ModLoadError load_mod(ModHandle& mod, std::string& error_param); + void check_dependencies(ModHandle& mod, std::vector>& errors); + CodeModLoadError init_mod_code(uint8_t* rdram, const std::unordered_map& section_vrom_map, ModHandle& mod, int32_t load_address, bool hooks_available, uint32_t& ram_used, std::string& error_param); + CodeModLoadError load_mod_code(uint8_t* rdram, ModHandle& mod, uint32_t base_event_index, std::string& error_param); + CodeModLoadError resolve_code_dependencies(ModHandle& mod, const std::unordered_set base_patched_funcs, std::string& error_param); void add_opened_mod(ModManifest&& manifest, std::vector&& game_indices, std::vector&& detected_content_types); void close_mods(); @@ -275,6 +279,8 @@ namespace recomp { std::unordered_map patched_funcs; std::unordered_map loaded_mods_by_id; std::vector loaded_code_mods; + // Code handle for vanilla code that was regenerated to add hooks. + std::unique_ptr regenerated_code_handle; // Map of hook definition to the entry hook slot's index. std::unordered_map hook_slots; // Tracks which hook slots have already been processed. Used to regenerate vanilla functions as needed diff --git a/librecomp/include/librecomp/overlays.hpp b/librecomp/include/librecomp/overlays.hpp index 82848bd..339910a 100644 --- a/librecomp/include/librecomp/overlays.hpp +++ b/librecomp/include/librecomp/overlays.hpp @@ -31,7 +31,9 @@ namespace recomp { void init_overlays(); const std::unordered_map& get_vrom_to_section_map(); + uint32_t get_section_ram_addr(uint16_t code_section_index); recomp_func_t* get_func_by_section_rom_function_vram(uint32_t section_rom, uint32_t function_vram); + bool get_func_entry_by_section_index_function_offset(uint16_t code_section_index, uint32_t function_offset, FuncEntry& func_out); recomp_func_t* get_func_by_section_index_function_offset(uint16_t code_section_index, uint32_t function_offset); recomp_func_t* get_base_export(const std::string& export_name); size_t get_base_event_index(const std::string& event_name); diff --git a/librecomp/include/librecomp/sections.h b/librecomp/include/librecomp/sections.h index 039c3cf..91789d3 100644 --- a/librecomp/include/librecomp/sections.h +++ b/librecomp/include/librecomp/sections.h @@ -9,6 +9,7 @@ typedef struct { recomp_func_t* func; uint32_t offset; + uint32_t rom_size; } FuncEntry; typedef struct { diff --git a/librecomp/src/mod_manifest.cpp b/librecomp/src/mod_manifest.cpp index aec8260..1762fe5 100644 --- a/librecomp/src/mod_manifest.cpp +++ b/librecomp/src/mod_manifest.cpp @@ -581,6 +581,14 @@ std::string recomp::mods::error_to_string(CodeModLoadError error) { return "Event for callback not found"; case CodeModLoadError::InvalidFunctionReplacement: return "Function to be replaced does not exist"; + case CodeModLoadError::HooksUnavailable: + // This error will occur if the ROM's GameEntry is set as having compressed code, but no + // ROM decompression routine has been provided. + return "Function hooks are currently unavailable in this project"; + case CodeModLoadError::InvalidHook: + return "Function to be hooked does not exist"; + case CodeModLoadError::CannotBeHooked: + return "Function is not hookable"; case CodeModLoadError::FailedToFindReplacement: return "Failed to find replacement function"; case CodeModLoadError::BaseRecompConflict: diff --git a/librecomp/src/mods.cpp b/librecomp/src/mods.cpp index 61d544e..7d259fb 100644 --- a/librecomp/src/mods.cpp +++ b/librecomp/src/mods.cpp @@ -753,12 +753,110 @@ std::vector recomp::mods::ModContext::get_mod_details( return ret; } +struct PatchedSection { + uint32_t rom_addr; + uint32_t ram_addr; + size_t first_func_index; +}; + +struct PatchedFunction { + uint32_t section_offset; + uint32_t size; +}; + +struct PatchedList { + std::vector sections; + std::vector functions; +}; + +N64Recomp::Context context_from_patched_function_list(const PatchedList& patchlist, std::span rom) { + N64Recomp::Context ret{}; + + // TODO avoid copying the whole ROM into the context somehow. + ret.rom.assign(rom.begin(), rom.end()); + + ret.sections.resize(patchlist.sections.size()); + ret.section_functions.resize(patchlist.sections.size()); + ret.functions.resize(patchlist.functions.size()); + + for (size_t section_index = 0; section_index < patchlist.sections.size(); section_index++) { + const PatchedSection& section_in = patchlist.sections[section_index]; + N64Recomp::Section& section_out = ret.sections[section_index]; + + size_t cur_num_funcs; + if (section_index == patchlist.sections.size() - 1) { + cur_num_funcs = patchlist.functions.size() - section_in.first_func_index; + } + else { + cur_num_funcs = patchlist.sections[section_index + 1].first_func_index - section_in.first_func_index; + } + + section_out.rom_addr = section_in.rom_addr; + section_out.ram_addr = section_in.ram_addr; + section_out.size = 0; + section_out.bss_size = 0; + section_out.function_addrs.resize(cur_num_funcs); + section_out.relocs = std::vector{}; + section_out.name = "patch_section_" + std::to_string(section_index); + section_out.bss_section_index = 0; + section_out.executable = true; + section_out.relocatable = false; + section_out.has_mips32_relocs = false; + + std::vector& section_funcs_out = ret.section_functions[section_index]; + section_funcs_out.resize(cur_num_funcs); + + for (size_t section_function_index = 0; section_function_index < cur_num_funcs; section_function_index++) { + // Get the global index of the function within the context. + size_t function_index = section_in.first_func_index + section_function_index; + section_funcs_out[section_function_index] = function_index; + + // Populate the fields of the function. + const PatchedFunction& function_in = patchlist.functions[function_index]; + N64Recomp::Function& function_out = ret.functions[function_index]; + function_out.vram = section_out.ram_addr + function_in.section_offset; + function_out.rom = section_out.rom_addr + function_in.section_offset; + function_out.words.resize(function_in.size / sizeof(uint32_t)); + function_out.name = "patch_function_" + std::to_string(function_index); + function_out.section_index = section_index; + function_out.ignored = false; + function_out.reimplemented = false; + function_out.stubbed = false; + function_out.function_hooks.clear(); + + // Copy the function's words. + const uint32_t* func_words = reinterpret_cast(rom.data() + function_out.rom); + function_out.words.assign(func_words, func_words + function_in.size / sizeof(uint32_t)); + + // Add the function to the lookup table. + ret.functions_by_vram[function_out.vram].push_back(function_index); + } + } + + return ret; +} + std::vector recomp::mods::ModContext::load_mods(const GameEntry& game_entry, uint8_t* rdram, int32_t load_address, uint32_t& ram_used) { std::vector ret{}; ram_used = 0; num_events = recomp::overlays::num_base_events(); loaded_code_mods.clear(); + std::span decompressed_rom{}; + + // Decompress the rom if needed. + std::vector decompressed_rom_data{}; + if (game_entry.has_compressed_code) { + if (game_entry.decompression_routine != nullptr) { + decompressed_rom_data = game_entry.decompression_routine(recomp::get_rom()); + } + decompressed_rom = std::span{decompressed_rom_data}; + } + // Otherwise, assign the regular rom as the decompressed rom since no decompression is needed. + else { + decompressed_rom = recomp::get_rom(); + } + // Collect the set of functions patched by the base recomp. std::unordered_set base_patched_funcs = recomp::overlays::get_base_patched_funcs(); @@ -829,7 +927,7 @@ std::vector recomp::mods::ModContext::load_mo uint32_t cur_ram_used = 0; auto& mod = opened_mods[mod_index]; std::string cur_error_param; - CodeModLoadError cur_error = init_mod_code(rdram, section_vrom_map, mod, load_address, cur_ram_used, cur_error_param); + CodeModLoadError cur_error = init_mod_code(rdram, section_vrom_map, mod, load_address, !decompressed_rom.empty(), cur_ram_used, cur_error_param); if (cur_error != CodeModLoadError::Good) { if (cur_error_param.empty()) { ret.emplace_back(mod.manifest.mod_id, ModLoadError::FailedToLoadCode, error_to_string(cur_error)); @@ -854,6 +952,9 @@ std::vector recomp::mods::ModContext::load_mo // Set up the event callbacks based on the number of events allocated. recomp::mods::setup_events(num_events); + // TODO if any hooks have been made but the decompressed rom isn't available, + // present an error and stop loading mods. + // Set up the hook slots based on the number of unique hooks. recomp::mods::setup_hooks(hook_slots.size()); @@ -904,7 +1005,163 @@ std::vector recomp::mods::ModContext::load_mo } // Regenerate any remaining hook slots that weren't handled during mod recompilation. - // TODO + + // List of unprocessed hooks and their hook index. + std::vector> unprocessed_hooks; + for (const auto& [def, index] : hook_slots) { + if (!processed_hook_slots[index]) { + unprocessed_hooks.emplace_back(std::make_pair(def, index)); + } + } + + // Sort the unprocessed hooks by section and vram. + std::sort(unprocessed_hooks.begin(), unprocessed_hooks.end(), + [](const std::pair& lhs, const std::pair& rhs) { + if (lhs.first.section_rom == rhs.first.section_rom) { + return lhs.first.function_vram < rhs.first.function_vram; + } + else { + return lhs.first.section_rom < rhs.first.section_rom; + } + } + ); + + // Collect the unprocessed hooks into a patch list. + // Hooks have been sorted by their section address and function address at this point so they + // can be gathered by section into the patch list. + PatchedList patchlist{}; + uint32_t cur_section_rom = 0xFFFFFFFF; + uint32_t cur_section_vram = 0xFFFFFFFF; + uint16_t cur_section_index = 0xFFFF; + uint32_t cur_function_vram = 0xFFFFFFFF; + + // While scanning, also track the hook slot indices for recompilation and the native functions so they can be patched. + std::vector func_ptrs{}; + // Maps function index within context to hook slot index. + std::unordered_map entry_func_hooks{}; + std::unordered_map return_func_hooks{}; + + for (size_t hook_index = 0; hook_index < unprocessed_hooks.size(); hook_index++) { + const auto& cur_hook = unprocessed_hooks[hook_index]; + const auto& cur_hook_def = cur_hook.first; + size_t cur_hook_slot_index = cur_hook.second; + + if (cur_hook_def.section_rom != cur_section_rom) { + // Get the index of the section. + auto find_section_it = section_vrom_map.find(cur_hook_def.section_rom); + if (find_section_it == section_vrom_map.end()) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: 0x" << cur_hook_def.section_rom << + " func: 0x" << std::setfill('0') << std::setw(8) << cur_hook_def.function_vram; + ret.emplace_back(ModLoadErrorDetails{ + "", ModLoadError::FailedToLoadCode, error_to_string(CodeModLoadError::InvalidHook) + ":" + error_param_stream.str() + }); + unload_mods(); + return ret; + } + + uint16_t section_index = find_section_it->second; + + // Allocate a new section. + auto& section_out = patchlist.sections.emplace_back(PatchedSection{ + .rom_addr = cur_hook_def.section_rom, + .ram_addr = recomp::overlays::get_section_ram_addr(section_index), + .first_func_index = patchlist.functions.size() + }); + + // Update the tracked section fields. + cur_section_rom = section_out.rom_addr; + cur_section_vram = section_out.ram_addr; + cur_section_index = section_index; + + // Reset the tracked function vram to prevent issues when two functions have the same vram in different sections. + cur_function_vram = 0xFFFFFFFF; + } + + if (cur_hook_def.function_vram != cur_function_vram) { + uint32_t function_section_offset = cur_hook_def.function_vram - cur_section_vram; + FuncEntry func_entry{}; + bool found_func = recomp::overlays::get_func_entry_by_section_index_function_offset(cur_section_index, function_section_offset, func_entry); + + if (!found_func) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: 0x" << cur_hook_def.section_rom << + " func: 0x" << std::setfill('0') << std::setw(8) << cur_hook_def.function_vram; + ret.emplace_back(ModLoadErrorDetails{ + "", ModLoadError::FailedToLoadCode, error_to_string(CodeModLoadError::InvalidHook) + ":" + error_param_stream.str() + }); + unload_mods(); + return ret; + } + + uint32_t function_rom_size = func_entry.rom_size; + + // A size of 0 means the function can't be hooked (e.g. it's a native reimplemented function). + if (function_rom_size == 0) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: 0x" << cur_hook_def.section_rom << + " func: 0x" << std::setfill('0') << std::setw(8) << cur_hook_def.function_vram; + ret.emplace_back(ModLoadErrorDetails{ + "", ModLoadError::FailedToLoadCode, error_to_string(CodeModLoadError::CannotBeHooked) + ":" + error_param_stream.str() + }); + unload_mods(); + return ret; + } + + // Allocate a new function. + patchlist.functions.emplace_back(PatchedFunction{ + .section_offset = function_section_offset, + .size = function_rom_size + }); + func_ptrs.push_back(func_entry.func); + + // Update the tracked function address. + cur_function_vram = cur_hook_def.function_vram; + } + + // Record the hooks in the function to hook mapping. + size_t func_index = patchlist.functions.size() - 1; + if (cur_hook_def.at_return) { + return_func_hooks[func_index] = cur_hook_slot_index; + } + else { + entry_func_hooks[func_index] = cur_hook_slot_index; + } + } + + // Generate the recompiler context. + N64Recomp::Context hook_context = context_from_patched_function_list(patchlist, decompressed_rom); + hook_context.use_lookup_for_all_function_calls = true; + + // Regenerate the functions using the live recompiler. + ModCodeHandleInputs handle_inputs{ + .base_event_index = 0, // No events in vanilla functions, so this doesn't matter. + .recomp_trigger_event = recomp_trigger_event, + .get_function = get_function, + .cop0_status_write = cop0_status_write, + .cop0_status_read = cop0_status_read, + .switch_error = switch_error, + .do_break = do_break, + .reference_section_addresses = section_addresses, + }; + regenerated_code_handle = std::make_unique(hook_context, handle_inputs, std::move(entry_func_hooks), std::move(return_func_hooks)); + + if (!regenerated_code_handle->good()) { + regenerated_code_handle.reset(); + ret.emplace_back(ModLoadErrorDetails{ + "", ModLoadError::FailedToLoadCode, error_to_string(CodeModLoadError::InternalError) + }); + unload_mods(); + return ret; + } + + // Patch the functions that were regenerated. + for (size_t patched_func_index = 0; patched_func_index < func_ptrs.size(); patched_func_index++) { + patch_func(func_ptrs[patched_func_index], regenerated_code_handle->get_function_handle(patched_func_index)); + } active_game = mod_game_index; return ret; @@ -940,7 +1197,7 @@ void recomp::mods::ModContext::check_dependencies(recomp::mods::ModHandle& mod, } } -recomp::mods::CodeModLoadError recomp::mods::ModContext::init_mod_code(uint8_t* rdram, const std::unordered_map& section_vrom_map, recomp::mods::ModHandle& mod, int32_t load_address, uint32_t& ram_used, std::string& error_param) { +recomp::mods::CodeModLoadError recomp::mods::ModContext::init_mod_code(uint8_t* rdram, const std::unordered_map& section_vrom_map, ModHandle& mod, int32_t load_address, bool hooks_available, uint32_t& ram_used, std::string& error_param) { // Load the mod symbol data from the file provided in the manifest. bool binary_syms_exists = false; std::vector syms_data = mod.manifest.file_handle->read_file(std::string{ modpaths::binary_syms_path }, binary_syms_exists); @@ -965,6 +1222,11 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::init_mod_code(uint8_t* return CodeModLoadError::FailedToParseSyms; } + // Prevent loading the mod if hooks aren't available and it has any hooks. + if (!hooks_available && !mod.recompiler_context->hooks.empty()) { + return CodeModLoadError::HooksUnavailable; + } + // Set all reference sections as relocatable, since the only relocations present in a mod's context // are ones that target relocatable sections. mod.recompiler_context->set_all_reference_sections_relocatable(); @@ -1113,7 +1375,7 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::load_mod_code(uint8_t* // Use a dynamic library code handle. This feature isn't meant to be used by end users, but provides a more debuggable // experience than the live recompiler for mod developers. - // Enabled if the mod's filename ends with ".offline.dll". + // Enabled if the mod's filename ends with ".offline.nrm". if (mod.manifest.mod_root_path.filename().string().ends_with(".offline.nrm")) { // Hooks can't be generated for native mods, so return an error if any of the functions this mod replaces are also hooked by another mod. if (!entry_func_hooks.empty() || !return_func_hooks.empty()) { diff --git a/librecomp/src/overlays.cpp b/librecomp/src/overlays.cpp index 538c173..53354b6 100644 --- a/librecomp/src/overlays.cpp +++ b/librecomp/src/overlays.cpp @@ -112,6 +112,10 @@ const std::unordered_map& recomp::overlays::get_vrom_to_sect return code_sections_by_rom; } +uint32_t recomp::overlays::get_section_ram_addr(uint16_t code_section_index) { + return sections_info.code_sections[code_section_index].ram_addr; +} + void recomp::overlays::add_loaded_function(int32_t ram, recomp_func_t* func) { func_map[ram] = func; } @@ -259,23 +263,34 @@ void recomp::overlays::init_overlays() { } // Finds a function given a section's index and the function's offset into the section. -recomp_func_t* recomp::overlays::get_func_by_section_index_function_offset(uint16_t code_section_index, uint32_t function_offset) { +bool recomp::overlays::get_func_entry_by_section_index_function_offset(uint16_t code_section_index, uint32_t function_offset, FuncEntry& func_out) { if (code_section_index >= sections_info.num_code_sections) { - return nullptr; + return false; } SectionTableEntry* section = §ions_info.code_sections[code_section_index]; if (function_offset >= section->size) { - return nullptr; + return false; } for (size_t func_index = 0; func_index < section->num_funcs; func_index++) { if (section->funcs[func_index].offset == function_offset) { - return section->funcs[func_index].func; + func_out = section->funcs[func_index]; + return true; } } - return nullptr; + return false; +} + +// Finds a function given a section's index and the function's offset into the section and returns its native pointer. +recomp_func_t* recomp::overlays::get_func_by_section_index_function_offset(uint16_t code_section_index, uint32_t function_offset) { + FuncEntry entry; + if (!get_func_entry_by_section_index_function_offset(code_section_index, function_offset, entry)) { + return nullptr; + } + + return entry.func; } // Finds a function given a section's rom address and the function's vram address. diff --git a/librecomp/src/pi.cpp b/librecomp/src/pi.cpp index a3ebfd4..ede0c63 100644 --- a/librecomp/src/pi.cpp +++ b/librecomp/src/pi.cpp @@ -21,6 +21,10 @@ void recomp::set_rom_contents(std::vector&& new_rom) { rom = std::move(new_rom); } +std::span recomp::get_rom() { + return rom; +} + constexpr uint32_t k1_to_phys(uint32_t addr) { return addr & 0x1FFFFFFF; } diff --git a/ultramodern/include/ultramodern/renderer_context.hpp b/ultramodern/include/ultramodern/renderer_context.hpp index 8d13d75..a299a15 100644 --- a/ultramodern/include/ultramodern/renderer_context.hpp +++ b/ultramodern/include/ultramodern/renderer_context.hpp @@ -90,7 +90,7 @@ namespace ultramodern { /** * This callback is optional. If not provided a library default will be used. */ - get_graphics_api_name_t *get_graphics_api_name; + get_graphics_api_name_t *get_graphics_api_name = nullptr; }; void set_callbacks(const callbacks_t& callbacks);