From eac57752ef7532de1e387e019508bdcf5b6bc6ec Mon Sep 17 00:00:00 2001 From: Mr-Wiseguy Date: Tue, 9 Jul 2024 16:47:05 -0400 Subject: [PATCH] Add N64Recomp as submodule and initial mod loading (temporarily using a DLL instead of Lua recompilation) --- .gitmodules | 3 + N64Recomp | 1 + librecomp/CMakeLists.txt | 3 +- librecomp/include/librecomp/addresses.hpp | 4 + librecomp/include/librecomp/mods.hpp | 11 +- librecomp/include/librecomp/overlays.hpp | 1 + librecomp/src/mod_manifest.cpp | 2 + librecomp/src/mods.cpp | 142 ++++++++++++++++++++++ librecomp/src/overlays.cpp | 35 +++++- librecomp/src/recomp.cpp | 43 ++++++- 10 files changed, 234 insertions(+), 11 deletions(-) create mode 160000 N64Recomp create mode 100644 librecomp/src/mods.cpp diff --git a/.gitmodules b/.gitmodules index dd3bf89..27fa707 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "thirdparty/miniz"] path = thirdparty/miniz url = https://github.com/richgel999/miniz +[submodule "N64Recomp"] + path = N64Recomp + url = https://github.com/N64Recomp/N64Recomp diff --git a/N64Recomp b/N64Recomp new file mode 160000 index 0000000..a9f9b0d --- /dev/null +++ b/N64Recomp @@ -0,0 +1 @@ +Subproject commit a9f9b0ddc4c7f860017d247c02d8507ca301352f diff --git a/librecomp/CMakeLists.txt b/librecomp/CMakeLists.txt index ce96ced..d0b151b 100644 --- a/librecomp/CMakeLists.txt +++ b/librecomp/CMakeLists.txt @@ -49,6 +49,7 @@ if (WIN32) endif() add_subdirectory(${PROJECT_SOURCE_DIR}/../thirdparty/miniz ${CMAKE_BINARY_DIR}/miniz) +add_subdirectory(${PROJECT_SOURCE_DIR}/../N64Recomp ${CMAKE_BINARY_DIR}/N64Recomp EXCLUDE_FROM_ALL) -target_link_libraries(librecomp PRIVATE ultramodern) +target_link_libraries(librecomp PRIVATE ultramodern N64Recomp) target_link_libraries(librecomp PUBLIC miniz) diff --git a/librecomp/include/librecomp/addresses.hpp b/librecomp/include/librecomp/addresses.hpp index 6285104..dc3bbcd 100644 --- a/librecomp/include/librecomp/addresses.hpp +++ b/librecomp/include/librecomp/addresses.hpp @@ -9,6 +9,10 @@ namespace recomp { constexpr int32_t cart_handle = 0x80800000; constexpr int32_t drive_handle = (int32_t)(cart_handle + sizeof(OSPiHandle)); constexpr int32_t flash_handle = (int32_t)(drive_handle + sizeof(OSPiHandle)); + constexpr int32_t flash_handle_end = (int32_t)(flash_handle + sizeof(OSPiHandle)); + constexpr int32_t patch_rdram_start = 0x80801000; + static_assert(patch_rdram_start >= flash_handle_end); + constexpr int32_t mod_rdram_start = 0x81000000; // Flashram occupies the same physical address as sram, but that issue is avoided because libultra exposes // a high-level interface for flashram. Because that high-level interface is reimplemented, low level accesses diff --git a/librecomp/include/librecomp/mods.hpp b/librecomp/include/librecomp/mods.hpp index 6c57a78..316e0e3 100644 --- a/librecomp/include/librecomp/mods.hpp +++ b/librecomp/include/librecomp/mods.hpp @@ -30,6 +30,13 @@ namespace recomp { InnerFileDoesNotExist }; + enum class ModLoadError { + Good, + FailedToLoadSyms, + FailedToLoadBinary, + InvalidFunctionReplacement, + }; + struct ModHandle { virtual ~ModHandle() = default; virtual std::vector read_file(const std::string& filepath, bool& exists) const = 0; @@ -77,8 +84,8 @@ namespace recomp { std::unique_ptr mod_handle; }; - ModManifest open_mod(const std::filesystem::path& mod_path, ModOpenError& error, std::string& error_string); - bool load_mod_(uint8_t* rdram, int32_t target_vram, const std::filesystem::path& symbol_file, const std::filesystem::path& binary_file); + ModManifest open_mod(const std::filesystem::path& mod_path, ModOpenError& error, std::string& error_param); + ModLoadError load_mod(uint8_t* rdram, const ModManifest& manifest, int32_t load_address, uint32_t& ram_used, std::string& error_param); std::string error_to_string(ModOpenError); } diff --git a/librecomp/include/librecomp/overlays.hpp b/librecomp/include/librecomp/overlays.hpp index e7a4fa2..fa9cc39 100644 --- a/librecomp/include/librecomp/overlays.hpp +++ b/librecomp/include/librecomp/overlays.hpp @@ -24,6 +24,7 @@ namespace recomp { void read_patch_data(uint8_t* rdram, gpr patch_data_address); void init_overlays(); + recomp_func_t* get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram); } }; diff --git a/librecomp/src/mod_manifest.cpp b/librecomp/src/mod_manifest.cpp index ef77ea9..55d875e 100644 --- a/librecomp/src/mod_manifest.cpp +++ b/librecomp/src/mod_manifest.cpp @@ -392,6 +392,8 @@ recomp::mods::ModManifest recomp::mods::open_mod(const std::filesystem::path& mo return {}; } + ret.mod_root_path = mod_path; + // Return the loaded mod manifest error = ModOpenError::Good; return ret; diff --git a/librecomp/src/mods.cpp b/librecomp/src/mods.cpp new file mode 100644 index 0000000..580afa9 --- /dev/null +++ b/librecomp/src/mods.cpp @@ -0,0 +1,142 @@ +#include +#include +#define WIN32_LEAN_AND_MEAN +#include + +#include "librecomp/mods.hpp" +#include "librecomp/overlays.hpp" +#include "n64recomp.h" + +void unprotect(void* target_func, DWORD* old_flags) { + BOOL result = VirtualProtect(target_func, + 16, + PAGE_READWRITE, + old_flags); +} + +void protect(void* target_func, DWORD old_flags) { + DWORD dummy_old_flags; + BOOL result = VirtualProtect(target_func, + 16, + old_flags, + &dummy_old_flags); +} + +void patch_func(void* target_func, void* replacement_func) { + static uint8_t movabs_rax[] = {0x48, 0xB8}; + static uint8_t jmp_rax[] = {0xFF, 0xE0}; + uint8_t* target_func_u8 = reinterpret_cast(target_func); + size_t offset = 0; + + auto write_bytes = [&](void* bytes, size_t count) { + memcpy(target_func_u8 + offset, bytes, count); + offset += count; + }; + + DWORD old_flags; + unprotect(target_func, &old_flags); + write_bytes(movabs_rax, sizeof(movabs_rax)); + write_bytes(&replacement_func, sizeof(&replacement_func)); + write_bytes(jmp_rax, sizeof(jmp_rax)); + protect(target_func, old_flags); +} + +recomp::mods::ModLoadError recomp::mods::load_mod(uint8_t* rdram, const ModManifest& manifest, int32_t load_address, uint32_t& ram_used, std::string& error_param) { + N64Recomp::Context context_out{}; + N64Recomp::ModContext mod_context_out{}; + std::vector section_load_addresses{}; + + { + // Load the mod symbol data from the file provided in the manifest. + bool binary_syms_exists = false; + std::vector syms_data = manifest.mod_handle->read_file(manifest.binary_syms_path, binary_syms_exists); + + if (!binary_syms_exists) { + return recomp::mods::ModLoadError::FailedToLoadSyms; + } + + // Load the binary data from the file provided in the manifest. + bool binary_exists = false; + std::vector binary_data = manifest.mod_handle->read_file(manifest.binary_path, binary_exists); + + if (!binary_exists) { + return recomp::mods::ModLoadError::FailedToLoadBinary; + } + + std::span binary_span {reinterpret_cast(binary_data.data()), binary_data.size() }; + + // Parse the symbol file into the recompiler contexts. + N64Recomp::ModSymbolsError symbol_load_error = N64Recomp::parse_mod_symbols(syms_data, binary_span, {}, context_out, mod_context_out); + if (symbol_load_error != N64Recomp::ModSymbolsError::Good) { + return ModLoadError::FailedToLoadSyms; + } + + section_load_addresses.resize(context_out.sections.size()); + + // Copy each section's binary into rdram, leaving room for the section's bss before the next one. + int32_t cur_section_addr = load_address; + for (size_t section_index = 0; section_index < context_out.sections.size(); section_index++) { + const auto& section = context_out.sections[section_index]; + for (size_t i = 0; i < section.size; i++) { + MEM_B(i, (gpr)cur_section_addr) = binary_data[section.rom_addr + i]; + } + section_load_addresses[section_index] = cur_section_addr; + cur_section_addr += section.size + section.bss_size; + } + + ram_used = cur_section_addr - load_address; + } + + // TODO temporary solution for loading mod DLLs, replace with LuaJIT recompilation (including patching LO16/HI16 relocs). + // N64Recomp::recompile_function(...); + static HMODULE mod_dll; + std::filesystem::path dll_path = manifest.mod_root_path; + dll_path.replace_extension(".dll"); + mod_dll = LoadLibraryW(dll_path.c_str()); + + if (!mod_dll) { + printf("Failed to open mod dll: %ls\n", dll_path.c_str()); + return ModLoadError::Good; + } + + // TODO track replacements by mod to find conflicts + uint32_t total_func_count = 0; + for (size_t section_index = 0; section_index < context_out.sections.size(); section_index++) { + const auto& section = context_out.sections[section_index]; + const auto& mod_section = mod_context_out.section_info[section_index]; + // TODO check that section original_vrom is nonzero if it has replacements. + for (const auto& replacement : mod_section.replacements) { + recomp_func_t* to_replace = recomp::overlays::get_func_by_section_ram(mod_section.original_rom_addr, replacement.original_vram); + + if (to_replace == nullptr) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: 0x" << mod_section.original_rom_addr << + " func: 0x" << std::setfill('0') << std::setw(8) << replacement.original_vram; + error_param = error_param_stream.str(); + return ModLoadError::InvalidFunctionReplacement; + } + + uint32_t section_func_index = replacement.func_index; + + // TODO temporary solution for loading mod DLLs, replace with LuaJIT recompilation. + std::string section_func_name = "mod_func_" + std::to_string(total_func_count + section_func_index); + void* replacement_func = GetProcAddress(mod_dll, section_func_name.c_str()); + + if (!replacement_func) { + printf("Failed to find func in dll: %s\n", section_func_name.c_str()); + return ModLoadError::Good; + } + + printf("found replacement func: 0x%016llX\n", (uintptr_t)to_replace); + + patch_func(to_replace, replacement_func); + } + total_func_count += mod_section.replacements.size(); + } + + // TODO perform mips32 relocations + + return ModLoadError::Good; +} + diff --git a/librecomp/src/overlays.cpp b/librecomp/src/overlays.cpp index 7ad7ea5..c12a14d 100644 --- a/librecomp/src/overlays.cpp +++ b/librecomp/src/overlays.cpp @@ -173,13 +173,11 @@ extern "C" void unload_overlays(int32_t ram_addr, uint32_t size) { } } +std::unordered_map sections_by_rom{}; + void recomp::overlays::init_overlays() { section_addresses = (int32_t *)calloc(sections_info.total_num_sections, sizeof(int32_t)); - for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) { - section_addresses[sections_info.code_sections[section_index].index] = sections_info.code_sections[section_index].ram_addr; - } - // Sort the executable sections by rom address std::sort(§ions_info.code_sections[0], §ions_info.code_sections[sections_info.num_code_sections], [](const SectionTableEntry& a, const SectionTableEntry& b) { @@ -187,9 +185,38 @@ void recomp::overlays::init_overlays() { } ); + for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) { + SectionTableEntry* code_section = §ions_info.code_sections[section_index]; + + section_addresses[sections_info.code_sections[section_index].index] = code_section->ram_addr; + sections_by_rom[code_section->rom_addr] = code_section; + } + load_patch_functions(); } +recomp_func_t* recomp::overlays::get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram) { + auto find_section_it = sections_by_rom.find(section_rom); + if (find_section_it == sections_by_rom.end()) { + return nullptr; + } + + SectionTableEntry* section = find_section_it->second; + if (function_vram < section->ram_addr || function_vram >= section->ram_addr + section->size) { + return nullptr; + } + + uint32_t func_offset = function_vram - section->ram_addr; + + for (size_t func_index = 0; func_index < section->num_funcs; func_index++) { + if (section->funcs[func_index].offset == func_offset) { + return section->funcs[func_index].func; + } + } + + return nullptr; +} + extern "C" recomp_func_t * get_function(int32_t addr) { auto func_find = func_map.find(addr); if (func_find == func_map.end()) { diff --git a/librecomp/src/recomp.cpp b/librecomp/src/recomp.cpp index 2726894..789b5a5 100644 --- a/librecomp/src/recomp.cpp +++ b/librecomp/src/recomp.cpp @@ -10,13 +10,16 @@ #include #include #include +#include -#include "recomp.h" -#include "overlays.hpp" -#include "game.hpp" +#include "librecomp/recomp.h" +#include "librecomp/overlays.hpp" +#include "librecomp/game.hpp" #include "xxHash/xxh3.h" #include "ultramodern/ultramodern.hpp" #include "ultramodern/error_handling.hpp" +#include "librecomp/addresses.hpp" +#include "librecomp/mods.hpp" #ifdef _MSC_VER inline uint32_t byteswap(uint32_t val) { @@ -322,7 +325,7 @@ void init(uint8_t* rdram, recomp_context* ctx, gpr entrypoint) { recomp::do_rom_read(rdram, entrypoint, 0x10001000, 0x100000); // Read in any extra data from patches - recomp::overlays::read_patch_data(rdram, (gpr)(s32)0x80801000); + recomp::overlays::read_patch_data(rdram, (gpr)recomp::patch_rdram_start); // Set up context floats ctx->f_odd = &ctx->f0.u32h; @@ -373,6 +376,9 @@ void ultramodern::quit() { current_game.reset(); } +// TODO temporary test mod loading, remove this when mod management is done +recomp::mods::ModManifest testmod_manifest; + void recomp::start( uint32_t rdram_size, ultramodern::renderer::WindowHandle window_handle, @@ -442,6 +448,35 @@ void recomp::start( ultramodern::load_shader_cache(game_entry.cache_data); init(rdram, &context, game_entry.entrypoint_address); + + // TODO temporary test mod loading, remove this when mod management is done + recomp::mods::ModOpenError error; + std::string error_param; + testmod_manifest = recomp::mods::open_mod("testmod_dir", error, error_param); + + if (error != recomp::mods::ModOpenError::Good) { + printf("Mod invalid: %s", recomp::mods::error_to_string(error).c_str()); + if (!error_param.empty()) { + printf(": \"%s\"", error_param.c_str()); + } + printf("\n"); + return; + } + + int32_t cur_mod_ram_addr = recomp::mod_rdram_start; + uint32_t mod_ram_size = 0; + recomp::mods::ModLoadError load_error = recomp::mods::load_mod(rdram, testmod_manifest, cur_mod_ram_addr, mod_ram_size, error_param); + + if (load_error != recomp::mods::ModLoadError::Good) { + printf("Failed to load mod\n"); + printf(" Error code: %d", (int)load_error); + if (!error_param.empty()) { + printf(" (%s)", error_param.c_str()); + } + printf("\n"); + return; + } + try { game_entry.entrypoint(rdram, &context); } catch (ultramodern::thread_terminated& terminated) {