Add N64Recomp as submodule and initial mod loading (temporarily using a DLL instead of Lua recompilation)

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "thirdparty/miniz"]
 	path = thirdparty/miniz
 	url = https://github.com/richgel999/miniz
+[submodule "N64Recomp"]
+	path = N64Recomp
+	url = https://github.com/N64Recomp/N64Recomp

N64Recomp

@@ -0,0 +1 @@
+Subproject commit a9f9b0ddc4c7f860017d247c02d8507ca301352f

librecomp/CMakeLists.txt

@@ -49,6 +49,7 @@ if (WIN32)
 endif()
 
 add_subdirectory(${PROJECT_SOURCE_DIR}/../thirdparty/miniz ${CMAKE_BINARY_DIR}/miniz)
+add_subdirectory(${PROJECT_SOURCE_DIR}/../N64Recomp ${CMAKE_BINARY_DIR}/N64Recomp EXCLUDE_FROM_ALL)
 
-target_link_libraries(librecomp PRIVATE ultramodern)
+target_link_libraries(librecomp PRIVATE ultramodern N64Recomp)
 target_link_libraries(librecomp PUBLIC miniz)

librecomp/include/librecomp/addresses.hpp

@@ -9,6 +9,10 @@ namespace recomp {
     constexpr int32_t cart_handle = 0x80800000;
     constexpr int32_t drive_handle = (int32_t)(cart_handle + sizeof(OSPiHandle));
     constexpr int32_t flash_handle = (int32_t)(drive_handle + sizeof(OSPiHandle));
+    constexpr int32_t flash_handle_end = (int32_t)(flash_handle + sizeof(OSPiHandle));
+    constexpr int32_t patch_rdram_start = 0x80801000;
+    static_assert(patch_rdram_start >= flash_handle_end);
+    constexpr int32_t mod_rdram_start = 0x81000000;
 
     // Flashram occupies the same physical address as sram, but that issue is avoided because libultra exposes
     // a high-level interface for flashram. Because that high-level interface is reimplemented, low level accesses

librecomp/include/librecomp/mods.hpp

@@ -30,6 +30,13 @@ namespace recomp {
             InnerFileDoesNotExist
         };
 
+        enum class ModLoadError {
+            Good,
+            FailedToLoadSyms,
+            FailedToLoadBinary,
+            InvalidFunctionReplacement,
+        };
+
         struct ModHandle {
             virtual ~ModHandle() = default;
             virtual std::vector<char> read_file(const std::string& filepath, bool& exists) const = 0;
@@ -77,8 +84,8 @@ namespace recomp {
             std::unique_ptr<ModHandle> mod_handle;
         };
 
-        ModManifest open_mod(const std::filesystem::path& mod_path, ModOpenError& error, std::string& error_string);
-        bool load_mod_(uint8_t* rdram, int32_t target_vram, const std::filesystem::path& symbol_file, const std::filesystem::path& binary_file);
+        ModManifest open_mod(const std::filesystem::path& mod_path, ModOpenError& error, std::string& error_param);
+        ModLoadError load_mod(uint8_t* rdram, const ModManifest& manifest, int32_t load_address, uint32_t& ram_used, std::string& error_param);
 
         std::string error_to_string(ModOpenError);
     }

librecomp/include/librecomp/overlays.hpp

@@ -24,6 +24,7 @@ namespace recomp {
         void read_patch_data(uint8_t* rdram, gpr patch_data_address);
 
         void init_overlays();
+        recomp_func_t* get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram);
     }
 };
 

librecomp/src/mod_manifest.cpp

@@ -392,6 +392,8 @@ recomp::mods::ModManifest recomp::mods::open_mod(const std::filesystem::path& mo
         return {};
     }
 
+    ret.mod_root_path = mod_path;
+
     // Return the loaded mod manifest
     error = ModOpenError::Good;
     return ret;

librecomp/src/mods.cpp

@@ -0,0 +1,142 @@
+#include <span>
+#include <fstream>
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+
+#include "librecomp/mods.hpp"
+#include "librecomp/overlays.hpp"
+#include "n64recomp.h"
+
+void unprotect(void* target_func, DWORD* old_flags) {
+    BOOL result = VirtualProtect(target_func,
+        16,
+        PAGE_READWRITE,
+        old_flags);
+}
+
+void protect(void* target_func, DWORD old_flags) {
+    DWORD dummy_old_flags;
+    BOOL result = VirtualProtect(target_func,
+        16,
+        old_flags,
+        &dummy_old_flags);
+}
+
+void patch_func(void* target_func, void* replacement_func) {
+    static uint8_t movabs_rax[] = {0x48, 0xB8};
+    static uint8_t jmp_rax[] = {0xFF, 0xE0};
+    uint8_t* target_func_u8 = reinterpret_cast<uint8_t*>(target_func);
+    size_t offset = 0;
+
+    auto write_bytes = [&](void* bytes, size_t count) {
+        memcpy(target_func_u8 + offset, bytes, count);
+        offset += count;
+    };
+
+    DWORD old_flags;
+    unprotect(target_func, &old_flags);
+    write_bytes(movabs_rax, sizeof(movabs_rax));
+    write_bytes(&replacement_func, sizeof(&replacement_func));
+    write_bytes(jmp_rax, sizeof(jmp_rax));
+    protect(target_func, old_flags);
+}
+
+recomp::mods::ModLoadError recomp::mods::load_mod(uint8_t* rdram, const ModManifest& manifest, int32_t load_address, uint32_t& ram_used, std::string& error_param) {
+    N64Recomp::Context context_out{};
+    N64Recomp::ModContext mod_context_out{};
+    std::vector<int32_t> section_load_addresses{};
+
+    {
+        // Load the mod symbol data from the file provided in the manifest.
+        bool binary_syms_exists = false;
+        std::vector<char> syms_data = manifest.mod_handle->read_file(manifest.binary_syms_path, binary_syms_exists);
+
+        if (!binary_syms_exists) {
+            return recomp::mods::ModLoadError::FailedToLoadSyms;
+        }
+        
+        // Load the binary data from the file provided in the manifest.
+        bool binary_exists = false;
+        std::vector<char> binary_data = manifest.mod_handle->read_file(manifest.binary_path, binary_exists);
+
+        if (!binary_exists) {
+            return recomp::mods::ModLoadError::FailedToLoadBinary;
+        }
+
+        std::span<uint8_t> binary_span {reinterpret_cast<uint8_t*>(binary_data.data()), binary_data.size() };
+
+        // Parse the symbol file into the recompiler contexts.
+        N64Recomp::ModSymbolsError symbol_load_error = N64Recomp::parse_mod_symbols(syms_data, binary_span, {}, context_out, mod_context_out);
+        if (symbol_load_error != N64Recomp::ModSymbolsError::Good) {
+            return ModLoadError::FailedToLoadSyms;
+        }
+        
+        section_load_addresses.resize(context_out.sections.size());
+        
+        // Copy each section's binary into rdram, leaving room for the section's bss before the next one.
+        int32_t cur_section_addr = load_address;
+        for (size_t section_index = 0; section_index < context_out.sections.size(); section_index++) {
+            const auto& section = context_out.sections[section_index];
+            for (size_t i = 0; i < section.size; i++) {
+                MEM_B(i, (gpr)cur_section_addr) = binary_data[section.rom_addr + i];
+            }
+            section_load_addresses[section_index] = cur_section_addr;
+            cur_section_addr += section.size + section.bss_size;
+        }
+
+        ram_used = cur_section_addr - load_address;
+    }
+
+    // TODO temporary solution for loading mod DLLs, replace with LuaJIT recompilation (including patching LO16/HI16 relocs).
+    // N64Recomp::recompile_function(...);
+    static HMODULE mod_dll;
+    std::filesystem::path dll_path = manifest.mod_root_path;
+    dll_path.replace_extension(".dll");
+    mod_dll = LoadLibraryW(dll_path.c_str());
+
+    if (!mod_dll) {
+        printf("Failed to open mod dll: %ls\n", dll_path.c_str());
+        return ModLoadError::Good;
+    }
+
+    // TODO track replacements by mod to find conflicts
+    uint32_t total_func_count = 0;
+    for (size_t section_index = 0; section_index < context_out.sections.size(); section_index++) {
+        const auto& section = context_out.sections[section_index];
+        const auto& mod_section = mod_context_out.section_info[section_index];
+        // TODO check that section original_vrom is nonzero if it has replacements.
+        for (const auto& replacement : mod_section.replacements) {
+            recomp_func_t* to_replace = recomp::overlays::get_func_by_section_ram(mod_section.original_rom_addr, replacement.original_vram);
+
+            if (to_replace == nullptr) {
+                std::stringstream error_param_stream{};
+                error_param_stream << std::hex <<
+                    "section: 0x" << mod_section.original_rom_addr <<
+                    " func: 0x" << std::setfill('0') << std::setw(8) << replacement.original_vram;
+                error_param = error_param_stream.str();
+                return ModLoadError::InvalidFunctionReplacement;
+            }
+
+            uint32_t section_func_index = replacement.func_index;
+
+            // TODO temporary solution for loading mod DLLs, replace with LuaJIT recompilation.
+            std::string section_func_name = "mod_func_" + std::to_string(total_func_count + section_func_index);
+            void* replacement_func = GetProcAddress(mod_dll, section_func_name.c_str());
+
+            if (!replacement_func) {
+                printf("Failed to find func in dll: %s\n", section_func_name.c_str());
+                return ModLoadError::Good;
+            }
+
+            printf("found replacement func: 0x%016llX\n", (uintptr_t)to_replace);
+
+            patch_func(to_replace, replacement_func);
+        }
+        total_func_count += mod_section.replacements.size();
+    }
+
+    // TODO perform mips32 relocations
+
+    return ModLoadError::Good;
+}
+

librecomp/src/overlays.cpp

@@ -173,13 +173,11 @@ extern "C" void unload_overlays(int32_t ram_addr, uint32_t size) {
     }
 }
 
+std::unordered_map<uint32_t, SectionTableEntry*> sections_by_rom{};
+
 void recomp::overlays::init_overlays() {
     section_addresses = (int32_t *)calloc(sections_info.total_num_sections, sizeof(int32_t));
 
-    for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) {
-        section_addresses[sections_info.code_sections[section_index].index] = sections_info.code_sections[section_index].ram_addr;
-    }
-
     // Sort the executable sections by rom address
     std::sort(&sections_info.code_sections[0], &sections_info.code_sections[sections_info.num_code_sections],
         [](const SectionTableEntry& a, const SectionTableEntry& b) {
@@ -187,9 +185,38 @@ void recomp::overlays::init_overlays() {
         }
     );
 
+    for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) {
+        SectionTableEntry* code_section = &sections_info.code_sections[section_index];
+
+        section_addresses[sections_info.code_sections[section_index].index] = code_section->ram_addr;
+        sections_by_rom[code_section->rom_addr] = code_section;        
+    }
+
     load_patch_functions();
 }
 
+recomp_func_t* recomp::overlays::get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram) {
+    auto find_section_it = sections_by_rom.find(section_rom);
+    if (find_section_it == sections_by_rom.end()) {
+        return nullptr;
+    }
+
+    SectionTableEntry* section = find_section_it->second;
+    if (function_vram < section->ram_addr || function_vram >= section->ram_addr + section->size) {
+        return nullptr;
+    }
+
+    uint32_t func_offset = function_vram - section->ram_addr;
+
+    for (size_t func_index = 0; func_index < section->num_funcs; func_index++) {
+        if (section->funcs[func_index].offset == func_offset) {
+            return section->funcs[func_index].func;
+        }
+    }
+
+    return nullptr;
+}
+
 extern "C" recomp_func_t * get_function(int32_t addr) {
     auto func_find = func_map.find(addr);
     if (func_find == func_map.end()) {

librecomp/src/recomp.cpp

@@ -10,13 +10,16 @@
 #include <optional>
 #include <mutex>
 #include <array>
+#include <cinttypes>
 
-#include "recomp.h"
-#include "overlays.hpp"
-#include "game.hpp"
+#include "librecomp/recomp.h"
+#include "librecomp/overlays.hpp"
+#include "librecomp/game.hpp"
 #include "xxHash/xxh3.h"
 #include "ultramodern/ultramodern.hpp"
 #include "ultramodern/error_handling.hpp"
+#include "librecomp/addresses.hpp"
+#include "librecomp/mods.hpp"
 
 #ifdef _MSC_VER
 inline uint32_t byteswap(uint32_t val) {
@@ -322,7 +325,7 @@ void init(uint8_t* rdram, recomp_context* ctx, gpr entrypoint) {
     recomp::do_rom_read(rdram, entrypoint, 0x10001000, 0x100000);
 
     // Read in any extra data from patches
-    recomp::overlays::read_patch_data(rdram, (gpr)(s32)0x80801000);
+    recomp::overlays::read_patch_data(rdram, (gpr)recomp::patch_rdram_start);
 
     // Set up context floats
     ctx->f_odd = &ctx->f0.u32h;
@@ -373,6 +376,9 @@ void ultramodern::quit() {
     current_game.reset();
 }
 
+// TODO temporary test mod loading, remove this when mod management is done
+recomp::mods::ModManifest testmod_manifest;
+
 void recomp::start(
     uint32_t rdram_size,
     ultramodern::renderer::WindowHandle window_handle,
@@ -442,6 +448,35 @@ void recomp::start(
 
                     ultramodern::load_shader_cache(game_entry.cache_data);
                     init(rdram, &context, game_entry.entrypoint_address);
+
+                    // TODO temporary test mod loading, remove this when mod management is done
+                    recomp::mods::ModOpenError error;
+                    std::string error_param;
+                    testmod_manifest = recomp::mods::open_mod("testmod_dir", error, error_param);
+
+                    if (error != recomp::mods::ModOpenError::Good) {
+                        printf("Mod invalid: %s", recomp::mods::error_to_string(error).c_str());
+                        if (!error_param.empty()) {
+                            printf(": \"%s\"", error_param.c_str());
+                        }
+                        printf("\n");
+                        return;
+                    }
+
+                    int32_t cur_mod_ram_addr = recomp::mod_rdram_start;
+                    uint32_t mod_ram_size = 0;
+                    recomp::mods::ModLoadError load_error = recomp::mods::load_mod(rdram, testmod_manifest, cur_mod_ram_addr, mod_ram_size, error_param);
+
+                    if (load_error != recomp::mods::ModLoadError::Good) {
+                        printf("Failed to load mod\n");
+                        printf("  Error code: %d", (int)load_error);
+                        if (!error_param.empty()) {
+                            printf(" (%s)", error_param.c_str());
+                        }
+                        printf("\n");
+                        return;
+                    }
+
                     try {
                         game_entry.entrypoint(rdram, &context);
                     } catch (ultramodern::thread_terminated& terminated) {