From d17a3f34cb223ef51865ae99d1eb8339ab796043 Mon Sep 17 00:00:00 2001 From: Wiseguy <68165316+Mr-Wiseguy@users.noreply.github.com> Date: Thu, 2 Jan 2025 21:25:20 -0500 Subject: [PATCH] Integrate the live recompiler for mod loading (#76) --- N64Recomp | 2 +- librecomp/CMakeLists.txt | 2 +- librecomp/include/librecomp/addresses.hpp | 2 +- librecomp/include/librecomp/mods.hpp | 94 +++--- librecomp/include/librecomp/recomp.h | 293 ------------------ librecomp/src/eep.cpp | 2 +- librecomp/src/flash.cpp | 2 +- librecomp/src/mod_manifest.cpp | 4 +- librecomp/src/mods.cpp | 243 +++++++++++---- librecomp/src/pi.cpp | 2 +- librecomp/src/recomp.cpp | 4 +- .../include/ultramodern/renderer_context.hpp | 2 + 12 files changed, 244 insertions(+), 408 deletions(-) delete mode 100644 librecomp/include/librecomp/recomp.h diff --git a/N64Recomp b/N64Recomp index d5ab742..fc69604 160000 --- a/N64Recomp +++ b/N64Recomp @@ -1 +1 @@ -Subproject commit d5ab74220da3d8468d6873ed29afbcad03b890f5 +Subproject commit fc696046da3e703450559154d9370ca74c197f8b diff --git a/librecomp/CMakeLists.txt b/librecomp/CMakeLists.txt index 33c52d3..c12c651 100644 --- a/librecomp/CMakeLists.txt +++ b/librecomp/CMakeLists.txt @@ -56,5 +56,5 @@ endif() add_subdirectory(${PROJECT_SOURCE_DIR}/../thirdparty/miniz ${CMAKE_CURRENT_BINARY_DIR}/miniz) add_subdirectory(${PROJECT_SOURCE_DIR}/../N64Recomp ${CMAKE_CURRENT_BINARY_DIR}/N64Recomp EXCLUDE_FROM_ALL) -target_link_libraries(librecomp PRIVATE ultramodern N64Recomp) +target_link_libraries(librecomp PRIVATE ultramodern N64Recomp LiveRecomp) target_link_libraries(librecomp PUBLIC miniz) diff --git a/librecomp/include/librecomp/addresses.hpp b/librecomp/include/librecomp/addresses.hpp index 0918f4c..b1b50cb 100644 --- a/librecomp/include/librecomp/addresses.hpp +++ b/librecomp/include/librecomp/addresses.hpp @@ -3,7 +3,7 @@ #include #include "ultramodern/ultra64.h" -#include "librecomp/recomp.h" +#include "recomp.h" namespace recomp { // 512GB (kseg0 size) diff --git a/librecomp/include/librecomp/mods.hpp b/librecomp/include/librecomp/mods.hpp index 8a9e04f..bdf5085 100644 --- a/librecomp/include/librecomp/mods.hpp +++ b/librecomp/include/librecomp/mods.hpp @@ -19,12 +19,13 @@ #include "miniz.h" #include "miniz_zip.h" +#include "recomp.h" #include "librecomp/game.hpp" -#include "librecomp/recomp.h" #include "librecomp/sections.h" namespace N64Recomp { class Context; + struct LiveGeneratorOutput; }; namespace recomp { @@ -71,6 +72,7 @@ namespace recomp { FailedToLoadNativeCode, FailedToLoadNativeLibrary, FailedToFindNativeExport, + FailedToRecompile, InvalidReferenceSymbol, InvalidImport, InvalidCallbackEvent, @@ -258,16 +260,8 @@ namespace recomp { virtual bool good() = 0; virtual uint32_t get_api_version() = 0; virtual void set_imported_function(size_t import_index, GenericFunction func) = 0; - virtual void set_reference_symbol_pointer(size_t symbol_index, recomp_func_t* ptr) = 0; - virtual void set_base_event_index(uint32_t global_event_index) = 0; + virtual CodeModLoadError populate_reference_symbols(const N64Recomp::Context& recompiler_context, std::string& error_param) = 0; virtual uint32_t get_base_event_index() = 0; - virtual void set_recomp_trigger_event_pointer(void (*ptr)(uint8_t* rdram, recomp_context* ctx, uint32_t index)) = 0; - virtual void set_get_function_pointer(recomp_func_t* (*ptr)(int32_t)) = 0; - virtual void set_cop0_status_write_pointer(void (*ptr)(recomp_context* ctx, gpr value)) = 0; - virtual void set_cop0_status_read_pointer(gpr (*ptr)(recomp_context* ctx)) = 0; - virtual void set_switch_error_pointer(void (*ptr)(const char* func, uint32_t vram, uint32_t jtbl)) = 0; - virtual void set_do_break_pointer(void (*ptr)(uint32_t vram)) = 0; - virtual void set_reference_section_addresses_pointer(int32_t* ptr) = 0; virtual void set_local_section_address(size_t section_index, int32_t address) = 0; virtual GenericFunction get_function_handle(size_t func_index) = 0; }; @@ -293,9 +287,9 @@ namespace recomp { size_t num_exports() const; size_t num_events() const; - CodeModLoadError populate_exports(std::string& error_param); + void populate_exports(); bool get_export_function(const std::string& export_name, GenericFunction& out) const; - CodeModLoadError populate_events(size_t base_event_index, std::string& error_param); + void populate_events(); bool get_global_event_index(const std::string& event_name, size_t& event_index_out) const; CodeModLoadError load_native_library(const NativeLibraryManifest& lib_manifest, std::string& error_param); @@ -325,44 +319,29 @@ namespace recomp { // Whether this mod can be toggled at runtime. bool runtime_toggleable; }; + + struct ModCodeHandleInputs { + uint32_t base_event_index; + void (*recomp_trigger_event)(uint8_t* rdram, recomp_context* ctx, uint32_t index); + recomp_func_t* (*get_function)(int32_t vram); + void (*cop0_status_write)(recomp_context* ctx, gpr value); + gpr (*cop0_status_read)(recomp_context* ctx); + void (*switch_error)(const char* func, uint32_t vram, uint32_t jtbl); + void (*do_break)(uint32_t vram); + int32_t* reference_section_addresses; + }; - class NativeCodeHandle : public ModCodeHandle { + class DynamicLibraryCodeHandle : public ModCodeHandle { public: - NativeCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context); - ~NativeCodeHandle() = default; + DynamicLibraryCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context, const ModCodeHandleInputs& inputs); + ~DynamicLibraryCodeHandle() = default; bool good() final; uint32_t get_api_version() final; void set_imported_function(size_t import_index, GenericFunction func) final; - void set_reference_symbol_pointer(size_t symbol_index, recomp_func_t* ptr) final { - reference_symbol_funcs[symbol_index] = ptr; - }; - void set_base_event_index(uint32_t global_event_index) final { - *base_event_index = global_event_index; - }; + CodeModLoadError populate_reference_symbols(const N64Recomp::Context& context, std::string& error_param) final; uint32_t get_base_event_index() final { return *base_event_index; } - void set_recomp_trigger_event_pointer(void (*ptr)(uint8_t* rdram, recomp_context* ctx, uint32_t index)) final { - *recomp_trigger_event = ptr; - }; - void set_get_function_pointer(recomp_func_t* (*ptr)(int32_t)) final { - *get_function = ptr; - }; - void set_cop0_status_write_pointer(void (*ptr)(recomp_context* ctx, gpr value)) final { - *cop0_status_write = ptr; - } - void set_cop0_status_read_pointer(gpr (*ptr)(recomp_context* ctx)) final { - *cop0_status_read = ptr; - } - void set_switch_error_pointer(void (*ptr)(const char* func, uint32_t vram, uint32_t jtbl)) final { - *switch_error = ptr; - } - void set_do_break_pointer(void (*ptr)(uint32_t vram)) final { - *do_break = ptr; - } - void set_reference_section_addresses_pointer(int32_t* ptr) final { - *reference_section_addresses = ptr; - }; void set_local_section_address(size_t section_index, int32_t address) final { section_addresses[section_index] = address; }; @@ -387,12 +366,41 @@ namespace recomp { int32_t* section_addresses; }; + class LiveRecompilerCodeHandle : public ModCodeHandle { + public: + LiveRecompilerCodeHandle(const N64Recomp::Context& context, const ModCodeHandleInputs& inputs); + + ~LiveRecompilerCodeHandle() = default; + + // Disable copying. + LiveRecompilerCodeHandle(const LiveRecompilerCodeHandle& rhs) = delete; + LiveRecompilerCodeHandle& operator=(const LiveRecompilerCodeHandle& rhs) = delete; + + bool good() final { return is_good; } + uint32_t get_api_version() final { return 1; } + void set_imported_function(size_t import_index, GenericFunction func) final; + CodeModLoadError populate_reference_symbols(const N64Recomp::Context& context, std::string& error_param) final; + uint32_t get_base_event_index() final { + return base_event_index; + } + void set_local_section_address(size_t section_index, int32_t address) final { + section_addresses[section_index] = address; + } + GenericFunction get_function_handle(size_t func_index) final; + private: + uint32_t base_event_index; + std::unique_ptr recompiler_output; + void set_bad(); + bool is_good = false; + std::unique_ptr section_addresses; + }; + void setup_events(size_t num_events); void register_event_callback(size_t event_index, GenericFunction callback); void reset_events(); CodeModLoadError validate_api_version(uint32_t api_version, std::string& error_param); - + void initialize_mod_recompiler(); void scan_mods(); void enable_mod(const std::string& mod_id, bool enabled); bool is_mod_enabled(const std::string& mod_id); diff --git a/librecomp/include/librecomp/recomp.h b/librecomp/include/librecomp/recomp.h deleted file mode 100644 index 39b0d8f..0000000 --- a/librecomp/include/librecomp/recomp.h +++ /dev/null @@ -1,293 +0,0 @@ -#ifndef __RECOMP_H__ -#define __RECOMP_H__ - -#include -#include -#include -#include - -// Compiler definition to disable inter-procedural optimization, allowing multiple functions to be in a single file without breaking interposition. -#if defined(_MSC_VER) && !defined(__clang__) - // MSVC's __declspec(noinline) seems to disable inter-procedural optimization entirely, so it's all that's needed. - #define RECOMP_FUNC __declspec(noinline) -#elif defined(__clang__) - // Clang has no dedicated IPO attribute, so we use a combination of other attributes to give the desired behavior. - // The inline keyword allows multiple definitions during linking, and extern forces clang to emit an externally visible definition. - // Weak forces Clang to not perform any IPO as the symbol can be interposed, which prevents actual inlining due to the inline keyword. - // Add noinline on for good measure, which doesn't conflict with the inline keyword as they have different meanings. - #define RECOMP_FUNC extern inline __attribute__((weak,noinline)) -#elif defined(__GNUC__) - // Use GCC's attribute for disabling inter-procedural optimizations. - #define RECOMP_FUNC __attribute__((noipa)) -#else - #error "No RECOMP_FUNC definition for this compiler" -#endif - -typedef uint64_t gpr; - -#define SIGNED(val) \ - ((int64_t)(val)) - -#define ADD32(a, b) \ - ((gpr)(int32_t)((a) + (b))) - -#define SUB32(a, b) \ - ((gpr)(int32_t)((a) - (b))) - -#define MEM_W(offset, reg) \ - (*(int32_t*)(rdram + ((((reg) + (offset))) - 0xFFFFFFFF80000000))) - -#define MEM_H(offset, reg) \ - (*(int16_t*)(rdram + ((((reg) + (offset)) ^ 2) - 0xFFFFFFFF80000000))) - -#define MEM_B(offset, reg) \ - (*(int8_t*)(rdram + ((((reg) + (offset)) ^ 3) - 0xFFFFFFFF80000000))) - -#define MEM_HU(offset, reg) \ - (*(uint16_t*)(rdram + ((((reg) + (offset)) ^ 2) - 0xFFFFFFFF80000000))) - -#define MEM_BU(offset, reg) \ - (*(uint8_t*)(rdram + ((((reg) + (offset)) ^ 3) - 0xFFFFFFFF80000000))) - -#define SD(val, offset, reg) { \ - *(uint32_t*)(rdram + ((((reg) + (offset) + 4)) - 0xFFFFFFFF80000000)) = (uint32_t)((gpr)(val) >> 0); \ - *(uint32_t*)(rdram + ((((reg) + (offset) + 0)) - 0xFFFFFFFF80000000)) = (uint32_t)((gpr)(val) >> 32); \ -} - -static inline uint64_t load_doubleword(uint8_t* rdram, gpr reg, gpr offset) { - uint64_t ret = 0; - uint64_t lo = (uint64_t)(uint32_t)MEM_W(reg, offset + 4); - uint64_t hi = (uint64_t)(uint32_t)MEM_W(reg, offset + 0); - ret = (lo << 0) | (hi << 32); - return ret; -} - -#define LD(offset, reg) \ - load_doubleword(rdram, offset, reg) - -static inline gpr do_lwl(uint8_t* rdram, gpr initial_value, gpr offset, gpr reg) { - // Calculate the overall address - gpr address = (offset + reg); - - // Load the aligned word - gpr word_address = address & ~0x3; - uint32_t loaded_value = MEM_W(0, word_address); - - // Mask the existing value and shift the loaded value appropriately - gpr misalignment = address & 0x3; - gpr masked_value = initial_value & ~(0xFFFFFFFFu << (misalignment * 8)); - loaded_value <<= (misalignment * 8); - - // Cast to int32_t to sign extend first - return (gpr)(int32_t)(masked_value | loaded_value); -} - -static inline gpr do_lwr(uint8_t* rdram, gpr initial_value, gpr offset, gpr reg) { - // Calculate the overall address - gpr address = (offset + reg); - - // Load the aligned word - gpr word_address = address & ~0x3; - uint32_t loaded_value = MEM_W(0, word_address); - - // Mask the existing value and shift the loaded value appropriately - gpr misalignment = address & 0x3; - gpr masked_value = initial_value & ~(0xFFFFFFFFu >> (24 - misalignment * 8)); - loaded_value >>= (24 - misalignment * 8); - - // Cast to int32_t to sign extend first - return (gpr)(int32_t)(masked_value | loaded_value); -} - -static inline void do_swl(uint8_t* rdram, gpr offset, gpr reg, gpr val) { - // Calculate the overall address - gpr address = (offset + reg); - - // Get the initial value of the aligned word - gpr word_address = address & ~0x3; - uint32_t initial_value = MEM_W(0, word_address); - - // Mask the initial value and shift the input value appropriately - gpr misalignment = address & 0x3; - uint32_t masked_initial_value = initial_value & ~(0xFFFFFFFFu >> (misalignment * 8)); - uint32_t shifted_input_value = ((uint32_t)val) >> (misalignment * 8); - MEM_W(0, word_address) = masked_initial_value | shifted_input_value; -} - -static inline void do_swr(uint8_t* rdram, gpr offset, gpr reg, gpr val) { - // Calculate the overall address - gpr address = (offset + reg); - - // Get the initial value of the aligned word - gpr word_address = address & ~0x3; - uint32_t initial_value = MEM_W(0, word_address); - - // Mask the initial value and shift the input value appropriately - gpr misalignment = address & 0x3; - uint32_t masked_initial_value = initial_value & ~(0xFFFFFFFFu << (24 - misalignment * 8)); - uint32_t shifted_input_value = ((uint32_t)val) << (24 - misalignment * 8); - MEM_W(0, word_address) = masked_initial_value | shifted_input_value; -} - -#define S32(val) \ - ((int32_t)(val)) - -#define U32(val) \ - ((uint32_t)(val)) - -#define S64(val) \ - ((int64_t)(val)) - -#define U64(val) \ - ((uint64_t)(val)) - -#define MUL_S(val1, val2) \ - ((val1) * (val2)) - -#define MUL_D(val1, val2) \ - ((val1) * (val2)) - -#define DIV_S(val1, val2) \ - ((val1) / (val2)) - -#define DIV_D(val1, val2) \ - ((val1) / (val2)) - -#define CVT_S_W(val) \ - ((float)((int32_t)(val))) - -#define CVT_D_W(val) \ - ((double)((int32_t)(val))) - -#define CVT_D_S(val) \ - ((double)(val)) - -#define CVT_S_D(val) \ - ((float)(val)) - -#define TRUNC_W_S(val) \ - ((int32_t)(val)) - -#define TRUNC_W_D(val) \ - ((int32_t)(val)) - -#define TRUNC_L_S(val) \ - ((int64_t)(val)) - -#define TRUNC_L_D(val) \ - ((int64_t)(val)) - -#define DEFAULT_ROUNDING_MODE 0 - -static inline int32_t do_cvt_w_s(float val, unsigned int rounding_mode) { - switch (rounding_mode) { - case 0: // round to nearest value - return (int32_t)lroundf(val); - case 1: // round to zero (truncate) - return (int32_t)val; - case 2: // round to positive infinity (ceil) - return (int32_t)ceilf(val); - case 3: // round to negative infinity (floor) - return (int32_t)floorf(val); - } - assert(0); - return 0; -} - -#define CVT_W_S(val) \ - do_cvt_w_s(val, rounding_mode) - -static inline int32_t do_cvt_w_d(double val, unsigned int rounding_mode) { - switch (rounding_mode) { - case 0: // round to nearest value - return (int32_t)lround(val); - case 1: // round to zero (truncate) - return (int32_t)val; - case 2: // round to positive infinity (ceil) - return (int32_t)ceil(val); - case 3: // round to negative infinity (floor) - return (int32_t)floor(val); - } - assert(0); - return 0; -} - -#define CVT_W_D(val) \ - do_cvt_w_d(val, rounding_mode) - -#define NAN_CHECK(val) \ - assert(val == val) - -//#define NAN_CHECK(val) - -typedef union { - double d; - struct { - float fl; - float fh; - }; - struct { - uint32_t u32l; - uint32_t u32h; - }; - uint64_t u64; -} fpr; - -typedef struct { - gpr r0, r1, r2, r3, r4, r5, r6, r7, - r8, r9, r10, r11, r12, r13, r14, r15, - r16, r17, r18, r19, r20, r21, r22, r23, - r24, r25, r26, r27, r28, r29, r30, r31; - fpr f0, f1, f2, f3, f4, f5, f6, f7, - f8, f9, f10, f11, f12, f13, f14, f15, - f16, f17, f18, f19, f20, f21, f22, f23, - f24, f25, f26, f27, f28, f29, f30, f31; - uint64_t hi, lo; - uint32_t* f_odd; - uint32_t status_reg; - uint8_t mips3_float_mode; -} recomp_context; - -// Checks if the target is an even float register or that mips3 float mode is enabled -#define CHECK_FR(ctx, idx) \ - assert(((idx) & 1) == 0 || (ctx)->mips3_float_mode) - -#ifdef __cplusplus -extern "C" { -#endif - -void cop0_status_write(recomp_context* ctx, gpr value); -gpr cop0_status_read(recomp_context* ctx); -void switch_error(const char* func, uint32_t vram, uint32_t jtbl); -void do_break(uint32_t vram); - -typedef void (recomp_func_t)(uint8_t* rdram, recomp_context* ctx); - -recomp_func_t* get_function(int32_t vram); - -#define LOOKUP_FUNC(val) \ - get_function((int32_t)(val)) - -extern int32_t* section_addresses; - -#define LO16(x) \ - ((x) & 0xFFFF) - -#define HI16(x) \ - (((x) >> 16) + (((x) >> 15) & 1)) - -#define RELOC_HI16(section_index, offset) \ - HI16(section_addresses[section_index] + (offset)) - -#define RELOC_LO16(section_index, offset) \ - LO16(section_addresses[section_index] + (offset)) - -void recomp_syscall_handler(uint8_t* rdram, recomp_context* ctx, int32_t instruction_vram); - -void pause_self(uint8_t *rdram); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/librecomp/src/eep.cpp b/librecomp/src/eep.cpp index 7a11b44..048246e 100644 --- a/librecomp/src/eep.cpp +++ b/librecomp/src/eep.cpp @@ -1,4 +1,4 @@ -#include "librecomp/recomp.h" +#include "recomp.h" #include "librecomp/game.hpp" #include "ultramodern/ultra64.h" diff --git a/librecomp/src/flash.cpp b/librecomp/src/flash.cpp index af3c1a7..b9fd333 100644 --- a/librecomp/src/flash.cpp +++ b/librecomp/src/flash.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "librecomp/recomp.h" +#include "recomp.h" #include "librecomp/addresses.hpp" #include "librecomp/game.hpp" diff --git a/librecomp/src/mod_manifest.cpp b/librecomp/src/mod_manifest.cpp index ba3073f..477ec52 100644 --- a/librecomp/src/mod_manifest.cpp +++ b/librecomp/src/mod_manifest.cpp @@ -2,7 +2,7 @@ #include "json/json.hpp" -#include "n64recomp.h" +#include "recompiler/context.h" #include "librecomp/mods.hpp" recomp::mods::ZipModFileHandle::~ZipModFileHandle() { @@ -571,6 +571,8 @@ std::string recomp::mods::error_to_string(CodeModLoadError error) { return "Failed to load mod library"; case CodeModLoadError::FailedToFindNativeExport: return "Failed to find native export"; + case CodeModLoadError::FailedToRecompile: + return "Failed to recompile mod"; case CodeModLoadError::InvalidReferenceSymbol: return "Reference symbol does not exist"; case CodeModLoadError::InvalidImport: diff --git a/librecomp/src/mods.cpp b/librecomp/src/mods.cpp index 9b48e71..acc458a 100644 --- a/librecomp/src/mods.cpp +++ b/librecomp/src/mods.cpp @@ -6,7 +6,8 @@ #include "librecomp/mods.hpp" #include "librecomp/overlays.hpp" #include "librecomp/game.hpp" -#include "n64recomp.h" +#include "recompiler/context.h" +#include "recompiler/live_recompiler.h" // Architecture detection. @@ -240,13 +241,11 @@ size_t recomp::mods::ModHandle::num_events() const { return recompiler_context->event_symbols.size(); } -recomp::mods::CodeModLoadError recomp::mods::ModHandle::populate_exports(std::string& error_param) { +void recomp::mods::ModHandle::populate_exports() { for (size_t func_index : recompiler_context->exported_funcs) { const auto& func_handle = recompiler_context->functions[func_index]; exports_by_name.emplace(func_handle.name, func_index); } - - return CodeModLoadError::Good; } recomp::mods::CodeModLoadError recomp::mods::ModHandle::load_native_library(const recomp::mods::NativeLibraryManifest& lib_manifest, std::string& error_param) { @@ -309,14 +308,11 @@ bool recomp::mods::ModHandle::get_export_function(const std::string& export_name return false; } -recomp::mods::CodeModLoadError recomp::mods::ModHandle::populate_events(size_t base_event_index, std::string& error_param) { +void recomp::mods::ModHandle::populate_events() { for (size_t event_index = 0; event_index < recompiler_context->event_symbols.size(); event_index++) { const N64Recomp::EventSymbol& event = recompiler_context->event_symbols[event_index]; events_by_name.emplace(event.base.name, event_index); } - - code_handle->set_base_event_index(base_event_index); - return CodeModLoadError::Good; } bool recomp::mods::ModHandle::get_global_event_index(const std::string& event_name, size_t& event_index_out) const { @@ -329,7 +325,7 @@ bool recomp::mods::ModHandle::get_global_event_index(const std::string& event_na return true; } -recomp::mods::NativeCodeHandle::NativeCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context) { +recomp::mods::DynamicLibraryCodeHandle::DynamicLibraryCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context, const ModCodeHandleInputs& inputs) { is_good = true; // Load the DLL. dynamic_lib = std::make_unique(dll_path); @@ -366,22 +362,33 @@ recomp::mods::NativeCodeHandle::NativeCodeHandle(const std::filesystem::path& dl is_good &= dynamic_lib->get_dll_symbol(do_break, "do_break"); is_good &= dynamic_lib->get_dll_symbol(reference_section_addresses, "reference_section_addresses"); is_good &= dynamic_lib->get_dll_symbol(section_addresses, "section_addresses"); + + if (is_good) { + *base_event_index = inputs.base_event_index; + *recomp_trigger_event = inputs.recomp_trigger_event; + *get_function = inputs.get_function; + *cop0_status_write = inputs.cop0_status_write; + *cop0_status_read = inputs.cop0_status_read; + *switch_error = inputs.switch_error; + *do_break = inputs.do_break; + *reference_section_addresses = inputs.reference_section_addresses; + } } -bool recomp::mods::NativeCodeHandle::good() { +bool recomp::mods::DynamicLibraryCodeHandle::good() { return dynamic_lib->good() && is_good; } -uint32_t recomp::mods::NativeCodeHandle::get_api_version() { +uint32_t recomp::mods::DynamicLibraryCodeHandle::get_api_version() { return dynamic_lib->get_api_version(); } -void recomp::mods::NativeCodeHandle::set_bad() { +void recomp::mods::DynamicLibraryCodeHandle::set_bad() { dynamic_lib.reset(); is_good = false; } -void recomp::mods::NativeCodeHandle::set_imported_function(size_t import_index, GenericFunction func) { +void recomp::mods::DynamicLibraryCodeHandle::set_imported_function(size_t import_index, GenericFunction func) { std::visit(overloaded { [this, import_index](recomp_func_t* native_func) { imported_funcs[import_index] = native_func; @@ -389,6 +396,95 @@ void recomp::mods::NativeCodeHandle::set_imported_function(size_t import_index, }, func); } +recomp::mods::CodeModLoadError recomp::mods::DynamicLibraryCodeHandle::populate_reference_symbols(const N64Recomp::Context& context, std::string& error_param) { + size_t reference_symbol_index = 0; + for (const auto& section : context.sections) { + for (const auto& reloc : section.relocs) { + if (reloc.type == N64Recomp::RelocType::R_MIPS_26 && reloc.reference_symbol && context.is_regular_reference_section(reloc.target_section)) { + recomp_func_t* cur_func = recomp::overlays::get_func_by_section_index_function_offset(reloc.target_section, reloc.target_section_offset); + if (cur_func == nullptr) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: " << reloc.target_section << + " func offset: 0x" << reloc.target_section_offset; + error_param = error_param_stream.str(); + return CodeModLoadError::InvalidReferenceSymbol; + } + reference_symbol_funcs[reference_symbol_index] = cur_func; + reference_symbol_index++; + } + } + } + return CodeModLoadError::Good; +} + +recomp::mods::LiveRecompilerCodeHandle::LiveRecompilerCodeHandle(const N64Recomp::Context& context, const ModCodeHandleInputs& inputs) { + section_addresses = std::make_unique(context.sections.size()); + base_event_index = inputs.base_event_index; + + N64Recomp::LiveGeneratorInputs recompiler_inputs{ + .base_event_index = inputs.base_event_index, + .cop0_status_write = inputs.cop0_status_write, + .cop0_status_read = inputs.cop0_status_read, + .switch_error = inputs.switch_error, + .do_break = inputs.do_break, + .get_function = inputs.get_function, + .syscall_handler = nullptr, // TODO hook this up + .pause_self = pause_self, + .trigger_event = inputs.recomp_trigger_event, + .reference_section_addresses = inputs.reference_section_addresses, + .local_section_addresses = section_addresses.get(), + }; + + N64Recomp::LiveGenerator generator{ context.functions.size(), recompiler_inputs }; + std::vector> dummy_static_funcs{}; + + for (size_t func_index = 0; func_index < context.functions.size(); func_index++) { + std::ostringstream dummy_ostream{}; + + if (!N64Recomp::recompile_function_live(generator, context, func_index, dummy_ostream, dummy_static_funcs, true)) { + is_good = false; + break; + } + } + + // Generate the code. + recompiler_output = std::make_unique(generator.finish()); + is_good = recompiler_output->good; +} + +void recomp::mods::LiveRecompilerCodeHandle::set_imported_function(size_t import_index, GenericFunction func) { + std::visit(overloaded { + [this, import_index](recomp_func_t* native_func) { + recompiler_output->populate_import_symbol_jumps(import_index, native_func); + } + }, func); +} + +recomp::mods::CodeModLoadError recomp::mods::LiveRecompilerCodeHandle::populate_reference_symbols(const N64Recomp::Context& context, std::string& error_param) { + size_t num_reference_jumps = recompiler_output->num_reference_symbol_jumps(); + for (size_t jump_index = 0; jump_index < num_reference_jumps; jump_index++) { + N64Recomp::ReferenceJumpDetails jump_details = recompiler_output->get_reference_symbol_jump_details(jump_index); + + recomp_func_t* cur_func = recomp::overlays::get_func_by_section_index_function_offset(jump_details.section, jump_details.section_offset); + if (cur_func == nullptr) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: " << jump_details.section << + " func offset: 0x" << jump_details.section_offset; + error_param = error_param_stream.str(); + return CodeModLoadError::InvalidReferenceSymbol; + } + + recompiler_output->set_reference_symbol_jump(jump_index, cur_func); + } + return CodeModLoadError::Good; +} + +recomp::mods::GenericFunction recomp::mods::LiveRecompilerCodeHandle::get_function_handle(size_t func_index) { + return GenericFunction{ recompiler_output->functions[func_index] }; +} + void patch_func(recomp_func_t* target_func, recomp::mods::GenericFunction replacement_func) { uint8_t* target_func_u8 = reinterpret_cast(target_func); size_t offset = 0; @@ -655,6 +751,7 @@ std::vector recomp::mods::ModContext::load_mo std::vector ret{}; ram_used = 0; num_events = recomp::overlays::num_base_events(); + loaded_code_mods.clear(); auto find_index_it = mod_game_ids.find(mod_game_id); if (find_index_it == mod_game_ids.end()) { @@ -722,7 +819,12 @@ std::vector recomp::mods::ModContext::load_mo std::string cur_error_param; CodeModLoadError cur_error = load_mod_code(rdram, section_vrom_map, mod, load_address, cur_ram_used, cur_error_param); if (cur_error != CodeModLoadError::Good) { - ret.emplace_back(mod.manifest.mod_id, ModLoadError::FailedToLoadCode, error_to_string(cur_error) + ":" + cur_error_param); + if (cur_error_param.empty()) { + ret.emplace_back(mod.manifest.mod_id, ModLoadError::FailedToLoadCode, error_to_string(cur_error)); + } + else { + ret.emplace_back(mod.manifest.mod_id, ModLoadError::FailedToLoadCode, error_to_string(cur_error) + ":" + cur_error_param); + } } else { load_address += cur_ram_used; @@ -745,7 +847,12 @@ std::vector recomp::mods::ModContext::load_mo std::string cur_error_param; CodeModLoadError cur_error = resolve_code_dependencies(mod, cur_error_param); if (cur_error != CodeModLoadError::Good) { - ret.emplace_back(mod.manifest.mod_id, ModLoadError::FailedToLoadCode, error_to_string(cur_error) + ":" + cur_error_param); + if (cur_error_param.empty()) { + ret.emplace_back(mod.manifest.mod_id, ModLoadError::FailedToLoadCode, error_to_string(cur_error)); + } + else { + ret.emplace_back(mod.manifest.mod_id, ModLoadError::FailedToLoadCode, error_to_string(cur_error) + ":" + cur_error_param); + } } } @@ -814,6 +921,19 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::load_mod_code(uint8_t* return CodeModLoadError::FailedToParseSyms; } + // Set all reference sections as relocatable, since the only relocations present in a mod's context + // are ones that target relocatable sections. + mod.recompiler_context->set_all_reference_sections_relocatable(); + // Disable validation of reference symbols (so we can skip populating them). Validation will still happen + // later on in the live recompilation process. + mod.recompiler_context->skip_validating_reference_symbols = true; + + // Populate the mod's export map. + mod.populate_exports(); + + // Populate the mod's event map and set its base event index. + mod.populate_events(); + // Validate that the dependencies present in the symbol file are all present in the mod's manifest as well. for (const auto& [cur_dep_id, cur_dep_index] : mod.recompiler_context->dependencies_by_name) { // Handle special dependency names. @@ -875,14 +995,33 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::load_mod_code(uint8_t* ram_used = cur_section_addr - load_address; - // TODO implement LuaJIT recompilation and allow it instead of native code loading via a mod manifest flag. - std::string cur_error_param; CodeModLoadError cur_error; - if (1) { + ModCodeHandleInputs handle_inputs{ + .base_event_index = static_cast(num_events), + .recomp_trigger_event = recomp_trigger_event, + .get_function = get_function, + .cop0_status_write = cop0_status_write, + .cop0_status_read = cop0_status_read, + .switch_error = switch_error, + .do_break = do_break, + .reference_section_addresses = section_addresses, + }; + + // Allocate the event indices used by the mod. + num_events += mod.num_events(); + + // Copy the mod's binary into the recompiler context so it can be analyzed during code loading. + // TODO move it instead, right now the move can't be done because of a signedness difference in the types. + mod.recompiler_context->rom.assign(binary_span.begin(), binary_span.end()); + + // Use a dynamic library code handle. This feature isn't meant to be used by end users, but provides a more debuggable + // experience than the live recompiler for mod developers. + // Enabled if the mod's filename ends with ".offline.dll". + if (mod.manifest.mod_root_path.filename().string().ends_with(".offline.nrm")) { std::filesystem::path dll_path = mod.manifest.mod_root_path; dll_path.replace_extension(DynamicLibrary::PlatformExtension); - mod.code_handle = std::make_unique(dll_path, *mod.recompiler_context); + mod.code_handle = std::make_unique(dll_path, *mod.recompiler_context, handle_inputs); if (!mod.code_handle->good()) { mod.code_handle.reset(); error_param = dll_path.string(); @@ -901,13 +1040,15 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::load_mod_code(uint8_t* return cur_error; } } - - // Populate the mod's export map. - cur_error = mod.populate_exports(cur_error_param); - - if (cur_error != CodeModLoadError::Good) { - error_param = std::move(cur_error_param); - return cur_error; + // Live recompiler code handle. + else { + mod.code_handle = std::make_unique(*mod.recompiler_context, handle_inputs); + + if (!mod.code_handle->good()) { + mod.code_handle.reset(); + error_param = {}; + return CodeModLoadError::FailedToRecompile; + } } // Load any native libraries specified by the mod and validate/register the expors. @@ -920,17 +1061,6 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::load_mod_code(uint8_t* } } - // Populate the mod's event map and set its base event index. - cur_error = mod.populate_events(num_events, cur_error_param); - - if (cur_error != CodeModLoadError::Good) { - error_param = std::move(cur_error_param); - return cur_error; - } - - // Allocate the event indices used by the mod. - num_events += mod.num_events(); - // Add each function from the mod into the function lookup table. for (size_t func_index = 0; func_index < mod.recompiler_context->functions.size(); func_index++) { const auto& func = mod.recompiler_context->functions[func_index]; @@ -953,25 +1083,13 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::load_mod_code(uint8_t* } recomp::mods::CodeModLoadError recomp::mods::ModContext::resolve_code_dependencies(recomp::mods::ModHandle& mod, std::string& error_param) { - // Reference symbols from the base recomp.1:1 with relocs for offline mods. - // TODO this won't be needed for LuaJIT recompilation, so move this logic into the code handle. - size_t reference_symbol_index = 0; - for (const auto& section : mod.recompiler_context->sections) { - for (const auto& reloc : section.relocs) { - if (reloc.type == N64Recomp::RelocType::R_MIPS_26 && reloc.reference_symbol && mod.recompiler_context->is_regular_reference_section(reloc.target_section)) { - recomp_func_t* cur_func = recomp::overlays::get_func_by_section_index_function_offset(reloc.target_section, reloc.target_section_offset); - if (cur_func == nullptr) { - std::stringstream error_param_stream{}; - error_param_stream << std::hex << - "section: " << reloc.target_section << - " func offset: 0x" << reloc.target_section_offset; - error_param = error_param_stream.str(); - return CodeModLoadError::InvalidReferenceSymbol; - } - mod.code_handle->set_reference_symbol_pointer(reference_symbol_index, cur_func); - reference_symbol_index++; - } - } + // Reference symbols. + std::string reference_syms_error_param{}; + CodeModLoadError reference_syms_error = mod.code_handle->populate_reference_symbols(*mod.recompiler_context, reference_syms_error_param); + + if (reference_syms_error != CodeModLoadError::Good) { + error_param = std::move(reference_syms_error_param); + return reference_syms_error; } // Create a list of dependencies ordered by their index in the recompiler context. @@ -1055,14 +1173,7 @@ recomp::mods::CodeModLoadError recomp::mods::ModContext::resolve_code_dependenci recomp::mods::register_event_callback(event_index, func); } - // Populate the mod's state fields. - mod.code_handle->set_recomp_trigger_event_pointer(recomp_trigger_event); - mod.code_handle->set_get_function_pointer(get_function); - mod.code_handle->set_cop0_status_write_pointer(cop0_status_write); - mod.code_handle->set_cop0_status_read_pointer(cop0_status_read); - mod.code_handle->set_switch_error_pointer(switch_error); - mod.code_handle->set_do_break_pointer(do_break); - mod.code_handle->set_reference_section_addresses_pointer(section_addresses); + // Populate the relocated section addresses for the mod. for (size_t section_index = 0; section_index < mod.section_load_addresses.size(); section_index++) { mod.code_handle->set_local_section_address(section_index, mod.section_load_addresses[section_index]); } @@ -1109,3 +1220,7 @@ void recomp::mods::ModContext::unload_mods() { num_events = recomp::overlays::num_base_events(); active_game = (size_t)-1; } + +void recomp::mods::initialize_mod_recompiler() { + N64Recomp::live_recompiler_init(); +} diff --git a/librecomp/src/pi.cpp b/librecomp/src/pi.cpp index f7f0705..a3ebfd4 100644 --- a/librecomp/src/pi.cpp +++ b/librecomp/src/pi.cpp @@ -4,7 +4,7 @@ #include #include #include -#include "librecomp/recomp.h" +#include "recomp.h" #include "librecomp/addresses.hpp" #include "librecomp/game.hpp" #include "librecomp/files.hpp" diff --git a/librecomp/src/recomp.cpp b/librecomp/src/recomp.cpp index 499749a..1c173da 100644 --- a/librecomp/src/recomp.cpp +++ b/librecomp/src/recomp.cpp @@ -15,7 +15,7 @@ #include #include -#include "librecomp/recomp.h" +#include "recomp.h" #include "librecomp/overlays.hpp" #include "librecomp/game.hpp" #include "xxHash/xxh3.h" @@ -632,6 +632,8 @@ void recomp::start( } } + recomp::mods::initialize_mod_recompiler(); + // Allocate rdram without comitting it. Use a platform-specific virtual allocation function // that initializes to zero. Protect the region above the memory size to catch accesses to invalid addresses. uint8_t* rdram; diff --git a/ultramodern/include/ultramodern/renderer_context.hpp b/ultramodern/include/ultramodern/renderer_context.hpp index 21f377e..8d13d75 100644 --- a/ultramodern/include/ultramodern/renderer_context.hpp +++ b/ultramodern/include/ultramodern/renderer_context.hpp @@ -18,6 +18,8 @@ # undef LockMask # undef Always # undef Success +# undef False +# undef True #endif #include "ultra64.h"