diff --git a/.gitmodules b/.gitmodules index 84f5a89..27fa707 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,9 @@ [submodule "thirdparty/xxHash"] path = thirdparty/xxHash url = https://github.com/Cyan4973/xxHash.git +[submodule "thirdparty/miniz"] + path = thirdparty/miniz + url = https://github.com/richgel999/miniz +[submodule "N64Recomp"] + path = N64Recomp + url = https://github.com/N64Recomp/N64Recomp diff --git a/N64Recomp b/N64Recomp new file mode 160000 index 0000000..cc71b31 --- /dev/null +++ b/N64Recomp @@ -0,0 +1 @@ +Subproject commit cc71b31b09a927f558e142598ffcca7d146b454b diff --git a/librecomp/CMakeLists.txt b/librecomp/CMakeLists.txt index 7318386..fb8c884 100644 --- a/librecomp/CMakeLists.txt +++ b/librecomp/CMakeLists.txt @@ -15,6 +15,9 @@ add_library(librecomp STATIC "${CMAKE_CURRENT_SOURCE_DIR}/src/files.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/flash.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/math_routines.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mods.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mod_events.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mod_manifest.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/overlays.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/pak.cpp" "${CMAKE_CURRENT_SOURCE_DIR}/src/pi.cpp" @@ -46,4 +49,8 @@ if (WIN32) add_compile_definitions(NOMINMAX) endif() -target_link_libraries(librecomp PRIVATE ultramodern) +add_subdirectory(${PROJECT_SOURCE_DIR}/../thirdparty/miniz ${CMAKE_CURRENT_BINARY_DIR}/miniz) +add_subdirectory(${PROJECT_SOURCE_DIR}/../N64Recomp ${CMAKE_CURRENT_BINARY_DIR}/N64Recomp EXCLUDE_FROM_ALL) + +target_link_libraries(librecomp PRIVATE ultramodern N64Recomp) +target_link_libraries(librecomp PUBLIC miniz) diff --git a/librecomp/include/librecomp/addresses.hpp b/librecomp/include/librecomp/addresses.hpp new file mode 100644 index 0000000..dc3bbcd --- /dev/null +++ b/librecomp/include/librecomp/addresses.hpp @@ -0,0 +1,25 @@ +#ifndef __RECOMP_ADDRESSES_HPP__ +#define __RECOMP_ADDRESSES_HPP__ + +#include +#include "ultramodern/ultra64.h" + +namespace recomp { + // We need a place in rdram to hold the PI handles, so pick an address in extended rdram + constexpr int32_t cart_handle = 0x80800000; + constexpr int32_t drive_handle = (int32_t)(cart_handle + sizeof(OSPiHandle)); + constexpr int32_t flash_handle = (int32_t)(drive_handle + sizeof(OSPiHandle)); + constexpr int32_t flash_handle_end = (int32_t)(flash_handle + sizeof(OSPiHandle)); + constexpr int32_t patch_rdram_start = 0x80801000; + static_assert(patch_rdram_start >= flash_handle_end); + constexpr int32_t mod_rdram_start = 0x81000000; + + // Flashram occupies the same physical address as sram, but that issue is avoided because libultra exposes + // a high-level interface for flashram. Because that high-level interface is reimplemented, low level accesses + // that involve physical addresses don't need to be handled for flashram. + constexpr uint32_t sram_base = 0x08000000; + constexpr uint32_t rom_base = 0x10000000; + constexpr uint32_t drive_base = 0x06000000; +} + +#endif diff --git a/librecomp/include/librecomp/game.hpp b/librecomp/include/librecomp/game.hpp index 4c2b304..8712fc6 100644 --- a/librecomp/include/librecomp/game.hpp +++ b/librecomp/include/librecomp/game.hpp @@ -13,6 +13,7 @@ namespace recomp { uint64_t rom_hash; std::string internal_name; std::u8string game_id; + std::string mod_game_id; std::span cache_data; bool is_enabled; @@ -21,34 +22,59 @@ namespace recomp { std::u8string stored_filename() const; }; - enum class RomValidationError { - Good, - FailedToOpen, - NotARom, - IncorrectRom, - NotYet, - IncorrectVersion, - OtherError - }; - void register_config_path(std::filesystem::path path); - bool register_game(const recomp::GameEntry& entry); - void check_all_stored_roms(); - bool load_stored_rom(std::u8string& game_id); - RomValidationError select_rom(const std::filesystem::path& rom_path, std::u8string& game_id); - bool is_rom_valid(std::u8string& game_id); - bool is_rom_loaded(); - void set_rom_contents(std::vector&& new_rom); - void do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr, size_t num_bytes); - void do_rom_pio(uint8_t* rdram, gpr ram_address, uint32_t physical_addr); + struct Version { + int major = -1; + int minor = -1; + int patch = -1; + std::string suffix; - /** - * The following arguments contain mandatory callbacks that need to be registered (i.e., can't be `nullptr`): - * - `rsp_callbacks` - * - `renderer_callbacks` - * - * It must be called only once and it must be called before `ultramodern::preinit`. - */ + std::string to_string() const { + return std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(patch) + suffix; + } + + static bool from_string(const std::string& str, Version& out); + + auto operator<=>(const Version& rhs) const { + if (major != rhs.major) { + return major <=> rhs.major; + } + if (minor != rhs.minor) { + return minor <=> rhs.minor; + } + return patch <=> rhs.patch; + } + }; + enum class RomValidationError { + Good, + FailedToOpen, + NotARom, + IncorrectRom, + NotYet, + IncorrectVersion, + OtherError + }; + void register_config_path(std::filesystem::path path); + bool register_game(const recomp::GameEntry& entry); + void check_all_stored_roms(); + bool load_stored_rom(std::u8string& game_id); + RomValidationError select_rom(const std::filesystem::path& rom_path, std::u8string& game_id); + bool is_rom_valid(std::u8string& game_id); + bool is_rom_loaded(); + void set_rom_contents(std::vector&& new_rom); + void do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr, size_t num_bytes); + void do_rom_pio(uint8_t* rdram, gpr ram_address, uint32_t physical_addr); + const Version& get_project_version(); + + /** + * The following arguments contain mandatory callbacks that need to be registered (i.e., can't be `nullptr`): + * - `rsp_callbacks` + * - `renderer_callbacks` + * + * It must be called only once and it must be called before `ultramodern::preinit`. + */ void start( + uint32_t rdram_size, + const Version& project_version, ultramodern::renderer::WindowHandle window_handle, const recomp::rsp::callbacks_t& rsp_callbacks, const ultramodern::renderer::callbacks_t& renderer_callbacks, @@ -60,8 +86,8 @@ namespace recomp { const ultramodern::threads::callbacks_t& threads_callbacks ); - void start_game(const std::u8string& game_id); - std::u8string current_game_id(); + void start_game(const std::u8string& game_id); + std::u8string current_game_id(); } #endif diff --git a/librecomp/include/librecomp/mods.hpp b/librecomp/include/librecomp/mods.hpp new file mode 100644 index 0000000..1c438c2 --- /dev/null +++ b/librecomp/include/librecomp/mods.hpp @@ -0,0 +1,316 @@ +#ifndef __RECOMP_MODS_HPP__ +#define __RECOMP_MODS_HPP__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MINIZ_NO_DEFLATE_APIS +#define MINIZ_NO_ARCHIVE_WRITING_APIS +#include "miniz.h" +#include "miniz_zip.h" + +#include "librecomp/game.hpp" +#include "librecomp/recomp.h" +#include "librecomp/sections.h" + +namespace N64Recomp { + class Context; +}; + +namespace recomp { + namespace mods { + enum class ModOpenError { + Good, + DoesNotExist, + NotAFileOrFolder, + FileError, + InvalidZip, + NoManifest, + FailedToParseManifest, + InvalidManifestSchema, + UnrecognizedManifestField, + IncorrectManifestFieldType, + InvalidVersionString, + InvalidMinimumRecompVersionString, + InvalidDependencyString, + MissingManifestField, + DuplicateMod, + WrongGame + }; + + std::string error_to_string(ModOpenError); + + enum class ModLoadError { + Good, + InvalidGame, + MinimumRecompVersionNotMet, + HasSymsButNoBinary, + HasBinaryButNoSyms, + FailedToParseSyms, + FailedToLoadNativeCode, + FailedToLoadNativeLibrary, + FailedToFindNativeExport, + InvalidReferenceSymbol, + InvalidImport, + InvalidCallbackEvent, + InvalidFunctionReplacement, + FailedToFindReplacement, + ReplacementConflict, + MissingDependencyInManifest, + MissingDependency, + WrongDependencyVersion, + ModConflict, + DuplicateExport, + NoSpecifiedApiVersion, + UnsupportedApiVersion, + }; + + std::string error_to_string(ModLoadError); + + struct ModFileHandle { + virtual ~ModFileHandle() = default; + virtual std::vector read_file(const std::string& filepath, bool& exists) const = 0; + virtual bool file_exists(const std::string& filepath) const = 0; + }; + + struct ZipModFileHandle final : public ModFileHandle { + FILE* file_handle = nullptr; + std::unique_ptr archive; + + ZipModFileHandle() = default; + ZipModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error); + ~ZipModFileHandle() final; + + std::vector read_file(const std::string& filepath, bool& exists) const final; + bool file_exists(const std::string& filepath) const final; + }; + + struct LooseModFileHandle final : public ModFileHandle { + std::filesystem::path root_path; + + LooseModFileHandle() = default; + LooseModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error); + ~LooseModFileHandle() final; + + std::vector read_file(const std::string& filepath, bool& exists) const final; + bool file_exists(const std::string& filepath) const final; + }; + + struct NativeLibraryManifest { + std::string name; + std::vector exports; + }; + + struct Dependency { + std::string mod_id; + Version version; + }; + + struct ModDetails { + std::string mod_id; + Version version; + std::vector authors; + std::vector dependencies; + }; + + struct ModManifest { + std::filesystem::path mod_root_path; + + std::vector mod_game_ids; + std::string mod_id; + std::vector authors; + std::vector dependencies; + std::unordered_map dependencies_by_id; + Version minimum_recomp_version; + Version version; + + std::vector native_libraries; + std::unique_ptr file_handle; + }; + + struct ModOpenErrorDetails { + std::filesystem::path mod_path; + ModOpenError error; + std::string error_param; + ModOpenErrorDetails() = default; + ModOpenErrorDetails(const std::filesystem::path& mod_path_, ModOpenError error_, const std::string& error_param_) : + mod_path(mod_path_), error(error_), error_param(error_param_) {} + }; + + struct ModLoadErrorDetails { + std::string mod_id; + ModLoadError error; + std::string error_param; + ModLoadErrorDetails() = default; + ModLoadErrorDetails(const std::string& mod_id_, ModLoadError error_, const std::string& error_param_) : + mod_id(mod_id_), error(error_), error_param(error_param_) {} + }; + + void scan_mods(); + void enable_mod(const std::string& mod_id, bool enabled); + bool is_mod_enabled(const std::string& mod_id); + std::vector get_mod_details(const std::string& mod_game_id); + + // Internal functions, TODO move to an internal header. + struct PatchData { + std::array replaced_bytes; + std::string mod_id; + }; + + using GenericFunction = std::variant; + + class ModHandle; + class ModContext { + public: + ModContext(); + ~ModContext(); + + void register_game(const std::string& mod_game_id); + std::vector scan_mod_folder(const std::filesystem::path& mod_folder); + void enable_mod(const std::string& mod_id, bool enabled); + bool is_mod_enabled(const std::string& mod_id); + size_t num_opened_mods(); + std::vector load_mods(const std::string& mod_game_id, uint8_t* rdram, int32_t load_address, uint32_t& ram_used); + void unload_mods(); + std::vector get_mod_details(const std::string& mod_game_id); + private: + ModOpenError open_mod(const std::filesystem::path& mod_path, std::string& error_param); + ModLoadError load_mod(uint8_t* rdram, const std::unordered_map& section_map, recomp::mods::ModHandle& handle, int32_t load_address, uint32_t& ram_used, std::string& error_param); + void check_dependencies(recomp::mods::ModHandle& mod, std::vector>& errors); + ModLoadError load_mod_code(recomp::mods::ModHandle& mod, std::string& error_param); + ModLoadError resolve_dependencies(recomp::mods::ModHandle& mod, std::string& error_param); + void add_opened_mod(ModManifest&& manifest, std::vector&& game_indices); + + // Maps game mod ID to the mod's internal integer ID. + std::unordered_map mod_game_ids; + std::vector opened_mods; + std::unordered_set mod_ids; + std::unordered_set enabled_mods; + std::unordered_map patched_funcs; + std::unordered_map loaded_mods_by_id; + size_t num_events = 0; + }; + + class ModCodeHandle { + public: + virtual ~ModCodeHandle() {} + virtual bool good() = 0; + virtual uint32_t get_api_version() = 0; + virtual void set_imported_function(size_t import_index, GenericFunction func) = 0; + virtual void set_reference_symbol_pointer(size_t symbol_index, recomp_func_t* ptr) = 0; + virtual void set_base_event_index(uint32_t global_event_index) = 0; + virtual uint32_t get_base_event_index() = 0; + virtual void set_recomp_trigger_event_pointer(void (*ptr)(uint8_t* rdram, recomp_context* ctx, uint32_t index)) = 0; + virtual void set_get_function_pointer(recomp_func_t* (*ptr)(int32_t)) = 0; + virtual void set_reference_section_addresses_pointer(int32_t* ptr) = 0; + virtual void set_local_section_address(size_t section_index, int32_t address) = 0; + virtual GenericFunction get_function_handle(size_t func_index) = 0; + }; + + class DynamicLibrary; + class ModHandle { + public: + // TODO make these private and expose methods for the functionality they're currently used in. + ModManifest manifest; + std::unique_ptr code_handle; + std::unique_ptr recompiler_context; + std::vector section_load_addresses; + + ModHandle(ModManifest&& manifest, std::vector&& game_indices); + ModHandle(const ModHandle& rhs) = delete; + ModHandle& operator=(const ModHandle& rhs) = delete; + ModHandle(ModHandle&& rhs); + ModHandle& operator=(ModHandle&& rhs); + ~ModHandle(); + + size_t num_exports() const; + size_t num_events() const; + + ModLoadError populate_exports(std::string& error_param); + bool get_export_function(const std::string& export_name, GenericFunction& out) const; + ModLoadError populate_events(size_t base_event_index, std::string& error_param); + bool get_global_event_index(const std::string& event_name, size_t& event_index_out) const; + ModLoadError load_native_library(const NativeLibraryManifest& lib_manifest, std::string& error_param); + + bool is_for_game(size_t game_index) const { + auto find_it = std::find(game_indices.begin(), game_indices.end(), game_index); + return find_it != game_indices.end(); + } + private: + // Mapping of export name to function index. + std::unordered_map exports_by_name; + // Mapping of export name to native library function pointer. + std::unordered_map native_library_exports; + // Mapping of event name to local index. + std::unordered_map events_by_name; + // Loaded dynamic libraries. + std::vector> native_libraries; // Vector of pointers so that implementation can be elsewhere. + // Games that this mod supports. + std::vector game_indices; + }; + + class NativeCodeHandle : public ModCodeHandle { + public: + NativeCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context); + ~NativeCodeHandle() = default; + bool good() final; + uint32_t get_api_version() final; + void set_imported_function(size_t import_index, GenericFunction func) final; + void set_reference_symbol_pointer(size_t symbol_index, recomp_func_t* ptr) final { + reference_symbol_funcs[symbol_index] = ptr; + }; + void set_base_event_index(uint32_t global_event_index) final { + *base_event_index = global_event_index; + }; + uint32_t get_base_event_index() final { + return *base_event_index; + } + void set_recomp_trigger_event_pointer(void (*ptr)(uint8_t* rdram, recomp_context* ctx, uint32_t index)) final { + *recomp_trigger_event = ptr; + }; + void set_get_function_pointer(recomp_func_t* (*ptr)(int32_t)) final { + *get_function = ptr; + }; + void set_reference_section_addresses_pointer(int32_t* ptr) final { + *reference_section_addresses = ptr; + }; + void set_local_section_address(size_t section_index, int32_t address) final { + section_addresses[section_index] = address; + }; + GenericFunction get_function_handle(size_t func_index) final { + return GenericFunction{ functions[func_index] }; + } + private: + void set_bad(); + bool is_good = false; + std::unique_ptr dynamic_lib; + std::vector functions; + recomp_func_t** imported_funcs; + recomp_func_t** reference_symbol_funcs; + uint32_t* base_event_index; + void (**recomp_trigger_event)(uint8_t* rdram, recomp_context* ctx, uint32_t index); + recomp_func_t* (**get_function)(int32_t vram); + int32_t** reference_section_addresses; + int32_t* section_addresses; + }; + + void setup_events(size_t num_events); + void register_event_callback(size_t event_index, GenericFunction callback); + void reset_events(); + ModLoadError validate_api_version(uint32_t api_version, std::string& error_param); + } +}; + +extern "C" void recomp_trigger_event(uint8_t* rdram, recomp_context* ctx, uint32_t event_index); + +#endif diff --git a/librecomp/include/librecomp/overlays.hpp b/librecomp/include/librecomp/overlays.hpp index e7a4fa2..33a1859 100644 --- a/librecomp/include/librecomp/overlays.hpp +++ b/librecomp/include/librecomp/overlays.hpp @@ -3,6 +3,8 @@ #include #include +#include +#include #include "sections.h" namespace recomp { @@ -21,9 +23,18 @@ namespace recomp { void register_overlays(const overlay_section_table_data_t& sections, const overlays_by_index_t& overlays); void register_patches(const char* patch_data, size_t patch_size, SectionTableEntry* code_sections, size_t num_sections); + void register_base_exports(const FunctionExport* exports); + void register_base_events(char const* const* event_names); void read_patch_data(uint8_t* rdram, gpr patch_data_address); void init_overlays(); + const std::unordered_map& get_vrom_to_section_map(); + recomp_func_t* get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram); + recomp_func_t* get_base_export(const std::string& export_name); + size_t get_base_event_index(const std::string& event_name); + size_t num_base_events(); + + void add_loaded_function(int32_t ram_addr, recomp_func_t* func); } }; diff --git a/librecomp/include/librecomp/sections.h b/librecomp/include/librecomp/sections.h index 1d1b228..039c3cf 100644 --- a/librecomp/include/librecomp/sections.h +++ b/librecomp/include/librecomp/sections.h @@ -20,4 +20,9 @@ typedef struct { size_t index; } SectionTableEntry; +typedef struct { + const char* name; + uint32_t ram_addr; +} FunctionExport; + #endif diff --git a/librecomp/src/flash.cpp b/librecomp/src/flash.cpp index 43407e8..4bbb79e 100644 --- a/librecomp/src/flash.cpp +++ b/librecomp/src/flash.cpp @@ -2,7 +2,8 @@ #include #include #include -#include "recomp.h" +#include "librecomp/recomp.h" +#include "librecomp/addresses.hpp" // TODO move this out into ultramodern code @@ -21,7 +22,7 @@ void save_clear(uint32_t start, uint32_t size, char value); std::array write_buffer; extern "C" void osFlashInit_recomp(uint8_t * rdram, recomp_context * ctx) { - ctx->r2 = ultramodern::flash_handle; + ctx->r2 = recomp::flash_handle; } extern "C" void osFlashReadStatus_recomp(uint8_t * rdram, recomp_context * ctx) { diff --git a/librecomp/src/mod_events.cpp b/librecomp/src/mod_events.cpp new file mode 100644 index 0000000..538dad4 --- /dev/null +++ b/librecomp/src/mod_events.cpp @@ -0,0 +1,56 @@ +#include +#include "librecomp/mods.hpp" +#include "librecomp/overlays.hpp" +#include "ultramodern/error_handling.hpp" + +template +struct overloaded : Ts... { using Ts::operator()...; }; +template +overloaded(Ts...) -> overloaded; + +// Vector of callbacks for each registered event. +std::vector> event_callbacks{}; + +extern "C" { + // This can stay at 0 since the base events are always first in the list. + uint32_t builtin_base_event_index = 0; +} + +extern "C" void recomp_trigger_event(uint8_t* rdram, recomp_context* ctx, uint32_t event_index) { + // Sanity check the event index. + if (event_index >= event_callbacks.size()) { + printf("Event %u triggered, but only %zu events have been registered!\n", event_index, event_callbacks.size()); + assert(false); + ultramodern::error_handling::message_box("Encountered an error with loaded mods: event index out of bounds"); + ULTRAMODERN_QUICK_EXIT(); + } + + // Copy the initial context state to restore it after running each callback. + recomp_context initial_context = *ctx; + + // Call every callback attached to the event. + const std::vector& callbacks = event_callbacks[event_index]; + for (recomp::mods::GenericFunction func : callbacks) { + // Run the callback. + std::visit(overloaded { + [rdram, ctx](recomp_func_t* native_func) { + native_func(rdram, ctx); + }, + }, func); + + // Restore the original context. + *ctx = initial_context; + } +} + +void recomp::mods::setup_events(size_t num_events) { + event_callbacks.resize(num_events); +} + +void recomp::mods::register_event_callback(size_t event_index, GenericFunction callback) { + event_callbacks[event_index].emplace_back(callback); +} + +void recomp::mods::reset_events() { + event_callbacks.clear(); +} diff --git a/librecomp/src/mod_manifest.cpp b/librecomp/src/mod_manifest.cpp new file mode 100644 index 0000000..4207840 --- /dev/null +++ b/librecomp/src/mod_manifest.cpp @@ -0,0 +1,536 @@ +#include + +#include "json/json.hpp" + +#include "n64recomp.h" +#include "librecomp/mods.hpp" + +recomp::mods::ZipModFileHandle::~ZipModFileHandle() { + if (file_handle) { + fclose(file_handle); + file_handle = nullptr; + } + + if (archive) { + mz_zip_reader_end(archive.get()); + } + archive = {}; +} + +recomp::mods::ZipModFileHandle::ZipModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error) { +#ifdef _WIN32 + if (_wfopen_s(&file_handle, mod_path.c_str(), L"rb") != 0) { + error = ModOpenError::FileError; + return; + } +#else + file_handle = fopen(mod_path.c_str(), "rb"); + if (!file_handle) { + error = ModOpenError::FileError; + return; + } +#endif + archive = std::make_unique(); + if (!mz_zip_reader_init_cfile(archive.get(), file_handle, 0, 0)) { + error = ModOpenError::InvalidZip; + return; + } + + error = ModOpenError::Good; +} + +std::vector recomp::mods::ZipModFileHandle::read_file(const std::string& filepath, bool& exists) const { + std::vector ret{}; + + mz_uint32 file_index; + if (!mz_zip_reader_locate_file_v2(archive.get(), filepath.c_str(), nullptr, MZ_ZIP_FLAG_CASE_SENSITIVE, &file_index)) { + exists = false; + return ret; + } + + mz_zip_archive_file_stat stat; + if (!mz_zip_reader_file_stat(archive.get(), file_index, &stat)) { + exists = false; + return ret; + } + + ret.resize(stat.m_uncomp_size); + if (!mz_zip_reader_extract_to_mem(archive.get(), file_index, ret.data(), ret.size(), 0)) { + exists = false; + return {}; + } + + exists = true; + return ret; +} + +bool recomp::mods::ZipModFileHandle::file_exists(const std::string& filepath) const { + mz_uint32 file_index; + if (!mz_zip_reader_locate_file_v2(archive.get(), filepath.c_str(), nullptr, MZ_ZIP_FLAG_CASE_SENSITIVE, &file_index)) { + return false; + } + + return true; +} + +recomp::mods::LooseModFileHandle::~LooseModFileHandle() { + // Nothing to do here, members will be destroyed automatically. +} + +recomp::mods::LooseModFileHandle::LooseModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error) { + root_path = mod_path; + + std::error_code ec; + if (!std::filesystem::is_directory(root_path, ec)) { + error = ModOpenError::NotAFileOrFolder; + } + + if (ec) { + error = ModOpenError::FileError; + } + + error = ModOpenError::Good; +} + +std::vector recomp::mods::LooseModFileHandle::read_file(const std::string& filepath, bool& exists) const { + std::vector ret{}; + std::filesystem::path full_path = root_path / filepath; + + std::error_code ec; + if (!std::filesystem::is_regular_file(full_path, ec) || ec) { + exists = false; + return ret; + } + + std::ifstream file{ full_path, std::ios::binary }; + + if (!file.good()) { + exists = false; + return ret; + } + + file.seekg(0, std::ios::end); + size_t file_size = file.tellg(); + file.seekg(0, std::ios::beg); + + ret.resize(file_size); + file.read(ret.data(), ret.size()); + + exists = true; + return ret; +} + +bool recomp::mods::LooseModFileHandle::file_exists(const std::string& filepath) const { + std::filesystem::path full_path = root_path / filepath; + + std::error_code ec; + if (!std::filesystem::is_regular_file(full_path, ec) || ec) { + return false; + } + + return true; +} + +enum class ManifestField { + GameModId, + Id, + Version, + Authors, + MinimumRecompVersion, + Dependencies, + NativeLibraries, +}; + +const std::string game_mod_id_key = "game_id"; +const std::string mod_id_key = "id"; +const std::string version_key = "version"; +const std::string authors_key = "authors"; +const std::string minimum_recomp_version_key = "minimum_recomp_version"; +const std::string dependencies_key = "dependencies"; +const std::string native_libraries_key = "native_libraries"; + +std::unordered_map field_map { + { game_mod_id_key, ManifestField::GameModId }, + { mod_id_key, ManifestField::Id }, + { version_key, ManifestField::Version }, + { authors_key, ManifestField::Authors }, + { minimum_recomp_version_key, ManifestField::MinimumRecompVersion }, + { dependencies_key, ManifestField::Dependencies }, + { native_libraries_key, ManifestField::NativeLibraries }, +}; + +template +bool get_to(const nlohmann::json& val, T2& out) { + const T1* ptr = val.get_ptr(); + if (ptr == nullptr) { + return false; + } + + out = *ptr; + return true; +} + +template +bool get_to_vec(const nlohmann::json& val, std::vector& out) { + const nlohmann::json::array_t* ptr = val.get_ptr(); + if (ptr == nullptr) { + return false; + } + + out.clear(); + + for (const nlohmann::json& cur_val : *ptr) { + const T1* temp_ptr = cur_val.get_ptr(); + if (temp_ptr == nullptr) { + out.clear(); + return false; + } + + out.emplace_back(*temp_ptr); + } + + return true; +} + +static bool parse_dependency(const std::string& val, recomp::mods::Dependency& out) { + recomp::mods::Dependency ret; + + bool validated_name; + bool validated_version; + + // Check if there's a version number specified. + size_t colon_pos = val.find(':'); + if (colon_pos == std::string::npos) { + // No version present, so just validate the dependency's id. + validated_name = N64Recomp::validate_mod_id(std::string_view{val}); + ret.mod_id = val; + validated_version = true; + ret.version.minor = 0; + ret.version.major = 0; + ret.version.patch = 0; + } + else { + // Version present, validate both the id and version. + ret.mod_id = val.substr(0, colon_pos); + validated_name = N64Recomp::validate_mod_id(ret.mod_id); + validated_version = recomp::Version::from_string(val.substr(colon_pos + 1), ret.version); + } + + if (validated_name && validated_version) { + out = std::move(ret); + return true; + } + + return false; +} + +recomp::mods::ModOpenError parse_manifest(recomp::mods::ModManifest& ret, const std::vector& manifest_data, std::string& error_param) { + using json = nlohmann::json; + json manifest_json = json::parse(manifest_data.begin(), manifest_data.end(), nullptr, false); + + if (manifest_json.is_discarded()) { + return recomp::mods::ModOpenError::FailedToParseManifest; + } + + if (!manifest_json.is_object()) { + return recomp::mods::ModOpenError::InvalidManifestSchema; + } + + for (const auto& [key, val] : manifest_json.items()) { + const auto find_key_it = field_map.find(key); + if (find_key_it == field_map.end()) { + // Unrecognized field + error_param = key; + return recomp::mods::ModOpenError::UnrecognizedManifestField; + } + + ManifestField field = find_key_it->second; + switch (field) { + case ManifestField::GameModId: + { + std::string mod_game_id; + if (!get_to(val, mod_game_id)) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + ret.mod_game_ids.resize(1); + ret.mod_game_ids[0] = std::move(mod_game_id); + } + break; + case ManifestField::Id: + if (!get_to(val, ret.mod_id)) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + break; + case ManifestField::Version: + { + const std::string* version_str = val.get_ptr(); + if (version_str == nullptr) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + if (!recomp::Version::from_string(*version_str, ret.version)) { + error_param = *version_str; + return recomp::mods::ModOpenError::InvalidVersionString; + } + } + break; + case ManifestField::Authors: + if (!get_to_vec(val, ret.authors)) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + break; + case ManifestField::MinimumRecompVersion: + { + const std::string* version_str = val.get_ptr(); + if (version_str == nullptr) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + if (!recomp::Version::from_string(*version_str, ret.minimum_recomp_version)) { + error_param = *version_str; + return recomp::mods::ModOpenError::InvalidMinimumRecompVersionString; + } + ret.minimum_recomp_version.suffix.clear(); + } + break; + case ManifestField::Dependencies: + { + std::vector dep_strings{}; + if (!get_to_vec(val, dep_strings)) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + + for (const std::string& dep_string : dep_strings) { + recomp::mods::Dependency cur_dep; + if (!parse_dependency(dep_string, cur_dep)) { + error_param = dep_string; + return recomp::mods::ModOpenError::InvalidDependencyString; + } + + size_t dependency_index = ret.dependencies.size(); + ret.dependencies_by_id.emplace(cur_dep.mod_id, dependency_index); + ret.dependencies.emplace_back(std::move(cur_dep)); + } + } + break; + case ManifestField::NativeLibraries: + { + if (!val.is_object()) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + for (const auto& [lib_name, lib_exports] : val.items()) { + recomp::mods::NativeLibraryManifest& cur_lib = ret.native_libraries.emplace_back(); + + cur_lib.name = lib_name; + if (!get_to_vec(lib_exports, cur_lib.exports)) { + error_param = key; + return recomp::mods::ModOpenError::IncorrectManifestFieldType; + } + } + } + break; + } + } + + return recomp::mods::ModOpenError::Good; +} + +recomp::mods::ModOpenError validate_manifest(const recomp::mods::ModManifest& manifest, std::string& error_param) { + using namespace recomp::mods; + + // Check for required fields. + if (manifest.mod_game_ids.empty()) { + error_param = game_mod_id_key; + return ModOpenError::MissingManifestField; + } + if (manifest.mod_id.empty()) { + error_param = mod_id_key; + return ModOpenError::MissingManifestField; + } + if (manifest.version.major == -1 || manifest.version.major == -1 || manifest.version.major == -1) { + error_param = version_key; + return ModOpenError::MissingManifestField; + } + if (manifest.authors.empty()) { + error_param = authors_key; + return ModOpenError::MissingManifestField; + } + if (manifest.minimum_recomp_version.major == -1 || manifest.minimum_recomp_version.major == -1 || manifest.minimum_recomp_version.major == -1) { + error_param = minimum_recomp_version_key; + return ModOpenError::MissingManifestField; + } + + return ModOpenError::Good; +} + +recomp::mods::ModOpenError recomp::mods::ModContext::open_mod(const std::filesystem::path& mod_path, std::string& error_param) { + ModManifest manifest{}; + std::error_code ec; + error_param = ""; + + if (!std::filesystem::exists(mod_path, ec) || ec) { + return ModOpenError::DoesNotExist; + } + + // TODO support symlinks? + bool is_file = std::filesystem::is_regular_file(mod_path, ec); + if (ec) { + return ModOpenError::FileError; + } + + bool is_directory = std::filesystem::is_directory(mod_path, ec); + if (ec) { + return ModOpenError::FileError; + } + + // Load the directory or zip file. + ModOpenError handle_error; + if (is_file) { + manifest.file_handle = std::make_unique(mod_path, handle_error); + } + else if (is_directory) { + manifest.file_handle = std::make_unique(mod_path, handle_error); + } + else { + return ModOpenError::NotAFileOrFolder; + } + + if (handle_error != ModOpenError::Good) { + return handle_error; + } + + { + bool exists; + std::vector manifest_data = manifest.file_handle->read_file("manifest.json", exists); + if (!exists) { + return ModOpenError::NoManifest; + } + + ModOpenError parse_error = parse_manifest(manifest, manifest_data, error_param); + if (parse_error != ModOpenError::Good) { + return parse_error; + } + } + + // Check for this being a duplicate of another opened mod. + if (mod_ids.contains(manifest.mod_id)) { + error_param = manifest.mod_id; + return ModOpenError::DuplicateMod; + } + mod_ids.emplace(manifest.mod_id); + + ModOpenError validate_error = validate_manifest(manifest, error_param); + if (validate_error != ModOpenError::Good) { + return validate_error; + } + + // Check for this mod's game ids being valid. + std::vector game_indices; + for (const auto& mod_game_id : manifest.mod_game_ids) { + auto find_id_it = mod_game_ids.find(mod_game_id); + if (find_id_it == mod_game_ids.end()) { + error_param = mod_game_id; + return ModOpenError::WrongGame; + } + game_indices.emplace_back(find_id_it->second); + } + + // Store the loaded mod manifest in a new mod handle. + manifest.mod_root_path = mod_path; + add_opened_mod(std::move(manifest), std::move(game_indices)); + + return ModOpenError::Good; +} + +std::string recomp::mods::error_to_string(ModOpenError error) { + switch (error) { + case ModOpenError::Good: + return "Good"; + case ModOpenError::DoesNotExist: + return "Mod does not exist"; + case ModOpenError::NotAFileOrFolder: + return "Mod is not a file or folder"; + case ModOpenError::FileError: + return "Error reading mod file(s)"; + case ModOpenError::InvalidZip: + return "Mod is an invalid zip file"; + case ModOpenError::NoManifest: + return "Mod is missing a manifest.json"; + case ModOpenError::FailedToParseManifest: + return "Failed to parse mod's manifest.json"; + case ModOpenError::InvalidManifestSchema: + return "Mod's manifest.json has an invalid schema"; + case ModOpenError::UnrecognizedManifestField: + return "Unrecognized field in manifest.json"; + case ModOpenError::IncorrectManifestFieldType: + return "Incorrect type for field in manifest.json"; + case ModOpenError::InvalidVersionString: + return "Invalid version string in manifest.json"; + case ModOpenError::InvalidMinimumRecompVersionString: + return "Invalid minimum recomp version string in manifest.json"; + case ModOpenError::InvalidDependencyString: + return "Invalid dependency string in manifest.json"; + case ModOpenError::MissingManifestField: + return "Missing required field in manifest"; + case ModOpenError::DuplicateMod: + return "Duplicate mod found"; + case ModOpenError::WrongGame: + return "Mod is for a different game"; + } + return "Unknown mod opening error: " + std::to_string((int)error); +} + +std::string recomp::mods::error_to_string(ModLoadError error) { + switch (error) { + case ModLoadError::Good: + return "Good"; + case ModLoadError::InvalidGame: + return "Invalid game"; + case ModLoadError::MinimumRecompVersionNotMet: + return "Mod requires a newer version of this project"; + case ModLoadError::HasSymsButNoBinary: + return "Mod has a symbol file but no binary file"; + case ModLoadError::HasBinaryButNoSyms: + return "Mod has a binary file but no symbol file"; + case ModLoadError::FailedToParseSyms: + return "Failed to parse mod symbol file"; + case ModLoadError::FailedToLoadNativeCode: + return "Failed to load offline mod library"; + case ModLoadError::FailedToLoadNativeLibrary: + return "Failed to load mod library"; + case ModLoadError::FailedToFindNativeExport: + return "Failed to find native export"; + case ModLoadError::InvalidReferenceSymbol: + return "Reference symbol does not exist"; + case ModLoadError::InvalidImport: + return "Imported function not found"; + case ModLoadError::InvalidCallbackEvent: + return "Event for callback not found"; + case ModLoadError::InvalidFunctionReplacement: + return "Function to be replaced does not exist"; + case ModLoadError::FailedToFindReplacement: + return "Failed to find replacement function"; + case ModLoadError::ReplacementConflict: + return "Attempted to replace a function that cannot be replaced"; + case ModLoadError::MissingDependencyInManifest: + return "Dependency is present in mod symbols but not in the manifest"; + case ModLoadError::MissingDependency: + return "Missing dependency"; + case ModLoadError::WrongDependencyVersion: + return "Wrong dependency version"; + case ModLoadError::ModConflict: + return "Conflicts with other mod"; + case ModLoadError::DuplicateExport: + return "Duplicate exports in mod"; + case ModLoadError::NoSpecifiedApiVersion: + return "Mod DLL does not specify an API version"; + case ModLoadError::UnsupportedApiVersion: + return "Mod DLL has an unsupported API version"; + } + return "Unknown mod loading error " + std::to_string((int)error); +} diff --git a/librecomp/src/mods.cpp b/librecomp/src/mods.cpp new file mode 100644 index 0000000..f2a01db --- /dev/null +++ b/librecomp/src/mods.cpp @@ -0,0 +1,937 @@ +#include +#include +#include + +#include "librecomp/mods.hpp" +#include "librecomp/overlays.hpp" +#include "librecomp/game.hpp" +#include "n64recomp.h" + +// Architecture detection. + +// MSVC x86_64 +#if defined (_M_AMD64) && (_M_AMD64 == 100) && !defined (_M_ARM64EC) +# define IS_X86_64 +// GCC/Clang x86_64 +#elif defined(__x86_64__) +# define IS_X86_64 +// MSVC/GCC/Clang ARM64 +#elif defined(__ARM_ARCH_ISA_A64) +# define IS_ARM64 +#else +# error "Unsupported architecture!" +#endif + + +#if defined(_WIN32) +#define PATHFMT "%ls" +#else +#define PATHFMT "%s" +#endif + +template +struct overloaded : Ts... { using Ts::operator()...; }; +template +overloaded(Ts...) -> overloaded; + +#if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# include "Windows.h" + +class recomp::mods::DynamicLibrary { +public: + static constexpr std::string_view PlatformExtension = ".dll"; + DynamicLibrary() = default; + DynamicLibrary(const std::filesystem::path& path) { + native_handle = LoadLibraryW(path.c_str()); + + if (good()) { + uint32_t* recomp_api_version; + if (get_dll_symbol(recomp_api_version, "recomp_api_version")) { + api_version = *recomp_api_version; + } + else { + api_version = (uint32_t)-1; + } + } + } + ~DynamicLibrary() { + unload(); + } + DynamicLibrary(const DynamicLibrary&) = delete; + DynamicLibrary& operator=(const DynamicLibrary&) = delete; + DynamicLibrary(DynamicLibrary&&) = delete; + DynamicLibrary& operator=(DynamicLibrary&&) = delete; + + void unload() { + if (native_handle != nullptr) { + FreeLibrary(native_handle); + } + native_handle = nullptr; + } + + bool good() const { + return native_handle != nullptr; + } + + template + bool get_dll_symbol(T& out, const char* name) const { + out = (T)GetProcAddress(native_handle, name); + if (out == nullptr) { + return false; + } + return true; + }; + + uint32_t get_api_version() { + return api_version; + } +private: + HMODULE native_handle; + uint32_t api_version; +}; + +void unprotect(void* target_func, uint64_t* old_flags) { + DWORD old_flags_dword; + BOOL result = VirtualProtect(target_func, + 16, + PAGE_READWRITE, + &old_flags_dword); + *old_flags = old_flags_dword; + (void)result; +} + +void protect(void* target_func, uint64_t old_flags) { + DWORD dummy_old_flags; + BOOL result = VirtualProtect(target_func, + 16, + static_cast(old_flags), + &dummy_old_flags); + (void)result; +} +#else +# include +# include +# include + +class recomp::mods::DynamicLibrary { +public: + static constexpr std::string_view PlatformExtension = ".so"; + DynamicLibrary() = default; + DynamicLibrary(const std::filesystem::path& path) { + native_handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + + if (good()) { + uint32_t* recomp_api_version; + if (get_dll_symbol(recomp_api_version, "recomp_api_version")) { + api_version = *recomp_api_version; + } + else { + api_version = (uint32_t)-1; + } + } + } + ~DynamicLibrary() { + unload(); + } + DynamicLibrary(const DynamicLibrary&) = delete; + DynamicLibrary& operator=(const DynamicLibrary&) = delete; + DynamicLibrary(DynamicLibrary&&) = delete; + DynamicLibrary& operator=(DynamicLibrary&&) = delete; + + void unload() { + if (native_handle != nullptr) { + dlclose(native_handle); + } + native_handle = nullptr; + } + + bool good() const { + return native_handle != nullptr; + } + + template + bool get_dll_symbol(T& out, const char* name) const { + out = (T)dlsym(native_handle, name); + if (out == nullptr) { + return false; + } + return true; + }; + + uint32_t get_api_version() { + return api_version; + } +private: + void* native_handle; + uint32_t api_version; +}; + +void unprotect(void* target_func, uint64_t* old_flags) { + // Align the address to a page boundary. + uintptr_t page_start = (uintptr_t)target_func; + int page_size = getpagesize(); + page_start = (page_start / page_size) * page_size; + + int result = mprotect((void*)page_start, page_size, PROT_READ | PROT_WRITE); + *old_flags = 0; + (void)result; +} + +void protect(void* target_func, uint64_t old_flags) { + // Align the address to a page boundary. + uintptr_t page_start = (uintptr_t)target_func; + int page_size = getpagesize(); + page_start = (page_start / page_size) * page_size; + + int result = mprotect((void*)page_start, page_size, PROT_READ | PROT_EXEC); + (void)result; +} +#endif + +namespace modpaths { + const std::string binary_path = "mod_binary.bin"; + const std::string binary_syms_path = "mod_syms.bin"; +}; + +recomp::mods::ModLoadError recomp::mods::validate_api_version(uint32_t api_version, std::string& error_param) { + switch (api_version) { + case 1: + return ModLoadError::Good; + case (uint32_t)-1: + return ModLoadError::NoSpecifiedApiVersion; + default: + error_param = std::to_string(api_version); + return ModLoadError::UnsupportedApiVersion; + } +} + +recomp::mods::ModHandle::ModHandle(ModManifest&& manifest, std::vector&& game_indices) : + manifest(std::move(manifest)), + code_handle(), + recompiler_context{std::make_unique()}, + game_indices{std::move(game_indices)} +{ + +} + +recomp::mods::ModHandle::ModHandle(ModHandle&& rhs) = default; +recomp::mods::ModHandle& recomp::mods::ModHandle::operator=(ModHandle&& rhs) = default; +recomp::mods::ModHandle::~ModHandle() = default; + +size_t recomp::mods::ModHandle::num_exports() const { + return recompiler_context->exported_funcs.size(); +} + +size_t recomp::mods::ModHandle::num_events() const { + return recompiler_context->event_symbols.size(); +} + +recomp::mods::ModLoadError recomp::mods::ModHandle::populate_exports(std::string& error_param) { + for (size_t func_index : recompiler_context->exported_funcs) { + const auto& func_handle = recompiler_context->functions[func_index]; + exports_by_name.emplace(func_handle.name, func_index); + } + + return ModLoadError::Good; +} + +recomp::mods::ModLoadError recomp::mods::ModHandle::load_native_library(const recomp::mods::NativeLibraryManifest& lib_manifest, std::string& error_param) { + std::string lib_filename = lib_manifest.name + std::string{DynamicLibrary::PlatformExtension}; + std::filesystem::path lib_path = manifest.mod_root_path.parent_path() / lib_filename; + + std::unique_ptr& lib = native_libraries.emplace_back(std::make_unique(lib_path)); + + if (!lib->good()) { + error_param = lib_filename; + return ModLoadError::FailedToLoadNativeLibrary; + } + + std::string api_error_param; + ModLoadError api_error = validate_api_version(lib->get_api_version(), api_error_param); + + if (api_error != ModLoadError::Good) { + if (api_error_param.empty()) { + error_param = lib_filename; + } + else { + error_param = lib_filename + ":" + api_error_param; + } + return api_error; + } + + for (const std::string& export_name : lib_manifest.exports) { + recomp_func_t* cur_func; + if (native_library_exports.contains(export_name)) { + error_param = export_name; + return ModLoadError::DuplicateExport; + } + if (!lib->get_dll_symbol(cur_func, export_name.c_str())) { + error_param = lib_manifest.name + ":" + export_name; + return ModLoadError::FailedToFindNativeExport; + } + native_library_exports.emplace(export_name, cur_func); + } + + return ModLoadError::Good; +} + +bool recomp::mods::ModHandle::get_export_function(const std::string& export_name, GenericFunction& out) const { + // First, check the code exports. + auto code_find_it = exports_by_name.find(export_name); + if (code_find_it != exports_by_name.end()) { + out = code_handle->get_function_handle(code_find_it->second); + return true; + } + + // Next, check the native library exports. + auto native_find_it = native_library_exports.find(export_name); + if (native_find_it != native_library_exports.end()) { + out = native_find_it->second; + return true; + } + + + // Nothing found. + return false; +} + +recomp::mods::ModLoadError recomp::mods::ModHandle::populate_events(size_t base_event_index, std::string& error_param) { + for (size_t event_index = 0; event_index < recompiler_context->event_symbols.size(); event_index++) { + const N64Recomp::EventSymbol& event = recompiler_context->event_symbols[event_index]; + events_by_name.emplace(event.base.name, event_index); + } + + code_handle->set_base_event_index(base_event_index); + return ModLoadError::Good; +} + +bool recomp::mods::ModHandle::get_global_event_index(const std::string& event_name, size_t& event_index_out) const { + auto find_it = events_by_name.find(event_name); + if (find_it == events_by_name.end()) { + return false; + } + + event_index_out = code_handle->get_base_event_index() + find_it->second; + return true; +} + +recomp::mods::NativeCodeHandle::NativeCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context) { + is_good = true; + // Load the DLL. + dynamic_lib = std::make_unique(dll_path); + if (!dynamic_lib->good()) { + is_good = false; + return; + } + + // Fill out the list of function pointers. + functions.resize(context.functions.size()); + for (size_t i = 0; i < functions.size(); i++) { + if(!context.functions[i].name.empty()) { + is_good &= dynamic_lib->get_dll_symbol(functions[i], context.functions[i].name.c_str()); + } + else { + std::string func_name = "mod_func_" + std::to_string(i); + is_good &= dynamic_lib->get_dll_symbol(functions[i], func_name.c_str()); + } + if (!is_good) { + return; + } + } + + // Get the standard exported symbols. + is_good = true; + is_good &= dynamic_lib->get_dll_symbol(imported_funcs, "imported_funcs"); + is_good &= dynamic_lib->get_dll_symbol(reference_symbol_funcs, "reference_symbol_funcs"); + is_good &= dynamic_lib->get_dll_symbol(base_event_index, "base_event_index"); + is_good &= dynamic_lib->get_dll_symbol(recomp_trigger_event, "recomp_trigger_event"); + is_good &= dynamic_lib->get_dll_symbol(get_function, "get_function"); + is_good &= dynamic_lib->get_dll_symbol(reference_section_addresses, "reference_section_addresses"); + is_good &= dynamic_lib->get_dll_symbol(section_addresses, "section_addresses"); +} + +bool recomp::mods::NativeCodeHandle::good() { + return dynamic_lib->good() && is_good; +} + +uint32_t recomp::mods::NativeCodeHandle::get_api_version() { + return dynamic_lib->get_api_version(); +} + +void recomp::mods::NativeCodeHandle::set_bad() { + dynamic_lib.reset(); + is_good = false; +} + +void recomp::mods::NativeCodeHandle::set_imported_function(size_t import_index, GenericFunction func) { + std::visit(overloaded { + [this, import_index](recomp_func_t* native_func) { + imported_funcs[import_index] = native_func; + } + }, func); +} + +void patch_func(recomp_func_t* target_func, recomp::mods::GenericFunction replacement_func) { + uint8_t* target_func_u8 = reinterpret_cast(target_func); + size_t offset = 0; + + auto write_bytes = [&](const void* bytes, size_t count) { + memcpy(target_func_u8 + offset, bytes, count); + offset += count; + }; + + uint64_t old_flags; + unprotect(target_func_u8, &old_flags); + +#if defined(IS_X86_64) + static const uint8_t movabs_rax[] = {0x48, 0xB8}; + static const uint8_t jmp_rax[] = {0xFF, 0xE0}; + std::visit(overloaded { + [&write_bytes](recomp_func_t* native_func) { + write_bytes(movabs_rax, sizeof(movabs_rax)); + write_bytes(&native_func, sizeof(&native_func)); + write_bytes(jmp_rax, sizeof(jmp_rax)); + } + }, replacement_func); +#elif defined(IS_ARM64) + ultramodern::error_handling::message_box("Mod loading not currently implemented on ARM CPUs!\n"); +#else +# error "Unsupported architecture" +#endif + + protect(target_func_u8, old_flags); +} + +void unpatch_func(void* target_func, const recomp::mods::PatchData& data) { + uint64_t old_flags; + unprotect(target_func, &old_flags); + memcpy(target_func, data.replaced_bytes.data(), data.replaced_bytes.size()); + protect(target_func, old_flags); +} + +void recomp::mods::ModContext::add_opened_mod(ModManifest&& manifest, std::vector&& game_indices) { + opened_mods.emplace_back(std::move(manifest), std::move(game_indices)); +} + +recomp::mods::ModLoadError recomp::mods::ModContext::load_mod(uint8_t* rdram, const std::unordered_map& section_vrom_map, recomp::mods::ModHandle& handle, int32_t load_address, uint32_t& ram_used, std::string& error_param) { + using namespace recomp::mods; + handle.section_load_addresses.clear(); + + // Check that the mod's minimum recomp version is met. + if (get_project_version() < handle.manifest.minimum_recomp_version) { + error_param = handle.manifest.minimum_recomp_version.to_string(); + return recomp::mods::ModLoadError::MinimumRecompVersionNotMet; + } + + // Load the mod symbol data from the file provided in the manifest. + bool binary_syms_exists = false; + std::vector syms_data = handle.manifest.file_handle->read_file(modpaths::binary_syms_path, binary_syms_exists); + + // Load the binary data from the file provided in the manifest. + bool binary_exists = false; + std::vector binary_data = handle.manifest.file_handle->read_file(modpaths::binary_path, binary_exists); + + if (binary_syms_exists && !binary_exists) { + return recomp::mods::ModLoadError::HasSymsButNoBinary; + } + + if (binary_exists && !binary_syms_exists) { + return recomp::mods::ModLoadError::HasBinaryButNoSyms; + } + + std::span binary_span {reinterpret_cast(binary_data.data()), binary_data.size() }; + + // Parse the symbol file into the recompiler context. + N64Recomp::ModSymbolsError symbol_load_error = N64Recomp::parse_mod_symbols(syms_data, binary_span, section_vrom_map, *handle.recompiler_context); + if (symbol_load_error != N64Recomp::ModSymbolsError::Good) { + return ModLoadError::FailedToParseSyms; + } + + const std::vector& mod_sections = handle.recompiler_context->sections; + handle.section_load_addresses.resize(mod_sections.size()); + + // Copy each section's binary into rdram, leaving room for the section's bss before the next one. + int32_t cur_section_addr = load_address; + for (size_t section_index = 0; section_index < mod_sections.size(); section_index++) { + const auto& section = mod_sections[section_index]; + for (size_t i = 0; i < section.size; i++) { + MEM_B(i, (gpr)cur_section_addr) = binary_data[section.rom_addr + i]; + } + handle.section_load_addresses[section_index] = cur_section_addr; + cur_section_addr += section.size + section.bss_size; + + } + + // Iterate over each section again after loading them to perform R_MIPS_32 relocations. + for (size_t section_index = 0; section_index < mod_sections.size(); section_index++) { + const auto& section = mod_sections[section_index]; + uint32_t cur_section_original_vram = section.ram_addr; + uint32_t cur_section_loaded_vram = handle.section_load_addresses[section_index]; + + // Perform mips32 relocations for this section. + for (const auto& reloc : section.relocs) { + if (reloc.type == N64Recomp::RelocType::R_MIPS_32 && !reloc.reference_symbol) { + if (reloc.target_section >= mod_sections.size()) { + return ModLoadError::FailedToParseSyms; + } + // Get the ram address of the word that's being relocated and read its original value. + int32_t reloc_word_addr = reloc.address - cur_section_original_vram + cur_section_loaded_vram; + uint32_t reloc_word = MEM_W(0, reloc_word_addr); + + // Determine the original and loaded addresses of the section that the relocation points to. + uint32_t target_section_original_vram = mod_sections[reloc.target_section].ram_addr; + uint32_t target_section_loaded_vram = handle.section_load_addresses[reloc.target_section]; + + uint32_t reloc_word_old = reloc_word; + + // Recalculate the word and write it back into ram. + reloc_word += (target_section_loaded_vram - target_section_original_vram); + MEM_W(0, reloc_word_addr) = reloc_word; + } + } + } + + ram_used = cur_section_addr - load_address; + + return ModLoadError::Good; +} + +void recomp::mods::ModContext::register_game(const std::string& mod_game_id) { + mod_game_ids.emplace(mod_game_id, mod_game_ids.size()); +} + +std::vector recomp::mods::ModContext::scan_mod_folder(const std::filesystem::path& mod_folder) { + std::vector ret{}; + std::error_code ec; + for (const auto& mod_path : std::filesystem::directory_iterator{mod_folder, std::filesystem::directory_options::skip_permission_denied, ec}) { + if ((mod_path.is_regular_file() && mod_path.path().extension() == ".nrm") || mod_path.is_directory()) { + printf("Opening mod " PATHFMT "\n", mod_path.path().stem().c_str()); + std::string open_error_param; + ModOpenError open_error = open_mod(mod_path, open_error_param); + + if (open_error != ModOpenError::Good) { + ret.emplace_back(mod_path.path(), open_error, open_error_param); + } + } + else { + printf("Skipping non-mod " PATHFMT PATHFMT "\n", mod_path.path().stem().c_str(), mod_path.path().extension().c_str()); + } + } + + return ret; +} + +// Nothing needed for these two, they just need to be explicitly declared outside the header to allow forward declaration of ModHandle. +recomp::mods::ModContext::ModContext() = default; +recomp::mods::ModContext::~ModContext() = default; + +void recomp::mods::ModContext::enable_mod(const std::string& mod_id, bool enabled) { + if (enabled) { + enabled_mods.emplace(mod_id); + } + else { + enabled_mods.erase(mod_id); + } +} + +bool recomp::mods::ModContext::is_mod_enabled(const std::string& mod_id) { + return enabled_mods.contains(mod_id); +} + +size_t recomp::mods::ModContext::num_opened_mods() { + return opened_mods.size(); +} + +std::vector recomp::mods::ModContext::get_mod_details(const std::string& mod_game_id) { + std::vector ret{}; + bool all_games = mod_game_id.empty(); + size_t game_index = (size_t)-1; + + auto find_game_it = mod_game_ids.find(mod_game_id); + if (find_game_it != mod_game_ids.end()) { + game_index = find_game_it->second; + } + + for (const ModHandle& mod : opened_mods) { + if (all_games || mod.is_for_game(game_index)) { + std::vector cur_dependencies{}; + + ret.emplace_back(ModDetails{ + .mod_id = mod.manifest.mod_id, + .version = mod.manifest.version, + .authors = mod.manifest.authors, + .dependencies = mod.manifest.dependencies + }); + } + } + + return ret; +} + +std::vector recomp::mods::ModContext::load_mods(const std::string& mod_game_id, uint8_t* rdram, int32_t load_address, uint32_t& ram_used) { + std::vector ret{}; + ram_used = 0; + num_events = recomp::overlays::num_base_events(); + + auto find_index_it = mod_game_ids.find(mod_game_id); + if (find_index_it == mod_game_ids.end()) { + ret.emplace_back(mod_game_id, ModLoadError::InvalidGame, std::string{}); + return ret; + } + + size_t mod_game_index = find_index_it->second; + + if (!patched_funcs.empty()) { + printf("Mods already loaded!\n"); + return {}; + } + + const std::unordered_map& section_vrom_map = recomp::overlays::get_vrom_to_section_map(); + + std::vector active_mods{}; + + // Find and load active mods. + for (size_t mod_index = 0; mod_index < opened_mods.size(); mod_index++) { + auto& mod = opened_mods[mod_index]; + if (mod.is_for_game(mod_game_index) && enabled_mods.contains(mod.manifest.mod_id)) { + active_mods.push_back(mod_index); + loaded_mods_by_id.emplace(mod.manifest.mod_id, mod_index); + + printf("Loading mod %s\n", mod.manifest.mod_id.c_str()); + uint32_t cur_ram_used = 0; + std::string load_error_param; + ModLoadError load_error = load_mod(rdram, section_vrom_map, mod, load_address, cur_ram_used, load_error_param); + + if (load_error != ModLoadError::Good) { + ret.emplace_back(mod.manifest.mod_id, load_error, load_error_param); + } + else { + load_address += cur_ram_used; + ram_used += cur_ram_used; + } + } + } + + // Exit early if errors were found. + if (!ret.empty()) { + unload_mods(); + return ret; + } + + // Check that mod dependencies are met. + for (size_t mod_index : active_mods) { + auto& mod = opened_mods[mod_index]; + std::vector> cur_errors; + check_dependencies(mod, cur_errors); + + if (!cur_errors.empty()) { + for (auto const& [cur_error, cur_error_param] : cur_errors) { + ret.emplace_back(mod.manifest.mod_id, cur_error, cur_error_param); + } + } + } + + // Exit early if errors were found. + if (!ret.empty()) { + unload_mods(); + return ret; + } + + // Load the code and exports from all mods. + for (size_t mod_index : active_mods) { + auto& mod = opened_mods[mod_index]; + std::string cur_error_param; + ModLoadError cur_error = load_mod_code(mod, cur_error_param); + if (cur_error != ModLoadError::Good) { + ret.emplace_back(mod.manifest.mod_id, cur_error, cur_error_param); + } + } + + // Exit early if errors were found. + if (!ret.empty()) { + unload_mods(); + return ret; + } + + // Set up the event callbacks based on the number of events allocated. + recomp::mods::setup_events(num_events); + + // Resolve dependencies for all mods. + for (size_t mod_index : active_mods) { + auto& mod = opened_mods[mod_index]; + std::string cur_error_param; + ModLoadError cur_error = resolve_dependencies(mod, cur_error_param); + if (cur_error != ModLoadError::Good) { + ret.emplace_back(mod.manifest.mod_id, cur_error, cur_error_param); + } + } + + // Exit early if errors were found. + if (!ret.empty()) { + unload_mods(); + return ret; + } + + return ret; +} + +void recomp::mods::ModContext::check_dependencies(recomp::mods::ModHandle& mod, std::vector>& errors) { + errors.clear(); + for (const auto& [cur_dep_id, cur_dep_index] : mod.recompiler_context->dependencies_by_name) { + // Handle special dependency names. + if (cur_dep_id == N64Recomp::DependencyBaseRecomp || cur_dep_id == N64Recomp::DependencySelf) { + continue; + } + + // Find the dependency in the mod manifest to get its version. + auto find_manifest_dep_it = mod.manifest.dependencies_by_id.find(cur_dep_id); + if (find_manifest_dep_it == mod.manifest.dependencies_by_id.end()) { + errors.emplace_back(ModLoadError::MissingDependencyInManifest, cur_dep_id); + continue; + } + + const auto& cur_dep = mod.manifest.dependencies[find_manifest_dep_it->second]; + + // Look for the dependency in the loaded mod mapping. + auto find_loaded_dep_it = loaded_mods_by_id.find(cur_dep_id); + if (find_loaded_dep_it == loaded_mods_by_id.end()) { + errors.emplace_back(ModLoadError::MissingDependency, cur_dep_id); + continue; + } + + const ModHandle& dep_mod = opened_mods[find_loaded_dep_it->second]; + if (cur_dep.version > dep_mod.manifest.version) + { + std::stringstream error_param_stream{}; + error_param_stream << "requires mod \"" << cur_dep.mod_id << "\" " << + (int)cur_dep.version.major << "." << (int)cur_dep.version.minor << "." << (int)cur_dep.version.patch << ", got " << + (int)dep_mod.manifest.version.major << "." << (int)dep_mod.manifest.version.minor << "." << (int)dep_mod.manifest.version.patch << ""; + errors.emplace_back(ModLoadError::WrongDependencyVersion, error_param_stream.str()); + } + } +} + +recomp::mods::ModLoadError recomp::mods::ModContext::load_mod_code(recomp::mods::ModHandle& mod, std::string& error_param) { + // TODO implement LuaJIT recompilation and allow it instead of native code loading via a mod manifest flag. + std::filesystem::path dll_path = mod.manifest.mod_root_path; + dll_path.replace_extension(DynamicLibrary::PlatformExtension); + mod.code_handle = std::make_unique(dll_path, *mod.recompiler_context); + if (!mod.code_handle->good()) { + mod.code_handle.reset(); + error_param = dll_path.string(); + return ModLoadError::FailedToLoadNativeCode; + } + + std::string cur_error_param; + ModLoadError cur_error = validate_api_version(mod.code_handle->get_api_version(), cur_error_param); + + if (cur_error != ModLoadError::Good) { + if (cur_error_param.empty()) { + error_param = dll_path.filename().string(); + } + else { + error_param = dll_path.filename().string() + ":" + std::move(cur_error_param); + } + return cur_error; + } + + // Populate the mod's export map. + cur_error = mod.populate_exports(cur_error_param); + + if (cur_error != ModLoadError::Good) { + error_param = std::move(cur_error_param); + return cur_error; + } + + // Load any native libraries specified by the mod and validate/register the expors. + std::filesystem::path parent_path = mod.manifest.mod_root_path.parent_path(); + for (const recomp::mods::NativeLibraryManifest& cur_lib_manifest: mod.manifest.native_libraries) { + cur_error = mod.load_native_library(cur_lib_manifest, cur_error_param); + if (cur_error != ModLoadError::Good) { + error_param = std::move(cur_error_param); + return cur_error; + } + } + + // Populate the mod's event map and set its base event index. + cur_error = mod.populate_events(num_events, cur_error_param); + + if (cur_error != ModLoadError::Good) { + error_param = std::move(cur_error_param); + return cur_error; + } + + // Allocate the event indices used by the mod. + num_events += mod.num_events(); + + // Add each function from the mod into the function lookup table. + const std::vector& mod_sections = mod.recompiler_context->sections; + for (size_t func_index = 0; func_index < mod.recompiler_context->functions.size(); func_index++) { + const auto& func = mod.recompiler_context->functions[func_index]; + if (func.section_index >= mod_sections.size()) { + return ModLoadError::FailedToParseSyms; + } + // Calculate the loaded address of this function. + int32_t func_address = func.vram - mod_sections[func.section_index].ram_addr + mod.section_load_addresses[func.section_index]; + + // Get the handle to the function and add it to the lookup table based on its type. + recomp::mods::GenericFunction func_handle = mod.code_handle->get_function_handle(func_index); + std::visit(overloaded{ + [func_address](recomp_func_t* native_func) { + recomp::overlays::add_loaded_function(func_address, native_func); + } + }, func_handle); + } + + return ModLoadError::Good; +} + +recomp::mods::ModLoadError recomp::mods::ModContext::resolve_dependencies(recomp::mods::ModHandle& mod, std::string& error_param) { + // Reference symbols from the base recomp. + for (size_t reference_sym_index = 0; reference_sym_index < mod.recompiler_context->num_regular_reference_symbols(); reference_sym_index++) { + const N64Recomp::ReferenceSymbol& reference_sym = mod.recompiler_context->get_regular_reference_symbol(reference_sym_index); + uint32_t reference_section_vrom = mod.recompiler_context->get_reference_section_rom(reference_sym.section_index); + uint32_t reference_section_vram = mod.recompiler_context->get_reference_section_vram(reference_sym.section_index); + uint32_t reference_symbol_vram = reference_section_vram + reference_sym.section_offset; + + recomp_func_t* found_func = recomp::overlays::get_func_by_section_ram(reference_section_vrom, reference_symbol_vram); + + if (found_func == nullptr) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: 0x" << reference_section_vrom << + " func: 0x" << std::setfill('0') << std::setw(8) << reference_symbol_vram; + error_param = error_param_stream.str(); + return ModLoadError::InvalidReferenceSymbol; + } + + mod.code_handle->set_reference_symbol_pointer(reference_sym_index, found_func); + } + + // Create a list of dependencies ordered by their index in the recompiler context. + std::vector dependencies_ordered{}; + dependencies_ordered.resize(mod.recompiler_context->dependencies_by_name.size()); + + for (const auto& [dependency, dependency_index] : mod.recompiler_context->dependencies_by_name) { + dependencies_ordered[dependency_index] = dependency; + } + + // Imported symbols. + for (size_t import_index = 0; import_index < mod.recompiler_context->import_symbols.size(); import_index++) { + const N64Recomp::ImportSymbol& imported_func = mod.recompiler_context->import_symbols[import_index]; + const std::string& dependency_id = dependencies_ordered[imported_func.dependency_index]; + + GenericFunction func_handle{}; + bool did_find_func = false; + + if (dependency_id == N64Recomp::DependencyBaseRecomp) { + recomp_func_t* func_ptr = recomp::overlays::get_base_export(imported_func.base.name); + did_find_func = func_ptr != nullptr; + func_handle = func_ptr; + } + else if (dependency_id == N64Recomp::DependencySelf) { + did_find_func = mod.get_export_function(imported_func.base.name, func_handle); + } + else { + auto find_mod_it = loaded_mods_by_id.find(dependency_id); + if (find_mod_it == loaded_mods_by_id.end()) { + error_param = dependency_id; + return ModLoadError::MissingDependency; + } + const auto& dependency = opened_mods[find_mod_it->second]; + did_find_func = dependency.get_export_function(imported_func.base.name, func_handle); + } + + if (!did_find_func) { + error_param = dependency_id + ":" + imported_func.base.name; + return ModLoadError::InvalidImport; + } + + mod.code_handle->set_imported_function(import_index, func_handle); + } + + // Register callbacks. + for (const N64Recomp::Callback& callback : mod.recompiler_context->callbacks) { + const N64Recomp::DependencyEvent& dependency_event = mod.recompiler_context->dependency_events[callback.dependency_event_index]; + const std::string& dependency_id = dependencies_ordered[dependency_event.dependency_index]; + GenericFunction func = mod.code_handle->get_function_handle(callback.function_index); + size_t event_index = 0; + bool did_find_event = false; + + if (dependency_id == N64Recomp::DependencyBaseRecomp) { + event_index = recomp::overlays::get_base_event_index(dependency_event.event_name); + if (event_index != (size_t)-1) { + did_find_event = true; + } + } + else if (dependency_id == N64Recomp::DependencySelf) { + did_find_event = mod.get_global_event_index(dependency_event.event_name, event_index); + } + else { + auto find_mod_it = loaded_mods_by_id.find(dependency_id); + if (find_mod_it == loaded_mods_by_id.end()) { + error_param = dependency_id; + return ModLoadError::MissingDependency; + } + const auto& dependency_mod = opened_mods[find_mod_it->second]; + did_find_event = dependency_mod.get_global_event_index(dependency_event.event_name, event_index); + } + + if (!did_find_event) { + error_param = dependency_id + ":" + dependency_event.event_name; + return ModLoadError::InvalidCallbackEvent; + } + + recomp::mods::register_event_callback(event_index, func); + } + + // Populate the mod's state fields. + mod.code_handle->set_recomp_trigger_event_pointer(recomp_trigger_event); + mod.code_handle->set_get_function_pointer(get_function); + mod.code_handle->set_reference_section_addresses_pointer(section_addresses); + for (size_t section_index = 0; section_index < mod.section_load_addresses.size(); section_index++) { + mod.code_handle->set_local_section_address(section_index, mod.section_load_addresses[section_index]); + } + + // Apply all the function replacements in the mod. + for (const auto& replacement : mod.recompiler_context->replacements) { + recomp_func_t* to_replace = recomp::overlays::get_func_by_section_ram(replacement.original_section_vrom, replacement.original_vram); + + if (to_replace == nullptr) { + std::stringstream error_param_stream{}; + error_param_stream << std::hex << + "section: 0x" << replacement.original_section_vrom << + " func: 0x" << std::setfill('0') << std::setw(8) << replacement.original_vram; + error_param = error_param_stream.str(); + return ModLoadError::InvalidFunctionReplacement; + } + + // Check if this function has already been replaced. + auto find_patch_it = patched_funcs.find(to_replace); + if (find_patch_it != patched_funcs.end()) { + error_param = find_patch_it->second.mod_id; + return ModLoadError::ModConflict; + } + + // Copy the original bytes so they can be restored later after the mod is unloaded. + PatchData& cur_replacement_data = patched_funcs[to_replace]; + memcpy(cur_replacement_data.replaced_bytes.data(), reinterpret_cast(to_replace), cur_replacement_data.replaced_bytes.size()); + cur_replacement_data.mod_id = mod.manifest.mod_id; + + // Patch the function to redirect it to the replacement. + patch_func(to_replace, mod.code_handle->get_function_handle(replacement.func_index)); + } + + return ModLoadError::Good; +} + +void recomp::mods::ModContext::unload_mods() { + for (auto& [replacement_func, replacement_data] : patched_funcs) { + unpatch_func(reinterpret_cast(replacement_func), replacement_data); + } + patched_funcs.clear(); + loaded_mods_by_id.clear(); + recomp::mods::reset_events(); + num_events = recomp::overlays::num_base_events(); +} diff --git a/librecomp/src/overlays.cpp b/librecomp/src/overlays.cpp index 7ad7ea5..e546467 100644 --- a/librecomp/src/overlays.cpp +++ b/librecomp/src/overlays.cpp @@ -18,19 +18,6 @@ static SectionTableEntry* patch_code_sections = nullptr; size_t num_patch_code_sections = 0; static std::vector patch_data; -void recomp::overlays::register_overlays(const overlay_section_table_data_t& sections, const overlays_by_index_t& overlays) { - sections_info = sections; - overlays_info = overlays; -} - -void recomp::overlays::register_patches(const char* patch, std::size_t size, SectionTableEntry* sections, size_t num_sections) { - patch_code_sections = sections; - num_patch_code_sections = num_sections; - - patch_data.resize(size); - std::memcpy(patch_data.data(), patch, size); -} - struct LoadedSection { int32_t loaded_ram_addr; size_t section_table_index; @@ -45,8 +32,85 @@ struct LoadedSection { } }; -std::vector loaded_sections{}; -std::unordered_map func_map{}; +static std::unordered_map code_sections_by_rom{}; +static std::vector loaded_sections{}; +static std::unordered_map func_map{}; +static std::unordered_map base_exports{}; +static std::unordered_map base_events; + +extern "C" { +int32_t* section_addresses = nullptr; +} + +void recomp::overlays::register_overlays(const overlay_section_table_data_t& sections, const overlays_by_index_t& overlays) { + sections_info = sections; + overlays_info = overlays; +} + +void recomp::overlays::register_patches(const char* patch, std::size_t size, SectionTableEntry* sections, size_t num_sections) { + patch_code_sections = sections; + num_patch_code_sections = num_sections; + + patch_data.resize(size); + std::memcpy(patch_data.data(), patch, size); +} + +void recomp::overlays::register_base_exports(const FunctionExport* export_list) { + std::unordered_map patch_func_vram_map{}; + + // Iterate over all patch functions to set up a mapping of their vram address. + for (size_t patch_section_index = 0; patch_section_index < num_patch_code_sections; patch_section_index++) { + const SectionTableEntry* cur_section = &patch_code_sections[patch_section_index]; + + for (size_t func_index = 0; func_index < cur_section->num_funcs; func_index++) { + const FuncEntry* cur_func = &cur_section->funcs[func_index]; + patch_func_vram_map.emplace(cur_section->ram_addr + cur_func->offset, cur_func->func); + } + } + + // Iterate over exports, using the vram mapping to create a name mapping. + for (const FunctionExport* cur_export = &export_list[0]; cur_export->name != nullptr; cur_export++) { + auto it = patch_func_vram_map.find(cur_export->ram_addr); + if (it == patch_func_vram_map.end()) { + assert(false && "Failed to find exported function in patch function sections!"); + } + base_exports.emplace(cur_export->name, it->second); + } +} + +recomp_func_t* recomp::overlays::get_base_export(const std::string& export_name) { + auto it = base_exports.find(export_name); + if (it == base_exports.end()) { + return nullptr; + } + return it->second; +} + +void recomp::overlays::register_base_events(char const* const* event_names) { + for (size_t event_index = 0; event_names[event_index] != nullptr; event_index++) { + base_events.emplace(event_names[event_index], event_index); + } +} + +size_t recomp::overlays::get_base_event_index(const std::string& event_name) { + auto it = base_events.find(event_name); + if (it == base_events.end()) { + return (size_t)-1; + } + return it->second; +} + +size_t recomp::overlays::num_base_events() { + return base_events.size(); +} + +const std::unordered_map& recomp::overlays::get_vrom_to_section_map() { + return code_sections_by_rom; +} + +void recomp::overlays::add_loaded_function(int32_t ram, recomp_func_t* func) { + func_map[ram] = func; +} void load_overlay(size_t section_table_index, int32_t ram) { const SectionTableEntry& section = sections_info.code_sections[section_table_index]; @@ -83,10 +147,6 @@ void recomp::overlays::read_patch_data(uint8_t* rdram, gpr patch_data_address) { } } -extern "C" { -int32_t* section_addresses = nullptr; -} - extern "C" void load_overlays(uint32_t rom, int32_t ram_addr, uint32_t size) { // Search for the first section that's included in the loaded rom range // Sections were sorted by `init_overlays` so we can use the bounds functions @@ -174,12 +234,9 @@ extern "C" void unload_overlays(int32_t ram_addr, uint32_t size) { } void recomp::overlays::init_overlays() { + func_map.clear(); section_addresses = (int32_t *)calloc(sections_info.total_num_sections, sizeof(int32_t)); - for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) { - section_addresses[sections_info.code_sections[section_index].index] = sections_info.code_sections[section_index].ram_addr; - } - // Sort the executable sections by rom address std::sort(§ions_info.code_sections[0], §ions_info.code_sections[sections_info.num_code_sections], [](const SectionTableEntry& a, const SectionTableEntry& b) { @@ -187,9 +244,38 @@ void recomp::overlays::init_overlays() { } ); + for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) { + SectionTableEntry* code_section = §ions_info.code_sections[section_index]; + + section_addresses[sections_info.code_sections[section_index].index] = code_section->ram_addr; + code_sections_by_rom[code_section->rom_addr] = section_index; + } + load_patch_functions(); } +recomp_func_t* recomp::overlays::get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram) { + auto find_section_it = code_sections_by_rom.find(section_rom); + if (find_section_it == code_sections_by_rom.end()) { + return nullptr; + } + + SectionTableEntry* section = §ions_info.code_sections[find_section_it->second]; + if (function_vram < section->ram_addr || function_vram >= section->ram_addr + section->size) { + return nullptr; + } + + uint32_t func_offset = function_vram - section->ram_addr; + + for (size_t func_index = 0; func_index < section->num_funcs; func_index++) { + if (section->funcs[func_index].offset == func_offset) { + return section->funcs[func_index].func; + } + } + + return nullptr; +} + extern "C" recomp_func_t * get_function(int32_t addr) { auto func_find = func_map.find(addr); if (func_find == func_map.end()) { diff --git a/librecomp/src/pi.cpp b/librecomp/src/pi.cpp index 855211c..96e178f 100644 --- a/librecomp/src/pi.cpp +++ b/librecomp/src/pi.cpp @@ -4,9 +4,10 @@ #include #include #include -#include "recomp.h" -#include "game.hpp" -#include "files.hpp" +#include "librecomp/recomp.h" +#include "librecomp/addresses.hpp" +#include "librecomp/game.hpp" +#include "librecomp/files.hpp" #include #include @@ -20,13 +21,6 @@ void recomp::set_rom_contents(std::vector&& new_rom) { rom = std::move(new_rom); } -// Flashram occupies the same physical address as sram, but that issue is avoided because libultra exposes -// a high-level interface for flashram. Because that high-level interface is reimplemented, low level accesses -// that involve physical addresses don't need to be handled for flashram. -constexpr uint32_t sram_base = 0x08000000; -constexpr uint32_t rom_base = 0x10000000; -constexpr uint32_t drive_base = 0x06000000; - constexpr uint32_t k1_to_phys(uint32_t addr) { return addr & 0x1FFFFFFF; } @@ -42,21 +36,21 @@ extern "C" void __osPiRelAccess_recomp(uint8_t* rdram, recomp_context* ctx) { } extern "C" void osCartRomInit_recomp(uint8_t* rdram, recomp_context* ctx) { - OSPiHandle* handle = TO_PTR(OSPiHandle, ultramodern::cart_handle); + OSPiHandle* handle = TO_PTR(OSPiHandle, recomp::cart_handle); handle->type = 0; // cart - handle->baseAddress = phys_to_k1(rom_base); + handle->baseAddress = phys_to_k1(recomp::rom_base); handle->domain = 0; - ctx->r2 = (gpr)ultramodern::cart_handle; + ctx->r2 = (gpr)recomp::cart_handle; } extern "C" void osDriveRomInit_recomp(uint8_t * rdram, recomp_context * ctx) { - OSPiHandle* handle = TO_PTR(OSPiHandle, ultramodern::drive_handle); + OSPiHandle* handle = TO_PTR(OSPiHandle, recomp::drive_handle); handle->type = 1; // bulk - handle->baseAddress = phys_to_k1(drive_base); + handle->baseAddress = phys_to_k1(recomp::drive_base); handle->domain = 0; - ctx->r2 = (gpr)ultramodern::drive_handle; + ctx->r2 = (gpr)recomp::drive_handle; } extern "C" void osCreatePiManager_recomp(uint8_t* rdram, recomp_context* ctx) { @@ -70,7 +64,7 @@ void recomp::do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr assert((physical_addr & 0x1) == 0 && "Only PI DMA from aligned ROM addresses is currently supported"); assert((ram_address & 0x7) == 0 && "Only PI DMA to aligned RDRAM addresses is currently supported"); assert((num_bytes & 0x1) == 0 && "Only PI DMA with aligned sizes is currently supported"); - uint8_t* rom_addr = rom.data() + physical_addr - rom_base; + uint8_t* rom_addr = rom.data() + physical_addr - recomp::rom_base; for (size_t i = 0; i < num_bytes; i++) { MEM_B(i, ram_address) = *rom_addr; rom_addr++; @@ -80,7 +74,7 @@ void recomp::do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr void recomp::do_rom_pio(uint8_t* rdram, gpr ram_address, uint32_t physical_addr) { assert((physical_addr & 0x3) == 0 && "PIO not 4-byte aligned in device, currently unsupported"); assert((ram_address & 0x3) == 0 && "PIO not 4-byte aligned in RDRAM, currently unsupported"); - uint8_t* rom_addr = rom.data() + physical_addr - rom_base; + uint8_t* rom_addr = rom.data() + physical_addr - recomp::rom_base; MEM_B(0, ram_address) = *rom_addr++; MEM_B(1, ram_address) = *rom_addr++; MEM_B(2, ram_address) = *rom_addr++; @@ -213,15 +207,15 @@ void do_dma(RDRAM_ARG PTR(OSMesgQueue) mq, gpr rdram_address, uint32_t physical_ // TODO asynchronous transfer // TODO implement unaligned DMA correctly if (direction == 0) { - if (physical_addr >= rom_base) { + if (physical_addr >= recomp::rom_base) { // read cart rom recomp::do_rom_read(rdram, rdram_address, physical_addr, size); // Send a message to the mq to indicate that the transfer completed osSendMesg(rdram, mq, 0, OS_MESG_NOBLOCK); - } else if (physical_addr >= sram_base) { + } else if (physical_addr >= recomp::sram_base) { // read sram - save_read(rdram, rdram_address, physical_addr - sram_base, size); + save_read(rdram, rdram_address, physical_addr - recomp::sram_base, size); // Send a message to the mq to indicate that the transfer completed osSendMesg(rdram, mq, 0, OS_MESG_NOBLOCK); @@ -229,12 +223,12 @@ void do_dma(RDRAM_ARG PTR(OSMesgQueue) mq, gpr rdram_address, uint32_t physical_ fprintf(stderr, "[WARN] PI DMA read from unknown region, phys address 0x%08X\n", physical_addr); } } else { - if (physical_addr >= rom_base) { + if (physical_addr >= recomp::rom_base) { // write cart rom throw std::runtime_error("ROM DMA write unimplemented"); - } else if (physical_addr >= sram_base) { + } else if (physical_addr >= recomp::sram_base) { // write sram - save_write(rdram, rdram_address, physical_addr - sram_base, size); + save_write(rdram, rdram_address, physical_addr - recomp::sram_base, size); // Send a message to the mq to indicate that the transfer completed osSendMesg(rdram, mq, 0, OS_MESG_NOBLOCK); @@ -248,7 +242,7 @@ extern "C" void osPiStartDma_recomp(RDRAM_ARG recomp_context* ctx) { uint32_t mb = ctx->r4; uint32_t pri = ctx->r5; uint32_t direction = ctx->r6; - uint32_t devAddr = ctx->r7 | rom_base; + uint32_t devAddr = ctx->r7 | recomp::rom_base; gpr dramAddr = MEM_W(0x10, ctx->r29); uint32_t size = MEM_W(0x14, ctx->r29); PTR(OSMesgQueue) mq = MEM_W(0x18, ctx->r29); @@ -284,7 +278,7 @@ extern "C" void osEPiReadIo_recomp(RDRAM_ARG recomp_context * ctx) { gpr dramAddr = ctx->r6; uint32_t physical_addr = k1_to_phys(devAddr); - if (physical_addr > rom_base) { + if (physical_addr > recomp::rom_base) { // cart rom recomp::do_rom_pio(PASS_RDRAM dramAddr, physical_addr); } else { diff --git a/librecomp/src/recomp.cpp b/librecomp/src/recomp.cpp index 75f8a05..82b6d52 100644 --- a/librecomp/src/recomp.cpp +++ b/librecomp/src/recomp.cpp @@ -6,17 +6,29 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include -#include "recomp.h" -#include "overlays.hpp" -#include "game.hpp" +#include "librecomp/recomp.h" +#include "librecomp/overlays.hpp" +#include "librecomp/game.hpp" #include "xxHash/xxh3.h" #include "ultramodern/ultramodern.hpp" #include "ultramodern/error_handling.hpp" +#include "librecomp/addresses.hpp" +#include "librecomp/mods.hpp" + +#if defined(_WIN32) +#define PATHFMT "%ls" +#else +#define PATHFMT "%s" +#endif #ifdef _MSC_VER inline uint32_t byteswap(uint32_t val) { @@ -37,10 +49,16 @@ enum GameStatus { // Mutexes std::mutex game_roms_mutex; std::mutex current_game_mutex; +std::mutex mod_context_mutex{}; // Global variables std::filesystem::path config_path; +// Maps game_id to the game's entry. std::unordered_map game_roms {}; +// The global mod context. +std::unique_ptr mod_context = std::make_unique(); +// The project's version. +recomp::Version project_version; std::u8string recomp::GameEntry::stored_filename() const { return game_id + u8".z64"; @@ -51,11 +69,30 @@ void recomp::register_config_path(std::filesystem::path path) { } bool recomp::register_game(const recomp::GameEntry& entry) { - std::lock_guard lock(game_roms_mutex); - game_roms.insert({ entry.game_id, entry }); + // TODO verify that there's no game with this ID already. + { + std::lock_guard lock(game_roms_mutex); + game_roms.insert({ entry.game_id, entry }); + } + if (!entry.mod_game_id.empty()) { + std::lock_guard lock(mod_context_mutex); + mod_context->register_game(entry.mod_game_id); + } + return true; } +void recomp::mods::scan_mods() { + std::vector mod_open_errors; + { + std::lock_guard mod_lock{ mod_context_mutex }; + mod_open_errors = mod_context->scan_mod_folder(config_path / "mods"); + } + for (const auto& cur_error : mod_open_errors) { + printf("Error opening mod " PATHFMT ": %s (%s)\n", cur_error.mod_path.c_str(), recomp::mods::error_to_string(cur_error.error).c_str(), cur_error.error_param.c_str()); + } +} + bool check_hash(const std::vector& rom_data, uint64_t expected_hash) { uint64_t calculated_hash = XXH3_64bits(rom_data.data(), rom_data.size()); return calculated_hash == expected_hash; @@ -134,6 +171,70 @@ bool recomp::load_stored_rom(std::u8string& game_id) { return true; } +const recomp::Version& recomp::get_project_version() { + return project_version; +} + +bool recomp::Version::from_string(const std::string& str, Version& out) { + std::array period_indices; + size_t num_periods = 0; + size_t cur_pos = 0; + uint16_t major; + uint16_t minor; + uint16_t patch; + std::string suffix; + + // Find the 2 required periods. + cur_pos = str.find('.', cur_pos); + period_indices[0] = cur_pos; + cur_pos = str.find('.', cur_pos + 1); + period_indices[1] = cur_pos; + + // Check that both were found. + if (period_indices[0] == std::string::npos || period_indices[1] == std::string::npos) { + return false; + } + + // Parse the 3 numbers formed by splitting the string via the periods. + std::array parse_results; + std::array parse_starts { 0, period_indices[0] + 1, period_indices[1] + 1 }; + std::array parse_ends { period_indices[0], period_indices[1], str.size() }; + parse_results[0] = std::from_chars(str.data() + parse_starts[0], str.data() + parse_ends[0], major); + parse_results[1] = std::from_chars(str.data() + parse_starts[1], str.data() + parse_ends[1], minor); + parse_results[2] = std::from_chars(str.data() + parse_starts[2], str.data() + parse_ends[2], patch); + + // Check that the first two parsed correctly. + auto did_parse = [&](size_t i) { + return parse_results[i].ec == std::errc{} && parse_results[i].ptr == str.data() + parse_ends[i]; + }; + + if (!did_parse(0) || !did_parse(1)) { + return false; + } + + // Check that the third had a successful parse, but not necessarily read all the characters. + if (parse_results[2].ec != std::errc{}) { + return false; + } + + // Allow a plus or minus directly after the third number. + if (parse_results[2].ptr != str.data() + parse_ends[2]) { + if (*parse_results[2].ptr == '+' || *parse_results[2].ptr == '-') { + suffix = str.substr(std::distance(str.data(), parse_results[2].ptr)); + } + // Failed to parse, as nothing is allowed directly after the last number besides a plus or minus. + else { + return false; + } + } + + out.major = major; + out.minor = minor; + out.patch = patch; + out.suffix = std::move(suffix); + return true; +} + const std::array first_rom_bytes { 0x80, 0x37, 0x12, 0x40 }; enum class ByteswapType { @@ -322,7 +423,7 @@ void init(uint8_t* rdram, recomp_context* ctx, gpr entrypoint) { recomp::do_rom_read(rdram, entrypoint, 0x10001000, 0x100000); // Read in any extra data from patches - recomp::overlays::read_patch_data(rdram, (gpr)(s32)0x80801000); + recomp::overlays::read_patch_data(rdram, (gpr)recomp::patch_rdram_start); // Set up context floats ctx->f_odd = &ctx->f0.u32h; @@ -373,7 +474,83 @@ void ultramodern::quit() { current_game.reset(); } +void recomp::mods::enable_mod(const std::string& mod_id, bool enabled) { + std::lock_guard lock { mod_context_mutex }; + return mod_context->enable_mod(mod_id, enabled); +} + +bool recomp::mods::is_mod_enabled(const std::string& mod_id) { + std::lock_guard lock { mod_context_mutex }; + return mod_context->is_mod_enabled(mod_id); +} + +std::vector recomp::mods::get_mod_details(const std::string& mod_game_id) { + std::lock_guard lock { mod_context_mutex }; + return mod_context->get_mod_details(mod_game_id); +} + +bool wait_for_game_started(uint8_t* rdram, recomp_context* context) { + game_status.wait(GameStatus::None); + + switch (game_status.load()) { + // TODO refactor this to allow a project to specify what entrypoint function to run for a give game. + case GameStatus::Running: + { + if (!recomp::load_stored_rom(current_game.value())) { + ultramodern::error_handling::message_box("Error opening stored ROM! Please restart this program."); + } + + auto find_it = game_roms.find(current_game.value()); + const recomp::GameEntry& game_entry = find_it->second; + + init(rdram, context, game_entry.entrypoint_address); + + if (!game_entry.mod_game_id.empty()) { + uint32_t mod_ram_used = 0; + std::vector mod_load_errors; + { + std::lock_guard lock { mod_context_mutex }; + mod_load_errors = mod_context->load_mods(game_entry.mod_game_id, rdram, recomp::mod_rdram_start, mod_ram_used); + } + + if (!mod_load_errors.empty()) { + std::ostringstream mod_error_stream; + mod_error_stream << "Error loading mods:\n\n"; + for (const auto& cur_error : mod_load_errors) { + mod_error_stream << cur_error.mod_id.c_str() << ": " << recomp::mods::error_to_string(cur_error.error); + if (!cur_error.error_param.empty()) { + mod_error_stream << " (" << cur_error.error_param.c_str() << ")"; + } + mod_error_stream << "\n"; + } + ultramodern::error_handling::message_box(mod_error_stream.str().c_str()); + game_status.store(GameStatus::None); + return false; + } + } + + ultramodern::init_saving(rdram); + ultramodern::load_shader_cache(game_entry.cache_data); + + try { + game_entry.entrypoint(rdram, context); + } catch (ultramodern::thread_terminated& terminated) { + + } + } + return true; + + case GameStatus::Quit: + return true; + + case GameStatus::None: + return true; + } +} + void recomp::start( + uint32_t rdram_size, + const recomp::Version& version, ultramodern::renderer::WindowHandle window_handle, const recomp::rsp::callbacks_t& rsp_callbacks, const ultramodern::renderer::callbacks_t& renderer_callbacks, @@ -384,6 +561,7 @@ void recomp::start( const ultramodern::error_handling::callbacks_t& error_handling_callbacks, const ultramodern::threads::callbacks_t& threads_callbacks ) { + project_version = version; recomp::check_all_stored_roms(); recomp::rsp::set_callbacks(rsp_callbacks); @@ -413,8 +591,8 @@ void recomp::start( } // Allocate rdram_buffer - std::unique_ptr rdram_buffer = std::make_unique(ultramodern::rdram_size); - std::memset(rdram_buffer.get(), 0, ultramodern::rdram_size); + std::unique_ptr rdram_buffer = std::make_unique(rdram_size); + std::memset(rdram_buffer.get(), 0, rdram_size); std::thread game_thread{[](ultramodern::renderer::WindowHandle window_handle, uint8_t* rdram) { debug_printf("[Recomp] Starting\n"); @@ -423,40 +601,10 @@ void recomp::start( ultramodern::preinit(rdram, window_handle); - game_status.wait(GameStatus::None); recomp_context context{}; - switch (game_status.load()) { - // TODO refactor this to allow a project to specify what entrypoint function to run for a give game. - case GameStatus::Running: - { - if (!recomp::load_stored_rom(current_game.value())) { - ultramodern::error_handling::message_box("Error opening stored ROM! Please restart this program."); - } - - ultramodern::init_saving(rdram); - - auto find_it = game_roms.find(current_game.value()); - const recomp::GameEntry& game_entry = find_it->second; - - ultramodern::load_shader_cache(game_entry.cache_data); - init(rdram, &context, game_entry.entrypoint_address); - try { - game_entry.entrypoint(rdram, &context); - } catch (ultramodern::thread_terminated& terminated) { - - } - } - break; - - case GameStatus::Quit: - break; - - case GameStatus::None: - break; - } - - debug_printf("[Recomp] Quitting\n"); + // Loop until the game starts. + while (!wait_for_game_started(rdram, &context)) {} }, window_handle, rdram_buffer.get()}; while (!exited) { diff --git a/thirdparty/miniz b/thirdparty/miniz new file mode 160000 index 0000000..8573fd7 --- /dev/null +++ b/thirdparty/miniz @@ -0,0 +1 @@ +Subproject commit 8573fd7cd6f49b262a0ccc447f3c6acfc415e556 diff --git a/ultramodern/include/ultramodern/ultramodern.hpp b/ultramodern/include/ultramodern/ultramodern.hpp index b144eaa..07019d4 100644 --- a/ultramodern/include/ultramodern/ultramodern.hpp +++ b/ultramodern/include/ultramodern/ultramodern.hpp @@ -27,11 +27,6 @@ struct UltraThreadContext { namespace ultramodern { -// We need a place in rdram to hold the PI handles, so pick an address in extended rdram -constexpr uint32_t rdram_size = 1024 * 1024 * 16; // 16MB to give extra room for anything custom -constexpr int32_t cart_handle = 0x80800000; -constexpr int32_t drive_handle = (int32_t)(cart_handle + sizeof(OSPiHandle)); -constexpr int32_t flash_handle = (int32_t)(drive_handle + sizeof(OSPiHandle)); constexpr uint32_t save_size = 1024 * 1024 / 8; // Maximum save size, 1Mbit for flash // Initialization.