thirdpartymirrors/N64ModernRuntime
1
0
Fork 0
mirror of https://github.com/N64Recomp/N64ModernRuntime.git synced 2025-12-16 13:02:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Runtime Mod Support PR 1 (Mod framework and mod loading) (#60)

Browse source 
* Added miniz and implemented mod manifest loading from zip

* Add more mod loading error enums

* Added ability to load mods from directories instead of zip files

* Added validation for mod contents and mod load error strings

* Add checks for required fields existing in manifest

* Renamed load_mod to open_mod and ModLoadError to ModOpenError

* Made rdram size an argument in recomp::start, reorganized some PI address constants

* Add N64Recomp as submodule and initial mod loading (temporarily using a DLL instead of Lua recompilation)

* Created ModContext and exposed functionality for searching mod folders, activating mods, and loading active mods

* Implemented per-game mod contexts

* Add duplicate mod detection

* Added function conflict detection, recovery from failed mod loading and message box with load errors

* Update N64Recomp for new modding infrastructure, begin adding support for offline compiled mods

* Reorganized mod loading, added mod dependency validation

* Reorganized mod code handle code

* Implement importing functions from other mods

* Implement mod events and callbacks

* Add support for creating events in patches

* Add support for exporting functions from native libraries in mods

* Removed per-game mod subdirectories and added the mod's corresponding game id to the manifest

* Added version parsing with version number as argument to recomp initialization and minimum recomp versions for mods

* Changed mod binary and mod symbol files to use fixed paths, removed them from the manifest

* Expose function for getting the details for currently opened mods

* Add dependencies and authors to manifest and update N64Recomp submodule

* Implement mod loading on posix systems

* Implement mip32 relocs for mod loading and add mod functions to function lookup table

* Add CPU architecture detection and gate current mod function hooking behind x86_64 define

* Fix build on compilers that don't properly support aggregate initialization

* Fix compilation on ARM64 and Apple

* Fix compilation on MacOS x86_64

* Update N64Recomp commit after merge

* Fix whitespace in game.hpp
This commit is contained in:
Wiseguy 2024-09-09 23:16:07 -04:00 • committed by GitHub
parent 0a53855333
commit 45e9f7a6cb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 2276 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitmodules vendored
View file

@ -1,3 +1,9 @@
[submodule "thirdparty/xxHash"]
path = thirdparty/xxHash
url = https://github.com/Cyan4973/xxHash.git
[submodule "thirdparty/miniz"]
path = thirdparty/miniz
url = https://github.com/richgel999/miniz
[submodule "N64Recomp"]
path = N64Recomp
url = https://github.com/N64Recomp/N64Recomp

1
N64Recomp Submodule

@ -0,0 +1 @@
Subproject commit cc71b31b09a927f558e142598ffcca7d146b454b

9
librecomp/CMakeLists.txt
View file

@ -15,6 +15,9 @@ add_library(librecomp STATIC
"${CMAKE_CURRENT_SOURCE_DIR}/src/files.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/flash.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/math_routines.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/mods.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/mod_events.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/mod_manifest.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/overlays.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/pak.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/pi.cpp"
@ -46,4 +49,8 @@ if (WIN32)
add_compile_definitions(NOMINMAX)
endif()
target_link_libraries(librecomp PRIVATE ultramodern)
add_subdirectory(${PROJECT_SOURCE_DIR}/../thirdparty/miniz ${CMAKE_CURRENT_BINARY_DIR}/miniz)
add_subdirectory(${PROJECT_SOURCE_DIR}/../N64Recomp ${CMAKE_CURRENT_BINARY_DIR}/N64Recomp EXCLUDE_FROM_ALL)
target_link_libraries(librecomp PRIVATE ultramodern N64Recomp)
target_link_libraries(librecomp PUBLIC miniz)

25
librecomp/include/librecomp/addresses.hpp Normal file
View file

@ -0,0 +1,25 @@
#ifndef __RECOMP_ADDRESSES_HPP__
#define __RECOMP_ADDRESSES_HPP__
#include <cstdint>
#include "ultramodern/ultra64.h"
namespace recomp {
// We need a place in rdram to hold the PI handles, so pick an address in extended rdram
constexpr int32_t cart_handle = 0x80800000;
constexpr int32_t drive_handle = (int32_t)(cart_handle + sizeof(OSPiHandle));
constexpr int32_t flash_handle = (int32_t)(drive_handle + sizeof(OSPiHandle));
constexpr int32_t flash_handle_end = (int32_t)(flash_handle + sizeof(OSPiHandle));
constexpr int32_t patch_rdram_start = 0x80801000;
static_assert(patch_rdram_start >= flash_handle_end);
constexpr int32_t mod_rdram_start = 0x81000000;
// Flashram occupies the same physical address as sram, but that issue is avoided because libultra exposes
// a high-level interface for flashram. Because that high-level interface is reimplemented, low level accesses
// that involve physical addresses don't need to be handled for flashram.
constexpr uint32_t sram_base = 0x08000000;
constexpr uint32_t rom_base = 0x10000000;
constexpr uint32_t drive_base = 0x06000000;
}
#endif

82
librecomp/include/librecomp/game.hpp
View file

@ -13,6 +13,7 @@ namespace recomp {
uint64_t rom_hash;
std::string internal_name;
std::u8string game_id;
std::string mod_game_id;
std::span<const char> cache_data;
bool is_enabled;
@ -21,34 +22,59 @@ namespace recomp {
std::u8string stored_filename() const;
};
enum class RomValidationError {
Good,
FailedToOpen,
NotARom,
IncorrectRom,
NotYet,
IncorrectVersion,
OtherError
};
void register_config_path(std::filesystem::path path);
bool register_game(const recomp::GameEntry& entry);
void check_all_stored_roms();
bool load_stored_rom(std::u8string& game_id);
RomValidationError select_rom(const std::filesystem::path& rom_path, std::u8string& game_id);
bool is_rom_valid(std::u8string& game_id);
bool is_rom_loaded();
void set_rom_contents(std::vector<uint8_t>&& new_rom);
void do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr, size_t num_bytes);
void do_rom_pio(uint8_t* rdram, gpr ram_address, uint32_t physical_addr);
struct Version {
int major = -1;
int minor = -1;
int patch = -1;
std::string suffix;
/**
* The following arguments contain mandatory callbacks that need to be registered (i.e., can't be `nullptr`):
* - `rsp_callbacks`
* - `renderer_callbacks`
*
* It must be called only once and it must be called before `ultramodern::preinit`.
*/
std::string to_string() const {
return std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(patch) + suffix;
}
static bool from_string(const std::string& str, Version& out);
auto operator<=>(const Version& rhs) const {
if (major != rhs.major) {
return major <=> rhs.major;
}
if (minor != rhs.minor) {
return minor <=> rhs.minor;
}
return patch <=> rhs.patch;
}
};
enum class RomValidationError {
Good,
FailedToOpen,
NotARom,
IncorrectRom,
NotYet,
IncorrectVersion,
OtherError
};
void register_config_path(std::filesystem::path path);
bool register_game(const recomp::GameEntry& entry);
void check_all_stored_roms();
bool load_stored_rom(std::u8string& game_id);
RomValidationError select_rom(const std::filesystem::path& rom_path, std::u8string& game_id);
bool is_rom_valid(std::u8string& game_id);
bool is_rom_loaded();
void set_rom_contents(std::vector<uint8_t>&& new_rom);
void do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr, size_t num_bytes);
void do_rom_pio(uint8_t* rdram, gpr ram_address, uint32_t physical_addr);
const Version& get_project_version();
/**
* The following arguments contain mandatory callbacks that need to be registered (i.e., can't be `nullptr`):
* - `rsp_callbacks`
* - `renderer_callbacks`
*
* It must be called only once and it must be called before `ultramodern::preinit`.
*/
void start(
uint32_t rdram_size,
const Version& project_version,
ultramodern::renderer::WindowHandle window_handle,
const recomp::rsp::callbacks_t& rsp_callbacks,
const ultramodern::renderer::callbacks_t& renderer_callbacks,
@ -60,8 +86,8 @@ namespace recomp {
const ultramodern::threads::callbacks_t& threads_callbacks
);
void start_game(const std::u8string& game_id);
std::u8string current_game_id();
void start_game(const std::u8string& game_id);
std::u8string current_game_id();
}
#endif

316
librecomp/include/librecomp/mods.hpp Normal file
View file

@ -0,0 +1,316 @@
#ifndef __RECOMP_MODS_HPP__
#define __RECOMP_MODS_HPP__
#include <filesystem>
#include <string>
#include <fstream>
#include <cstdio>
#include <vector>
#include <memory>
#include <tuple>
#include <unordered_set>
#include <unordered_map>
#include <array>
#include <cstddef>
#include <variant>
#define MINIZ_NO_DEFLATE_APIS
#define MINIZ_NO_ARCHIVE_WRITING_APIS
#include "miniz.h"
#include "miniz_zip.h"
#include "librecomp/game.hpp"
#include "librecomp/recomp.h"
#include "librecomp/sections.h"
namespace N64Recomp {
class Context;
};
namespace recomp {
namespace mods {
enum class ModOpenError {
Good,
DoesNotExist,
NotAFileOrFolder,
FileError,
InvalidZip,
NoManifest,
FailedToParseManifest,
InvalidManifestSchema,
UnrecognizedManifestField,
IncorrectManifestFieldType,
InvalidVersionString,
InvalidMinimumRecompVersionString,
InvalidDependencyString,
MissingManifestField,
DuplicateMod,
WrongGame
};
std::string error_to_string(ModOpenError);
enum class ModLoadError {
Good,
InvalidGame,
MinimumRecompVersionNotMet,
HasSymsButNoBinary,
HasBinaryButNoSyms,
FailedToParseSyms,
FailedToLoadNativeCode,
FailedToLoadNativeLibrary,
FailedToFindNativeExport,
InvalidReferenceSymbol,
InvalidImport,
InvalidCallbackEvent,
InvalidFunctionReplacement,
FailedToFindReplacement,
ReplacementConflict,
MissingDependencyInManifest,
MissingDependency,
WrongDependencyVersion,
ModConflict,
DuplicateExport,
NoSpecifiedApiVersion,
UnsupportedApiVersion,
};
std::string error_to_string(ModLoadError);
struct ModFileHandle {
virtual ~ModFileHandle() = default;
virtual std::vector<char> read_file(const std::string& filepath, bool& exists) const = 0;
virtual bool file_exists(const std::string& filepath) const = 0;
};
struct ZipModFileHandle final : public ModFileHandle {
FILE* file_handle = nullptr;
std::unique_ptr<mz_zip_archive> archive;
ZipModFileHandle() = default;
ZipModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error);
~ZipModFileHandle() final;
std::vector<char> read_file(const std::string& filepath, bool& exists) const final;
bool file_exists(const std::string& filepath) const final;
};
struct LooseModFileHandle final : public ModFileHandle {
std::filesystem::path root_path;
LooseModFileHandle() = default;
LooseModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error);
~LooseModFileHandle() final;
std::vector<char> read_file(const std::string& filepath, bool& exists) const final;
bool file_exists(const std::string& filepath) const final;
};
struct NativeLibraryManifest {
std::string name;
std::vector<std::string> exports;
};
struct Dependency {
std::string mod_id;
Version version;
};
struct ModDetails {
std::string mod_id;
Version version;
std::vector<std::string> authors;
std::vector<Dependency> dependencies;
};
struct ModManifest {
std::filesystem::path mod_root_path;
std::vector<std::string> mod_game_ids;
std::string mod_id;
std::vector<std::string> authors;
std::vector<Dependency> dependencies;
std::unordered_map<std::string, size_t> dependencies_by_id;
Version minimum_recomp_version;
Version version;
std::vector<NativeLibraryManifest> native_libraries;
std::unique_ptr<ModFileHandle> file_handle;
};
struct ModOpenErrorDetails {
std::filesystem::path mod_path;
ModOpenError error;
std::string error_param;
ModOpenErrorDetails() = default;
ModOpenErrorDetails(const std::filesystem::path& mod_path_, ModOpenError error_, const std::string& error_param_) :
mod_path(mod_path_), error(error_), error_param(error_param_) {}
};
struct ModLoadErrorDetails {
std::string mod_id;
ModLoadError error;
std::string error_param;
ModLoadErrorDetails() = default;
ModLoadErrorDetails(const std::string& mod_id_, ModLoadError error_, const std::string& error_param_) :
mod_id(mod_id_), error(error_), error_param(error_param_) {}
};
void scan_mods();
void enable_mod(const std::string& mod_id, bool enabled);
bool is_mod_enabled(const std::string& mod_id);
std::vector<ModDetails> get_mod_details(const std::string& mod_game_id);
// Internal functions, TODO move to an internal header.
struct PatchData {
std::array<std::byte, 16> replaced_bytes;
std::string mod_id;
};
using GenericFunction = std::variant<recomp_func_t*>;
class ModHandle;
class ModContext {
public:
ModContext();
~ModContext();
void register_game(const std::string& mod_game_id);
std::vector<ModOpenErrorDetails> scan_mod_folder(const std::filesystem::path& mod_folder);
void enable_mod(const std::string& mod_id, bool enabled);
bool is_mod_enabled(const std::string& mod_id);
size_t num_opened_mods();
std::vector<ModLoadErrorDetails> load_mods(const std::string& mod_game_id, uint8_t* rdram, int32_t load_address, uint32_t& ram_used);
void unload_mods();
std::vector<ModDetails> get_mod_details(const std::string& mod_game_id);
private:
ModOpenError open_mod(const std::filesystem::path& mod_path, std::string& error_param);
ModLoadError load_mod(uint8_t* rdram, const std::unordered_map<uint32_t, uint16_t>& section_map, recomp::mods::ModHandle& handle, int32_t load_address, uint32_t& ram_used, std::string& error_param);
void check_dependencies(recomp::mods::ModHandle& mod, std::vector<std::pair<recomp::mods::ModLoadError, std::string>>& errors);
ModLoadError load_mod_code(recomp::mods::ModHandle& mod, std::string& error_param);
ModLoadError resolve_dependencies(recomp::mods::ModHandle& mod, std::string& error_param);
void add_opened_mod(ModManifest&& manifest, std::vector<size_t>&& game_indices);
// Maps game mod ID to the mod's internal integer ID.
std::unordered_map<std::string, size_t> mod_game_ids;
std::vector<ModHandle> opened_mods;
std::unordered_set<std::string> mod_ids;
std::unordered_set<std::string> enabled_mods;
std::unordered_map<recomp_func_t*, PatchData> patched_funcs;
std::unordered_map<std::string, size_t> loaded_mods_by_id;
size_t num_events = 0;
};
class ModCodeHandle {
public:
virtual ~ModCodeHandle() {}
virtual bool good() = 0;
virtual uint32_t get_api_version() = 0;
virtual void set_imported_function(size_t import_index, GenericFunction func) = 0;
virtual void set_reference_symbol_pointer(size_t symbol_index, recomp_func_t* ptr) = 0;
virtual void set_base_event_index(uint32_t global_event_index) = 0;
virtual uint32_t get_base_event_index() = 0;
virtual void set_recomp_trigger_event_pointer(void (*ptr)(uint8_t* rdram, recomp_context* ctx, uint32_t index)) = 0;
virtual void set_get_function_pointer(recomp_func_t* (*ptr)(int32_t)) = 0;
virtual void set_reference_section_addresses_pointer(int32_t* ptr) = 0;
virtual void set_local_section_address(size_t section_index, int32_t address) = 0;
virtual GenericFunction get_function_handle(size_t func_index) = 0;
};
class DynamicLibrary;
class ModHandle {
public:
// TODO make these private and expose methods for the functionality they're currently used in.
ModManifest manifest;
std::unique_ptr<ModCodeHandle> code_handle;
std::unique_ptr<N64Recomp::Context> recompiler_context;
std::vector<uint32_t> section_load_addresses;
ModHandle(ModManifest&& manifest, std::vector<size_t>&& game_indices);
ModHandle(const ModHandle& rhs) = delete;
ModHandle& operator=(const ModHandle& rhs) = delete;
ModHandle(ModHandle&& rhs);
ModHandle& operator=(ModHandle&& rhs);
~ModHandle();
size_t num_exports() const;
size_t num_events() const;
ModLoadError populate_exports(std::string& error_param);
bool get_export_function(const std::string& export_name, GenericFunction& out) const;
ModLoadError populate_events(size_t base_event_index, std::string& error_param);
bool get_global_event_index(const std::string& event_name, size_t& event_index_out) const;
ModLoadError load_native_library(const NativeLibraryManifest& lib_manifest, std::string& error_param);
bool is_for_game(size_t game_index) const {
auto find_it = std::find(game_indices.begin(), game_indices.end(), game_index);
return find_it != game_indices.end();
}
private:
// Mapping of export name to function index.
std::unordered_map<std::string, size_t> exports_by_name;
// Mapping of export name to native library function pointer.
std::unordered_map<std::string, recomp_func_t*> native_library_exports;
// Mapping of event name to local index.
std::unordered_map<std::string, size_t> events_by_name;
// Loaded dynamic libraries.
std::vector<std::unique_ptr<DynamicLibrary>> native_libraries; // Vector of pointers so that implementation can be elsewhere.
// Games that this mod supports.
std::vector<size_t> game_indices;
};
class NativeCodeHandle : public ModCodeHandle {
public:
NativeCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context);
~NativeCodeHandle() = default;
bool good() final;
uint32_t get_api_version() final;
void set_imported_function(size_t import_index, GenericFunction func) final;
void set_reference_symbol_pointer(size_t symbol_index, recomp_func_t* ptr) final {
reference_symbol_funcs[symbol_index] = ptr;
};
void set_base_event_index(uint32_t global_event_index) final {
*base_event_index = global_event_index;
};
uint32_t get_base_event_index() final {
return *base_event_index;
}
void set_recomp_trigger_event_pointer(void (*ptr)(uint8_t* rdram, recomp_context* ctx, uint32_t index)) final {
*recomp_trigger_event = ptr;
};
void set_get_function_pointer(recomp_func_t* (*ptr)(int32_t)) final {
*get_function = ptr;
};
void set_reference_section_addresses_pointer(int32_t* ptr) final {
*reference_section_addresses = ptr;
};
void set_local_section_address(size_t section_index, int32_t address) final {
section_addresses[section_index] = address;
};
GenericFunction get_function_handle(size_t func_index) final {
return GenericFunction{ functions[func_index] };
}
private:
void set_bad();
bool is_good = false;
std::unique_ptr<DynamicLibrary> dynamic_lib;
std::vector<recomp_func_t*> functions;
recomp_func_t** imported_funcs;
recomp_func_t** reference_symbol_funcs;
uint32_t* base_event_index;
void (**recomp_trigger_event)(uint8_t* rdram, recomp_context* ctx, uint32_t index);
recomp_func_t* (**get_function)(int32_t vram);
int32_t** reference_section_addresses;
int32_t* section_addresses;
};
void setup_events(size_t num_events);
void register_event_callback(size_t event_index, GenericFunction callback);
void reset_events();
ModLoadError validate_api_version(uint32_t api_version, std::string& error_param);
}
};
extern "C" void recomp_trigger_event(uint8_t* rdram, recomp_context* ctx, uint32_t event_index);
#endif

11
librecomp/include/librecomp/overlays.hpp
View file

@ -3,6 +3,8 @@
#include <cstdint>
#include <cstddef>
#include <string>
#include <unordered_map>
#include "sections.h"
namespace recomp {
@ -21,9 +23,18 @@ namespace recomp {
void register_overlays(const overlay_section_table_data_t& sections, const overlays_by_index_t& overlays);
void register_patches(const char* patch_data, size_t patch_size, SectionTableEntry* code_sections, size_t num_sections);
void register_base_exports(const FunctionExport* exports);
void register_base_events(char const* const* event_names);
void read_patch_data(uint8_t* rdram, gpr patch_data_address);
void init_overlays();
const std::unordered_map<uint32_t, uint16_t>& get_vrom_to_section_map();
recomp_func_t* get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram);
recomp_func_t* get_base_export(const std::string& export_name);
size_t get_base_event_index(const std::string& event_name);
size_t num_base_events();
void add_loaded_function(int32_t ram_addr, recomp_func_t* func);
}
};

5
librecomp/include/librecomp/sections.h
View file

@ -20,4 +20,9 @@ typedef struct {
size_t index;
} SectionTableEntry;
typedef struct {
const char* name;
uint32_t ram_addr;
} FunctionExport;
#endif

5
librecomp/src/flash.cpp
View file

@ -2,7 +2,8 @@
#include <cassert>
#include <ultramodern/ultra64.h>
#include <ultramodern/ultramodern.hpp>
#include "recomp.h"
#include "librecomp/recomp.h"
#include "librecomp/addresses.hpp"
// TODO move this out into ultramodern code
@ -21,7 +22,7 @@ void save_clear(uint32_t start, uint32_t size, char value);
std::array<char, page_size> write_buffer;
extern "C" void osFlashInit_recomp(uint8_t * rdram, recomp_context * ctx) {
ctx->r2 = ultramodern::flash_handle;
ctx->r2 = recomp::flash_handle;
}
extern "C" void osFlashReadStatus_recomp(uint8_t * rdram, recomp_context * ctx) {

56
librecomp/src/mod_events.cpp Normal file
View file

@ -0,0 +1,56 @@
#include <vector>
#include "librecomp/mods.hpp"
#include "librecomp/overlays.hpp"
#include "ultramodern/error_handling.hpp"
template<class... Ts>
struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts>
overloaded(Ts...) -> overloaded<Ts...>;
// Vector of callbacks for each registered event.
std::vector<std::vector<recomp::mods::GenericFunction>> event_callbacks{};
extern "C" {
// This can stay at 0 since the base events are always first in the list.
uint32_t builtin_base_event_index = 0;
}
extern "C" void recomp_trigger_event(uint8_t* rdram, recomp_context* ctx, uint32_t event_index) {
// Sanity check the event index.
if (event_index >= event_callbacks.size()) {
printf("Event %u triggered, but only %zu events have been registered!\n", event_index, event_callbacks.size());
assert(false);
ultramodern::error_handling::message_box("Encountered an error with loaded mods: event index out of bounds");
ULTRAMODERN_QUICK_EXIT();
}
// Copy the initial context state to restore it after running each callback.
recomp_context initial_context = *ctx;
// Call every callback attached to the event.
const std::vector<recomp::mods::GenericFunction>& callbacks = event_callbacks[event_index];
for (recomp::mods::GenericFunction func : callbacks) {
// Run the callback.
std::visit(overloaded {
[rdram, ctx](recomp_func_t* native_func) {
native_func(rdram, ctx);
},
}, func);
// Restore the original context.
*ctx = initial_context;
}
}
void recomp::mods::setup_events(size_t num_events) {
event_callbacks.resize(num_events);
}
void recomp::mods::register_event_callback(size_t event_index, GenericFunction callback) {
event_callbacks[event_index].emplace_back(callback);
}
void recomp::mods::reset_events() {
event_callbacks.clear();
}

536
librecomp/src/mod_manifest.cpp Normal file
View file

@ -0,0 +1,536 @@
#include <unordered_map>
#include "json/json.hpp"
#include "n64recomp.h"
#include "librecomp/mods.hpp"
recomp::mods::ZipModFileHandle::~ZipModFileHandle() {
if (file_handle) {
fclose(file_handle);
file_handle = nullptr;
}
if (archive) {
mz_zip_reader_end(archive.get());
}
archive = {};
}
recomp::mods::ZipModFileHandle::ZipModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error) {
#ifdef _WIN32
if (_wfopen_s(&file_handle, mod_path.c_str(), L"rb") != 0) {
error = ModOpenError::FileError;
return;
}
#else
file_handle = fopen(mod_path.c_str(), "rb");
if (!file_handle) {
error = ModOpenError::FileError;
return;
}
#endif
archive = std::make_unique<mz_zip_archive>();
if (!mz_zip_reader_init_cfile(archive.get(), file_handle, 0, 0)) {
error = ModOpenError::InvalidZip;
return;
}
error = ModOpenError::Good;
}
std::vector<char> recomp::mods::ZipModFileHandle::read_file(const std::string& filepath, bool& exists) const {
std::vector<char> ret{};
mz_uint32 file_index;
if (!mz_zip_reader_locate_file_v2(archive.get(), filepath.c_str(), nullptr, MZ_ZIP_FLAG_CASE_SENSITIVE, &file_index)) {
exists = false;
return ret;
}
mz_zip_archive_file_stat stat;
if (!mz_zip_reader_file_stat(archive.get(), file_index, &stat)) {
exists = false;
return ret;
}
ret.resize(stat.m_uncomp_size);
if (!mz_zip_reader_extract_to_mem(archive.get(), file_index, ret.data(), ret.size(), 0)) {
exists = false;
return {};
}
exists = true;
return ret;
}
bool recomp::mods::ZipModFileHandle::file_exists(const std::string& filepath) const {
mz_uint32 file_index;
if (!mz_zip_reader_locate_file_v2(archive.get(), filepath.c_str(), nullptr, MZ_ZIP_FLAG_CASE_SENSITIVE, &file_index)) {
return false;
}
return true;
}
recomp::mods::LooseModFileHandle::~LooseModFileHandle() {
// Nothing to do here, members will be destroyed automatically.
}
recomp::mods::LooseModFileHandle::LooseModFileHandle(const std::filesystem::path& mod_path, ModOpenError& error) {
root_path = mod_path;
std::error_code ec;
if (!std::filesystem::is_directory(root_path, ec)) {
error = ModOpenError::NotAFileOrFolder;
}
if (ec) {
error = ModOpenError::FileError;
}
error = ModOpenError::Good;
}
std::vector<char> recomp::mods::LooseModFileHandle::read_file(const std::string& filepath, bool& exists) const {
std::vector<char> ret{};
std::filesystem::path full_path = root_path / filepath;
std::error_code ec;
if (!std::filesystem::is_regular_file(full_path, ec) || ec) {
exists = false;
return ret;
}
std::ifstream file{ full_path, std::ios::binary };
if (!file.good()) {
exists = false;
return ret;
}
file.seekg(0, std::ios::end);
size_t file_size = file.tellg();
file.seekg(0, std::ios::beg);
ret.resize(file_size);
file.read(ret.data(), ret.size());
exists = true;
return ret;
}
bool recomp::mods::LooseModFileHandle::file_exists(const std::string& filepath) const {
std::filesystem::path full_path = root_path / filepath;
std::error_code ec;
if (!std::filesystem::is_regular_file(full_path, ec) || ec) {
return false;
}
return true;
}
enum class ManifestField {
GameModId,
Id,
Version,
Authors,
MinimumRecompVersion,
Dependencies,
NativeLibraries,
};
const std::string game_mod_id_key = "game_id";
const std::string mod_id_key = "id";
const std::string version_key = "version";
const std::string authors_key = "authors";
const std::string minimum_recomp_version_key = "minimum_recomp_version";
const std::string dependencies_key = "dependencies";
const std::string native_libraries_key = "native_libraries";
std::unordered_map<std::string, ManifestField> field_map {
{ game_mod_id_key, ManifestField::GameModId },
{ mod_id_key, ManifestField::Id },
{ version_key, ManifestField::Version },
{ authors_key, ManifestField::Authors },
{ minimum_recomp_version_key, ManifestField::MinimumRecompVersion },
{ dependencies_key, ManifestField::Dependencies },
{ native_libraries_key, ManifestField::NativeLibraries },
};
template <typename T1, typename T2>
bool get_to(const nlohmann::json& val, T2& out) {
const T1* ptr = val.get_ptr<const T1*>();
if (ptr == nullptr) {
return false;
}
out = *ptr;
return true;
}
template <typename T1, typename T2>
bool get_to_vec(const nlohmann::json& val, std::vector<T2>& out) {
const nlohmann::json::array_t* ptr = val.get_ptr<const nlohmann::json::array_t*>();
if (ptr == nullptr) {
return false;
}
out.clear();
for (const nlohmann::json& cur_val : *ptr) {
const T1* temp_ptr = cur_val.get_ptr<const T1*>();
if (temp_ptr == nullptr) {
out.clear();
return false;
}
out.emplace_back(*temp_ptr);
}
return true;
}
static bool parse_dependency(const std::string& val, recomp::mods::Dependency& out) {
recomp::mods::Dependency ret;
bool validated_name;
bool validated_version;
// Check if there's a version number specified.
size_t colon_pos = val.find(':');
if (colon_pos == std::string::npos) {
// No version present, so just validate the dependency's id.
validated_name = N64Recomp::validate_mod_id(std::string_view{val});
ret.mod_id = val;
validated_version = true;
ret.version.minor = 0;
ret.version.major = 0;
ret.version.patch = 0;
}
else {
// Version present, validate both the id and version.
ret.mod_id = val.substr(0, colon_pos);
validated_name = N64Recomp::validate_mod_id(ret.mod_id);
validated_version = recomp::Version::from_string(val.substr(colon_pos + 1), ret.version);
}
if (validated_name && validated_version) {
out = std::move(ret);
return true;
}
return false;
}
recomp::mods::ModOpenError parse_manifest(recomp::mods::ModManifest& ret, const std::vector<char>& manifest_data, std::string& error_param) {
using json = nlohmann::json;
json manifest_json = json::parse(manifest_data.begin(), manifest_data.end(), nullptr, false);
if (manifest_json.is_discarded()) {
return recomp::mods::ModOpenError::FailedToParseManifest;
}
if (!manifest_json.is_object()) {
return recomp::mods::ModOpenError::InvalidManifestSchema;
}
for (const auto& [key, val] : manifest_json.items()) {
const auto find_key_it = field_map.find(key);
if (find_key_it == field_map.end()) {
// Unrecognized field
error_param = key;
return recomp::mods::ModOpenError::UnrecognizedManifestField;
}
ManifestField field = find_key_it->second;
switch (field) {
case ManifestField::GameModId:
{
std::string mod_game_id;
if (!get_to<json::string_t>(val, mod_game_id)) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
ret.mod_game_ids.resize(1);
ret.mod_game_ids[0] = std::move(mod_game_id);
}
break;
case ManifestField::Id:
if (!get_to<json::string_t>(val, ret.mod_id)) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
break;
case ManifestField::Version:
{
const std::string* version_str = val.get_ptr<const std::string*>();
if (version_str == nullptr) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
if (!recomp::Version::from_string(*version_str, ret.version)) {
error_param = *version_str;
return recomp::mods::ModOpenError::InvalidVersionString;
}
}
break;
case ManifestField::Authors:
if (!get_to_vec<std::string>(val, ret.authors)) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
break;
case ManifestField::MinimumRecompVersion:
{
const std::string* version_str = val.get_ptr<const std::string*>();
if (version_str == nullptr) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
if (!recomp::Version::from_string(*version_str, ret.minimum_recomp_version)) {
error_param = *version_str;
return recomp::mods::ModOpenError::InvalidMinimumRecompVersionString;
}
ret.minimum_recomp_version.suffix.clear();
}
break;
case ManifestField::Dependencies:
{
std::vector<std::string> dep_strings{};
if (!get_to_vec<std::string>(val, dep_strings)) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
for (const std::string& dep_string : dep_strings) {
recomp::mods::Dependency cur_dep;
if (!parse_dependency(dep_string, cur_dep)) {
error_param = dep_string;
return recomp::mods::ModOpenError::InvalidDependencyString;
}
size_t dependency_index = ret.dependencies.size();
ret.dependencies_by_id.emplace(cur_dep.mod_id, dependency_index);
ret.dependencies.emplace_back(std::move(cur_dep));
}
}
break;
case ManifestField::NativeLibraries:
{
if (!val.is_object()) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
for (const auto& [lib_name, lib_exports] : val.items()) {
recomp::mods::NativeLibraryManifest& cur_lib = ret.native_libraries.emplace_back();
cur_lib.name = lib_name;
if (!get_to_vec<std::string>(lib_exports, cur_lib.exports)) {
error_param = key;
return recomp::mods::ModOpenError::IncorrectManifestFieldType;
}
}
}
break;
}
}
return recomp::mods::ModOpenError::Good;
}
recomp::mods::ModOpenError validate_manifest(const recomp::mods::ModManifest& manifest, std::string& error_param) {
using namespace recomp::mods;
// Check for required fields.
if (manifest.mod_game_ids.empty()) {
error_param = game_mod_id_key;
return ModOpenError::MissingManifestField;
}
if (manifest.mod_id.empty()) {
error_param = mod_id_key;
return ModOpenError::MissingManifestField;
}
if (manifest.version.major == -1 || manifest.version.major == -1 || manifest.version.major == -1) {
error_param = version_key;
return ModOpenError::MissingManifestField;
}
if (manifest.authors.empty()) {
error_param = authors_key;
return ModOpenError::MissingManifestField;
}
if (manifest.minimum_recomp_version.major == -1 || manifest.minimum_recomp_version.major == -1 || manifest.minimum_recomp_version.major == -1) {
error_param = minimum_recomp_version_key;
return ModOpenError::MissingManifestField;
}
return ModOpenError::Good;
}
recomp::mods::ModOpenError recomp::mods::ModContext::open_mod(const std::filesystem::path& mod_path, std::string& error_param) {
ModManifest manifest{};
std::error_code ec;
error_param = "";
if (!std::filesystem::exists(mod_path, ec) || ec) {
return ModOpenError::DoesNotExist;
}
// TODO support symlinks?
bool is_file = std::filesystem::is_regular_file(mod_path, ec);
if (ec) {
return ModOpenError::FileError;
}
bool is_directory = std::filesystem::is_directory(mod_path, ec);
if (ec) {
return ModOpenError::FileError;
}
// Load the directory or zip file.
ModOpenError handle_error;
if (is_file) {
manifest.file_handle = std::make_unique<recomp::mods::ZipModFileHandle>(mod_path, handle_error);
}
else if (is_directory) {
manifest.file_handle = std::make_unique<recomp::mods::LooseModFileHandle>(mod_path, handle_error);
}
else {
return ModOpenError::NotAFileOrFolder;
}
if (handle_error != ModOpenError::Good) {
return handle_error;
}
{
bool exists;
std::vector<char> manifest_data = manifest.file_handle->read_file("manifest.json", exists);
if (!exists) {
return ModOpenError::NoManifest;
}
ModOpenError parse_error = parse_manifest(manifest, manifest_data, error_param);
if (parse_error != ModOpenError::Good) {
return parse_error;
}
}
// Check for this being a duplicate of another opened mod.
if (mod_ids.contains(manifest.mod_id)) {
error_param = manifest.mod_id;
return ModOpenError::DuplicateMod;
}
mod_ids.emplace(manifest.mod_id);
ModOpenError validate_error = validate_manifest(manifest, error_param);
if (validate_error != ModOpenError::Good) {
return validate_error;
}
// Check for this mod's game ids being valid.
std::vector<size_t> game_indices;
for (const auto& mod_game_id : manifest.mod_game_ids) {
auto find_id_it = mod_game_ids.find(mod_game_id);
if (find_id_it == mod_game_ids.end()) {
error_param = mod_game_id;
return ModOpenError::WrongGame;
}
game_indices.emplace_back(find_id_it->second);
}
// Store the loaded mod manifest in a new mod handle.
manifest.mod_root_path = mod_path;
add_opened_mod(std::move(manifest), std::move(game_indices));
return ModOpenError::Good;
}
std::string recomp::mods::error_to_string(ModOpenError error) {
switch (error) {
case ModOpenError::Good:
return "Good";
case ModOpenError::DoesNotExist:
return "Mod does not exist";
case ModOpenError::NotAFileOrFolder:
return "Mod is not a file or folder";
case ModOpenError::FileError:
return "Error reading mod file(s)";
case ModOpenError::InvalidZip:
return "Mod is an invalid zip file";
case ModOpenError::NoManifest:
return "Mod is missing a manifest.json";
case ModOpenError::FailedToParseManifest:
return "Failed to parse mod's manifest.json";
case ModOpenError::InvalidManifestSchema:
return "Mod's manifest.json has an invalid schema";
case ModOpenError::UnrecognizedManifestField:
return "Unrecognized field in manifest.json";
case ModOpenError::IncorrectManifestFieldType:
return "Incorrect type for field in manifest.json";
case ModOpenError::InvalidVersionString:
return "Invalid version string in manifest.json";
case ModOpenError::InvalidMinimumRecompVersionString:
return "Invalid minimum recomp version string in manifest.json";
case ModOpenError::InvalidDependencyString:
return "Invalid dependency string in manifest.json";
case ModOpenError::MissingManifestField:
return "Missing required field in manifest";
case ModOpenError::DuplicateMod:
return "Duplicate mod found";
case ModOpenError::WrongGame:
return "Mod is for a different game";
}
return "Unknown mod opening error: " + std::to_string((int)error);
}
std::string recomp::mods::error_to_string(ModLoadError error) {
switch (error) {
case ModLoadError::Good:
return "Good";
case ModLoadError::InvalidGame:
return "Invalid game";
case ModLoadError::MinimumRecompVersionNotMet:
return "Mod requires a newer version of this project";
case ModLoadError::HasSymsButNoBinary:
return "Mod has a symbol file but no binary file";
case ModLoadError::HasBinaryButNoSyms:
return "Mod has a binary file but no symbol file";
case ModLoadError::FailedToParseSyms:
return "Failed to parse mod symbol file";
case ModLoadError::FailedToLoadNativeCode:
return "Failed to load offline mod library";
case ModLoadError::FailedToLoadNativeLibrary:
return "Failed to load mod library";
case ModLoadError::FailedToFindNativeExport:
return "Failed to find native export";
case ModLoadError::InvalidReferenceSymbol:
return "Reference symbol does not exist";
case ModLoadError::InvalidImport:
return "Imported function not found";
case ModLoadError::InvalidCallbackEvent:
return "Event for callback not found";
case ModLoadError::InvalidFunctionReplacement:
return "Function to be replaced does not exist";
case ModLoadError::FailedToFindReplacement:
return "Failed to find replacement function";
case ModLoadError::ReplacementConflict:
return "Attempted to replace a function that cannot be replaced";
case ModLoadError::MissingDependencyInManifest:
return "Dependency is present in mod symbols but not in the manifest";
case ModLoadError::MissingDependency:
return "Missing dependency";
case ModLoadError::WrongDependencyVersion:
return "Wrong dependency version";
case ModLoadError::ModConflict:
return "Conflicts with other mod";
case ModLoadError::DuplicateExport:
return "Duplicate exports in mod";
case ModLoadError::NoSpecifiedApiVersion:
return "Mod DLL does not specify an API version";
case ModLoadError::UnsupportedApiVersion:
return "Mod DLL has an unsupported API version";
}
return "Unknown mod loading error " + std::to_string((int)error);
}

937
librecomp/src/mods.cpp Normal file
View file

@ -0,0 +1,937 @@
#include <span>
#include <fstream>
#include <sstream>
#include "librecomp/mods.hpp"
#include "librecomp/overlays.hpp"
#include "librecomp/game.hpp"
#include "n64recomp.h"
// Architecture detection.
// MSVC x86_64
#if defined (_M_AMD64) && (_M_AMD64 == 100) && !defined (_M_ARM64EC)
# define IS_X86_64
// GCC/Clang x86_64
#elif defined(__x86_64__)
# define IS_X86_64
// MSVC/GCC/Clang ARM64
#elif defined(__ARM_ARCH_ISA_A64)
# define IS_ARM64
#else
# error "Unsupported architecture!"
#endif
#if defined(_WIN32)
#define PATHFMT "%ls"
#else
#define PATHFMT "%s"
#endif
template<class... Ts>
struct overloaded : Ts... { using Ts::operator()...; };
template<class... Ts>
overloaded(Ts...) -> overloaded<Ts...>;
#if defined(_WIN32)
# define WIN32_LEAN_AND_MEAN
# include "Windows.h"
class recomp::mods::DynamicLibrary {
public:
static constexpr std::string_view PlatformExtension = ".dll";
DynamicLibrary() = default;
DynamicLibrary(const std::filesystem::path& path) {
native_handle = LoadLibraryW(path.c_str());
if (good()) {
uint32_t* recomp_api_version;
if (get_dll_symbol(recomp_api_version, "recomp_api_version")) {
api_version = *recomp_api_version;
}
else {
api_version = (uint32_t)-1;
}
}
}
~DynamicLibrary() {
unload();
}
DynamicLibrary(const DynamicLibrary&) = delete;
DynamicLibrary& operator=(const DynamicLibrary&) = delete;
DynamicLibrary(DynamicLibrary&&) = delete;
DynamicLibrary& operator=(DynamicLibrary&&) = delete;
void unload() {
if (native_handle != nullptr) {
FreeLibrary(native_handle);
}
native_handle = nullptr;
}
bool good() const {
return native_handle != nullptr;
}
template <typename T>
bool get_dll_symbol(T& out, const char* name) const {
out = (T)GetProcAddress(native_handle, name);
if (out == nullptr) {
return false;
}
return true;
};
uint32_t get_api_version() {
return api_version;
}
private:
HMODULE native_handle;
uint32_t api_version;
};
void unprotect(void* target_func, uint64_t* old_flags) {
DWORD old_flags_dword;
BOOL result = VirtualProtect(target_func,
16,
PAGE_READWRITE,
&old_flags_dword);
*old_flags = old_flags_dword;
(void)result;
}
void protect(void* target_func, uint64_t old_flags) {
DWORD dummy_old_flags;
BOOL result = VirtualProtect(target_func,
16,
static_cast<DWORD>(old_flags),
&dummy_old_flags);
(void)result;
}
#else
# include <unistd.h>
# include <dlfcn.h>
# include <sys/mman.h>
class recomp::mods::DynamicLibrary {
public:
static constexpr std::string_view PlatformExtension = ".so";
DynamicLibrary() = default;
DynamicLibrary(const std::filesystem::path& path) {
native_handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL);
if (good()) {
uint32_t* recomp_api_version;
if (get_dll_symbol(recomp_api_version, "recomp_api_version")) {
api_version = *recomp_api_version;
}
else {
api_version = (uint32_t)-1;
}
}
}
~DynamicLibrary() {
unload();
}
DynamicLibrary(const DynamicLibrary&) = delete;
DynamicLibrary& operator=(const DynamicLibrary&) = delete;
DynamicLibrary(DynamicLibrary&&) = delete;
DynamicLibrary& operator=(DynamicLibrary&&) = delete;
void unload() {
if (native_handle != nullptr) {
dlclose(native_handle);
}
native_handle = nullptr;
}
bool good() const {
return native_handle != nullptr;
}
template <typename T>
bool get_dll_symbol(T& out, const char* name) const {
out = (T)dlsym(native_handle, name);
if (out == nullptr) {
return false;
}
return true;
};
uint32_t get_api_version() {
return api_version;
}
private:
void* native_handle;
uint32_t api_version;
};
void unprotect(void* target_func, uint64_t* old_flags) {
// Align the address to a page boundary.
uintptr_t page_start = (uintptr_t)target_func;
int page_size = getpagesize();
page_start = (page_start / page_size) * page_size;
int result = mprotect((void*)page_start, page_size, PROT_READ | PROT_WRITE);
*old_flags = 0;
(void)result;
}
void protect(void* target_func, uint64_t old_flags) {
// Align the address to a page boundary.
uintptr_t page_start = (uintptr_t)target_func;
int page_size = getpagesize();
page_start = (page_start / page_size) * page_size;
int result = mprotect((void*)page_start, page_size, PROT_READ | PROT_EXEC);
(void)result;
}
#endif
namespace modpaths {
const std::string binary_path = "mod_binary.bin";
const std::string binary_syms_path = "mod_syms.bin";
};
recomp::mods::ModLoadError recomp::mods::validate_api_version(uint32_t api_version, std::string& error_param) {
switch (api_version) {
case 1:
return ModLoadError::Good;
case (uint32_t)-1:
return ModLoadError::NoSpecifiedApiVersion;
default:
error_param = std::to_string(api_version);
return ModLoadError::UnsupportedApiVersion;
}
}
recomp::mods::ModHandle::ModHandle(ModManifest&& manifest, std::vector<size_t>&& game_indices) :
manifest(std::move(manifest)),
code_handle(),
recompiler_context{std::make_unique<N64Recomp::Context>()},
game_indices{std::move(game_indices)}
{
}
recomp::mods::ModHandle::ModHandle(ModHandle&& rhs) = default;
recomp::mods::ModHandle& recomp::mods::ModHandle::operator=(ModHandle&& rhs) = default;
recomp::mods::ModHandle::~ModHandle() = default;
size_t recomp::mods::ModHandle::num_exports() const {
return recompiler_context->exported_funcs.size();
}
size_t recomp::mods::ModHandle::num_events() const {
return recompiler_context->event_symbols.size();
}
recomp::mods::ModLoadError recomp::mods::ModHandle::populate_exports(std::string& error_param) {
for (size_t func_index : recompiler_context->exported_funcs) {
const auto& func_handle = recompiler_context->functions[func_index];
exports_by_name.emplace(func_handle.name, func_index);
}
return ModLoadError::Good;
}
recomp::mods::ModLoadError recomp::mods::ModHandle::load_native_library(const recomp::mods::NativeLibraryManifest& lib_manifest, std::string& error_param) {
std::string lib_filename = lib_manifest.name + std::string{DynamicLibrary::PlatformExtension};
std::filesystem::path lib_path = manifest.mod_root_path.parent_path() / lib_filename;
std::unique_ptr<DynamicLibrary>& lib = native_libraries.emplace_back(std::make_unique<DynamicLibrary>(lib_path));
if (!lib->good()) {
error_param = lib_filename;
return ModLoadError::FailedToLoadNativeLibrary;
}
std::string api_error_param;
ModLoadError api_error = validate_api_version(lib->get_api_version(), api_error_param);
if (api_error != ModLoadError::Good) {
if (api_error_param.empty()) {
error_param = lib_filename;
}
else {
error_param = lib_filename + ":" + api_error_param;
}
return api_error;
}
for (const std::string& export_name : lib_manifest.exports) {
recomp_func_t* cur_func;
if (native_library_exports.contains(export_name)) {
error_param = export_name;
return ModLoadError::DuplicateExport;
}
if (!lib->get_dll_symbol(cur_func, export_name.c_str())) {
error_param = lib_manifest.name + ":" + export_name;
return ModLoadError::FailedToFindNativeExport;
}
native_library_exports.emplace(export_name, cur_func);
}
return ModLoadError::Good;
}
bool recomp::mods::ModHandle::get_export_function(const std::string& export_name, GenericFunction& out) const {
// First, check the code exports.
auto code_find_it = exports_by_name.find(export_name);
if (code_find_it != exports_by_name.end()) {
out = code_handle->get_function_handle(code_find_it->second);
return true;
}
// Next, check the native library exports.
auto native_find_it = native_library_exports.find(export_name);
if (native_find_it != native_library_exports.end()) {
out = native_find_it->second;
return true;
}
// Nothing found.
return false;
}
recomp::mods::ModLoadError recomp::mods::ModHandle::populate_events(size_t base_event_index, std::string& error_param) {
for (size_t event_index = 0; event_index < recompiler_context->event_symbols.size(); event_index++) {
const N64Recomp::EventSymbol& event = recompiler_context->event_symbols[event_index];
events_by_name.emplace(event.base.name, event_index);
}
code_handle->set_base_event_index(base_event_index);
return ModLoadError::Good;
}
bool recomp::mods::ModHandle::get_global_event_index(const std::string& event_name, size_t& event_index_out) const {
auto find_it = events_by_name.find(event_name);
if (find_it == events_by_name.end()) {
return false;
}
event_index_out = code_handle->get_base_event_index() + find_it->second;
return true;
}
recomp::mods::NativeCodeHandle::NativeCodeHandle(const std::filesystem::path& dll_path, const N64Recomp::Context& context) {
is_good = true;
// Load the DLL.
dynamic_lib = std::make_unique<DynamicLibrary>(dll_path);
if (!dynamic_lib->good()) {
is_good = false;
return;
}
// Fill out the list of function pointers.
functions.resize(context.functions.size());
for (size_t i = 0; i < functions.size(); i++) {
if(!context.functions[i].name.empty()) {
is_good &= dynamic_lib->get_dll_symbol(functions[i], context.functions[i].name.c_str());
}
else {
std::string func_name = "mod_func_" + std::to_string(i);
is_good &= dynamic_lib->get_dll_symbol(functions[i], func_name.c_str());
}
if (!is_good) {
return;
}
}
// Get the standard exported symbols.
is_good = true;
is_good &= dynamic_lib->get_dll_symbol(imported_funcs, "imported_funcs");
is_good &= dynamic_lib->get_dll_symbol(reference_symbol_funcs, "reference_symbol_funcs");
is_good &= dynamic_lib->get_dll_symbol(base_event_index, "base_event_index");
is_good &= dynamic_lib->get_dll_symbol(recomp_trigger_event, "recomp_trigger_event");
is_good &= dynamic_lib->get_dll_symbol(get_function, "get_function");
is_good &= dynamic_lib->get_dll_symbol(reference_section_addresses, "reference_section_addresses");
is_good &= dynamic_lib->get_dll_symbol(section_addresses, "section_addresses");
}
bool recomp::mods::NativeCodeHandle::good() {
return dynamic_lib->good() && is_good;
}
uint32_t recomp::mods::NativeCodeHandle::get_api_version() {
return dynamic_lib->get_api_version();
}
void recomp::mods::NativeCodeHandle::set_bad() {
dynamic_lib.reset();
is_good = false;
}
void recomp::mods::NativeCodeHandle::set_imported_function(size_t import_index, GenericFunction func) {
std::visit(overloaded {
[this, import_index](recomp_func_t* native_func) {
imported_funcs[import_index] = native_func;
}
}, func);
}
void patch_func(recomp_func_t* target_func, recomp::mods::GenericFunction replacement_func) {
uint8_t* target_func_u8 = reinterpret_cast<uint8_t*>(target_func);
size_t offset = 0;
auto write_bytes = [&](const void* bytes, size_t count) {
memcpy(target_func_u8 + offset, bytes, count);
offset += count;
};
uint64_t old_flags;
unprotect(target_func_u8, &old_flags);
#if defined(IS_X86_64)
static const uint8_t movabs_rax[] = {0x48, 0xB8};
static const uint8_t jmp_rax[] = {0xFF, 0xE0};
std::visit(overloaded {
[&write_bytes](recomp_func_t* native_func) {
write_bytes(movabs_rax, sizeof(movabs_rax));
write_bytes(&native_func, sizeof(&native_func));
write_bytes(jmp_rax, sizeof(jmp_rax));
}
}, replacement_func);
#elif defined(IS_ARM64)
ultramodern::error_handling::message_box("Mod loading not currently implemented on ARM CPUs!\n");
#else
# error "Unsupported architecture"
#endif
protect(target_func_u8, old_flags);
}
void unpatch_func(void* target_func, const recomp::mods::PatchData& data) {
uint64_t old_flags;
unprotect(target_func, &old_flags);
memcpy(target_func, data.replaced_bytes.data(), data.replaced_bytes.size());
protect(target_func, old_flags);
}
void recomp::mods::ModContext::add_opened_mod(ModManifest&& manifest, std::vector<size_t>&& game_indices) {
opened_mods.emplace_back(std::move(manifest), std::move(game_indices));
}
recomp::mods::ModLoadError recomp::mods::ModContext::load_mod(uint8_t* rdram, const std::unordered_map<uint32_t, uint16_t>& section_vrom_map, recomp::mods::ModHandle& handle, int32_t load_address, uint32_t& ram_used, std::string& error_param) {
using namespace recomp::mods;
handle.section_load_addresses.clear();
// Check that the mod's minimum recomp version is met.
if (get_project_version() < handle.manifest.minimum_recomp_version) {
error_param = handle.manifest.minimum_recomp_version.to_string();
return recomp::mods::ModLoadError::MinimumRecompVersionNotMet;
}
// Load the mod symbol data from the file provided in the manifest.
bool binary_syms_exists = false;
std::vector<char> syms_data = handle.manifest.file_handle->read_file(modpaths::binary_syms_path, binary_syms_exists);
// Load the binary data from the file provided in the manifest.
bool binary_exists = false;
std::vector<char> binary_data = handle.manifest.file_handle->read_file(modpaths::binary_path, binary_exists);
if (binary_syms_exists && !binary_exists) {
return recomp::mods::ModLoadError::HasSymsButNoBinary;
}
if (binary_exists && !binary_syms_exists) {
return recomp::mods::ModLoadError::HasBinaryButNoSyms;
}
std::span<uint8_t> binary_span {reinterpret_cast<uint8_t*>(binary_data.data()), binary_data.size() };
// Parse the symbol file into the recompiler context.
N64Recomp::ModSymbolsError symbol_load_error = N64Recomp::parse_mod_symbols(syms_data, binary_span, section_vrom_map, *handle.recompiler_context);
if (symbol_load_error != N64Recomp::ModSymbolsError::Good) {
return ModLoadError::FailedToParseSyms;
}
const std::vector<N64Recomp::Section>& mod_sections = handle.recompiler_context->sections;
handle.section_load_addresses.resize(mod_sections.size());
// Copy each section's binary into rdram, leaving room for the section's bss before the next one.
int32_t cur_section_addr = load_address;
for (size_t section_index = 0; section_index < mod_sections.size(); section_index++) {
const auto& section = mod_sections[section_index];
for (size_t i = 0; i < section.size; i++) {
MEM_B(i, (gpr)cur_section_addr) = binary_data[section.rom_addr + i];
}
handle.section_load_addresses[section_index] = cur_section_addr;
cur_section_addr += section.size + section.bss_size;
}
// Iterate over each section again after loading them to perform R_MIPS_32 relocations.
for (size_t section_index = 0; section_index < mod_sections.size(); section_index++) {
const auto& section = mod_sections[section_index];
uint32_t cur_section_original_vram = section.ram_addr;
uint32_t cur_section_loaded_vram = handle.section_load_addresses[section_index];
// Perform mips32 relocations for this section.
for (const auto& reloc : section.relocs) {
if (reloc.type == N64Recomp::RelocType::R_MIPS_32 && !reloc.reference_symbol) {
if (reloc.target_section >= mod_sections.size()) {
return ModLoadError::FailedToParseSyms;
}
// Get the ram address of the word that's being relocated and read its original value.
int32_t reloc_word_addr = reloc.address - cur_section_original_vram + cur_section_loaded_vram;
uint32_t reloc_word = MEM_W(0, reloc_word_addr);
// Determine the original and loaded addresses of the section that the relocation points to.
uint32_t target_section_original_vram = mod_sections[reloc.target_section].ram_addr;
uint32_t target_section_loaded_vram = handle.section_load_addresses[reloc.target_section];
uint32_t reloc_word_old = reloc_word;
// Recalculate the word and write it back into ram.
reloc_word += (target_section_loaded_vram - target_section_original_vram);
MEM_W(0, reloc_word_addr) = reloc_word;
}
}
}
ram_used = cur_section_addr - load_address;
return ModLoadError::Good;
}
void recomp::mods::ModContext::register_game(const std::string& mod_game_id) {
mod_game_ids.emplace(mod_game_id, mod_game_ids.size());
}
std::vector<recomp::mods::ModOpenErrorDetails> recomp::mods::ModContext::scan_mod_folder(const std::filesystem::path& mod_folder) {
std::vector<recomp::mods::ModOpenErrorDetails> ret{};
std::error_code ec;
for (const auto& mod_path : std::filesystem::directory_iterator{mod_folder, std::filesystem::directory_options::skip_permission_denied, ec}) {
if ((mod_path.is_regular_file() && mod_path.path().extension() == ".nrm") || mod_path.is_directory()) {
printf("Opening mod " PATHFMT "\n", mod_path.path().stem().c_str());
std::string open_error_param;
ModOpenError open_error = open_mod(mod_path, open_error_param);
if (open_error != ModOpenError::Good) {
ret.emplace_back(mod_path.path(), open_error, open_error_param);
}
}
else {
printf("Skipping non-mod " PATHFMT PATHFMT "\n", mod_path.path().stem().c_str(), mod_path.path().extension().c_str());
}
}
return ret;
}
// Nothing needed for these two, they just need to be explicitly declared outside the header to allow forward declaration of ModHandle.
recomp::mods::ModContext::ModContext() = default;
recomp::mods::ModContext::~ModContext() = default;
void recomp::mods::ModContext::enable_mod(const std::string& mod_id, bool enabled) {
if (enabled) {
enabled_mods.emplace(mod_id);
}
else {
enabled_mods.erase(mod_id);
}
}
bool recomp::mods::ModContext::is_mod_enabled(const std::string& mod_id) {
return enabled_mods.contains(mod_id);
}
size_t recomp::mods::ModContext::num_opened_mods() {
return opened_mods.size();
}
std::vector<recomp::mods::ModDetails> recomp::mods::ModContext::get_mod_details(const std::string& mod_game_id) {
std::vector<ModDetails> ret{};
bool all_games = mod_game_id.empty();
size_t game_index = (size_t)-1;
auto find_game_it = mod_game_ids.find(mod_game_id);
if (find_game_it != mod_game_ids.end()) {
game_index = find_game_it->second;
}
for (const ModHandle& mod : opened_mods) {
if (all_games || mod.is_for_game(game_index)) {
std::vector<Dependency> cur_dependencies{};
ret.emplace_back(ModDetails{
.mod_id = mod.manifest.mod_id,
.version = mod.manifest.version,
.authors = mod.manifest.authors,
.dependencies = mod.manifest.dependencies
});
}
}
return ret;
}
std::vector<recomp::mods::ModLoadErrorDetails> recomp::mods::ModContext::load_mods(const std::string& mod_game_id, uint8_t* rdram, int32_t load_address, uint32_t& ram_used) {
std::vector<recomp::mods::ModLoadErrorDetails> ret{};
ram_used = 0;
num_events = recomp::overlays::num_base_events();
auto find_index_it = mod_game_ids.find(mod_game_id);
if (find_index_it == mod_game_ids.end()) {
ret.emplace_back(mod_game_id, ModLoadError::InvalidGame, std::string{});
return ret;
}
size_t mod_game_index = find_index_it->second;
if (!patched_funcs.empty()) {
printf("Mods already loaded!\n");
return {};
}
const std::unordered_map<uint32_t, uint16_t>& section_vrom_map = recomp::overlays::get_vrom_to_section_map();
std::vector<size_t> active_mods{};
// Find and load active mods.
for (size_t mod_index = 0; mod_index < opened_mods.size(); mod_index++) {
auto& mod = opened_mods[mod_index];
if (mod.is_for_game(mod_game_index) && enabled_mods.contains(mod.manifest.mod_id)) {
active_mods.push_back(mod_index);
loaded_mods_by_id.emplace(mod.manifest.mod_id, mod_index);
printf("Loading mod %s\n", mod.manifest.mod_id.c_str());
uint32_t cur_ram_used = 0;
std::string load_error_param;
ModLoadError load_error = load_mod(rdram, section_vrom_map, mod, load_address, cur_ram_used, load_error_param);
if (load_error != ModLoadError::Good) {
ret.emplace_back(mod.manifest.mod_id, load_error, load_error_param);
}
else {
load_address += cur_ram_used;
ram_used += cur_ram_used;
}
}
}
// Exit early if errors were found.
if (!ret.empty()) {
unload_mods();
return ret;
}
// Check that mod dependencies are met.
for (size_t mod_index : active_mods) {
auto& mod = opened_mods[mod_index];
std::vector<std::pair<ModLoadError, std::string>> cur_errors;
check_dependencies(mod, cur_errors);
if (!cur_errors.empty()) {
for (auto const& [cur_error, cur_error_param] : cur_errors) {
ret.emplace_back(mod.manifest.mod_id, cur_error, cur_error_param);
}
}
}
// Exit early if errors were found.
if (!ret.empty()) {
unload_mods();
return ret;
}
// Load the code and exports from all mods.
for (size_t mod_index : active_mods) {
auto& mod = opened_mods[mod_index];
std::string cur_error_param;
ModLoadError cur_error = load_mod_code(mod, cur_error_param);
if (cur_error != ModLoadError::Good) {
ret.emplace_back(mod.manifest.mod_id, cur_error, cur_error_param);
}
}
// Exit early if errors were found.
if (!ret.empty()) {
unload_mods();
return ret;
}
// Set up the event callbacks based on the number of events allocated.
recomp::mods::setup_events(num_events);
// Resolve dependencies for all mods.
for (size_t mod_index : active_mods) {
auto& mod = opened_mods[mod_index];
std::string cur_error_param;
ModLoadError cur_error = resolve_dependencies(mod, cur_error_param);
if (cur_error != ModLoadError::Good) {
ret.emplace_back(mod.manifest.mod_id, cur_error, cur_error_param);
}
}
// Exit early if errors were found.
if (!ret.empty()) {
unload_mods();
return ret;
}
return ret;
}
void recomp::mods::ModContext::check_dependencies(recomp::mods::ModHandle& mod, std::vector<std::pair<recomp::mods::ModLoadError, std::string>>& errors) {
errors.clear();
for (const auto& [cur_dep_id, cur_dep_index] : mod.recompiler_context->dependencies_by_name) {
// Handle special dependency names.
if (cur_dep_id == N64Recomp::DependencyBaseRecomp || cur_dep_id == N64Recomp::DependencySelf) {
continue;
}
// Find the dependency in the mod manifest to get its version.
auto find_manifest_dep_it = mod.manifest.dependencies_by_id.find(cur_dep_id);
if (find_manifest_dep_it == mod.manifest.dependencies_by_id.end()) {
errors.emplace_back(ModLoadError::MissingDependencyInManifest, cur_dep_id);
continue;
}
const auto& cur_dep = mod.manifest.dependencies[find_manifest_dep_it->second];
// Look for the dependency in the loaded mod mapping.
auto find_loaded_dep_it = loaded_mods_by_id.find(cur_dep_id);
if (find_loaded_dep_it == loaded_mods_by_id.end()) {
errors.emplace_back(ModLoadError::MissingDependency, cur_dep_id);
continue;
}
const ModHandle& dep_mod = opened_mods[find_loaded_dep_it->second];
if (cur_dep.version > dep_mod.manifest.version)
{
std::stringstream error_param_stream{};
error_param_stream << "requires mod \"" << cur_dep.mod_id << "\" " <<
(int)cur_dep.version.major << "." << (int)cur_dep.version.minor << "." << (int)cur_dep.version.patch << ", got " <<
(int)dep_mod.manifest.version.major << "." << (int)dep_mod.manifest.version.minor << "." << (int)dep_mod.manifest.version.patch << "";
errors.emplace_back(ModLoadError::WrongDependencyVersion, error_param_stream.str());
}
}
}
recomp::mods::ModLoadError recomp::mods::ModContext::load_mod_code(recomp::mods::ModHandle& mod, std::string& error_param) {
// TODO implement LuaJIT recompilation and allow it instead of native code loading via a mod manifest flag.
std::filesystem::path dll_path = mod.manifest.mod_root_path;
dll_path.replace_extension(DynamicLibrary::PlatformExtension);
mod.code_handle = std::make_unique<NativeCodeHandle>(dll_path, *mod.recompiler_context);
if (!mod.code_handle->good()) {
mod.code_handle.reset();
error_param = dll_path.string();
return ModLoadError::FailedToLoadNativeCode;
}
std::string cur_error_param;
ModLoadError cur_error = validate_api_version(mod.code_handle->get_api_version(), cur_error_param);
if (cur_error != ModLoadError::Good) {
if (cur_error_param.empty()) {
error_param = dll_path.filename().string();
}
else {
error_param = dll_path.filename().string() + ":" + std::move(cur_error_param);
}
return cur_error;
}
// Populate the mod's export map.
cur_error = mod.populate_exports(cur_error_param);
if (cur_error != ModLoadError::Good) {
error_param = std::move(cur_error_param);
return cur_error;
}
// Load any native libraries specified by the mod and validate/register the expors.
std::filesystem::path parent_path = mod.manifest.mod_root_path.parent_path();
for (const recomp::mods::NativeLibraryManifest& cur_lib_manifest: mod.manifest.native_libraries) {
cur_error = mod.load_native_library(cur_lib_manifest, cur_error_param);
if (cur_error != ModLoadError::Good) {
error_param = std::move(cur_error_param);
return cur_error;
}
}
// Populate the mod's event map and set its base event index.
cur_error = mod.populate_events(num_events, cur_error_param);
if (cur_error != ModLoadError::Good) {
error_param = std::move(cur_error_param);
return cur_error;
}
// Allocate the event indices used by the mod.
num_events += mod.num_events();
// Add each function from the mod into the function lookup table.
const std::vector<N64Recomp::Section>& mod_sections = mod.recompiler_context->sections;
for (size_t func_index = 0; func_index < mod.recompiler_context->functions.size(); func_index++) {
const auto& func = mod.recompiler_context->functions[func_index];
if (func.section_index >= mod_sections.size()) {
return ModLoadError::FailedToParseSyms;
}
// Calculate the loaded address of this function.
int32_t func_address = func.vram - mod_sections[func.section_index].ram_addr + mod.section_load_addresses[func.section_index];
// Get the handle to the function and add it to the lookup table based on its type.
recomp::mods::GenericFunction func_handle = mod.code_handle->get_function_handle(func_index);
std::visit(overloaded{
[func_address](recomp_func_t* native_func) {
recomp::overlays::add_loaded_function(func_address, native_func);
}
}, func_handle);
}
return ModLoadError::Good;
}
recomp::mods::ModLoadError recomp::mods::ModContext::resolve_dependencies(recomp::mods::ModHandle& mod, std::string& error_param) {
// Reference symbols from the base recomp.
for (size_t reference_sym_index = 0; reference_sym_index < mod.recompiler_context->num_regular_reference_symbols(); reference_sym_index++) {
const N64Recomp::ReferenceSymbol& reference_sym = mod.recompiler_context->get_regular_reference_symbol(reference_sym_index);
uint32_t reference_section_vrom = mod.recompiler_context->get_reference_section_rom(reference_sym.section_index);
uint32_t reference_section_vram = mod.recompiler_context->get_reference_section_vram(reference_sym.section_index);
uint32_t reference_symbol_vram = reference_section_vram + reference_sym.section_offset;
recomp_func_t* found_func = recomp::overlays::get_func_by_section_ram(reference_section_vrom, reference_symbol_vram);
if (found_func == nullptr) {
std::stringstream error_param_stream{};
error_param_stream << std::hex <<
"section: 0x" << reference_section_vrom <<
" func: 0x" << std::setfill('0') << std::setw(8) << reference_symbol_vram;
error_param = error_param_stream.str();
return ModLoadError::InvalidReferenceSymbol;
}
mod.code_handle->set_reference_symbol_pointer(reference_sym_index, found_func);
}
// Create a list of dependencies ordered by their index in the recompiler context.
std::vector<std::string> dependencies_ordered{};
dependencies_ordered.resize(mod.recompiler_context->dependencies_by_name.size());
for (const auto& [dependency, dependency_index] : mod.recompiler_context->dependencies_by_name) {
dependencies_ordered[dependency_index] = dependency;
}
// Imported symbols.
for (size_t import_index = 0; import_index < mod.recompiler_context->import_symbols.size(); import_index++) {
const N64Recomp::ImportSymbol& imported_func = mod.recompiler_context->import_symbols[import_index];
const std::string& dependency_id = dependencies_ordered[imported_func.dependency_index];
GenericFunction func_handle{};
bool did_find_func = false;
if (dependency_id == N64Recomp::DependencyBaseRecomp) {
recomp_func_t* func_ptr = recomp::overlays::get_base_export(imported_func.base.name);
did_find_func = func_ptr != nullptr;
func_handle = func_ptr;
}
else if (dependency_id == N64Recomp::DependencySelf) {
did_find_func = mod.get_export_function(imported_func.base.name, func_handle);
}
else {
auto find_mod_it = loaded_mods_by_id.find(dependency_id);
if (find_mod_it == loaded_mods_by_id.end()) {
error_param = dependency_id;
return ModLoadError::MissingDependency;
}
const auto& dependency = opened_mods[find_mod_it->second];
did_find_func = dependency.get_export_function(imported_func.base.name, func_handle);
}
if (!did_find_func) {
error_param = dependency_id + ":" + imported_func.base.name;
return ModLoadError::InvalidImport;
}
mod.code_handle->set_imported_function(import_index, func_handle);
}
// Register callbacks.
for (const N64Recomp::Callback& callback : mod.recompiler_context->callbacks) {
const N64Recomp::DependencyEvent& dependency_event = mod.recompiler_context->dependency_events[callback.dependency_event_index];
const std::string& dependency_id = dependencies_ordered[dependency_event.dependency_index];
GenericFunction func = mod.code_handle->get_function_handle(callback.function_index);
size_t event_index = 0;
bool did_find_event = false;
if (dependency_id == N64Recomp::DependencyBaseRecomp) {
event_index = recomp::overlays::get_base_event_index(dependency_event.event_name);
if (event_index != (size_t)-1) {
did_find_event = true;
}
}
else if (dependency_id == N64Recomp::DependencySelf) {
did_find_event = mod.get_global_event_index(dependency_event.event_name, event_index);
}
else {
auto find_mod_it = loaded_mods_by_id.find(dependency_id);
if (find_mod_it == loaded_mods_by_id.end()) {
error_param = dependency_id;
return ModLoadError::MissingDependency;
}
const auto& dependency_mod = opened_mods[find_mod_it->second];
did_find_event = dependency_mod.get_global_event_index(dependency_event.event_name, event_index);
}
if (!did_find_event) {
error_param = dependency_id + ":" + dependency_event.event_name;
return ModLoadError::InvalidCallbackEvent;
}
recomp::mods::register_event_callback(event_index, func);
}
// Populate the mod's state fields.
mod.code_handle->set_recomp_trigger_event_pointer(recomp_trigger_event);
mod.code_handle->set_get_function_pointer(get_function);
mod.code_handle->set_reference_section_addresses_pointer(section_addresses);
for (size_t section_index = 0; section_index < mod.section_load_addresses.size(); section_index++) {
mod.code_handle->set_local_section_address(section_index, mod.section_load_addresses[section_index]);
}
// Apply all the function replacements in the mod.
for (const auto& replacement : mod.recompiler_context->replacements) {
recomp_func_t* to_replace = recomp::overlays::get_func_by_section_ram(replacement.original_section_vrom, replacement.original_vram);
if (to_replace == nullptr) {
std::stringstream error_param_stream{};
error_param_stream << std::hex <<
"section: 0x" << replacement.original_section_vrom <<
" func: 0x" << std::setfill('0') << std::setw(8) << replacement.original_vram;
error_param = error_param_stream.str();
return ModLoadError::InvalidFunctionReplacement;
}
// Check if this function has already been replaced.
auto find_patch_it = patched_funcs.find(to_replace);
if (find_patch_it != patched_funcs.end()) {
error_param = find_patch_it->second.mod_id;
return ModLoadError::ModConflict;
}
// Copy the original bytes so they can be restored later after the mod is unloaded.
PatchData& cur_replacement_data = patched_funcs[to_replace];
memcpy(cur_replacement_data.replaced_bytes.data(), reinterpret_cast<void*>(to_replace), cur_replacement_data.replaced_bytes.size());
cur_replacement_data.mod_id = mod.manifest.mod_id;
// Patch the function to redirect it to the replacement.
patch_func(to_replace, mod.code_handle->get_function_handle(replacement.func_index));
}
return ModLoadError::Good;
}
void recomp::mods::ModContext::unload_mods() {
for (auto& [replacement_func, replacement_data] : patched_funcs) {
unpatch_func(reinterpret_cast<void*>(replacement_func), replacement_data);
}
patched_funcs.clear();
loaded_mods_by_id.clear();
recomp::mods::reset_events();
num_events = recomp::overlays::num_base_events();
}

132
librecomp/src/overlays.cpp
View file

@ -18,19 +18,6 @@ static SectionTableEntry* patch_code_sections = nullptr;
size_t num_patch_code_sections = 0;
static std::vector<char> patch_data;
void recomp::overlays::register_overlays(const overlay_section_table_data_t& sections, const overlays_by_index_t& overlays) {
sections_info = sections;
overlays_info = overlays;
}
void recomp::overlays::register_patches(const char* patch, std::size_t size, SectionTableEntry* sections, size_t num_sections) {
patch_code_sections = sections;
num_patch_code_sections = num_sections;
patch_data.resize(size);
std::memcpy(patch_data.data(), patch, size);
}
struct LoadedSection {
int32_t loaded_ram_addr;
size_t section_table_index;
@ -45,8 +32,85 @@ struct LoadedSection {
}
};
std::vector<LoadedSection> loaded_sections{};
std::unordered_map<int32_t, recomp_func_t*> func_map{};
static std::unordered_map<uint32_t, uint16_t> code_sections_by_rom{};
static std::vector<LoadedSection> loaded_sections{};
static std::unordered_map<int32_t, recomp_func_t*> func_map{};
static std::unordered_map<std::string, recomp_func_t*> base_exports{};
static std::unordered_map<std::string, size_t> base_events;
extern "C" {
int32_t* section_addresses = nullptr;
}
void recomp::overlays::register_overlays(const overlay_section_table_data_t& sections, const overlays_by_index_t& overlays) {
sections_info = sections;
overlays_info = overlays;
}
void recomp::overlays::register_patches(const char* patch, std::size_t size, SectionTableEntry* sections, size_t num_sections) {
patch_code_sections = sections;
num_patch_code_sections = num_sections;
patch_data.resize(size);
std::memcpy(patch_data.data(), patch, size);
}
void recomp::overlays::register_base_exports(const FunctionExport* export_list) {
std::unordered_map<uint32_t, recomp_func_t*> patch_func_vram_map{};
// Iterate over all patch functions to set up a mapping of their vram address.
for (size_t patch_section_index = 0; patch_section_index < num_patch_code_sections; patch_section_index++) {
const SectionTableEntry* cur_section = &patch_code_sections[patch_section_index];
for (size_t func_index = 0; func_index < cur_section->num_funcs; func_index++) {
const FuncEntry* cur_func = &cur_section->funcs[func_index];
patch_func_vram_map.emplace(cur_section->ram_addr + cur_func->offset, cur_func->func);
}
}
// Iterate over exports, using the vram mapping to create a name mapping.
for (const FunctionExport* cur_export = &export_list[0]; cur_export->name != nullptr; cur_export++) {
auto it = patch_func_vram_map.find(cur_export->ram_addr);
if (it == patch_func_vram_map.end()) {
assert(false && "Failed to find exported function in patch function sections!");
}
base_exports.emplace(cur_export->name, it->second);
}
}
recomp_func_t* recomp::overlays::get_base_export(const std::string& export_name) {
auto it = base_exports.find(export_name);
if (it == base_exports.end()) {
return nullptr;
}
return it->second;
}
void recomp::overlays::register_base_events(char const* const* event_names) {
for (size_t event_index = 0; event_names[event_index] != nullptr; event_index++) {
base_events.emplace(event_names[event_index], event_index);
}
}
size_t recomp::overlays::get_base_event_index(const std::string& event_name) {
auto it = base_events.find(event_name);
if (it == base_events.end()) {
return (size_t)-1;
}
return it->second;
}
size_t recomp::overlays::num_base_events() {
return base_events.size();
}
const std::unordered_map<uint32_t, uint16_t>& recomp::overlays::get_vrom_to_section_map() {
return code_sections_by_rom;
}
void recomp::overlays::add_loaded_function(int32_t ram, recomp_func_t* func) {
func_map[ram] = func;
}
void load_overlay(size_t section_table_index, int32_t ram) {
const SectionTableEntry& section = sections_info.code_sections[section_table_index];
@ -83,10 +147,6 @@ void recomp::overlays::read_patch_data(uint8_t* rdram, gpr patch_data_address) {
}
}
extern "C" {
int32_t* section_addresses = nullptr;
}
extern "C" void load_overlays(uint32_t rom, int32_t ram_addr, uint32_t size) {
// Search for the first section that's included in the loaded rom range
// Sections were sorted by `init_overlays` so we can use the bounds functions
@ -174,12 +234,9 @@ extern "C" void unload_overlays(int32_t ram_addr, uint32_t size) {
}
void recomp::overlays::init_overlays() {
func_map.clear();
section_addresses = (int32_t *)calloc(sections_info.total_num_sections, sizeof(int32_t));
for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) {
section_addresses[sections_info.code_sections[section_index].index] = sections_info.code_sections[section_index].ram_addr;
}
// Sort the executable sections by rom address
std::sort(&sections_info.code_sections[0], &sections_info.code_sections[sections_info.num_code_sections],
[](const SectionTableEntry& a, const SectionTableEntry& b) {
@ -187,9 +244,38 @@ void recomp::overlays::init_overlays() {
}
);
for (size_t section_index = 0; section_index < sections_info.num_code_sections; section_index++) {
SectionTableEntry* code_section = &sections_info.code_sections[section_index];
section_addresses[sections_info.code_sections[section_index].index] = code_section->ram_addr;
code_sections_by_rom[code_section->rom_addr] = section_index;
}
load_patch_functions();
}
recomp_func_t* recomp::overlays::get_func_by_section_ram(uint32_t section_rom, uint32_t function_vram) {
auto find_section_it = code_sections_by_rom.find(section_rom);
if (find_section_it == code_sections_by_rom.end()) {
return nullptr;
}
SectionTableEntry* section = &sections_info.code_sections[find_section_it->second];
if (function_vram < section->ram_addr || function_vram >= section->ram_addr + section->size) {
return nullptr;
}
uint32_t func_offset = function_vram - section->ram_addr;
for (size_t func_index = 0; func_index < section->num_funcs; func_index++) {
if (section->funcs[func_index].offset == func_offset) {
return section->funcs[func_index].func;
}
}
return nullptr;
}
extern "C" recomp_func_t * get_function(int32_t addr) {
auto func_find = func_map.find(addr);
if (func_find == func_map.end()) {

46
librecomp/src/pi.cpp
View file

@ -4,9 +4,10 @@
#include <cstring>
#include <string>
#include <mutex>
#include "recomp.h"
#include "game.hpp"
#include "files.hpp"
#include "librecomp/recomp.h"
#include "librecomp/addresses.hpp"
#include "librecomp/game.hpp"
#include "librecomp/files.hpp"
#include <ultramodern/ultra64.h>
#include <ultramodern/ultramodern.hpp>
@ -20,13 +21,6 @@ void recomp::set_rom_contents(std::vector<uint8_t>&& new_rom) {
rom = std::move(new_rom);
}
// Flashram occupies the same physical address as sram, but that issue is avoided because libultra exposes
// a high-level interface for flashram. Because that high-level interface is reimplemented, low level accesses
// that involve physical addresses don't need to be handled for flashram.
constexpr uint32_t sram_base = 0x08000000;
constexpr uint32_t rom_base = 0x10000000;
constexpr uint32_t drive_base = 0x06000000;
constexpr uint32_t k1_to_phys(uint32_t addr) {
return addr & 0x1FFFFFFF;
}
@ -42,21 +36,21 @@ extern "C" void __osPiRelAccess_recomp(uint8_t* rdram, recomp_context* ctx) {
}
extern "C" void osCartRomInit_recomp(uint8_t* rdram, recomp_context* ctx) {
OSPiHandle* handle = TO_PTR(OSPiHandle, ultramodern::cart_handle);
OSPiHandle* handle = TO_PTR(OSPiHandle, recomp::cart_handle);
handle->type = 0; // cart
handle->baseAddress = phys_to_k1(rom_base);
handle->baseAddress = phys_to_k1(recomp::rom_base);
handle->domain = 0;
ctx->r2 = (gpr)ultramodern::cart_handle;
ctx->r2 = (gpr)recomp::cart_handle;
}
extern "C" void osDriveRomInit_recomp(uint8_t * rdram, recomp_context * ctx) {
OSPiHandle* handle = TO_PTR(OSPiHandle, ultramodern::drive_handle);
OSPiHandle* handle = TO_PTR(OSPiHandle, recomp::drive_handle);
handle->type = 1; // bulk
handle->baseAddress = phys_to_k1(drive_base);
handle->baseAddress = phys_to_k1(recomp::drive_base);
handle->domain = 0;
ctx->r2 = (gpr)ultramodern::drive_handle;
ctx->r2 = (gpr)recomp::drive_handle;
}
extern "C" void osCreatePiManager_recomp(uint8_t* rdram, recomp_context* ctx) {
@ -70,7 +64,7 @@ void recomp::do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr
assert((physical_addr & 0x1) == 0 && "Only PI DMA from aligned ROM addresses is currently supported");
assert((ram_address & 0x7) == 0 && "Only PI DMA to aligned RDRAM addresses is currently supported");
assert((num_bytes & 0x1) == 0 && "Only PI DMA with aligned sizes is currently supported");
uint8_t* rom_addr = rom.data() + physical_addr - rom_base;
uint8_t* rom_addr = rom.data() + physical_addr - recomp::rom_base;
for (size_t i = 0; i < num_bytes; i++) {
MEM_B(i, ram_address) = *rom_addr;
rom_addr++;
@ -80,7 +74,7 @@ void recomp::do_rom_read(uint8_t* rdram, gpr ram_address, uint32_t physical_addr
void recomp::do_rom_pio(uint8_t* rdram, gpr ram_address, uint32_t physical_addr) {
assert((physical_addr & 0x3) == 0 && "PIO not 4-byte aligned in device, currently unsupported");
assert((ram_address & 0x3) == 0 && "PIO not 4-byte aligned in RDRAM, currently unsupported");
uint8_t* rom_addr = rom.data() + physical_addr - rom_base;
uint8_t* rom_addr = rom.data() + physical_addr - recomp::rom_base;
MEM_B(0, ram_address) = *rom_addr++;
MEM_B(1, ram_address) = *rom_addr++;
MEM_B(2, ram_address) = *rom_addr++;
@ -213,15 +207,15 @@ void do_dma(RDRAM_ARG PTR(OSMesgQueue) mq, gpr rdram_address, uint32_t physical_
// TODO asynchronous transfer
// TODO implement unaligned DMA correctly
if (direction == 0) {
if (physical_addr >= rom_base) {
if (physical_addr >= recomp::rom_base) {
// read cart rom
recomp::do_rom_read(rdram, rdram_address, physical_addr, size);
// Send a message to the mq to indicate that the transfer completed
osSendMesg(rdram, mq, 0, OS_MESG_NOBLOCK);
} else if (physical_addr >= sram_base) {
} else if (physical_addr >= recomp::sram_base) {
// read sram
save_read(rdram, rdram_address, physical_addr - sram_base, size);
save_read(rdram, rdram_address, physical_addr - recomp::sram_base, size);
// Send a message to the mq to indicate that the transfer completed
osSendMesg(rdram, mq, 0, OS_MESG_NOBLOCK);
@ -229,12 +223,12 @@ void do_dma(RDRAM_ARG PTR(OSMesgQueue) mq, gpr rdram_address, uint32_t physical_
fprintf(stderr, "[WARN] PI DMA read from unknown region, phys address 0x%08X\n", physical_addr);
}
} else {
if (physical_addr >= rom_base) {
if (physical_addr >= recomp::rom_base) {
// write cart rom
throw std::runtime_error("ROM DMA write unimplemented");
} else if (physical_addr >= sram_base) {
} else if (physical_addr >= recomp::sram_base) {
// write sram
save_write(rdram, rdram_address, physical_addr - sram_base, size);
save_write(rdram, rdram_address, physical_addr - recomp::sram_base, size);
// Send a message to the mq to indicate that the transfer completed
osSendMesg(rdram, mq, 0, OS_MESG_NOBLOCK);
@ -248,7 +242,7 @@ extern "C" void osPiStartDma_recomp(RDRAM_ARG recomp_context* ctx) {
uint32_t mb = ctx->r4;
uint32_t pri = ctx->r5;
uint32_t direction = ctx->r6;
uint32_t devAddr = ctx->r7 | rom_base;
uint32_t devAddr = ctx->r7 | recomp::rom_base;
gpr dramAddr = MEM_W(0x10, ctx->r29);
uint32_t size = MEM_W(0x14, ctx->r29);
PTR(OSMesgQueue) mq = MEM_W(0x18, ctx->r29);
@ -284,7 +278,7 @@ extern "C" void osEPiReadIo_recomp(RDRAM_ARG recomp_context * ctx) {
gpr dramAddr = ctx->r6;
uint32_t physical_addr = k1_to_phys(devAddr);
if (physical_addr > rom_base) {
if (physical_addr > recomp::rom_base) {
// cart rom
recomp::do_rom_pio(PASS_RDRAM dramAddr, physical_addr);
} else {

228
librecomp/src/recomp.cpp
View file

@ -6,17 +6,29 @@
#include <unordered_map>
#include <unordered_set>
#include <fstream>
#include <sstream>
#include <iostream>
#include <optional>
#include <mutex>
#include <array>
#include <cinttypes>
#include <cuchar>
#include <charconv>
#include "recomp.h"
#include "overlays.hpp"
#include "game.hpp"
#include "librecomp/recomp.h"
#include "librecomp/overlays.hpp"
#include "librecomp/game.hpp"
#include "xxHash/xxh3.h"
#include "ultramodern/ultramodern.hpp"
#include "ultramodern/error_handling.hpp"
#include "librecomp/addresses.hpp"
#include "librecomp/mods.hpp"
#if defined(_WIN32)
#define PATHFMT "%ls"
#else
#define PATHFMT "%s"
#endif
#ifdef _MSC_VER
inline uint32_t byteswap(uint32_t val) {
@ -37,10 +49,16 @@ enum GameStatus {
// Mutexes
std::mutex game_roms_mutex;
std::mutex current_game_mutex;
std::mutex mod_context_mutex{};
// Global variables
std::filesystem::path config_path;
// Maps game_id to the game's entry.
std::unordered_map<std::u8string, recomp::GameEntry> game_roms {};
// The global mod context.
std::unique_ptr<recomp::mods::ModContext> mod_context = std::make_unique<recomp::mods::ModContext>();
// The project's version.
recomp::Version project_version;
std::u8string recomp::GameEntry::stored_filename() const {
return game_id + u8".z64";
@ -51,11 +69,30 @@ void recomp::register_config_path(std::filesystem::path path) {
}
bool recomp::register_game(const recomp::GameEntry& entry) {
std::lock_guard<std::mutex> lock(game_roms_mutex);
game_roms.insert({ entry.game_id, entry });
// TODO verify that there's no game with this ID already.
{
std::lock_guard<std::mutex> lock(game_roms_mutex);
game_roms.insert({ entry.game_id, entry });
}
if (!entry.mod_game_id.empty()) {
std::lock_guard<std::mutex> lock(mod_context_mutex);
mod_context->register_game(entry.mod_game_id);
}
return true;
}
void recomp::mods::scan_mods() {
std::vector<recomp::mods::ModOpenErrorDetails> mod_open_errors;
{
std::lock_guard mod_lock{ mod_context_mutex };
mod_open_errors = mod_context->scan_mod_folder(config_path / "mods");
}
for (const auto& cur_error : mod_open_errors) {
printf("Error opening mod " PATHFMT ": %s (%s)\n", cur_error.mod_path.c_str(), recomp::mods::error_to_string(cur_error.error).c_str(), cur_error.error_param.c_str());
}
}
bool check_hash(const std::vector<uint8_t>& rom_data, uint64_t expected_hash) {
uint64_t calculated_hash = XXH3_64bits(rom_data.data(), rom_data.size());
return calculated_hash == expected_hash;
@ -134,6 +171,70 @@ bool recomp::load_stored_rom(std::u8string& game_id) {
return true;
}
const recomp::Version& recomp::get_project_version() {
return project_version;
}
bool recomp::Version::from_string(const std::string& str, Version& out) {
std::array<size_t, 2> period_indices;
size_t num_periods = 0;
size_t cur_pos = 0;
uint16_t major;
uint16_t minor;
uint16_t patch;
std::string suffix;
// Find the 2 required periods.
cur_pos = str.find('.', cur_pos);
period_indices[0] = cur_pos;
cur_pos = str.find('.', cur_pos + 1);
period_indices[1] = cur_pos;
// Check that both were found.
if (period_indices[0] == std::string::npos || period_indices[1] == std::string::npos) {
return false;
}
// Parse the 3 numbers formed by splitting the string via the periods.
std::array<std::from_chars_result, 3> parse_results;
std::array<size_t, 3> parse_starts { 0, period_indices[0] + 1, period_indices[1] + 1 };
std::array<size_t, 3> parse_ends { period_indices[0], period_indices[1], str.size() };
parse_results[0] = std::from_chars(str.data() + parse_starts[0], str.data() + parse_ends[0], major);
parse_results[1] = std::from_chars(str.data() + parse_starts[1], str.data() + parse_ends[1], minor);
parse_results[2] = std::from_chars(str.data() + parse_starts[2], str.data() + parse_ends[2], patch);
// Check that the first two parsed correctly.
auto did_parse = [&](size_t i) {
return parse_results[i].ec == std::errc{} && parse_results[i].ptr == str.data() + parse_ends[i];
};
if (!did_parse(0) || !did_parse(1)) {
return false;
}
// Check that the third had a successful parse, but not necessarily read all the characters.
if (parse_results[2].ec != std::errc{}) {
return false;
}
// Allow a plus or minus directly after the third number.
if (parse_results[2].ptr != str.data() + parse_ends[2]) {
if (*parse_results[2].ptr == '+' || *parse_results[2].ptr == '-') {
suffix = str.substr(std::distance(str.data(), parse_results[2].ptr));
}
// Failed to parse, as nothing is allowed directly after the last number besides a plus or minus.
else {
return false;
}
}
out.major = major;
out.minor = minor;
out.patch = patch;
out.suffix = std::move(suffix);
return true;
}
const std::array<uint8_t, 4> first_rom_bytes { 0x80, 0x37, 0x12, 0x40 };
enum class ByteswapType {
@ -322,7 +423,7 @@ void init(uint8_t* rdram, recomp_context* ctx, gpr entrypoint) {
recomp::do_rom_read(rdram, entrypoint, 0x10001000, 0x100000);
// Read in any extra data from patches
recomp::overlays::read_patch_data(rdram, (gpr)(s32)0x80801000);
recomp::overlays::read_patch_data(rdram, (gpr)recomp::patch_rdram_start);
// Set up context floats
ctx->f_odd = &ctx->f0.u32h;
@ -373,7 +474,83 @@ void ultramodern::quit() {
current_game.reset();
}
void recomp::mods::enable_mod(const std::string& mod_id, bool enabled) {
std::lock_guard lock { mod_context_mutex };
return mod_context->enable_mod(mod_id, enabled);
}
bool recomp::mods::is_mod_enabled(const std::string& mod_id) {
std::lock_guard lock { mod_context_mutex };
return mod_context->is_mod_enabled(mod_id);
}
std::vector<recomp::mods::ModDetails> recomp::mods::get_mod_details(const std::string& mod_game_id) {
std::lock_guard lock { mod_context_mutex };
return mod_context->get_mod_details(mod_game_id);
}
bool wait_for_game_started(uint8_t* rdram, recomp_context* context) {
game_status.wait(GameStatus::None);
switch (game_status.load()) {
// TODO refactor this to allow a project to specify what entrypoint function to run for a give game.
case GameStatus::Running:
{
if (!recomp::load_stored_rom(current_game.value())) {
ultramodern::error_handling::message_box("Error opening stored ROM! Please restart this program.");
}
auto find_it = game_roms.find(current_game.value());
const recomp::GameEntry& game_entry = find_it->second;
init(rdram, context, game_entry.entrypoint_address);
if (!game_entry.mod_game_id.empty()) {
uint32_t mod_ram_used = 0;
std::vector<recomp::mods::ModLoadErrorDetails> mod_load_errors;
{
std::lock_guard lock { mod_context_mutex };
mod_load_errors = mod_context->load_mods(game_entry.mod_game_id, rdram, recomp::mod_rdram_start, mod_ram_used);
}
if (!mod_load_errors.empty()) {
std::ostringstream mod_error_stream;
mod_error_stream << "Error loading mods:\n\n";
for (const auto& cur_error : mod_load_errors) {
mod_error_stream << cur_error.mod_id.c_str() << ": " << recomp::mods::error_to_string(cur_error.error);
if (!cur_error.error_param.empty()) {
mod_error_stream << " (" << cur_error.error_param.c_str() << ")";
}
mod_error_stream << "\n";
}
ultramodern::error_handling::message_box(mod_error_stream.str().c_str());
game_status.store(GameStatus::None);
return false;
}
}
ultramodern::init_saving(rdram);
ultramodern::load_shader_cache(game_entry.cache_data);
try {
game_entry.entrypoint(rdram, context);
} catch (ultramodern::thread_terminated& terminated) {
}
}
return true;
case GameStatus::Quit:
return true;
case GameStatus::None:
return true;
}
}
void recomp::start(
uint32_t rdram_size,
const recomp::Version& version,
ultramodern::renderer::WindowHandle window_handle,
const recomp::rsp::callbacks_t& rsp_callbacks,
const ultramodern::renderer::callbacks_t& renderer_callbacks,
@ -384,6 +561,7 @@ void recomp::start(
const ultramodern::error_handling::callbacks_t& error_handling_callbacks,
const ultramodern::threads::callbacks_t& threads_callbacks
) {
project_version = version;
recomp::check_all_stored_roms();
recomp::rsp::set_callbacks(rsp_callbacks);
@ -413,8 +591,8 @@ void recomp::start(
}
// Allocate rdram_buffer
std::unique_ptr<uint8_t[]> rdram_buffer = std::make_unique<uint8_t[]>(ultramodern::rdram_size);
std::memset(rdram_buffer.get(), 0, ultramodern::rdram_size);
std::unique_ptr<uint8_t[]> rdram_buffer = std::make_unique<uint8_t[]>(rdram_size);
std::memset(rdram_buffer.get(), 0, rdram_size);
std::thread game_thread{[](ultramodern::renderer::WindowHandle window_handle, uint8_t* rdram) {
debug_printf("[Recomp] Starting\n");
@ -423,40 +601,10 @@ void recomp::start(
ultramodern::preinit(rdram, window_handle);
game_status.wait(GameStatus::None);
recomp_context context{};
switch (game_status.load()) {
// TODO refactor this to allow a project to specify what entrypoint function to run for a give game.
case GameStatus::Running:
{
if (!recomp::load_stored_rom(current_game.value())) {
ultramodern::error_handling::message_box("Error opening stored ROM! Please restart this program.");
}
ultramodern::init_saving(rdram);
auto find_it = game_roms.find(current_game.value());
const recomp::GameEntry& game_entry = find_it->second;
ultramodern::load_shader_cache(game_entry.cache_data);
init(rdram, &context, game_entry.entrypoint_address);
try {
game_entry.entrypoint(rdram, &context);
} catch (ultramodern::thread_terminated& terminated) {
}
}
break;
case GameStatus::Quit:
break;
case GameStatus::None:
break;
}
debug_printf("[Recomp] Quitting\n");
// Loop until the game starts.
while (!wait_for_game_started(rdram, &context)) {}
}, window_handle, rdram_buffer.get()};
while (!exited) {

1
thirdparty/miniz vendored Submodule

@ -0,0 +1 @@
Subproject commit 8573fd7cd6f49b262a0ccc447f3c6acfc415e556

5
ultramodern/include/ultramodern/ultramodern.hpp
View file

@ -27,11 +27,6 @@ struct UltraThreadContext {
namespace ultramodern {
// We need a place in rdram to hold the PI handles, so pick an address in extended rdram
constexpr uint32_t rdram_size = 1024 * 1024 * 16; // 16MB to give extra room for anything custom
constexpr int32_t cart_handle = 0x80800000;
constexpr int32_t drive_handle = (int32_t)(cart_handle + sizeof(OSPiHandle));
constexpr int32_t flash_handle = (int32_t)(drive_handle + sizeof(OSPiHandle));
constexpr uint32_t save_size = 1024 * 1024 / 8; // Maximum save size, 1Mbit for flash
// Initialization.