From 7045b4345724667ae41d7195ad1f12d616d0f589 Mon Sep 17 00:00:00 2001 From: Dario Date: Tue, 21 Jan 2025 22:52:39 -0300 Subject: [PATCH] Config storage for mods. --- librecomp/include/librecomp/mods.hpp | 31 ++++- librecomp/src/mod_manifest.cpp | 118 +++++++++++++++- librecomp/src/mods.cpp | 200 +++++++++++++++++++++++++-- librecomp/src/recomp.cpp | 33 +++-- 4 files changed, 356 insertions(+), 26 deletions(-) diff --git a/librecomp/include/librecomp/mods.hpp b/librecomp/include/librecomp/mods.hpp index 4dcbea4..af16c70 100644 --- a/librecomp/include/librecomp/mods.hpp +++ b/librecomp/include/librecomp/mods.hpp @@ -13,6 +13,9 @@ #include #include #include +#include + +#include "blockingconcurrentqueue.h" #define MINIZ_NO_DEFLATE_APIS #define MINIZ_NO_ARCHIVE_WRITING_APIS @@ -55,6 +58,9 @@ struct std::hash namespace recomp { namespace mods { + static constexpr std::string_view mods_directory = "mods"; + static constexpr std::string_view mod_config_directory = "mod_config"; + enum class ModOpenError { Good, DoesNotExist, @@ -194,6 +200,13 @@ namespace recomp { struct ConfigSchema { std::vector options; + std::unordered_map options_by_id; + }; + + typedef std::variant ConfigValueVariant; + + struct ConfigStorage { + std::unordered_map value_map; }; struct ModDetails { @@ -302,6 +315,9 @@ namespace recomp { void unload_mods(); std::vector get_mod_details(const std::string& mod_game_id); const ConfigSchema &get_mod_config_schema(const std::string &mod_id) const; + void set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value); + ConfigValueVariant get_mod_config_value(const std::string &mod_id, const std::string &option_id); + void set_mod_config_path(const std::filesystem::path &path); ModContentTypeId register_content_type(const ModContentType& type); bool register_container_type(const std::string& extension, const std::vector& content_types, bool requires_manifest); ModContentTypeId get_code_content_type() const { return code_content_type_id; } @@ -313,13 +329,14 @@ namespace recomp { CodeModLoadError init_mod_code(uint8_t* rdram, const std::unordered_map& section_vrom_map, ModHandle& mod, int32_t load_address, bool hooks_available, uint32_t& ram_used, std::string& error_param); CodeModLoadError load_mod_code(uint8_t* rdram, ModHandle& mod, uint32_t base_event_index, std::string& error_param); CodeModLoadError resolve_code_dependencies(ModHandle& mod, const std::unordered_map& base_patched_funcs, std::string& error_param); - void add_opened_mod(ModManifest&& manifest, std::vector&& game_indices, std::vector&& detected_content_types); + void add_opened_mod(ModManifest&& manifest, ConfigStorage&& config_storage, std::vector&& game_indices, std::vector&& detected_content_types); void close_mods(); std::vector regenerate_with_hooks( const std::vector>& sorted_unprocessed_hooks, const std::unordered_map& section_vrom_map, const std::unordered_map& base_patched_funcs, std::span decompressed_rom); + void dirty_mod_configuration_thread_process(); static void on_code_mod_enabled(ModContext& context, const ModHandle& mod); @@ -329,10 +346,15 @@ namespace recomp { std::unordered_map mod_game_ids; std::vector opened_mods; std::unordered_map opened_mods_by_id; + std::mutex opened_mods_mutex; std::unordered_set mod_ids; std::unordered_set enabled_mods; std::unordered_map patched_funcs; std::unordered_map loaded_mods_by_id; + std::unique_ptr dirty_mod_configuration_thread; + moodycamel::BlockingConcurrentQueue dirty_mod_configuration_thread_queue; + std::filesystem::path mod_config_path; + std::mutex mod_config_storage_mutex; std::vector loaded_code_mods; // Code handle for vanilla code that was regenerated to add hooks. std::unique_ptr regenerated_code_handle; @@ -366,13 +388,14 @@ namespace recomp { public: // TODO make these private and expose methods for the functionality they're currently used in. ModManifest manifest; + ConfigStorage config_storage; std::unique_ptr code_handle; std::unique_ptr recompiler_context; std::vector section_load_addresses; // Content types present in this mod. std::vector content_types; - ModHandle(const ModContext& context, ModManifest&& manifest, std::vector&& game_indices, std::vector&& content_types); + ModHandle(const ModContext& context, ModManifest&& manifest, ConfigStorage&& config_storage, std::vector&& game_indices, std::vector&& content_types); ModHandle(const ModHandle& rhs) = delete; ModHandle& operator=(const ModHandle& rhs) = delete; ModHandle(ModHandle&& rhs); @@ -502,12 +525,14 @@ namespace recomp { CodeModLoadError validate_api_version(uint32_t api_version, std::string& error_param); - void initialize_mod_recompiler(); + void initialize_mods(); void scan_mods(); void enable_mod(const std::string& mod_id, bool enabled); bool is_mod_enabled(const std::string& mod_id); bool is_mod_auto_enabled(const std::string& mod_id); const ConfigSchema &get_mod_config_schema(const std::string &mod_id); + void set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value); + ConfigValueVariant get_mod_config_value(const std::string &mod_id, const std::string &option_id); ModContentTypeId register_mod_content_type(const ModContentType& type); bool register_mod_container_type(const std::string& extension, const std::vector& content_types, bool requires_manifest); } diff --git a/librecomp/src/mod_manifest.cpp b/librecomp/src/mod_manifest.cpp index 515c365..6d85acd 100644 --- a/librecomp/src/mod_manifest.cpp +++ b/librecomp/src/mod_manifest.cpp @@ -3,8 +3,38 @@ #include "json/json.hpp" #include "recompiler/context.h" +#include "librecomp/files.hpp" #include "librecomp/mods.hpp" +static bool read_json(std::ifstream input_file, nlohmann::json &json_out) { + if (!input_file.good()) { + return false; + } + + try { + input_file >> json_out; + } + catch (nlohmann::json::parse_error &) { + return false; + } + return true; +} + +static bool read_json_with_backups(const std::filesystem::path &path, nlohmann::json &json_out) { + // Try reading and parsing the base file. + if (read_json(std::ifstream{ path }, json_out)) { + return true; + } + + // Try reading and parsing the backup file. + if (read_json(recomp::open_input_backup_file(path), json_out)) { + return true; + } + + // Both reads failed. + return false; +} + recomp::mods::ZipModFileHandle::~ZipModFileHandle() { if (file_handle) { fclose(file_handle); @@ -469,7 +499,9 @@ recomp::mods::ModOpenError parse_manifest_config_schema_option(const nlohmann::j break; } - ret.config_schema.options.push_back(option); + ret.config_schema.options_by_id.emplace(option.id, ret.config_schema.options.size()); + ret.config_schema.options.emplace_back(option); + return recomp::mods::ModOpenError::Good; } @@ -606,6 +638,83 @@ recomp::mods::ModOpenError parse_manifest(recomp::mods::ModManifest& ret, const return recomp::mods::ModOpenError::Good; } +bool parse_mod_config_storage(const std::filesystem::path &path, const std::string &expected_mod_id, recomp::mods::ConfigStorage &config_storage, const recomp::mods::ConfigSchema &config_schema) { + using json = nlohmann::json; + json config_json; + if (!read_json_with_backups(path, config_json)) { + return false; + } + + auto mod_id = config_json.find("mod_id"); + if (mod_id != config_json.end()) { + std::string mod_id_str; + if (get_to(*mod_id, mod_id_str)) { + if (*mod_id != expected_mod_id) { + // The mod's ID doesn't match. + return false; + } + } + else { + // The mod ID is not a string. + return false; + } + } + else { + // The configuration file doesn't have a mod ID. + return false; + } + + auto storage_json = config_json.find("storage"); + if (storage_json == config_json.end()) { + // The configuration file doesn't have a storage object. + return false; + } + + if (!storage_json->is_object()) { + // The storage key does not correspond to an object. + return false; + } + + // Only parse the object for known option types based on the schema. + int64_t value_int64; + double value_double; + std::string value_str; + for (const recomp::mods::ConfigOption &option : config_schema.options) { + auto option_json = storage_json->find(option.id); + if (option_json == storage_json->end()) { + // Option doesn't exist in storage. + continue; + } + + switch (option.type) { + case recomp::mods::ConfigOptionType::Enum: + if (get_to(*option_json, value_int64)) { + config_storage.value_map[option.id] = uint32_t(value_int64); + } + + break; + case recomp::mods::ConfigOptionType::Number: + if (get_to(*option_json, value_double)) { + config_storage.value_map[option.id] = value_double; + } + + break; + case recomp::mods::ConfigOptionType::String: { + if (get_to(*option_json, value_str)) { + config_storage.value_map[option.id] = value_str; + } + + break; + } + default: + assert(false && "Unknown option type."); + break; + } + } + + return true; +} + recomp::mods::ModOpenError recomp::mods::ModContext::open_mod(const std::filesystem::path& mod_path, std::string& error_param, const std::vector& supported_content_types, bool requires_manifest) { ModManifest manifest{}; std::error_code ec; @@ -724,9 +833,14 @@ recomp::mods::ModOpenError recomp::mods::ModContext::open_mod(const std::filesys } } + // Read the mod config if it exists. + ConfigStorage config_storage; + std::filesystem::path config_path = mod_config_path / (manifest.mod_id + ".json"); + parse_mod_config_storage(config_path, manifest.mod_id, config_storage, manifest.config_schema); + // Store the loaded mod manifest in a new mod handle. manifest.mod_root_path = mod_path; - add_opened_mod(std::move(manifest), std::move(game_indices), std::move(detected_content_types)); + add_opened_mod(std::move(manifest), std::move(config_storage), std::move(game_indices), std::move(detected_content_types)); return ModOpenError::Good; } diff --git a/librecomp/src/mods.cpp b/librecomp/src/mods.cpp index 0d4fe3e..c3da7f4 100644 --- a/librecomp/src/mods.cpp +++ b/librecomp/src/mods.cpp @@ -3,6 +3,7 @@ #include #include +#include "librecomp/files.hpp" #include "librecomp/mods.hpp" #include "librecomp/overlays.hpp" #include "librecomp/game.hpp" @@ -213,8 +214,9 @@ recomp::mods::CodeModLoadError recomp::mods::validate_api_version(uint32_t api_v } } -recomp::mods::ModHandle::ModHandle(const ModContext& context, ModManifest&& manifest, std::vector&& game_indices, std::vector&& content_types) : +recomp::mods::ModHandle::ModHandle(const ModContext& context, ModManifest&& manifest, ConfigStorage&& config_storage, std::vector&& game_indices, std::vector&& content_types) : manifest(std::move(manifest)), + config_storage(std::move(config_storage)), code_handle(), recompiler_context{std::make_unique()}, content_types{std::move(content_types)}, @@ -539,10 +541,11 @@ void unpatch_func(void* target_func, const recomp::mods::PatchData& data) { protect(target_func, old_flags); } -void recomp::mods::ModContext::add_opened_mod(ModManifest&& manifest, std::vector&& game_indices, std::vector&& detected_content_types) { +void recomp::mods::ModContext::add_opened_mod(ModManifest&& manifest, ConfigStorage&& config_storage, std::vector&& game_indices, std::vector&& detected_content_types) { + std::unique_lock lock(opened_mods_mutex); size_t mod_index = opened_mods.size(); opened_mods_by_id.emplace(manifest.mod_id, mod_index); - opened_mods.emplace_back(*this, std::move(manifest), std::move(game_indices), std::move(detected_content_types)); + opened_mods.emplace_back(*this, std::move(manifest), std::move(config_storage), std::move(game_indices), std::move(detected_content_types)); } recomp::mods::ModLoadError recomp::mods::ModContext::load_mod(recomp::mods::ModHandle& mod, std::string& error_param) { @@ -567,12 +570,107 @@ void recomp::mods::ModContext::register_game(const std::string& mod_game_id) { } void recomp::mods::ModContext::close_mods() { + std::unique_lock lock(opened_mods_mutex); opened_mods_by_id.clear(); opened_mods.clear(); mod_ids.clear(); enabled_mods.clear(); } +bool save_mod_config_storage(const std::filesystem::path &path, const std::string &mod_id, const recomp::Version &mod_version, const recomp::mods::ConfigStorage &config_storage, const recomp::mods::ConfigSchema &config_schema) { + nlohmann::json config_json; + config_json["mod_id"] = mod_id; + config_json["mod_version"] = mod_version.to_string(); + config_json["recomp_version"] = recomp::get_project_version().to_string(); + + nlohmann::json &storage_json = config_json["storage"]; + for (auto it : config_storage.value_map) { + auto id_it = config_schema.options_by_id.find(it.first); + if (id_it == config_schema.options_by_id.end()) { + continue; + } + + const recomp::mods::ConfigOption &config_option = config_schema.options[id_it->second]; + switch (config_option.type) { + case recomp::mods::ConfigOptionType::Enum: + storage_json[it.first] = std::get(config_option.variant).options[std::get(it.second)]; + break; + case recomp::mods::ConfigOptionType::Number: + storage_json[it.first] = std::get(it.second); + break; + case recomp::mods::ConfigOptionType::String: + storage_json[it.first] = std::get(it.second); + break; + default: + assert(false && "Unknown config type."); + break; + } + } + + std::ofstream output_file = recomp::open_output_file_with_backup(path); + if (!output_file.good()) { + return false; + } + + output_file << std::setw(4) << config_json; + output_file.close(); + + return recomp::finalize_output_file_with_backup(path); +} + +void recomp::mods::ModContext::dirty_mod_configuration_thread_process() { + using namespace std::chrono_literals; + std::string mod_id; + std::unordered_set pending_mods; + std::unordered_map pending_mod_storage; + std::unordered_map pending_mod_schema; + std::unordered_map pending_mod_version; + std::filesystem::path config_path; + bool active = true; + while (active) { + // Wait for at least one mod to require writing. + dirty_mod_configuration_thread_queue.wait_dequeue(mod_id); + + if (!mod_id.empty()) { + pending_mods.emplace(mod_id); + } + else { + active = false; + } + + // Clear out the entire queue to coalesce all writes with a timeout. + while (active && dirty_mod_configuration_thread_queue.wait_dequeue_timed(mod_id, 1s)) { + if (!mod_id.empty()) { + pending_mods.emplace(mod_id); + } + else { + active = false; + } + } + + if (active && !pending_mods.empty()) { + { + std::unique_lock opened_mods_lock(opened_mods_mutex); + for (const std::string &id : pending_mods) { + auto it = opened_mods_by_id.find(id); + if (it != opened_mods_by_id.end()) { + const ModHandle &mod = opened_mods[it->second]; + std::unique_lock config_storage_lock(mod_config_storage_mutex); + pending_mod_storage[id] = mod.config_storage; + pending_mod_schema[id] = mod.manifest.config_schema; + pending_mod_version[id] = mod.manifest.version; + } + } + } + + for (const std::string &id : pending_mods) { + config_path = mod_config_path / std::string(id + ".json"); + save_mod_config_storage(config_path, id, pending_mod_version[id], pending_mod_storage[id], pending_mod_schema[id]); + } + } + } +} + std::vector recomp::mods::ModContext::scan_mod_folder(const std::filesystem::path& mod_folder) { std::vector ret{}; std::error_code ec; @@ -623,6 +721,8 @@ recomp::mods::ModContext::ModContext() { // Register the default mod container type (.nrm) and allow it to have any content type by passing an empty vector. register_container_type(std::string{ modpaths::default_mod_extension }, {}, true); + + dirty_mod_configuration_thread = std::make_unique(&ModContext::dirty_mod_configuration_thread_process, this); } void recomp::mods::ModContext::on_code_mod_enabled(ModContext& context, const ModHandle& mod) { @@ -635,8 +735,11 @@ void recomp::mods::ModContext::on_code_mod_enabled(ModContext& context, const Mo } } -// Nothing needed for this, it just need to be explicitly declared outside the header to allow forward declaration of ModHandle. -recomp::mods::ModContext::~ModContext() = default; +recomp::mods::ModContext::~ModContext() { + dirty_mod_configuration_thread_queue.enqueue(std::string()); + dirty_mod_configuration_thread->join(); + dirty_mod_configuration_thread.reset(); +} recomp::mods::ModContentTypeId recomp::mods::ModContext::register_content_type(const ModContentType& type) { size_t ret = content_types.size(); @@ -892,6 +995,89 @@ const recomp::mods::ConfigSchema &recomp::mods::ModContext::get_mod_config_schem return mod.manifest.config_schema; } +void recomp::mods::ModContext::set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value) { + // Check that the mod exists. + auto find_it = opened_mods_by_id.find(mod_id); + if (find_it == opened_mods_by_id.end()) { + return; + } + + ModHandle &mod = opened_mods[find_it->second]; + std::unique_lock lock(mod_config_storage_mutex); + auto option_by_id_it = mod.manifest.config_schema.options_by_id.find(option_id); + if (option_by_id_it != mod.manifest.config_schema.options_by_id.end()) { + // Only accept setting values if the value exists and the variant is the right type. + const ConfigOption &option = mod.manifest.config_schema.options[option_by_id_it->second]; + switch (option.type) { + case ConfigOptionType::Enum: + if (std::holds_alternative(value)) { + if (std::get(value) < std::get(option.variant).options.size()) { + mod.config_storage.value_map[option_id] = value; + } + } + + break; + case ConfigOptionType::Number: + if (std::holds_alternative(value)) { + mod.config_storage.value_map[option_id] = value; + } + + break; + case ConfigOptionType::String: + if (std::holds_alternative(value)) { + mod.config_storage.value_map[option_id] = value; + } + + break; + default: + assert(false && "Unknown config option type."); + return; + } + } + + // Notify the asynchronous thread it should save the configuration for this mod. + dirty_mod_configuration_thread_queue.enqueue(mod_id); +} + +recomp::mods::ConfigValueVariant recomp::mods::ModContext::get_mod_config_value(const std::string &mod_id, const std::string &option_id) { + // Check that the mod exists. + auto find_it = opened_mods_by_id.find(mod_id); + if (find_it == opened_mods_by_id.end()) { + return std::monostate(); + } + + const ModHandle &mod = opened_mods[find_it->second]; + std::unique_lock lock(mod_config_storage_mutex); + auto it = mod.config_storage.value_map.find(option_id); + if (it != mod.config_storage.value_map.end()) { + return it->second; + } + else { + // Attempt to see if we can find a default value from the schema. + auto option_by_id_it = mod.manifest.config_schema.options_by_id.find(option_id); + if (option_by_id_it == mod.manifest.config_schema.options_by_id.end()) { + return std::monostate(); + } + + const ConfigOption &option = mod.manifest.config_schema.options[option_by_id_it->second]; + switch (option.type) { + case ConfigOptionType::Enum: + return std::get(option.variant).default_value; + case ConfigOptionType::Number: + return std::get(option.variant).default_value; + case ConfigOptionType::String: + return std::get(option.variant).default_value; + default: + assert(false && "Unknown config option type."); + return std::monostate(); + } + } +} + +void recomp::mods::ModContext::set_mod_config_path(const std::filesystem::path &path) { + mod_config_path = path; +} + std::vector recomp::mods::ModContext::load_mods(const GameEntry& game_entry, uint8_t* rdram, int32_t load_address, uint32_t& ram_used) { std::vector ret{}; ram_used = 0; @@ -1872,7 +2058,3 @@ void recomp::mods::ModContext::unload_mods() { num_events = recomp::overlays::num_base_events(); active_game = (size_t)-1; } - -void recomp::mods::initialize_mod_recompiler() { - N64Recomp::live_recompiler_init(); -} diff --git a/librecomp/src/recomp.cpp b/librecomp/src/recomp.cpp index fe0513b..45070a2 100644 --- a/librecomp/src/recomp.cpp +++ b/librecomp/src/recomp.cpp @@ -23,6 +23,7 @@ #include "ultramodern/error_handling.hpp" #include "librecomp/addresses.hpp" #include "librecomp/mods.hpp" +#include "recompiler/live_recompiler.h" #ifdef _WIN32 # define WIN32_LEAN_AND_MEAN @@ -37,16 +38,6 @@ #define PATHFMT "%s" #endif -#ifdef _MSC_VER -inline uint32_t byteswap(uint32_t val) { - return _byteswap_ulong(val); -} -#else -constexpr uint32_t byteswap(uint32_t val) { - return __builtin_bswap32(val); -} -#endif - enum GameStatus { None, Running, @@ -91,11 +82,18 @@ bool recomp::register_game(const recomp::GameEntry& entry) { return true; } +void recomp::mods::initialize_mods() { + N64Recomp::live_recompiler_init(); + std::filesystem::create_directories(config_path / mods_directory); + std::filesystem::create_directories(config_path / mod_config_directory); + mod_context->set_mod_config_path(config_path / mod_config_directory); +} + void recomp::mods::scan_mods() { std::vector mod_open_errors; { std::lock_guard mod_lock{ mod_context_mutex }; - mod_open_errors = mod_context->scan_mod_folder(config_path / "mods"); + mod_open_errors = mod_context->scan_mod_folder(config_path / mods_directory); } for (const auto& cur_error : mod_open_errors) { printf("Error opening mod " PATHFMT ": %s (%s)\n", cur_error.mod_path.c_str(), recomp::mods::error_to_string(cur_error.error).c_str(), cur_error.error_param.c_str()); @@ -520,6 +518,16 @@ const recomp::mods::ConfigSchema &recomp::mods::get_mod_config_schema(const std: return mod_context->get_mod_config_schema(mod_id); } +void recomp::mods::set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value) { + std::lock_guard lock{ mod_context_mutex }; + return mod_context->set_mod_config_value(mod_id, option_id, value); +} + +recomp::mods::ConfigValueVariant recomp::mods::get_mod_config_value(const std::string &mod_id, const std::string &option_id) { + std::lock_guard lock{ mod_context_mutex }; + return mod_context->get_mod_config_value(mod_id, option_id); +} + std::vector recomp::mods::get_mod_details(const std::string& mod_game_id) { std::lock_guard lock { mod_context_mutex }; return mod_context->get_mod_details(mod_game_id); @@ -650,7 +658,8 @@ void recomp::start( } } - recomp::mods::initialize_mod_recompiler(); + recomp::mods::initialize_mods(); + recomp::mods::scan_mods(); // Allocate rdram without comitting it. Use a platform-specific virtual allocation function // that initializes to zero. Protect the region above the memory size to catch accesses to invalid addresses.