Config storage for mods.

This commit is contained in:
Dario 2025-01-21 22:52:39 -03:00 committed by Mr-Wiseguy
parent 293e09f0ed
commit 7045b43457
4 changed files with 356 additions and 26 deletions

View file

@ -13,6 +13,9 @@
#include <array>
#include <cstddef>
#include <variant>
#include <mutex>
#include "blockingconcurrentqueue.h"
#define MINIZ_NO_DEFLATE_APIS
#define MINIZ_NO_ARCHIVE_WRITING_APIS
@ -55,6 +58,9 @@ struct std::hash<recomp::mods::HookDefinition>
namespace recomp {
namespace mods {
static constexpr std::string_view mods_directory = "mods";
static constexpr std::string_view mod_config_directory = "mod_config";
enum class ModOpenError {
Good,
DoesNotExist,
@ -194,6 +200,13 @@ namespace recomp {
struct ConfigSchema {
std::vector<ConfigOption> options;
std::unordered_map<std::string, size_t> options_by_id;
};
typedef std::variant<std::monostate, uint32_t, double, std::string> ConfigValueVariant;
struct ConfigStorage {
std::unordered_map<std::string, ConfigValueVariant> value_map;
};
struct ModDetails {
@ -302,6 +315,9 @@ namespace recomp {
void unload_mods();
std::vector<ModDetails> get_mod_details(const std::string& mod_game_id);
const ConfigSchema &get_mod_config_schema(const std::string &mod_id) const;
void set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value);
ConfigValueVariant get_mod_config_value(const std::string &mod_id, const std::string &option_id);
void set_mod_config_path(const std::filesystem::path &path);
ModContentTypeId register_content_type(const ModContentType& type);
bool register_container_type(const std::string& extension, const std::vector<ModContentTypeId>& content_types, bool requires_manifest);
ModContentTypeId get_code_content_type() const { return code_content_type_id; }
@ -313,13 +329,14 @@ namespace recomp {
CodeModLoadError init_mod_code(uint8_t* rdram, const std::unordered_map<uint32_t, uint16_t>& section_vrom_map, ModHandle& mod, int32_t load_address, bool hooks_available, uint32_t& ram_used, std::string& error_param);
CodeModLoadError load_mod_code(uint8_t* rdram, ModHandle& mod, uint32_t base_event_index, std::string& error_param);
CodeModLoadError resolve_code_dependencies(ModHandle& mod, const std::unordered_map<recomp_func_t*, recomp::overlays::BasePatchedFunction>& base_patched_funcs, std::string& error_param);
void add_opened_mod(ModManifest&& manifest, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& detected_content_types);
void add_opened_mod(ModManifest&& manifest, ConfigStorage&& config_storage, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& detected_content_types);
void close_mods();
std::vector<ModLoadErrorDetails> regenerate_with_hooks(
const std::vector<std::pair<HookDefinition, size_t>>& sorted_unprocessed_hooks,
const std::unordered_map<uint32_t, uint16_t>& section_vrom_map,
const std::unordered_map<recomp_func_t*, overlays::BasePatchedFunction>& base_patched_funcs,
std::span<const uint8_t> decompressed_rom);
void dirty_mod_configuration_thread_process();
static void on_code_mod_enabled(ModContext& context, const ModHandle& mod);
@ -329,10 +346,15 @@ namespace recomp {
std::unordered_map<std::string, size_t> mod_game_ids;
std::vector<ModHandle> opened_mods;
std::unordered_map<std::string, size_t> opened_mods_by_id;
std::mutex opened_mods_mutex;
std::unordered_set<std::string> mod_ids;
std::unordered_set<std::string> enabled_mods;
std::unordered_map<recomp_func_t*, PatchData> patched_funcs;
std::unordered_map<std::string, size_t> loaded_mods_by_id;
std::unique_ptr<std::thread> dirty_mod_configuration_thread;
moodycamel::BlockingConcurrentQueue<std::string> dirty_mod_configuration_thread_queue;
std::filesystem::path mod_config_path;
std::mutex mod_config_storage_mutex;
std::vector<size_t> loaded_code_mods;
// Code handle for vanilla code that was regenerated to add hooks.
std::unique_ptr<LiveRecompilerCodeHandle> regenerated_code_handle;
@ -366,13 +388,14 @@ namespace recomp {
public:
// TODO make these private and expose methods for the functionality they're currently used in.
ModManifest manifest;
ConfigStorage config_storage;
std::unique_ptr<ModCodeHandle> code_handle;
std::unique_ptr<N64Recomp::Context> recompiler_context;
std::vector<uint32_t> section_load_addresses;
// Content types present in this mod.
std::vector<ModContentTypeId> content_types;
ModHandle(const ModContext& context, ModManifest&& manifest, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& content_types);
ModHandle(const ModContext& context, ModManifest&& manifest, ConfigStorage&& config_storage, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& content_types);
ModHandle(const ModHandle& rhs) = delete;
ModHandle& operator=(const ModHandle& rhs) = delete;
ModHandle(ModHandle&& rhs);
@ -502,12 +525,14 @@ namespace recomp {
CodeModLoadError validate_api_version(uint32_t api_version, std::string& error_param);
void initialize_mod_recompiler();
void initialize_mods();
void scan_mods();
void enable_mod(const std::string& mod_id, bool enabled);
bool is_mod_enabled(const std::string& mod_id);
bool is_mod_auto_enabled(const std::string& mod_id);
const ConfigSchema &get_mod_config_schema(const std::string &mod_id);
void set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value);
ConfigValueVariant get_mod_config_value(const std::string &mod_id, const std::string &option_id);
ModContentTypeId register_mod_content_type(const ModContentType& type);
bool register_mod_container_type(const std::string& extension, const std::vector<ModContentTypeId>& content_types, bool requires_manifest);
}

View file

@ -3,8 +3,38 @@
#include "json/json.hpp"
#include "recompiler/context.h"
#include "librecomp/files.hpp"
#include "librecomp/mods.hpp"
static bool read_json(std::ifstream input_file, nlohmann::json &json_out) {
if (!input_file.good()) {
return false;
}
try {
input_file >> json_out;
}
catch (nlohmann::json::parse_error &) {
return false;
}
return true;
}
static bool read_json_with_backups(const std::filesystem::path &path, nlohmann::json &json_out) {
// Try reading and parsing the base file.
if (read_json(std::ifstream{ path }, json_out)) {
return true;
}
// Try reading and parsing the backup file.
if (read_json(recomp::open_input_backup_file(path), json_out)) {
return true;
}
// Both reads failed.
return false;
}
recomp::mods::ZipModFileHandle::~ZipModFileHandle() {
if (file_handle) {
fclose(file_handle);
@ -469,7 +499,9 @@ recomp::mods::ModOpenError parse_manifest_config_schema_option(const nlohmann::j
break;
}
ret.config_schema.options.push_back(option);
ret.config_schema.options_by_id.emplace(option.id, ret.config_schema.options.size());
ret.config_schema.options.emplace_back(option);
return recomp::mods::ModOpenError::Good;
}
@ -606,6 +638,83 @@ recomp::mods::ModOpenError parse_manifest(recomp::mods::ModManifest& ret, const
return recomp::mods::ModOpenError::Good;
}
bool parse_mod_config_storage(const std::filesystem::path &path, const std::string &expected_mod_id, recomp::mods::ConfigStorage &config_storage, const recomp::mods::ConfigSchema &config_schema) {
using json = nlohmann::json;
json config_json;
if (!read_json_with_backups(path, config_json)) {
return false;
}
auto mod_id = config_json.find("mod_id");
if (mod_id != config_json.end()) {
std::string mod_id_str;
if (get_to<json::string_t>(*mod_id, mod_id_str)) {
if (*mod_id != expected_mod_id) {
// The mod's ID doesn't match.
return false;
}
}
else {
// The mod ID is not a string.
return false;
}
}
else {
// The configuration file doesn't have a mod ID.
return false;
}
auto storage_json = config_json.find("storage");
if (storage_json == config_json.end()) {
// The configuration file doesn't have a storage object.
return false;
}
if (!storage_json->is_object()) {
// The storage key does not correspond to an object.
return false;
}
// Only parse the object for known option types based on the schema.
int64_t value_int64;
double value_double;
std::string value_str;
for (const recomp::mods::ConfigOption &option : config_schema.options) {
auto option_json = storage_json->find(option.id);
if (option_json == storage_json->end()) {
// Option doesn't exist in storage.
continue;
}
switch (option.type) {
case recomp::mods::ConfigOptionType::Enum:
if (get_to<int64_t>(*option_json, value_int64)) {
config_storage.value_map[option.id] = uint32_t(value_int64);
}
break;
case recomp::mods::ConfigOptionType::Number:
if (get_to<double>(*option_json, value_double)) {
config_storage.value_map[option.id] = value_double;
}
break;
case recomp::mods::ConfigOptionType::String: {
if (get_to<json::string_t>(*option_json, value_str)) {
config_storage.value_map[option.id] = value_str;
}
break;
}
default:
assert(false && "Unknown option type.");
break;
}
}
return true;
}
recomp::mods::ModOpenError recomp::mods::ModContext::open_mod(const std::filesystem::path& mod_path, std::string& error_param, const std::vector<ModContentTypeId>& supported_content_types, bool requires_manifest) {
ModManifest manifest{};
std::error_code ec;
@ -724,9 +833,14 @@ recomp::mods::ModOpenError recomp::mods::ModContext::open_mod(const std::filesys
}
}
// Read the mod config if it exists.
ConfigStorage config_storage;
std::filesystem::path config_path = mod_config_path / (manifest.mod_id + ".json");
parse_mod_config_storage(config_path, manifest.mod_id, config_storage, manifest.config_schema);
// Store the loaded mod manifest in a new mod handle.
manifest.mod_root_path = mod_path;
add_opened_mod(std::move(manifest), std::move(game_indices), std::move(detected_content_types));
add_opened_mod(std::move(manifest), std::move(config_storage), std::move(game_indices), std::move(detected_content_types));
return ModOpenError::Good;
}

View file

@ -3,6 +3,7 @@
#include <sstream>
#include <functional>
#include "librecomp/files.hpp"
#include "librecomp/mods.hpp"
#include "librecomp/overlays.hpp"
#include "librecomp/game.hpp"
@ -213,8 +214,9 @@ recomp::mods::CodeModLoadError recomp::mods::validate_api_version(uint32_t api_v
}
}
recomp::mods::ModHandle::ModHandle(const ModContext& context, ModManifest&& manifest, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& content_types) :
recomp::mods::ModHandle::ModHandle(const ModContext& context, ModManifest&& manifest, ConfigStorage&& config_storage, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& content_types) :
manifest(std::move(manifest)),
config_storage(std::move(config_storage)),
code_handle(),
recompiler_context{std::make_unique<N64Recomp::Context>()},
content_types{std::move(content_types)},
@ -539,10 +541,11 @@ void unpatch_func(void* target_func, const recomp::mods::PatchData& data) {
protect(target_func, old_flags);
}
void recomp::mods::ModContext::add_opened_mod(ModManifest&& manifest, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& detected_content_types) {
void recomp::mods::ModContext::add_opened_mod(ModManifest&& manifest, ConfigStorage&& config_storage, std::vector<size_t>&& game_indices, std::vector<ModContentTypeId>&& detected_content_types) {
std::unique_lock lock(opened_mods_mutex);
size_t mod_index = opened_mods.size();
opened_mods_by_id.emplace(manifest.mod_id, mod_index);
opened_mods.emplace_back(*this, std::move(manifest), std::move(game_indices), std::move(detected_content_types));
opened_mods.emplace_back(*this, std::move(manifest), std::move(config_storage), std::move(game_indices), std::move(detected_content_types));
}
recomp::mods::ModLoadError recomp::mods::ModContext::load_mod(recomp::mods::ModHandle& mod, std::string& error_param) {
@ -567,12 +570,107 @@ void recomp::mods::ModContext::register_game(const std::string& mod_game_id) {
}
void recomp::mods::ModContext::close_mods() {
std::unique_lock lock(opened_mods_mutex);
opened_mods_by_id.clear();
opened_mods.clear();
mod_ids.clear();
enabled_mods.clear();
}
bool save_mod_config_storage(const std::filesystem::path &path, const std::string &mod_id, const recomp::Version &mod_version, const recomp::mods::ConfigStorage &config_storage, const recomp::mods::ConfigSchema &config_schema) {
nlohmann::json config_json;
config_json["mod_id"] = mod_id;
config_json["mod_version"] = mod_version.to_string();
config_json["recomp_version"] = recomp::get_project_version().to_string();
nlohmann::json &storage_json = config_json["storage"];
for (auto it : config_storage.value_map) {
auto id_it = config_schema.options_by_id.find(it.first);
if (id_it == config_schema.options_by_id.end()) {
continue;
}
const recomp::mods::ConfigOption &config_option = config_schema.options[id_it->second];
switch (config_option.type) {
case recomp::mods::ConfigOptionType::Enum:
storage_json[it.first] = std::get<recomp::mods::ConfigOptionEnum>(config_option.variant).options[std::get<uint32_t>(it.second)];
break;
case recomp::mods::ConfigOptionType::Number:
storage_json[it.first] = std::get<double>(it.second);
break;
case recomp::mods::ConfigOptionType::String:
storage_json[it.first] = std::get<std::string>(it.second);
break;
default:
assert(false && "Unknown config type.");
break;
}
}
std::ofstream output_file = recomp::open_output_file_with_backup(path);
if (!output_file.good()) {
return false;
}
output_file << std::setw(4) << config_json;
output_file.close();
return recomp::finalize_output_file_with_backup(path);
}
void recomp::mods::ModContext::dirty_mod_configuration_thread_process() {
using namespace std::chrono_literals;
std::string mod_id;
std::unordered_set<std::string> pending_mods;
std::unordered_map<std::string, ConfigStorage> pending_mod_storage;
std::unordered_map<std::string, ConfigSchema> pending_mod_schema;
std::unordered_map<std::string, Version> pending_mod_version;
std::filesystem::path config_path;
bool active = true;
while (active) {
// Wait for at least one mod to require writing.
dirty_mod_configuration_thread_queue.wait_dequeue(mod_id);
if (!mod_id.empty()) {
pending_mods.emplace(mod_id);
}
else {
active = false;
}
// Clear out the entire queue to coalesce all writes with a timeout.
while (active && dirty_mod_configuration_thread_queue.wait_dequeue_timed(mod_id, 1s)) {
if (!mod_id.empty()) {
pending_mods.emplace(mod_id);
}
else {
active = false;
}
}
if (active && !pending_mods.empty()) {
{
std::unique_lock opened_mods_lock(opened_mods_mutex);
for (const std::string &id : pending_mods) {
auto it = opened_mods_by_id.find(id);
if (it != opened_mods_by_id.end()) {
const ModHandle &mod = opened_mods[it->second];
std::unique_lock config_storage_lock(mod_config_storage_mutex);
pending_mod_storage[id] = mod.config_storage;
pending_mod_schema[id] = mod.manifest.config_schema;
pending_mod_version[id] = mod.manifest.version;
}
}
}
for (const std::string &id : pending_mods) {
config_path = mod_config_path / std::string(id + ".json");
save_mod_config_storage(config_path, id, pending_mod_version[id], pending_mod_storage[id], pending_mod_schema[id]);
}
}
}
}
std::vector<recomp::mods::ModOpenErrorDetails> recomp::mods::ModContext::scan_mod_folder(const std::filesystem::path& mod_folder) {
std::vector<recomp::mods::ModOpenErrorDetails> ret{};
std::error_code ec;
@ -623,6 +721,8 @@ recomp::mods::ModContext::ModContext() {
// Register the default mod container type (.nrm) and allow it to have any content type by passing an empty vector.
register_container_type(std::string{ modpaths::default_mod_extension }, {}, true);
dirty_mod_configuration_thread = std::make_unique<std::thread>(&ModContext::dirty_mod_configuration_thread_process, this);
}
void recomp::mods::ModContext::on_code_mod_enabled(ModContext& context, const ModHandle& mod) {
@ -635,8 +735,11 @@ void recomp::mods::ModContext::on_code_mod_enabled(ModContext& context, const Mo
}
}
// Nothing needed for this, it just need to be explicitly declared outside the header to allow forward declaration of ModHandle.
recomp::mods::ModContext::~ModContext() = default;
recomp::mods::ModContext::~ModContext() {
dirty_mod_configuration_thread_queue.enqueue(std::string());
dirty_mod_configuration_thread->join();
dirty_mod_configuration_thread.reset();
}
recomp::mods::ModContentTypeId recomp::mods::ModContext::register_content_type(const ModContentType& type) {
size_t ret = content_types.size();
@ -892,6 +995,89 @@ const recomp::mods::ConfigSchema &recomp::mods::ModContext::get_mod_config_schem
return mod.manifest.config_schema;
}
void recomp::mods::ModContext::set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value) {
// Check that the mod exists.
auto find_it = opened_mods_by_id.find(mod_id);
if (find_it == opened_mods_by_id.end()) {
return;
}
ModHandle &mod = opened_mods[find_it->second];
std::unique_lock lock(mod_config_storage_mutex);
auto option_by_id_it = mod.manifest.config_schema.options_by_id.find(option_id);
if (option_by_id_it != mod.manifest.config_schema.options_by_id.end()) {
// Only accept setting values if the value exists and the variant is the right type.
const ConfigOption &option = mod.manifest.config_schema.options[option_by_id_it->second];
switch (option.type) {
case ConfigOptionType::Enum:
if (std::holds_alternative<uint32_t>(value)) {
if (std::get<uint32_t>(value) < std::get<ConfigOptionEnum>(option.variant).options.size()) {
mod.config_storage.value_map[option_id] = value;
}
}
break;
case ConfigOptionType::Number:
if (std::holds_alternative<double>(value)) {
mod.config_storage.value_map[option_id] = value;
}
break;
case ConfigOptionType::String:
if (std::holds_alternative<std::string>(value)) {
mod.config_storage.value_map[option_id] = value;
}
break;
default:
assert(false && "Unknown config option type.");
return;
}
}
// Notify the asynchronous thread it should save the configuration for this mod.
dirty_mod_configuration_thread_queue.enqueue(mod_id);
}
recomp::mods::ConfigValueVariant recomp::mods::ModContext::get_mod_config_value(const std::string &mod_id, const std::string &option_id) {
// Check that the mod exists.
auto find_it = opened_mods_by_id.find(mod_id);
if (find_it == opened_mods_by_id.end()) {
return std::monostate();
}
const ModHandle &mod = opened_mods[find_it->second];
std::unique_lock lock(mod_config_storage_mutex);
auto it = mod.config_storage.value_map.find(option_id);
if (it != mod.config_storage.value_map.end()) {
return it->second;
}
else {
// Attempt to see if we can find a default value from the schema.
auto option_by_id_it = mod.manifest.config_schema.options_by_id.find(option_id);
if (option_by_id_it == mod.manifest.config_schema.options_by_id.end()) {
return std::monostate();
}
const ConfigOption &option = mod.manifest.config_schema.options[option_by_id_it->second];
switch (option.type) {
case ConfigOptionType::Enum:
return std::get<ConfigOptionEnum>(option.variant).default_value;
case ConfigOptionType::Number:
return std::get<ConfigOptionNumber>(option.variant).default_value;
case ConfigOptionType::String:
return std::get<ConfigOptionString>(option.variant).default_value;
default:
assert(false && "Unknown config option type.");
return std::monostate();
}
}
}
void recomp::mods::ModContext::set_mod_config_path(const std::filesystem::path &path) {
mod_config_path = path;
}
std::vector<recomp::mods::ModLoadErrorDetails> recomp::mods::ModContext::load_mods(const GameEntry& game_entry, uint8_t* rdram, int32_t load_address, uint32_t& ram_used) {
std::vector<recomp::mods::ModLoadErrorDetails> ret{};
ram_used = 0;
@ -1872,7 +2058,3 @@ void recomp::mods::ModContext::unload_mods() {
num_events = recomp::overlays::num_base_events();
active_game = (size_t)-1;
}
void recomp::mods::initialize_mod_recompiler() {
N64Recomp::live_recompiler_init();
}

View file

@ -23,6 +23,7 @@
#include "ultramodern/error_handling.hpp"
#include "librecomp/addresses.hpp"
#include "librecomp/mods.hpp"
#include "recompiler/live_recompiler.h"
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
@ -37,16 +38,6 @@
#define PATHFMT "%s"
#endif
#ifdef _MSC_VER
inline uint32_t byteswap(uint32_t val) {
return _byteswap_ulong(val);
}
#else
constexpr uint32_t byteswap(uint32_t val) {
return __builtin_bswap32(val);
}
#endif
enum GameStatus {
None,
Running,
@ -91,11 +82,18 @@ bool recomp::register_game(const recomp::GameEntry& entry) {
return true;
}
void recomp::mods::initialize_mods() {
N64Recomp::live_recompiler_init();
std::filesystem::create_directories(config_path / mods_directory);
std::filesystem::create_directories(config_path / mod_config_directory);
mod_context->set_mod_config_path(config_path / mod_config_directory);
}
void recomp::mods::scan_mods() {
std::vector<recomp::mods::ModOpenErrorDetails> mod_open_errors;
{
std::lock_guard mod_lock{ mod_context_mutex };
mod_open_errors = mod_context->scan_mod_folder(config_path / "mods");
mod_open_errors = mod_context->scan_mod_folder(config_path / mods_directory);
}
for (const auto& cur_error : mod_open_errors) {
printf("Error opening mod " PATHFMT ": %s (%s)\n", cur_error.mod_path.c_str(), recomp::mods::error_to_string(cur_error.error).c_str(), cur_error.error_param.c_str());
@ -520,6 +518,16 @@ const recomp::mods::ConfigSchema &recomp::mods::get_mod_config_schema(const std:
return mod_context->get_mod_config_schema(mod_id);
}
void recomp::mods::set_mod_config_value(const std::string &mod_id, const std::string &option_id, const ConfigValueVariant &value) {
std::lock_guard lock{ mod_context_mutex };
return mod_context->set_mod_config_value(mod_id, option_id, value);
}
recomp::mods::ConfigValueVariant recomp::mods::get_mod_config_value(const std::string &mod_id, const std::string &option_id) {
std::lock_guard lock{ mod_context_mutex };
return mod_context->get_mod_config_value(mod_id, option_id);
}
std::vector<recomp::mods::ModDetails> recomp::mods::get_mod_details(const std::string& mod_game_id) {
std::lock_guard lock { mod_context_mutex };
return mod_context->get_mod_details(mod_game_id);
@ -650,7 +658,8 @@ void recomp::start(
}
}
recomp::mods::initialize_mod_recompiler();
recomp::mods::initialize_mods();
recomp::mods::scan_mods();
// Allocate rdram without comitting it. Use a platform-specific virtual allocation function
// that initializes to zero. Protect the region above the memory size to catch accesses to invalid addresses.