From ec28e164db3abeb7769cf0d7c364bc8198d115ed Mon Sep 17 00:00:00 2001 From: Isaac0-dev <62234577+Isaac0-dev@users.noreply.github.com> Date: Sun, 10 May 2026 01:13:08 +1000 Subject: [PATCH] optimised loading the mod cache (#1222) * use hashmaps in mod cache * use templates, suggested by peachy an attempt to do what peachy is talking about * that looks unusual * change data to key parameter --- data/dynos_cmap.cpp | 210 ++++++++++++++++++---------------------- data/dynos_cmap.cpp.h | 11 ++- src/pc/mods/mod_cache.c | 128 ++++++++++++------------ src/pc/mods/mod_cache.h | 2 +- 4 files changed, 173 insertions(+), 178 deletions(-) diff --git a/data/dynos_cmap.cpp b/data/dynos_cmap.cpp index 5929885f6..70b0ca4ad 100644 --- a/data/dynos_cmap.cpp +++ b/data/dynos_cmap.cpp @@ -3,186 +3,164 @@ #include #include #include - -enum class MapType { - Ordered, - Unordered -}; +#include // Ordered maps can be iterated by key order // Unordered maps have the fastest lookup times (also called a hash map) -class HMap { +template +class IHMap { public: - HMap(MapType type = MapType::Ordered) : mMapType(type) { - switch (mMapType) { - case MapType::Ordered: - mOrderedMap = std::make_unique>(); - break; - case MapType::Unordered: - mUnorderedMap = std::make_unique>(); - break; - } + virtual ~IHMap() = default; + + virtual void* get(const KeyType& key) = 0; + virtual void put(const KeyType& key, void* value) = 0; + virtual void erase(const KeyType& key) = 0; + virtual void clear() = 0; + virtual size_t size() const = 0; + virtual void* begin() = 0; + virtual void* next() = 0; +}; + +template +class HMap final : public IHMap { +private: + using MapType = typename std::conditional< + UseUnordered, + std::unordered_map, + std::map + >::type; + +public: + void* get(const KeyType& key) override { + auto it = mMap.find(key); + return (it != mMap.end()) ? it->second : nullptr; } - void* get(int64_t key) { - switch (mMapType) { - case MapType::Ordered: { - auto it = mOrderedMap->find(key); - if (it != mOrderedMap->end()) { - return it->second; - } - break; - } - case MapType::Unordered: { - auto it = mUnorderedMap->find(key); - if (it != mUnorderedMap->end()) { - return it->second; - } - break; - } - } - return nullptr; + void put(const KeyType& key, void* value) override { + mMap.insert_or_assign(key, value); } - void put(int64_t key, void* value) { - switch (mMapType) { - case MapType::Ordered: - mOrderedMap->insert_or_assign(key, value); - break; - case MapType::Unordered: - mUnorderedMap->insert_or_assign(key, value); - break; - } + void erase(const KeyType& key) override { + mMap.erase(key); } - void erase(int64_t key) { - switch (mMapType) { - case MapType::Ordered: - mOrderedMap->erase(key); - break; - case MapType::Unordered: - mUnorderedMap->erase(key); - break; - } + void clear() override { + mMap.clear(); } - void clear() { - switch (mMapType) { - case MapType::Ordered: - mOrderedMap->clear(); - break; - case MapType::Unordered: - mUnorderedMap->clear(); - break; - } + size_t size() const override { + return mMap.size(); } - size_t size() const { - switch (mMapType) { - case MapType::Ordered: - return mOrderedMap->size(); - case MapType::Unordered: - return mUnorderedMap->size(); - } - return 0; + void* begin() override { + if (mMap.empty()) return nullptr; + mIterator = mMap.begin(); + return mIterator->second; } - void* begin() { - switch (mMapType) { - case MapType::Ordered: { - auto& orderedMap = *mOrderedMap; - if (orderedMap.empty()) { return nullptr; } - mOrderedIterator = mOrderedMap->begin(); - return mOrderedIterator->second; - } - case MapType::Unordered: { - auto& unorderedMap = *mUnorderedMap; - if (unorderedMap.empty()) { return nullptr; } - mUnorderedIterator = mUnorderedMap->begin(); - return mUnorderedIterator->second; - } - } - return nullptr; - } - - void* next() { - switch (mMapType) { - case MapType::Ordered: { - if (++mOrderedIterator != mOrderedMap->end()) { - return mOrderedIterator->second; - } - break; - } - case MapType::Unordered: { - if (++mUnorderedIterator != mUnorderedMap->end()) { - return mUnorderedIterator->second; - } - break; - } + void* next() override { + if (++mIterator != mMap.end()) { + return mIterator->second; } return nullptr; } private: - MapType mMapType; - - std::unique_ptr> mOrderedMap; - typename std::map::iterator mOrderedIterator; - - std::unique_ptr> mUnorderedMap; - typename std::unordered_map::iterator mUnorderedIterator; + MapType mMap; + typename MapType::iterator mIterator; }; extern "C" { void* hmap_create(bool useUnordered) { - return new HMap(useUnordered ? MapType::Unordered : MapType::Ordered); + if (useUnordered) { + return new HMap(); + } + return new HMap(); } void* hmap_get(void* map, int64_t key) { if (!map) { return NULL; } - HMap* hmap = reinterpret_cast(map); + IHMap* hmap = static_cast*>(map); return hmap->get(key); } void hmap_put(void* map, int64_t key, void* value) { if (!map) { return; } - HMap* hmap = reinterpret_cast(map); + IHMap* hmap = static_cast*>(map); hmap->put(key, value); } void hmap_del(void* map, int64_t key) { if (!map) { return; } - HMap* hmap = reinterpret_cast(map); + IHMap* hmap = static_cast*>(map); hmap->erase(key); } void hmap_clear(void* map) { if (!map) { return; } - HMap* hmap = reinterpret_cast(map); + IHMap* hmap = static_cast*>(map); hmap->clear(); } void hmap_destroy(void* map) { if (!map) { return; } - delete reinterpret_cast(map); + delete static_cast*>(map); } size_t hmap_len(void* map) { if (!map) { return 0; } - HMap* hmap = reinterpret_cast(map); + IHMap* hmap = static_cast*>(map); return hmap->size(); } void* hmap_begin(void* map) { if (!map) { return NULL; } - HMap* hmap = reinterpret_cast(map); + IHMap* hmap = static_cast*>(map); return hmap->begin(); } void* hmap_next(void* map) { if (!map) { return NULL; } - HMap* hmap = reinterpret_cast(map); + IHMap* hmap = static_cast*>(map); return hmap->next(); } + +// Data/String map (for larger keys) +void* hmap_data_create(void) { + return new HMap(); +} + +void* hmap_data_get(void* map, const char* key, size_t len) { + if (!map) { return NULL; } + std::string keyString(key, len); + return static_cast*>(map)->get(keyString); +} + +void hmap_data_put(void* map, const char* key, size_t len, void* value) { + if (!map) { return; } + std::string keyString(key, len); + static_cast*>(map)->put(keyString, value); +} + +void hmap_data_del(void* map, const char* key, size_t len) { + if (!map) { return; } + std::string keyString(key, len); + static_cast*>(map)->erase(keyString); +} + +void hmap_data_clear(void* map) { + if (!map) { return; } + static_cast*>(map)->clear(); +} + +void hmap_data_destroy(void* map) { + if (!map) { return; } + delete static_cast*>(map); +} + +size_t hmap_data_len(void* map) { + if (!map) { return 0; } + return static_cast*>(map)->size(); +} } diff --git a/data/dynos_cmap.cpp.h b/data/dynos_cmap.cpp.h index 09d6553d8..9c21153f4 100644 --- a/data/dynos_cmap.cpp.h +++ b/data/dynos_cmap.cpp.h @@ -1,5 +1,7 @@ #ifndef DYNOS_CMAP_CPP_H #define DYNOS_CMAP_CPP_H +#endif + #ifndef __cplusplus void* hmap_create(bool useUnordered); @@ -13,5 +15,12 @@ size_t hmap_len(void* map); void* hmap_begin(void* map); void* hmap_next(void* map); +void* hmap_data_create(void); +void* hmap_data_get(void* map, const char* key, size_t len); +void hmap_data_put(void* map, const char* key, size_t len, void* value); +void hmap_data_del(void* map, const char* key, size_t len); +void hmap_data_clear(void* map); +void hmap_data_destroy(void* map); +size_t hmap_data_len(void* map); + #endif -#endif \ No newline at end of file diff --git a/src/pc/mods/mod_cache.c b/src/pc/mods/mod_cache.c index 3df1f378d..0ad2a0718 100644 --- a/src/pc/mods/mod_cache.c +++ b/src/pc/mods/mod_cache.c @@ -10,27 +10,58 @@ #include "pc/utils/md5.h" #include "pc/lua/smlua_hooks.h" #include "pc/loading.h" +#include "data/dynos_cmap.cpp.h" #define MOD_CACHE_FILENAME "mod.cache" #define MOD_CACHE_VERSION 7 #define MD5_BUFFER_SIZE 1024 -static struct ModCacheEntry* sModCacheEntries = NULL; +static struct ModCacheEntry** sModCacheEntries = NULL; static size_t sModCacheLength = 0; static size_t sModLengthCapacity = 0; +static void* sPathMap = NULL; // lookup by file path +static void* sDataMap = NULL; // lookup by file data md5 hash + static void mod_cache_remove_node(struct ModCacheEntry* node) { + + // remove from hashmaps + if (node->path) { + hmap_data_del(sPathMap, node->path, strlen(node->path)); + } + hmap_data_del(sDataMap, (const char*)node->dataHash, 16); + if (node->path) { free(node->path); node->path = NULL; } - if (node != &sModCacheEntries[sModCacheLength - 1]) - memcpy(node, &sModCacheEntries[sModCacheLength - 1], sizeof(struct ModCacheEntry)); + size_t index = node->arrayIndex; + size_t lastIndex = sModCacheLength - 1; + if (index != lastIndex) { + struct ModCacheEntry* lastNode = sModCacheEntries[lastIndex]; + sModCacheEntries[index] = lastNode; + lastNode->arrayIndex = index; + } + sModCacheEntries[lastIndex] = NULL; sModCacheLength--; + free(node); } void mod_cache_shutdown(void) { LOG_INFO("Shutting down mod cache."); + + // destroy maps + hmap_data_destroy(sPathMap); + hmap_data_destroy(sDataMap); + sPathMap = NULL; + sDataMap = NULL; + + for (size_t i = 0; i < sModCacheLength; i++) { + if (sModCacheEntries[i]->path) { + free(sModCacheEntries[i]->path); + } + free(sModCacheEntries[i]); + } sModCacheLength = 0; sModLengthCapacity = 0; free(sModCacheEntries); @@ -78,15 +109,6 @@ void mod_cache_md5(const char* inPath, u8* outDataPath) { MD5_Final(outDataPath, &ctx); } -static u64 mod_cache_fnv1a(const char* str) { - u64 hash = 0xCBF29CE484222325; - while (*str) { - hash *= 0x100000001B3; - hash ^= *str++; - } - return hash; -} - static bool mod_cache_is_valid(struct ModCacheEntry* node) { if (node == NULL || node->path == NULL || strlen(node->path) == 0) { return false; @@ -98,38 +120,24 @@ static bool mod_cache_is_valid(struct ModCacheEntry* node) { struct ModCacheEntry* mod_cache_get_from_hash(u8* dataHash) { if (dataHash == NULL) { return NULL; } - for (size_t i = 0; i < sModCacheLength;) { - struct ModCacheEntry* node = &sModCacheEntries[i]; - if (!memcmp(node->dataHash, dataHash, 16)) { - if (mod_cache_is_valid(node)) { - return node; - } else { - mod_cache_remove_node(node); - continue; - } - } - i++; - } + + struct ModCacheEntry* node = hmap_data_get(sDataMap, (const char*) dataHash, 16); + if (!node) { return NULL; } + if (mod_cache_is_valid(node)) { return node; } + + mod_cache_remove_node(node); return NULL; } struct ModCacheEntry* mod_cache_get_from_path(const char* path, bool validate) { if (path == NULL || strlen(path) == 0) { return NULL; } - u64 pathHash = mod_cache_fnv1a(path); - for (size_t i = 0; i < sModCacheLength;) { - struct ModCacheEntry* node = &sModCacheEntries[i]; - if (node->pathHash == pathHash && !strcmp(node->path, path)) { - if (!validate) { - return node; - } else if (mod_cache_is_valid(node)) { - return node; - } else { - mod_cache_remove_node(node); - continue; - } - } - i++; - } + + struct ModCacheEntry* node = hmap_data_get(sPathMap, path, strlen(path)); + if (!node) { return NULL; } + if (!validate) { return node; } + if (mod_cache_is_valid(node)) { return node; } + + mod_cache_remove_node(node); return NULL; } @@ -148,7 +156,6 @@ void mod_cache_add_internal(u8* dataHash, u64 lastLoaded, char* inPath) { return; } normalize_path((char*)path); - u64 pathHash = mod_cache_fnv1a(path); bool foundNonZero = false; for (u8 i = 0; i < 16; i++) { @@ -163,34 +170,35 @@ void mod_cache_add_internal(u8* dataHash, u64 lastLoaded, char* inPath) { return; } + if (!sPathMap) { sPathMap = hmap_data_create(); } + if (!sDataMap) { sDataMap = hmap_data_create(); } + + struct ModCacheEntry* existing = hmap_data_get(sPathMap, path, strlen(path)); + if (existing) { + mod_cache_remove_node(existing); + } + if (sModCacheEntries == NULL) { sModLengthCapacity = 16; sModCacheLength = 0; - sModCacheEntries = calloc(sModLengthCapacity, sizeof(struct ModCacheEntry)); + sModCacheEntries = calloc(sModLengthCapacity, sizeof(struct ModCacheEntry *)); } else if (sModCacheLength == sModLengthCapacity) { sModLengthCapacity *= 2; - sModCacheEntries = realloc(sModCacheEntries, sizeof(struct ModCacheEntry) * sModLengthCapacity); + sModCacheEntries = realloc(sModCacheEntries, sizeof(struct ModCacheEntry *) * sModLengthCapacity); } - struct ModCacheEntry node = {}; - memcpy(node.dataHash, dataHash, sizeof(u8) * 16); + struct ModCacheEntry *node = malloc(sizeof(struct ModCacheEntry)); + sModCacheEntries[sModCacheLength] = node; + memcpy(node->dataHash, dataHash, sizeof(u8) * 16); if (lastLoaded == 0) { lastLoaded = clock(); } - node.lastLoaded = lastLoaded; - node.path = (char*)path; - node.pathHash = pathHash; + node->lastLoaded = lastLoaded; + node->path = path; + node->arrayIndex = sModCacheLength; + sModCacheLength++; - for (size_t i = 0; i < sModCacheLength;) { - struct ModCacheEntry* n = &sModCacheEntries[i]; - - // found old hash, remove it - if (n->pathHash == pathHash && !strcmp(n->path, path)) { - LOG_INFO("Removing old node: %s", node->path); - mod_cache_remove_node(n); - } else { - i++; - } - } - memcpy(&sModCacheEntries[sModCacheLength++], &node, sizeof(node)); + // insert into hashmaps + hmap_data_put(sPathMap, path, strlen(path), node); + hmap_data_put(sDataMap, (const char*) dataHash, 16, node); } void mod_cache_add(struct Mod* mod, struct ModFile* file, bool useFilePath) { @@ -325,7 +333,7 @@ void mod_cache_save(void) { fwrite(&t, sizeof(u8), 1, fp); for (size_t i = 0; i < sModCacheLength; i++) { - struct ModCacheEntry* node = &sModCacheEntries[i]; + struct ModCacheEntry *node = sModCacheEntries[i]; if (node->path == NULL) { continue; } u16 pathLen = strlen(node->path); if (pathLen == 0) { continue; } diff --git a/src/pc/mods/mod_cache.h b/src/pc/mods/mod_cache.h index 4993fe447..4a8ba11e2 100644 --- a/src/pc/mods/mod_cache.h +++ b/src/pc/mods/mod_cache.h @@ -7,7 +7,7 @@ struct ModCacheEntry { u8 dataHash[16]; u64 lastLoaded; char* path; - u64 pathHash; + size_t arrayIndex; }; void mod_cache_md5(const char* inPath, u8* outDataPath);