thirdpartymirrors/UnleashedRecomp
1
0
Fork 0
mirror of https://github.com/hedge-dev/UnleashedRecomp.git synced 2026-04-29 22:01:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Cache append/merge archive lookups.

Browse source
This commit is contained in:
Skyth 2024-12-31 17:33:26 +03:00
parent 66a13734d1
commit e2f89b08fa
1 changed files with 287 additions and 227 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
UnleashedRecomp/mod/mod_loader.cpp
View file

@ -50,11 +50,11 @@ std::filesystem::path ModLoader::ResolvePath(std::string_view path)
return {}; return {};
} }
thread_local xxHashMap<std::filesystem::path> s_pathCache; thread_local xxHashMap<std::filesystem::path> s_cache;
XXH64_hash_t hash = XXH3_64bits(path.data(), path.size()); XXH64_hash_t hash = XXH3_64bits(path.data(), path.size());
auto findResult = s_pathCache.find(hash); auto findResult = s_cache.find(hash);
if (findResult != s_pathCache.end()) if (findResult != s_cache.end())
return findResult->second; return findResult->second;
std::string pathStr(path); std::string pathStr(path);
@ -75,11 +75,11 @@ std::filesystem::path ModLoader::ResolvePath(std::string_view path)
{ {
std::filesystem::path modPath = includeDir / fsPath; std::filesystem::path modPath = includeDir / fsPath;
if (std::filesystem::exists(modPath)) if (std::filesystem::exists(modPath))
return s_pathCache.emplace(hash, modPath).first->second; return s_cache.emplace(hash, modPath).first->second;
} }
} }
return s_pathCache.emplace(hash, std::filesystem::path{}).first->second; return s_cache.emplace(hash, std::filesystem::path{}).first->second;
} }
std::vector<std::filesystem::path>* ModLoader::GetIncludeDirectories(size_t modIndex) std::vector<std::filesystem::path>* ModLoader::GetIncludeDirectories(size_t modIndex)
@ -262,10 +262,7 @@ PPC_FUNC(sub_82E0D3E8)
} }
thread_local ankerl::unordered_dense::set<std::string> s_fileNames; thread_local ankerl::unordered_dense::set<std::string> s_fileNames;
thread_local std::vector<uint8_t> s_fileData;
thread_local std::filesystem::path s_tempPath;
s_fileNames.clear(); s_fileNames.clear();
s_fileData.clear();
auto parseArlFileData = [&](const uint8_t* arlFileData, size_t arlFileSize) auto parseArlFileData = [&](const uint8_t* arlFileData, size_t arlFileSize)
{ {
@ -311,26 +308,14 @@ PPC_FUNC(sub_82E0D3E8)
} }
}; };
std::u8string_view arlFilePathU8(reinterpret_cast<const char8_t*>(base + PPC_LOAD_U32(ctx.r4.u32)));
std::filesystem::path arlFilePath;
std::filesystem::path arFilePath;
std::filesystem::path appendArlFilePath;
auto r3 = ctx.r3; auto r3 = ctx.r3;
auto r4 = ctx.r4; auto r4 = ctx.r4;
auto r5 = ctx.r5; auto r5 = ctx.r5;
auto r6 = ctx.r6; auto r6 = ctx.r6;
for (auto& mod : g_mods)
{
for (auto& includeDir : mod.includeDirs)
{
auto loadFile = [&]<typename TFunction>(const std::filesystem::path& filePath, const TFunction& function) auto loadFile = [&]<typename TFunction>(const std::filesystem::path& filePath, const TFunction& function)
{ {
if (mod.type == ModType::UMM && mod.readOnly.contains(filePath)) std::ifstream stream(filePath, std::ios::binary);
return false;
std::ifstream stream(includeDir / filePath, std::ios::binary);
if (stream.good()) if (stream.good())
{ {
be<uint32_t> signature{}; be<uint32_t> signature{};
@ -355,6 +340,8 @@ PPC_FUNC(sub_82E0D3E8)
} }
else else
{ {
thread_local std::vector<uint8_t> s_fileData;
s_fileData.resize(arlFileSize); s_fileData.resize(arlFileSize);
stream.read(reinterpret_cast<char*>(s_fileData.data()), arlFileSize); stream.read(reinterpret_cast<char*>(s_fileData.data()), arlFileSize);
stream.close(); stream.close();
@ -368,6 +355,52 @@ PPC_FUNC(sub_82E0D3E8)
return false; return false;
}; };
thread_local xxHashMap<std::vector<std::pair<std::filesystem::path, bool>>> s_cache;
std::u8string_view arlFilePathU8(reinterpret_cast<const char8_t*>(base + PPC_LOAD_U32(ctx.r4.u32)));
XXH64_hash_t hash = XXH3_64bits(arlFilePathU8.data(), arlFilePathU8.size());
auto findResult = s_cache.find(hash);
if (findResult != s_cache.end())
{
for (const auto& [arlFilePath, isArchiveList] : findResult->second)
{
if (isArchiveList)
loadFile(arlFilePath, parseArlFileData);
else
loadFile(arlFilePath, parseArFileData);
}
}
else
{
std::vector<std::pair<std::filesystem::path, bool>> arlFilePaths;
std::filesystem::path arlFilePath;
std::filesystem::path arFilePath;
std::filesystem::path appendArlFilePath;
for (auto& mod : g_mods)
{
for (auto& includeDir : mod.includeDirs)
{
auto loadUncachedFile = [&](const std::filesystem::path& filePath, bool isArchiveList)
{
if (mod.type == ModType::UMM && mod.readOnly.contains(filePath))
return false;
std::filesystem::path combinedFilePath = includeDir / filePath;
bool success;
if (isArchiveList)
success = loadFile(combinedFilePath, parseArlFileData);
else
success = loadFile(combinedFilePath, parseArFileData);
if (success)
arlFilePaths.emplace_back(std::move(combinedFilePath), isArchiveList);
return success;
};
if (mod.type == ModType::UMM) if (mod.type == ModType::UMM)
{ {
if (mod.merge) if (mod.merge)
@ -378,7 +411,7 @@ PPC_FUNC(sub_82E0D3E8)
arlFilePath += ".arl"; arlFilePath += ".arl";
} }
if (!loadFile(arlFilePath, parseArlFileData)) if (!loadUncachedFile(arlFilePath, true))
{ {
if (arFilePath.empty()) if (arFilePath.empty())
{ {
@ -386,14 +419,16 @@ PPC_FUNC(sub_82E0D3E8)
arFilePath += ".ar"; arFilePath += ".ar";
} }
if (!loadFile(arFilePath, parseArFileData)) if (!loadUncachedFile(arFilePath, false))
{ {
thread_local std::filesystem::path s_tempPath;
for (uint32_t i = 0; ; i++) for (uint32_t i = 0; ; i++)
{ {
s_tempPath = arFilePath; s_tempPath = arFilePath;
s_tempPath += fmt::format(".{:02}", i); s_tempPath += fmt::format(".{:02}", i);
if (!loadFile(s_tempPath, parseArFileData)) if (!loadUncachedFile(s_tempPath, false))
break; break;
} }
} }
@ -413,11 +448,14 @@ PPC_FUNC(sub_82E0D3E8)
appendArlFilePath += ".arl"; appendArlFilePath += ".arl";
} }
loadFile(appendArlFilePath, parseArlFileData); loadUncachedFile(appendArlFilePath, true);
} }
} }
} }
s_cache.emplace(hash, std::move(arlFilePaths));
}
ctx.r3 = r3; ctx.r3 = r3;
ctx.r4 = r4; ctx.r4 = r4;
ctx.r5 = r5; ctx.r5 = r5;
@ -480,13 +518,6 @@ PPC_FUNC(sub_82E0B500)
return; return;
} }
auto r3 = ctx.r3; // Callback
auto r4 = ctx.r4; // Database
auto r5 = ctx.r5; // Name
auto r6 = ctx.r6; // Data
auto r7 = ctx.r7; // Size
auto r8 = ctx.r8; // Callback data
std::u8string_view arFilePathU8(reinterpret_cast<const char8_t*>(base + PPC_LOAD_U32(ctx.r5.u32))); std::u8string_view arFilePathU8(reinterpret_cast<const char8_t*>(base + PPC_LOAD_U32(ctx.r5.u32)));
size_t index = arFilePathU8.find(u8".ar.00"); size_t index = arFilePathU8.find(u8".ar.00");
if (index == (arFilePathU8.size() - 6)) if (index == (arFilePathU8.size() - 6))
@ -507,22 +538,16 @@ PPC_FUNC(sub_82E0B500)
} }
} }
thread_local std::filesystem::path s_tempFilePath; auto r3 = ctx.r3; // Callback
s_tempFilePath.clear(); auto r4 = ctx.r4; // Database
auto r5 = ctx.r5; // Name
auto r6 = ctx.r6; // Data
auto r7 = ctx.r7; // Size
auto r8 = ctx.r8; // Callback data
std::filesystem::path arFilePath;
std::filesystem::path appendArFilePath;
for (auto& mod : g_mods)
{
for (auto& includeDir : mod.includeDirs)
{
auto loadArchive = [&](const std::filesystem::path& arFilePath) auto loadArchive = [&](const std::filesystem::path& arFilePath)
{ {
if (mod.type == ModType::UMM && mod.readOnly.contains(arFilePath)) std::ifstream stream(arFilePath, std::ios::binary);
return false;
std::ifstream stream(includeDir / arFilePath, std::ios::binary);
if (stream.good()) if (stream.good())
{ {
stream.seekg(0, std::ios::end); stream.seekg(0, std::ios::end);
@ -566,18 +591,50 @@ PPC_FUNC(sub_82E0B500)
return false; return false;
}; };
thread_local xxHashMap<std::vector<std::filesystem::path>> s_cache;
XXH64_hash_t hash = XXH3_64bits(arFilePathU8.data(), arFilePathU8.size());
auto findResult = s_cache.find(hash);
if (findResult != s_cache.end())
{
for (const auto& arFilePath : findResult->second)
loadArchive(arFilePath);
}
else
{
std::vector<std::filesystem::path> arFilePaths;
std::filesystem::path arFilePath;
std::filesystem::path appendArFilePath;
for (auto& mod : g_mods)
{
for (auto& includeDir : mod.includeDirs)
{
auto loadUncachedArchive = [&](const std::filesystem::path& arFilePath)
{
if (mod.type == ModType::UMM && mod.readOnly.contains(arFilePath))
return false;
std::filesystem::path combinedFilePath = includeDir / arFilePath;
bool success = loadArchive(combinedFilePath);
if (success)
arFilePaths.emplace_back(std::move(combinedFilePath));
return success;
};
auto loadArchives = [&](const std::filesystem::path& arFilePath) auto loadArchives = [&](const std::filesystem::path& arFilePath)
{ {
s_tempFilePath = arFilePath; thread_local std::filesystem::path s_tempPath;
s_tempFilePath += "l"; s_tempPath = arFilePath;
s_tempPath += "l";
if (mod.type == ModType::UMM && mod.readOnly.contains(s_tempFilePath)) if (mod.type == ModType::UMM && mod.readOnly.contains(s_tempPath))
return; return;
std::ifstream stream(includeDir / s_tempFilePath, std::ios::binary); std::ifstream stream(includeDir / s_tempPath, std::ios::binary);
if (stream.good()) if (stream.good())
{ {
// TODO: Should cache this instead of re-opening the file.
be<uint32_t> signature{}; be<uint32_t> signature{};
uint32_t splitCount{}; uint32_t splitCount{};
stream.read(reinterpret_cast<char*>(&signature), sizeof(signature)); stream.read(reinterpret_cast<char*>(&signature), sizeof(signature));
@ -608,27 +665,27 @@ PPC_FUNC(sub_82E0B500)
if (splitCount == 0) if (splitCount == 0)
{ {
loadArchive(arFilePath); loadUncachedArchive(arFilePath);
} }
else else
{ {
for (uint32_t i = 0; i < splitCount; i++) for (uint32_t i = 0; i < splitCount; i++)
{ {
s_tempFilePath = arFilePath; s_tempPath = arFilePath;
s_tempFilePath += fmt::format(".{:02}", i); s_tempPath += fmt::format(".{:02}", i);
loadArchive(s_tempFilePath); loadUncachedArchive(s_tempPath);
} }
} }
} }
else if (mod.type == ModType::UMM) else if (mod.type == ModType::UMM)
{ {
if (!loadArchive(arFilePath)) if (!loadUncachedArchive(arFilePath))
{ {
for (uint32_t i = 0; ; i++) for (uint32_t i = 0; ; i++)
{ {
s_tempFilePath = arFilePath; s_tempPath = arFilePath;
s_tempFilePath += fmt::format(".{:02}", i); s_tempPath += fmt::format(".{:02}", i);
if (!loadArchive(s_tempFilePath)) if (!loadUncachedArchive(s_tempPath))
break; break;
} }
} }
@ -662,6 +719,9 @@ PPC_FUNC(sub_82E0B500)
} }
} }
s_cache.emplace(hash, std::move(arFilePaths));
}
ctx.r3 = r3; ctx.r3 = r3;
ctx.r4 = r4; ctx.r4 = r4;
ctx.r5 = r5; ctx.r5 = r5;