Implement shader cache writing.

ShaderRecomp/CMakeLists.txt

@@ -16,7 +16,11 @@ add_executable(ShaderRecomp
     shader_recompiler.h)
 
 find_package(directx-dxc CONFIG REQUIRED)
-target_link_libraries(ShaderRecomp PRIVATE Microsoft::DirectXShaderCompiler)
+find_package(xxhash CONFIG REQUIRED)
+
+target_link_libraries(ShaderRecomp PRIVATE
+    Microsoft::DirectXShaderCompiler
+    xxHash::xxhash)
 
 target_precompile_headers(ShaderRecomp PRIVATE pch.h)
 

ShaderRecomp/main.cpp

@@ -2,11 +2,11 @@
 #include "shader_recompiler.h"
 #include "dxc_compiler.h"
 
-static std::unique_ptr<uint8_t[]> readAllBytes(const char* filePath)
+static std::unique_ptr<uint8_t[]> readAllBytes(const char* filePath, size_t& fileSize)
 {
     FILE* file = fopen(filePath, "rb");
     fseek(file, 0, SEEK_END);
-    long fileSize = ftell(file);
+    fileSize = ftell(file);
     fseek(file, 0, SEEK_SET);
     auto data = std::make_unique<uint8_t[]>(fileSize);
     fread(data.get(), 1, fileSize, file);
@@ -21,59 +21,107 @@ static void writeAllBytes(const char* filePath, const void* data, size_t dataSiz
     fclose(file);
 }
 
+struct ShaderCacheEntry
+{
+    XXH64_hash_t hash = 0;
+    uint32_t dxilOffset = 0;
+    uint32_t dxilSize = 0;
+    uint32_t spirvOffset = 0;
+    uint32_t spirvSize = 0;
+    uint64_t reserved = 0;
+};
+
+struct ShaderHolder
+{
+    std::string filePath;
+    size_t fileSize = 0;
+    std::unique_ptr<uint8_t[]> fileData;
+    IDxcBlob* dxil = nullptr;
+    IDxcBlob* spirv = nullptr;
+    ShaderCacheEntry entry;
+};
+
+struct ShaderCacheHeader
+{
+    uint32_t version = 0;
+    uint32_t count = 0;
+    uint32_t reserved0 = 0;
+    uint32_t reserved1 = 0;
+};
+
 int main(int argc, char** argv)
 {
     if (std::filesystem::is_directory(argv[1]))
     {
-        std::vector<std::string> filePaths;
+        std::map<XXH64_hash_t, ShaderHolder> shaders;
 
         for (auto& file : std::filesystem::directory_iterator(argv[1]))
         {
             auto extension = file.path().extension();
             if (extension == ".xpu" || extension == ".xvu")
-                filePaths.push_back(file.path().string());
+            {
+                ShaderHolder shader;
+                shader.filePath = file.path().string();
+                shader.fileData = readAllBytes(shader.filePath.c_str(), shader.fileSize);
+                shader.entry.hash = XXH3_64bits(shader.fileData.get(), shader.fileSize);
+
+                shaders.emplace(shader.entry.hash, std::move(shader));
+            }
         }
 
-        std::for_each(std::execution::par_unseq, filePaths.begin(), filePaths.end(), [&](auto& filePath)
+        std::for_each(std::execution::par_unseq, shaders.begin(), shaders.end(), [&](auto& hashShaderPair)
             {
-                printf("%s\n", filePath.c_str());
+                auto& shader = hashShaderPair.second;
+                printf("%s\n", shader.filePath.c_str());
 
                 thread_local ShaderRecompiler recompiler;
                 recompiler = {};
-                recompiler.recompile(readAllBytes(filePath.c_str()).get());
+                recompiler.recompile(shader.fileData.get());
-                writeAllBytes((filePath + ".hlsl").c_str(), recompiler.out.data(), recompiler.out.size());
+                writeAllBytes((shader.filePath + ".hlsl").c_str(), recompiler.out.data(), recompiler.out.size());
 
                 thread_local DxcCompiler dxcCompiler;
 
-                auto dxil = dxcCompiler.compile(recompiler.out, recompiler.isPixelShader, false);
+                shader.dxil = dxcCompiler.compile(recompiler.out, recompiler.isPixelShader, false);
-                auto spirv = dxcCompiler.compile(recompiler.out, recompiler.isPixelShader, true);
+                shader.spirv = dxcCompiler.compile(recompiler.out, recompiler.isPixelShader, true);
 
-                assert(dxil != nullptr && spirv != nullptr);
+                assert(shader.dxil != nullptr && shader.spirv != nullptr);
-                assert(*(reinterpret_cast<uint32_t*>(dxil->GetBufferPointer()) + 1) != 0 && "DXIL was not signed properly!");
+                assert(*(reinterpret_cast<uint32_t*>(shader.dxil->GetBufferPointer()) + 1) != 0 && "DXIL was not signed properly!");
-
-                std::string outFilePath = argv[2];
-                outFilePath += '/';
-                outFilePath += filePath.substr(filePath.find_last_of("\\/") + 1);
-
-                FILE* file = fopen(outFilePath.c_str(), "wb");
-
-                struct
-                {
-                    uint32_t version;
-                    uint32_t dxilSize;
-                    uint32_t spirvSize;
-                } header;
-
-                header.version = 0;
-                header.dxilSize = uint32_t(dxil->GetBufferSize());
-                header.spirvSize = uint32_t(spirv->GetBufferSize());
-
-                fwrite(&header, sizeof(header), 1, file);
-                fwrite(dxil->GetBufferPointer(), 1, dxil->GetBufferSize(), file);
-                fwrite(spirv->GetBufferPointer(), 1, spirv->GetBufferSize(), file);
-
-                fclose(file);
             });
+
+        FILE* file = fopen(argv[2], "wb");
+
+        ShaderCacheHeader header{};
+        header.version = 0;
+        header.count = shaders.size();
+        fwrite(&header, sizeof(header), 1, file);
+
+        uint32_t headerSize = sizeof(ShaderCacheHeader) + shaders.size() * sizeof(ShaderCacheEntry);
+        uint32_t dxilOffset = headerSize;
+        uint32_t spirvOffset = 0;
+
+        for (auto& [_, shader] : shaders)
+        {
+            shader.entry.dxilOffset = dxilOffset;
+            shader.entry.dxilSize = uint32_t(shader.dxil->GetBufferSize());
+            shader.entry.spirvOffset = spirvOffset;
+            shader.entry.spirvSize = uint32_t(shader.spirv->GetBufferSize());
+            dxilOffset += shader.entry.dxilSize;
+            spirvOffset += shader.entry.spirvSize;
+        }
+
+        for (auto& [_, shader] : shaders)
+        {
+            shader.entry.spirvOffset += dxilOffset;
+            fwrite(&shader.entry, sizeof(shader.entry), 1, file);
+        }
+
+        for (auto& [_, shader] : shaders)
+            fwrite(shader.dxil->GetBufferPointer(), 1, shader.dxil->GetBufferSize(), file);
+
+        for (auto& [_, shader] : shaders)
+            fwrite(shader.spirv->GetBufferPointer(), 1, shader.spirv->GetBufferSize(), file);
+
+        fclose(file);
     }
     else
     {

ShaderRecomp/pch.h

@@ -9,8 +9,10 @@
 #include <execution>
 #include <filesystem>
 #include <format>
+#include <map>
 #include <string>
 #include <unordered_map>
+#include <xxhash.h>
 
 template<typename T>
 struct be

vcpkg.json

@@ -1,6 +1,7 @@
 {
-  "builtin-baseline": "e63bd09dc0b7204467705c1c7c71d0e2a3f8860b",
+    "builtin-baseline": "e63bd09dc0b7204467705c1c7c71d0e2a3f8860b",
-  "dependencies": [
+    "dependencies": [
-    "directx-dxc"
+        "directx-dxc",
-  ]
+        "xxhash"
+    ]
 }