refactor(cleanup): introduce a pipeline cache

lsfg-vk-backend/src/lsfgvk.cpp

@@ -26,6 +26,7 @@
 #include <array>
 #include <cstddef>
 #include <cstdint>
+#include <cstdlib>
 #include <exception>
 #include <filesystem>
 #include <functional>
@@ -175,13 +176,27 @@ Instance::Instance(
 }
 
 namespace {
+    /// find the cache file path
+    std::filesystem::path findCacheFilePath() {
+        const char* xdgCacheHome = std::getenv("XDG_CACHE_HOME");
+        if (xdgCacheHome && *xdgCacheHome != '\0')
+            return std::filesystem::path(xdgCacheHome) / "lsfg-vk_pipeline_cache.bin";
+
+        const char* home = std::getenv("HOME");
+        if (home && *home != '\0')
+            return std::filesystem::path(home) / ".cache" / "lsfg-vk_pipeline_cache.bin";
+
+        return{"/tmp/lsfg-vk_pipeline_cache.bin"};
+    }
     /// create a Vulkan instance
     vk::Vulkan createVulkanInstance(vk::PhysicalDeviceSelector selectPhysicalDevice) {
         try {
             return{
                 "lsfg-vk", vk::version{1, 1, 0},
                 "lsfg-vk-engine", vk::version{1, 1, 0},
-                selectPhysicalDevice
+                selectPhysicalDevice,
+                false, std::nullopt,
+                findCacheFilePath()
             };
         } catch (const std::exception& e) {
             throw lsfgvk::error("Unable to initialize Vulkan", e);
@@ -239,6 +254,7 @@ InstanceImpl::InstanceImpl(vk::PhysicalDeviceSelector selectPhysicalDevice,
 #ifdef LSFGVK__RENDERDOC_INTEGRATION
     this->renderdoc = loadRenderDocIntegration();
 #endif
+    vk.persistPipelineCache(); // will silently fail
 }
 
 Context& Instance::openContext(std::pair<int, int> sourceFds, const std::vector<int>& destFds,

lsfg-vk-common/include/lsfg-vk-common/vulkan/vulkan.hpp

@@ -4,6 +4,7 @@
 
 #include <bitset>
 #include <cstdint>
+#include <filesystem>
 #include <functional>
 #include <optional>
 #include <string>
@@ -96,6 +97,9 @@ namespace vk {
         PFN_vkDestroyDescriptorSetLayout DestroyDescriptorSetLayout;
         PFN_vkCreatePipelineLayout CreatePipelineLayout;
         PFN_vkDestroyPipelineLayout DestroyPipelineLayout;
+        PFN_vkCreatePipelineCache CreatePipelineCache;
+        PFN_vkDestroyPipelineCache DestroyPipelineCache;
+        PFN_vkGetPipelineCacheData GetPipelineCacheData;
         PFN_vkCreateComputePipelines CreateComputePipelines;
         PFN_vkDestroyPipeline DestroyPipeline;
 
@@ -147,12 +151,15 @@ namespace vk {
         /// @param engineVersion version of the engine
         /// @param selectPhysicalDevice function to select the physical device
         /// @param isGraphical whether the device is graphical (rather than compute)
+        /// @param setLoaderData optional function to set loader data
+        /// @param cachefile optional path to pipeline cache file
         /// @throws ls::vulkan_error on failure
         Vulkan(const std::string& appName, version appVersion,
             const std::string& engineName, version engineVersion,
             PhysicalDeviceSelector selectPhysicalDevice,
             bool isGraphical = false,
-            std::optional<PFN_vkSetDeviceLoaderData> setLoaderData = std::nullopt);
+            std::optional<PFN_vkSetDeviceLoaderData> setLoaderData = std::nullopt,
+            const std::optional<std::filesystem::path>& cachefile = std::nullopt);
 
         /// create based on an existing externally managed vulkan instance.
         /// @param instance vulkan instance handle
@@ -161,13 +168,16 @@ namespace vk {
         /// @param instanceFuncs instance function pointers
         /// @param deviceFuncs device function pointers
         /// @param isGraphical whether the device is graphical (rather than compute)
+        /// @param setLoaderData optional function to set loader data
+        /// @param cachefile optional path to pipeline cache file
         /// @throws ls::vulkan_error on failure
         Vulkan(VkInstance instance, VkDevice device,
             VkPhysicalDevice physdev,
             VulkanInstanceFuncs instanceFuncs,
             VulkanDeviceFuncs deviceFuncs,
             bool isGraphical = true,
-            std::optional<PFN_vkSetDeviceLoaderData> setLoaderData = std::nullopt);
+            std::optional<PFN_vkSetDeviceLoaderData> setLoaderData = std::nullopt,
+            const std::optional<std::filesystem::path>& cachefile = std::nullopt);
 
         /// find a memory type index
         /// @param validTypes bitset of valid memory types
@@ -176,6 +186,9 @@ namespace vk {
         [[nodiscard]] std::optional<uint32_t> findMemoryTypeIndex(
             std::bitset<32> validTypes, bool hostVisibility) const;
 
+        /// persist the pipeline cache to file, silently failing on error
+        void persistPipelineCache() const noexcept;
+
         /// get the vulkan instance
         /// @return the instance handle
         [[nodiscard]] const auto& inst() const { return this->instance.get(); }
@@ -188,6 +201,9 @@ namespace vk {
         /// get the command pool
         /// @return the command pool handle
         [[nodiscard]] const auto& cmdpool() const { return this->cmdPool.get(); }
+        /// get the pipeline cache
+        /// @return the pipeline cache handle
+        [[nodiscard]] const auto& cache() const { return this->pipelineCache.get(); }
         /// get the compute queue
         /// @return the compute queue handle
         [[nodiscard]] const auto& queue() const { return this->computeQueue; }
@@ -220,5 +236,7 @@ namespace vk {
         VkQueue computeQueue;
 
         ls::owned_ptr<VkCommandPool> cmdPool;
+        ls::owned_ptr<VkPipelineCache> pipelineCache;
+        std::optional<std::filesystem::path> cachefile;
     };
 }

lsfg-vk-common/src/vulkan/shader.cpp

@@ -132,12 +132,10 @@ namespace {
             .layout = pipelineLayout
         };
         auto res = vk.df().CreateComputePipelines(vk.dev(),
-            VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &handle);
+            vk.cache(), 1, &pipelineInfo, nullptr, &handle);
         if (res != VK_SUCCESS)
             throw ls::vulkan_error(res, "vkCreateComputePipelines() failed");
 
-        // TODO: ponder pipeline cache
-
         return ls::owned_ptr<VkPipeline>(
             new VkPipeline(handle),
             [dev = vk.dev(), defunc = vk.df().DestroyPipeline](VkPipeline& pipeline) {

lsfg-vk-common/src/vulkan/vulkan.cpp

@@ -4,7 +4,11 @@
 
 #include <array>
 #include <bitset>
+#include <cstddef>
 #include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <ios>
 #include <optional>
 #include <string>
 #include <vector>
@@ -235,6 +239,47 @@ namespace {
             }
         );
     }
+
+    /// try to read the pipeline cache from file
+    void readCacheFile(const std::filesystem::path& cachefile, std::vector<uint8_t>& data) {
+        std::ifstream file(cachefile, std::ios::binary | std::ios::ate);
+        if (!file.is_open())
+            return;
+
+        const std::streamsize size = file.tellg();
+        data = std::vector<uint8_t>(static_cast<size_t>(size));
+
+        file.seekg(0, std::ios::beg);
+        if (!file.read(reinterpret_cast<char*>(data.data()), size))
+            return;
+    }
+
+    /// create a pipeline cache
+    ls::owned_ptr<VkPipelineCache> createPipelineCache(
+            const VulkanDeviceFuncs& fd, VkDevice device,
+            const std::optional<std::filesystem::path>& cachefile) {
+        VkPipelineCache handle{};
+
+        std::vector<uint8_t> cache{};
+        if (cachefile && std::filesystem::exists(*cachefile))
+            readCacheFile(*cachefile, cache);
+
+        const VkPipelineCacheCreateInfo pipelineCacheInfo{
+            .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
+            .initialDataSize = cache.size(),
+            .pInitialData = cache.data()
+        };
+        auto res = fd.CreatePipelineCache(device, &pipelineCacheInfo, nullptr, &handle);
+        if (res != VK_SUCCESS)
+            throw ls::vulkan_error(res, "vkCreatePipelineCache() failed");
+
+        return ls::owned_ptr<VkPipelineCache>(
+            new VkPipelineCache(handle),
+            [dev = device, defunc = fd.DestroyPipelineCache](VkPipelineCache& cache) {
+                defunc(dev, cache, nullptr);
+            }
+        );
+    }
 }
 
 /// initialize vulkan instance function pointers
@@ -323,8 +368,10 @@ VulkanDeviceFuncs vk::initVulkanDeviceFuncs(const VulkanInstanceFuncs& f, VkDevi
             "vkDestroyDescriptorSetLayout"),
         .CreatePipelineLayout = dpa<PFN_vkCreatePipelineLayout>(f, d, "vkCreatePipelineLayout"),
         .DestroyPipelineLayout = dpa<PFN_vkDestroyPipelineLayout>(f, d, "vkDestroyPipelineLayout"),
-        .CreateComputePipelines = dpa<PFN_vkCreateComputePipelines>(f, d,
-            "vkCreateComputePipelines"),
+        .CreatePipelineCache = dpa<PFN_vkCreatePipelineCache>(f, d, "vkCreatePipelineCache"),
+        .DestroyPipelineCache = dpa<PFN_vkDestroyPipelineCache>(f, d, "vkDestroyPipelineCache"),
+        .GetPipelineCacheData = dpa<PFN_vkGetPipelineCacheData>(f, d, "vkGetPipelineCacheData"),
+        .CreateComputePipelines = dpa<PFN_vkCreateComputePipelines>(f, d, "vkCreateComputePipelines"),
         .DestroyPipeline = dpa<PFN_vkDestroyPipeline>(f, d, "vkDestroyPipeline"),
 
         .SignalSemaphoreKHR = dpa<PFN_vkSignalSemaphoreKHR>(f, d, "vkSignalSemaphoreKHR"),
@@ -350,7 +397,8 @@ Vulkan::Vulkan(const std::string& appName, version appVersion,
         const std::string& engineName, version engineVersion,
         PhysicalDeviceSelector selectPhysicalDevice,
         bool isGraphical,
-        std::optional<PFN_vkSetDeviceLoaderData> setLoaderData) :
+        std::optional<PFN_vkSetDeviceLoaderData> setLoaderData,
+        const std::optional<std::filesystem::path>& cachefile) :
     instance(createInstance(
         appName, appVersion,
         engineName, engineVersion
@@ -379,7 +427,11 @@ Vulkan::Vulkan(const std::string& appName, version appVersion,
     cmdPool(createCommandPool(this->device_funcs,
         *this->device,
         this->queueFamilyIdx
-    )) {
+    )),
+    pipelineCache(createPipelineCache(this->device_funcs,
+        *this->device, cachefile
+    )),
+    cachefile(cachefile) {
 }
 
 Vulkan::Vulkan(VkInstance instance, VkDevice device,
@@ -387,7 +439,8 @@ Vulkan::Vulkan(VkInstance instance, VkDevice device,
         VulkanInstanceFuncs instanceFuncs,
         VulkanDeviceFuncs deviceFuncs,
         bool isGraphical,
-        std::optional<PFN_vkSetDeviceLoaderData> setLoaderData) :
+        std::optional<PFN_vkSetDeviceLoaderData> setLoaderData,
+        const std::optional<std::filesystem::path>& cachefile) :
     instance(new VkInstance(instance)),
     instance_funcs(instanceFuncs),
     phys_dev(physdev),
@@ -403,7 +456,11 @@ Vulkan::Vulkan(VkInstance instance, VkDevice device,
     cmdPool(createCommandPool(this->device_funcs,
         *this->device,
         this->queueFamilyIdx
-    )) {
+    )),
+    pipelineCache(createPipelineCache(this->device_funcs,
+        *this->device, cachefile
+    )),
+    cachefile(cachefile) {
 }
 
 std::optional<uint32_t> Vulkan::findMemoryTypeIndex(
@@ -422,3 +479,31 @@ std::optional<uint32_t> Vulkan::findMemoryTypeIndex(
 
     return std::nullopt;
 }
+
+void Vulkan::persistPipelineCache() const noexcept {
+    if (!this->cachefile)
+        return;
+
+    size_t cacheSize{};
+    auto res = this->device_funcs.GetPipelineCacheData(*this->device,
+        *this->pipelineCache,
+        &cacheSize, nullptr);
+    if (res != VK_SUCCESS)
+        return;
+
+    std::vector<uint8_t> cacheData(cacheSize);
+    res = this->device_funcs.GetPipelineCacheData(*this->device,
+        *this->pipelineCache,
+        &cacheSize, cacheData.data());
+    if (res != VK_SUCCESS)
+        return;
+
+    std::ofstream file(*this->cachefile, std::ios::binary | std::ios::trunc);
+    if (!file.is_open())
+        return;
+
+    file.write(reinterpret_cast<const char*>(cacheData.data()),
+        static_cast<std::streamsize>(cacheData.size()));
+    if (!file.good())
+        return;
+}

lsfg-vk-layer/src/context/instance.hpp

@@ -27,13 +27,6 @@ namespace lsfgvk::layer {
         /// ensure the layer is up-to-date
         void update();
 
-        // /// required instance extensions
-        // /// @return list of extension names
-        // [[nodiscard]] std::vector<const char*> instanceExtensions() const;
-        // /// required device extensions
-        // /// @return list of extension names
-        // [[nodiscard]] std::vector<const char*> deviceExtensions() const;
-
         /// modify instance create info
         /// @param createInfo original create info
         /// @param finish function to call after modification