test(dualgpu): Replace synchronization primitives with shareable ones

This will increase latency a bunch, both due to not being on-device only, but also due to CPU overhead in building and submitting the command buffer. This is just a temporary test and not ready for production anyways

lsfg-vk-backend/include/lsfg-vk-backend/lsfgvk.hpp

@@ -79,17 +79,8 @@ namespace lsfgvk::backend {
         /// - false: VK_FORMAT_R8G8B8A8_UNORM
         /// - true: VK_FORMAT_R16G16B16A16_SFLOAT
         ///
-        /// The application and library must keep track of the frame index. When the next frame
-        /// is ready, signal the syncFd with one increment (with the first trigger being 1).
-        /// Each generated frame will increment the semaphore by one:
-        /// - Application signals 1 -> Start generating with (curr, next) source images
-        /// - Library signals 1 -> First frame between (curr, next) is ready
-        /// - Library signals N -> N-th frame between (curr, next) is ready
-        /// - Application signals N+1 -> Start generating with (next, curr) source images
-        ///
         /// @param sourceFds Pair of file descriptors for the source images alternated between.
         /// @param destFds Vector with file descriptors to import output images from.
-        /// @param syncFd File descriptor for the timeline semaphore used for synchronization.
         /// @param width Width of the images.
         /// @param height Height of the images.
         /// @param hdr Whether the images are HDR.
@@ -101,7 +92,6 @@ namespace lsfgvk::backend {
         Context& openContext(
             std::pair<int, int> sourceFds,
             const std::vector<int>& destFds,
-            int syncFd,
             uint32_t width, uint32_t height,
             bool hdr, float flow, bool perf
         );
@@ -110,9 +100,11 @@ namespace lsfgvk::backend {
         /// Schedule a new set of generated frames.
         ///
         /// @param context Context to use.
+        /// @param waitFd File descriptor to wait on before starting frame generation.
+        /// @param syncFds Vector of file descriptors to emplace, signalled when generated frames are ready.
         /// @throws backend::error on failure
         ///
-        void scheduleFrames(Context& context);
+        void scheduleFrames(Context& context, int waitFd, std::vector<int>& syncFds);
 
         ///
         /// Close a frame generation context

lsfg-vk-backend/src/lsfgvk.cpp

@@ -11,6 +11,7 @@
 #include "lsfg-vk-common/vulkan/command_buffer.hpp"
 #include "lsfg-vk-common/vulkan/fence.hpp"
 #include "lsfg-vk-common/vulkan/image.hpp"
+#include "lsfg-vk-common/vulkan/semaphore.hpp"
 #include "lsfg-vk-common/vulkan/timeline_semaphore.hpp"
 #include "lsfg-vk-common/vulkan/vulkan.hpp"
 #include "shaderchains/alpha0.hpp"
@@ -99,18 +100,18 @@ namespace lsfgvk::backend {
         /// create a context
         /// (see lsfg-vk documentation)
         ContextImpl(const InstanceImpl& instance,
-            std::pair<int, int> sourceFds, const std::vector<int>& destFds, int syncFd,
+            std::pair<int, int> sourceFds, const std::vector<int>& destFds,
             VkExtent2D extent, bool hdr, float flow, bool perf);
 
         /// schedule frames
         /// (see lsfg-vk documentation)
-        void scheduleFrames();
+        void scheduleFrames(int waitFd, std::vector<int>& syncFds);
     private:
         std::pair<vk::Image, vk::Image> sourceImages;
         std::vector<vk::Image> destImages;
         vk::Image blackImage;
 
-        vk::TimelineSemaphore syncSemaphore; // imported
+        ls::lazy<vk::Semaphore> syncSemaphore; // imported
         vk::TimelineSemaphore prepassSemaphore;
         size_t idx{1};
         size_t fidx{0}; // real frame index
@@ -126,6 +127,8 @@ namespace lsfgvk::backend {
         Beta0 beta0;
         Beta1 beta1;
         struct Pass {
+            ls::lazy<vk::Semaphore> sync2Semaphore; // imported
+
             std::vector<Gamma0> gamma0;
             std::vector<Gamma1> gamma1;
 
@@ -274,11 +277,11 @@ InstanceImpl::InstanceImpl(vk::PhysicalDeviceSelector selectPhysicalDevice,
 }
 
 Context& Instance::openContext(std::pair<int, int> sourceFds, const std::vector<int>& destFds,
-        int syncFd, uint32_t width, uint32_t height,
+        uint32_t width, uint32_t height,
         bool hdr, float flow, bool perf) {
     const VkExtent2D extent{ width, height };
     return *this->m_contexts.emplace_back(std::make_unique<ContextImpl>(*this->m_impl,
-        sourceFds, destFds, syncFd,
+        sourceFds, destFds,
         extent, hdr, flow, perf
     )).get();
 }
@@ -326,14 +329,6 @@ namespace {
             throw backend::error("Unable to create black image", e);
         }
     }
-    /// import timeline semaphore
-    vk::TimelineSemaphore importTimelineSemaphore(const vk::Vulkan& vk, int syncFd) {
-        try {
-            return{vk, 0, syncFd};
-        } catch (const std::exception& e) {
-            throw backend::error("Unable to import timeline semaphore", e);
-        }
-    }
     /// create prepass semaphores
     vk::TimelineSemaphore createPrepassSemaphore(const vk::Vulkan& vk) {
         try {
@@ -400,14 +395,13 @@ namespace {
 }
 
 ContextImpl::ContextImpl(const InstanceImpl& instance,
-            std::pair<int, int> sourceFds, const std::vector<int>& destFds, int syncFd,
+            std::pair<int, int> sourceFds, const std::vector<int>& destFds,
             VkExtent2D extent, bool hdr, float flow, bool perf) :
         sourceImages(importImages(instance.getVulkan(), sourceFds,
             extent, hdr ? VK_FORMAT_R16G16B16A16_SFLOAT : VK_FORMAT_R8G8B8A8_UNORM)),
         destImages(importImages(instance.getVulkan(), destFds,
             extent, hdr ? VK_FORMAT_R16G16B16A16_SFLOAT : VK_FORMAT_R8G8B8A8_UNORM)),
         blackImage(createBlackImage(instance.getVulkan())),
-        syncSemaphore(importTimelineSemaphore(instance.getVulkan(), syncFd)),
         prepassSemaphore(createPrepassSemaphore(instance.getVulkan())),
         cmdbufs(createCommandBuffers(instance.getVulkan(), destFds.size() + 1)),
         cmdbufFence(instance.getVulkan()),
@@ -549,7 +543,7 @@ ContextImpl::ContextImpl(const InstanceImpl& instance,
     cmdbuf.submit(ctx.vk); // wait for completion
 }
 
-void Instance::scheduleFrames(Context& context) {
+void Instance::scheduleFrames(Context& context, int waitFd, std::vector<int>& syncFds) {
 #ifdef LSFGVK_TESTING_RENDERDOC
     const auto& impl = this->m_impl;
     if (impl->getRenderDocAPI()) {
@@ -559,7 +553,7 @@ void Instance::scheduleFrames(Context& context) {
     }
 #endif
     try {
-        context.scheduleFrames();
+        context.scheduleFrames(waitFd, syncFds);
     } catch (const std::exception& e) {
         throw backend::error("Unable to schedule frames", e);
     }
@@ -573,12 +567,17 @@ void Instance::scheduleFrames(Context& context) {
 #endif
 }
 
-void Context::scheduleFrames() {
+void Context::scheduleFrames(int waitFd, std::vector<int>& syncFds) {
     // wait for previous pre-pass to complete
     if (this->fidx && !this->cmdbufFence.wait(this->ctx.vk))
         throw backend::error("Timeout waiting for previous frame to complete");
     this->cmdbufFence.reset(this->ctx.vk);
 
+    // import sync semaphore
+    this->syncSemaphore.emplace(ctx.vk, waitFd);
+    for (size_t i = 0; i < this->destImages.size(); ++i)
+        this->passes.at(i).sync2Semaphore.emplace(ctx.vk, std::nullopt, true);
+
     // schedule pre-pass
     const auto& cmdbuf = this->cmdbufs.at(0);
     cmdbuf.begin(ctx.vk);
@@ -593,13 +592,15 @@ void Context::scheduleFrames() {
 
     cmdbuf.end(ctx.vk);
     cmdbuf.submit(this->ctx.vk,
-        {}, this->syncSemaphore.handle(), this->idx,
+        { this->syncSemaphore->handle() }, VK_NULL_HANDLE, 0,
         {}, this->prepassSemaphore.handle(), this->idx
     );
 
     this->idx++;
 
     // schedule main passes
+    syncFds.clear();
+
     for (size_t i = 0; i < this->destImages.size(); i++) {
         const auto& cmdbuf = this->cmdbufs.at(i + 1);
         cmdbuf.begin(ctx.vk);
@@ -618,9 +619,11 @@ void Context::scheduleFrames() {
         cmdbuf.end(ctx.vk);
         cmdbuf.submit(this->ctx.vk,
             {}, this->prepassSemaphore.handle(), this->idx - 1,
-            {}, this->syncSemaphore.handle(), this->idx + i,
+            { pass.sync2Semaphore->handle() }, VK_NULL_HANDLE, 0,
             i == this->destImages.size() - 1 ? this->cmdbufFence.handle() : VK_NULL_HANDLE
         );
+
+        syncFds.push_back(pass.sync2Semaphore->exportToFd(ctx.vk));
     }
 
     this->idx += this->destImages.size();

lsfg-vk-common/include/lsfg-vk-common/helpers/pointers.hpp

@@ -25,8 +25,9 @@ namespace ls {
         /// @throws std::logic_error if value already present
         template<typename... Args>
         T& emplace(Args&&... args) {
-            if (this->opt.has_value())
+            // FIXME: should not be commented out
-                throw std::logic_error("lazy: value already present");
+            // if (this->opt.has_value())
+            //     throw std::logic_error("lazy: value already present");
 
             this->opt.emplace(std::forward<Args>(args)...);
             return *this->opt;

lsfg-vk-common/include/lsfg-vk-common/vulkan/semaphore.hpp

@@ -15,9 +15,18 @@ namespace vk {
     public:
         /// create a semaphore
         /// @param vk the vulkan instance
-        /// @param fd optional file descriptor to import the semaphore from
+        /// @param importFd optional file descriptor to import from
+        /// @param markExport whether to mark the semaphore as exportable
         /// @throws ls::vulkan_error on failure
-        Semaphore(const vk::Vulkan& vk, std::optional<int> fd = std::nullopt);
+        Semaphore(const vk::Vulkan& vk,
+            std::optional<int> importFd = std::nullopt,
+            bool markExport = false);
+
+        /// export the semaphore to a file descriptor
+        /// @param vk the vulkan instance
+        /// @return the exported file descriptor
+        /// @throws ls::vulkan_error on failure
+        [[nodiscard]] int exportToFd(const vk::Vulkan& vk) const;
 
         /// get the underlying VkSemaphore handle
         /// @return the VkSemaphore handle

lsfg-vk-common/src/vulkan/semaphore.cpp

@@ -13,28 +13,30 @@ using namespace vk;
 
 namespace {
     /// create a semaphore
-    ls::owned_ptr<VkSemaphore> createSemaphore(const vk::Vulkan& vk, std::optional<int> fd) {
+    ls::owned_ptr<VkSemaphore> createSemaphore(const vk::Vulkan& vk,
+            std::optional<int> importFd, bool markExport) {
         VkSemaphore handle{};
 
         const VkExportSemaphoreCreateInfo exportInfo{
             .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
-            .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
+            .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
         };
         const VkSemaphoreCreateInfo semaphoreInfo{
             .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
-            .pNext = fd.has_value() ? &exportInfo : nullptr
+            .pNext = (importFd.has_value() || markExport) ? &exportInfo : nullptr,
         };
         auto res = vk.df().CreateSemaphore(vk.dev(), &semaphoreInfo, VK_NULL_HANDLE, &handle);
         if (res != VK_SUCCESS)
             throw ls::vulkan_error(res, "vkCreateSemaphore() failed");
 
-        if (fd.has_value()) {
+        if (importFd.has_value()) {
             // import semaphore from fd
             const VkImportSemaphoreFdInfoKHR importInfo{
                 .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
                 .semaphore = handle,
-                .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
+                .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
-                .fd = *fd // closes the fd
+                .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
+                .fd = *importFd // closes the fd
             };
             res = vk.df().ImportSemaphoreFdKHR(vk.dev(), &importInfo);
             if (res != VK_SUCCESS)
@@ -50,5 +52,20 @@ namespace {
     }
 }
 
-Semaphore::Semaphore(const vk::Vulkan& vk, std::optional<int> fd)
+Semaphore::Semaphore(const vk::Vulkan& vk,
-    : semaphore(createSemaphore(vk, fd)) {}
+        std::optional<int> importFd, bool markExport)
+    : semaphore(createSemaphore(vk, importFd, markExport)) {}
+
+int Semaphore::exportToFd(const vk::Vulkan& vk) const {
+    int fd{};
+    const VkSemaphoreGetFdInfoKHR getFdInfo{
+        .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
+        .semaphore = *this->semaphore,
+        .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
+    };
+    auto res = vk.df().GetSemaphoreFdKHR(vk.dev(), &getFdInfo, &fd);
+    if (res != VK_SUCCESS)
+        throw ls::vulkan_error(res, "vkGetSemaphoreFdKHR() failed");
+
+    return fd;
+}

lsfg-vk-common/src/vulkan/timeline_semaphore.cpp

@@ -20,11 +20,11 @@ namespace {
 
         const VkExportSemaphoreCreateInfo exportInfo{
             .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
-            .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
+            .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
         };
         const VkSemaphoreTypeCreateInfo typeInfo{
             .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
-            .pNext = ( importFd.has_value() || exportFd.has_value() ) ? &exportInfo : nullptr,
+            .pNext = (importFd.has_value() || exportFd.has_value()) ? &exportInfo : nullptr,
             .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
             .initialValue = initial
         };
@@ -41,7 +41,7 @@ namespace {
             const VkImportSemaphoreFdInfoKHR importInfo{
                 .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
                 .semaphore = handle,
-                .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
+                .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
                 .fd = *importFd // closes the fd
             };
             res = vk.df().ImportSemaphoreFdKHR(vk.dev(), &importInfo);
@@ -54,7 +54,7 @@ namespace {
             const VkSemaphoreGetFdInfoKHR getFdInfo{
                 .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
                 .semaphore = handle,
-                .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
+                .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
             };
             int fd{};
             res = vk.df().GetSemaphoreFdKHR(vk.dev(), &getFdInfo, &fd);

lsfg-vk-layer/src/instance.cpp

@@ -120,31 +120,7 @@ void Root::modifyDeviceCreateInfo(VkDeviceCreateInfo& createInfo,
     );
     createInfo.enabledExtensionCount = static_cast<uint32_t>(extensions.size());
     createInfo.ppEnabledExtensionNames = extensions.data();
-
+    
-    bool isFeatureEnabled = false;
-    auto* featureInfo = reinterpret_cast<VkBaseInStructure*>(const_cast<void*>(createInfo.pNext));
-    while (featureInfo) {
-        if (featureInfo->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES) {
-            auto* features = reinterpret_cast<VkPhysicalDeviceVulkan12Features*>(featureInfo);
-            features->timelineSemaphore = VK_TRUE;
-            isFeatureEnabled = true;
-        } else if (featureInfo->sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES) {
-            auto* features = reinterpret_cast<VkPhysicalDeviceTimelineSemaphoreFeatures*>(featureInfo);
-            features->timelineSemaphore = VK_TRUE;
-            isFeatureEnabled = true;
-        }
-
-        featureInfo = const_cast<VkBaseInStructure*>(featureInfo->pNext);
-    }
-
-    VkPhysicalDeviceTimelineSemaphoreFeatures timelineFeatures{
-        .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES,
-        .pNext = const_cast<void*>(createInfo.pNext),
-        .timelineSemaphore = VK_TRUE
-    };
-    if (!isFeatureEnabled)
-        createInfo.pNext = &timelineFeatures;
-
     finish();
 }
 

lsfg-vk-layer/src/swapchain.cpp

@@ -90,13 +90,10 @@ Swapchain::Swapchain(const vk::Vulkan& vk, backend::Instance& backend,
             VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT,
             std::nullopt, &fd);
 
-    int syncFd{};
-    this->syncSemaphore.emplace(vk, 0, std::nullopt, &syncFd);
-
     try {
         this->ctx = ls::owned_ptr<ls::R<backend::Context>>(
             new ls::R<backend::Context>(backend.openContext(
-                { sourceFds.at(0), sourceFds.at(1) }, destinationFds, syncFd,
+                { sourceFds.at(0), sourceFds.at(1) }, destinationFds,
                 extent.width, extent.height,
                 hdr, 1.0F / this->profile.flow_scale, this->profile.performance_mode
             )),
@@ -135,13 +132,6 @@ VkResult Swapchain::present(const vk::Vulkan& vk,
     const auto& swapchainImage = this->info.images.at(imageIdx);
     const auto& sourceImage = this->sourceImages.at(this->fidx % 2);
 
-    // schedule frame generation
-    try {
-        this->instance.get().scheduleFrames(this->ctx.get());
-    } catch (const std::exception& e) {
-        throw ls::error("failed to schedule frames", e);
-    }
-
     // update present mode when not using pacing
     if (this->profile.pacing == ls::Pacing::None) {
 #pragma clang diagnostic push
@@ -196,12 +186,29 @@ VkResult Swapchain::present(const vk::Vulkan& vk,
         }
     );
 
+    this->syncSemaphore.emplace(vk, std::nullopt, true);
+
     cmdbuf.end(vk);
     cmdbuf.submit(vk,
         semaphores, VK_NULL_HANDLE, 0,
-        {}, this->syncSemaphore->handle(), this->idx++
+        { this->syncSemaphore->handle() }, VK_NULL_HANDLE, 0
     );
 
+    this->idx++;
+
+    // schedule frame generation
+    std::vector<int> waitFds;
+
+    try {
+        this->instance.get().scheduleFrames(
+            this->ctx.get(),
+            this->syncSemaphore->exportToFd(vk),
+            waitFds
+        );
+    } catch (const std::exception& e) {
+        throw ls::error("failed to schedule frames", e);
+    }
+
     for (size_t i = 0; i < this->destinationImages.size(); i++) {
         auto& pcs = this->postCopySemaphores.at(this->idx % this->postCopySemaphores.size());
         auto& destinationImage = this->destinationImages.at(i);
@@ -250,7 +257,12 @@ VkResult Swapchain::present(const vk::Vulkan& vk,
             }
         );
 
-        std::vector<VkSemaphore> waitSemaphores{ pass.acquireSemaphore.handle() };
+        pass.sync2Semaphore.emplace(vk, waitFds.at(i));
+
+        std::vector<VkSemaphore> waitSemaphores{
+            pass.acquireSemaphore.handle(),
+            pass.sync2Semaphore->handle()
+        };
         if (i) { // non-first pass
             const auto& prevPCS = this->postCopySemaphores.at((this->idx - 1) % this->postCopySemaphores.size());
             waitSemaphores.push_back(prevPCS.second.handle());
@@ -263,7 +275,7 @@ VkResult Swapchain::present(const vk::Vulkan& vk,
 
         cmdbuf.end(vk);
         cmdbuf.submit(vk,
-            waitSemaphores, this->syncSemaphore->handle(), this->idx,
+            waitSemaphores, VK_NULL_HANDLE, 0,
             signalSemaphores, VK_NULL_HANDLE, 0,
             i == this->destinationImages.size() - 1 ? this->renderFence->handle() : VK_NULL_HANDLE
         );

lsfg-vk-layer/src/swapchain.hpp

@@ -61,11 +61,12 @@ namespace lsfgvk::layer {
     private:
         std::vector<vk::Image> sourceImages;
         std::vector<vk::Image> destinationImages;
-        ls::lazy<vk::TimelineSemaphore> syncSemaphore;
+        ls::lazy<vk::Semaphore> syncSemaphore;
 
         ls::lazy<vk::CommandBuffer> renderCommandBuffer;
         ls::lazy<vk::Fence> renderFence;
         struct RenderPass {
+            ls::lazy<vk::Semaphore> sync2Semaphore;
             vk::CommandBuffer commandBuffer;
             vk::Semaphore acquireSemaphore;
         };