refactor(cleanup): refactor dll & shader extraction logic

This commit is contained in:
PancakeTAS 2025-11-17 20:50:22 +01:00
parent c9caf38cbb
commit 0660faa094
No known key found for this signature in database
13 changed files with 249 additions and 346 deletions

View file

@ -31,3 +31,4 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug")
endif()
add_subdirectory(lsfg-vk-common)
add_subdirectory(lsfg-vk-backend)

View file

@ -1,70 +0,0 @@
#pragma once
#include "core/device.hpp"
#include "core/pipeline.hpp"
#include "core/shadermodule.hpp"
#include <vulkan/vulkan_core.h>
#include <cstdint>
#include <cstddef>
#include <functional>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace LSFG::Pool {
///
/// Shader pool for each Vulkan device.
///
class ShaderPool {
public:
ShaderPool() noexcept = default;
///
/// Create the shader pool.
///
/// @param source Function to retrieve shader source code by name.
/// @param fp16 If true, use the FP16 variant of shaders.
///
/// @throws std::runtime_error if the shader pool cannot be created.
///
ShaderPool(
const std::function<std::vector<uint8_t>(const std::string&, bool)>& source,
bool fp16)
: source(source), fp16(fp16) {}
///
/// Retrieve a shader module by name or create it.
///
/// @param name Name of the shader module
/// @param types Descriptor types for the shader module
/// @return Shader module
///
/// @throws LSFG::vulkan_error if the shader module cannot be created.
///
Core::ShaderModule getShader(
const Core::Device& device, const std::string& name,
const std::vector<std::pair<size_t, VkDescriptorType>>& types);
///
/// Retrieve a pipeline shader module by name or create it.
///
/// @param name Name of the shader module
/// @return Pipeline shader module or empty
///
/// @throws LSFG::vulkan_error if the shader module cannot be created.
///
Core::Pipeline getPipeline(
const Core::Device& device, const std::string& name);
private:
std::function<std::vector<uint8_t>(const std::string&, bool)> source;
bool fp16{false};
std::unordered_map<std::string, Core::ShaderModule> shaders;
std::unordered_map<std::string, Core::Pipeline> pipelines;
};
}

View file

@ -1,48 +0,0 @@
#include "pool/shaderpool.hpp"
#include "core/shadermodule.hpp"
#include "core/device.hpp"
#include "core/pipeline.hpp"
#include <vulkan/vulkan_core.h>
#include <cstddef>
#include <stdexcept>
#include <string>
#include <vector>
#include <utility>
using namespace LSFG;
using namespace LSFG::Pool;
Core::ShaderModule ShaderPool::getShader(
const Core::Device& device, const std::string& name,
const std::vector<std::pair<size_t, VkDescriptorType>>& types) {
auto it = shaders.find(name);
if (it != shaders.end())
return it->second;
// grab the shader
auto bytecode = this->source(name, this->fp16);
if (bytecode.empty())
throw std::runtime_error("Shader code is empty: " + name);
// create the shader module
Core::ShaderModule shader(device, bytecode, types);
shaders[name] = shader;
return shader;
}
Core::Pipeline ShaderPool::getPipeline(
const Core::Device& device, const std::string& name) {
auto it = pipelines.find(name);
if (it != pipelines.end())
return it->second;
// grab the shader module
auto shader = this->getShader(device, name, {});
// create the pipeline
Core::Pipeline pipeline(device, shader);
pipelines[name] = pipeline;
return pipeline;
}

View file

@ -1,22 +0,0 @@
#pragma once
#include <unordered_map>
#include <cstdint>
#include <string>
#include <vector>
namespace DLL {
///
/// Parse all resources from a DLL file.
///
/// *Shouldn't* cause any segmentation faults.
///
/// @param filename Path to the DLL file.
/// @return A map of resource IDs to their binary data.
///
/// @throws std::runtime_error on various failure points.
///
std::unordered_map<uint32_t, std::vector<uint8_t>> parse_dll(const std::string& filename);
}

View file

@ -1,27 +0,0 @@
#pragma once
#include <cstdint>
#include <string>
#include <vector>
namespace Extract {
///
/// Extract all known shaders.
///
/// @throws std::runtime_error if shader extraction fails.
///
void extractShaders();
///
/// Get a shader by name.
///
/// @param name The name of the shader to get.
/// @param fp16 If true, use the FP16 variant of shaders.
/// @return The shader bytecode.
///
/// @throws std::runtime_error if the shader is not found.
///
std::vector<uint8_t> getShader(const std::string& name, bool fp16);
}

View file

@ -0,0 +1,36 @@
Checks:
# enable basic checks
- "clang-analyzer-*"
# configure performance checks
- "performance-*"
- "-performance-enum-size"
# configure readability and bugprone checks
- "readability-*"
- "bugprone-*"
- "misc-*"
- "-readability-braces-around-statements"
- "-readability-function-cognitive-complexity"
- "-readability-identifier-length"
- "-readability-implicit-bool-conversion"
- "-readability-magic-numbers"
- "-readability-math-missing-parentheses"
- "-readability-named-parameter"
- "-bugprone-easily-swappable-parameters"
# configure modernization
- "modernize-*"
- "-modernize-use-trailing-return-type"
# configure cppcoreguidelines
- "cppcoreguidelines-*"
- "-cppcoreguidelines-avoid-magic-numbers"
- "-cppcoreguidelines-pro-type-reinterpret-cast"
- "-cppcoreguidelines-macro-usage"
# disable slow and pointless checks
- "-modernize-use-std-numbers"
- "-modernize-type-traits"
- "-cppcoreguidelines-owning-memory"
- "-cppcoreguidelines-macro-to-enum"
- "-readability-container-contains"
- "-bugprone-reserved-identifier"
- "-bugprone-stringview-nullptr"
- "-bugprone-standalone-empty"
- "-misc-unused-using-decls"

View file

@ -0,0 +1,14 @@
set(BACKEND_SOURCES
"src/extraction/dll_reader.cpp"
"src/extraction/shader_registry.cpp")
add_library(lsfg-vk-backend STATIC ${BACKEND_SOURCES})
target_include_directories(lsfg-vk-backend
PUBLIC include)
target_link_libraries(lsfg-vk-backend
PUBLIC lsfg-vk-common)
set_target_properties(lsfg-vk-backend PROPERTIES
CXX_VISIBILITY_PRESET hidden)

View file

@ -1,7 +1,8 @@
#include "extract/dll.hpp"
#include "dll_reader.hpp"
#include <unordered_map>
#include <stdexcept>
#include <unordered_map>
#include <filesystem>
#include <algorithm>
#include <iostream>
#include <optional>
@ -9,11 +10,12 @@
#include <cstdint>
#include <fstream>
#include <utility>
#include <string>
#include <vector>
#include <array>
#include <span>
using namespace extr;
/// DOS file header
struct DOSHeader {
uint16_t magic; // 0x5A4D
@ -76,7 +78,7 @@ namespace {
const T* safe_cast(const std::vector<uint8_t>& data, size_t offset) {
const size_t end = offset + sizeof(T);
if (end > data.size() || end < offset)
throw std::runtime_error("Buffer overflow during safe cast");
throw std::runtime_error("buffer overflow/underflow during safe cast");
return reinterpret_cast<const T*>(&data.at(offset));
}
@ -85,41 +87,42 @@ namespace {
std::span<const T> span_cast(const std::vector<uint8_t>& data, size_t offset, size_t count) {
const size_t end = offset + (count * sizeof(T));
if (end > data.size() || end < offset)
throw std::runtime_error("Buffer overflow during safe cast");
throw std::runtime_error("buffer overflow/underflow during safe cast");
return std::span<const T>(reinterpret_cast<const T*>(&data.at(offset)), count);
}
}
#pragma clang diagnostic pop
std::unordered_map<uint32_t, std::vector<uint8_t>> DLL::parse_dll(const std::string& filename) {
std::ifstream file(filename, std::ios::binary | std::ios::ate);
std::unordered_map<uint32_t, std::vector<uint8_t>> extr::extractResourcesFromDLL(
const std::filesystem::path& dll) {
std::ifstream file(dll, std::ios::binary | std::ios::ate);
if (!file.is_open())
throw std::runtime_error("Failed to open Lossless.dll");
throw std::runtime_error("failed to open dll file");
const auto size = file.tellg();
file.seekg(0, std::ios::beg);
std::vector<uint8_t> data(static_cast<size_t>(size));
if (!file.read(reinterpret_cast<char*>(data.data()), size))
throw std::runtime_error("Failed to read Lossless.dll");
throw std::runtime_error("failed to read dll file");
// parse dos header
size_t fileOffset = 0;
const auto* dosHdr = safe_cast<const DOSHeader>(data, 0);
if (dosHdr->magic != 0x5A4D)
throw std::runtime_error("Invalid DOS header magic number");
throw std::runtime_error("dos header magic number is incorrect");
// parse pe header
fileOffset += static_cast<size_t>(dosHdr->pe_offset);
const auto* peHdr = safe_cast<const PEHeader>(data, fileOffset);
if (peHdr->signature != 0x00004550)
throw std::runtime_error("Invalid PE header signature");
throw std::runtime_error("pe header signature is incorrect");
// parse optional pe header
fileOffset += sizeof(PEHeader);
const auto* peOptHdr = safe_cast<const PEOptionalHeader>(data, fileOffset);
if (peOptHdr->magic != 0x20B)
throw std::runtime_error("Unsupported PE format (not PE32+)");
throw std::runtime_error("pe format is not PE32+");
const auto& [rsrc_rva, rsrc_size] = peOptHdr->resource_table;
// locate section containing resources
@ -134,13 +137,13 @@ std::unordered_map<uint32_t, std::vector<uint8_t>> DLL::parse_dll(const std::str
break;
}
if (!rsrc_offset)
throw std::runtime_error("Failed to locate resource section");
throw std::runtime_error("unable to locate resource section");
// parse resource directory
fileOffset = rsrc_offset.value();
const auto* rsrcDir = safe_cast<const ResourceDirectory>(data, fileOffset);
if (rsrcDir->id_count < 3)
throw std::runtime_error("Incorrect resource directory");
throw std::runtime_error("resource directory does not have enough entries");
// find resource table with data type
std::optional<size_t> rsrc_tbl_offset;
@ -151,18 +154,18 @@ std::unordered_map<uint32_t, std::vector<uint8_t>> DLL::parse_dll(const std::str
if (rsrcDirEntry.id != 10) // RT_RCDATA
continue;
if ((rsrcDirEntry.offset & 0x80000000) == 0)
throw std::runtime_error("Expected resource directory, but found data entry");
throw std::runtime_error("expected resource directory, found data entry");
rsrc_tbl_offset.emplace(rsrcDirEntry.offset & 0x7FFFFFFF);
}
if (!rsrc_tbl_offset)
throw std::runtime_error("Failed to locate RT_RCDATA directory");
throw std::runtime_error("unabele to locate RT_RCDATA directory");
// parse data type resource directory
fileOffset = rsrc_offset.value() + rsrc_tbl_offset.value();
const auto* rsrcTbl = safe_cast<const ResourceDirectory>(data, fileOffset);
if (rsrcTbl->id_count < 1)
throw std::runtime_error("Incorrect RT_RCDATA directory");
throw std::runtime_error("RT_RCDATA directory does not have enough entries");
// collect all resources
fileOffset += sizeof(ResourceDirectory);
@ -171,7 +174,7 @@ std::unordered_map<uint32_t, std::vector<uint8_t>> DLL::parse_dll(const std::str
std::unordered_map<uint32_t, std::vector<uint8_t>> resources;
for (const auto& rsrcTblEntry : rsrcTblEntries) {
if ((rsrcTblEntry.offset & 0x80000000) == 0)
throw std::runtime_error("Expected resource directory, but found data entry");
throw std::runtime_error("expected resource directory, found data entry");
// skip over language directory
fileOffset = rsrc_offset.value() + (rsrcTblEntry.offset & 0x7FFFFFFF);
@ -182,19 +185,19 @@ std::unordered_map<uint32_t, std::vector<uint8_t>> DLL::parse_dll(const std::str
fileOffset += sizeof(ResourceDirectory);
const auto* langDirEntry = safe_cast<const ResourceDirectoryEntry>(data, fileOffset);
if ((langDirEntry->offset & 0x80000000) != 0)
throw std::runtime_error("Expected resource data entry, but found directory");
throw std::runtime_error("expected resource data entry, but found directory");
// parse resource data entry
fileOffset = rsrc_offset.value() + (langDirEntry->offset & 0x7FFFFFFF);
const auto* entry = safe_cast<const ResourceDataEntry>(data, fileOffset);
if (entry->offset < rsrc_rva || entry->offset > (rsrc_rva + rsrc_size))
throw std::runtime_error("Resource data entry points outside resource section");
throw std::runtime_error("resource data entry points outside resource section");
// extract resource
std::vector<uint8_t> resource(entry->size);
fileOffset = (entry->offset - rsrc_rva) + rsrc_offset.value();
if (fileOffset + entry->size > data.size())
throw std::runtime_error("Resource data entry points outside file");
throw std::runtime_error("resource data entry points outside file");
std::copy_n(&data.at(fileOffset), entry->size, resource.data());
resources.emplace(rsrcTblEntry.id, std::move(resource));
}

View file

@ -0,0 +1,17 @@
#pragma once
#include <cstdint>
#include <filesystem>
#include <unordered_map>
#include <vector>
namespace extr {
/// extract all resources from a DLL file
/// @param dll path to the DLL file
/// @return map of resource IDs to their binary data
/// @throws std::runtime_error on various failure points
std::unordered_map<uint32_t, std::vector<uint8_t>> extractResourcesFromDLL(
const std::filesystem::path& dll);
}

View file

@ -0,0 +1,117 @@
#include "shader_registry.hpp"
#include "lsfg-vk-common/vulkan/shader.hpp"
#include "lsfg-vk-common/vulkan/vulkan.hpp"
#include <cstddef>
#include <cstdint>
#include <stdexcept>
#include <string>
#include <unordered_map>
#include <vector>
using namespace extr;
namespace {
/// get the source code for a shader
const std::vector<uint8_t>& getShaderSource(uint32_t id, bool fp16, bool perf,
const std::unordered_map<uint32_t, std::vector<uint8_t>>& resources) {
const size_t BASE_OFFSET = 49;
const size_t OFFSET_PERF = 23;
const size_t OFFSET_FP16 = 49;
auto it = resources.find(BASE_OFFSET + id +
(perf ? OFFSET_PERF : 0) +
(fp16 ? OFFSET_FP16 : 0));
if (it == resources.end())
throw std::runtime_error("unable to find shader with id: " + std::to_string(id));
return it->second;
}
}
ShaderRegistry extr::buildShaderRegistry(const vk::Vulkan& vk, bool fp16,
const std::unordered_map<uint32_t, std::vector<uint8_t>>& resources) {
#define SHADER(id, p1, p2, p3, p4) \
vk::Shader(vk, getShaderSource(id, fp16, PERF, resources), \
p1, p2, p3, p4)
return {
#define PERF false
.mipmaps = SHADER(255, 1, 7, 1, 1),
.generate = SHADER(256, 5, 1, 1, 2),
.quality = {
.alpha = {
SHADER(267, 1, 2, 0, 1),
SHADER(268, 2, 2, 0, 1),
SHADER(269, 2, 4, 0, 1),
SHADER(270, 4, 4, 0, 1)
},
.beta = {
SHADER(275, 12, 2, 0, 1),
SHADER(276, 2, 2, 0, 1),
SHADER(277, 2, 2, 0, 1),
SHADER(278, 2, 2, 0, 1),
SHADER(279, 2, 6, 1, 1)
},
.gamma = {
SHADER(257, 9, 3, 1, 2),
SHADER(259, 3, 4, 0, 1),
SHADER(260, 4, 4, 0, 1),
SHADER(261, 4, 4, 0, 1),
SHADER(262, 6, 1, 1, 2)
},
.delta = {
SHADER(257, 9, 3, 1, 2),
SHADER(263, 3, 4, 0, 1),
SHADER(264, 4, 4, 0, 1),
SHADER(265, 4, 4, 0, 1),
SHADER(266, 6, 1, 1, 2),
SHADER(258, 10, 2, 1, 2),
SHADER(271, 2, 2, 0, 1),
SHADER(272, 2, 2, 0, 1),
SHADER(273, 2, 2, 0, 1),
SHADER(274, 3, 1, 1, 2)
}
},
#undef PERF
#define PERF true
.performance = {
.alpha = {
SHADER(267, 1, 1, 0, 1),
SHADER(268, 1, 1, 0, 1),
SHADER(269, 1, 2, 0, 1),
SHADER(270, 2, 2, 0, 1)
},
.beta = {
SHADER(275, 6, 2, 0, 1),
SHADER(276, 2, 2, 0, 1),
SHADER(277, 2, 2, 0, 1),
SHADER(278, 2, 2, 0, 1),
SHADER(279, 2, 6, 1, 1)
},
.gamma = {
SHADER(257, 5, 3, 1, 2),
SHADER(259, 3, 2, 0, 1),
SHADER(260, 2, 2, 0, 1),
SHADER(261, 2, 2, 0, 1),
SHADER(262, 4, 1, 1, 2)
},
.delta = {
SHADER(257, 5, 3, 1, 2),
SHADER(263, 3, 2, 0, 1),
SHADER(264, 2, 2, 0, 1),
SHADER(265, 2, 2, 0, 1),
SHADER(266, 4, 1, 1, 2),
SHADER(258, 6, 1, 1, 2),
SHADER(271, 1, 1, 0, 1),
SHADER(272, 1, 1, 0, 1),
SHADER(273, 1, 1, 0, 1),
SHADER(274, 2, 1, 1, 2)
}
},
#undef PERF
.is_fp16 = fp16
};
#undef SHADER
}

View file

@ -0,0 +1,40 @@
#pragma once
#include "lsfg-vk-common/vulkan/shader.hpp"
#include <array>
#include <cstdint>
#include <unordered_map>
#include <vector>
namespace extr {
/// shader collection struct
struct Shaders {
std::array<vk::Shader, 4> alpha;
std::array<vk::Shader, 5> beta;
std::array<vk::Shader, 5> gamma;
std::array<vk::Shader, 10> delta;
};
/// shader registry struct
struct ShaderRegistry {
vk::Shader mipmaps;
vk::Shader generate;
Shaders quality;
Shaders performance;
bool is_fp16; //!< whether the fp16 shader variants were loaded
};
/// build a shader registry from resources
/// @param vk Vulkan instance
/// @param fp16 whether to load fp16 variants
/// @param resources map of resource IDs to their binary data
/// @return constructed shader registry
/// @throws std::runtime_error if shaders are missing
/// @throws vk::vulkan_error on Vulkan errors
ShaderRegistry buildShaderRegistry(const vk::Vulkan& vk, bool fp16,
const std::unordered_map<uint32_t, std::vector<uint8_t>>& resources);
}

View file

@ -61,16 +61,4 @@ namespace ls {
T* ptr{};
std::function<void(T&)> deleter{};
};
/// turn a vector of images into a vector of references
template<typename T>
std::vector<ls::R<const T>> refs(const std::vector<T>& images) {
std::vector<ls::R<const T>> result;
result.reserve(images.size());
for (const auto& img : images)
result.push_back(std::ref(img));
return result;
}
}

View file

@ -1,146 +0,0 @@
#include "extract/extract.hpp"
#include "config/config.hpp"
#include "extract/dll.hpp"
#include <iostream>
#include <unordered_map>
#include <filesystem>
#include <stdexcept>
#include <cstdint>
#include <cstdlib>
#include <string>
#include <vector>
#include <array>
using namespace Extract;
const uint32_t NO = 49; // native offset
const uint32_t PO = NO + 23; // performance+native offset
const uint32_t FP = 49; // fp32 offset
const std::unordered_map<std::string, uint32_t> nameIdxTable = {{
{ "mipmaps", 255 + NO },
{ "alpha[0]", 267 + NO },
{ "alpha[1]", 268 + NO },
{ "alpha[2]", 269 + NO },
{ "alpha[3]", 270 + NO },
{ "beta[0]", 275 + NO },
{ "beta[1]", 276 + NO },
{ "beta[2]", 277 + NO },
{ "beta[3]", 278 + NO },
{ "beta[4]", 279 + NO },
{ "gamma[0]", 257 + NO },
{ "gamma[1]", 259 + NO },
{ "gamma[2]", 260 + NO },
{ "gamma[3]", 261 + NO },
{ "gamma[4]", 262 + NO },
{ "delta[0]", 257 + NO },
{ "delta[1]", 263 + NO },
{ "delta[2]", 264 + NO },
{ "delta[3]", 265 + NO },
{ "delta[4]", 266 + NO },
{ "delta[5]", 258 + NO },
{ "delta[6]", 271 + NO },
{ "delta[7]", 272 + NO },
{ "delta[8]", 273 + NO },
{ "delta[9]", 274 + NO },
{ "generate", 256 + NO },
{ "p_alpha[0]", 267 + PO },
{ "p_alpha[1]", 268 + PO },
{ "p_alpha[2]", 269 + PO },
{ "p_alpha[3]", 270 + PO },
{ "p_beta[0]", 275 + PO },
{ "p_beta[1]", 276 + PO },
{ "p_beta[2]", 277 + PO },
{ "p_beta[3]", 278 + PO },
{ "p_beta[4]", 279 + PO },
{ "p_gamma[0]", 257 + PO },
{ "p_gamma[1]", 259 + PO },
{ "p_gamma[2]", 260 + PO },
{ "p_gamma[3]", 261 + PO },
{ "p_gamma[4]", 262 + PO },
{ "p_delta[0]", 257 + PO },
{ "p_delta[1]", 263 + PO },
{ "p_delta[2]", 264 + PO },
{ "p_delta[3]", 265 + PO },
{ "p_delta[4]", 266 + PO },
{ "p_delta[5]", 258 + PO },
{ "p_delta[6]", 271 + PO },
{ "p_delta[7]", 272 + PO },
{ "p_delta[8]", 273 + PO },
{ "p_delta[9]", 274 + PO },
}};
namespace {
auto& shaders() {
static std::unordered_map<uint32_t, std::array<std::vector<uint8_t>, 2>> shaderData;
return shaderData;
}
const std::vector<std::filesystem::path> PATHS{{
".local/share/Steam/steamapps/common",
".steam/steam/steamapps/common",
".steam/debian-installation/steamapps/common",
".var/app/com.valvesoftware.Steam/.local/share/Steam/steamapps/common",
"snap/steam/common/.local/share/Steam/steamapps/common"
}};
std::string getDllPath() {
// overriden path
std::string dllPath = Config::globalConf.dll;
if (!dllPath.empty())
return dllPath;
// home based paths
const char* home = getenv("HOME");
const std::string homeStr = home ? home : "";
for (const auto& base : PATHS) {
const std::filesystem::path path =
std::filesystem::path(homeStr) / base / "Lossless Scaling" / "Lossless.dll";
if (std::filesystem::exists(path))
return path.string();
}
// xdg home
const char* dataDir = getenv("XDG_DATA_HOME");
if (dataDir && *dataDir != '\0')
return std::string(dataDir) + "/Steam/steamapps/common/Lossless Scaling/Lossless.dll";
// final fallback
return "Lossless.dll";
}
}
void Extract::extractShaders() {
if (!shaders().empty())
return;
// parse the dll
const auto resources = DLL::parse_dll(getDllPath());
std::cerr << "lsfg-vk: Extracted " << resources.size() << " resources from dll.\n";
// ensure all shaders are present
for (const auto& [name, idx] : nameIdxTable) {
auto fp16 = resources.find(idx);
if (fp16 == resources.end())
throw std::runtime_error("Shader not found: " + name + " (FP16).\n- Is Lossless Scaling up to date?");
auto fp32 = resources.find(idx + FP);
if (fp32 == resources.end())
throw std::runtime_error("Shader not found: " + name + " (FP32).\n- Is Lossless Scaling up to date?");
shaders().emplace(idx, std::array<std::vector<uint8_t>, 2>{
fp32->second,
fp16->second
});
}
}
std::vector<uint8_t> Extract::getShader(const std::string& name, bool fp16) {
if (shaders().empty())
throw std::runtime_error("Shaders are not loaded.");
auto hit = nameIdxTable.find(name);
if (hit == nameIdxTable.end())
throw std::runtime_error("Shader hash not found: " + name);
auto sit = shaders().find(hit->second);
if (sit == shaders().end())
throw std::runtime_error("Shader not found: " + name);
return fp16 ? sit->second.at(1) : sit->second.at(0);
}