Merge branch 'eid/new-shader-lookup' into 'master'

Decouple RHI shader loading from lump system

See merge request kart-krew-dev/ring-racers-internal!2507

assets/CMakeLists.txt

@@ -39,7 +39,6 @@ set(SRB2_ASSETS_GAME
 	"scripts.pk3"
 	"staffghosts.pk3"
 	"unlocks.pk3"
-	"shaders.pk3"
 )
 list(TRANSFORM SRB2_ASSETS_GAME PREPEND "/")
 list(TRANSFORM SRB2_ASSETS_GAME PREPEND "${SRB2_ASSET_DIRECTORY_ABSOLUTE}")

src/d_main.cpp

@@ -1408,7 +1408,6 @@ static void IdentifyVersion(void)
 	D_AddFile(startupiwads, va(spandf,srb2waddir,"data","maps.pk3"));
 	D_AddFile(startupiwads, va(spandf,srb2waddir,"data","unlocks.pk3"));
 	D_AddFile(startupiwads, va(spandf,srb2waddir,"data","staffghosts.pk3"));
-	D_AddFile(startupiwads, va(spandf,srb2waddir,"data","shaders.pk3"));
 #ifdef USE_PATCH_FILE
 	D_AddFile(startupiwads, va(pandf,srb2waddir,"patch.pk3"));
 #endif
@@ -1733,7 +1732,6 @@ void D_SRB2Main(void)
 	mainwads++; W_VerifyFileMD5(mainwads, ASSET_HASH_MAPS_PK3);						// maps.pk3
 	mainwads++; W_VerifyFileMD5(mainwads, ASSET_HASH_UNLOCKS_PK3);					// unlocks.pk3
 	mainwads++; W_VerifyFileMD5(mainwads, ASSET_HASH_STAFFGHOSTS_PK3);				// staffghosts.pk3
-	mainwads++; W_VerifyFileMD5(mainwads, ASSET_HASH_SHADERS_PK3);					// shaders.pk3
 #ifdef USE_PATCH_FILE
 	mainwads++; W_VerifyFileMD5(mainwads, ASSET_HASH_PATCH_PK3);					// patch.pk3
 #endif
@@ -1748,7 +1746,6 @@ void D_SRB2Main(void)
 	mainwads++;	// maps.pk3
 	mainwads++; // unlocks.pk3
 	mainwads++; // staffghosts.pk3
-	mainwads++; // shaders.pk3
 #ifdef USE_PATCH_FILE
 	mainwads++; // patch.pk3
 #endif
@@ -1773,6 +1770,8 @@ void D_SRB2Main(void)
 
 	M_PasswordInit();
 
+	W_InitShaderLookup("shaders.pk3");
+
 	//---------------------------------------------------- READY SCREEN
 	// we need to check for dedicated before initialization of some subsystems
 

src/sdl/i_video.cpp

@@ -366,10 +366,10 @@ static boolean IgnoreMouse(void)
 
 static void SDLdoGrabMouse(void)
 {
-	SDL_ShowCursor(SDL_DISABLE);
-	SDL_SetWindowGrab(window, SDL_TRUE);
-	if (SDL_SetRelativeMouseMode(SDL_TRUE) == 0) // already warps mouse if successful
-		wrapmouseok = SDL_TRUE; // TODO: is wrapmouseok or HalfWarpMouse needed anymore?
+	// SDL_ShowCursor(SDL_DISABLE);
+	// SDL_SetWindowGrab(window, SDL_TRUE);
+	// if (SDL_SetRelativeMouseMode(SDL_TRUE) == 0) // already warps mouse if successful
+	// 	wrapmouseok = SDL_TRUE; // TODO: is wrapmouseok or HalfWarpMouse needed anymore?
 }
 
 static void SDLdoUngrabMouse(void)

src/sdl/rhi_gl2_platform.cpp

@@ -36,40 +36,25 @@ void SdlGl2Platform::present()
 
 static std::array<std::string, 2> glsllist_lump_names(const char* name)
 {
-	std::string vertex_list_name = fmt::format("rhi_glsllist_{}_vertex", name);
-	std::string fragment_list_name = fmt::format("rhi_glsllist_{}_fragment", name);
+	std::string vertex_list_name = fmt::format("rhi_glsllist_{}_vertex.txt", name);
+	std::string fragment_list_name = fmt::format("rhi_glsllist_{}_fragment.txt", name);
 
 	return {std::move(vertex_list_name), std::move(fragment_list_name)};
 }
 
 static std::vector<std::string> get_sources_from_glsllist_lump(const char* lumpname)
 {
-	std::string shaderspk3 = "shaders.pk3";
-	INT32 shaderwadnum = -1;
-	for (INT32 wadnum = 0; wadnum <= mainwads; wadnum++)
-	{
-		std::string wadname = std::string(wadfiles[wadnum]->filename);
-		if (wadname.find(shaderspk3) != std::string::npos)
-		{
-			shaderwadnum = wadnum;
-			break;
-		}
-	}
-
-	if (shaderwadnum < 0)
-	{
-		throw std::runtime_error("Unable to identify the shaders.pk3 wadnum");
-	}
-
-	UINT16 glsllist_lump_num = W_CheckNumForLongNamePwad(lumpname, shaderwadnum, 0);
-	if (glsllist_lump_num == INT16_MAX)
+	size_t buffer_size;
+	if (!W_ReadShader(lumpname, &buffer_size, nullptr))
 	{
 		throw std::runtime_error(fmt::format("Unable to find glsllist lump {}", lumpname));
 	}
-
 	std::string glsllist_lump_data;
-	glsllist_lump_data.resize(W_LumpLengthPwad(shaderwadnum, glsllist_lump_num));
-	W_ReadLumpPwad(shaderwadnum, glsllist_lump_num, glsllist_lump_data.data());
+	glsllist_lump_data.resize(buffer_size);
+	if (!W_ReadShader(lumpname, &buffer_size, glsllist_lump_data.data()))
+	{
+		throw std::runtime_error(fmt::format("Unable to read glsllist lump {}", lumpname));
+	}
 
 	std::istringstream glsllist(glsllist_lump_data);
 	std::vector<std::string> sources;
@@ -90,15 +75,24 @@ static std::vector<std::string> get_sources_from_glsllist_lump(const char* lumpn
 			line.pop_back();
 		}
 
-		UINT16 source_lump_num = W_CheckNumForLongNamePwad(line.c_str(), shaderwadnum, 0);
-		if (source_lump_num == INT16_MAX)
+		// Compat: entries not ending in .glsl should append, for new shader file lookup system
+		size_t glsl_pos = line.find(".glsl");
+		if (line.size() < 5 || glsl_pos == line.npos || glsl_pos != line.size() - 5)
 		{
-			throw std::runtime_error(fmt::format("Unable to find glsl source lump lump {}", lumpname));
+			line.append(".glsl");
 		}
 
+		size_t source_lump_size;
+		if (!W_ReadShader(line.c_str(), &source_lump_size, nullptr))
+		{
+			throw std::runtime_error(fmt::format("Unable to find glsl source lump lump {}", line));
+		}
 		std::string source_lump;
-		source_lump.resize(W_LumpLengthPwad(shaderwadnum, source_lump_num));
-		W_ReadLumpPwad(shaderwadnum, source_lump_num, source_lump.data());
+		source_lump.resize(source_lump_size);
+		if (!W_ReadShader(line.c_str(), &source_lump_size, source_lump.data()))
+		{
+			throw std::runtime_error(fmt::format("Unable to read glsl source lump lump {}", line));
+		}
 
 		sources.emplace_back(source_lump);
 	}

src/w_wad.cpp

@@ -113,6 +113,10 @@ static UINT16 lumpnumcacheindex = 0;
 UINT16 numwadfiles = 0; // number of active wadfiles
 wadfile_t *wadfiles[MAX_WADFILES]; // 0 to numwadfiles-1 are valid
 
+static FILE *g_shaderspk3file;
+static UINT16 g_shaderspk3numlumps;
+static lumpinfo_t *g_shaderspk3lumps;
+
 // W_Shutdown
 // Closes all of the WAD files before quitting
 // If not done on a Mac then open wad files
@@ -138,6 +142,24 @@ void W_Shutdown(void)
 		Z_Free(wad->lumpinfo);
 		Z_Free(wad);
 	}
+
+	// Cleanup the separate shader lookup
+	if (g_shaderspk3file)
+	{
+		while (g_shaderspk3numlumps--)
+		{
+			lumpinfo_t *lump = &g_shaderspk3lumps[g_shaderspk3numlumps];
+			Z_Free(lump->longname);
+			if (lump->fullname != lump->longname)
+			{
+				Z_Free(lump->fullname);
+			}
+		}
+		Z_Free(g_shaderspk3lumps);
+		g_shaderspk3lumps = NULL;
+		fclose(g_shaderspk3file);
+		g_shaderspk3file = NULL;
+	}
 }
 
 //===========================================================================
@@ -2423,6 +2445,197 @@ int W_VerifyNMUSlumps(const char *filename, boolean exit_on_error)
 	return status;
 }
 
+void W_InitShaderLookup(const char *filename)
+{
+	I_Assert(g_shaderspk3file == NULL);
+
+	FILE* handle;
+	char filename_buf[2048];
+
+	g_shaderspk3file = NULL;
+	g_shaderspk3lumps = NULL;
+	g_shaderspk3numlumps = 0;
+
+	strncpy(filename_buf, filename, 2048);
+	filename_buf[2048 - 1] = '\0';
+
+	if ((handle = fopen(filename_buf, "rb")) == NULL)
+	{
+		nameonly(filename_buf);
+
+		if (findfile(filename_buf, NULL, true))
+		{
+			if ((handle = fopen(filename_buf, "rb")) == NULL)
+			{
+				return;
+			}
+		}
+		else
+		{
+			return;
+		}
+	}
+	else
+	{
+		return;
+	}
+
+	// It is acceptable to fail opening the pk3 lookup.
+	// The shader pk3 lookup is only needed to build a lookup directory of the zip
+	// for later. We always check for the flat file shader anyway.
+
+	UINT16 numlumps;
+	lumpinfo_t *shader_lumps = ResGetLumpsZip(handle, &numlumps);
+	if (shader_lumps == NULL)
+	{
+		return;
+	}
+	g_shaderspk3file = handle;
+	g_shaderspk3lumps = shader_lumps;
+	g_shaderspk3numlumps = numlumps;
+}
+
+static boolean ReadShaderFlatFile(const char *filename, size_t *size, void *dest)
+{
+	FILE* flat_handle = NULL;
+	char filename_buf[2048];
+	char filename_only_buf[512];
+
+	strncpy(filename_buf, filename, 2048);
+	filename_buf[2048 - 1] = '\0';
+
+	if ((flat_handle = fopen(filename_buf, "rb")) == NULL)
+	{
+		nameonly(filename_buf);
+		strncpy(filename_only_buf, filename_buf, 512);
+		filename_only_buf[512 - 1] = '\0';
+		sprintf(filename_buf, "shaders/%s", filename_only_buf);
+		if (findfile(filename_buf, NULL, true))
+		{
+			if ((flat_handle = fopen(filename_buf, "rb")) == NULL)
+			{
+				return false;
+			}
+		}
+		else
+		{
+			return false;
+		}
+	}
+
+	// idk, pray it's not >2gb. ansi c made mistakes
+	fseek(flat_handle, 0, SEEK_END);
+	*size = ftell(flat_handle);
+	fseek(flat_handle, 0, SEEK_SET);
+	if (dest)
+	{
+		fread(dest, *size, 1, flat_handle);
+	}
+
+	fclose(flat_handle);
+	return true;
+}
+
+boolean W_ReadShader(const char *filename, size_t *size, void *dest)
+{
+	I_Assert(filename != NULL);
+	I_Assert(size != NULL);
+
+	if (ReadShaderFlatFile(filename, size, dest))
+	{
+		return true;
+	}
+
+	UINT32 hash = quickncasehash(filename, 512);
+
+	lumpinfo_t* lump = NULL;
+	for (int i = 0 ; i < g_shaderspk3numlumps; ++i)
+	{
+		lump = &g_shaderspk3lumps[i];
+		UINT32 lumpnamehash = quickncasehash(lump->fullname, 512);
+		if (lumpnamehash == hash)
+		{
+			break;
+		}
+		lump = NULL;
+	}
+
+	if (lump == NULL)
+	{
+		return false;
+	}
+
+	size_t sizelocal = lump->size;
+	if (dest == NULL)
+	{
+		*size = sizelocal;
+		return true;
+	}
+
+	if (fseek(g_shaderspk3file, lump->position, SEEK_SET) != 0)
+		I_Error("Failed to seek shaders pk3 to offset of file: %s", strerror(errno));
+
+	switch (lump->compression)
+	{
+	case CM_NOCOMPRESSION:
+		if (fread(dest, sizelocal, 1, g_shaderspk3file) != 0)
+			I_Error("Failed to read file in shaders pk3: %s", strerror(errno));
+		break;
+#ifdef HAVE_ZLIB
+	case CM_DEFLATE:
+	{
+		UINT8 *rawData; // The lump's raw data.
+		UINT8 *decData; // Lump's decompressed real data.
+
+		int zErr; // Helper var.
+		z_stream strm;
+		unsigned long rawSize = lump->disksize;
+		unsigned long decSize = (unsigned long)size;
+
+		rawData = static_cast<UINT8*>(Z_Malloc(rawSize, PU_STATIC, NULL));
+		decData = static_cast<UINT8*>(dest);
+
+		if (fread(rawData, 1, rawSize, g_shaderspk3file) < rawSize)
+			I_Error("Failed to read compressed file in shaders pk3: %s", strerror(errno));
+
+		strm.zalloc = Z_NULL;
+		strm.zfree = Z_NULL;
+		strm.opaque = Z_NULL;
+
+		strm.total_in = strm.avail_in = rawSize;
+		strm.total_out = strm.avail_out = decSize;
+
+		strm.next_in = rawData;
+		strm.next_out = decData;
+
+		zErr = inflateInit2(&strm, -15);
+		if (zErr == Z_OK)
+		{
+			zErr = inflate(&strm, Z_SYNC_FLUSH);
+			if (zErr != Z_OK && zErr != Z_STREAM_END)
+			{
+				zerr(zErr);
+			}
+			(void)inflateEnd(&strm);
+		}
+		else
+		{
+			size = 0;
+			zerr(zErr);
+		}
+
+		Z_Free(rawData);
+	}
+		break;
+#endif
+	default:
+		return false;
+	}
+
+	*size = sizelocal;
+	return true;
+}
+
 /** \brief Generates a virtual resource used for level data loading.
  *
  * \param lumpnum_t reference

src/w_wad.h

@@ -221,6 +221,10 @@ void W_VerifyFileMD5(UINT16 wadfilenum, const char *matchmd5);
 
 int W_VerifyNMUSlumps(const char *filename, boolean exit_on_error);
 
+/// Initialize non-legacy GL shader lookup, which lives outside the lump management system.
+void W_InitShaderLookup(const char *filename);
+boolean W_ReadShader(const char *filename, size_t *size, void *dest);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif