Merge branch 'faster-startup' into 'master'

Startup: calculate file checksum in background thread

See merge request KartKrew/Kart!1442

src/CMakeLists.txt

@@ -92,7 +92,7 @@ add_executable(SRB2SDL2 MACOSX_BUNDLE WIN32
 	v_video.cpp
 	s_sound.c
 	sounds.c
-	w_wad.c
+	w_wad.cpp
 	filesrch.c
 	mserv.c
 	http-mserv.c

src/d_netcmd.c

@@ -4308,7 +4308,7 @@ static void Command_Addfile(void)
 
 			for (i = 0; i < numwadfiles; i++)
 			{
-				if (!memcmp(wadfiles[i]->md5sum, md5sum, 16))
+				if (!memcmp(W_GetFileMD5(wadfiles[i]), md5sum, 16))
 				{
 					CONS_Alert(CONS_ERROR, M_GetText("%s is already loaded\n"), fn);
 					valid = false;

src/d_netfil.c

@@ -208,7 +208,7 @@ UINT8 *PutFileNeeded(UINT16 firstfile)
 		count++;
 		WRITEUINT32(p, wadfiles[i]->filesize);
 		WRITESTRINGN(p, wadfilename, MAX_WADPATH);
-		WRITEMEM(p, wadfiles[i]->md5sum, 16);
+		WRITEMEM(p, W_GetFileMD5(wadfiles[i]), 16);
 	}
 	if (netbuffer->packettype == PT_MOREFILESNEEDED)
 		netbuffer->u.filesneededcfg.num = count;
@@ -576,7 +576,7 @@ INT32 CL_CheckFiles(void)
 				return 2;
 
 			// For the sake of speed, only bother with a md5 check
-			if (memcmp(wadfiles[j]->md5sum, fileneeded[i].md5sum, 16))
+			if (memcmp(W_GetFileMD5(wadfiles[j]), fileneeded[i].md5sum, 16))
 				return 2;
 
 			// It's accounted for! let's keep going.
@@ -611,7 +611,7 @@ INT32 CL_CheckFiles(void)
 		{
 			nameonly(strcpy(wadfilename, wadfiles[j]->filename));
 			if (!stricmp(wadfilename, fileneeded[i].filename) &&
-				!memcmp(wadfiles[j]->md5sum, fileneeded[i].md5sum, 16))
+				!memcmp(W_GetFileMD5(wadfiles[j]), fileneeded[i].md5sum, 16))
 			{
 				CONS_Debug(DBG_NETPLAY, "already loaded\n");
 				fileneeded[i].status = FS_OPEN;

src/filesrch.c

@@ -837,7 +837,7 @@ boolean preparefilemenu(boolean samedepth, boolean replayhut)
 
 							if (strcmp(dent->d_name, filenamebuf[i]))
 								continue;
-							if (cv_addons_md5.value && !checkfilemd5(menupath, wadfiles[i]->md5sum))
+							if (cv_addons_md5.value && !checkfilemd5(menupath, W_GetFileMD5(wadfiles[i])))
 								continue;
 
 							ext |= EXT_LOADED;

src/g_demo.c

@@ -2053,7 +2053,7 @@ static void G_SaveDemoExtraFiles(UINT8 **pp)
 	{
 		nameonly(( filename = va("%s", wadfiles[i]->filename) ));
 		WRITESTRINGL((*pp), filename, MAX_WADPATH);
-		WRITEMEM((*pp), wadfiles[i]->md5sum, 16);
+		WRITEMEM((*pp), W_GetFileMD5(wadfiles[i]), 16);
 
 		totalfiles++;
 	}
@@ -2089,7 +2089,7 @@ static void G_LoadDemoExtraFiles(UINT8 **pp)
 
 			for (j = 0; j < numwadfiles; ++j)
 			{
-				if (memcmp(md5sum, wadfiles[j]->md5sum, 16) == 0)
+				if (memcmp(md5sum, W_GetFileMD5(wadfiles[j]), 16) == 0)
 				{
 					alreadyloaded = true;
 					break;
@@ -2192,7 +2192,7 @@ static UINT8 G_CheckDemoExtraFiles(savebuffer_t *info, boolean quick)
 				else
 					continue;
 
-				if (memcmp(md5sum, wadfiles[j]->md5sum, 16) == 0)
+				if (memcmp(md5sum, W_GetFileMD5(wadfiles[j]), 16) == 0)
 				{
 					alreadyloaded = true;
 

src/r_patch.cpp

@@ -40,7 +40,7 @@ patch_t *Patch_Create(softwarepatch_t *source, size_t srcsize, void *dest)
 		patch->topoffset  = SHORT(source->topoffset);
 		patch->columnofs  = static_cast<INT32*>(Z_Calloc(size, PU_PATCH_DATA, NULL));
 
-		for (col = 0; col < source->width; col++)
+		for (col = 0; col < patch->width; col++)
 		{
 			// This makes the column offsets relative to the column data itself,
 			// instead of the entire patch data

src/r_picformats.h

@@ -46,6 +46,7 @@ typedef enum
 
 typedef enum
 {
+	PICFLAGS_NONE = 0,
 	PICFLAGS_XFLIP = 1,
 	PICFLAGS_YFLIP = 1<<1
 } pictureflags_t;

src/typedef.h

@@ -443,6 +443,7 @@ TYPEDEF (lumpinfo_t);
 TYPEDEF (virtlump_t);
 TYPEDEF (virtres_t);
 TYPEDEF (wadfile_t);
+TYPEDEF (wadfile_private_t);
 
 #undef TYPEDEF
 #undef TYPEDEF2

src/w_wad.cpp

@@ -40,6 +40,13 @@
 #include "lzf.h"
 #endif
 
+#include <algorithm>
+#include <cstddef>
+#include <mutex>
+
+#include "cxxutil.hpp"
+#include "wad_private.hpp"
+
 #include "doomdef.h"
 #include "doomstat.h"
 #include "doomtype.h"
@@ -82,6 +89,14 @@
 #define O_BINARY 0
 #endif
 
+using namespace srb2::wad;
+
+namespace srb2::wad
+{
+
+std::mutex g_wadfiles_mutex;
+
+}; // namespace srb2::wad
 
 typedef struct
 {
@@ -120,6 +135,8 @@ void W_Shutdown(void)
 	{
 		wadfile_t *wad = wadfiles[numwadfiles];
 
+		delete wad->internal_state;
+
 		fclose(wad->handle);
 		Z_Free(wad->filename);
 		while (wad->numlumps--)
@@ -227,7 +244,7 @@ static inline void W_LoadDehackedLumpsPK3(UINT16 wadnum, boolean mainfile)
 		{
 			lumpinfo_t *lump_p = &wadfiles[wadnum]->lumpinfo[posStart];
 			size_t length = strlen(wadfiles[wadnum]->filename) + 1 + strlen(lump_p->fullname); // length of file name, '|', and lump name
-			char *name = malloc(length + 1);
+			char *name = static_cast<char*>(malloc(length + 1));
 			sprintf(name, "%s|%s", wadfiles[wadnum]->filename, lump_p->fullname);
 			name[length] = '\0';
 			CONS_Printf(M_GetText("Loading SOC from %s\n"), name);
@@ -256,7 +273,7 @@ static inline void W_LoadDehackedLumps(UINT16 wadnum, boolean mainfile)
 			if (memcmp(lump_p->name,"SOC_",4)==0) // Check for generic SOC lump
 			{	// shameless copy+paste of code from LUA_LoadLump
 				size_t length = strlen(wadfiles[wadnum]->filename) + 1 + strlen(lump_p->fullname); // length of file name, '|', and lump name
-				char *name = malloc(length + 1);
+				char *name = static_cast<char*>(malloc(length + 1));
 				sprintf(name, "%s|%s", wadfiles[wadnum]->filename, lump_p->fullname);
 				name[length] = '\0';
 
@@ -286,7 +303,7 @@ static inline void W_LoadDehackedLumps(UINT16 wadnum, boolean mainfile)
   * \param resblock resulting MD5 checksum
   * \return 0 if MD5 checksum was made, and is at resblock, 1 if error was found
   */
-static inline INT32 W_MakeFileMD5(const char *filename, void *resblock)
+INT32 W_MakeFileMD5(const char *filename, void *resblock)
 {
 #ifdef NOMD5
 	(void)filename;
@@ -337,7 +354,7 @@ static restype_t ResourceFileDetect (const char* filename)
  */
 static lumpinfo_t* ResGetLumpsStandalone (FILE* handle, UINT16* numlumps, const char* lumpname)
 {
-	lumpinfo_t* lumpinfo = Z_Calloc(sizeof (*lumpinfo), PU_STATIC, NULL);
+	lumpinfo_t* lumpinfo = static_cast<lumpinfo_t*>(Z_Calloc(sizeof (*lumpinfo), PU_STATIC, NULL));
 	lumpinfo->position = 0;
 	fseek(handle, 0, SEEK_END);
 	lumpinfo->size = ftell(handle);
@@ -346,12 +363,12 @@ static lumpinfo_t* ResGetLumpsStandalone (FILE* handle, UINT16* numlumps, const
 	lumpinfo->hash = quickncasehash(lumpname, 8);
 
 	// Allocate the lump's long name.
-	lumpinfo->longname = Z_Malloc(9 * sizeof(char), PU_STATIC, NULL);
+	lumpinfo->longname = static_cast<char*>(Z_Malloc(9 * sizeof(char), PU_STATIC, NULL));
 	strcpy(lumpinfo->longname, lumpname);
 	lumpinfo->longname[8] = '\0';
 
 	// Allocate the lump's full name.
-	lumpinfo->fullname = Z_Malloc(9 * sizeof(char), PU_STATIC, NULL);
+	lumpinfo->fullname = static_cast<char*>(Z_Malloc(9 * sizeof(char), PU_STATIC, NULL));
 	strcpy(lumpinfo->fullname, lumpname);
 	lumpinfo->fullname[8] = '\0';
 
@@ -395,7 +412,7 @@ static lumpinfo_t* ResGetLumpsWad (FILE* handle, UINT16* nlmp, const char* filen
 
 	// read wad file directory
 	i = header.numlumps * sizeof (*fileinfo);
-	fileinfov = fileinfo = malloc(i);
+	fileinfov = fileinfo = static_cast<filelump_t*>(malloc(i));
 	if (fseek(handle, header.infotableofs, SEEK_SET) == -1
 		|| fread(fileinfo, 1, i, handle) < i)
 	{
@@ -407,7 +424,7 @@ static lumpinfo_t* ResGetLumpsWad (FILE* handle, UINT16* nlmp, const char* filen
 	numlumps = header.numlumps;
 
 	// fill in lumpinfo for this wad
-	lump_p = lumpinfo = Z_Malloc(numlumps * sizeof (*lumpinfo), PU_STATIC, NULL);
+	lump_p = lumpinfo = static_cast<lumpinfo_t*>(Z_Malloc(numlumps * sizeof (*lumpinfo), PU_STATIC, NULL));
 	for (i = 0; i < numlumps; i++, lump_p++, fileinfo++)
 	{
 		lump_p->position = LONG(fileinfo->filepos);
@@ -488,7 +505,7 @@ static lumpinfo_t* ResGetLumpsWad (FILE* handle, UINT16* nlmp, const char* filen
 				namelen = strlen(trimname);
 
 			// Allocate the lump's long and full name (save on memory).
-			lump_p->longname = lump_p->fullname = Z_Calloc(namelen * sizeof(char), PU_STATIC, NULL);
+			lump_p->longname = lump_p->fullname = static_cast<char*>(Z_Calloc(namelen * sizeof(char), PU_STATIC, NULL));
 			strncpy(lump_p->longname, trimname, namelen);
 			lump_p->longname[namelen-1] = '\0';
 
@@ -505,7 +522,7 @@ static lumpinfo_t* ResGetLumpsWad (FILE* handle, UINT16* nlmp, const char* filen
 			lump_p->hash = quickncasehash(lump_p->name, 8);
 
 			// Allocate the lump's long and full name (save on memory).
-			lump_p->longname = lump_p->fullname = Z_Malloc(9 * sizeof(char), PU_STATIC, NULL);
+			lump_p->longname = lump_p->fullname = static_cast<char*>(Z_Malloc(9 * sizeof(char), PU_STATIC, NULL));
 			strncpy(lump_p->longname, fileinfo->name, 8);
 			lump_p->longname[8] = '\0';
 		}
@@ -519,24 +536,49 @@ static lumpinfo_t* ResGetLumpsWad (FILE* handle, UINT16* nlmp, const char* filen
  */
 static boolean ResFindSignature (FILE* handle, char endPat[], UINT32 startpos)
 {
+	//the Wii U has rather slow filesystem access, and fgetc is *unbearable*
+	//so I reimplemented this function to buffer 128k chunks
 	char *s;
 	int c;
 
+	fseek(handle, 0, SEEK_END);
+	size_t len = ftell(handle);
+
 	fseek(handle, startpos, SEEK_SET);
+	size_t remaining = len - startpos;
+	size_t chunkpos = startpos;
+
 	s = endPat;
-	while((c = fgetc(handle)) != EOF)
+
-	{
+	//128k buffers
-		if (*s != c && s > endPat) // No match?
+	size_t buffer_size = std::min(128 * 1024 * sizeof(char), remaining);
-			s = endPat; // We "reset" the counter by sending the s pointer back to the start of the array.
+	char* buffer = static_cast<char*>(malloc(buffer_size));
-		if (*s == c)
+
-		{
+	size_t bytes_read = 0;
-			s++;
+	while ((bytes_read = fread(buffer, 1, buffer_size, handle)) > 0) {
-			if (*s == 0x00) // The array pointer has reached the key char which marks the end. It means we have matched the signature.
+		for (size_t i = 0; i < bytes_read; i++) {
+			c = (int)buffer[i];
+
+			if (*s != c && s > endPat) // No match?
+				s = endPat; // We "reset" the counter by sending the s pointer back to the start of the array.
+			if (*s == c)
 			{
-				return true;
+				s++;
+				if (*s == 0x00) // The array pointer has reached the key char which marks the end. It means we have matched the signature.
+				{
+					//the original function would leave the FILE* seeked to the end of the match
+					size_t foundpos = chunkpos + i + 1;
+					fseek(handle, foundpos, SEEK_SET);
+
+					free(buffer);
+					return true;
+				}
 			}
 		}
+		chunkpos += bytes_read;
 	}
+
+	free(buffer);
 	return false;
 }
 
@@ -599,7 +641,6 @@ typedef struct zlentry_s
 static lumpinfo_t* ResGetLumpsZip (FILE* handle, UINT16* nlmp)
 {
     zend_t zend;
-    zentry_t zentry;
     zlentry_t zlentry;
 
 	UINT16 numlumps = *nlmp;
@@ -613,7 +654,7 @@ static lumpinfo_t* ResGetLumpsZip (FILE* handle, UINT16* nlmp)
 	// Look for central directory end signature near end of file.
 	// Contains entry number (number of lumps), and central directory start offset.
 	fseek(handle, 0, SEEK_END);
-	if (!ResFindSignature(handle, pat_end, max(0, ftell(handle) - (22 + 65536))))
+	if (!ResFindSignature(handle, pat_end, std::max(0l, ftell(handle) - (22 + 65536))))
 	{
 		CONS_Alert(CONS_ERROR, "Missing central directory\n");
 		return NULL;
@@ -625,42 +666,44 @@ static lumpinfo_t* ResGetLumpsZip (FILE* handle, UINT16* nlmp)
 		CONS_Alert(CONS_ERROR, "Corrupt central directory (%s)\n", M_FileError(handle));
 		return NULL;
 	}
-	numlumps = zend.entries;
+	numlumps = SHORT(zend.entries);
 
-	lump_p = lumpinfo = Z_Malloc(numlumps * sizeof (*lumpinfo), PU_STATIC, NULL);
+	lump_p = lumpinfo = static_cast<lumpinfo_t*>(Z_Malloc(numlumps * sizeof (*lumpinfo), PU_STATIC, NULL));
+
+	fseek(handle, LONG(zend.cdiroffset), SEEK_SET);
+
+	char *cdir = static_cast<char*>(Z_MallocAlign(LONG(zend.cdirsize), PU_STATIC, &cdir, 7));
+	auto cdir_finally = srb2::finally([cdir] { Z_Free(cdir); });
+
+	if (fread(cdir, 1, LONG(zend.cdirsize), handle) < static_cast<UINT32>(LONG(zend.cdirsize)))
+	{
+		CONS_Alert(CONS_ERROR, "Failed to read central directory (%s)\n", M_FileError(handle));
+		Z_Free(lumpinfo);
+		return NULL;
+	}
+
+	size_t offset = 0;
 
-	fseek(handle, zend.cdiroffset, SEEK_SET);
 	for (i = 0; i < numlumps; i++, lump_p++)
 	{
+		zentry_t *zentry = reinterpret_cast<zentry_t*>(cdir + offset);
 		char* fullname;
 		char* trimname;
 		char* dotpos;
 
-		if (fread(&zentry, 1, sizeof(zentry_t), handle) < sizeof(zentry_t))
+		if (memcmp(zentry->signature, pat_central, 4))
-		{
-			CONS_Alert(CONS_ERROR, "Failed to read central directory (%s)\n", M_FileError(handle));
-			Z_Free(lumpinfo);
-			return NULL;
-		}
-		if (memcmp(zentry.signature, pat_central, 4))
 		{
 			CONS_Alert(CONS_ERROR, "Central directory is corrupt\n");
 			Z_Free(lumpinfo);
 			return NULL;
 		}
 
-		lump_p->position = zentry.offset; // NOT ACCURATE YET: we still need to read the local entry to find our true position
+		lump_p->position = LONG(zentry->offset); // NOT ACCURATE YET: we still need to read the local entry to find our true position
-		lump_p->disksize = zentry.compsize;
+		lump_p->disksize = LONG(zentry->compsize);
-		lump_p->size = zentry.size;
+		lump_p->size = LONG(zentry->size);
 
-		fullname = malloc(zentry.namelen + 1);
+		fullname = static_cast<char*>(malloc(SHORT(zentry->namelen) + 1));
-		if (fgets(fullname, zentry.namelen + 1, handle) != fullname)
+		strlcpy(fullname, (char*)(zentry + 1), SHORT(zentry->namelen) + 1);
-		{
-			CONS_Alert(CONS_ERROR, "Unable to read lumpname (%s)\n", M_FileError(handle));
-			Z_Free(lumpinfo);
-			free(fullname);
-			return NULL;
-		}
 
 		// Strip away file address and extension for the 8char name.
 		if ((trimname = strrchr(fullname, '/')) != 0)
@@ -672,16 +715,16 @@ static lumpinfo_t* ResGetLumpsZip (FILE* handle, UINT16* nlmp)
 			dotpos = fullname + strlen(fullname); // Watch for files without extension.
 
 		memset(lump_p->name, '\0', 9); // Making sure they're initialized to 0. Is it necessary?
-		strncpy(lump_p->name, trimname, min(8, dotpos - trimname));
+		strncpy(lump_p->name, trimname, std::min(static_cast<std::ptrdiff_t>(8), dotpos - trimname));
 		lump_p->hash = quickncasehash(lump_p->name, 8);
 
-		lump_p->longname = Z_Calloc(dotpos - trimname + 1, PU_STATIC, NULL);
+		lump_p->longname = static_cast<char*>(Z_Calloc(dotpos - trimname + 1, PU_STATIC, NULL));
 		strlcpy(lump_p->longname, trimname, dotpos - trimname + 1);
 
-		lump_p->fullname = Z_Calloc(zentry.namelen + 1, PU_STATIC, NULL);
+		lump_p->fullname = static_cast<char*>(Z_Calloc(SHORT(zentry->namelen) + 1, PU_STATIC, NULL));
-		strncpy(lump_p->fullname, fullname, zentry.namelen);
+		strncpy(lump_p->fullname, fullname, SHORT(zentry->namelen));
 
-		switch(zentry.compression)
+		switch(SHORT(zentry->compression))
 		{
 		case 0:
 			lump_p->compression = CM_NOCOMPRESSION;
@@ -703,12 +746,7 @@ static lumpinfo_t* ResGetLumpsZip (FILE* handle, UINT16* nlmp)
 		free(fullname);
 
 		// skip and ignore comments/extra fields
-		if (fseek(handle, zentry.xtralen + zentry.commlen, SEEK_CUR) != 0)
+		offset += sizeof *zentry + SHORT(zentry->namelen) + SHORT(zentry->xtralen) + SHORT(zentry->commlen);
-		{
-			CONS_Alert(CONS_ERROR, "Central directory is corrupt\n");
-			Z_Free(lumpinfo);
-			return NULL;
-		}
 	}
 
 	// Adjust lump position values properly
@@ -722,7 +760,7 @@ static lumpinfo_t* ResGetLumpsZip (FILE* handle, UINT16* nlmp)
 			return NULL;
 		}
 
-		lump_p->position += sizeof(zlentry_t) + zlentry.namelen + zlentry.xtralen;
+		lump_p->position += sizeof(zlentry_t) + SHORT(zlentry.namelen) + SHORT(zlentry.xtralen);
 	}
 
 	*nlmp = numlumps;
@@ -766,6 +804,7 @@ UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup)
 	size_t i;
 #endif
 	UINT8 md5sum[16];
+	const boolean md5_in_background = startup;
 	int important;
 
 	if (!(refreshdirmenu & REFRESHDIR_ADDFILE))
@@ -807,21 +846,24 @@ UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup)
 	important = !important;
 
 #ifndef NOMD5
-	//
+	if (!md5_in_background)
-	// w-waiiiit!
-	// Let's not add a wad file if the MD5 matches
-	// an MD5 of an already added WAD file!
-	//
-	W_MakeFileMD5(filename, md5sum);
-
-	for (i = 0; i < numwadfiles; i++)
 	{
-		if (!memcmp(wadfiles[i]->md5sum, md5sum, 16))
+		//
+		// w-waiiiit!
+		// Let's not add a wad file if the MD5 matches
+		// an MD5 of an already added WAD file!
+		//
+		W_MakeFileMD5(filename, md5sum);
+
+		for (i = 0; i < numwadfiles; i++)
 		{
-			CONS_Alert(CONS_ERROR, M_GetText("%s is already loaded\n"), filename);
+			if (!memcmp(W_GetFileMD5(wadfiles[i]), md5sum, 16))
-			if (handle)
+			{
-				fclose(handle);
+				CONS_Alert(CONS_ERROR, M_GetText("%s is already loaded\n"), filename);
-			return W_InitFileError(filename, false);
+				if (handle)
+					fclose(handle);
+				return W_InitFileError(filename, false);
+			}
 		}
 	}
 #endif
@@ -867,7 +909,7 @@ UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup)
 	//
 	// link wad file to search files
 	//
-	wadfile = Z_Malloc(sizeof (*wadfile), PU_STATIC, NULL);
+	wadfile = static_cast<wadfile_t*>(Z_Malloc(sizeof (*wadfile), PU_STATIC, NULL));
 	wadfile->filename = Z_StrDup(filename);
 	wadfile->type = type;
 	wadfile->handle = handle;
@@ -878,21 +920,35 @@ UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup)
 	wadfile->filesize = (unsigned)ftell(handle);
 	wadfile->type = type;
 
-	// already generated, just copy it over
-	M_Memcpy(&wadfile->md5sum, &md5sum, 16);
-
 	//
 	// set up caching
 	//
 	Z_Calloc(numlumps * sizeof (*wadfile->lumpcache), PU_STATIC, &wadfile->lumpcache);
 	Z_Calloc(numlumps * sizeof (*wadfile->patchcache), PU_STATIC, &wadfile->patchcache);
 
+	wadfile->internal_state = new wadfile_private_t(wadfile);
+
+	if (md5_in_background)
+	{
+		wadfile->internal_state->md5sum_.request();
+	}
+	else
+	{
+		// already generated, just copy it over
+		wadfile->internal_state->md5sum_.fill(md5sum);
+	}
+
 	//
 	// add the wadfile
 	//
 	CONS_Printf(M_GetText("Added file %s (%u lumps)\n"), filename, numlumps);
-	wadfiles[numwadfiles] = wadfile;
+
-	numwadfiles++; // must come BEFORE W_LoadDehackedLumps, so any addfile called by COM_BufInsertText called by Lua doesn't overwrite what we just loaded
+	{
+		std::lock_guard _(g_wadfiles_mutex);
+
+		wadfiles[numwadfiles] = wadfile;
+		numwadfiles++; // must come BEFORE W_LoadDehackedLumps, so any addfile called by COM_BufInsertText called by Lua doesn't overwrite what we just loaded
+	}
 
 #ifdef HWRENDER
 	// Read shaders from file
@@ -1596,8 +1652,8 @@ size_t W_ReadLumpHeaderPwad(UINT16 wad, UINT16 lump, void *dest, size_t size, si
 			char *decData; // Lump's decompressed real data.
 			size_t retval; // Helper var, lzf_decompress returns 0 when an error occurs.
 
-			rawData = Z_Malloc(l->disksize, PU_STATIC, NULL);
+			rawData = static_cast<char*>(Z_Malloc(l->disksize, PU_STATIC, NULL));
-			decData = Z_Malloc(l->size, PU_STATIC, NULL);
+			decData = static_cast<char*>(Z_Malloc(l->size, PU_STATIC, NULL));
 
 			if (fread(rawData, 1, l->disksize, handle) < l->disksize)
 				I_Error("wad %d, lump %d: cannot read compressed data", wad, lump);
@@ -1645,8 +1701,8 @@ size_t W_ReadLumpHeaderPwad(UINT16 wad, UINT16 lump, void *dest, size_t size, si
 			unsigned long rawSize = l->disksize;
 			unsigned long decSize = size;
 
-			rawData = Z_Malloc(rawSize, PU_STATIC, NULL);
+			rawData = static_cast<UINT8*>(Z_Malloc(rawSize, PU_STATIC, NULL));
-			decData = dest;
+			decData = static_cast<UINT8*>(dest);
 
 			if (fread(rawData, 1, rawSize, handle) < rawSize)
 				I_Error("wad %d, lump %d: cannot read compressed data", wad, lump);
@@ -1861,13 +1917,13 @@ static void *MakePatch(void *lumpdata, size_t size, INT32 tag, void *cache)
 #ifndef NO_PNG_LUMPS
 	if (Picture_IsLumpPNG((UINT8 *)lumpdata, len))
 	{
-		ptr = Picture_PNGConvert((UINT8 *)lumpdata, PICFMT_DOOMPATCH, NULL, NULL, NULL, NULL, len, &len, 0);
+		ptr = Picture_PNGConvert((UINT8 *)lumpdata, PICFMT_DOOMPATCH, NULL, NULL, NULL, NULL, len, &len, PICFLAGS_NONE);
 	}
 #endif
 
 	dest = Z_Calloc(sizeof(patch_t), tag, cache);
 
-	Patch_Create(ptr, len, dest);
+	Patch_Create(static_cast<softwarepatch_t*>(ptr), len, dest);
 
 	return dest;
 }
@@ -1910,7 +1966,7 @@ void *W_CachePatchNumPwad(UINT16 wad, UINT16 lump, INT32 tag)
 	if (!TestValidLump(wad, lump))
 		return NULL;
 
-	patch = W_CacheSoftwarePatchNumPwad(wad, lump, tag);
+	patch = static_cast<patch_t*>(W_CacheSoftwarePatchNumPwad(wad, lump, tag));
 
 #ifdef HWRENDER
 	// Software-only compile cache the data without conversion
@@ -1977,7 +2033,8 @@ void *W_CachePatchLongName(const char *name, INT32 tag)
   */
 #define MD5_FORMAT \
 	"%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x"
-static void PrintMD5String(const UINT8 *md5, char *buf)
+void PrintMD5String(const UINT8 *md5, char *buf);
+void PrintMD5String(const UINT8 *md5, char *buf)
 {
 	snprintf(buf, 2*MD5_LEN+1, MD5_FORMAT,
 		md5[0], md5[1], md5[2], md5[3],
@@ -2024,20 +2081,15 @@ void W_VerifyFileMD5(UINT16 wadfilenum, const char *matchmd5)
 		else realmd5[ix>>1] = (UINT8)(n<<4);
 	}
 
-	if (memcmp(realmd5, wadfiles[wadfilenum]->md5sum, 16))
+	wadfiles[wadfilenum]->internal_state->md5sum_.expect(realmd5);
-	{
-		char actualmd5text[2*MD5_LEN+1];
-		PrintMD5String(wadfiles[wadfilenum]->md5sum, actualmd5text);
-#ifdef _DEBUG
-		CONS_Printf
-#else
-		I_Error
-#endif
-			(M_GetText("File is old, is corrupt or has been modified: %s (found md5: %s, wanted: %s)\n"), wadfiles[wadfilenum]->filename, actualmd5text, matchmd5);
-	}
 #endif
 }
 
+const UINT8* W_GetFileMD5(const wadfile_t* wadfile)
+{
+	return wadfile->internal_state->md5sum_.get();
+}
+
 // Verify versions for different archive
 // formats. checklist assumed to be valid.
 
@@ -2121,14 +2173,11 @@ W_VerifyPK3 (FILE *fp, lumpchecklist_t *checklist, boolean status)
 	int verified = true;
 
     zend_t zend;
-    zentry_t zentry;
     zlentry_t zlentry;
 
 	long file_size;/* size of zip file */
 	long data_size;/* size of data inside zip file */
 
-	long old_position;
-
 	UINT16 numlumps;
 	size_t i;
 
@@ -2145,7 +2194,7 @@ W_VerifyPK3 (FILE *fp, lumpchecklist_t *checklist, boolean status)
 	fseek(fp, 0, SEEK_END);
 	file_size = ftell(fp);
 
-	if (!ResFindSignature(fp, pat_end, max(0, ftell(fp) - (22 + 65536))))
+	if (!ResFindSignature(fp, pat_end, std::max(0l, ftell(fp) - (22 + 65536))))
 		return true;
 
 	fseek(fp, -4, SEEK_CUR);
@@ -2154,25 +2203,32 @@ W_VerifyPK3 (FILE *fp, lumpchecklist_t *checklist, boolean status)
 
 	data_size = sizeof zend;
 
-	numlumps = zend.entries;
+	numlumps = SHORT(zend.entries);
+
+	fseek(fp, LONG(zend.cdiroffset), SEEK_SET);
+
+	char *cdir = static_cast<char*>(malloc(LONG(zend.cdirsize)));
+	auto cdir_finally = srb2::finally([cdir] { free(cdir); });
+
+	if (fread(cdir, 1, LONG(zend.cdirsize), fp) < static_cast<UINT32>(LONG(zend.cdirsize)))
+		return true;
+
+	size_t offset = 0;
 
-	fseek(fp, zend.cdiroffset, SEEK_SET);
 	for (i = 0; i < numlumps; i++)
 	{
+		zentry_t *zentry = reinterpret_cast<zentry_t*>(cdir + offset);
 		char* fullname;
 		char* trimname;
 		char* dotpos;
 
-		if (fread(&zentry, 1, sizeof(zentry_t), fp) < sizeof(zentry_t))
+		if (memcmp(zentry->signature, pat_central, 4) != 0)
-			return true;
-		if (memcmp(zentry.signature, pat_central, 4))
 			return true;
 
 		if (verified == true)
 		{
-			fullname = malloc(zentry.namelen + 1);
+			fullname = static_cast<char*>(malloc(SHORT(zentry->namelen) + 1));
-			if (fgets(fullname, zentry.namelen + 1, fp) != fullname)
+			strlcpy(fullname, (char*)(zentry + 1), SHORT(zentry->namelen) + 1);
-				return true;
 
 			// Strip away file address and extension for the 8char name.
 			if ((trimname = strrchr(fullname, '/')) != 0)
@@ -2186,7 +2242,7 @@ W_VerifyPK3 (FILE *fp, lumpchecklist_t *checklist, boolean status)
 					dotpos = fullname + strlen(fullname); // Watch for files without extension.
 
 				memset(lumpname, '\0', 9); // Making sure they're initialized to 0. Is it necessary?
-				strncpy(lumpname, trimname, min(8, dotpos - trimname));
+				strncpy(lumpname, trimname, std::min(static_cast<std::ptrdiff_t>(8), dotpos - trimname));
 
 				if (! W_VerifyName(lumpname, checklist, status))
 					verified = false;
@@ -2197,32 +2253,21 @@ W_VerifyPK3 (FILE *fp, lumpchecklist_t *checklist, boolean status)
 			}
 
 			free(fullname);
+		}
 
-			// skip and ignore comments/extra fields
+		offset += sizeof *zentry + SHORT(zentry->namelen) + SHORT(zentry->xtralen) + SHORT(zentry->commlen);
-			if (fseek(fp, zentry.xtralen + zentry.commlen, SEEK_CUR) != 0)
-				return true;
-		}
-		else
-		{
-			if (fseek(fp, zentry.namelen + zentry.xtralen + zentry.commlen, SEEK_CUR) != 0)
-				return true;
-		}
 
 		data_size +=
-			sizeof zentry + zentry.namelen + zentry.xtralen + zentry.commlen;
+			sizeof *zentry + SHORT(zentry->namelen) + SHORT(zentry->xtralen) + SHORT(zentry->commlen);
 
-		old_position = ftell(fp);
+		if (fseek(fp, LONG(zentry->offset), SEEK_SET) != 0)
-
-		if (fseek(fp, zentry.offset, SEEK_SET) != 0)
 			return true;
 
 		if (fread(&zlentry, 1, sizeof(zlentry_t), fp) < sizeof (zlentry_t))
 			return true;
 
 		data_size +=
-			sizeof zlentry + zlentry.namelen + zlentry.xtralen + zlentry.compsize;
+			sizeof zlentry + SHORT(zlentry.namelen) + SHORT(zlentry.xtralen) + LONG(zlentry.compsize);
-
-		fseek(fp, old_position, SEEK_SET);
 	}
 
 	if (data_size < file_size)
@@ -2366,15 +2411,15 @@ virtres_t* vres_GetMap(lumpnum_t lumpnum)
 		size_t *vsizecache;
 
 		// Remember that we're assuming that the WAD will have a specific set of lumps in a specific order.
-		UINT8 *wadData = W_CacheLumpNum(lumpnum, PU_LEVEL);
+		UINT8 *wadData = static_cast<UINT8*>(W_CacheLumpNum(lumpnum, PU_LEVEL));
-		filelump_t *fileinfo = (filelump_t *)(wadData + ((wadinfo_t *)wadData)->infotableofs);
+		filelump_t *fileinfo = (filelump_t *)(wadData + LONG(((wadinfo_t *)wadData)->infotableofs));
 
-		i = ((wadinfo_t *)wadData)->numlumps;
+		i = LONG(((wadinfo_t *)wadData)->numlumps);
-		vsizecache = Z_Malloc(sizeof(size_t)*i, PU_LEVEL, NULL);
+		vsizecache = static_cast<size_t*>(Z_Malloc(sizeof(size_t)*i, PU_LEVEL, NULL));
 
 		for (realentry = 0; realentry < i; realentry++)
 		{
-			vsizecache[realentry] = (size_t)(((filelump_t *)(fileinfo + realentry))->size);
+			vsizecache[realentry] = (size_t)(LONG(((filelump_t *)(fileinfo + realentry))->size));
 
 			if (!vsizecache[realentry])
 				continue;
@@ -2382,7 +2427,7 @@ virtres_t* vres_GetMap(lumpnum_t lumpnum)
 			numlumps++;
 		}
 
-		vlumps = Z_Malloc(sizeof(virtlump_t)*numlumps, PU_LEVEL, NULL);
+		vlumps = static_cast<virtlump_t*>(Z_Malloc(sizeof(virtlump_t)*numlumps, PU_LEVEL, NULL));
 
 		// Build the lumps, skipping over empty entries.
 		for (i = 0, realentry = 0; i < numlumps; realentry++)
@@ -2393,8 +2438,10 @@ virtres_t* vres_GetMap(lumpnum_t lumpnum)
 			// Play it safe with the name in this case.
 			memcpy(vlumps[i].name, (fileinfo + realentry)->name, 8);
 			vlumps[i].name[8] = '\0';
-			vlumps[i].data = Z_Malloc(vlumps[i].size, PU_LEVEL, NULL); // This is memory inefficient, sorry about that.
+			vlumps[i].data = static_cast<UINT8*>(
-			memcpy(vlumps[i].data, wadData + (fileinfo + realentry)->filepos, vlumps[i].size);
+				Z_Malloc(vlumps[i].size, PU_LEVEL, NULL) // This is memory inefficient, sorry about that.
+			);
+			memcpy(vlumps[i].data, wadData + LONG((fileinfo + realentry)->filepos), vlumps[i].size);
 			i++;
 		}
 
@@ -2414,16 +2461,16 @@ virtres_t* vres_GetMap(lumpnum_t lumpnum)
 		}
 		numlumps++;
 
-		vlumps = Z_Malloc(sizeof(virtlump_t)*numlumps, PU_LEVEL, NULL);
+		vlumps = static_cast<virtlump_t*>(Z_Malloc(sizeof(virtlump_t)*numlumps, PU_LEVEL, NULL));
 		for (i = 0; i < numlumps; i++, lumpnum++)
 		{
 			vlumps[i].size = W_LumpLength(lumpnum);
 			memcpy(vlumps[i].name, W_CheckNameForNum(lumpnum), 8);
 			vlumps[i].name[8] = '\0';
-			vlumps[i].data = W_CacheLumpNum(lumpnum, PU_LEVEL);
+			vlumps[i].data = static_cast<UINT8*>(W_CacheLumpNum(lumpnum, PU_LEVEL));
 		}
 	}
-	vres = Z_Malloc(sizeof(virtres_t), PU_LEVEL, NULL);
+	vres = static_cast<virtres_t*>(Z_Malloc(sizeof(virtres_t), PU_LEVEL, NULL));
 	vres->vlumps = vlumps;
 	vres->numlumps = numlumps;
 
@@ -2493,7 +2540,7 @@ void *vres_GetPatch(virtlump_t *vlump, INT32 tag)
 	if (!vlump)
 		return NULL;
 
-	patch = MakePatch(vlump->data, vlump->size, tag, NULL);
+	patch = static_cast<patch_t*>(MakePatch(vlump->data, vlump->size, tag, NULL));
 
 #ifdef HWRENDER
 	// Software-only compile cache the data without conversion

src/w_wad.h

@@ -130,7 +130,7 @@ struct wadfile_t
 	UINT16 numlumps; // this wad's number of resources
 	FILE *handle;
 	UINT32 filesize; // for network
-	UINT8 md5sum[16];
+	wadfile_private_t *internal_state;
 
 	boolean important; // also network - !W_VerifyNMUSlumps
 };
@@ -150,6 +150,8 @@ FILE *W_OpenWadFile(const char **filename, boolean useerrors);
 // Load and add a wadfile to the active wad files, returns numbers of lumps, INT16_MAX on error
 UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup);
 
+wadfile_private_t *W_InitPrivate(wadfile_t *wadfile);
+
 // W_InitMultipleFiles returns 1 if all is okay, 0 otherwise,
 // so that it stops with a message if a file was not found, but not if all is okay.
 // W_InitMultipleFiles exits if a file was not found, but not if all is okay.
@@ -218,6 +220,8 @@ void *W_CacheSoftwarePatchNum(lumpnum_t lumpnum, INT32 tag);
 void W_UnlockCachedPatch(void *patch);
 
 void W_VerifyFileMD5(UINT16 wadfilenum, const char *matchmd5);
+INT32 W_MakeFileMD5(const char *filename, void *resblock);
+const UINT8 *W_GetFileMD5(const wadfile_t *wadfile); // this function may block!
 
 int W_VerifyNMUSlumps(const char *filename, boolean exit_on_error);
 

src/wad_private.hpp

@@ -0,0 +1,183 @@
+// DR. ROBOTNIK'S RING RACERS
+//-----------------------------------------------------------------------------
+// Copyright (C) 2023 by Kart Krew.
+//
+// This program is free software distributed under the
+// terms of the GNU General Public License, version 2.
+// See the 'LICENSE' file for more details.
+//-----------------------------------------------------------------------------
+
+#ifndef WAD_PRIVATE_HPP
+#define WAD_PRIVATE_HPP
+
+#include <array>
+#include <atomic>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <mutex>
+#include <optional>
+#include <thread>
+
+#include "cxxutil.hpp"
+
+#include "doomtype.h"
+#include "typedef.h"
+#include "w_wad.h"
+
+extern "C" void PrintMD5String(const UINT8 *md5, char *buf);
+
+namespace srb2::wad
+{
+
+// Mutex for accessing wadfiles and numwadfiles from other
+// threads. WARNING: this doesn't cover the lumpcache.
+extern std::mutex g_wadfiles_mutex;
+
+}; // namespace srb2::wad::detail
+
+struct wadfile_private_t
+{
+	struct ChecksumState
+	{
+		explicit ChecksumState(wadfile_t* wad) : wad_(wad) {}
+
+		const std::uint8_t* get() const
+		{
+			std::unique_lock lock(mutex_);
+			cv_.wait(lock, [this]() -> bool { return valid_; });
+			return md5sum_;
+		}
+
+		void fill(const std::uint8_t chk[16])
+		{
+			SRB2_ASSERT(!valid_ && !thread_.joinable());
+
+			std::memcpy(md5sum_, chk, 16);
+			valid_ = true;
+		}
+
+		void request()
+		{
+			SRB2_ASSERT(!valid_ && !thread_.joinable());
+
+			thread_ = std::thread(&ChecksumState::worker, this);
+		}
+
+		void expect(const std::uint8_t chk[16])
+		{
+			std::lock_guard _(mutex_);
+
+			expected_md5sum_ = std::array<uint8_t, 16>();
+			std::memcpy(expected_md5sum_->data(), chk, 16);
+
+			if (valid_)
+			{
+				check_expected();
+			}
+		}
+
+		void join()
+		{
+			if (thread_.joinable())
+			{
+				thread_.join();
+			}
+		}
+
+	private:
+		wadfile_t* wad_;
+		std::uint8_t md5sum_[16];
+		std::optional<std::array<uint8_t, 16>> expected_md5sum_;
+		std::atomic_bool valid_ = false;
+		std::thread thread_;
+		mutable std::mutex mutex_;
+		mutable std::condition_variable cv_;
+
+		void worker()
+		{
+			W_MakeFileMD5(wad_->filename, md5sum_);
+
+			{
+				std::lock_guard _(mutex_);
+				check_expected();
+				valid_ = true;
+			}
+
+			cv_.notify_all();
+
+			check_collisions();
+		}
+
+		void check_collisions()
+		{
+			std::lock_guard _(srb2::wad::g_wadfiles_mutex);
+
+			for (UINT16 i = 0; i < numwadfiles; ++i)
+			{
+				const ChecksumState& other = wadfiles[i]->internal_state->md5sum_;
+
+				// That's us!
+				if (&other == this)
+				{
+					continue;
+				}
+
+				// Don't block for threads in progress,
+				// because they'll do their own check when
+				// they're done.
+				if (!other.valid_)
+				{
+					continue;
+				}
+
+				if (!std::memcmp(other.md5sum_, md5sum_, 16))
+				{
+					// FIXME: I_Error from a thread other than
+					// main kind of messes up the program
+					// state. It gets the message to the user,
+					// but should ideally be replaced by some
+					// communication with the main thread.
+					I_Error(
+						"MD5 checksum for '%s' matches a file already loaded.\n"
+						"Was this file added twice? Check the command line parameters.\n",
+						wad_->filename
+					);
+				}
+			}
+		}
+
+		void check_expected() const
+		{
+			if (!expected_md5sum_ || !std::memcmp(md5sum_, expected_md5sum_->data(), 16))
+			{
+				return;
+			}
+
+			char got[33];
+			char wanted[33];
+
+			PrintMD5String(md5sum_, got);
+			PrintMD5String(expected_md5sum_->data(), wanted);
+
+			I_Error(
+					"File is old, is corrupt or has been modified: %s (found md5: %s, wanted: %s)\n",
+					wad_->filename,
+					got,
+					wanted
+			);
+		}
+	};
+
+	ChecksumState md5sum_;
+
+	explicit wadfile_private_t(wadfile_t* wad) : md5sum_(wad) {}
+
+	~wadfile_private_t()
+	{
+		md5sum_.join();
+	}
+};
+
+#endif // WAD_PRIVATE_HPP