diff --git a/src/d_netcmd.c b/src/d_netcmd.c index faa8c4b16..41c6c4fda 100644 --- a/src/d_netcmd.c +++ b/src/d_netcmd.c @@ -4308,7 +4308,7 @@ static void Command_Addfile(void) for (i = 0; i < numwadfiles; i++) { - if (!memcmp(wadfiles[i]->md5sum, md5sum, 16)) + if (!memcmp(W_GetFileMD5(wadfiles[i]), md5sum, 16)) { CONS_Alert(CONS_ERROR, M_GetText("%s is already loaded\n"), fn); valid = false; diff --git a/src/d_netfil.c b/src/d_netfil.c index cb6897a8e..a010ffb1a 100644 --- a/src/d_netfil.c +++ b/src/d_netfil.c @@ -208,7 +208,7 @@ UINT8 *PutFileNeeded(UINT16 firstfile) count++; WRITEUINT32(p, wadfiles[i]->filesize); WRITESTRINGN(p, wadfilename, MAX_WADPATH); - WRITEMEM(p, wadfiles[i]->md5sum, 16); + WRITEMEM(p, W_GetFileMD5(wadfiles[i]), 16); } if (netbuffer->packettype == PT_MOREFILESNEEDED) netbuffer->u.filesneededcfg.num = count; @@ -576,7 +576,7 @@ INT32 CL_CheckFiles(void) return 2; // For the sake of speed, only bother with a md5 check - if (memcmp(wadfiles[j]->md5sum, fileneeded[i].md5sum, 16)) + if (memcmp(W_GetFileMD5(wadfiles[j]), fileneeded[i].md5sum, 16)) return 2; // It's accounted for! let's keep going. @@ -611,7 +611,7 @@ INT32 CL_CheckFiles(void) { nameonly(strcpy(wadfilename, wadfiles[j]->filename)); if (!stricmp(wadfilename, fileneeded[i].filename) && - !memcmp(wadfiles[j]->md5sum, fileneeded[i].md5sum, 16)) + !memcmp(W_GetFileMD5(wadfiles[j]), fileneeded[i].md5sum, 16)) { CONS_Debug(DBG_NETPLAY, "already loaded\n"); fileneeded[i].status = FS_OPEN; diff --git a/src/filesrch.c b/src/filesrch.c index 78232c26e..609f408c9 100644 --- a/src/filesrch.c +++ b/src/filesrch.c @@ -837,7 +837,7 @@ boolean preparefilemenu(boolean samedepth, boolean replayhut) if (strcmp(dent->d_name, filenamebuf[i])) continue; - if (cv_addons_md5.value && !checkfilemd5(menupath, wadfiles[i]->md5sum)) + if (cv_addons_md5.value && !checkfilemd5(menupath, W_GetFileMD5(wadfiles[i]))) continue; ext |= EXT_LOADED; diff --git a/src/g_demo.c b/src/g_demo.c index 95f5a0ac3..da30fe68c 100644 --- a/src/g_demo.c +++ b/src/g_demo.c @@ -2053,7 +2053,7 @@ static void G_SaveDemoExtraFiles(UINT8 **pp) { nameonly(( filename = va("%s", wadfiles[i]->filename) )); WRITESTRINGL((*pp), filename, MAX_WADPATH); - WRITEMEM((*pp), wadfiles[i]->md5sum, 16); + WRITEMEM((*pp), W_GetFileMD5(wadfiles[i]), 16); totalfiles++; } @@ -2089,7 +2089,7 @@ static void G_LoadDemoExtraFiles(UINT8 **pp) for (j = 0; j < numwadfiles; ++j) { - if (memcmp(md5sum, wadfiles[j]->md5sum, 16) == 0) + if (memcmp(md5sum, W_GetFileMD5(wadfiles[j]), 16) == 0) { alreadyloaded = true; break; @@ -2192,7 +2192,7 @@ static UINT8 G_CheckDemoExtraFiles(savebuffer_t *info, boolean quick) else continue; - if (memcmp(md5sum, wadfiles[j]->md5sum, 16) == 0) + if (memcmp(md5sum, W_GetFileMD5(wadfiles[j]), 16) == 0) { alreadyloaded = true; diff --git a/src/typedef.h b/src/typedef.h index 746a234b1..735728ae9 100644 --- a/src/typedef.h +++ b/src/typedef.h @@ -443,6 +443,7 @@ TYPEDEF (lumpinfo_t); TYPEDEF (virtlump_t); TYPEDEF (virtres_t); TYPEDEF (wadfile_t); +TYPEDEF (wadfile_private_t); #undef TYPEDEF #undef TYPEDEF2 diff --git a/src/w_wad.cpp b/src/w_wad.cpp index d2ea6c2ac..096916c65 100644 --- a/src/w_wad.cpp +++ b/src/w_wad.cpp @@ -42,6 +42,10 @@ #include #include +#include + +#include "cxxutil.hpp" +#include "wad_private.hpp" #include "doomdef.h" #include "doomstat.h" @@ -85,6 +89,14 @@ #define O_BINARY 0 #endif +using namespace srb2::wad; + +namespace srb2::wad +{ + +std::mutex g_wadfiles_mutex; + +}; // namespace srb2::wad typedef struct { @@ -123,6 +135,8 @@ void W_Shutdown(void) { wadfile_t *wad = wadfiles[numwadfiles]; + delete wad->internal_state; + fclose(wad->handle); Z_Free(wad->filename); while (wad->numlumps--) @@ -289,7 +303,7 @@ static inline void W_LoadDehackedLumps(UINT16 wadnum, boolean mainfile) * \param resblock resulting MD5 checksum * \return 0 if MD5 checksum was made, and is at resblock, 1 if error was found */ -static inline INT32 W_MakeFileMD5(const char *filename, void *resblock) +INT32 W_MakeFileMD5(const char *filename, void *resblock) { #ifdef NOMD5 (void)filename; @@ -790,6 +804,7 @@ UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup) size_t i; #endif UINT8 md5sum[16]; + const boolean md5_in_background = startup; int important; if (!(refreshdirmenu & REFRESHDIR_ADDFILE)) @@ -831,21 +846,24 @@ UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup) important = !important; #ifndef NOMD5 - // - // w-waiiiit! - // Let's not add a wad file if the MD5 matches - // an MD5 of an already added WAD file! - // - W_MakeFileMD5(filename, md5sum); - - for (i = 0; i < numwadfiles; i++) + if (!md5_in_background) { - if (!memcmp(wadfiles[i]->md5sum, md5sum, 16)) + // + // w-waiiiit! + // Let's not add a wad file if the MD5 matches + // an MD5 of an already added WAD file! + // + W_MakeFileMD5(filename, md5sum); + + for (i = 0; i < numwadfiles; i++) { - CONS_Alert(CONS_ERROR, M_GetText("%s is already loaded\n"), filename); - if (handle) - fclose(handle); - return W_InitFileError(filename, false); + if (!memcmp(W_GetFileMD5(wadfiles[i]), md5sum, 16)) + { + CONS_Alert(CONS_ERROR, M_GetText("%s is already loaded\n"), filename); + if (handle) + fclose(handle); + return W_InitFileError(filename, false); + } } } #endif @@ -902,21 +920,35 @@ UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup) wadfile->filesize = (unsigned)ftell(handle); wadfile->type = type; - // already generated, just copy it over - M_Memcpy(&wadfile->md5sum, &md5sum, 16); - // // set up caching // Z_Calloc(numlumps * sizeof (*wadfile->lumpcache), PU_STATIC, &wadfile->lumpcache); Z_Calloc(numlumps * sizeof (*wadfile->patchcache), PU_STATIC, &wadfile->patchcache); + wadfile->internal_state = new wadfile_private_t(wadfile); + + if (md5_in_background) + { + wadfile->internal_state->md5sum_.request(); + } + else + { + // already generated, just copy it over + wadfile->internal_state->md5sum_.fill(md5sum); + } + // // add the wadfile // CONS_Printf(M_GetText("Added file %s (%u lumps)\n"), filename, numlumps); - wadfiles[numwadfiles] = wadfile; - numwadfiles++; // must come BEFORE W_LoadDehackedLumps, so any addfile called by COM_BufInsertText called by Lua doesn't overwrite what we just loaded + + { + std::lock_guard _(g_wadfiles_mutex); + + wadfiles[numwadfiles] = wadfile; + numwadfiles++; // must come BEFORE W_LoadDehackedLumps, so any addfile called by COM_BufInsertText called by Lua doesn't overwrite what we just loaded + } #ifdef HWRENDER // Read shaders from file @@ -2001,7 +2033,8 @@ void *W_CachePatchLongName(const char *name, INT32 tag) */ #define MD5_FORMAT \ "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x" -static void PrintMD5String(const UINT8 *md5, char *buf) +void PrintMD5String(const UINT8 *md5, char *buf); +void PrintMD5String(const UINT8 *md5, char *buf) { snprintf(buf, 2*MD5_LEN+1, MD5_FORMAT, md5[0], md5[1], md5[2], md5[3], @@ -2048,20 +2081,15 @@ void W_VerifyFileMD5(UINT16 wadfilenum, const char *matchmd5) else realmd5[ix>>1] = (UINT8)(n<<4); } - if (memcmp(realmd5, wadfiles[wadfilenum]->md5sum, 16)) - { - char actualmd5text[2*MD5_LEN+1]; - PrintMD5String(wadfiles[wadfilenum]->md5sum, actualmd5text); -#ifdef _DEBUG - CONS_Printf -#else - I_Error -#endif - (M_GetText("File is old, is corrupt or has been modified: %s (found md5: %s, wanted: %s)\n"), wadfiles[wadfilenum]->filename, actualmd5text, matchmd5); - } + wadfiles[wadfilenum]->internal_state->md5sum_.expect(realmd5); #endif } +const UINT8* W_GetFileMD5(const wadfile_t* wadfile) +{ + return wadfile->internal_state->md5sum_.get(); +} + // Verify versions for different archive // formats. checklist assumed to be valid. diff --git a/src/w_wad.h b/src/w_wad.h index 31ca26e69..ed7bb15b0 100644 --- a/src/w_wad.h +++ b/src/w_wad.h @@ -130,7 +130,7 @@ struct wadfile_t UINT16 numlumps; // this wad's number of resources FILE *handle; UINT32 filesize; // for network - UINT8 md5sum[16]; + wadfile_private_t *internal_state; boolean important; // also network - !W_VerifyNMUSlumps }; @@ -150,6 +150,8 @@ FILE *W_OpenWadFile(const char **filename, boolean useerrors); // Load and add a wadfile to the active wad files, returns numbers of lumps, INT16_MAX on error UINT16 W_InitFile(const char *filename, boolean mainfile, boolean startup); +wadfile_private_t *W_InitPrivate(wadfile_t *wadfile); + // W_InitMultipleFiles returns 1 if all is okay, 0 otherwise, // so that it stops with a message if a file was not found, but not if all is okay. // W_InitMultipleFiles exits if a file was not found, but not if all is okay. @@ -218,6 +220,8 @@ void *W_CacheSoftwarePatchNum(lumpnum_t lumpnum, INT32 tag); void W_UnlockCachedPatch(void *patch); void W_VerifyFileMD5(UINT16 wadfilenum, const char *matchmd5); +INT32 W_MakeFileMD5(const char *filename, void *resblock); +const UINT8 *W_GetFileMD5(const wadfile_t *wadfile); // this function may block! int W_VerifyNMUSlumps(const char *filename, boolean exit_on_error); diff --git a/src/wad_private.hpp b/src/wad_private.hpp new file mode 100644 index 000000000..5d50e9dc3 --- /dev/null +++ b/src/wad_private.hpp @@ -0,0 +1,183 @@ +// DR. ROBOTNIK'S RING RACERS +//----------------------------------------------------------------------------- +// Copyright (C) 2023 by Kart Krew. +// +// This program is free software distributed under the +// terms of the GNU General Public License, version 2. +// See the 'LICENSE' file for more details. +//----------------------------------------------------------------------------- + +#ifndef WAD_PRIVATE_HPP +#define WAD_PRIVATE_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cxxutil.hpp" + +#include "doomtype.h" +#include "typedef.h" +#include "w_wad.h" + +extern "C" void PrintMD5String(const UINT8 *md5, char *buf); + +namespace srb2::wad +{ + +// Mutex for accessing wadfiles and numwadfiles from other +// threads. WARNING: this doesn't cover the lumpcache. +extern std::mutex g_wadfiles_mutex; + +}; // namespace srb2::wad::detail + +struct wadfile_private_t +{ + struct ChecksumState + { + explicit ChecksumState(wadfile_t* wad) : wad_(wad) {} + + const std::uint8_t* get() const + { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this]() -> bool { return valid_; }); + return md5sum_; + } + + void fill(const std::uint8_t chk[16]) + { + SRB2_ASSERT(!valid_ && !thread_.joinable()); + + std::memcpy(md5sum_, chk, 16); + valid_ = true; + } + + void request() + { + SRB2_ASSERT(!valid_ && !thread_.joinable()); + + thread_ = std::thread(&ChecksumState::worker, this); + } + + void expect(const std::uint8_t chk[16]) + { + std::lock_guard _(mutex_); + + expected_md5sum_ = std::array(); + std::memcpy(expected_md5sum_->data(), chk, 16); + + if (valid_) + { + check_expected(); + } + } + + void join() + { + if (thread_.joinable()) + { + thread_.join(); + } + } + + private: + wadfile_t* wad_; + std::uint8_t md5sum_[16]; + std::optional> expected_md5sum_; + std::atomic_bool valid_ = false; + std::thread thread_; + mutable std::mutex mutex_; + mutable std::condition_variable cv_; + + void worker() + { + W_MakeFileMD5(wad_->filename, md5sum_); + + { + std::lock_guard _(mutex_); + check_expected(); + valid_ = true; + } + + cv_.notify_all(); + + check_collisions(); + } + + void check_collisions() + { + std::lock_guard _(srb2::wad::g_wadfiles_mutex); + + for (UINT16 i = 0; i < numwadfiles; ++i) + { + const ChecksumState& other = wadfiles[i]->internal_state->md5sum_; + + // That's us! + if (&other == this) + { + continue; + } + + // Don't block for threads in progress, + // because they'll do their own check when + // they're done. + if (!other.valid_) + { + continue; + } + + if (!std::memcmp(other.md5sum_, md5sum_, 16)) + { + // FIXME: I_Error from a thread other than + // main kind of messes up the program + // state. It gets the message to the user, + // but should ideally be replaced by some + // communication with the main thread. + I_Error( + "MD5 checksum for '%s' matches a file already loaded.\n" + "Was this file added twice? Check the command line parameters.\n", + wad_->filename + ); + } + } + } + + void check_expected() const + { + if (!expected_md5sum_ || !std::memcmp(md5sum_, expected_md5sum_->data(), 16)) + { + return; + } + + char got[33]; + char wanted[33]; + + PrintMD5String(md5sum_, got); + PrintMD5String(expected_md5sum_->data(), wanted); + + I_Error( + "File is old, is corrupt or has been modified: %s (found md5: %s, wanted: %s)\n", + wad_->filename, + got, + wanted + ); + } + }; + + ChecksumState md5sum_; + + explicit wadfile_private_t(wadfile_t* wad) : md5sum_(wad) {} + + ~wadfile_private_t() + { + md5sum_.join(); + } +}; + +#endif // WAD_PRIVATE_HPP