Rewrite of hash/cache system

data/dynos.c.h

@@ -29,11 +29,11 @@ void dynos_packs_set_enabled(s32 index, bool value);
 void dynos_generate_packs(const char* directory);
 
 // -- geos -- //
-void dynos_add_actor_custom(const char *modPath, const char* geoName);
+void dynos_add_actor_custom(const char *filePath, const char* geoName);
 const void* dynos_geolayout_get(const char *name);
 
 // -- collisions -- //
-void dynos_add_collision(const char *modPath, const char* collisionName);
+void dynos_add_collision(const char *filePath, const char* collisionName);
 Collision* dynos_collision_get(const char* collisionName);
 
 // -- movtexqcs -- //
@@ -42,7 +42,7 @@ struct MovtexQuadCollection* dynos_movtexqc_get_from_id(u32 id);
 struct MovtexQuadCollection* dynos_movtexqc_get_from_index(s32 index);
 
 // -- levels -- //
-void dynos_add_level(s32 modIndex, const char *modPath, const char* levelName);
+void dynos_add_level(s32 modIndex, const char *filePath, const char* levelName);
 const char* dynos_level_get_token(u32 index);
 Trajectory* dynos_level_get_trajectory(const char* name);
 void dynos_level_load_background(void *ptr);

data/dynos.cpp.h

@@ -743,7 +743,7 @@ s32              DynOS_Builtin_Func_GetIndexFromData(const void* aData);
 // Actor Manager
 //
 
-void DynOS_Actor_AddCustom(const SysPath &aPackFolder, const char *aActorName);
+void DynOS_Actor_AddCustom(const SysPath &aFilename, const char *aActorName);
 s32 DynOS_Actor_GetCount();
 const char *DynOS_Actor_GetName(s32 aIndex);
 const void *DynOS_Actor_GetLayoutFromIndex(s32 aIndex);
@@ -756,7 +756,7 @@ bool DynOS_Actor_IsCustom(s32 aIndex);
 //
 
 Array<Pair<const char*, GfxData*>> &DynOS_Lvl_GetArray();
-void  DynOS_Lvl_Activate(s32 modIndex, const SysPath &aPackFolder, const char *aLevelName);
+void  DynOS_Lvl_Activate(s32 modIndex, const SysPath &aFilePath, const char *aLevelName);
 DataNode<TexData>*  DynOS_Lvl_GetTexture(void *aPtr);
 GfxData* DynOS_Lvl_GetActiveGfx(void);
 const char* DynOS_Lvl_GetToken(u32 index);
@@ -769,7 +769,7 @@ void* DynOS_Lvl_Override(void *aCmd);
 // Col Manager
 //
 
-void DynOS_Col_Activate(const SysPath &aPackFolder, const char *aCollisionName);
+void DynOS_Col_Activate(const SysPath &aFilePath, const char *aCollisionName);
 Collision* DynOS_Col_Get(const char* collisionName);
 
 //
@@ -803,7 +803,7 @@ void DynOS_Anim_Table_Load(FILE *aFile, GfxData *aGfxData);
 DataNode<Collision>* DynOS_Col_Parse(GfxData* aGfxData, DataNode<Collision>* aNode, bool aDisplayPercent);
 void DynOS_Col_Write(FILE* aFile, GfxData* aGfxData, DataNode<Collision> *aNode);
 DataNode<Collision>* DynOS_Col_Load(FILE *aFile, GfxData *aGfxData);
-DataNode<Collision>* DynOS_Col_LoadFromBinary(const SysPath &aPackFolder, const char *aCollisionName);
+DataNode<Collision>* DynOS_Col_LoadFromBinary(const SysPath &aFilename, const char *aCollisionName);
 void DynOS_Col_Generate(const SysPath &aPackFolder, Array<Pair<u64, String>> _ActorsFolders, GfxData *_GfxData);
 
 DataNode<GeoLayout>* DynOS_Geo_Parse(GfxData* aGfxData, DataNode<GeoLayout>* aNode, bool aDisplayPercent);
@@ -866,11 +866,11 @@ void *DynOS_Pointer_Load(FILE *aFile, GfxData *aGfxData, u32 aValue);
 
 void DynOS_GfxDynCmd_Load(FILE *aFile, GfxData *aGfxData);
 
-GfxData *DynOS_Actor_LoadFromBinary(const SysPath &aPackFolder, const char *aActorName);
+GfxData *DynOS_Actor_LoadFromBinary(const SysPath &aPackFolder, const char *aActorName, const SysPath &aFilename);
 void DynOS_Actor_GeneratePack(const SysPath &aPackFolder);
 
 DataNode<LevelScript>* DynOS_Lvl_Parse(GfxData* aGfxData, DataNode<LevelScript>* aNode, bool aDisplayPercent);
-GfxData *DynOS_Lvl_LoadFromBinary(const SysPath &aPackFolder, const char *aLevelName);
+GfxData *DynOS_Lvl_LoadFromBinary(const SysPath &aFilename, const char *aLevelName);
 void DynOS_Lvl_GeneratePack(const SysPath &aPackFolder);
 s64 DynOS_Lvl_ParseLevelScriptConstants(const String& _Arg, bool* found);
 

data/dynos_bin_actor.cpp

@@ -72,7 +72,7 @@ static bool DynOS_Actor_WriteBinary(const SysPath &aOutputFilename, GfxData *aGf
  // Reading //
 /////////////
 
-GfxData *DynOS_Actor_LoadFromBinary(const SysPath &aPackFolder, const char *aActorName) {
+GfxData *DynOS_Actor_LoadFromBinary(const SysPath &aPackFolder, const char *aActorName, const SysPath &aFilename) {
     struct DynosGfxDataCache { SysPath mPackFolder; Array<Pair<const char *, GfxData *>> mGfxData; };
     static Array<DynosGfxDataCache *> sDynosGfxDataCache = {};
 
@@ -96,8 +96,7 @@ GfxData *DynOS_Actor_LoadFromBinary(const SysPath &aPackFolder, const char *aAct
 
     // Load data from binary file
     GfxData *_GfxData = NULL;
-    SysPath _Filename = fstring("%s/%s.bin", aPackFolder.begin(), aActorName);
+    FILE *_File = fopen(aFilename.c_str(), "rb");
-    FILE *_File = fopen(_Filename.c_str(), "rb");
     if (_File) {
         _GfxData = New<GfxData>();
         for (bool _Done = false; !_Done;) {

data/dynos_bin_col.cpp

@@ -664,11 +664,10 @@ DataNode<Collision>* DynOS_Col_Load(FILE *aFile, GfxData *aGfxData) {
     return _Node;
 }
 
-DataNode<Collision>* DynOS_Col_LoadFromBinary(const SysPath &aPackFolder, const char *aCollisionName) {
+DataNode<Collision>* DynOS_Col_LoadFromBinary(const SysPath &aFilename, const char *aCollisionName) {
     // Load data from binary file
     DataNode<Collision>* collisionNode = NULL;
-    SysPath _Filename = fstring("%s/%s.col", aPackFolder.begin(), aCollisionName);
+    FILE *_File = fopen(aFilename.c_str(), "rb");
-    FILE *_File = fopen(_Filename.c_str(), "rb");
     if (_File) {
         u8 type = ReadBytes<u8>(_File);
         if (type == DATA_TYPE_COLLISION) {

data/dynos_bin_lvl.cpp

@@ -1972,14 +1972,13 @@ static DataNode<LevelScript>* DynOS_Lvl_Load(FILE *aFile, GfxData *aGfxData) {
     return _Node;
 }
 
-GfxData *DynOS_Lvl_LoadFromBinary(const SysPath &aPackFolder, const char *aLevelName) {
+GfxData *DynOS_Lvl_LoadFromBinary(const SysPath &aFilename, const char *aLevelName) {
     struct DynosGfxDataCache { SysPath mPackFolder; Array<Pair<const char *, GfxData *>> mGfxData; };
     static Array<DynosGfxDataCache *> sDynosGfxDataCache;
 
     // Load data from binary file
     GfxData *_GfxData = NULL;
-    SysPath _Filename = fstring("%s/%s.lvl", aPackFolder.begin(), aLevelName);
+    FILE *_File = fopen(aFilename.c_str(), "rb");
-    FILE *_File = fopen(_Filename.c_str(), "rb");
     if (_File) {
         _GfxData = New<GfxData>();
         for (bool _Done = false; !_Done;) {

data/dynos_c.cpp

@@ -84,8 +84,8 @@ void dynos_generate_packs(const char* directory) {
 
 // -- geos -- //
 
-void dynos_add_actor_custom(const char *modPath, const char* geoName) {
+void dynos_add_actor_custom(const char *filePath, const char* geoName) {
-    DynOS_Actor_AddCustom(modPath, geoName);
+    DynOS_Actor_AddCustom(filePath, geoName);
 }
 
 const void* dynos_geolayout_get(const char *name) {
@@ -94,8 +94,8 @@ const void* dynos_geolayout_get(const char *name) {
 
 // -- collisions -- //
 
-void dynos_add_collision(const char *modPath, const char* collisionName) {
+void dynos_add_collision(const char *filePath, const char* collisionName) {
-    DynOS_Col_Activate(modPath, collisionName);
+    DynOS_Col_Activate(filePath, collisionName);
 }
 
 Collision* dynos_collision_get(const char* collisionName) {
@@ -124,8 +124,8 @@ struct MovtexQuadCollection* dynos_movtexqc_get_from_index(s32 index) {
 
 // -- levels -- //
 
-void dynos_add_level(s32 modIndex, const char *modPath, const char* levelName) {
+void dynos_add_level(s32 modIndex, const char *filePath, const char* levelName) {
-    DynOS_Lvl_Activate(modIndex, modPath, levelName);
+    DynOS_Lvl_Activate(modIndex, filePath, levelName);
 }
 
 const char* dynos_level_get_token(u32 index) {

data/dynos_gfx_update.cpp

@@ -140,7 +140,8 @@ void DynOS_Gfx_Update() {
                 if (_Enabled[i] && _ActorGfx->mPackIndex == -1) {
 
                     // Load Gfx data from binary
-                    GfxData *_GfxData = DynOS_Actor_LoadFromBinary(pDynosPacks[i]->mPath, DynOS_Actor_GetName(_ActorIndex));
+                    SysPath _Filename = fstring("%s/%s.bin", pDynosPacks[i]->mPath.begin(), DynOS_Actor_GetName(_ActorIndex));
+                    GfxData *_GfxData = DynOS_Actor_LoadFromBinary(pDynosPacks[i]->mPath, DynOS_Actor_GetName(_ActorIndex), _Filename);
                     if (_GfxData) {
                         _ActorGfx->mPackIndex = i;
                         _ActorGfx->mGfxData   = _GfxData;

data/dynos_mgr_actor.cpp

@@ -5,7 +5,7 @@ static Array<Pair<const char*, void *>> sDynosCustomActors;
 // TODO: the cleanup/refactor didn't really go as planned.
 //       clean up the actor management code more
 
-void DynOS_Actor_AddCustom(const SysPath &aPackFolder, const char *aActorName) {
+void DynOS_Actor_AddCustom(const SysPath &aFilename, const char *aActorName) {
     // check for duplicates
     bool isUnique = true;
     s32 foundIndex = -1;
@@ -21,7 +21,7 @@ void DynOS_Actor_AddCustom(const SysPath &aPackFolder, const char *aActorName) {
     char* actorName = (char*)calloc(1, sizeof(char) * (actorLen + 1));
     strcpy(actorName, aActorName);
 
-    GfxData *_GfxData = DynOS_Actor_LoadFromBinary(aPackFolder, actorName);
+    GfxData *_GfxData = DynOS_Actor_LoadFromBinary(aFilename, actorName, aFilename);
     if (!_GfxData) {
         free(actorName);
         return;

data/dynos_mgr_col.cpp

@@ -2,7 +2,7 @@
 
 static Array<Pair<const char*, DataNode<Collision>*>> sDynosCollisions;
 
-void DynOS_Col_Activate(const SysPath &aPackFolder, const char *aCollisionName) {
+void DynOS_Col_Activate(const SysPath &aFilename, const char *aCollisionName) {
     // check for duplicates
     for (s32 i = 0; i < sDynosCollisions.Count(); ++i) {
         if (!strcmp(sDynosCollisions[i].first, aCollisionName)) {
@@ -16,7 +16,7 @@ void DynOS_Col_Activate(const SysPath &aPackFolder, const char *aCollisionName)
     strcpy(collisionName, aCollisionName);
 
     // Load
-    DataNode<Collision>* _Node = DynOS_Col_LoadFromBinary(aPackFolder, collisionName);
+    DataNode<Collision>* _Node = DynOS_Col_LoadFromBinary(aFilename, collisionName);
     if (!_Node) {
         free(collisionName);
         return;

data/dynos_mgr_lvl.cpp

@@ -17,7 +17,7 @@ Array<Pair<const char*, GfxData*>> &DynOS_Lvl_GetArray() {
     return sDynosCustomLevelScripts;
 }
 
-void DynOS_Lvl_Activate(s32 modIndex, const SysPath &aPackFolder, const char *aLevelName) {
+void DynOS_Lvl_Activate(s32 modIndex, const SysPath &aFilename, const char *aLevelName) {
     // make sure vanilla levels were parsed
     DynOS_Level_GetCount();
 
@@ -32,7 +32,7 @@ void DynOS_Lvl_Activate(s32 modIndex, const SysPath &aPackFolder, const char *aL
     char* levelName = (char*)calloc(1, sizeof(char) * (levelLen + 1));
     strcpy(levelName, aLevelName);
 
-    GfxData* _Node = DynOS_Lvl_LoadFromBinary(aPackFolder, levelName);
+    GfxData* _Node = DynOS_Lvl_LoadFromBinary(aFilename, levelName);
     if (!_Node) {
         free(levelName);
         return;

src/pc/lua/smlua.c

@@ -31,15 +31,9 @@ static void smlua_exec_str(char* str) {
 static void smlua_load_script(struct Mod* mod, struct ModFile* file, u16 remoteIndex) {
     lua_State* L = gLuaState;
 
-    char fullPath[SYS_MAX_PATH] = { 0 };
-    if (!mod_file_full_path(fullPath, mod, file)) {
-        LOG_ERROR("Failed to concat path: '%s' + '%s", mod->relativePath, file->relativePath);
-        return;
-    }
-
     gLuaInitializingScript = 1;
-    if (luaL_loadfile(L, fullPath) != LUA_OK) {
+    if (luaL_loadfile(L, file->cachedPath) != LUA_OK) {
-        LOG_LUA("Failed to load lua script '%s'.", fullPath);
+        LOG_LUA("Failed to load lua script '%s'.", file->cachedPath);
         puts(smlua_to_string(L, lua_gettop(L)));
         return;
     }
@@ -82,7 +76,7 @@ static void smlua_load_script(struct Mod* mod, struct ModFile* file, u16 remoteI
 
     // run chunks
     if (lua_pcall(L, 0, LUA_MULTRET, 0) != LUA_OK) {
-        LOG_LUA("Failed to execute lua script '%s'.", fullPath);
+        LOG_LUA("Failed to execute lua script '%s'.", file->cachedPath);
         puts(smlua_to_string(L, lua_gettop(L)));
         smlua_dump_stack();
         gLuaInitializingScript = 0;

src/pc/lua/utils/smlua_audio_utils.c

@@ -108,19 +108,10 @@ void smlua_audio_utils_replace_sequence(u8 sequenceId, u8 bankId, u8 defaultVolu
     for (s32 i = 0; i < gLuaActiveMod->fileCount; i++) {
         struct ModFile* file = &gLuaActiveMod->files[i];
         if (!strcmp(file->relativePath, m64path)) {
-
-            char fullPath[SYS_MAX_PATH] = { 0 };
-            if (snprintf(fullPath, SYS_MAX_PATH - 1, "%s/%s", gLuaActiveMod->basePath, m64path) < 0) {
-                LOG_ERROR("Failed to concat full path to m64: %s", m64Name);
-                return;
-            }
-            normalize_path(fullPath);
-
-
             struct AudioOverride* override = &sAudioOverrides[sequenceId];
             smlua_audio_utils_reset(override);
-            LOG_INFO("Loading audio: %s", fullPath);
+            LOG_INFO("Loading audio: %s", file->cachedPath);
-            override->filename = strdup(fullPath);
+            override->filename = strdup(file->cachedPath);
             override->enabled = true;
             override->bank = bankId;
             sound_set_background_music_default_volume(sequenceId, defaultVolume);

src/pc/mods/mod.c

@@ -7,13 +7,7 @@
 #include "pc/utils/md5.h"
 #include "pc/debuglog.h"
 
-static void mod_activate_bin(struct Mod* mod, struct ModFile* file) {
+static void mod_activate_bin(struct ModFile* file) {
-    char dynosPath[SYS_MAX_PATH] = { 0 };
-    if (snprintf(dynosPath, SYS_MAX_PATH - 1, "%s/actors", mod->basePath) < 0) {
-        LOG_ERROR("Failed to concat dynos path");
-        return;
-    }
-
     // copy geo name
     char geoName[64] = { 0 };
     if (snprintf(geoName, 63, "%s", path_basename(file->relativePath)) < 0) {
@@ -32,17 +26,11 @@ static void mod_activate_bin(struct Mod* mod, struct ModFile* file) {
     }
 
     // Add to custom actors
-    LOG_INFO("Activating DynOS bin: '%s', '%s'", dynosPath, geoName);
+    LOG_INFO("Activating DynOS bin: '%s', '%s'", file->cachedPath, geoName);
-    dynos_add_actor_custom(dynosPath, geoName);
+    dynos_add_actor_custom(file->cachedPath, geoName);
 }
 
-static void mod_activate_col(struct Mod* mod, struct ModFile* file) {
+static void mod_activate_col(struct ModFile* file) {
-    char dynosPath[SYS_MAX_PATH] = { 0 };
-    if (snprintf(dynosPath, SYS_MAX_PATH - 1, "%s/actors", mod->basePath) < 0) {
-        LOG_ERROR("Failed to concat dynos path");
-        return;
-    }
-
     // copy geo name
     char colName[64] = { 0 };
     if (snprintf(colName, 63, "%s", path_basename(file->relativePath)) < 0) {
@@ -61,17 +49,11 @@ static void mod_activate_col(struct Mod* mod, struct ModFile* file) {
     }
 
     // Add to custom actors
-    LOG_INFO("Activating DynOS col: '%s', '%s'", dynosPath, colName);
+    LOG_INFO("Activating DynOS col: '%s', '%s'", file->cachedPath, colName);
-    dynos_add_collision(dynosPath, colName);
+    dynos_add_collision(file->cachedPath, colName);
 }
 
 static void mod_activate_lvl(struct Mod* mod, struct ModFile* file) {
-    char dynosPath[SYS_MAX_PATH] = { 0 };
-    if (snprintf(dynosPath, SYS_MAX_PATH - 1, "%s/levels", mod->basePath) < 0) {
-        LOG_ERROR("Failed to concat dynos path");
-        return;
-    }
-
     // copy geo name
     char lvlName[64] = { 0 };
     if (snprintf(lvlName, 63, "%s", path_basename(file->relativePath)) < 0) {
@@ -90,26 +72,25 @@ static void mod_activate_lvl(struct Mod* mod, struct ModFile* file) {
     }
 
     // Add to levels
-    LOG_INFO("Activating DynOS lvl: '%s', '%s'", dynosPath, lvlName);
+    LOG_INFO("Activating DynOS lvl: '%s', '%s'", file->cachedPath, lvlName);
-    dynos_add_level(mod->index, dynosPath, lvlName);
+    dynos_add_level(mod->index, file->cachedPath, lvlName);
 }
 
 void mod_activate(struct Mod* mod) {
     // activate dynos models
     for (int i = 0; i < mod->fileCount; i++) {
         struct ModFile* file = &mod->files[i];
-        normalize_path(file->relativePath);
+        mod_cache_add(mod, file);
         if (str_ends_with(file->relativePath, ".bin")) {
-            mod_activate_bin(mod, file);
+            mod_activate_bin(file);
         }
         if (str_ends_with(file->relativePath, ".col")) {
-            mod_activate_col(mod, file);
+            mod_activate_col(file);
         }
         if (str_ends_with(file->relativePath, ".lvl")) {
             mod_activate_lvl(mod, file);
         }
     }
-    mod_md5_hash(mod);
 }
 
 void mod_clear(struct Mod* mod) {
@@ -119,6 +100,10 @@ void mod_clear(struct Mod* mod) {
             fclose(file->fp);
             file->fp = NULL;
         }
+        if (file->cachedPath != NULL) {
+            free((char*)file->cachedPath);
+            file->cachedPath = NULL;
+        }
     }
 
     if (mod->name != NULL) {
@@ -517,114 +502,10 @@ bool mod_load(struct Mods* mods, char* basePath, char* modName) {
     if (isDirectory) {
         for (int i = 0; i < mod->fileCount; i++) {
             struct ModFile* file = &mod->files[i];
+            mod_cache_add(mod, file);
             LOG_INFO("      - %s", file->relativePath);
         }
     }
 
-    // hash and cache if we haven't before
-    struct ModCacheEntry* cache = mod_cache_get_from_path(fullPath);
-    if (cache == NULL) {
-        mod_md5_hash(mod);
-        mod_cache_add(mod->dataHash, 0, strdup(fullPath));
-    }
-
     return true;
 }
-
-#define MD5_BUFFER_SIZE 1024
-
-void mod_md5_hash(struct Mod* mod) {
-    char path[SYS_MAX_PATH] = { 0 };
-    u8 buffer[MD5_BUFFER_SIZE] = { 0 };
-
-    mod->hashProcessed = false;
-    mod_set_loading_order(mod);
-
-    MD5_CTX ctx = { 0 };
-    MD5_Init(&ctx);
-
-    for (u32 i = 0; i < mod->fileCount; i++) {
-        struct ModFile* file = &mod->files[i];
-        if (!concat_path(path, mod->basePath, file->relativePath)) {
-            LOG_ERROR("Failed to combine path for mod hashing.");
-            return;
-        }
-        normalize_path(path);
-
-        // open file pointer
-        FILE* fp = fopen(path, "rb");
-        if (fp == NULL) {
-            LOG_ERROR("Failed to open filepointer for mod hashing: '%s'.", path);
-            continue;
-        }
-
-        // read bytes and md5 them
-        size_t readBytes = 0;
-        do {
-            readBytes = fread(buffer, sizeof(u8), MD5_BUFFER_SIZE, fp);
-            MD5_Update(&ctx, buffer, readBytes);
-        } while (readBytes >= MD5_BUFFER_SIZE);
-
-        // close file pointer
-        fclose(fp);
-    }
-
-    // finish computing
-    MD5_Final(mod->dataHash, &ctx);
-    mod->hashProcessed = true;
-
-    if (mod->isDirectory) {
-        mod_cache_add(mod->dataHash, 0, strdup(mod->basePath));
-    } else {
-        if (!concat_path(path, mod->basePath, mod->files[0].relativePath)) {
-            LOG_ERROR("Failed to combine path for mod hashing.");
-            return;
-        }
-        mod_cache_add(mod->dataHash, 0, strdup(path));
-    }
-}
-
-void mod_load_from_cache(struct Mod* mod) {
-    mod->loadedFromCache = false;
-    struct ModCacheEntry* cache = mod_cache_get_from_hash(mod->dataHash);
-    if (cache == NULL) { return; }
-
-    // remember previous base path and hash
-    char oldBasePath[SYS_MAX_PATH] = { 0 };
-    snprintf(oldBasePath, SYS_MAX_PATH, "%s", mod->basePath);
-    u8 oldDataHash[16] = { 0 };
-    memcpy(oldDataHash, mod->dataHash, sizeof(u8) * 16);
-
-    // override base path
-    if (mod->isDirectory) {
-        snprintf(mod->basePath, SYS_MAX_PATH-1, "%s", cache->path);
-    } else {
-        path_get_folder(cache->path, mod->basePath);
-    }
-
-    // hash our local version of the mod
-    mod_md5_hash(mod);
-
-    // check if hashes match
-    if (mod->hashProcessed && !memcmp(mod->dataHash, oldDataHash, sizeof(u8) * 16)) {
-        // close file pointers
-        for (s32 i = 0; i < mod->fileCount; i++) {
-            struct ModFile* file = &mod->files[i];
-            if (file->fp != NULL) {
-                fclose(file->fp);
-                file->fp = NULL;
-            }
-        }
-
-        // mod is loaded and enabled
-        mod->loadedFromCache = true;
-        mod->enabled = true;
-        LOG_INFO("Loaded from cache: %s", mod->name);
-        return;
-    }
-
-    // error condition, load old base path and hash
-    snprintf(mod->basePath, SYS_MAX_PATH, "%s", oldBasePath);
-    memcpy(mod->dataHash, oldDataHash, sizeof(u8) * 16);
-    LOG_INFO("Could not load from cache: %s", mod->name);
-}

src/pc/mods/mod.h

@@ -14,6 +14,9 @@ struct ModFile {
     FILE* fp;
     u64 curOffset;
     bool complete;
+
+    u8 dataHash[16];
+    char* cachedPath;
 };
 
 struct Mod {
@@ -29,15 +32,10 @@ struct Mod {
     bool enabled;
     bool selectable;
     size_t size;
-    u8 dataHash[16];
-    bool hashProcessed;
-    bool loadedFromCache;
 };
 
 void mod_activate(struct Mod* mod);
 void mod_clear(struct Mod* mod);
 bool mod_load(struct Mods* mods, char* basePath, char* modName);
-void mod_md5_hash(struct Mod* mod);
-void mod_load_from_cache(struct Mod* mod);
 
 #endif

src/pc/mods/mod_cache.c

@@ -2,10 +2,13 @@
 #include "mod_cache.h"
 #include "mods.h"
 #include "mod.h"
+#include "mods_utils.h"
 #include "pc/debuglog.h"
+#include "pc/utils/md5.h"
 
 #define MOD_CACHE_FILENAME "mod.cache"
-#define MOD_CACHE_VERSION 2
+#define MOD_CACHE_VERSION 3
+#define MD5_BUFFER_SIZE 1024
 
 struct ModCacheEntry* sModCacheHead = NULL;
 
@@ -29,7 +32,10 @@ void mod_cache_shutdown(void) {
 
 struct ModCacheEntry* mod_cache_get_from_hash(u8* dataHash) {
     struct ModCacheEntry* node = sModCacheHead;
+    char str[128] = { 0 };
+    MD5_ToString(dataHash, str);
     while (node != NULL) {
+        MD5_ToString(node->dataHash, str);
         if (!memcmp(node->dataHash, dataHash, 16)) {
             return node;
         }
@@ -49,7 +55,7 @@ struct ModCacheEntry* mod_cache_get_from_path(const char* path) {
     return NULL;
 }
 
-void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path) {
+void mod_cache_add_internal(u8* dataHash, u64 lastLoaded, const char* path) {
     if (mod_cache_get_from_hash(dataHash)) {
         return;
     }
@@ -88,6 +94,56 @@ void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path) {
     }
 }
 
+void mod_cache_md5(const char* inPath, u8* outDataPath) {
+    char cpath[SYS_MAX_PATH] = { 0 };
+    u8 buffer[MD5_BUFFER_SIZE] = { 0 };
+
+    MD5_CTX ctx = { 0 };
+    MD5_Init(&ctx);
+
+    snprintf(cpath, SYS_MAX_PATH-1, "%s", inPath);
+    normalize_path(cpath);
+
+    // open file pointer
+    FILE* fp = fopen(cpath, "rb");
+    if (fp == NULL) {
+        LOG_ERROR("Failed to open filepointer for mod hashing: '%s'.", cpath);
+        return;
+    }
+
+    // read bytes and md5 them
+    size_t readBytes = 0;
+    do {
+        readBytes = fread(buffer, sizeof(u8), MD5_BUFFER_SIZE, fp);
+        MD5_Update(&ctx, buffer, readBytes);
+    } while (readBytes >= MD5_BUFFER_SIZE);
+
+    // close file pointer
+    fclose(fp);
+
+    // finish computing
+    MD5_Final(outDataPath, &ctx);
+}
+
+void mod_cache_add(struct Mod* mod, struct ModFile* file) {
+    // if we already have a cached path, don't do anything
+    if (file->cachedPath != NULL) {
+        return;
+    }
+
+    // build the path
+    char modFilePath[SYS_MAX_PATH] = { 0 };
+    if (!concat_path(modFilePath, mod->basePath, file->relativePath)) {
+        return;
+    }
+    normalize_path(modFilePath);
+
+    // hash and cache
+    file->cachedPath = strdup(modFilePath);
+    mod_cache_md5(file->cachedPath, file->dataHash);
+    mod_cache_add_internal(file->dataHash, 0, file->cachedPath);
+}
+
 void mod_cache_load(void) {
     mod_cache_shutdown();
     LOG_INFO("Loading mod cache");
@@ -107,6 +163,7 @@ void mod_cache_load(void) {
         return;
     }
 
+    u16 count = 0;
     while (true) {
         u8 dataHash[16] = { 0 };
         u64 lastLoaded = 0;
@@ -122,7 +179,8 @@ void mod_cache_load(void) {
         const char* path = calloc(pathLen + 1, sizeof(u8));
         fread((char*)path, sizeof(u8), pathLen + 1, fp);
 
-        mod_cache_add(dataHash, lastLoaded, path);
+        mod_cache_add_internal(dataHash, lastLoaded, path);
+        count++;
     }
     LOG_INFO("Loading mod cache complete");
 

src/pc/mods/mod_cache.h

@@ -1,6 +1,7 @@
 #ifndef MOD_CACHE_H
 
 #include "types.h"
+#include "mod.h"
 
 struct ModCacheEntry {
     u8 dataHash[16];
@@ -12,7 +13,7 @@ struct ModCacheEntry {
 void mod_cache_shutdown(void);
 struct ModCacheEntry* mod_cache_get_from_hash(u8* dataHash);
 struct ModCacheEntry* mod_cache_get_from_path(const char* path);
-void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path);
+void mod_cache_add(struct Mod* mod, struct ModFile* modFile);
 void mod_cache_load(void);
 void mod_cache_save(void);
 

src/pc/mods/mods.c

@@ -81,15 +81,9 @@ void mods_activate(struct Mods* mods) {
             for (int j = 0; j < mod->fileCount; j++) {
                 struct ModFile* file = &mod->files[j];
 
-                char fullPath[SYS_MAX_PATH] = { 0 };
+                file->fp = fopen(file->cachedPath, "rb");
-                if (!mod_file_full_path(fullPath, mod, file)) {
-                    LOG_ERROR("Failed to concat path: '%s' + '%s'", mod->basePath, file->relativePath);
-                    continue;
-                }
-
-                file->fp = fopen(fullPath, "rb");
                 if (file->fp == NULL) {
-                    LOG_ERROR("Failed to open file '%s'", fullPath);
+                    LOG_ERROR("Failed to open file '%s'", file->cachedPath);
                     continue;
                 }
 

src/pc/network/packets/packet_download.c

@@ -57,23 +57,24 @@ static void mark_groups_loaded_from_hash(void) {
     u64 fileStartOffset = 0;
     for (u64 modIndex = 0; modIndex < gRemoteMods.entryCount; modIndex++) {
         struct Mod* mod = gRemoteMods.entries[modIndex];
-
+        for (u64 fileIndex = 0; fileIndex < mod->fileCount; fileIndex++) {
-        if (mod->loadedFromCache) {
+            struct ModFile* file = &mod->files[fileIndex];
-            // if we loaded from cache, mark bytes as downloaded
+            if (file->cachedPath != NULL) {
-            sTotalDownloadBytes += mod->size;
+                // if we loaded from cache, mark bytes as downloaded
-            LOG_INFO("Loaded from cache: %s, %llu", mod->name, (u64)mod->size);
+                sTotalDownloadBytes += file->size;
-        } else {
+                LOG_INFO("Loaded from cache: %s, %llu", file->cachedPath, (u64)file->size);
-            // if we haven't loaded from cache, we need this offset group
+            } else {
-            u64 ogIndexStart = fileStartOffset / GROUP_SIZE;
+                // if we haven't loaded from cache, we need this offset group
-            u64 ogIndexEnd = (fileStartOffset + mod->size) / GROUP_SIZE;
+                u64 ogIndexStart = fileStartOffset / GROUP_SIZE;
-            do {
+                u64 ogIndexEnd = (fileStartOffset + mod->size) / GROUP_SIZE;
-                LOG_INFO("Marking group as required: %llu (%s)", ogIndexStart, mod->name);
+                do {
-                offsetGroupRequired[ogIndexStart] = 1;
+                    LOG_INFO("Marking group as required: %llu (%s)", ogIndexStart, file->relativePath);
-                ogIndexStart++;
+                    offsetGroupRequired[ogIndexStart] = 1;
-            } while (ogIndexStart <= ogIndexEnd);
+                    ogIndexStart++;
+                } while (ogIndexStart <= ogIndexEnd);
+            }
+            fileStartOffset += file->size;
         }
-
-        fileStartOffset += mod->size;
     }
 
     for (u64 ogIndex = 0; ogIndex < sOffsetGroupCount; ogIndex++) {
@@ -184,6 +185,7 @@ static void network_update_offset_groups(void) {
                 fclose(modFile->fp);
                 modFile->fp = NULL;
             }
+            mod->enabled = true;
         }
         LOG_INFO("Download complete!");
         network_send_join_request();
@@ -305,8 +307,6 @@ static void open_mod_file(struct Mod* mod, struct ModFile* file) {
         return;
     }
     LOG_INFO("Opened mod file pointer: %s", fullPath);
-
-    mod->enabled = true;
 }
 
 void network_receive_download(struct Packet* p) {
@@ -378,7 +378,7 @@ after_group:;
             u64 fileWriteLength = MIN((modFile->size - fileWriteOffset), (chunkLength - chunkPour));
 
             // read from file, filling chunk
-            if (!mod->loadedFromCache) {
+            if (!modFile->cachedPath) {
                 open_mod_file(mod, modFile);
                 fseek(modFile->fp, fileWriteOffset, SEEK_SET);
                 fwrite(&chunk[chunkPour], sizeof(u8), fileWriteLength, modFile->fp);

src/pc/network/packets/packet_mod_list.c

@@ -4,6 +4,7 @@
 #include "pc/mods/mods_utils.h"
 #include "pc/djui/djui.h"
 #include "pc/debuglog.h"
+#include "pc/mods/mod_cache.h"
 
 void network_send_mod_list_request(void) {
     SOFT_ASSERT(gNetworkType == NT_CLIENT);
@@ -63,7 +64,6 @@ void network_send_mod_list(void) {
         packet_write(&p, mod->relativePath, sizeof(u8) * relativePathLength);
         packet_write(&p, &modSize, sizeof(u64));
         packet_write(&p, &mod->isDirectory, sizeof(u8));
-        packet_write(&p, &mod->dataHash[0], sizeof(u8) * 16);
         packet_write(&p, &mod->fileCount, sizeof(u16));
         network_send_to(0, &p);
         LOG_INFO("    '%s': %llu", mod->name, (u64)mod->size);
@@ -79,6 +79,7 @@ void network_send_mod_list(void) {
             packet_write(&p, &relativePathLength, sizeof(u16));
             packet_write(&p, file->relativePath, sizeof(u8) * relativePathLength);
             packet_write(&p, &fileSize, sizeof(u64));
+            packet_write(&p, &file->dataHash[0], sizeof(u8) * 16);
             network_send_to(0, &p);
             LOG_INFO("      '%s': %llu", file->relativePath, (u64)file->size);
         }
@@ -184,7 +185,6 @@ void network_receive_mod_list_entry(struct Packet* p) {
     packet_read(p, mod->relativePath, relativePathLength * sizeof(u8));
     packet_read(p, &mod->size, sizeof(u64));
     packet_read(p, &mod->isDirectory, sizeof(u8));
-    packet_read(p, &mod->dataHash, sizeof(u8) * 16);
     normalize_path(mod->relativePath);
     LOG_INFO("    '%s': %llu", mod->name, (u64)mod->size);
 
@@ -258,9 +258,18 @@ void network_receive_mod_list_file(struct Packet* p) {
     packet_read(p, &relativePathLength, sizeof(u16));
     packet_read(p, file->relativePath, relativePathLength * sizeof(u8));
     packet_read(p, &file->size, sizeof(u64));
+    packet_read(p, &file->dataHash, sizeof(u8) * 16);
     file->fp = NULL;
     LOG_INFO("      '%s': %llu", file->relativePath, (u64)file->size);
 
+    struct ModCacheEntry* cache = mod_cache_get_from_hash(file->dataHash);
+    if (cache != NULL) {
+        LOG_INFO("Found file in cache: %s -> %s", file->relativePath, cache->path);
+        if (file->cachedPath != NULL) {
+            free((char*)file->cachedPath);
+        }
+        file->cachedPath = strdup(cache->path);
+    }
 }
 
 void network_receive_mod_list_done(struct Packet* p) {
@@ -277,7 +286,6 @@ void network_receive_mod_list_done(struct Packet* p) {
     for (u16 i = 0; i < gRemoteMods.entryCount; i++) {
         struct Mod* mod = gRemoteMods.entries[i];
         totalSize += mod->size;
-        mod_load_from_cache(mod);
     }
     gRemoteMods.size = totalSize;