More work on mod hashing/caching

developer/network.sh

@@ -16,9 +16,9 @@ if [ ! -f "$FILE" ]; then
 fi
 
 # no debug, direct
-$FILE --server 27015 --configfile sm64config_server.txt  &
+$FILE --server 7777 --configfile sm64config_server.txt  &
 sleep 2
-$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt  &
+$FILE --client 127.0.0.1 7777 --configfile sm64config_client.txt  &
 exit
 
 # no debug, discord
@@ -27,11 +27,11 @@ exit
 #exit
 
 # debug on server
-#$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt  & > /dev/null
+#$FILE --client 127.0.0.1 7777 --configfile sm64config_client.txt  & > /dev/null
-#$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 27015 --configfile sm64config_server.txt' -ex 'quit'
+#$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 7777 --configfile sm64config_server.txt' -ex 'quit'
 #exit
 
 # debug on client
-$FILE --server 27015 --configfile sm64config_server.txt & > /dev/null
+$FILE --server 7777 --configfile sm64config_server.txt & > /dev/null
-$WINPTY cgdb $FILE -ex 'network_receive_download' -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 27015 --configfile sm64config_client.txt' -ex 'quit'
+$WINPTY cgdb $FILE -ex 'network_receive_download' -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 7777 --configfile sm64config_client.txt' -ex 'quit'
 exit

src/pc/mods/mod.c

@@ -525,7 +525,7 @@ bool mod_load(struct Mods* mods, char* basePath, char* modName) {
     struct ModCacheEntry* cache = mod_cache_get_from_path(fullPath);
     if (cache == NULL) {
         mod_md5_hash(mod);
-        mod_cache_add(mod->dataHash, strdup(fullPath));
+        mod_cache_add(mod->dataHash, 0, strdup(fullPath));
     }
 
     return true;
@@ -572,13 +572,13 @@ void mod_md5_hash(struct Mod* mod) {
     mod->hashProcessed = true;
 
     if (mod->isDirectory) {
-        mod_cache_add(mod->dataHash, strdup(mod->basePath));
+        mod_cache_add(mod->dataHash, 0, strdup(mod->basePath));
     } else {
         if (!concat_path(path, mod->basePath, mod->files[0].relativePath)) {
             LOG_ERROR("Failed to combine path for mod hashing.");
             return;
         }
-        mod_cache_add(mod->dataHash, strdup(path));
+        mod_cache_add(mod->dataHash, 0, strdup(path));
     }
 }
 

src/pc/mods/mod_cache.c

@@ -5,7 +5,7 @@
 #include "pc/debuglog.h"
 
 #define MOD_CACHE_FILENAME "mod.cache"
-#define MOD_CACHE_VERSION 1
+#define MOD_CACHE_VERSION 2
 
 struct ModCacheEntry* sModCacheHead = NULL;
 
@@ -49,13 +49,15 @@ struct ModCacheEntry* mod_cache_get_from_path(const char* path) {
     return NULL;
 }
 
-void mod_cache_add(u8* dataHash, const char* path) {
+void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path) {
     if (mod_cache_get_from_hash(dataHash)) {
         return;
     }
 
     struct ModCacheEntry* node = calloc(1, sizeof(struct ModCacheEntry));
     memcpy(node->dataHash, dataHash, 16);
+    if (lastLoaded == 0) { lastLoaded = clock(); }
+    node->lastLoaded = lastLoaded;
     node->path = (char*)path;
     node->next = NULL;
 
@@ -107,17 +109,20 @@ void mod_cache_load(void) {
 
     while (true) {
         u8 dataHash[16] = { 0 };
+        u64 lastLoaded = 0;
         u16 pathLen;
 
         if (fread(dataHash, sizeof(u8), 16, fp) == 0) {
             break;
         }
+
+        fread(&lastLoaded, sizeof(u64), 1, fp);
         fread(&pathLen, sizeof(u16), 1, fp);
 
         const char* path = calloc(pathLen + 1, sizeof(u8));
         fread((char*)path, sizeof(u8), pathLen + 1, fp);
 
-        mod_cache_add(dataHash, path);
+        mod_cache_add(dataHash, lastLoaded, path);
     }
     LOG_INFO("Loading mod cache complete");
 
@@ -138,6 +143,7 @@ void mod_cache_save(void) {
     struct ModCacheEntry* node = sModCacheHead;
     while (node != NULL) {
         fwrite(node->dataHash, sizeof(u8), 16, fp);
+        fwrite(node->lastLoaded, sizeof(u64), 1, fp);
         u16 pathLen = strlen(node->path);
         fwrite(&pathLen, sizeof(u16), 1, fp);
         fwrite(node->path, sizeof(u8), pathLen + 1, fp);

src/pc/mods/mod_cache.h

@@ -4,6 +4,7 @@
 
 struct ModCacheEntry {
     u8 dataHash[16];
+    u64 lastLoaded;
     char* path;
     struct ModCacheEntry* next;
 };
@@ -11,7 +12,7 @@ struct ModCacheEntry {
 void mod_cache_shutdown(void);
 struct ModCacheEntry* mod_cache_get_from_hash(u8* dataHash);
 struct ModCacheEntry* mod_cache_get_from_path(const char* path);
-void mod_cache_add(u8* dataHash, const char* path);
+void mod_cache_add(u8* dataHash, u64 lastLoaded, const char* path);
 void mod_cache_load(void);
 void mod_cache_save(void);
 

src/pc/mods/mods.c

@@ -62,6 +62,7 @@ void mods_activate(struct Mods* mods) {
     }
 
     // copy enabled entries
+    gActiveMods.entryCount = 0;
     gActiveMods.size = 0;
     for (int i = 0; i < mods->entryCount; i++) {
         struct Mod* mod = mods->entries[i];
@@ -213,5 +214,4 @@ void mods_shutdown(void) {
     mods_clear(&gRemoteMods);
     mods_clear(&gActiveMods);
     mods_clear(&gLocalMods);
-    mods_delete_tmp();
 }

src/pc/mods/mods_utils.c

@@ -94,20 +94,6 @@ static void mods_delete_folder(char* path) {
     rmdir(path);
 }
 
-void mods_delete_tmp(void) {
-    // ensure tmpPath exists
-    char tmpPath[SYS_MAX_PATH] = { 0 };
-    if (snprintf(tmpPath, SYS_MAX_PATH - 1, "%s", fs_get_write_path(TMP_DIRECTORY)) < 0) {
-        LOG_ERROR("Failed to concat tmp path");
-        return;
-    }
-
-    // sanity
-    if (strlen(tmpPath) < 1) { return; }
-
-    // delete
-    mods_delete_folder(tmpPath);
-}
 //////////////////////////////////////////////////////////////////////////////////////////
 
 bool mod_file_full_path(char* destination, struct Mod* mod, struct ModFile* modFile) {

src/pc/mods/mods_utils.h

@@ -7,7 +7,6 @@
 
 void mods_size_enforce(struct Mods* mods);
 void mods_update_selectable(void);
-void mods_delete_tmp(void);
 
 bool mod_file_full_path(char* destination, struct Mod* mod, struct ModFile* modFile);
 bool mod_file_create_directories(struct Mod* mod, struct ModFile* modFile);

src/pc/network/network.c

@@ -176,6 +176,9 @@ bool network_allow_unknown_local_index(enum PacketType packetType) {
         || (packetType == PACKET_ACK)
         || (packetType == PACKET_MOD_LIST_REQUEST)
         || (packetType == PACKET_MOD_LIST)
+        || (packetType == PACKET_MOD_LIST_ENTRY)
+        || (packetType == PACKET_MOD_LIST_FILE)
+        || (packetType == PACKET_MOD_LIST_DONE)
         || (packetType == PACKET_DOWNLOAD_REQUEST)
         || (packetType == PACKET_DOWNLOAD)
         || (packetType == PACKET_KEEP_ALIVE)

src/pc/network/packets/packet.c

@@ -83,6 +83,9 @@ void packet_process(struct Packet* p) {
         case PACKET_MOD_LIST:                network_receive_mod_list(p);                break;
         case PACKET_DOWNLOAD_REQUEST:        network_receive_download_request(p);        break;
         case PACKET_DOWNLOAD:                network_receive_download(p);                break;
+        case PACKET_MOD_LIST_ENTRY:          network_receive_mod_list_entry(p);          break;
+        case PACKET_MOD_LIST_FILE:           network_receive_mod_list_file(p);           break;
+        case PACKET_MOD_LIST_DONE:           network_receive_mod_list_done(p);           break;
 
         case PACKET_LUA_SYNC_TABLE_REQUEST:  network_receive_lua_sync_table_request(p);  break;
         case PACKET_LUA_SYNC_TABLE:          network_receive_lua_sync_table(p);          break;

src/pc/network/packets/packet.h

@@ -58,6 +58,9 @@ enum PacketType {
     PACKET_MOD_LIST,
     PACKET_DOWNLOAD_REQUEST,
     PACKET_DOWNLOAD,
+    PACKET_MOD_LIST_ENTRY,
+    PACKET_MOD_LIST_FILE,
+    PACKET_MOD_LIST_DONE,
 
     PACKET_LUA_SYNC_TABLE_REQUEST,
     PACKET_LUA_SYNC_TABLE,
@@ -327,6 +330,9 @@ void network_send_mod_list_request(void);
 void network_receive_mod_list_request(UNUSED struct Packet* p);
 void network_send_mod_list(void);
 void network_receive_mod_list(struct Packet* p);
+void network_receive_mod_list_entry(struct Packet* p);
+void network_receive_mod_list_file(struct Packet* p);
+void network_receive_mod_list_done(struct Packet* p);
 
 // packet_download.c
 void network_start_download_requests(void);

src/pc/network/packets/packet_download.c

@@ -61,7 +61,7 @@ static void mark_groups_loaded_from_hash(void) {
         if (mod->loadedFromCache) {
             // if we loaded from cache, mark bytes as downloaded
             sTotalDownloadBytes += mod->size;
-            LOG_INFO("Loaded from cache: %s, %lu", mod->name, mod->size);
+            LOG_INFO("Loaded from cache: %s, %llu", mod->name, mod->size);
         } else {
             // if we haven't loaded from cache, we need this offset group
             u64 ogIndexStart = fileStartOffset / GROUP_SIZE;

src/pc/network/packets/packet_mod_list.c

@@ -32,6 +32,8 @@ void network_receive_mod_list_request(UNUSED struct Packet* p) {
 void network_send_mod_list(void) {
     SOFT_ASSERT(gNetworkType == NT_SERVER);
 
+    packet_ordered_begin();
+
     struct Packet p = { 0 };
     packet_init(&p, PACKET_MOD_LIST, true, PLMT_NONE);
 
@@ -40,6 +42,7 @@ void network_send_mod_list(void) {
     LOG_INFO("sending version: %s", version);
     packet_write(&p, &version, sizeof(u8) * MAX_VERSION_LENGTH);
     packet_write(&p, &gActiveMods.entryCount, sizeof(u16));
+    network_send_to(0, &p);
 
     LOG_INFO("sent mod list (%u):", gActiveMods.entryCount);
     for (u16 i = 0; i < gActiveMods.entryCount; i++) {
@@ -51,6 +54,9 @@ void network_send_mod_list(void) {
         u16 relativePathLength = strlen(mod->relativePath);
         u64 modSize = mod->size;
 
+        struct Packet p = { 0 };
+        packet_init(&p, PACKET_MOD_LIST_ENTRY, true, PLMT_NONE);
+        packet_write(&p, &i, sizeof(u16));
         packet_write(&p, &nameLength, sizeof(u16));
         packet_write(&p, mod->name, sizeof(u8) * nameLength);
         packet_write(&p, &relativePathLength, sizeof(u16));
@@ -58,20 +64,32 @@ void network_send_mod_list(void) {
         packet_write(&p, &modSize, sizeof(u64));
         packet_write(&p, &mod->isDirectory, sizeof(u8));
         packet_write(&p, &mod->dataHash[0], sizeof(u8) * 16);
+        packet_write(&p, &mod->fileCount, sizeof(u16));
+        network_send_to(0, &p);
         LOG_INFO("    '%s': %llu", mod->name, (u64)mod->size);
 
-        packet_write(&p, &mod->fileCount, sizeof(u16));
         for (u16 j = 0; j < mod->fileCount; j++) {
+            struct Packet p = { 0 };
+            packet_init(&p, PACKET_MOD_LIST_FILE, true, PLMT_NONE);
             struct ModFile* file = &mod->files[j];
             u16 relativePathLength = strlen(file->relativePath);
             u64 fileSize = file->size;
+            packet_write(&p, &i, sizeof(u16));
+            packet_write(&p, &j, sizeof(u16));
             packet_write(&p, &relativePathLength, sizeof(u16));
             packet_write(&p, file->relativePath, sizeof(u8) * relativePathLength);
             packet_write(&p, &fileSize, sizeof(u64));
+            network_send_to(0, &p);
             LOG_INFO("      '%s': %llu", file->relativePath, (u64)file->size);
         }
     }
-    network_send_to(0, &p);
+
+    struct Packet p2 = { 0 };
+    packet_init(&p2, PACKET_MOD_LIST_DONE, true, PLMT_NONE);
+    network_send_to(0, &p2);
+
+    packet_ordered_end();
+
 }
 
 void network_receive_mod_list(struct Packet* p) {
@@ -79,7 +97,7 @@ void network_receive_mod_list(struct Packet* p) {
 
     if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
         if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
-            LOG_ERROR("Received download from known local index '%d'", p->localIndex);
+            LOG_ERROR("Received mod list from known local index '%d'", p->localIndex);
             return;
         }
     }
@@ -118,72 +136,147 @@ void network_receive_mod_list(struct Packet* p) {
     }
 
     LOG_INFO("received mod list (%u):", gRemoteMods.entryCount);
+}
+
+void network_receive_mod_list_entry(struct Packet* p) {
+    SOFT_ASSERT(gNetworkType == NT_CLIENT);
+
+    // make sure it was sent by the server
+    if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
+        if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
+            LOG_ERROR("Received download from known local index '%d'", p->localIndex);
+            return;
+        }
+    }
+
+    // get mod index
+    u16 modIndex = 0;
+    packet_read(p, &modIndex, sizeof(u16));
+    if (modIndex >= gRemoteMods.entryCount) {
+        LOG_ERROR("Received mod outside of known range");
+        return;
+    }
+
+    // allocate mod entry
+    gRemoteMods.entries[modIndex] = calloc(1, sizeof(struct Mod));
+    struct Mod* mod = gRemoteMods.entries[modIndex];
+    if (mod == NULL) {
+        LOG_ERROR("Failed to allocate remote mod!");
+        return;
+    }
+
+    // get name length
+    u16 nameLength = 0;
+    packet_read(p, &nameLength, sizeof(u16));
+    if (nameLength > 31) {
+        LOG_ERROR("Received name with invalid length!");
+        return;
+    }
+
+    // get name
+    char name[32] = { 0 };
+    packet_read(p, name, nameLength * sizeof(u8));
+    mod->name = strdup(name);
+
+    // get other fields
+    u16 relativePathLength = 0;
+    packet_read(p, &relativePathLength, sizeof(u16));
+    packet_read(p, mod->relativePath, relativePathLength * sizeof(u8));
+    packet_read(p, &mod->size, sizeof(u64));
+    packet_read(p, &mod->isDirectory, sizeof(u8));
+    packet_read(p, &mod->dataHash, sizeof(u8) * 16);
+    normalize_path(mod->relativePath);
+    LOG_INFO("    '%s': %llu", mod->name, (u64)mod->size);
+
+    // figure out base path
+    if (mod->isDirectory) {
+        if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s/%s", gRemoteModsBasePath, mod->relativePath) < 0) {
+            LOG_ERROR("Failed save remote base path!");
+            return;
+        }
+        normalize_path(mod->basePath);
+    } else {
+        if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s", gRemoteModsBasePath) < 0) {
+            LOG_ERROR("Failed save remote base path!");
+            return;
+        }
+    }
+
+    // sanity check mod size
+    if (mod->size >= MAX_MOD_SIZE) {
+        djui_popup_create("Server had too large of a mod.\nQuitting.", 4);
+        network_shutdown(false);
+        return;
+    }
+
+    // get file count and allocate them
+    packet_read(p, &mod->fileCount, sizeof(u16));
+    mod->files = calloc(mod->fileCount, sizeof(struct ModFile));
+    if (mod->files == NULL) {
+        LOG_ERROR("Failed to allocate mod files!");
+        return;
+    }
+}
+
+void network_receive_mod_list_file(struct Packet* p) {
+    SOFT_ASSERT(gNetworkType == NT_CLIENT);
+
+    if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
+        if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
+            LOG_ERROR("Received download from known local index '%d'", p->localIndex);
+            return;
+        }
+    }
+
+    // get mod index
+    u16 modIndex = 0;
+    packet_read(p, &modIndex, sizeof(u16));
+    if (modIndex >= gRemoteMods.entryCount) {
+        LOG_ERROR("Received mod outside of known range");
+        return;
+    }
+    struct Mod* mod = gRemoteMods.entries[modIndex];
+    if (mod == NULL) {
+        LOG_ERROR("Received mod file for null mod");
+        return;
+    }
+
+    // get file index
+    u16 fileIndex = 0;
+    packet_read(p, &fileIndex, sizeof(u16));
+    if (fileIndex >= mod->fileCount) {
+        LOG_ERROR("Received mod file outside of known range");
+        return;
+    }
+    struct ModFile* file = &mod->files[fileIndex];
+    if (mod == NULL) {
+        LOG_ERROR("Received null mod file");
+        return;
+    }
+
+    u16 relativePathLength = 0;
+    packet_read(p, &relativePathLength, sizeof(u16));
+    packet_read(p, file->relativePath, relativePathLength * sizeof(u8));
+    packet_read(p, &file->size, sizeof(u64));
+    file->fp = NULL;
+    LOG_INFO("      '%s': %llu", file->relativePath, (u64)file->size);
+
+}
+
+void network_receive_mod_list_done(struct Packet* p) {
+    SOFT_ASSERT(gNetworkType == NT_CLIENT);
+
+    if (p->localIndex != UNKNOWN_LOCAL_INDEX) {
+        if (gNetworkPlayerServer == NULL || gNetworkPlayerServer->localIndex != p->localIndex) {
+            LOG_ERROR("Received download from known local index '%d'", p->localIndex);
+            return;
+        }
+    }
+
     size_t totalSize = 0;
     for (u16 i = 0; i < gRemoteMods.entryCount; i++) {
-        gRemoteMods.entries[i] = calloc(1, sizeof(struct Mod));
         struct Mod* mod = gRemoteMods.entries[i];
-        if (mod == NULL) {
-            LOG_ERROR("Failed to allocate remote mod!");
-            return;
-        }
-
-        char name[32] = { 0 };
-        u16 nameLength = 0;
-
-        packet_read(p, &nameLength, sizeof(u16));
-        if (nameLength > 31) {
-            LOG_ERROR("Received name with invalid length!");
-            return;
-        }
-        packet_read(p, name, nameLength * sizeof(u8));
-        mod->name = strdup(name);
-
-        u16 relativePathLength = 0;
-        packet_read(p, &relativePathLength, sizeof(u16));
-        packet_read(p, mod->relativePath, relativePathLength * sizeof(u8));
-        packet_read(p, &mod->size, sizeof(u64));
-        packet_read(p, &mod->isDirectory, sizeof(u8));
-        packet_read(p, &mod->dataHash, sizeof(u8) * 16);
-        normalize_path(mod->relativePath);
         totalSize += mod->size;
-        LOG_INFO("    '%s': %llu", mod->name, (u64)mod->size);
-
-        if (mod->isDirectory) {
-            if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s/%s", gRemoteModsBasePath, mod->relativePath) < 0) {
-                LOG_ERROR("Failed save remote base path!");
-                return;
-            }
-            normalize_path(mod->basePath);
-        } else {
-            if (snprintf(mod->basePath, SYS_MAX_PATH - 1, "%s", gRemoteModsBasePath) < 0) {
-                LOG_ERROR("Failed save remote base path!");
-                return;
-            }
-        }
-
-        if (mod->size >= MAX_MOD_SIZE) {
-            djui_popup_create("Server had too large of a mod.\nQuitting.", 4);
-            network_shutdown(false);
-            return;
-        }
-
-        packet_read(p, &mod->fileCount, sizeof(u16));
-        mod->files = calloc(mod->fileCount, sizeof(struct ModFile));
-        if (mod->files == NULL) {
-            LOG_ERROR("Failed to allocate mod files!");
-            return;
-        }
-
-        for (u16 j = 0; j < mod->fileCount; j++) {
-            struct ModFile* file = &mod->files[j];
-            u16 relativePathLength = 0;
-            packet_read(p, &relativePathLength, sizeof(u16));
-            packet_read(p, file->relativePath, relativePathLength * sizeof(u8));
-            packet_read(p, &file->size, sizeof(u64));
-            file->fp = NULL;
-            LOG_INFO("      '%s': %llu", file->relativePath, (u64)file->size);
-        }
-
         mod_load_from_cache(mod);
     }
     gRemoteMods.size = totalSize;

src/pc/network/packets/packet_reliable.c

@@ -125,6 +125,9 @@ static float network_adjust_max_elapsed(enum PacketType packetType, float maxEla
         case PACKET_DOWNLOAD:
         case PACKET_MOD_LIST_REQUEST:
         case PACKET_MOD_LIST:
+        case PACKET_MOD_LIST_ENTRY:
+        case PACKET_MOD_LIST_FILE:
+        case PACKET_MOD_LIST_DONE:
             return 0.2f + maxElapsed * 2.0f;
         default:
             return maxElapsed;