From 693a77078d230e6ba71d1a249122a314cb4ed5d5 Mon Sep 17 00:00:00 2001 From: Khangaroo Date: Fri, 15 Nov 2024 21:44:57 -0500 Subject: [PATCH] Low-hanging fruit mod loading optimizations (#493) * Low-hanging fruit mod loading optimizations * Fix DynOS fread spam and don't use a linked list for mod cache (still O(n^2), but much faster) --- data/dynos_bin_read.cpp | 9 ++- src/pc/fs/fs.c | 15 +++++ src/pc/mods/mod.c | 14 +++-- src/pc/mods/mod.h | 1 + src/pc/mods/mod_cache.c | 118 +++++++++++++++++++--------------------- src/pc/mods/mod_cache.h | 2 +- 6 files changed, 89 insertions(+), 70 deletions(-) diff --git a/data/dynos_bin_read.cpp b/data/dynos_bin_read.cpp index b3676844..d6e3c72f 100644 --- a/data/dynos_bin_read.cpp +++ b/data/dynos_bin_read.cpp @@ -36,14 +36,18 @@ char *DynOS_Read_Buffer(FILE* aFile, GfxData* aGfxData) { aGfxData->mModelIdentifier = (u32) _Length; } - // Remove comments + char *_OrigFileBuffer = New(_Length + 1); + char *pOrigFileBuffer = _OrigFileBuffer; rewind(aFile); + _OrigFileBuffer[fread(_OrigFileBuffer, 1, _Length, aFile)] = 0; + + // Remove comments char *_FileBuffer = New(_Length + 1); char *pFileBuffer = _FileBuffer; char _Previous = 0; char _Current = 0; s32 _CommentType = 0; - while (fread(&_Current, 1, 1, aFile)) { + while ((_Current = *pOrigFileBuffer++)) { if (_CommentType == COMMENT_NONE) { if (_Current == '/') { _CommentType = COMMENT_START; @@ -79,6 +83,7 @@ char *DynOS_Read_Buffer(FILE* aFile, GfxData* aGfxData) { _Previous = _Current; } *(pFileBuffer++) = 0; + Delete(_OrigFileBuffer); // Remove ifdef blocks // Doesn't support nested blocks diff --git a/src/pc/fs/fs.c b/src/pc/fs/fs.c index 4e0746bf..843ac1fe 100644 --- a/src/pc/fs/fs.c +++ b/src/pc/fs/fs.c @@ -11,6 +11,7 @@ #include #ifdef _WIN32 #include +#include #endif #include "macros.h" @@ -292,18 +293,32 @@ bool fs_sys_filename_is_portable(char const *filename) { /* these operate on the real file system */ bool fs_sys_path_exists(const char *name) { +#ifdef _WIN32 + return GetFileAttributesA(name) != INVALID_FILE_ATTRIBUTES; +#else struct stat st; return (stat(name, &st) == 0); +#endif } bool fs_sys_file_exists(const char *name) { +#ifdef _WIN32 + DWORD attribs = GetFileAttributesA(name); + return attribs != INVALID_FILE_ATTRIBUTES && !(attribs & FILE_ATTRIBUTE_DIRECTORY); +#else struct stat st; return (stat(name, &st) == 0 && S_ISREG(st.st_mode)); +#endif } bool fs_sys_dir_exists(const char *name) { +#ifdef _WIN32 + DWORD attribs = GetFileAttributesA(name); + return attribs != INVALID_FILE_ATTRIBUTES && (attribs & FILE_ATTRIBUTE_DIRECTORY); +#else struct stat st; return (stat(name, &st) == 0 && S_ISDIR(st.st_mode)); +#endif } bool fs_sys_dir_is_empty(const char *name) { diff --git a/src/pc/mods/mod.c b/src/pc/mods/mod.c index 48d60d6d..32b6efd7 100644 --- a/src/pc/mods/mod.c +++ b/src/pc/mods/mod.c @@ -204,18 +204,22 @@ void mod_clear(struct Mod* mod) { } mod->fileCount = 0; + mod->fileCapacity = 0; mod->size = 0; free(mod); } static struct ModFile* mod_allocate_file(struct Mod* mod, char* relativePath) { // actual allocation - u16 fileIndex = mod->fileCount++; - mod->files = realloc(mod->files, sizeof(struct ModFile) * mod->fileCount); - if (mod->files == NULL) { - LOG_ERROR("Failed to allocate file: '%s'", relativePath); - return NULL; + if (mod->fileCount == mod->fileCapacity) { + mod->fileCapacity = (mod->fileCapacity == 0) ? 16 : (mod->fileCapacity * 2); + mod->files = realloc(mod->files, sizeof(struct ModFile) * mod->fileCapacity); + if (mod->files == NULL) { + LOG_ERROR("Failed to allocate file: '%s'", relativePath); + return NULL; + } } + u16 fileIndex = mod->fileCount++; // clear memory struct ModFile* file = &mod->files[fileIndex]; diff --git a/src/pc/mods/mod.h b/src/pc/mods/mod.h index 7c16979d..27ef834a 100644 --- a/src/pc/mods/mod.h +++ b/src/pc/mods/mod.h @@ -30,6 +30,7 @@ struct Mod { struct ModFile* files; s32 index; u16 fileCount; + u16 fileCapacity; bool isDirectory; bool enabled; bool selectable; diff --git a/src/pc/mods/mod_cache.c b/src/pc/mods/mod_cache.c index 6f2c066e..fa1a662d 100644 --- a/src/pc/mods/mod_cache.c +++ b/src/pc/mods/mod_cache.c @@ -15,25 +15,26 @@ #define MOD_CACHE_VERSION 7 #define MD5_BUFFER_SIZE 1024 -struct ModCacheEntry* sModCacheHead = NULL; +static struct ModCacheEntry* sModCacheEntries = NULL; +static size_t sModCacheLength = 0; +static size_t sModLengthCapacity = 0; -static void mod_cache_remove_node(struct ModCacheEntry* node, struct ModCacheEntry* parent) { - if (node == NULL) { return; } - if (node == sModCacheHead) { sModCacheHead = node->next; } - if (parent != NULL) { parent->next = node->next; } - //LOG_INFO("Removing node: %s", node->path); +static void mod_cache_remove_node(struct ModCacheEntry* node) { if (node->path) { free(node->path); node->path = NULL; } - free(node); + if (node != &sModCacheEntries[sModCacheLength - 1]) + memcpy(node, &sModCacheEntries[sModCacheLength - 1], sizeof(struct ModCacheEntry)); + sModCacheLength--; } void mod_cache_shutdown(void) { LOG_INFO("Shutting down mod cache."); - while (sModCacheHead) { - mod_cache_remove_node(sModCacheHead, NULL); - } + sModCacheLength = 0; + sModLengthCapacity = 0; + free(sModCacheEntries); + sModCacheEntries = NULL; } void mod_cache_md5(const char* inPath, u8* outDataPath) { @@ -77,6 +78,15 @@ void mod_cache_md5(const char* inPath, u8* outDataPath) { MD5_Final(outDataPath, &ctx); } +static u64 mod_cache_fnv1a(const char* str) { + u64 hash = 0xCBF29CE484222325; + while (*str) { + hash *= 0x100000001B3; + hash ^= *str++; + } + return hash; +} + static bool mod_cache_is_valid(struct ModCacheEntry* node) { if (node == NULL || node->path == NULL || strlen(node->path) == 0) { return false; @@ -88,42 +98,37 @@ static bool mod_cache_is_valid(struct ModCacheEntry* node) { struct ModCacheEntry* mod_cache_get_from_hash(u8* dataHash) { if (dataHash == NULL) { return NULL; } - struct ModCacheEntry* node = sModCacheHead; - struct ModCacheEntry* prev = NULL; - while (node != NULL) { - struct ModCacheEntry* next = node->next; + for (size_t i = 0; i < sModCacheLength;) { + struct ModCacheEntry* node = &sModCacheEntries[i]; if (!memcmp(node->dataHash, dataHash, 16)) { if (mod_cache_is_valid(node)) { return node; } else { - mod_cache_remove_node(node, prev); - node = prev; + mod_cache_remove_node(node); + continue; } } - prev = node; - node = next; + i++; } return NULL; } struct ModCacheEntry* mod_cache_get_from_path(const char* path, bool validate) { if (path == NULL || strlen(path) == 0) { return NULL; } - struct ModCacheEntry* node = sModCacheHead; - struct ModCacheEntry* prev = NULL; - while (node != NULL) { - struct ModCacheEntry* next = node->next; - if (!strcmp(node->path, path)) { + u64 pathHash = mod_cache_fnv1a(path); + for (size_t i = 0; i < sModCacheLength;) { + struct ModCacheEntry* node = &sModCacheEntries[i]; + if (node->pathHash == pathHash && !strcmp(node->path, path)) { if (!validate) { return node; } else if (mod_cache_is_valid(node)) { return node; } else { - mod_cache_remove_node(node, prev); - node = prev; + mod_cache_remove_node(node); + continue; } } - prev = node; - node = next; + i++; } return NULL; } @@ -143,6 +148,7 @@ void mod_cache_add_internal(u8* dataHash, u64 lastLoaded, char* inPath) { return; } normalize_path((char*)path); + u64 pathHash = mod_cache_fnv1a(path); bool foundNonZero = false; for (u8 i = 0; i < 16; i++) { @@ -157,43 +163,34 @@ void mod_cache_add_internal(u8* dataHash, u64 lastLoaded, char* inPath) { return; } - struct ModCacheEntry* node = calloc(1, sizeof(struct ModCacheEntry)); - memcpy(node->dataHash, dataHash, sizeof(u8) * 16); - if (lastLoaded == 0) { lastLoaded = clock(); } - node->lastLoaded = lastLoaded; - node->path = (char*)path; - node->next = NULL; - - if (sModCacheHead == NULL) { - sModCacheHead = node; - LOG_INFO("Added head: %s", node->path); - return; + if (sModCacheEntries == NULL) { + sModLengthCapacity = 16; + sModCacheLength = 0; + sModCacheEntries = calloc(sModLengthCapacity, sizeof(struct ModCacheEntry)); + } else if (sModCacheLength == sModLengthCapacity) { + sModLengthCapacity *= 2; + sModCacheEntries = realloc(sModCacheEntries, sizeof(struct ModCacheEntry) * sModLengthCapacity); } - struct ModCacheEntry* n = sModCacheHead; - struct ModCacheEntry* prev = NULL; - while (n != NULL) { - struct ModCacheEntry* next = n->next; + struct ModCacheEntry node = {}; + memcpy(node.dataHash, dataHash, sizeof(u8) * 16); + if (lastLoaded == 0) { lastLoaded = clock(); } + node.lastLoaded = lastLoaded; + node.path = (char*)path; + node.pathHash = pathHash; - // found end of list, add it - if (next == NULL) { - LOG_INFO("Added node: %s", node->path); - if (n != node) { n->next = node; } - return; - } + for (size_t i = 0; i < sModCacheLength;) { + struct ModCacheEntry* n = &sModCacheEntries[i]; // found old hash, remove it - if (!strcmp(n->path, path)) { + if (n->pathHash == pathHash && !strcmp(n->path, path)) { LOG_INFO("Removing old node: %s", node->path); - mod_cache_remove_node(n, prev); + mod_cache_remove_node(n); } else { - prev = n; + i++; } - - n = next; } - - LOG_ERROR("Did not add node for some reason?"); + memcpy(&sModCacheEntries[sModCacheLength++], &node, sizeof(node)); } void mod_cache_add(struct Mod* mod, struct ModFile* file, bool useFilePath) { @@ -329,19 +326,16 @@ void mod_cache_save(void) { u8 t = *gBehaviorOffset != 0; fwrite(&t, sizeof(u8), 1, fp); - struct ModCacheEntry* node = sModCacheHead; - while (node != NULL) { - struct ModCacheEntry* next = node->next; - if (node->path == NULL) { goto iterate; } + for (size_t i = 0; i < sModCacheLength; i++) { + struct ModCacheEntry* node = &sModCacheEntries[i]; + if (node->path == NULL) { continue; } u16 pathLen = strlen(node->path); - if (pathLen == 0) { goto iterate; } + if (pathLen == 0) { continue; } fwrite(node->dataHash, sizeof(u8), 16, fp); fwrite(&node->lastLoaded, sizeof(u64), 1, fp); fwrite(&pathLen, sizeof(u16), 1, fp); fwrite(node->path, sizeof(u8), pathLen + 1, fp); -iterate: - node = next; } fclose(fp); diff --git a/src/pc/mods/mod_cache.h b/src/pc/mods/mod_cache.h index f43cf8c6..4993fe44 100644 --- a/src/pc/mods/mod_cache.h +++ b/src/pc/mods/mod_cache.h @@ -7,7 +7,7 @@ struct ModCacheEntry { u8 dataHash[16]; u64 lastLoaded; char* path; - struct ModCacheEntry* next; + u64 pathHash; }; void mod_cache_md5(const char* inPath, u8* outDataPath);