add texture preloading

when EXTERNAL_TEXTURES is defined, the texture hashmap in gfx_pc.c uses texture names as keys

all textures are precached on startup if EXTERNAL_TEXTURES is defined and 'precache' is true in the config

src/pc/configfile.c

@@ -66,7 +66,9 @@ unsigned int configKeyStickUp[MAX_BINDS]    = { 0x0011,   VK_INVALID, VK_INVALID
 unsigned int configKeyStickDown[MAX_BINDS]  = { 0x001F,   VK_INVALID, VK_INVALID };
 unsigned int configKeyStickLeft[MAX_BINDS]  = { 0x001E,   VK_INVALID, VK_INVALID };
 unsigned int configKeyStickRight[MAX_BINDS] = { 0x0020,   VK_INVALID, VK_INVALID };
-
+#ifdef EXTERNAL_TEXTURES
+bool configPrecacheRes = false;
+#endif
 #ifdef BETTERCAMERA
 // BetterCamera settings
 unsigned int configCameraXSens   = 50;
@@ -105,6 +107,9 @@ static const struct ConfigOption options[] = {
     {.name = "key_stickdown",        .type = CONFIG_TYPE_BIND, .uintValue = configKeyStickDown},
     {.name = "key_stickleft",        .type = CONFIG_TYPE_BIND, .uintValue = configKeyStickLeft},
     {.name = "key_stickright",       .type = CONFIG_TYPE_BIND, .uintValue = configKeyStickRight},
+    #ifdef EXTERNAL_TEXTURES
+    {.name = "precache",             .type = CONFIG_TYPE_BOOL, .boolValue = &configPrecacheRes},
+    #endif
     #ifdef BETTERCAMERA
     {.name = "bettercam_enable",     .type = CONFIG_TYPE_BOOL, .boolValue = &configEnableCamera},
     {.name = "bettercam_mouse_look", .type = CONFIG_TYPE_BOOL, .boolValue = &configCameraMouse},

src/pc/configfile.h

@@ -35,6 +35,9 @@ extern unsigned int configKeyStickUp[];
 extern unsigned int configKeyStickDown[];
 extern unsigned int configKeyStickLeft[];
 extern unsigned int configKeyStickRight[];
+#ifdef EXTERNAL_TEXTURES
+extern bool         configPrecacheRes;
+#endif
 #ifdef BETTERCAMERA
 extern unsigned int configCameraXSens;
 extern unsigned int configCameraYSens;

src/pc/gfx/gfx_pc.c

@@ -46,6 +46,17 @@
 #define MAX_LIGHTS 2
 #define MAX_VERTICES 64
 
+#ifdef EXTERNAL_TEXTURES
+# define MAX_CACHED_TEXTURES 4096 // for preloading purposes
+# define HASH_SHIFT 0
+#else
+# define MAX_CACHED_TEXTURES 512
+# define HASH_SHIFT 5
+#endif
+
+#define HASHMAP_LEN (MAX_CACHED_TEXTURES * 2)
+#define HASH_MASK (HASHMAP_LEN - 1)
+
 struct RGBA {
     uint8_t r, g, b, a;
 };
@@ -72,8 +83,8 @@ struct TextureHashmapNode {
     bool linear_filter;
 };
 static struct {
-    struct TextureHashmapNode *hashmap[1024];
+    struct TextureHashmapNode *hashmap[HASHMAP_LEN];
-    struct TextureHashmapNode pool[512];
+    struct TextureHashmapNode pool[MAX_CACHED_TEXTURES];
     uint32_t pool_pos;
 } gfx_texture_cache;
 
@@ -161,41 +172,17 @@ static size_t buf_vbo_num_tris;
 static struct GfxWindowManagerAPI *gfx_wapi;
 static struct GfxRenderingAPI *gfx_rapi;
 
-#if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) && !defined(__APPLE__)
+#ifdef EXTERNAL_TEXTURES
-// old mingw
+static inline size_t string_hash(const uint8_t *str) {
-# include <_mingw.h>
+    size_t h = 0;
-# define NO_CLOCK_GETTIME
+    for (const uint8_t *p = str; *p; p++)
+        h = 31 * h + *p;
+    return h;
+}
 #endif
 
-#ifdef NO_CLOCK_GETTIME
-
-#if defined(_WIN32)
-#include <windows.h>
-#define CLOCK_MONOTONIC 0
-// https://stackoverflow.com/questions/5404277/porting-clock-gettime-to-windows
-struct timespec { long tv_sec; long tv_nsec; };
-int clock_gettime(int arg, struct timespec *spec) {
-    __int64 wintime;
-    GetSystemTimeAsFileTime((FILETIME*)&wintime);
-    wintime      -= 116444736000000000LL;     //1jan1601 to 1jan1970
-    spec->tv_sec  = wintime / 10000000LL;     //seconds
-    spec->tv_nsec = wintime % 10000000LL*100; //nano-seconds
-    return 0;
-}
-#else // _WIN32
-#error "Add a clock_gettime() impl for your platform!"
-#endif // _WIN32
-
-#else // NO_CLOCK_GETTIME
-
-#include <time.h>
-
-#endif // NO_CLOCK_GETTIME
-
 static unsigned long get_time(void) {
-    struct timespec ts;
+    return 0;
-    clock_gettime(CLOCK_MONOTONIC, &ts);
-    return (unsigned long)ts.tv_sec * 1000000 + ts.tv_nsec / 1000;
 }
 
 static void gfx_flush(void) {
@@ -287,11 +274,19 @@ static struct ColorCombiner *gfx_lookup_or_create_color_combiner(uint32_t cc_id)
 }
 
 static bool gfx_texture_cache_lookup(int tile, struct TextureHashmapNode **n, const uint8_t *orig_addr, uint32_t fmt, uint32_t siz) {
+    #ifdef EXTERNAL_TEXTURES // hash and compare the data (i.e. the texture name) itself
+    size_t hash = string_hash(orig_addr);
+    #define CMPADDR(x, y) (x && !sys_strcasecmp(x, y))
+    #else // hash and compare the address
     size_t hash = (uintptr_t)orig_addr;
-    hash = (hash >> 5) & 0x3ff;
+    #define CMPADDR(x, y) x == y
+    #endif
+
+    hash = (hash >> HASH_SHIFT) & HASH_MASK;
+
     struct TextureHashmapNode **node = &gfx_texture_cache.hashmap[hash];
     while (*node != NULL && *node - gfx_texture_cache.pool < gfx_texture_cache.pool_pos) {
-        if ((*node)->texture_addr == orig_addr && (*node)->fmt == fmt && (*node)->siz == siz) {
+        if (CMPADDR((*node)->texture_addr, orig_addr) && (*node)->fmt == fmt && (*node)->siz == siz) {
             gfx_rapi->select_texture(tile, (*node)->texture_id);
             *n = *node;
             return true;
@@ -302,7 +297,7 @@ static bool gfx_texture_cache_lookup(int tile, struct TextureHashmapNode **n, co
         // Pool is full. We just invalidate everything and start over.
         gfx_texture_cache.pool_pos = 0;
         node = &gfx_texture_cache.hashmap[hash];
-        //puts("Clearing texture cache");
+        // puts("Clearing texture cache");
     }
     *node = &gfx_texture_cache.pool[gfx_texture_cache.pool_pos++];
     if ((*node)->texture_addr == NULL) {
@@ -319,9 +314,11 @@ static bool gfx_texture_cache_lookup(int tile, struct TextureHashmapNode **n, co
     (*node)->siz = siz;
     *n = *node;
     return false;
+    #undef CMPADDR
 }
 
 #ifndef EXTERNAL_TEXTURES
+
 static void import_texture_rgba32(int tile) {
     uint32_t width = rdp.texture_tile.line_size_bytes / 2;
     uint32_t height = (rdp.loaded_texture[tile].size_bytes / 2) / rdp.texture_tile.line_size_bytes;
@@ -493,6 +490,89 @@ static void import_texture_ci8(int tile) {
     
     gfx_rapi->upload_texture(rgba32_buf, width, height);
 }
+
+#else // EXTERNAL_TEXTURES
+
+// this is taken straight from n64graphics
+static bool texname_to_texformat(const char *name, u8 *fmt, u8 *siz) {
+    static const struct {
+        const char *name;
+        const u8 format;
+        const u8 size;
+    } fmt_table[] = {
+        { "rgba16", G_IM_FMT_RGBA, G_IM_SIZ_16b },
+        { "rgba32", G_IM_FMT_RGBA, G_IM_SIZ_32b },
+        { "ia1",    G_IM_FMT_IA,   G_IM_SIZ_8b  }, // uhh
+        { "ia4",    G_IM_FMT_IA,   G_IM_SIZ_4b  },
+        { "ia8",    G_IM_FMT_IA,   G_IM_SIZ_8b  },
+        { "ia16",   G_IM_FMT_IA,   G_IM_SIZ_16b },
+        { "i4",     G_IM_FMT_I,    G_IM_SIZ_4b  },
+        { "i8",     G_IM_FMT_I,    G_IM_SIZ_8b  },
+        { "ci8",    G_IM_FMT_I,    G_IM_SIZ_8b  },
+        { "ci16",   G_IM_FMT_I,    G_IM_SIZ_16b },
+    };
+
+    char *fstr = strrchr(name, '.');
+    if (!fstr) return false; // no format string?
+    fstr++;
+
+    for (unsigned i = 0; i < sizeof(fmt_table) / sizeof(fmt_table[0]); ++i) {
+        if (!sys_strcasecmp(fstr, fmt_table[i].name)) {
+            *fmt = fmt_table[i].format;
+            *siz = fmt_table[i].size;
+            return true;
+        }
+    }
+
+    return false;
+}
+
+// calls import_texture() on every texture in the res folder
+// we can get the format and size from the texture files
+// and then cache them using gfx_texture_cache_lookup
+static bool preload_texture(const char *path) {
+    // strip off the extension
+    char texname[SYS_MAX_PATH];
+    strncpy(texname, path, sizeof(texname));
+    texname[sizeof(texname)-1] = 0;
+    char *dot = strrchr(texname, '.');
+    if (dot) *dot = 0;
+
+    // get the format and size from filename
+    u8 fmt, siz;
+    if (!texname_to_texformat(texname, &fmt, &siz)) {
+        fprintf(stderr, "unknown texture format: `%s`, skipping\n", texname);
+        return true; // just skip it, might be a stray skybox or something
+    }
+
+    // strip off the data path
+    const char *datapath = sys_data_path();
+    const unsigned int datalen = strlen(datapath);
+    const char *actualname = (!strncmp(texname, datapath, datalen)) ?
+        texname + datalen + 1 : texname;
+    // skip any separators
+    while (*actualname == '/' || *actualname == '\\') ++actualname;
+    // this will be stored in the hashtable, so make a copy
+    actualname = sys_strdup(actualname);
+    assert(actualname);
+
+    struct TextureHashmapNode *n;
+    if (!gfx_texture_cache_lookup(0, &n, actualname, fmt, siz)) {
+        // new texture, load it
+        int w, h;
+        u8 *data = stbi_load(path, &w, &h, NULL, 4);
+        if (!data) {
+            fprintf(stderr, "could not load texture: `%s`\n", path);
+            return false;
+        }
+        // upload it
+        gfx_rapi->upload_texture(data, w, h);
+        stbi_image_free(data);
+    }
+
+    return true;
+}
+
 #endif // EXTERNAL_TEXTURES
 
 static void import_texture(int tile) {
@@ -512,7 +592,7 @@ static void import_texture(int tile) {
     snprintf(fpath, sizeof(fpath), "%s/%s.png", sys_data_path(), texname);
     u8 *data = stbi_load(fpath, &w, &h, NULL, 4);
     if (!data) {
-        fprintf(stderr, "texture not found: `%s`\n", fpath);
+        fprintf(stderr, "could not load texture: `%s`\n", fpath);
         abort();
     }
     gfx_rapi->upload_texture(data, w, h);
@@ -1663,6 +1743,13 @@ void gfx_init(struct GfxWindowManagerAPI *wapi, struct GfxRenderingAPI *rapi) {
     for (size_t i = 0; i < sizeof(precomp_shaders) / sizeof(uint32_t); i++) {
         gfx_lookup_or_create_shader_program(precomp_shaders[i]);
     }
+    #ifdef EXTERNAL_TEXTURES
+    // preload all textures if needed
+    if (configPrecacheRes) {
+        printf("Precaching textures from `%s`\n", sys_data_path());
+        sys_dir_walk(sys_data_path(), preload_texture, true);
+    }
+    #endif
 }
 
 void gfx_start_frame(void) {

src/pc/platform.c

@@ -5,15 +5,85 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <dirent.h>
+#include <ctype.h>
 #ifdef _WIN32
 #include <direct.h>
 #endif
 
 #include "cliopts.h"
 
-static inline bool dir_exists(const char *path) {
+/* these are not available on some platforms, so might as well */
+
+char *sys_strlwr(char *src) {
+  for (unsigned char *p = (unsigned char *)src; *p; p++)
+     *p = tolower(*p);
+  return src;
+}
+
+char *sys_strdup(const char *src) {
+    const unsigned len = strlen(src) + 1;
+    char *newstr = malloc(len);
+    if (newstr) memcpy(newstr, src, len);
+    return newstr;
+}
+
+int sys_strcasecmp(const char *s1, const char *s2) {
+    const unsigned char *p1 = (const unsigned char *) s1;
+    const unsigned char *p2 = (const unsigned char *) s2;
+    int result;
+    if (p1 == p2)
+        return 0;
+    while ((result = tolower(*p1) - tolower(*p2++)) == 0)
+        if (*p1++ == '\0')
+            break;
+    return result;
+}
+
+/* file system stuff */
+
+bool sys_file_exists(const char *name) {
     struct stat st;
-    return (stat(path, &st) == 0 && S_ISDIR(st.st_mode));
+    return (stat(name, &st) == 0 && S_ISREG(st.st_mode));
+}
+
+bool sys_dir_exists(const char *name) {
+    struct stat st;
+    return (stat(name, &st) == 0 && S_ISDIR(st.st_mode));
+}
+
+bool sys_dir_walk(const char *base, walk_fn_t walk, const bool recur) {
+    char fullpath[SYS_MAX_PATH];
+    DIR *dir;
+    struct dirent *ent;
+
+    if (!(dir = opendir(base))) {
+        fprintf(stderr, "sys_dir_walk(): could not open `%s`\n", base);
+        return false;
+    }
+
+    bool ret = true;
+
+    while ((ent = readdir(dir)) != NULL) {
+        if (ent->d_name[0] == 0 || ent->d_name[0] == '.') continue; // skip ./.. and hidden files
+        snprintf(fullpath, sizeof(fullpath), "%s/%s", base, ent->d_name);
+        if (sys_dir_exists(fullpath)) {
+            if (recur) {
+                if (!sys_dir_walk(fullpath, walk, recur)) {
+                    ret = false;
+                    break;
+                }
+            }
+        } else {
+            if (!walk(fullpath)) {
+                ret = false;
+                break;
+            }
+        }
+    }
+
+    closedir(dir);
+    return ret;
 }
 
 bool sys_mkdir(const char *name) {
@@ -40,17 +110,17 @@ const char *sys_data_path(void) {
                 snprintf(path, sizeof(path), "%s%s", sys_exe_path(), gCLIOpts.DataPath + 1);
             else
                 snprintf(path, sizeof(path), "%s", gCLIOpts.DataPath);
-            if (dir_exists(path)) return path;
+            if (sys_dir_exists(path)) return path;
             printf("Warning: Specified data path ('%s') doesn't exist\n", path);
         }
 
         // then the executable directory
         snprintf(path, sizeof(path), "%s/" DATADIR, sys_exe_path());
-        if (dir_exists(path)) return path;
+        if (sys_dir_exists(path)) return path;
 
         // then the save path
         snprintf(path, sizeof(path), "%s/" DATADIR, sys_save_path());
-        if (dir_exists(path)) return path;
+        if (sys_dir_exists(path)) return path;
 
         #if defined(__linux__) || defined(__unix__)
         // on Linux/BSD try some common paths for read-only data
@@ -60,7 +130,7 @@ const char *sys_data_path(void) {
             "/opt/sm64pc/" DATADIR,
         };
         for (unsigned i = 0; i < sizeof(try) / sizeof(try[0]); ++i) {
-            if (dir_exists(try[i])) {
+            if (sys_dir_exists(try[i])) {
                 strcpy(path, try[i]);
                 return path;
             }
@@ -85,7 +155,7 @@ const char *sys_save_path(void) {
                 snprintf(path, sizeof(path), "%s%s", sys_exe_path(), gCLIOpts.SavePath + 1);
             else
                 snprintf(path, sizeof(path), "%s", gCLIOpts.SavePath);
-            if (!dir_exists(path) && !sys_mkdir(path)) {
+            if (!sys_dir_exists(path) && !sys_mkdir(path)) {
                 printf("Warning: Specified save path ('%s') doesn't exist and can't be created\n", path);
                 path[0] = 0; // doesn't exist and no write access
             }
@@ -101,7 +171,7 @@ const char *sys_save_path(void) {
                 SDL_free(sdlpath);
                 if (path[len-1] == '/' || path[len-1] == '\\')
                     path[len-1] = 0; // strip the trailing separator
-                if (!dir_exists(path) && !sys_mkdir(path))
+                if (!sys_dir_exists(path) && !sys_mkdir(path))
                     path[0] = 0;
             }
         }

src/pc/platform.h

@@ -2,13 +2,31 @@
 #define _SM64_PLATFORM_H_
 
 #include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
 
 /* Platform-specific functions and whatnot */
 
 #define DATADIR "res"
 #define SYS_MAX_PATH 1024 // FIXME: define this on different platforms
 
+// crossplatform impls of misc stuff
+char *sys_strdup(const char *src);
+char *sys_strlwr(char *src);
+int sys_strcasecmp(const char *s1, const char *s2);
+
+// filesystem stuff
 bool sys_mkdir(const char *name); // creates with 0777 by default
+bool sys_file_exists(const char *name);
+bool sys_dir_exists(const char *name);
+
+// receives the full path
+// should return `true` if traversal should continue
+typedef bool (*walk_fn_t)(const char *);
+// returns `true` if the directory was successfully opened and walk() didn't ever return false
+bool sys_dir_walk(const char *base, walk_fn_t walk, const bool recur);
+
+// path stuff
 const char *sys_data_path(void);
 const char *sys_save_path(void);
 const char *sys_exe_path(void);