Made downloads significantly faster

This commit is contained in:
MysterD 2022-04-10 21:53:58 -07:00
parent b60b5e3c07
commit f8ffc23422
9 changed files with 306 additions and 188 deletions

View file

@ -16,10 +16,10 @@ if [ ! -f "$FILE" ]; then
fi
# no debug, direct
#$FILE --server 27015 --configfile sm64config_server.txt &
#sleep 2
#$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt &
#exit
$FILE --server 27015 --configfile sm64config_server.txt &
sleep 2
$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt &
exit
# no debug, discord
#$FILE --discord 2 --configfile sm64config_server.txt &
@ -27,11 +27,11 @@ fi
#exit
# debug on server
$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt & > /dev/null
$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 27015 --configfile sm64config_server.txt' -ex 'quit'
exit
#$FILE --client 127.0.0.1 27015 --configfile sm64config_client.txt & > /dev/null
#$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --server 27015 --configfile sm64config_server.txt' -ex 'quit'
#exit
# debug on client
$FILE --server 27015 --configfile sm64config_server.txt & > /dev/null
$WINPTY cgdb $FILE -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 27015 --configfile sm64config_client.txt' -ex 'quit'
$WINPTY cgdb $FILE -ex 'network_receive_download' -ex 'break debug_breakpoint_here' -ex 'run --client 127.0.0.1 27015 --configfile sm64config_client.txt' -ex 'quit'
exit

View file

@ -115,6 +115,7 @@ void smlua_audio_utils_replace_sequence(u8 sequenceId, u8 bankId, u8 defaultVolu
struct AudioOverride* override = &sAudioOverrides[sequenceId];
smlua_audio_utils_reset(override);
LOG_INFO("Loading audio: %s", fullPath);
override->filename = strdup(fullPath);
override->enabled = true;
override->bank = bankId;

View file

@ -61,11 +61,13 @@ void mods_activate(struct Mods* mods) {
}
// copy enabled entries
gActiveMods.size = 0;
for (int i = 0; i < mods->entryCount; i++) {
struct Mod* mod = mods->entries[i];
if (mod->enabled) {
mod->index = gActiveMods.entryCount;
gActiveMods.entries[gActiveMods.entryCount++] = mod;
gActiveMods.size += mod->size;
mod_activate(mod);
}
}

View file

@ -131,7 +131,10 @@ bool mod_file_create_directories(struct Mod* mod, struct ModFile* modFile) {
while (*p != '\0') {
if (*p == '/' || *p == '\\') {
if (snprintf(tmpPath, index + 1, "%s", path) < 0) { }
if (!fs_sys_dir_exists(tmpPath)) { fs_sys_mkdir(tmpPath); }
if (!fs_sys_dir_exists(tmpPath)) {
fs_sys_mkdir(tmpPath);
LOG_INFO("Creating mod path: %s", tmpPath);
}
}
index++;
p++;

View file

@ -120,6 +120,11 @@ bool network_init(enum NetworkType inNetworkType) {
extern u8* gOverrideEeprom;
gOverrideEeprom = NULL;
if (gCurrLevelNum != (s16)gLevelValues.entryLevel) {
extern s16 gChangeLevelTransition;
gChangeLevelTransition = gLevelValues.entryLevel;
}
djui_chat_box_create();
}

View file

@ -331,9 +331,9 @@ void network_receive_mod_list(struct Packet* p);
// packet_download.c
void network_start_download_requests(void);
void network_send_next_download_request(void);
void network_send_download_request(u16 clientIndex, u16 serverIndex, u64 offset);
void network_send_download_request(u64 offset);
void network_receive_download_request(struct Packet* p);
void network_send_download(u16 clientIndex, u16 serverIndex, u64 offset);
void network_send_download(u64 offset);
void network_receive_download(struct Packet* p);
// packet_lua_sync_table.c

View file

@ -3,152 +3,263 @@
#include "pc/djui/djui.h"
#include "pc/mods/mods.h"
#include "pc/mods/mods_utils.h"
#define DISABLE_MODULE_LOG 1
#include "pc/debuglog.h"
#define CHUNK_SIZE 400
#define OFFSET_COUNT 40
#define CHUNK_SIZE 800
#define OFFSET_COUNT 50
#define GROUP_SIZE (CHUNK_SIZE * OFFSET_COUNT)
struct OffsetGroup {
u64 offset[OFFSET_COUNT];
bool rx[OFFSET_COUNT];
bool active;
};
static struct OffsetGroup sOffsetGroup[2] = { 0 };
static bool* sOffsetGroupsCompleted = NULL;
static u64 sOffsetGroupCount = 0;
static u64 sOffset[OFFSET_COUNT] = { 0 };
static bool sWaitingForOffset[OFFSET_COUNT] = { 0 };
u64 sTotalDownloadBytes = 0;
extern float gDownloadProgress;
static void network_start_offset_group(struct OffsetGroup* og);
static void network_update_offset_groups(void);
void network_start_download_requests(void) {
sTotalDownloadBytes = 0;
gDownloadProgress = 0;
for (s32 i = 0; i < gRemoteMods.entryCount; i++) {
struct Mod* mod = gRemoteMods.entries[i];
mod->enabled = true;
for (s32 j = 0; j < mod->fileCount; j++) {
struct ModFile* file = &mod->files[j];
char fullPath[SYS_MAX_PATH] = { 0 };
if (!mod_file_full_path(fullPath, mod, file)) {
LOG_ERROR("unable to concat full path!");
return;
}
mod_file_create_directories(mod, file);
file->fp = fopen(fullPath, "wb");
if (file->fp == NULL) {
LOG_ERROR("unable to open for write: '%s'", fullPath);
return;
sOffsetGroupCount = (gRemoteMods.size / GROUP_SIZE) + 1;
if (sOffsetGroupsCompleted != NULL) {
free(sOffsetGroupsCompleted);
}
sOffsetGroupsCompleted = calloc(sOffsetGroupCount, sizeof(bool));
memset(&sOffsetGroup[0], 0, sizeof(struct OffsetGroup));
memset(&sOffsetGroup[1], 0, sizeof(struct OffsetGroup));
network_update_offset_groups();
}
static void network_start_offset_group(struct OffsetGroup* og) {
// sanity check
if (og->active) {
for (u32 i = 0; i < OFFSET_COUNT; i++) {
assert(og->rx[i]);
}
}
// figure out the starting offset
bool foundIndex = false;
u64 offset = 0;
for (u32 i = 0; i < sOffsetGroupCount; i++) {
// skip this offset if its in progress
struct OffsetGroup* otherOg = (og == &sOffsetGroup[0])
? &sOffsetGroup[1]
: &sOffsetGroup[0];
if (otherOg->active && otherOg->offset[0] == (i * GROUP_SIZE)) {
continue;
}
if (!sOffsetGroupsCompleted[i]) {
offset = (i * GROUP_SIZE);
foundIndex = true;
break;
}
}
// sanity check
if (!foundIndex) {
LOG_INFO("Could not find offset group, may be near the end of the download");
return;
}
// set up offset group
for (u64 i = 0; i < OFFSET_COUNT; i++) {
og->offset[i] = offset + (i * CHUNK_SIZE);
og->rx[i] = (og->offset[i] >= gRemoteMods.size);
}
og->active = true;
// send download request
network_send_download_request(og->offset[0]);
}
static void network_update_offset_groups(void) {
SOFT_ASSERT(gNetworkType == NT_CLIENT);
// if no groups are active, start one
if (!sOffsetGroup[0].active && !sOffsetGroup[1].active) {
network_start_offset_group(&sOffsetGroup[0]);
return;
}
// figure out group progress
u32 groupProgress[2] = { 0 };
for (u32 i = 0; i < 2; i++) {
struct OffsetGroup* og = &sOffsetGroup[i];
// count how many chunks were received
for (u32 j = 0; j < OFFSET_COUNT; j++) {
if (og->rx[j]) { groupProgress[i]++; }
}
// mark finished if finished
if (groupProgress[i] >= OFFSET_COUNT) {
u64 groupIndex = (og->offset[0] / GROUP_SIZE);
if (!sOffsetGroupsCompleted[groupIndex]) {
LOG_INFO("Completed group: %llu [ %llu <---> %llu ]", groupIndex, og->offset[0], og->offset[0] + GROUP_SIZE);
sOffsetGroupsCompleted[groupIndex] = true;
}
}
}
network_send_next_download_request();
}
void network_send_next_download_request(void) {
SOFT_ASSERT(gNetworkType == NT_CLIENT);
for (s32 i = 0; i < gRemoteMods.entryCount; i++) {
struct Mod* mod = gRemoteMods.entries[i];
for (s32 j = 0; j < mod->fileCount; j++) {
struct ModFile* file = &mod->files[j];
if (file->complete) { continue; }
//LOG_INFO("sending download request: %d, %d, %lld", i, file->remoteIndex, file->curOffset);
network_send_download_request(i, j, file->curOffset);
// if all chunks were received, we're finished
bool completedDownload = true;
for (u64 i = 0; i < sOffsetGroupCount; i++) {
if (!sOffsetGroupsCompleted[i]) {
completedDownload = false;
break;
}
}
if (completedDownload) {
// close and flush all file pointers
for (u64 modIndex = 0; modIndex < gRemoteMods.entryCount; modIndex++) {
struct Mod* mod = gRemoteMods.entries[modIndex];
for (u64 fileIndex = 0; fileIndex < mod->fileCount; fileIndex++) {
struct ModFile* modFile = &mod->files[fileIndex];
if (modFile->fp == NULL) { continue; }
fflush(modFile->fp);
}
}
LOG_INFO("Download complete!");
network_send_join_request();
return;
}
// if one group is more than half complete, and the other group is complete, start the other group
for (u32 i = 0; i < 2; i++) {
u32 o = (i + 1) % 2;
struct OffsetGroup* otherOg = &sOffsetGroup[o];
if ((groupProgress[i] >= (OFFSET_COUNT/2)) && ((groupProgress[o] >= OFFSET_COUNT) || !otherOg->active)) {
network_start_offset_group(otherOg);
return;
}
}
network_send_join_request();
}
void network_send_download_request(u16 modIndex, u16 fileIndex, u64 offset) {
void network_send_download_request(u64 offset) {
SOFT_ASSERT(gNetworkType == NT_CLIENT);
struct Packet p = { 0 };
packet_init(&p, PACKET_DOWNLOAD_REQUEST, true, PLMT_NONE);
packet_write(&p, &modIndex, sizeof(u16));
packet_write(&p, &fileIndex, sizeof(u16));
packet_write(&p, &offset, sizeof(u64));
struct Mod* mod = gRemoteMods.entries[modIndex];
struct ModFile* file = &mod->files[fileIndex];
for (s32 i = 0; i < OFFSET_COUNT; i++) {
sOffset[i] = offset + CHUNK_SIZE * i;
sWaitingForOffset[i] = (sOffset[i] < file->size);
}
network_send_to((gNetworkPlayerServer != NULL) ? gNetworkPlayerServer->localIndex : 0, &p);
LOG_INFO("Requesting group: %llu [ %llu <---> %llu ]", (offset / GROUP_SIZE), offset, offset + GROUP_SIZE);
}
void network_receive_download_request(struct Packet* p) {
SOFT_ASSERT(gNetworkType == NT_SERVER);
u16 modIndex;
u16 fileIndex;
u64 offset;
packet_read(p, &modIndex, sizeof(u16));
packet_read(p, &fileIndex, sizeof(u16));
packet_read(p, &offset, sizeof(u64));
// receive requested offset
u64 requestOffset;
packet_read(p, &requestOffset, sizeof(u64));
if (modIndex >= gActiveMods.entryCount) {
LOG_ERROR("Requested download of invalid mod index %u:%llu", modIndex, offset);
return;
for (u64 i = 0; i < OFFSET_COUNT; i++) {
u64 sendOffset = requestOffset + (i * CHUNK_SIZE);
if (sendOffset >= gActiveMods.size) {
break;
}
network_send_download(sendOffset);
}
struct Mod* mod = gActiveMods.entries[modIndex];
if (fileIndex >= mod->fileCount) {
LOG_ERROR("Requested download of invalid file index %u:%llu", fileIndex, offset);
return;
}
struct ModFile* file = &mod->files[fileIndex];
for (s32 i = 0; i < OFFSET_COUNT; i++) {
u64 o = offset + CHUNK_SIZE * i;
if (o >= file->size) { break; }
network_send_download(modIndex, fileIndex, o);
}
LOG_INFO("Sending group: %llu [ %llu <---> %llu ]", (requestOffset / GROUP_SIZE), requestOffset, requestOffset + GROUP_SIZE);
}
void network_send_download(u16 modIndex, u16 fileIndex, u64 offset) {
SOFT_ASSERT(gNetworkType == NT_SERVER);
void network_send_download(u64 requestOffset) {
u8 chunk[CHUNK_SIZE] = { 0 };
u64 chunkFill = 0;
u64 fileStartOffset = 0;
if (modIndex >= gActiveMods.entryCount) {
LOG_ERROR("Requested download of invalid mod index %u:%llu", modIndex, offset);
return;
// fill up chunk
for (u64 modIndex = 0; modIndex < gActiveMods.entryCount; modIndex++) {
struct Mod* mod = gActiveMods.entries[modIndex];
// skip past mods to get to the right offset
if ((fileStartOffset + mod->size) < requestOffset) {
fileStartOffset += mod->size;
continue;
}
for (u64 fileIndex = 0; fileIndex < mod->fileCount; fileIndex++) {
struct ModFile* modFile = &mod->files[fileIndex];
// skip past mod files to get to the right offset
if ((fileStartOffset + modFile->size) < requestOffset) {
fileStartOffset += modFile->size;
continue;
}
// calculate file offset and read length
u64 fileReadOffset = MAX(((s64)requestOffset - (s64)fileStartOffset), 0);
u64 fileReadLength = MIN((modFile->size - fileReadOffset), (CHUNK_SIZE - chunkFill));
// read from file, filling chunk
fseek(modFile->fp, fileReadOffset, SEEK_SET);
fread(&chunk[chunkFill], sizeof(u8), fileReadLength, modFile->fp);
// increment counters
chunkFill += fileReadLength;
fileStartOffset += modFile->size;
// check if we've filled the chunk
if (chunkFill >= CHUNK_SIZE) {
goto after_filled;
}
}
}
after_filled:;
struct Mod* mod = gActiveMods.entries[modIndex];
if (fileIndex >= mod->fileCount) {
LOG_ERROR("Requested download of invalid file index %u:%llu", fileIndex, offset);
return;
}
struct ModFile* file = &mod->files[fileIndex];
if (offset >= file->size) {
LOG_ERROR("Requested download of invalid offset %u:%llu", modIndex, offset);
return;
}
if (file->fp == NULL) {
LOG_ERROR("Requested download of invalid file pointer %u:%llu", modIndex, offset);
return;
}
u16 chunkSize = 400;
if ((offset + chunkSize) > file->size) {
chunkSize = file->size - offset;
}
u8 chunk[400] = { 0 };
fseek(file->fp, offset, SEEK_SET);
fread(chunk, chunkSize, 1, file->fp);
// send the packet
struct Packet p = { 0 };
packet_init(&p, PACKET_DOWNLOAD, true, PLMT_NONE);
packet_write(&p, &modIndex, sizeof(u16));
packet_write(&p, &fileIndex, sizeof(u16));
packet_write(&p, &offset, sizeof(u64));
packet_write(&p, &chunkSize, sizeof(u16));
packet_write(&p, chunk, chunkSize * sizeof(u8));
packet_write(&p, &requestOffset, sizeof(u64));
packet_write(&p, &chunkFill, sizeof(u64));
packet_write(&p, &chunk, sizeof(u8) * chunkFill);
network_send_to(0, &p);
LOG_INFO("Sent chunk: offset %llu, length %llu", requestOffset, chunkFill);
}
static void open_mod_file(struct Mod* mod, struct ModFile* file) {
if (file->fp != NULL) {
return;
}
char fullPath[SYS_MAX_PATH] = { 0 };
if (!mod_file_full_path(fullPath, mod, file)) {
LOG_ERROR("unable to concat full path!");
return;
}
mod_file_create_directories(mod, file);
file->fp = fopen(fullPath, "wb");
if (file->fp == NULL) {
LOG_ERROR("unable to open for write: '%s'", fullPath);
return;
}
LOG_INFO("Opened mod file pointer: %s", fullPath);
mod->enabled = true;
}
void network_receive_download(struct Packet* p) {
@ -160,84 +271,78 @@ void network_receive_download(struct Packet* p) {
}
}
u16 modIndex;
u16 fileIndex;
u64 offset;
u16 chunkSize;
u8 chunk[400] = { 0 };
// read the chunk
u64 receiveOffset = 0;
u64 chunkLength = 0;
u8 chunk[CHUNK_SIZE] = { 0 };
packet_read(p, &receiveOffset, sizeof(u64));
packet_read(p, &chunkLength, sizeof(u64));
packet_read(p, &chunk, sizeof(u8) * chunkLength);
packet_read(p, &modIndex, sizeof(u16));
packet_read(p, &fileIndex, sizeof(u16));
packet_read(p, &offset, sizeof(u64));
packet_read(p, &chunkSize, sizeof(u16));
packet_read(p, chunk, chunkSize * sizeof(u8));
//LOG_INFO("Received download %u:%llu", clientIndex, offset);
if (modIndex >= gRemoteMods.entryCount) {
LOG_ERROR("Received download of invalid mod index %u:%llu", modIndex, offset);
return;
}
struct Mod* mod = gRemoteMods.entries[modIndex];
if (fileIndex >= mod->fileCount) {
LOG_ERROR("Received download of invalid file index %u:%llu", modIndex, offset);
return;
}
struct ModFile* file = &mod->files[fileIndex];
if (offset >= file->size) {
LOG_ERROR("Received download of invalid offset %u:%llu", modIndex, offset);
return;
}
if (file->fp == NULL) {
LOG_ERROR("Received download of invalid file pointer %u:%llu", modIndex, offset);
return;
}
if ((offset + chunkSize) > file->size) {
LOG_ERROR("Received download of invalid chunk size %u:%llu:%u -- %llu", modIndex, (u64)offset, chunkSize, (u64)file->size);
return;
}
// check if we're still waiting for chunks
bool found = false;
bool waiting = false;
for (s32 i = 0; i < OFFSET_COUNT; i++) {
if (sOffset[i] == offset) {
found = sWaitingForOffset[i];
sWaitingForOffset[i] = false;
// mark the offset group as received
for (u64 i = 0; i < 2; i++) {
struct OffsetGroup* og = &sOffsetGroup[i];
for (u64 j = 0; j < OFFSET_COUNT; j++) {
if (og->offset[j] != receiveOffset) {
continue;
}
if (og->rx[j]) {
LOG_INFO("Received duplicate chunk: %llu", receiveOffset);
return;
}
og->rx[j] = true;
goto after_group;
}
waiting = waiting || sWaitingForOffset[i];
}
after_group:;
if (!found) {
LOG_ERROR("Received download of unexpected offset [ %llu <-> %llu ] != %llu", file->curOffset, file->curOffset + CHUNK_SIZE * OFFSET_COUNT, offset);
return;
// write the chunk
u64 chunkPour = 0;
u64 fileStartOffset = 0;
for (u64 modIndex = 0; modIndex < gRemoteMods.entryCount; modIndex++) {
struct Mod* mod = gRemoteMods.entries[modIndex];
// skip past mods to get to the right offset
if ((fileStartOffset + mod->size) < receiveOffset) {
fileStartOffset += mod->size;
continue;
}
for (u64 fileIndex = 0; fileIndex < mod->fileCount; fileIndex++) {
struct ModFile* modFile = &mod->files[fileIndex];
// skip past mod files to get to the right offset
if ((fileStartOffset + modFile->size) < receiveOffset) {
fileStartOffset += modFile->size;
continue;
}
// calculate file offset and read length
u64 fileWriteOffset = MAX(((s64)receiveOffset - (s64)fileStartOffset), 0);
u64 fileWriteLength = MIN((modFile->size - fileWriteOffset), (chunkLength - chunkPour));
// read from file, filling chunk
open_mod_file(mod, modFile);
fseek(modFile->fp, fileWriteOffset, SEEK_SET);
fwrite(&chunk[chunkPour], sizeof(u8), fileWriteLength, modFile->fp);
// increment counters
chunkPour += fileWriteLength;
fileStartOffset += modFile->size;
// check if we've filled the chunk
if (chunkPour >= CHUNK_SIZE) {
goto after_poured;
}
}
}
after_poured:;
// write to the file
fseek(file->fp, offset, SEEK_SET);
fwrite(chunk, sizeof(u8) * chunkSize, 1, file->fp);
LOG_INFO("Received chunk: offset %llu, size %llu", receiveOffset, chunkLength);
// update progress
sTotalDownloadBytes += chunkSize;
sTotalDownloadBytes += chunkLength;
gDownloadProgress = (float)sTotalDownloadBytes / (float)gRemoteMods.size;
if (!waiting) {
// check if we're finished with this file
//LOG_INFO("Checking download of '%s': %lld, %lld", file->name, sOffset[OFFSET_COUNT - 1] + CHUNK_SIZE, file->size);
if (sOffset[OFFSET_COUNT - 1] + CHUNK_SIZE >= file->size) {
LOG_INFO("Finished download of '%s'", file->relativePath);
fclose(file->fp);
file->fp = NULL;
file->complete = true;
}
file->curOffset += (u64)CHUNK_SIZE * OFFSET_COUNT;
network_send_next_download_request();
}
network_update_offset_groups();
}

View file

@ -230,8 +230,6 @@ void network_receive_join(struct Packet* p) {
update_all_mario_stars();
fake_lvl_init_from_save_file();
extern s16 gChangeLevel;
gChangeLevel = gLevelValues.entryLevel;
mods_activate(&gRemoteMods);
djui_panel_modlist_create(NULL);
@ -239,4 +237,7 @@ void network_receive_join(struct Packet* p) {
network_send_network_players_request();
network_send_lua_sync_table_request();
extern s16 gChangeLevel;
gChangeLevel = gLevelValues.entryLevel;
}

View file

@ -170,6 +170,7 @@ void network_receive_mod_list(struct Packet* p) {
packet_read(p, &relativePathLength, sizeof(u16));
packet_read(p, file->relativePath, relativePathLength * sizeof(u8));
packet_read(p, &file->size, sizeof(u64));
file->fp = NULL;
if (mod->isDirectory && !strstr(file->relativePath, "actors") && !strstr(file->relativePath, "levels") && !strstr(file->relativePath, "sound")) {
char tmp[SYS_MAX_PATH];
if (snprintf(tmp, SYS_MAX_PATH, "%s-%s", mod->relativePath, file->relativePath) >= 0) {