etc/obs-StreamFX
0
0
Fork 0
mirror of https://github.com/Xaymar/obs-StreamFX synced 2024-12-29 11:01:23 +00:00
Code Issues Projects Releases Packages Wiki Activity

tools: Rate limit operation and fix strange buffer issues

Browse source 
Slightly relaxes the necessary amount of memory, as we are no longer loading everything all at once. Also for unknown reasons git interferes with other git processes running in the same repository only on Linux. This causes Linux runs of this tool to have strange issues detecting the authors, as git just quits with a success error code. Fun.
This commit is contained in:
Michael Fabian 'Xaymar' Dirks 2023-03-01 11:29:57 +01:00
parent a091b08259
commit c9ff7093d4
1 changed files with 401 additions and 369 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

224
tools/copyright.js
View file

@ -11,20 +11,82 @@ const OS = require("os");
const SECTION_START = "AUTOGENERATED COPYRIGHT HEADER START"; const SECTION_START = "AUTOGENERATED COPYRIGHT HEADER START";
const SECTION_END = "AUTOGENERATED COPYRIGHT HEADER END"; const SECTION_END = "AUTOGENERATED COPYRIGHT HEADER END";
const IGNORED = [ const IGNORED = [
".git", /^\.git$/gi,
"cmake/clang", /^cmake\/clang$/gi,
"cmake/version", /^cmake\/version$/gi,
"third-party", /^third-party$/gi,
] ]
let abortAllWork = false; let abortAllWork = false;
class RateLimiter {
constructor(limit = undefined) {
const OS = require("node:os");
this._limit = limit;
if (!this._limit) {
this._limit = Math.ceil(Math.max(2, OS.cpus().length / 3 * 2));
}
this._cur = this._limit;
this._pend = 0;
this._locks = [];
}
async run(runner) {
// Use Promises to spin-lock this execution path until there is a free slot.
this._pend += 1;
while (true) {
if (this._cur > 0) {
this._cur -= 1;
break;
} else {
await Promise.race(this._locks);
}
}
this._pend -= 1;
let data = {};
data.pri = new Promise((resolve, reject) => {
try {
if (runner.constructor.name == "AsyncFunction") {
runner().then((res) => {
resolve(res);
}, (err) => {
reject(err);
})
} else {
resolve(runner());
}
} catch (ex) {
reject(ex);
}
});
data.sec = data.pri.finally(() => {
// Remove this promise from the locks list.
let idx = this._locks.indexOf(data.pri);
if (idx >= 0) {
this._locks.splice(idx, 1);
}
let idx2 = this._locks.indexOf(data.sec);
if (idx2 >= 0) {
this._locks.splice(idx2, 1);
}
this._cur += 1;
//console.log(`Avail: ${this._cur} / ${this._limit}; Pending: ${this._pend}`)
});
this._locks.push(data.sec);
return await data.sec;
}
}
let gitRL = new RateLimiter(1);
let workRL = new RateLimiter();
async function isIgnored(path) { async function isIgnored(path) {
let rpath = PATH.relative(process.cwd(), path).replaceAll(PATH.sep, PATH.posix.sep); let rpath = PATH.relative(process.cwd(), path).replaceAll(PATH.sep, PATH.posix.sep);
for (let ignore of IGNORED) { for (let ignore of IGNORED) {
if (ignore instanceof RegExp) { if (ignore instanceof RegExp) {
if (ignore.global) { if (ignore.global) {
if (!rpath.matchAll(ignore).done) { let matches = rpath.matchAll(ignore);
for (let match of matches) {
return true; return true;
} }
} else { } else {
@ -37,6 +99,7 @@ async function isIgnored(path) {
} }
} }
return await gitRL.run(async () => {
return await new Promise((resolve, reject) => { return await new Promise((resolve, reject) => {
try { try {
let proc = CHILD_PROCESS.spawn("git", [ let proc = CHILD_PROCESS.spawn("git", [
@ -58,26 +121,15 @@ async function isIgnored(path) {
reject(ex); reject(ex);
} }
}); });
/* Sync alternative });
try {
return CHILD_PROCESS.spawnSync("git", [
"check-ignore",
path
], {
"cwd": PROCESS.cwd(),
"encoding": "utf8"
}).status == 0;
} catch (ex) {
return true;
}
*/
} }
async function git_retrieveAuthors(file) { async function git_retrieveAuthors(file) {
// git --no-pager log --date-order --reverse "--format=format:%aI|%aN <%aE>" -- file // git --no-pager log --date-order --reverse "--format=format:%aI|%aN <%aE>" -- file
let lines = await new Promise((resolve, reject) => { let lines = await gitRL.run(async () => {
return await new Promise((resolve, reject) => {
try { try {
let lines = ""; let chunks = [];
let proc = CHILD_PROCESS.spawn("git", [ let proc = CHILD_PROCESS.spawn("git", [
"--no-pager", "--no-pager",
"log", "log",
@ -90,19 +142,46 @@ async function git_retrieveAuthors(file) {
"cwd": PROCESS.cwd(), "cwd": PROCESS.cwd(),
"encoding": "utf8", "encoding": "utf8",
}); });
proc.stdout.on('data', (data) => { proc.stdout.on('data', (chunk) => {
lines += data.toString(); chunks.push(chunk);
}) });
proc.on('close', (code) => { proc.stdout.on('close', () => {
resolve(lines); let chunk = proc.stdout.read();
if (chunk) {
chunks.push(chunk);
}
}); });
proc.on('exit', (code) => { proc.on('exit', (code) => {
resolve(lines); // Merge all data into one buffer.
let length = 0;
for (let chunk of chunks) {
length += chunk.byteLength;
}
let buf = Buffer.alloc(length);
length = 0;
for (let chunk of chunks) {
if (!(chunk instanceof Buffer)) {
chunk = Buffer.from(chunk);
}
chunk.copy(buf, length, 0);
length += chunk.byteLength;
}
if (code == 0) {
if (buf) {
resolve(buf.toString());
} else {
reject(code);
}
} else {
reject(code);
}
}); });
} catch (ex) { } catch (ex) {
reject(ex); reject(ex);
} }
}); });
});
lines = lines.split(lines.indexOf("\r\n") >= 0 ? "\r\n" : "\n"); lines = lines.split(lines.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
let authors = new Map(); let authors = new Map();
@ -120,31 +199,6 @@ async function git_retrieveAuthors(file) {
} }
} }
return authors; return authors;
/* Sync Variant
try {
let data = await CHILD_PROCESS
let lines = data.stdout.toString().split("\n");
let authors = new Map();
for (let line of lines) {
let [date, name] = line.split("|");
let author = authors.get(name);
if (author) {
author.to = new Date(date)
} else {
authors.set(name, {
from: new Date(date),
to: new Date(date),
})
}
}
return authors;
} catch (ex) {
console.error(ex);
throw ex;
}
*/
} }
async function generateCopyright(file) { async function generateCopyright(file) {
@ -268,19 +322,20 @@ function makeHeader(file, copyright) {
throw new Error("Unrecognized file format.") throw new Error("Unrecognized file format.")
} }
async function addCopyright(file) { async function updateFile(file) {
await workRL.run(async () => {
try { try {
if (abortAllWork) { if (abortAllWork) {
return; return;
} }
// Async/Promises
// Copyright information. // Copyright information.
let copyright = await generateCopyright(file); let copyright = await generateCopyright(file);
let header = undefined; let header = undefined;
try { try {
header = makeHeader(file, copyright); header = makeHeader(file, copyright);
} catch (ex) { } catch (ex) {
console.log(`Skipping file '${file}'...`);
return; return;
} }
console.log(`Updating file '${file}'...`); console.log(`Updating file '${file}'...`);
@ -320,53 +375,24 @@ async function addCopyright(file) {
fd.write(content, 0, undefined, insert.byteLength); fd.write(content, 0, undefined, insert.byteLength);
} }
await fd.close(); await fd.close();
/* Sync variant (slow!)
let content = FS.readFileSync(file);
let eol = (content.indexOf("\r\n") != -1 ? OS.EOL : "\n");
let copyright = await generateCopyright(file);
let header = makeHeader(file, copyright);
let insert = Buffer.from(header.join(eol) + eol);
let startHeader = content.indexOf(header[0]);
let endHeader = content.indexOf(header[header.length - 1], startHeader + 1);
endHeader += header[header.length - 1].length + eol.length;
let fd = FS.openSync(file, "w+");
if ((startHeader >= 0) && (endHeader >= 0)) {
let pos = 0;
if (startHeader > 0) {
FS.writeSync(fd, content, 0, startHeader, 0);
pos += startHeader;
}
FS.writeSync(fd, insert, 0, undefined, pos);
pos += insert.byteLength;
FS.writeSync(fd, content, endHeader, undefined, pos);
} else {
FS.writeSync(fd, insert, 0, undefined, 0);
FS.writeSync(fd, content, 0, undefined, insert.byteLength);
}
FS.close(fd, (err) => {
if (err)
throw err;
})*/
} catch (ex) { } catch (ex) {
console.error(`Error processing '${file}'!: ${ex}`); console.error(`Error processing '${file}'!: ${ex}`);
abortAllWork = true;
PROCESS.exitCode = 1;
return; return;
} }
});
} }
async function addCopyrights(path) { async function scanPath(path) {
// Abort here if the user aborted the process, or if the path is ignored.
if (abortAllWork) { if (abortAllWork) {
return; return;
} }
if (await isIgnored(path)) {
return;
}
let promises = []; let promises = [];
await workRL.run(async () => {
let files = await FSPROMISES.readdir(path, { "withFileTypes": true }); let files = await FSPROMISES.readdir(path, { "withFileTypes": true });
for (let file of files) { for (let file of files) {
if (abortAllWork) { if (abortAllWork) {
@ -380,19 +406,21 @@ async function addCopyrights(path) {
} }
if (file.isDirectory()) { if (file.isDirectory()) {
//console.log(`Scanning path '${fullname}'...`); console.log(`Scanning path '${fullname}'...`);
promises.push(addCopyrights(fullname)); promises.push(scanPath(fullname));
} else { } else {
promises.push(addCopyright(fullname)); promises.push(updateFile(fullname));
} }
} }
});
await Promise.all(promises); await Promise.all(promises);
} }
(async function () { (async function () {
PROCESS.on("SIGINT", (ev) => { PROCESS.on("SIGINT", () => {
abortAllWork = true; abortAllWork = true;
PROCESS.exitCode = 1;
console.log("Sanely aborting all pending work..."); console.log("Sanely aborting all pending work...");
}) })
@ -416,11 +444,15 @@ async function addCopyrights(path) {
path = PATH.normalize(PATH.relative(process.cwd(), path)); path = PATH.normalize(PATH.relative(process.cwd(), path));
} }
let pathStat = await FSPROMISES.stat(path); if (!await isIgnored(path)) {
if (pathStat.isDirectory()) { if ((await FSPROMISES.stat(path)).isDirectory()) {
await addCopyrights(path); console.log(`Scanning path '${path}'...`);
await scanPath(path);
} else { } else {
await addCopyright(path); await updateFile(path);
}
} else {
console.log(`Ignoring path '${path}'...`);
} }
console.log("Done"); console.log("Done");
})(); })();