mirror of
https://github.com/Xaymar/obs-StreamFX
synced 2024-12-28 18:41:14 +00:00
tools: Rate limit operation and fix strange buffer issues
Slightly relaxes the necessary amount of memory, as we are no longer loading everything all at once. Also for unknown reasons git interferes with other git processes running in the same repository only on Linux. This causes Linux runs of this tool to have strange issues detecting the authors, as git just quits with a success error code. Fun.
This commit is contained in:
Michael Fabian 'Xaymar' Dirks
parent
a091b08259
commit
c9ff7093d4
1 changed files with 401 additions and 369 deletions
|
|
@ -11,20 +11,82 @@ const OS = require("os");
|
|||
const SECTION_START = "AUTOGENERATED COPYRIGHT HEADER START";
|
||||
const SECTION_END = "AUTOGENERATED COPYRIGHT HEADER END";
|
||||
const IGNORED = [
|
||||
".git",
|
||||
"cmake/clang",
|
||||
"cmake/version",
|
||||
"third-party",
|
||||
/^\.git$/gi,
|
||||
/^cmake\/clang$/gi,
|
||||
/^cmake\/version$/gi,
|
||||
/^third-party$/gi,
|
||||
]
|
||||
|
||||
let abortAllWork = false;
|
||||
class RateLimiter {
|
||||
constructor(limit = undefined) {
|
||||
const OS = require("node:os");
|
||||
this._limit = limit;
|
||||
if (!this._limit) {
|
||||
this._limit = Math.ceil(Math.max(2, OS.cpus().length / 3 * 2));
|
||||
}
|
||||
this._cur = this._limit;
|
||||
this._pend = 0;
|
||||
this._locks = [];
|
||||
}
|
||||
|
||||
async run(runner) {
|
||||
// Use Promises to spin-lock this execution path until there is a free slot.
|
||||
this._pend += 1;
|
||||
while (true) {
|
||||
if (this._cur > 0) {
|
||||
this._cur -= 1;
|
||||
break;
|
||||
} else {
|
||||
await Promise.race(this._locks);
|
||||
}
|
||||
}
|
||||
this._pend -= 1;
|
||||
|
||||
let data = {};
|
||||
data.pri = new Promise((resolve, reject) => {
|
||||
try {
|
||||
if (runner.constructor.name == "AsyncFunction") {
|
||||
runner().then((res) => {
|
||||
resolve(res);
|
||||
}, (err) => {
|
||||
reject(err);
|
||||
})
|
||||
} else {
|
||||
resolve(runner());
|
||||
}
|
||||
} catch (ex) {
|
||||
reject(ex);
|
||||
}
|
||||
});
|
||||
data.sec = data.pri.finally(() => {
|
||||
// Remove this promise from the locks list.
|
||||
let idx = this._locks.indexOf(data.pri);
|
||||
if (idx >= 0) {
|
||||
this._locks.splice(idx, 1);
|
||||
}
|
||||
let idx2 = this._locks.indexOf(data.sec);
|
||||
if (idx2 >= 0) {
|
||||
this._locks.splice(idx2, 1);
|
||||
}
|
||||
this._cur += 1;
|
||||
//console.log(`Avail: ${this._cur} / ${this._limit}; Pending: ${this._pend}`)
|
||||
});
|
||||
this._locks.push(data.sec);
|
||||
return await data.sec;
|
||||
}
|
||||
}
|
||||
|
||||
let gitRL = new RateLimiter(1);
|
||||
let workRL = new RateLimiter();
|
||||
|
||||
async function isIgnored(path) {
|
||||
let rpath = PATH.relative(process.cwd(), path).replaceAll(PATH.sep, PATH.posix.sep);
|
||||
for (let ignore of IGNORED) {
|
||||
if (ignore instanceof RegExp) {
|
||||
if (ignore.global) {
|
||||
if (!rpath.matchAll(ignore).done) {
|
||||
let matches = rpath.matchAll(ignore);
|
||||
for (let match of matches) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -37,6 +99,7 @@ async function isIgnored(path) {
|
|||
}
|
||||
}
|
||||
|
||||
return await gitRL.run(async () => {
|
||||
return await new Promise((resolve, reject) => {
|
||||
try {
|
||||
let proc = CHILD_PROCESS.spawn("git", [
|
||||
|
|
@ -58,26 +121,15 @@ async function isIgnored(path) {
|
|||
reject(ex);
|
||||
}
|
||||
});
|
||||
/* Sync alternative
|
||||
try {
|
||||
return CHILD_PROCESS.spawnSync("git", [
|
||||
"check-ignore",
|
||||
path
|
||||
], {
|
||||
"cwd": PROCESS.cwd(),
|
||||
"encoding": "utf8"
|
||||
}).status == 0;
|
||||
} catch (ex) {
|
||||
return true;
|
||||
}
|
||||
*/
|
||||
});
|
||||
}
|
||||
|
||||
async function git_retrieveAuthors(file) {
|
||||
// git --no-pager log --date-order --reverse "--format=format:%aI|%aN <%aE>" -- file
|
||||
let lines = await new Promise((resolve, reject) => {
|
||||
let lines = await gitRL.run(async () => {
|
||||
return await new Promise((resolve, reject) => {
|
||||
try {
|
||||
let lines = "";
|
||||
let chunks = [];
|
||||
let proc = CHILD_PROCESS.spawn("git", [
|
||||
"--no-pager",
|
||||
"log",
|
||||
|
|
@ -90,19 +142,46 @@ async function git_retrieveAuthors(file) {
|
|||
"cwd": PROCESS.cwd(),
|
||||
"encoding": "utf8",
|
||||
});
|
||||
proc.stdout.on('data', (data) => {
|
||||
lines += data.toString();
|
||||
})
|
||||
proc.on('close', (code) => {
|
||||
resolve(lines);
|
||||
proc.stdout.on('data', (chunk) => {
|
||||
chunks.push(chunk);
|
||||
});
|
||||
proc.stdout.on('close', () => {
|
||||
let chunk = proc.stdout.read();
|
||||
if (chunk) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
});
|
||||
proc.on('exit', (code) => {
|
||||
resolve(lines);
|
||||
// Merge all data into one buffer.
|
||||
let length = 0;
|
||||
for (let chunk of chunks) {
|
||||
length += chunk.byteLength;
|
||||
}
|
||||
let buf = Buffer.alloc(length);
|
||||
length = 0;
|
||||
for (let chunk of chunks) {
|
||||
if (!(chunk instanceof Buffer)) {
|
||||
chunk = Buffer.from(chunk);
|
||||
}
|
||||
chunk.copy(buf, length, 0);
|
||||
length += chunk.byteLength;
|
||||
}
|
||||
|
||||
if (code == 0) {
|
||||
if (buf) {
|
||||
resolve(buf.toString());
|
||||
} else {
|
||||
reject(code);
|
||||
}
|
||||
} else {
|
||||
reject(code);
|
||||
}
|
||||
});
|
||||
} catch (ex) {
|
||||
reject(ex);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
lines = lines.split(lines.indexOf("\r\n") >= 0 ? "\r\n" : "\n");
|
||||
let authors = new Map();
|
||||
|
|
@ -120,31 +199,6 @@ async function git_retrieveAuthors(file) {
|
|||
}
|
||||
}
|
||||
return authors;
|
||||
|
||||
/* Sync Variant
|
||||
try {
|
||||
let data = await CHILD_PROCESS
|
||||
let lines = data.stdout.toString().split("\n");
|
||||
let authors = new Map();
|
||||
for (let line of lines) {
|
||||
let [date, name] = line.split("|");
|
||||
|
||||
let author = authors.get(name);
|
||||
if (author) {
|
||||
author.to = new Date(date)
|
||||
} else {
|
||||
authors.set(name, {
|
||||
from: new Date(date),
|
||||
to: new Date(date),
|
||||
})
|
||||
}
|
||||
}
|
||||
return authors;
|
||||
} catch (ex) {
|
||||
console.error(ex);
|
||||
throw ex;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
async function generateCopyright(file) {
|
||||
|
|
@ -268,19 +322,20 @@ function makeHeader(file, copyright) {
|
|||
throw new Error("Unrecognized file format.")
|
||||
}
|
||||
|
||||
async function addCopyright(file) {
|
||||
async function updateFile(file) {
|
||||
await workRL.run(async () => {
|
||||
try {
|
||||
if (abortAllWork) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Async/Promises
|
||||
// Copyright information.
|
||||
let copyright = await generateCopyright(file);
|
||||
let header = undefined;
|
||||
try {
|
||||
header = makeHeader(file, copyright);
|
||||
} catch (ex) {
|
||||
console.log(`Skipping file '${file}'...`);
|
||||
return;
|
||||
}
|
||||
console.log(`Updating file '${file}'...`);
|
||||
|
|
@ -320,53 +375,24 @@ async function addCopyright(file) {
|
|||
fd.write(content, 0, undefined, insert.byteLength);
|
||||
}
|
||||
await fd.close();
|
||||
|
||||
/* Sync variant (slow!)
|
||||
let content = FS.readFileSync(file);
|
||||
let eol = (content.indexOf("\r\n") != -1 ? OS.EOL : "\n");
|
||||
|
||||
let copyright = await generateCopyright(file);
|
||||
let header = makeHeader(file, copyright);
|
||||
let insert = Buffer.from(header.join(eol) + eol);
|
||||
|
||||
let startHeader = content.indexOf(header[0]);
|
||||
let endHeader = content.indexOf(header[header.length - 1], startHeader + 1);
|
||||
endHeader += header[header.length - 1].length + eol.length;
|
||||
|
||||
let fd = FS.openSync(file, "w+");
|
||||
if ((startHeader >= 0) && (endHeader >= 0)) {
|
||||
let pos = 0;
|
||||
if (startHeader > 0) {
|
||||
FS.writeSync(fd, content, 0, startHeader, 0);
|
||||
pos += startHeader;
|
||||
}
|
||||
FS.writeSync(fd, insert, 0, undefined, pos);
|
||||
pos += insert.byteLength;
|
||||
FS.writeSync(fd, content, endHeader, undefined, pos);
|
||||
} else {
|
||||
FS.writeSync(fd, insert, 0, undefined, 0);
|
||||
FS.writeSync(fd, content, 0, undefined, insert.byteLength);
|
||||
}
|
||||
FS.close(fd, (err) => {
|
||||
if (err)
|
||||
throw err;
|
||||
})*/
|
||||
} catch (ex) {
|
||||
console.error(`Error processing '${file}'!: ${ex}`);
|
||||
abortAllWork = true;
|
||||
PROCESS.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function addCopyrights(path) {
|
||||
async function scanPath(path) {
|
||||
// Abort here if the user aborted the process, or if the path is ignored.
|
||||
if (abortAllWork) {
|
||||
return;
|
||||
}
|
||||
if (await isIgnored(path)) {
|
||||
return;
|
||||
}
|
||||
|
||||
let promises = [];
|
||||
|
||||
await workRL.run(async () => {
|
||||
let files = await FSPROMISES.readdir(path, { "withFileTypes": true });
|
||||
for (let file of files) {
|
||||
if (abortAllWork) {
|
||||
|
|
@ -380,19 +406,21 @@ async function addCopyrights(path) {
|
|||
}
|
||||
|
||||
if (file.isDirectory()) {
|
||||
//console.log(`Scanning path '${fullname}'...`);
|
||||
promises.push(addCopyrights(fullname));
|
||||
console.log(`Scanning path '${fullname}'...`);
|
||||
promises.push(scanPath(fullname));
|
||||
} else {
|
||||
promises.push(addCopyright(fullname));
|
||||
promises.push(updateFile(fullname));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
(async function () {
|
||||
PROCESS.on("SIGINT", (ev) => {
|
||||
PROCESS.on("SIGINT", () => {
|
||||
abortAllWork = true;
|
||||
PROCESS.exitCode = 1;
|
||||
console.log("Sanely aborting all pending work...");
|
||||
})
|
||||
|
||||
|
|
@ -416,11 +444,15 @@ async function addCopyrights(path) {
|
|||
path = PATH.normalize(PATH.relative(process.cwd(), path));
|
||||
}
|
||||
|
||||
let pathStat = await FSPROMISES.stat(path);
|
||||
if (pathStat.isDirectory()) {
|
||||
await addCopyrights(path);
|
||||
if (!await isIgnored(path)) {
|
||||
if ((await FSPROMISES.stat(path)).isDirectory()) {
|
||||
console.log(`Scanning path '${path}'...`);
|
||||
await scanPath(path);
|
||||
} else {
|
||||
await addCopyright(path);
|
||||
await updateFile(path);
|
||||
}
|
||||
} else {
|
||||
console.log(`Ignoring path '${path}'...`);
|
||||
}
|
||||
console.log("Done");
|
||||
})();
|
||||
|
|
|
|||
Loading…
Reference in a new issue