etc/Sharkey
0
0
Fork 0
mirror of https://activitypub.software/TransFem-org/Sharkey synced 2024-11-25 07:25:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

ファイルのダウンロードがタイムアウトしなくなっているのを修正など (#6233)

Browse source 
* Refactor download / file-info

* body read timeout on download Fix syuilo#6232
This commit is contained in:
MeiMei 2020-04-11 18:28:40 +09:00 committed by GitHub
parent 9fcf94b197
commit ca66acac2b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -102,6 +102,7 @@
"@types/websocket": "1.0.0",
"@types/ws": "7.2.3",
"@typescript-eslint/parser": "2.26.0",
"abort-controller": "3.0.0",
"animejs": "3.1.0",
"apexcharts": "3.17.1",
"autobind-decorator": "2.4.0",

48
src/misc/donwload-url.ts
View file

@ -1,55 +1,39 @@
import * as fs from 'fs';
import * as stream from 'stream';
import * as util from 'util';
import fetch from 'node-fetch';
import { httpAgent, httpsAgent } from './fetch';
import { AbortController } from 'abort-controller';
import config from '../config';
import * as chalk from 'chalk';
import Logger from '../services/logger';
const pipeline = util.promisify(stream.pipeline);
export async function downloadUrl(url: string, path: string) {
const logger = new Logger('download');
logger.info(`Downloading ${chalk.cyan(url)} ...`);
const controller = new AbortController();
setTimeout(() => {
controller.abort();
}, 11 * 1000);
const response = await fetch(new URL(url).href, {
headers: {
'User-Agent': config.userAgent
},
timeout: 10 * 1000,
signal: controller.signal,
agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent,
}).then(response => {
if (!response.ok) {
logger.error(`Got ${response.status} (${url})`);
throw response.status;
} else {
return response;
}
});
await new Promise((res, rej) => {
const writable = fs.createWriteStream(path);
if (!response.ok) {
logger.error(`Got ${response.status} (${url})`);
throw response.status;
}
response.body.on('error', (error: any) => {
logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
writable.close();
rej(error);
});
await pipeline(response.body, fs.createWriteStream(path));
writable.on('finish', () => {
logger.succ(`Download finished: ${chalk.cyan(url)}`);
res();
});
writable.on('error', error => {
logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, {
url: url,
e: error
});
rej(error);
});
response.body.pipe(writable);
});
logger.succ(`Download finished: ${chalk.cyan(url)}`);
}

29
src/misc/get-file-info.ts
View file

@ -1,10 +1,14 @@
import * as fs from 'fs';
import * as crypto from 'crypto';
import * as stream from 'stream';
import * as util from 'util';
import * as fileType from 'file-type';
import isSvg from 'is-svg';
import * as probeImageSize from 'probe-image-size';
import * as sharp from 'sharp';
const pipeline = util.promisify(stream.pipeline);
export type FileInfo = {
size: number;
md5: string;
@ -138,32 +142,17 @@ export async function checkSvg(path: string) {
* Get file size
*/
export async function getFileSize(path: string): Promise<number> {
return new Promise<number>((res, rej) => {
fs.stat(path, (err, stats) => {
if (err) return rej(err);
res(stats.size);
});
});
const getStat = util.promisify(fs.stat);
return (await getStat(path)).size;
}
/**
* Calculate MD5 hash
*/
async function calcHash(path: string): Promise<string> {
return new Promise<string>((res, rej) => {
const readable = fs.createReadStream(path);
const hash = crypto.createHash('md5');
const chunks: Buffer[] = [];
readable
.on('error', rej)
.pipe(hash)
.on('error', rej)
.on('data', chunk => chunks.push(chunk))
.on('end', () => {
const buffer = Buffer.concat(chunks);
res(buffer.toString('hex'));
});
});
const hash = crypto.createHash('md5').setEncoding('hex');
await pipeline(fs.createReadStream(path), hash);
return hash.read();
}
/**