mirror of
https://activitypub.software/TransFem-org/Sharkey
synced 2024-11-25 07:25:12 +00:00
Tune download (#2)
* s2-2 * allowedPrivateNetworks * style * Proxyの間にあると誤解しそうなのでconfigの記述順を変更 * Fix error handler
This commit is contained in:
parent
7e30910ab8
commit
e1a8b158e0
7 changed files with 106 additions and 23 deletions
|
@ -157,3 +157,7 @@ id: 'aid'
|
||||||
|
|
||||||
# Sign to ActivityPub GET request (default: false)
|
# Sign to ActivityPub GET request (default: false)
|
||||||
#signToActivityPubGet: true
|
#signToActivityPubGet: true
|
||||||
|
|
||||||
|
#allowedPrivateNetworks: [
|
||||||
|
# '127.0.0.1/32'
|
||||||
|
#]
|
||||||
|
|
|
@ -148,6 +148,7 @@
|
||||||
"http-signature": "1.3.5",
|
"http-signature": "1.3.5",
|
||||||
"idb-keyval": "5.1.3",
|
"idb-keyval": "5.1.3",
|
||||||
"insert-text-at-cursor": "0.3.0",
|
"insert-text-at-cursor": "0.3.0",
|
||||||
|
"ip-cidr": "3.0.4",
|
||||||
"is-svg": "4.3.1",
|
"is-svg": "4.3.1",
|
||||||
"js-yaml": "4.1.0",
|
"js-yaml": "4.1.0",
|
||||||
"jsdom": "16.7.0",
|
"jsdom": "16.7.0",
|
||||||
|
@ -184,6 +185,7 @@
|
||||||
"postcss": "8.3.6",
|
"postcss": "8.3.6",
|
||||||
"postcss-loader": "6.1.1",
|
"postcss-loader": "6.1.1",
|
||||||
"prismjs": "1.24.1",
|
"prismjs": "1.24.1",
|
||||||
|
"private-ip": "2.2.1",
|
||||||
"probe-image-size": "7.2.1",
|
"probe-image-size": "7.2.1",
|
||||||
"promise-limit": "2.7.0",
|
"promise-limit": "2.7.0",
|
||||||
"pug": "3.0.2",
|
"pug": "3.0.2",
|
||||||
|
|
|
@ -37,6 +37,8 @@ export type Source = {
|
||||||
proxySmtp?: string;
|
proxySmtp?: string;
|
||||||
proxyBypassHosts?: string[];
|
proxyBypassHosts?: string[];
|
||||||
|
|
||||||
|
allowedPrivateNetworks?: string[];
|
||||||
|
|
||||||
accesslog?: string;
|
accesslog?: string;
|
||||||
|
|
||||||
clusterLimit?: number;
|
clusterLimit?: number;
|
||||||
|
|
|
@ -1,13 +1,13 @@
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import * as stream from 'stream';
|
import * as stream from 'stream';
|
||||||
import * as util from 'util';
|
import * as util from 'util';
|
||||||
import { URL } from 'url';
|
import got, * as Got from 'got';
|
||||||
import fetch from 'node-fetch';
|
import { httpAgent, httpsAgent } from './fetch';
|
||||||
import { getAgentByUrl } from './fetch';
|
|
||||||
import { AbortController } from 'abort-controller';
|
|
||||||
import config from '@/config/index';
|
import config from '@/config/index';
|
||||||
import * as chalk from 'chalk';
|
import * as chalk from 'chalk';
|
||||||
import Logger from '@/services/logger';
|
import Logger from '@/services/logger';
|
||||||
|
import * as IPCIDR from 'ip-cidr';
|
||||||
|
const PrivateIp = require('private-ip');
|
||||||
|
|
||||||
const pipeline = util.promisify(stream.pipeline);
|
const pipeline = util.promisify(stream.pipeline);
|
||||||
|
|
||||||
|
@ -15,26 +15,57 @@ export async function downloadUrl(url: string, path: string) {
|
||||||
const logger = new Logger('download');
|
const logger = new Logger('download');
|
||||||
|
|
||||||
logger.info(`Downloading ${chalk.cyan(url)} ...`);
|
logger.info(`Downloading ${chalk.cyan(url)} ...`);
|
||||||
const controller = new AbortController();
|
|
||||||
setTimeout(() => {
|
|
||||||
controller.abort();
|
|
||||||
}, 60 * 1000);
|
|
||||||
|
|
||||||
const response = await fetch(new URL(url).href, {
|
const timeout = 30 * 1000;
|
||||||
|
const operationTimeout = 60 * 1000;
|
||||||
|
|
||||||
|
const req = got.stream(url, {
|
||||||
headers: {
|
headers: {
|
||||||
'User-Agent': config.userAgent
|
'User-Agent': config.userAgent
|
||||||
},
|
},
|
||||||
timeout: 10 * 1000,
|
timeout: {
|
||||||
signal: controller.signal,
|
lookup: timeout,
|
||||||
agent: getAgentByUrl,
|
connect: timeout,
|
||||||
|
secureConnect: timeout,
|
||||||
|
socket: timeout, // read timeout
|
||||||
|
response: timeout,
|
||||||
|
send: timeout,
|
||||||
|
request: operationTimeout, // whole operation timeout
|
||||||
|
},
|
||||||
|
agent: {
|
||||||
|
http: httpAgent,
|
||||||
|
https: httpsAgent,
|
||||||
|
},
|
||||||
|
retry: 0,
|
||||||
|
}).on('response', (res: Got.Response) => {
|
||||||
|
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) {
|
||||||
|
if (isPrivateIp(res.ip)) {
|
||||||
|
logger.warn(`Blocked address: ${res.ip}`);
|
||||||
|
req.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}).on('error', (e: any) => {
|
||||||
|
if (e.name === 'HTTPError') {
|
||||||
|
const statusCode = e.response?.statusCode;
|
||||||
|
const statusMessage = e.response?.statusMessage;
|
||||||
|
e.name = `StatusError`;
|
||||||
|
e.statusCode = statusCode;
|
||||||
|
e.message = `${statusCode} ${statusMessage}`;
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
await pipeline(req, fs.createWriteStream(path));
|
||||||
logger.error(`Got ${response.status} (${url})`);
|
|
||||||
throw response.status;
|
|
||||||
}
|
|
||||||
|
|
||||||
await pipeline(response.body, fs.createWriteStream(path));
|
|
||||||
|
|
||||||
logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
logger.succ(`Download finished: ${chalk.cyan(url)}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isPrivateIp(ip: string) {
|
||||||
|
for (const net of config.allowedPrivateNetworks || []) {
|
||||||
|
const cidr = new IPCIDR(net);
|
||||||
|
if (cidr.contains(ip)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return PrivateIp(ip);
|
||||||
|
}
|
||||||
|
|
|
@ -83,10 +83,10 @@ export default async function(ctx: Koa.Context) {
|
||||||
ctx.set('Content-Type', image.type);
|
ctx.set('Content-Type', image.type);
|
||||||
ctx.set('Cache-Control', 'max-age=31536000, immutable');
|
ctx.set('Cache-Control', 'max-age=31536000, immutable');
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
serverLogger.error(e);
|
serverLogger.error(e.statusCode);
|
||||||
|
|
||||||
if (typeof e == 'number' && e >= 400 && e < 500) {
|
if (typeof e.statusCode === 'number' && e.statusCode >= 400 && e.statusCode < 500) {
|
||||||
ctx.status = e;
|
ctx.status = e.statusCode;
|
||||||
ctx.set('Cache-Control', 'max-age=86400');
|
ctx.set('Cache-Control', 'max-age=86400');
|
||||||
} else {
|
} else {
|
||||||
ctx.status = 500;
|
ctx.status = 500;
|
||||||
|
|
|
@ -39,8 +39,8 @@ export async function proxyMedia(ctx: Koa.Context) {
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
serverLogger.error(e);
|
serverLogger.error(e);
|
||||||
|
|
||||||
if (typeof e == 'number' && e >= 400 && e < 500) {
|
if (typeof e.statusCode === 'number' && e.statusCode >= 400 && e.statusCode < 500) {
|
||||||
ctx.status = e;
|
ctx.status = e.statusCode;
|
||||||
} else {
|
} else {
|
||||||
ctx.status = 500;
|
ctx.status = 500;
|
||||||
}
|
}
|
||||||
|
|
44
yarn.lock
44
yarn.lock
|
@ -5742,6 +5742,27 @@ ioredis@^4.27.0:
|
||||||
redis-parser "^3.0.0"
|
redis-parser "^3.0.0"
|
||||||
standard-as-callback "^2.1.0"
|
standard-as-callback "^2.1.0"
|
||||||
|
|
||||||
|
ip-address@^7.1.0:
|
||||||
|
version "7.1.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-7.1.0.tgz#4a9c699e75b51cbeb18b38de8ed216efa1a490c5"
|
||||||
|
integrity sha512-V9pWC/VJf2lsXqP7IWJ+pe3P1/HCYGBMZrrnT62niLGjAfCbeiwXMUxaeHvnVlz19O27pvXP4azs+Pj/A0x+SQ==
|
||||||
|
dependencies:
|
||||||
|
jsbn "1.1.0"
|
||||||
|
sprintf-js "1.1.2"
|
||||||
|
|
||||||
|
ip-cidr@3.0.4:
|
||||||
|
version "3.0.4"
|
||||||
|
resolved "https://registry.yarnpkg.com/ip-cidr/-/ip-cidr-3.0.4.tgz#a915c47e00f47ea8d5f8ed662ea6161471c44375"
|
||||||
|
integrity sha512-pKNiqmBlTvEkhaLAa3+FOmYSY0/jjADVxxjA3NbujZZTT8mjLI90Q+6mwg6kd0fNm0RuAOkWJ1u1a/ETmlrPNQ==
|
||||||
|
dependencies:
|
||||||
|
ip-address "^7.1.0"
|
||||||
|
jsbn "^1.1.0"
|
||||||
|
|
||||||
|
ip-regex@^4.3.0:
|
||||||
|
version "4.3.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.3.0.tgz#687275ab0f57fa76978ff8f4dddc8a23d5990db5"
|
||||||
|
integrity sha512-B9ZWJxHHOHUhUjCPrMpLD4xEq35bUTClHM1S6CBU5ixQnkZmwipwgc96vAd7AAGM9TGHvJR+Uss+/Ak6UphK+Q==
|
||||||
|
|
||||||
ip@^1.1.5:
|
ip@^1.1.5:
|
||||||
version "1.1.5"
|
version "1.1.5"
|
||||||
resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a"
|
resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a"
|
||||||
|
@ -6238,6 +6259,11 @@ js-yaml@~3.7.0:
|
||||||
argparse "^1.0.7"
|
argparse "^1.0.7"
|
||||||
esprima "^2.6.0"
|
esprima "^2.6.0"
|
||||||
|
|
||||||
|
jsbn@1.1.0, jsbn@^1.1.0:
|
||||||
|
version "1.1.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040"
|
||||||
|
integrity sha1-sBMHyym2GKHtJux56RH4A8TaAEA=
|
||||||
|
|
||||||
jsbn@~0.1.0:
|
jsbn@~0.1.0:
|
||||||
version "0.1.1"
|
version "0.1.1"
|
||||||
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
|
resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513"
|
||||||
|
@ -7440,6 +7466,11 @@ nested-property@4.0.0:
|
||||||
resolved "https://registry.yarnpkg.com/nested-property/-/nested-property-4.0.0.tgz#a67b5a31991e701e03cdbaa6453bc5b1011bb88d"
|
resolved "https://registry.yarnpkg.com/nested-property/-/nested-property-4.0.0.tgz#a67b5a31991e701e03cdbaa6453bc5b1011bb88d"
|
||||||
integrity sha512-yFehXNWRs4cM0+dz7QxCd06hTbWbSkV0ISsqBfkntU6TOY4Qm3Q88fRRLOddkGh2Qq6dZvnKVAahfhjcUvLnyA==
|
integrity sha512-yFehXNWRs4cM0+dz7QxCd06hTbWbSkV0ISsqBfkntU6TOY4Qm3Q88fRRLOddkGh2Qq6dZvnKVAahfhjcUvLnyA==
|
||||||
|
|
||||||
|
netmask@^2.0.2:
|
||||||
|
version "2.0.2"
|
||||||
|
resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.0.2.tgz#8b01a07644065d536383835823bc52004ebac5e7"
|
||||||
|
integrity sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==
|
||||||
|
|
||||||
next-line@^1.1.0:
|
next-line@^1.1.0:
|
||||||
version "1.1.0"
|
version "1.1.0"
|
||||||
resolved "https://registry.yarnpkg.com/next-line/-/next-line-1.1.0.tgz#fcae57853052b6a9bae8208e40dd7d3c2d304603"
|
resolved "https://registry.yarnpkg.com/next-line/-/next-line-1.1.0.tgz#fcae57853052b6a9bae8208e40dd7d3c2d304603"
|
||||||
|
@ -8861,6 +8892,14 @@ prismjs@1.24.1:
|
||||||
resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.24.1.tgz#c4d7895c4d6500289482fa8936d9cdd192684036"
|
resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.24.1.tgz#c4d7895c4d6500289482fa8936d9cdd192684036"
|
||||||
integrity sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow==
|
integrity sha512-mNPsedLuk90RVJioIky8ANZEwYm5w9LcvCXrxHlwf4fNVSn8jEipMybMkWUyyF0JhnC+C4VcOVSBuHRKs1L5Ow==
|
||||||
|
|
||||||
|
private-ip@2.2.1:
|
||||||
|
version "2.2.1"
|
||||||
|
resolved "https://registry.yarnpkg.com/private-ip/-/private-ip-2.2.1.tgz#4fe167d04e12eca5c67cdcbd3224e86b38c79253"
|
||||||
|
integrity sha512-jN1WT/br/VNW9xEcwHr6DjtOKxQ5qOIqmh7o+co2TWgq56pZJw99iO3UT1tWdfgsQiyK9FqG4ji3ykwpjFqITA==
|
||||||
|
dependencies:
|
||||||
|
ip-regex "^4.3.0"
|
||||||
|
netmask "^2.0.2"
|
||||||
|
|
||||||
probe-image-size@7.2.1:
|
probe-image-size@7.2.1:
|
||||||
version "7.2.1"
|
version "7.2.1"
|
||||||
resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-7.2.1.tgz#df0c924e67e247bc94f8fcb0fad7f0081061fc44"
|
resolved "https://registry.yarnpkg.com/probe-image-size/-/probe-image-size-7.2.1.tgz#df0c924e67e247bc94f8fcb0fad7f0081061fc44"
|
||||||
|
@ -10103,6 +10142,11 @@ split@^1.0.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
through "2"
|
through "2"
|
||||||
|
|
||||||
|
sprintf-js@1.1.2:
|
||||||
|
version "1.1.2"
|
||||||
|
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.2.tgz#da1765262bf8c0f571749f2ad6c26300207ae673"
|
||||||
|
integrity sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==
|
||||||
|
|
||||||
sprintf-js@~1.0.2:
|
sprintf-js@~1.0.2:
|
||||||
version "1.0.3"
|
version "1.0.3"
|
||||||
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
|
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
|
||||||
|
|
Loading…
Reference in a new issue