perf(backend): Use undici instead of node-fetch and got (#9459)

* Implement? HttpFetchService

* ✌️

* remove node-fetch

* fix

* refactor

* fix

* gateway timeout

* UndiciFetcherクラスを追加 (仮コミット, ビルドもstartもさせていない)

* fix

* add logger and fix url preview

* fix ip check

* enhance logger and error handling

* fix

* fix

* clean up

* Use custom fetcher for ApRequest / ApResolver

* bypassProxyはproxyBypassHostsに判断を委譲するように

* set maxRedirections (default 3, ApRequest/ApResolver: 0)

* fix comment

* handle error s3 upload

* add debug message

* no return await

* Revert "no return await"

This reverts commit b5b0dc58a342393d260492e3a6f58304372f53b2.

* reduce maxSockets

* apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger

* Revert "apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger"

This reverts commit 997243915c8e1f8472da64f607f88c36cb1d5cb4.

* add logger

* fix

* change logger name

* safe

* デフォルトでUser-Agentを設定

.config/example.yml

@@ -122,10 +122,12 @@ id: 'aid'
 # Proxy for HTTP/HTTPS
 #proxy: http://127.0.0.1:3128
 
-#proxyBypassHosts: [
+proxyBypassHosts:
-#  'example.com',
+  - api.deepl.com
-#  '192.0.2.8'
+  - api-free.deepl.com
-#]
+  - www.recaptcha.net
+  - hcaptcha.com
+  - challenges.cloudflare.com
 
 # Proxy for SMTP/SMTPS
 #proxySmtp: http://127.0.0.1:3128   # use HTTP/1.1 CONNECT

packages/backend/package.json

@@ -77,7 +77,6 @@
 		"misskey-js": "0.0.14",
 		"ms": "3.0.0-canary.1",
 		"nested-property": "4.0.0",
-		"node-fetch": "3.3.0",
 		"nodemailer": "6.8.0",
 		"nsfwjs": "2.4.2",
 		"oauth": "^0.10.0",
@@ -118,6 +117,7 @@
 		"twemoji-parser": "14.0.0",
 		"typeorm": "0.3.11",
 		"ulid": "2.3.0",
+		"undici": "^5.14.0",
 		"unzipper": "0.10.11",
 		"uuid": "9.0.0",
 		"vary": "1.1.2",
@@ -180,6 +180,7 @@
 		"execa": "6.1.0",
 		"jest": "29.3.1",
 		"jest-mock": "^29.3.1",
+		"node-fetch": "3.3.0",
 		"typescript": "4.9.4"
 	}
 }

packages/backend/src/core/CaptchaService.ts

@@ -1,7 +1,4 @@
-import { Inject, Injectable } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
-import { DI } from '@/di-symbols.js';
-import type { UsersRepository } from '@/models/index.js';
-import type { Config } from '@/config.js';
 import { HttpRequestService } from '@/core/HttpRequestService.js';
 import { bindThis } from '@/decorators.js';
 
@@ -13,9 +10,6 @@ type CaptchaResponse = {
 @Injectable()
 export class CaptchaService {
 	constructor(
-		@Inject(DI.config)
-		private config: Config,
-
 		private httpRequestService: HttpRequestService,
 	) {
 	}
@@ -27,16 +21,16 @@ export class CaptchaService {
 			response,
 		});
 	
-		const res = await fetch(url, {
+		const res = await this.httpRequestService.fetch(
+			url,
+			{
 				method: 'POST',
 				body: params,
-			headers: {
-				'User-Agent': this.config.userAgent,
 			},
-			// TODO
+			{
-			//timeout: 10 * 1000,
+				noOkError: true,
-			agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy),
+			}
-		}).catch(err => {
+		).catch(err => {
 			throw `${err.message ?? err}`;
 		});
 	

packages/backend/src/core/DownloadService.ts

@@ -8,11 +8,12 @@ import got, * as Got from 'got';
 import chalk from 'chalk';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
 import { createTemp } from '@/misc/create-temp.js';
 import { StatusError } from '@/misc/status-error.js';
 import { LoggerService } from '@/core/LoggerService.js';
 import type Logger from '@/logger.js';
+import { buildConnector } from 'undici';
 
 const pipeline = util.promisify(stream.pipeline);
 import { bindThis } from '@/decorators.js';
@@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js';
 @Injectable()
 export class DownloadService {
 	private logger: Logger;
+	private undiciFetcher: UndiciFetcher;
 
 	constructor(
 		@Inject(DI.config)
@@ -29,6 +31,24 @@ export class DownloadService {
 		private loggerService: LoggerService,
 	) {
 		this.logger = this.loggerService.getLogger('download');
+
+		this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
+			{
+				connect: process.env.NODE_ENV === 'development' ?
+					this.httpRequestService.clientDefaults.connect
+					:
+					this.httpRequestService.getConnectorWithIpCheck(
+						buildConnector({
+							...this.httpRequestService.clientDefaults.connect,
+						}),
+						(ip) => !this.isPrivateIp(ip)
+					),
+				bodyTimeout: 30 * 1000,
+			},
+			{
+				connect: this.httpRequestService.clientDefaults.connect,
+			}
+		), this.logger);
 	}
 
 	@bindThis
@@ -39,59 +59,13 @@ export class DownloadService {
 		const operationTimeout = 60 * 1000;
 		const maxSize = this.config.maxFileSize ?? 262144000;
 
-		const req = got.stream(url, {
+		const response = await this.undiciFetcher.fetch(url);
-			headers: {
+
-				'User-Agent': this.config.userAgent,
+		if (response.body === null) {
-			},
+			throw new StatusError('No body', 400, 'No body');
-			timeout: {
-				lookup: timeout,
-				connect: timeout,
-				secureConnect: timeout,
-				socket: timeout,	// read timeout
-				response: timeout,
-				send: timeout,
-				request: operationTimeout,	// whole operation timeout
-			},
-			agent: {
-				http: this.httpRequestService.httpAgent,
-				https: this.httpRequestService.httpsAgent,
-			},
-			http2: false,	// default
-			retry: {
-				limit: 0,
-			},
-		}).on('response', (res: Got.Response) => {
-			if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
-				if (this.isPrivateIp(res.ip)) {
-					this.logger.warn(`Blocked address: ${res.ip}`);
-					req.destroy();
-				}
 		}
 
-			const contentLength = res.headers['content-length'];
+		await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
-			if (contentLength != null) {
-				const size = Number(contentLength);
-				if (size > maxSize) {
-					this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
-					req.destroy();
-				}
-			}
-		}).on('downloadProgress', (progress: Got.Progress) => {
-			if (progress.transferred > maxSize) {
-				this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
-				req.destroy();
-			}
-		});
-	
-		try {
-			await pipeline(req, fs.createWriteStream(path));
-		} catch (e) {
-			if (e instanceof Got.HTTPError) {
-				throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
-			} else {
-				throw e;
-			}
-		}
 
 		this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
 	}
@@ -124,6 +98,6 @@ export class DownloadService {
 			}
 		}
 
-		return PrivateIp(ip);
+		return PrivateIp(ip) ?? false;
 	}
 }

packages/backend/src/core/DriveService.ts

@@ -375,8 +375,19 @@ export class DriveService {
 			partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
 		});
 
-		const result = await upload.promise();
+		await upload.promise()
-		if (result) this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
+			.then(
+				result => {
+					if (result) {
+						this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
+					} else {
+						this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`);
+					}
+				},
+				err => {
+					this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
+				}
+			);
 	}
 
 	@bindThis
@@ -462,6 +473,8 @@ export class DriveService {
 			}
 		}
 
+		this.registerLogger.debug(`ADD DRIVE FILE: user ${user?.id ?? 'not set'}, name ${detectedName}, tmp ${path}`);
+
 		//#region Check drive usage
 		if (user && !isLink) {
 			const usage = await this.driveFileEntityService.calcDriveUsageOf(user);

packages/backend/src/core/FetchInstanceMetadataService.ts

@@ -1,7 +1,6 @@
 import { URL } from 'node:url';
 import { Inject, Injectable } from '@nestjs/common';
 import { JSDOM } from 'jsdom';
-import fetch from 'node-fetch';
 import tinycolor from 'tinycolor2';
 import type { Instance } from '@/models/entities/Instance.js';
 import type { InstancesRepository } from '@/models/index.js';
@@ -191,11 +190,7 @@ export class FetchInstanceMetadataService {
 	
 		const faviconUrl = url + '/favicon.ico';
 	
-		const favicon = await fetch(faviconUrl, {
+		const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true });
-			// TODO
-			//timeout: 10000,
-			agent: url => this.httpRequestService.getAgentByUrl(url),
-		});
 	
 		if (favicon.ok) {
 			return faviconUrl;

packages/backend/src/core/HttpRequestService.ts

@@ -1,67 +1,257 @@
 import * as http from 'node:http';
 import * as https from 'node:https';
 import CacheableLookup from 'cacheable-lookup';
-import fetch from 'node-fetch';
 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
 import { Inject, Injectable } from '@nestjs/common';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { StatusError } from '@/misc/status-error.js';
 import { bindThis } from '@/decorators.js';
-import type { Response } from 'node-fetch';
+import * as undici from 'undici';
-import type { URL } from 'node:url';
+import { LookupFunction } from 'node:net';
+import { LoggerService } from '@/core/LoggerService.js';
+import type Logger from '@/logger.js';
+
+// true to allow, false to deny
+export type IpChecker = (ip: string) => boolean;
+
+/* 
+ *  Child class to create and save Agent for fetch.
+ *  You should construct this when you want
+ *  to change timeout, size limit, socket connect function, etc.
+ */
+export class UndiciFetcher {
+	/**
+	 * Get http non-proxy agent (undici)
+	 */
+	public nonProxiedAgent: undici.Agent;
+
+	/**
+	 * Get http proxy or non-proxy agent (undici)
+	 */
+	public agent: undici.ProxyAgent | undici.Agent;
+
+	private proxyBypassHosts: string[];
+	private userAgent: string | undefined;
+
+	private logger: Logger | undefined;
+
+	constructor(
+		args: {
+			agentOptions: undici.Agent.Options;
+			proxy?: {
+				uri: string;
+				options?: undici.Agent.Options; // Override of agentOptions
+			},
+			proxyBypassHosts?: string[];
+			userAgent?: string;
+		},
+		logger?: Logger,
+	) {
+		this.logger = logger;
+		this.logger?.debug('UndiciFetcher constructor', args);
+
+		this.proxyBypassHosts = args.proxyBypassHosts ?? [];
+		this.userAgent = args.userAgent;
+
+		this.nonProxiedAgent = new undici.Agent({
+			...args.agentOptions,
+			connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
+				? (options, cb) => {
+					// Custom connector for debug
+					undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
+						this.logger?.debug('Socket connector called', socket);
+						if (err) {
+							this.logger?.debug(`Socket error`, err);
+							cb(new Error(`Error while socket connecting\n${err}`), null);
+							return;
+						}
+						this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
+						cb(null, socket);
+					});
+				} : args.agentOptions.connect,
+		});
+
+		this.agent = args.proxy
+			? new undici.ProxyAgent({
+				...args.agentOptions,
+				...args.proxy.options,
+
+				uri: args.proxy.uri,
+
+				connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function')
+					? (options, cb) => {
+						// Custom connector for debug
+						undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
+							this.logger?.debug('Socket connector called (secure)', socket);
+							if (err) {
+								this.logger?.debug(`Socket error`, err);
+								cb(new Error(`Error while socket connecting\n${err}`), null);
+								return;
+							}
+							this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
+							cb(null, socket);
+						});
+					} : (args.proxy?.options?.connect ?? args.agentOptions.connect),
+			})
+			: this.nonProxiedAgent;
+	}
+
+	/**
+	 * Get agent by URL
+	 * @param url URL
+	 * @param bypassProxy Allways bypass proxy
+	 */
+	@bindThis
+	public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
+		if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
+			return this.nonProxiedAgent;
+		} else {
+			return this.agent;
+		}
+	}
+
+	@bindThis
+	public async fetch(
+		url: string | URL,
+		options: undici.RequestInit = {},
+		privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }
+	): Promise<undici.Response> {
+		const res = await undici.fetch(url, {
+			dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
+			...options,
+			headers: {
+				'User-Agent': this.userAgent ?? '',
+				...(options.headers ?? {}),
+			},
+		}).catch((err) => {
+			this.logger?.error('fetch error', err);
+			throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
+		});
+		if (!res.ok && !privateOptions.noOkError) {
+			throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
+		}
+		return res;
+	}
+
+	@bindThis
+	public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
+		const res = await this.fetch(
+			url,
+			{
+				headers: Object.assign({
+					Accept: accept,
+				}, headers ?? {}),
+			}
+		);
+
+		return await res.json() as T;
+	}
+
+	@bindThis
+	public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
+		const res = await this.fetch(
+			url,
+			{
+				headers: Object.assign({
+					Accept: accept,
+				}, headers ?? {}),
+			}
+		);
+
+		return await res.text();
+	}
+}
 
 @Injectable()
 export class HttpRequestService {
-	/**
+	public defaultFetcher: UndiciFetcher;
-	 * Get http non-proxy agent
+	public fetch: UndiciFetcher['fetch'];
-	 */
+	public getHtml: UndiciFetcher['getHtml'];
+	public defaultJsonFetcher: UndiciFetcher;
+	public getJson: UndiciFetcher['getJson'];
+
+	//#region for old http/https, only used in S3Service
+	// http non-proxy agent
 	private http: http.Agent;
 
-	/**
+	// https non-proxy agent
-	 * Get https non-proxy agent
-	 */
 	private https: https.Agent;
 
-	/**
+	// http proxy or non-proxy agent
-	 * Get http proxy or non-proxy agent
-	 */
 	public httpAgent: http.Agent;
 
-	/**
+	// https proxy or non-proxy agent
-	 * Get https proxy or non-proxy agent
-	 */
 	public httpsAgent: https.Agent;
+	//#endregion
+
+	public readonly dnsCache: CacheableLookup;
+	public readonly clientDefaults: undici.Agent.Options;
+	private maxSockets: number;
+
+	private logger: Logger;
 
 	constructor(
 		@Inject(DI.config)
 		private config: Config,
+		private loggerService: LoggerService,
 	) {
-		const cache = new CacheableLookup({
+		this.logger = this.loggerService.getLogger('http-request');
+
+		this.dnsCache = new CacheableLookup({
 			maxTtl: 3600,	// 1hours
 			errorTtl: 30,	// 30secs
 			lookup: false,	// nativeのdns.lookupにfallbackしない
 		});
 
+		this.clientDefaults = {
+			keepAliveTimeout: 30 * 1000,
+			keepAliveMaxTimeout: 10 * 60 * 1000,
+			keepAliveTimeoutThreshold: 1 * 1000,
+			strictContentLength: true,
+			headersTimeout: 10 * 1000,
+			bodyTimeout: 10 * 1000,
+			maxHeaderSize: 16364, // default
+			maxResponseSize: 10 * 1024 * 1024,
+			maxRedirections: 3,
+			connect: {
+				timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
+				maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
+				lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
+			},
+		}
+
+		this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128);
+
+		this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger);
+
+		this.fetch = this.defaultFetcher.fetch;
+		this.getHtml = this.defaultFetcher.getHtml;
+
+		this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({
+			maxResponseSize: 1024 * 256,
+		}), this.logger);
+
+		this.getJson = this.defaultJsonFetcher.getJson;
+
+		//#region for old http/https, only used in S3Service
 		this.http = new http.Agent({
 			keepAlive: true,
 			keepAliveMsecs: 30 * 1000,
-			lookup: cache.lookup,
+			lookup: this.dnsCache.lookup,
 		} as http.AgentOptions);
 		
 		this.https = new https.Agent({
 			keepAlive: true,
 			keepAliveMsecs: 30 * 1000,
-			lookup: cache.lookup,
+			lookup: this.dnsCache.lookup,
 		} as https.AgentOptions);
 
-		const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
-		
 		this.httpAgent = config.proxy
 			? new HttpProxyAgent({
 				keepAlive: true,
 				keepAliveMsecs: 30 * 1000,
-				maxSockets,
+				maxSockets: this.maxSockets,
 				maxFreeSockets: 256,
 				scheduling: 'lifo',
 				proxy: config.proxy,
@@ -72,21 +262,42 @@ export class HttpRequestService {
 			? new HttpsProxyAgent({
 				keepAlive: true,
 				keepAliveMsecs: 30 * 1000,
-				maxSockets,
+				maxSockets: this.maxSockets,
 				maxFreeSockets: 256,
 				scheduling: 'lifo',
 				proxy: config.proxy,
 			})
 			: this.https;
+		//#endregion
+	}
+
+	@bindThis
+	public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
+		return {
+			agentOptions: {
+				...this.clientDefaults,
+				...opts,
+			},
+			...(this.config.proxy ? {
+			proxy: {
+				uri: this.config.proxy,
+				options: {
+					connections: this.maxSockets,
+					...proxyOpts,
+				}
+			}
+			} : {}),
+			userAgent: this.config.userAgent,
+		}
 	}
 
 	/**
-	 * Get agent by URL
+	 * Get http agent by URL
 	 * @param url URL
 	 * @param bypassProxy Allways bypass proxy
 	 */
 	@bindThis
-	public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
+	public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
 		if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
 			return url.protocol === 'http:' ? this.http : this.https;
 		} else {
@@ -94,67 +305,37 @@ export class HttpRequestService {
 		}
 	}
 
+	/**
+	 * check ip
+	 */
 	@bindThis
-	public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>): Promise<unknown> {
+	public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
-		const res = await this.getResponse({
+		return (options, cb) => {
-			url,
+			connector(options, (err, socket) => {
-			method: 'GET',
+				this.logger.debug('Socket connector (with ip checker) called', socket);
-			headers: Object.assign({
+				if (err) {
-				'User-Agent': this.config.userAgent,
+					this.logger.error(`Socket error`, err)
-				Accept: accept,
+					cb(new Error(`Error while socket connecting\n${err}`), null);
-			}, headers ?? {}),
+					return;
-			timeout,
+				}
-			size: 1024 * 256,
+
+				if (socket.remoteAddress == undefined) {
+					this.logger.error(`Socket error: remoteAddress is undefined`);
+					cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
+					return;
+				}
+
+				// allow
+				if (checkIp(socket.remoteAddress)) {
+					this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
+					cb(null, socket);
+					return;
+				}
+
+				this.logger.error('IP is not allowed', socket);
+				cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
+				socket.destroy();
 			});
-
+		};
-		return await res.json();
-	}
-
-	@bindThis
-	public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>): Promise<string> {
-		const res = await this.getResponse({
-			url,
-			method: 'GET',
-			headers: Object.assign({
-				'User-Agent': this.config.userAgent,
-				Accept: accept,
-			}, headers ?? {}),
-			timeout,
-		});
-
-		return await res.text();
-	}
-
-	@bindThis
-	public async getResponse(args: {
-		url: string,
-		method: string,
-		body?: string,
-		headers: Record<string, string>,
-		timeout?: number,
-		size?: number,
-	}): Promise<Response> {
-		const timeout = args.timeout ?? 10 * 1000;
-
-		const controller = new AbortController();
-		setTimeout(() => {
-			controller.abort();
-		}, timeout * 6);
-
-		const res = await fetch(args.url, {
-			method: args.method,
-			headers: args.headers,
-			body: args.body,
-			timeout,
-			size: args.size ?? 10 * 1024 * 1024,
-			agent: (url) => this.getAgentByUrl(url),
-			signal: controller.signal,
-		});
-
-		if (!res.ok) {
-			throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
-		}
-
-		return res;
 	}
 }

packages/backend/src/core/S3Service.ts

@@ -33,7 +33,7 @@ export class S3Service {
 				? false
 				: meta.objectStorageS3ForcePathStyle,
 			httpOptions: {
-				agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
+				agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
 			},
 		});
 	}

packages/backend/src/core/WebfingerService.ts

@@ -30,7 +30,7 @@ export class WebfingerService {
 	public async webfinger(query: string): Promise<IWebFinger> {
 		const url = this.genUrl(query);
 
-		return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger;
+		return await this.httpRequestService.getJson<IWebFinger>(url, 'application/jrd+json, application/json');
 	}
 
 	@bindThis

packages/backend/src/core/activitypub/ApRequestService.ts

@@ -5,8 +5,10 @@ import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import type { User } from '@/models/entities/User.js';
 import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import { bindThis } from '@/decorators.js';
+import type Logger from '@/logger.js';
 
 type Request = {
 	url: string;
@@ -28,13 +30,21 @@ type PrivateKey = {
 
 @Injectable()
 export class ApRequestService {
+	private undiciFetcher: UndiciFetcher;
+	private logger: Logger;
+
 	constructor(
 		@Inject(DI.config)
 		private config: Config,
 
 		private userKeypairStoreService: UserKeypairStoreService,
 		private httpRequestService: HttpRequestService,
+		private loggerService: LoggerService,
 	) {
+		this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
+		this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
+			maxRedirections: 0,
+		}), this.logger );
 	}
 
 	@bindThis
@@ -148,16 +158,17 @@ export class ApRequestService {
 			url,
 			body,
 			additionalHeaders: {
-				'User-Agent': this.config.userAgent,
 			},
 		});
 
-		await this.httpRequestService.getResponse({
+		await this.undiciFetcher.fetch(
 			url,
+			{
 				method: req.request.method,
 				headers: req.request.headers,
 				body,
-		});
+			}
+		);
 	}
 
 	/**
@@ -176,15 +187,16 @@ export class ApRequestService {
 			},
 			url,
 			additionalHeaders: {
-				'User-Agent': this.config.userAgent,
 			},
 		});
 
-		const res = await this.httpRequestService.getResponse({
+		const res = await this.httpRequestService.fetch(
 			url,
+			{
 				method: req.request.method,
 				headers: req.request.headers,
-		});
+			}
+		);
 
 		return await res.json();
 	}

packages/backend/src/core/activitypub/ApResolverService.ts

@@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js';
 import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js';
 import type { Config } from '@/config.js';
 import { MetaService } from '@/core/MetaService.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
 import { DI } from '@/di-symbols.js';
 import { UtilityService } from '@/core/UtilityService.js';
 import { bindThis } from '@/decorators.js';
@@ -12,11 +12,15 @@ import { isCollectionOrOrderedCollection } from './type.js';
 import { ApDbResolverService } from './ApDbResolverService.js';
 import { ApRendererService } from './ApRendererService.js';
 import { ApRequestService } from './ApRequestService.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import type { IObject, ICollection, IOrderedCollection } from './type.js';
+import type Logger from '@/logger.js';
 
 export class Resolver {
 	private history: Set<string>;
 	private user?: ILocalUser;
+	private undiciFetcher: UndiciFetcher;
+	private logger: Logger;
 
 	constructor(
 		private config: Config,
@@ -31,9 +35,14 @@ export class Resolver {
 		private httpRequestService: HttpRequestService,
 		private apRendererService: ApRendererService,
 		private apDbResolverService: ApDbResolverService,
+		private loggerService: LoggerService,
 		private recursionLimit = 100,
 	) {
 		this.history = new Set();
+		this.logger = this.loggerService?.getLogger('ap-resolve');  // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
+		this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
+			maxRedirections: 0,
+		}), this.logger);
 	}
 
 	@bindThis
@@ -96,8 +105,8 @@ export class Resolver {
 		}
 
 		const object = (this.user
-			? await this.apRequestService.signedGet(value, this.user)
+			? await this.apRequestService.signedGet(value, this.user) as IObject
-			: await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject;
+			: await this.undiciFetcher.getJson<IObject>(value, 'application/activity+json, application/ld+json'));
 
 		if (object == null || (
 			Array.isArray(object['@context']) ?

packages/backend/src/core/activitypub/LdSignatureService.ts

@@ -1,6 +1,5 @@
 import * as crypto from 'node:crypto';
 import { Inject, Injectable } from '@nestjs/common';
-import fetch from 'node-fetch';
 import { HttpRequestService } from '@/core/HttpRequestService.js';
 import { bindThis } from '@/decorators.js';
 import { CONTEXTS } from './misc/contexts.js';
@@ -116,14 +115,19 @@ class LdSignature {
 
 	@bindThis
 	private async fetchDocument(url: string) {
-		const json = await fetch(url, {
+		const json = await this.httpRequestService.fetch(
+			url,
+			{
 				headers: {
 					Accept: 'application/ld+json, application/json',
 				},
 				// TODO
 				//timeout: this.loderTimeout,
-			agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent,
+			},
-		}).then(res => {
+			{
+				noOkError: true,
+			}
+		).then(res => {
 			if (!res.ok) {
 				throw `${res.status} ${res.statusText}`;
 			} else {

packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts

@@ -33,8 +33,9 @@ export class WebhookDeliverProcessorService {
 		try {
 			this.logger.debug(`delivering ${job.data.webhookId}`);
 	
-			const res = await this.httpRequestService.getResponse({
+			const res = await this.httpRequestService.fetch(
-				url: job.data.to,
+				job.data.to,
+				{
 					method: 'POST',
 					headers: {
 						'User-Agent': 'Misskey-Hooks',
@@ -50,7 +51,8 @@ export class WebhookDeliverProcessorService {
 						type: job.data.type,
 						body: job.data.content,
 					}),
-			});
+				}
+			);
 	
 			this.webhooksRepository.update({ id: job.data.webhookId }, {
 				latestSentAt: new Date(),

packages/backend/src/server/api/endpoints/fetch-rss.ts

@@ -33,15 +33,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 		private httpRequestService: HttpRequestService,
 	) {
 		super(meta, paramDef, async (ps, me) => {
-			const res = await this.httpRequestService.getResponse({
+			const res = await this.httpRequestService.fetch(
-				url: ps.url,
+				ps.url,
+				{
 					method: 'GET',
-				headers: Object.assign({
+					headers: {
-					'User-Agent': config.userAgent,
 						Accept: 'application/rss+xml, */*',
-				}),
+					},
-				timeout: 5000,
+					// timeout: 5000,
-			});
+				}
+			);
 
 			const text = await res.text();
 

packages/backend/src/server/api/endpoints/notes/translate.ts

@@ -1,5 +1,4 @@
 import { URLSearchParams } from 'node:url';
-import fetch from 'node-fetch';
 import { Inject, Injectable } from '@nestjs/common';
 import type { NotesRepository } from '@/models/index.js';
 import { Endpoint } from '@/server/api/endpoint-base.js';
@@ -84,18 +83,20 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
 
 			const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
 
-			const res = await fetch(endpoint, {
+			const res = await this.httpRequestService.fetch(
+				endpoint,
+				{
 					method: 'POST',
 					headers: {
 						'Content-Type': 'application/x-www-form-urlencoded',
-					'User-Agent': config.userAgent,
 						Accept: 'application/json, */*',
 					},
-				body: params,
+					body: params.toString(),
-				// TODO
+				},
-				//timeout: 10000,
+				{
-				agent: (url) => this.httpRequestService.getAgentByUrl(url),
+					noOkError: false,
-			});
+				}
+			);
 
 			const json = (await res.json()) as {
 				translations: {

packages/backend/src/server/api/integration/DiscordServerService.ts

@@ -181,7 +181,7 @@ export class DiscordServerService {
 						}
 					}));
 
-				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
+				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
 					'Authorization': `Bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 
@@ -249,7 +249,7 @@ export class DiscordServerService {
 						}
 					}));
 
-				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
+				const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
 					'Authorization': `Bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 				if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') {

packages/backend/src/server/api/integration/GithubServerService.ts

@@ -174,7 +174,7 @@ export class GithubServerService {
 						}
 					}));
 
-				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
 					'Authorization': `bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 				if (typeof login !== 'string' || typeof id !== 'string') {
@@ -223,7 +223,7 @@ export class GithubServerService {
 							}
 						}));
 
-				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+				const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
 					'Authorization': `bearer ${accessToken}`,
 				})) as Record<string, unknown>;
 

packages/backend/src/server/web/UrlPreviewService.ts

@@ -63,9 +63,8 @@ export class UrlPreviewService {
 		this.logger.info(meta.summalyProxy
 			? `(Proxy) Getting preview of ${url}@${lang} ...`
 			: `Getting preview of ${url}@${lang} ...`);
-	
 		try {
-			const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({
+			const summary = meta.summalyProxy ? await this.httpRequestService.getJson<ReturnType<typeof summaly.default>>(`${meta.summalyProxy}?${query({
 				url: url,
 				lang: lang ?? 'ja-JP',
 			})}`) : await summaly.default(url, {

yarn.lock

@@ -4272,6 +4272,7 @@ __metadata:
     typeorm: 0.3.11
     typescript: 4.9.4
     ulid: 2.3.0
+    undici: ^5.14.0
     unzipper: 0.10.11
     uuid: 9.0.0
     vary: 1.1.2
@@ -16665,6 +16666,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"undici@npm:^5.14.0, undici@npm:^5.5.1":
+  version: 5.14.0
+  resolution: "undici@npm:5.14.0"
+  dependencies:
+    busboy: ^1.6.0
+  checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
+  languageName: node
+  linkType: hard
+
 "undici@npm:^5.2.0":
   version: 5.13.0
   resolution: "undici@npm:5.13.0"
@@ -16674,15 +16684,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"undici@npm:^5.5.1":
-  version: 5.14.0
-  resolution: "undici@npm:5.14.0"
-  dependencies:
-    busboy: ^1.6.0
-  checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
-  languageName: node
-  linkType: hard
-
 "union-value@npm:^1.0.0":
   version: 1.0.1
   resolution: "union-value@npm:1.0.1"