allow setting separate timeout / max size for imports - fixes #479

This commit is contained in:
dakkar 2024-06-03 16:29:19 +00:00 committed by Amelia Yukii
parent 55fc2879f3
commit 082e1d1afb
4 changed files with 38 additions and 10 deletions

View file

@ -287,5 +287,10 @@ checkActivityPubGetSignature: false
# Upload or download file size limits (bytes) # Upload or download file size limits (bytes)
#maxFileSize: 262144000 #maxFileSize: 262144000
# timeout and maximum size for imports (e.g. note imports)
#import:
# downloadTimeout: 30
# maxFileSize: 262144000
# PID File of master process # PID File of master process
#pidFile: /tmp/misskey.pid #pidFile: /tmp/misskey.pid

View file

@ -97,6 +97,12 @@ type Source = {
perChannelMaxNoteCacheCount?: number; perChannelMaxNoteCacheCount?: number;
perUserNotificationsMaxCount?: number; perUserNotificationsMaxCount?: number;
deactivateAntennaThreshold?: number; deactivateAntennaThreshold?: number;
import?: {
downloadTimeout: number;
maxFileSize: number;
};
pidFile: string; pidFile: string;
}; };
@ -177,6 +183,12 @@ export type Config = {
perChannelMaxNoteCacheCount: number; perChannelMaxNoteCacheCount: number;
perUserNotificationsMaxCount: number; perUserNotificationsMaxCount: number;
deactivateAntennaThreshold: number; deactivateAntennaThreshold: number;
import: {
downloadTimeout: number;
maxFileSize: number;
} | undefined;
pidFile: string; pidFile: string;
}; };
@ -284,6 +296,7 @@ export function loadConfig(): Config {
perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000, perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000,
perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500, perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500,
deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7), deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7),
import: config.import,
pidFile: config.pidFile, pidFile: config.pidFile,
}; };
} }
@ -425,4 +438,5 @@ function applyEnvOverrides(config: Source) {
_apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]); _apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]);
_apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]); _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]);
_apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]); _apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]);
_apply_top(['import', ['downloadTimeout', 'maxFileSize']]);
} }

View file

@ -35,14 +35,14 @@ export class DownloadService {
} }
@bindThis @bindThis
public async downloadUrl(url: string, path: string): Promise<{ public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{
filename: string; filename: string;
}> { }> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
const timeout = 30 * 1000; const timeout = options.timeout ?? 30 * 1000;
const operationTimeout = 60 * 1000; const operationTimeout = options.operationTimeout ?? 60 * 1000;
const maxSize = this.config.maxFileSize ?? 262144000; const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000;
const urlObj = new URL(url); const urlObj = new URL(url);
let filename = urlObj.pathname.split('/').pop() ?? 'untitled'; let filename = urlObj.pathname.split('/').pop() ?? 'untitled';

View file

@ -19,12 +19,16 @@ import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq'; import type * as Bull from 'bullmq';
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js'; import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
import type { Config } from '@/config.js';
@Injectable() @Injectable()
export class ImportNotesProcessorService { export class ImportNotesProcessorService {
private logger: Logger; private logger: Logger;
constructor( constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository) @Inject(DI.usersRepository)
private usersRepository: UsersRepository, private usersRepository: UsersRepository,
@ -73,6 +77,11 @@ export class ImportNotesProcessorService {
} }
} }
@bindThis
private downloadUrl(url: string, path:string): Promise<{filename: string}> {
return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize });
}
@bindThis @bindThis
private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> { private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise<any[]> {
type NotesMap = { type NotesMap = {
@ -176,7 +185,7 @@ export class ImportNotesProcessorService {
try { try {
await fsp.writeFile(destPath, '', 'binary'); await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath); await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
this.logger.error(e); this.logger.error(e);
@ -206,7 +215,7 @@ export class ImportNotesProcessorService {
try { try {
await fsp.writeFile(destPath, '', 'binary'); await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath); await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
this.logger.error(e); this.logger.error(e);
@ -239,7 +248,7 @@ export class ImportNotesProcessorService {
try { try {
await fsp.writeFile(destPath, '', 'binary'); await fsp.writeFile(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath); await this.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
this.logger.error(e); this.logger.error(e);
@ -297,7 +306,7 @@ export class ImportNotesProcessorService {
try { try {
await fsp.writeFile(path, '', 'utf-8'); await fsp.writeFile(path, '', 'utf-8');
await this.downloadService.downloadUrl(file.url, path); await this.downloadUrl(file.url, path);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') { if (e instanceof Error || typeof e === 'string') {
this.logger.error(e); this.logger.error(e);
@ -349,7 +358,7 @@ export class ImportNotesProcessorService {
if (!exists) { if (!exists) {
try { try {
await this.downloadService.downloadUrl(file.url, filePath); await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string)); this.logger.error(e instanceof Error ? e : new Error(e as string));
} }
@ -488,7 +497,7 @@ export class ImportNotesProcessorService {
if (!exists) { if (!exists) {
try { try {
await this.downloadService.downloadUrl(file.url, filePath); await this.downloadUrl(file.url, filePath);
} catch (e) { // TODO: 何度か再試行 } catch (e) { // TODO: 何度か再試行
this.logger.error(e instanceof Error ? e : new Error(e as string)); this.logger.error(e instanceof Error ? e : new Error(e as string));
} }