diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dce1a0496..188e146cdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ## Unreleased +### Note +- コントロールパネル内にあるサマリープロキシの設定個所がセキュリティから全般へ変更となります。 + ### General +- Enhance: URLプレビューの有効化・無効化を設定できるように #13569 - Enhance: アンテナでBotによるノートを除外できるように (Cherry-picked from https://github.com/MisskeyIO/misskey/pull/545) - Fix: Play作成時に設定した公開範囲が機能していない問題を修正 @@ -25,6 +29,7 @@ ### Server - Enhance: エンドポイント`antennas/update`の必須項目を`antennaId`のみに +- Enhance: misskey-dev/summaly@5.1.0の取り込み（プレビュー生成処理の効率化） - Fix: フォローリクエストを作成する際に既存のものは削除するように (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/440) diff --git a/locales/index.d.ts b/locales/index.d.ts index afb4adac6c..70586d7a87 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -4916,6 +4916,10 @@ export interface Locale extends ILocale { * リトライ */ "gameRetry": string; + /** + * 使用しない場合は空欄にしてください + */ + "notUsePleaseLeaveBlank": string; "_bubbleGame": { /** * 遊び方 @@ -9768,6 +9772,60 @@ export interface Locale extends ILocale { */ "header": string; }; + "_urlPreviewSetting": { + /** + * URLプレビューの設定 + */ + "title": string; + /** + * URLプレビューを有効にする + */ + "enable": string; + /** + * プレビュー取得時のタイムアウト(ms) + */ + "timeout": string; + /** + * プレビュー取得の所要時間がこの値を超えた場合、プレビューは生成されません。 + */ + "timeoutDescription": string; + /** + * Content-Lengthの最大値(byte) + */ + "maximumContentLength": string; + /** + * Content-Lengthがこの値を超えた場合、プレビューは生成されません。 + */ + "maximumContentLengthDescription": string; + /** + * Content-Lengthが取得できた場合のみプレビューを生成 + */ + "requireContentLength": string; + /** + * 相手サーバがContent-Lengthを返さない場合、プレビューは生成されません。 + */ + "requireContentLengthDescription": string; + /** + * User-Agent + */ + "userAgent": string; + /** + * プレビュー取得時に使用されるUser-Agentを設定します。空欄の場合、デフォルトのUser-Agentが使用されます。 + */ + "userAgentDescription": string; + /** + * プレビューを生成するプロキシのエンドポイント + */ + "summaryProxy": string; + /** + * Misskey本体ではなく、サマリープロキシを使用してプレビューを生成します。 + */ + "summaryProxyDescription": string; + /** + * プロキシには下記パラメータがクエリ文字列として連携されます。プロキシ側がこれらをサポートしない場合、設定値は無視されます。 + */ + "summaryProxyDescription2": string; + }; } declare const locales: { [lang: string]: Locale; diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index a64c83b10f..cada6d855f 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1225,6 +1225,7 @@ enableHorizontalSwipe: "スワイプしてタブを切り替える" loading: "読み込み中" surrender: "やめる" gameRetry: "リトライ" +notUsePleaseLeaveBlank: "使用しない場合は空欄にしてください" _bubbleGame: howToPlay: "遊び方" @@ -2602,3 +2603,17 @@ _offlineScreen: title: "オフライン - サーバーに接続できません" header: "サーバーに接続できません" +_urlPreviewSetting: + title: "URLプレビューの設定" + enable: "URLプレビューを有効にする" + timeout: "プレビュー取得時のタイムアウト(ms)" + timeoutDescription: "プレビュー取得の所要時間がこの値を超えた場合、プレビューは生成されません。" + maximumContentLength: "Content-Lengthの最大値(byte)" + maximumContentLengthDescription: "Content-Lengthがこの値を超えた場合、プレビューは生成されません。" + requireContentLength: "Content-Lengthが取得できた場合のみプレビューを生成" + requireContentLengthDescription: "相手サーバがContent-Lengthを返さない場合、プレビューは生成されません。" + userAgent: "User-Agent" + userAgentDescription: "プレビュー取得時に使用されるUser-Agentを設定します。空欄の場合、デフォルトのUser-Agentが使用されます。" + summaryProxy: "プレビューを生成するプロキシのエンドポイント" + summaryProxyDescription: "Misskey本体ではなく、サマリープロキシを使用してプレビューを生成します。" + summaryProxyDescription2: "プロキシには下記パラメータがクエリ文字列として連携されます。プロキシ側がこれらをサポートしない場合、設定値は無視されます。" diff --git a/packages/backend/migration/1710512074000-url-preview-meta.js b/packages/backend/migration/1710512074000-url-preview-meta.js new file mode 100644 index 0000000000..8af521bbf4 --- /dev/null +++ b/packages/backend/migration/1710512074000-url-preview-meta.js @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class UrlPreviewMeta1710512074000 { + name = 'UrlPreviewMeta1710512074000' + + async up(queryRunner) { + await queryRunner.query(` + alter table meta + rename column "summalyProxy" to "urlPreviewSummaryProxyUrl"; + alter table meta + add "urlPreviewEnabled" boolean default true not null; + alter table meta + add "urlPreviewTimeout" integer default 10000 not null; + alter table meta + add "urlPreviewMaximumContentLength" bigint default 10485760 not null; + alter table meta + add "urlPreviewRequireContentLength" boolean default false not null; + alter table meta + add "urlPreviewUserAgent" varchar(1024) default null; + `); + } + + async down(queryRunner) { + await queryRunner.query(` + alter table meta + rename column "urlPreviewSummaryProxyUrl" to "summalyProxy"; + alter table meta + drop column "urlPreviewEnabled"; + alter table meta + drop column "urlPreviewTimeout"; + alter table meta + drop column "urlPreviewMaximumContentLength"; + alter table meta + drop column "urlPreviewRequireContentLength"; + alter table meta + drop column "urlPreviewUserAgent"; + `); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index eaad96d5f6..d64fcc3d2a 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -80,7 +80,7 @@ "@fastify/static": "6.12.0", "@fastify/view": "8.2.0", "@misskey-dev/sharp-read-bmp": "1.2.0", - "@misskey-dev/summaly": "5.0.3", + "@misskey-dev/summaly": "5.1.0", "@nestjs/common": "10.3.3", "@nestjs/core": "10.3.3", "@nestjs/testing": "10.3.3", diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index b50d76288f..9d054ab6a1 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -111,6 +111,7 @@ export class MetaEntityService { policies: { ...DEFAULT_POLICIES, ...instance.policies }, mediaProxy: this.config.mediaProxy, + enableUrlPreview: instance.urlPreviewEnabled, }; return packed; diff --git a/packages/backend/src/models/Meta.ts b/packages/backend/src/models/Meta.ts index 66f19ce197..04a34bbbb4 100644 --- a/packages/backend/src/models/Meta.ts +++ b/packages/backend/src/models/Meta.ts @@ -277,12 +277,6 @@ export class MiMeta { }) public enableSensitiveMediaDetectionForVideos: boolean; - @Column('varchar', { - length: 1024, - nullable: true, - }) - public summalyProxy: string | null; - @Column('boolean', { default: false, }) @@ -588,4 +582,36 @@ export class MiMeta { default: 0, }) public notesPerOneAd: number; + + @Column('boolean', { + default: true, + }) + public urlPreviewEnabled: boolean; + + @Column('integer', { + default: 10000, + }) + public urlPreviewTimeout: number; + + @Column('bigint', { + default: 1024 * 1024 * 10, + }) + public urlPreviewMaximumContentLength: number; + + @Column('boolean', { + default: true, + }) + public urlPreviewRequireContentLength: boolean; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public urlPreviewSummaryProxyUrl: string | null; + + @Column('varchar', { + length: 1024, + nullable: true, + }) + public urlPreviewUserAgent: string | null; } diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 17789f3b46..473339a1ad 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -207,6 +207,10 @@ export const packedMetaLiteSchema = { type: 'string', optional: false, nullable: false, }, + enableUrlPreview: { + type: 'boolean', + optional: false, nullable: false, + }, backgroundImageUrl: { type: 'string', optional: false, nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index 88c5907bcc..f4ff573271 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -434,6 +434,8 @@ export const meta = { summalyProxy: { type: 'string', optional: false, nullable: true, + deprecated: true, + description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.', }, themeColor: { type: 'string', @@ -451,6 +453,30 @@ export const meta = { type: 'string', optional: false, nullable: false, }, + urlPreviewEnabled: { + type: 'boolean', + optional: false, nullable: false, + }, + urlPreviewTimeout: { + type: 'number', + optional: false, nullable: false, + }, + urlPreviewMaximumContentLength: { + type: 'number', + optional: false, nullable: false, + }, + urlPreviewRequireContentLength: { + type: 'boolean', + optional: false, nullable: false, + }, + urlPreviewUserAgent: { + type: 'string', + optional: false, nullable: true, + }, + urlPreviewSummaryProxyUrl: { + type: 'string', + optional: false, nullable: true, + }, }, }, } as const; @@ -533,7 +559,6 @@ export default class extends Endpoint { // eslint- setSensitiveFlagAutomatically: instance.setSensitiveFlagAutomatically, enableSensitiveMediaDetectionForVideos: instance.enableSensitiveMediaDetectionForVideos, proxyAccountId: instance.proxyAccountId, - summalyProxy: instance.summalyProxy, email: instance.email, smtpSecure: instance.smtpSecure, smtpHost: instance.smtpHost, @@ -577,6 +602,13 @@ export default class extends Endpoint { // eslint- perUserHomeTimelineCacheMax: instance.perUserHomeTimelineCacheMax, perUserListTimelineCacheMax: instance.perUserListTimelineCacheMax, notesPerOneAd: instance.notesPerOneAd, + summalyProxy: instance.urlPreviewSummaryProxyUrl, + urlPreviewEnabled: instance.urlPreviewEnabled, + urlPreviewTimeout: instance.urlPreviewTimeout, + urlPreviewMaximumContentLength: instance.urlPreviewMaximumContentLength, + urlPreviewRequireContentLength: instance.urlPreviewRequireContentLength, + urlPreviewUserAgent: instance.urlPreviewUserAgent, + urlPreviewSummaryProxyUrl: instance.urlPreviewSummaryProxyUrl, }; }); } diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index bffceef815..2f62d30ada 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -90,7 +90,6 @@ export const paramDef = { type: 'string', }, }, - summalyProxy: { type: 'string', nullable: true }, deeplAuthKey: { type: 'string', nullable: true }, deeplIsPro: { type: 'boolean' }, enableEmail: { type: 'boolean' }, @@ -150,6 +149,16 @@ export const paramDef = { type: 'string', }, }, + summalyProxy: { + type: 'string', nullable: true, + description: '[Deprecated] Use "urlPreviewSummaryProxyUrl" instead.', + }, + urlPreviewEnabled: { type: 'boolean' }, + urlPreviewTimeout: { type: 'integer' }, + urlPreviewMaximumContentLength: { type: 'integer' }, + urlPreviewRequireContentLength: { type: 'boolean' }, + urlPreviewUserAgent: { type: 'string', nullable: true }, + urlPreviewSummaryProxyUrl: { type: 'string', nullable: true }, }, required: [], } as const; @@ -353,10 +362,6 @@ export default class extends Endpoint { // eslint- set.langs = ps.langs.filter(Boolean); } - if (ps.summalyProxy !== undefined) { - set.summalyProxy = ps.summalyProxy; - } - if (ps.enableEmail !== undefined) { set.enableEmail = ps.enableEmail; } @@ -581,6 +586,32 @@ export default class extends Endpoint { // eslint- set.bannedEmailDomains = ps.bannedEmailDomains; } + if (ps.urlPreviewEnabled !== undefined) { + set.urlPreviewEnabled = ps.urlPreviewEnabled; + } + + if (ps.urlPreviewTimeout !== undefined) { + set.urlPreviewTimeout = ps.urlPreviewTimeout; + } + + if (ps.urlPreviewMaximumContentLength !== undefined) { + set.urlPreviewMaximumContentLength = ps.urlPreviewMaximumContentLength; + } + + if (ps.urlPreviewRequireContentLength !== undefined) { + set.urlPreviewRequireContentLength = ps.urlPreviewRequireContentLength; + } + + if (ps.urlPreviewUserAgent !== undefined) { + const value = (ps.urlPreviewUserAgent ?? '').trim(); + set.urlPreviewUserAgent = value === '' ? null : ps.urlPreviewUserAgent; + } + + if (ps.summalyProxy !== undefined || ps.urlPreviewSummaryProxyUrl !== undefined) { + const value = ((ps.urlPreviewSummaryProxyUrl ?? ps.summalyProxy) ?? '').trim(); + set.urlPreviewSummaryProxyUrl = value === '' ? null : value; + } + const before = await this.metaService.fetch(true); await this.metaService.update(set); diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index c6a96e94cb..8f8f08a305 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { summaly } from '@misskey-dev/summaly'; +import { SummalyResult } from '@misskey-dev/summaly/built/summary.js'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; @@ -14,6 +15,7 @@ import { query } from '@/misc/prelude/url.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import { ApiError } from '@/server/api/error.js'; +import { MiMeta } from '@/models/Meta.js'; import type { FastifyRequest, FastifyReply } from 'fastify'; @Injectable() @@ -62,24 +64,25 @@ export class UrlPreviewService { const meta = await this.metaService.fetch(); - this.logger.info(meta.summalyProxy + if (!meta.urlPreviewEnabled) { + reply.code(403); + return { + error: new ApiError({ + message: 'URL preview is disabled', + code: 'URL_PREVIEW_DISABLED', + id: '58b36e13-d2f5-0323-b0c6-76aa9dabefb8', + }), + }; + } + + this.logger.info(meta.urlPreviewSummaryProxyUrl ? `(Proxy) Getting preview of ${url}@${lang} ...` : `Getting preview of ${url}@${lang} ...`); + try { - const summary = meta.summalyProxy ? - await this.httpRequestService.getJson>(`${meta.summalyProxy}?${query({ - url: url, - lang: lang ?? 'ja-JP', - })}`) - : - await summaly(url, { - followRedirects: false, - lang: lang ?? 'ja-JP', - agent: this.config.proxy ? { - http: this.httpRequestService.httpAgent, - https: this.httpRequestService.httpsAgent, - } : undefined, - }); + const summary = meta.urlPreviewSummaryProxyUrl + ? await this.fetchSummaryFromProxy(url, meta, lang) + : await this.fetchSummary(url, meta, lang); this.logger.succ(`Got preview of ${url}: ${summary.title}`); @@ -100,6 +103,7 @@ export class UrlPreviewService { return summary; } catch (err) { this.logger.warn(`Failed to get preview of ${url}: ${err}`); + reply.code(422); reply.header('Cache-Control', 'max-age=86400, immutable'); return { @@ -111,4 +115,37 @@ export class UrlPreviewService { }; } } + + private fetchSummary(url: string, meta: MiMeta, lang?: string): Promise { + const agent = this.config.proxy + ? { + http: this.httpRequestService.httpAgent, + https: this.httpRequestService.httpsAgent, + } + : undefined; + + return summaly(url, { + followRedirects: false, + lang: lang ?? 'ja-JP', + agent: agent, + userAgent: meta.urlPreviewUserAgent ?? undefined, + operationTimeout: meta.urlPreviewTimeout, + contentLengthLimit: meta.urlPreviewMaximumContentLength, + contentLengthRequired: meta.urlPreviewRequireContentLength, + }); + } + + private fetchSummaryFromProxy(url: string, meta: MiMeta, lang?: string): Promise { + const proxy = meta.urlPreviewSummaryProxyUrl!; + const queryStr = query({ + url: url, + lang: lang ?? 'ja-JP', + userAgent: meta.urlPreviewUserAgent ?? undefined, + operationTimeout: meta.urlPreviewTimeout, + contentLengthLimit: meta.urlPreviewMaximumContentLength, + contentLengthRequired: meta.urlPreviewRequireContentLength, + }); + + return this.httpRequestService.getJson(`${proxy}?${queryStr}`); + } } diff --git a/packages/frontend/src/components/MkLink.vue b/packages/frontend/src/components/MkLink.vue index 3f7aba2fe4..ca875242b4 100644 --- a/packages/frontend/src/components/MkLink.vue +++ b/packages/frontend/src/components/MkLink.vue @@ -18,6 +18,7 @@ import { defineAsyncComponent, ref } from 'vue'; import { url as local } from '@/config.js'; import { useTooltip } from '@/scripts/use-tooltip.js'; import * as os from '@/os.js'; +import { isEnabledUrlPreview } from '@/instance.js'; const props = withDefaults(defineProps<{ url: string; @@ -31,13 +32,15 @@ const target = self ? null : '_blank'; const el = ref(); -useTooltip(el, (showing) => { - os.popup(defineAsyncComponent(() => import('@/components/MkUrlPreviewPopup.vue')), { - showing, - url: props.url, - source: el.value instanceof HTMLElement ? el.value : el.value?.$el, - }, {}, 'closed'); -}); +if (isEnabledUrlPreview.value) { + useTooltip(el, (showing) => { + os.popup(defineAsyncComponent(() => import('@/components/MkUrlPreviewPopup.vue')), { + showing, + url: props.url, + source: el.value instanceof HTMLElement ? el.value : el.value?.$el, + }, {}, 'closed'); + }); +} diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 636bc62aaa..c6e36d80aa 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4810,6 +4810,7 @@ export type components = { enableServiceWorker: boolean; translatorAvailable: boolean; mediaProxy: string; + enableUrlPreview: boolean; backgroundImageUrl: string | null; impressumUrl: string | null; logoImageUrl: string | null; @@ -4962,11 +4963,21 @@ export type operations = { objectStorageS3ForcePathStyle: boolean; privacyPolicyUrl: string | null; repositoryUrl: string | null; + /** + * @deprecated + * @description [Deprecated] Use "urlPreviewSummaryProxyUrl" instead. + */ summalyProxy: string | null; themeColor: string | null; tosUrl: string | null; uri: string; version: string; + urlPreviewEnabled: boolean; + urlPreviewTimeout: number; + urlPreviewMaximumContentLength: number; + urlPreviewRequireContentLength: boolean; + urlPreviewUserAgent: string | null; + urlPreviewSummaryProxyUrl: string | null; }; }; }; @@ -8862,7 +8873,6 @@ export type operations = { maintainerName?: string | null; maintainerEmail?: string | null; langs?: string[]; - summalyProxy?: string | null; deeplAuthKey?: string | null; deeplIsPro?: boolean; enableEmail?: boolean; @@ -8916,6 +8926,14 @@ export type operations = { perUserListTimelineCacheMax?: number; notesPerOneAd?: number; silencedHosts?: string[] | null; + /** @description [Deprecated] Use "urlPreviewSummaryProxyUrl" instead. */ + summalyProxy?: string | null; + urlPreviewEnabled?: boolean; + urlPreviewTimeout?: number; + urlPreviewMaximumContentLength?: number; + urlPreviewRequireContentLength?: boolean; + urlPreviewUserAgent?: string | null; + urlPreviewSummaryProxyUrl?: string | null; }; }; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4c1e228a95..383b31b1f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -117,8 +117,8 @@ importers: specifier: 1.2.0 version: 1.2.0 '@misskey-dev/summaly': - specifier: 5.0.3 - version: 5.0.3 + specifier: 5.1.0 + version: 5.1.0 '@nestjs/common': specifier: 10.3.3 version: 10.3.3(reflect-metadata@0.2.1)(rxjs@7.8.1) @@ -4772,6 +4772,20 @@ packages: jschardet: 3.0.0 private-ip: 2.3.3 trace-redirect: 1.0.6 + dev: true + + /@misskey-dev/summaly@5.1.0: + resolution: {integrity: sha512-WAUrgX3/z4h4aI8Y/WVwmJcJ6Fa1Zf2LJCSS651t9MHoWVGABLsQ2KCXRGmlpk4i+cMDNIwweObUroosE7j8rg==} + dependencies: + cheerio: 1.0.0-rc.12 + escape-regexp: 0.0.1 + got: 12.6.1 + html-entities: 2.3.2 + iconv-lite: 0.6.3 + jschardet: 3.0.0 + private-ip: 2.3.3 + trace-redirect: 1.0.6 + dev: false /@mole-inc/bin-wrapper@8.0.1: resolution: {integrity: sha512-sTGoeZnjI8N4KS+sW2AN95gDBErhAguvkw/tWdCjeM8bvxpz5lqrnd0vOJABA1A+Ic3zED7PYoLP/RANLgVotA==}