From f086bc23f59bb2d201e8423e940ac41611e4d25a Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 6 Jul 2024 11:35:52 +0100 Subject: [PATCH 01/11] don't send real-time updates of replies to blocked users this is in line with the behaviour of a note's detailed view, which does not show me replies from users who blocked me --- packages/backend/src/core/GlobalEventService.ts | 1 + packages/backend/src/core/NoteCreateService.ts | 1 + packages/backend/src/server/api/stream/Connection.ts | 12 ++++++++++++ 3 files changed, 14 insertions(+) diff --git a/packages/backend/src/core/GlobalEventService.ts b/packages/backend/src/core/GlobalEventService.ts index 22871adb16..4c8a3dc050 100644 --- a/packages/backend/src/core/GlobalEventService.ts +++ b/packages/backend/src/core/GlobalEventService.ts @@ -135,6 +135,7 @@ export interface NoteEventTypes { }; replied: { id: MiNote['id']; + userId: MiUser['id']; }; } type NoteStreamEventTypes = { diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 41efa76f3f..44b066444d 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -831,6 +831,7 @@ export class NoteCreateService implements OnApplicationShutdown { if (data.reply) { this.globalEventService.publishNoteStream(data.reply.id, 'replied', { id: note.id, + userId: user.id, }); // 通知 if (data.reply.userHost === null) { diff --git a/packages/backend/src/server/api/stream/Connection.ts b/packages/backend/src/server/api/stream/Connection.ts index 41c0feccc7..7dd7db24e5 100644 --- a/packages/backend/src/server/api/stream/Connection.ts +++ b/packages/backend/src/server/api/stream/Connection.ts @@ -201,6 +201,18 @@ export default class Connection { @bindThis private async onNoteStreamMessage(data: GlobalEvents['note']['payload']) { + // we must not send to the frontend information about notes from + // users who blocked the logged-in user, even when they're replies + // to notes the logged-in user can see + if (data.type === 'replied') { + const noteUserId = data.body.body.userId; + if (noteUserId !== null) { + if (this.userIdsWhoBlockingMe.has(noteUserId)) { + return; + } + } + } + this.sendMessageToWs('noteUpdated', { id: data.body.id, type: data.type, From 6e1e3bc0eae4f6ed8ae3e60088e099bf5074d01f Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 21 Jul 2024 11:02:46 +0100 Subject: [PATCH 02/11] pass redis config as-is to postgres cache also, explicitly state that `path` is a valid config option for redis (tells `ioredis` to connect via UNIX socked instead of TCP socket) --- packages/backend/src/config.ts | 5 +++-- packages/backend/src/postgres.ts | 6 +----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 58c4d028aa..92774328cc 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -12,9 +12,10 @@ import * as Sentry from '@sentry/node'; import type { RedisOptions } from 'ioredis'; type RedisOptionsSource = Partial & { - host: string; - port: number; + host?: string; + port?: number; family?: number; + path?: string, pass: string; db?: number; prefix?: string; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 4a1b42383f..b7c418fa59 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -236,12 +236,8 @@ export function createPostgresDataSource(config: Config) { cache: !config.db.disableCache && process.env.NODE_ENV !== 'test' ? { // dbをcloseしても何故かredisのコネクションが内部的に残り続けるようで、テストの際に支障が出るため無効にする(キャッシュも含めてテストしたいため本当は有効にしたいが...) type: 'ioredis', options: { - host: config.redis.host, - port: config.redis.port, - family: config.redis.family ?? 0, - password: config.redis.pass, + ...config.redis, keyPrefix: `${config.redis.prefix}:query:`, - db: config.redis.db ?? 0, }, } : false, logging: log, From 3eff85a3d32c1f4ce4e7960fc1cfc0014a6b8467 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 21 Jul 2024 11:23:49 +0100 Subject: [PATCH 03/11] error out when we can't spawn workers - fixes #586 Setting `clusterLimit` to 0 means no workers are started, which usually breaks things. Also, some "hardening" things may prevent node from seeing how many CPUs the machine has, which has the same effect. With this commit we provide hopefully-useful error messages. --- packages/backend/src/boot/master.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 303ba94207..879249e231 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -180,7 +180,16 @@ async function connectDb(): Promise { */ async function spawnWorkers(limit = 1) { - const workers = Math.min(limit, os.cpus().length); + const cpuCount = os.cpus().length; + const workers = Math.min(limit, cpuCount); + if (workers === 0) { + const cause = cpuCount === 0 + ? 'you seem to have no CPUs (this may be caused by some "hardening" system)' + : "`config.clusterLimit` is 0 (if you don't want to use clustering, set the environment variable `MK_DISABLE_CLUSTERING` to a non-empty value instead)"; + bootLogger.error(`Configuration error: we can't create workers, ${cause}`, null, true); + process.exit(1); + } + bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`); await Promise.all([...Array(workers)].map(spawnWorker)); bootLogger.succ('All workers started'); From f18f30cb7914b7d8f7123f8e293fe8fc113ed474 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sun, 21 Jul 2024 13:29:54 +0100 Subject: [PATCH 04/11] ignore cpuCount when it's 0 - fixes #586 seems more useful this way --- packages/backend/src/boot/master.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 879249e231..f757ed64b9 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -112,6 +112,11 @@ export async function masterMain() { await server(); } + if (config.clusterLimit === 0) { + bootLogger.error("Configuration error: we can't create workers, `config.clusterLimit` is 0 (if you don't want to use clustering, set the environment variable `MK_DISABLE_CLUSTERING` to a non-empty value instead)", null, true); + process.exit(1); + } + await spawnWorkers(config.clusterLimit); } @@ -181,14 +186,8 @@ async function connectDb(): Promise { async function spawnWorkers(limit = 1) { const cpuCount = os.cpus().length; - const workers = Math.min(limit, cpuCount); - if (workers === 0) { - const cause = cpuCount === 0 - ? 'you seem to have no CPUs (this may be caused by some "hardening" system)' - : "`config.clusterLimit` is 0 (if you don't want to use clustering, set the environment variable `MK_DISABLE_CLUSTERING` to a non-empty value instead)"; - bootLogger.error(`Configuration error: we can't create workers, ${cause}`, null, true); - process.exit(1); - } + // in some weird environments, node can't count the CPUs; we trust the config in those cases + const workers = cpuCount === 0 ? limit : Math.min(limit, cpuCount); bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`); await Promise.all([...Array(workers)].map(spawnWorker)); From 015d601527c832608f499637731d624d642b64be Mon Sep 17 00:00:00 2001 From: dakkar Date: Mon, 29 Jul 2024 18:46:19 +0100 Subject: [PATCH 05/11] use correct user for preview/link of collapsed parent - fixes #575 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yes, again… I had only fixed it for `SkNote` ☹ --- packages/frontend/src/components/MkNote.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend/src/components/MkNote.vue b/packages/frontend/src/components/MkNote.vue index b8ce7ed830..b6ded3167b 100644 --- a/packages/frontend/src/components/MkNote.vue +++ b/packages/frontend/src/components/MkNote.vue @@ -14,7 +14,7 @@ SPDX-License-Identifier: AGPL-3.0-only >
- + : From 858ba188768017764c61c4a5591bdf2524a850e7 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Thu, 1 Aug 2024 08:24:50 -0400 Subject: [PATCH 06/11] fix: pass current user into `InstanceEntityService.packMany` --- packages/backend/src/core/entities/InstanceEntityService.ts | 3 ++- .../backend/src/server/api/endpoints/federation/instances.ts | 2 +- packages/backend/src/server/api/endpoints/federation/stats.ts | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/core/entities/InstanceEntityService.ts b/packages/backend/src/core/entities/InstanceEntityService.ts index 002a93397d..2dfb8d5356 100644 --- a/packages/backend/src/core/entities/InstanceEntityService.ts +++ b/packages/backend/src/core/entities/InstanceEntityService.ts @@ -63,8 +63,9 @@ export class InstanceEntityService { @bindThis public packMany( instances: MiInstance[], + me?: { id: MiUser['id']; } | null | undefined, ) { - return Promise.all(instances.map(x => this.pack(x))); + return Promise.all(instances.map(x => this.pack(x, me))); } } diff --git a/packages/backend/src/server/api/endpoints/federation/instances.ts b/packages/backend/src/server/api/endpoints/federation/instances.ts index c3f2247b69..c1ce3f2238 100644 --- a/packages/backend/src/server/api/endpoints/federation/instances.ts +++ b/packages/backend/src/server/api/endpoints/federation/instances.ts @@ -203,7 +203,7 @@ export default class extends Endpoint { // eslint- const instances = await query.limit(ps.limit).offset(ps.offset).getMany(); - return await this.instanceEntityService.packMany(instances); + return await this.instanceEntityService.packMany(instances, me); }); } } diff --git a/packages/backend/src/server/api/endpoints/federation/stats.ts b/packages/backend/src/server/api/endpoints/federation/stats.ts index bac54970ab..69900bff9a 100644 --- a/packages/backend/src/server/api/endpoints/federation/stats.ts +++ b/packages/backend/src/server/api/endpoints/federation/stats.ts @@ -107,9 +107,9 @@ export default class extends Endpoint { // eslint- const gotPubCount = topPubInstances.map(x => x.followingCount).reduce((a, b) => a + b, 0); return await awaitAll({ - topSubInstances: this.instanceEntityService.packMany(topSubInstances), + topSubInstances: this.instanceEntityService.packMany(topSubInstances, me), otherFollowersCount: Math.max(0, allSubCount - gotSubCount), - topPubInstances: this.instanceEntityService.packMany(topPubInstances), + topPubInstances: this.instanceEntityService.packMany(topPubInstances, me), otherFollowingCount: Math.max(0, allPubCount - gotPubCount), }); }); From 16601fa7663d8dcc865d36cd656cd9c27152ba62 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 3 Aug 2024 14:56:29 +0100 Subject: [PATCH 07/11] allow overriding more config settings via env vars `id` in particular is necessary if one wants to do a working basic configuration without a config file at all, thanks to @4censord for making me notice --- packages/backend/src/config.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 92774328cc..f91791cc54 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -437,7 +437,7 @@ function applyEnvOverrides(config: Source) { // these are all the settings that can be overridden - _apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts']]); + _apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications']]); _apply_top(['db', ['host', 'port', 'db', 'user', 'pass']]); _apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]); _apply_top([ @@ -448,7 +448,8 @@ function applyEnvOverrides(config: Source) { _apply_top([['sentryForFrontend', 'sentryForBackend'], 'options', ['dsn', 'profileSampleRate', 'serverName', 'includeLocalVariables', 'proxy', 'keepAlive', 'caCerts']]); _apply_top(['sentryForBackend', 'enableNodeProfiling']); _apply_top([['clusterLimit', 'deliverJobConcurrency', 'inboxJobConcurrency', 'relashionshipJobConcurrency', 'deliverJobPerSec', 'inboxJobPerSec', 'relashionshipJobPerSec', 'deliverJobMaxAttempts', 'inboxJobMaxAttempts']]); - _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'videoThumbnailGenerator']]); + _apply_top([['outgoingAddress', 'outgoingAddressFamily', 'proxy', 'proxySmtp', 'mediaProxy', 'proxyRemoteFiles','videoThumbnailGenerator']]); _apply_top([['maxFileSize', 'maxNoteLength', 'pidFile']]); _apply_top(['import', ['downloadTimeout', 'maxFileSize']]); + _apply_top([['signToActivityPubGet', 'checkActivityPubGetSignature']]); } From 955fda6e1c1d2c669d43434282c4f19720eb801f Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 3 Aug 2024 15:29:09 +0100 Subject: [PATCH 08/11] also allow `db.disableCache` --- packages/backend/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index f91791cc54..9596b3a722 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -438,7 +438,7 @@ function applyEnvOverrides(config: Source) { // these are all the settings that can be overridden _apply_top([['url', 'port', 'socket', 'chmodSocket', 'disableHsts', 'id', 'dbReplications']]); - _apply_top(['db', ['host', 'port', 'db', 'user', 'pass']]); + _apply_top(['db', ['host', 'port', 'db', 'user', 'pass', 'disableCache']]); _apply_top(['dbSlaves', Array.from((config.dbSlaves ?? []).keys()), ['host', 'port', 'db', 'user', 'pass']]); _apply_top([ ['redis', 'redisForPubsub', 'redisForJobQueue', 'redisForTimelines'], From 12aeaa5f956d1c66f16e8092998f14bf64c045b6 Mon Sep 17 00:00:00 2001 From: 4censord Date: Sat, 3 Aug 2024 17:42:59 +0200 Subject: [PATCH 09/11] Complain if no config-files are loaded --- package.json | 2 +- packages/backend/src/config.ts | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 519a8c453d..f60fd789db 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "build": "pnpm build-pre && pnpm -r build && pnpm build-assets", "build-storybook": "pnpm --filter frontend build-storybook", "build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api", - "start": "pnpm check:connect && cd packages/backend && node ./built/boot/entry.js", + "start": "pnpm check:connect && cd packages/backend && MK_WARNED_ABOUT_CONFIG=true node ./built/boot/entry.js", "start:test": "cd packages/backend && cross-env NODE_ENV=test node ./built/boot/entry.js", "init": "pnpm migrate", "migrate": "cd packages/backend && pnpm migrate", diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 92774328cc..15fc6fff28 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -222,8 +222,15 @@ export function loadConfig(): Config { JSON.parse(fs.readFileSync(`${_dirname}/../../../built/_vite_/manifest.json`, 'utf-8')) : { 'src/_boot_.ts': { file: 'src/_boot_.ts' } }; - const config = globSync(path).sort() - .map(path => fs.readFileSync(path, 'utf-8')) + const configFiles = globSync(path).sort(); + + if (configFiles.length === 0 + && !process.env['MK_WARNED_ABOUT_CONFIG']) { + console.log('No config files loaded, check if this is intentional'); + process.env['MK_WARNED_ABOUT_CONFIG'] = true; + } + + const config = configFiles.map(path => fs.readFileSync(path, 'utf-8')) .map(contents => yaml.load(contents) as Source) .reduce( (acc: Source, cur: Source) => Object.assign(acc, cur), From e1b0faa18f3b6d7b365f5fc10caaecd6e480644d Mon Sep 17 00:00:00 2001 From: 4censord Date: Sat, 3 Aug 2024 15:21:44 +0200 Subject: [PATCH 10/11] Fall back to port 3000 if none is specified Originally, because `parseInt` can return `NaN`, sharkey would happily try to listen on port `NaN`. --- packages/backend/src/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 15fc6fff28..44b0a05f24 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -256,7 +256,7 @@ export function loadConfig(): Config { version, publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl, url: url.origin, - port: config.port ?? parseInt(process.env.PORT ?? '', 10), + port: config.port ?? parseInt(process.env.PORT ?? '3000', 10), socket: config.socket, chmodSocket: config.chmodSocket, disableHsts: config.disableHsts, From 58d329e3bd5d0a8b620a8f9716fed4006231c6b0 Mon Sep 17 00:00:00 2001 From: dakkar Date: Mon, 5 Aug 2024 10:12:27 +0100 Subject: [PATCH 11/11] make sure that `sensitive` is a boolean - fixes #596 Some remote instances may send garbage. If the `sensitive` value passed to `driveService.uploadFromUrl` is not a boolean, eventually we'll get an exception from the database complaining about that. This change avoids that problem. --- packages/backend/src/core/activitypub/models/ApImageService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/models/ApImageService.ts b/packages/backend/src/core/activitypub/models/ApImageService.ts index 3d98c5b764..b281ac9728 100644 --- a/packages/backend/src/core/activitypub/models/ApImageService.ts +++ b/packages/backend/src/core/activitypub/models/ApImageService.ts @@ -82,7 +82,7 @@ export class ApImageService { url: image.url, user: actor, uri: image.url, - sensitive: image.sensitive, + sensitive: !!(image.sensitive), isLink: !shouldBeCached, comment: truncate(image.name ?? undefined, DB_MAX_IMAGE_COMMENT_LENGTH), });