diff --git a/packages/backend/package.json b/packages/backend/package.json index 19547c5033..00baef56d8 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -93,6 +93,7 @@ "@swc/core": "1.6.6", "@transfem-org/sfm-js": "0.24.5", "@twemoji/parser": "15.1.1", + "@types/psl": "^1.1.3", "accepts": "1.3.8", "ajv": "8.17.1", "archiver": "7.0.1", @@ -135,9 +136,9 @@ "json5": "2.2.3", "jsonld": "8.3.2", "jsrsasign": "11.1.0", + "juice": "11.0.0", "megalodon": "workspace:*", "meilisearch": "0.42.0", - "juice": "11.0.0", "microformats-parser": "2.0.2", "mime-types": "2.1.35", "misskey-js": "workspace:*", @@ -158,6 +159,7 @@ "probe-image-size": "7.2.3", "promise-limit": "2.7.0", "proxy-addr": "^2.0.7", + "psl": "^1.13.0", "pug": "3.0.3", "punycode": "2.3.1", "qrcode": "1.5.4", diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 4c6d539e16..dda4bd5fbd 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -4,9 +4,10 @@ */ import { URL } from 'node:url'; -import { toASCII } from 'punycode'; +import punycode from 'punycode/punycode.js'; import { Inject, Injectable } from '@nestjs/common'; import RE2 from 're2'; +import psl from 'psl'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { bindThis } from '@/decorators.js'; @@ -106,13 +107,13 @@ export class UtilityService { @bindThis public toPuny(host: string): string { - return toASCII(host.toLowerCase()); + return punycode.toASCII(host.toLowerCase()); } @bindThis public toPunyNullable(host: string | null | undefined): string | null { if (host == null) return null; - return toASCII(host.toLowerCase()); + return punycode.toASCII(host.toLowerCase()); } @bindThis @@ -122,6 +123,26 @@ export class UtilityService { return host; } + private specialSuffix(hostname: string): string | null { + // masto.host provides domain names for its clients, we have to + // treat it as if it were a public suffix + const mastoHost = hostname.match(/\.?([a-zA-Z0-9-]+\.masto\.host)$/i); + if (mastoHost) { + return mastoHost[1]; + } + + return null; + } + + @bindThis + public punyHostPSLDomain(url: string): string { + const urlObj = new URL(url); + const hostname = urlObj.hostname; + const domain = this.specialSuffix(hostname) ?? psl.get(hostname) ?? hostname; + const host = `${this.toPuny(domain)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`; + return host; + } + public isFederationAllowedHost(host: string): boolean { if (this.meta.federation === 'none') return false; if (this.meta.federation === 'specified' && !this.meta.federationHosts.some(x => `.${host.toLowerCase()}`.endsWith(`.${x}`))) return false; diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index eeff73385b..0dea615cb0 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -243,7 +243,7 @@ export class ApRequestService { if (alternate) { const href = alternate.getAttribute('href'); if (href) { - if (this.utilityService.punyHost(url) === this.utilityService.punyHost(href)) { + if (this.utilityService.punyHostPSLDomain(url) === this.utilityService.punyHostPSLDomain(href)) { return await this.signedGet(href, user, false); } } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index d4964d544d..e56b9ebc99 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -131,7 +131,7 @@ export class Resolver { throw new UnrecoverableError(`invalid AP object ${value}: missing id`); } - if (this.utilityService.punyHost(object.id) !== this.utilityService.punyHost(value)) { + if (this.utilityService.punyHostPSLDomain(object.id) !== this.utilityService.punyHostPSLDomain(value)) { throw new UnrecoverableError(`invalid AP object ${value}: id ${object.id} has different host`); } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 3d4a33ded2..b8aa67e9ea 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -192,8 +192,8 @@ export class ApNoteService { throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${entryUri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) { - throw new Error(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`); + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) { + throw new UnrecoverableError(`note url <> uri host mismatch: ${url} <> ${note.id} in ${entryUri}`); } } @@ -444,7 +444,7 @@ export class ApNoteService { throw new UnrecoverableError(`unexpected schema of note.url ${url} in ${noteUri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) { + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(note.id)) { throw new UnrecoverableError(`note url <> id host mismatch: ${url} <> ${note.id} in ${noteUri}`); } } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index cd6078b2ed..598486cd84 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -138,7 +138,7 @@ export class ApPersonService implements OnModuleInit { */ @bindThis private validateActor(x: IObject, uri: string): IActor { - const expectHost = this.utilityService.punyHost(uri); + const expectHost = this.utilityService.punyHostPSLDomain(uri); if (!isActor(x)) { throw new UnrecoverableError(`invalid Actor type '${x.type}' in ${uri}`); @@ -152,7 +152,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox type`); } - const inboxHost = this.utilityService.punyHost(x.inbox); + const inboxHost = this.utilityService.punyHostPSLDomain(x.inbox); if (inboxHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong inbox ${inboxHost}`); } @@ -160,7 +160,7 @@ export class ApPersonService implements OnModuleInit { const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined); if (sharedInboxObject != null) { const sharedInbox = getApId(sharedInboxObject); - if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHost(sharedInbox) === expectHost)) { + if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHostPSLDomain(sharedInbox) === expectHost)) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong shared inbox ${sharedInbox}`); } } @@ -170,7 +170,7 @@ export class ApPersonService implements OnModuleInit { if (xCollection != null) { const collectionUri = getApId(xCollection); if (typeof collectionUri === 'string' && collectionUri.length > 0) { - if (this.utilityService.punyHost(collectionUri) !== expectHost) { + if (this.utilityService.punyHostPSLDomain(collectionUri) !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong ${collection} ${collectionUri}`); } } else if (collectionUri != null) { @@ -202,7 +202,7 @@ export class ApPersonService implements OnModuleInit { x.summary = truncate(x.summary, summaryLength); } - const idHost = this.utilityService.punyHost(x.id); + const idHost = this.utilityService.punyHostPSLDomain(x.id); if (idHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong id ${x.id}`); } @@ -212,7 +212,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id type`); } - const publicKeyIdHost = this.utilityService.punyHost(x.publicKey.id); + const publicKeyIdHost = this.utilityService.punyHostPSLDomain(x.publicKey.id); if (publicKeyIdHost !== expectHost) { throw new UnrecoverableError(`invalid Actor ${uri} - wrong publicKey.id ${x.publicKey.id}`); } @@ -351,7 +351,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); } } @@ -563,7 +563,7 @@ export class ApPersonService implements OnModuleInit { throw new UnrecoverableError(`unexpected schema of person url ${url} in ${uri}`); } - if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { + if (this.utilityService.punyHostPSLDomain(url) !== this.utilityService.punyHostPSLDomain(person.id)) { throw new UnrecoverableError(`person url <> uri host mismatch: ${url} <> ${person.id} in ${uri}`); } } diff --git a/packages/backend/test/unit/UtilityService.ts b/packages/backend/test/unit/UtilityService.ts new file mode 100644 index 0000000000..d86e794f2f --- /dev/null +++ b/packages/backend/test/unit/UtilityService.ts @@ -0,0 +1,43 @@ +import * as assert from 'assert'; +import { Test } from '@nestjs/testing'; + +import { CoreModule } from '@/core/CoreModule.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { GlobalModule } from '@/GlobalModule.js'; + +describe('UtilityService', () => { + let utilityService: UtilityService; + + beforeAll(async () => { + const app = await Test.createTestingModule({ + imports: [GlobalModule, CoreModule], + }).compile(); + utilityService = app.get(UtilityService); + }); + + describe('punyHost', () => { + test('simple', () => { + assert.equal(utilityService.punyHost('http://www.foo.com'), 'www.foo.com'); + }); + test('japanese', () => { + assert.equal(utilityService.punyHost('http://www.新聞.com'), 'www.xn--efvv70d.com'); + }); + }); + + describe('punyHostPSLDomain', () => { + test('simple', () => { + assert.equal(utilityService.punyHostPSLDomain('http://www.foo.com'), 'foo.com'); + }); + test('japanese', () => { + assert.equal(utilityService.punyHostPSLDomain('http://www.新聞.com'), 'xn--efvv70d.com'); + }); + test('lower', () => { + assert.equal(utilityService.punyHostPSLDomain('http://foo.github.io'), 'foo.github.io'); + assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.github.io'), 'bar.github.io'); + }); + test('special', () => { + assert.equal(utilityService.punyHostPSLDomain('http://foo.masto.host'), 'foo.masto.host'); + assert.equal(utilityService.punyHostPSLDomain('http://foo.bar.masto.host'), 'bar.masto.host'); + }); + }); +}); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 26de19eaf1..cf97473d14 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -612,8 +612,8 @@ export async function initTestDb(justBorrow = false, initEntities?: any[]) { username: config.db.user, password: config.db.pass, database: config.db.db, - synchronize: true && !justBorrow, - dropSchema: true && !justBorrow, + synchronize: !justBorrow, + dropSchema: !justBorrow, entities: initEntities ?? entities, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d2d7035550..76e399d5dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -170,6 +170,9 @@ importers: '@twemoji/parser': specifier: 15.1.1 version: 15.1.1 + '@types/psl': + specifier: ^1.1.3 + version: 1.1.3 accepts: specifier: 1.3.8 version: 1.3.8 @@ -365,6 +368,9 @@ importers: proxy-addr: specifier: ^2.0.7 version: 2.0.7 + psl: + specifier: ^1.13.0 + version: 1.13.0 pug: specifier: 3.0.3 version: 3.0.3 @@ -4835,6 +4841,9 @@ packages: '@types/proxy-addr@2.0.3': resolution: {integrity: sha512-TgAHHO4tNG3HgLTUhB+hM4iwW6JUNeQHCLnF1DjaDA9c69PN+IasoFu2MYDhubFc+ZIw5c5t9DMtjvrD6R3Egg==} + '@types/psl@1.1.3': + resolution: {integrity: sha512-Iu174JHfLd7i/XkXY6VDrqSlPvTDQOtQI7wNAXKKOAADJ9TduRLkNdMgjGiMxSttUIZnomv81JAbAbC0DhggxA==} + '@types/pug@2.0.10': resolution: {integrity: sha512-Sk/uYFOBAB7mb74XcpizmH0KOR2Pv3D2Hmrh1Dmy5BmK3MpdSa5kqZcg6EKBdklU0bFXX9gCfzvpnyUehrPIuA==} @@ -5307,6 +5316,7 @@ packages: acorn-import-assertions@1.9.0: resolution: {integrity: sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==} + deprecated: package has been renamed to acorn-import-attributes peerDependencies: acorn: ^8 @@ -9736,8 +9746,8 @@ packages: pseudomap@1.0.2: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} - psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + psl@1.13.0: + resolution: {integrity: sha512-BFwmFXiJoFqlUpZ5Qssolv15DMyc84gTBds1BjsV1BfXEo1UyyD7GsmN67n7J77uRhoSNW1AXtXKPLcBFQn9Aw==} pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} @@ -13103,7 +13113,7 @@ snapshots: '@eslint/config-array@0.17.1': dependencies: '@eslint/object-schema': 2.1.4 - debug: 4.3.5(supports-color@8.1.1) + debug: 4.3.7 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -13125,7 +13135,7 @@ snapshots: '@eslint/eslintrc@3.1.0': dependencies: ajv: 6.12.6 - debug: 4.3.5(supports-color@8.1.1) + debug: 4.3.7 espree: 10.1.0 globals: 14.0.0 ignore: 5.3.1 @@ -15648,6 +15658,8 @@ snapshots: dependencies: '@types/node': 20.14.12 + '@types/psl@1.1.3': {} + '@types/pug@2.0.10': {} '@types/punycode@2.1.4': {} @@ -18435,7 +18447,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.5(supports-color@8.1.1) + debug: 4.3.7 escape-string-regexp: 4.0.0 eslint-scope: 8.0.2 eslint-visitor-keys: 4.0.0 @@ -22014,7 +22026,9 @@ snapshots: pseudomap@1.0.2: {} - psl@1.9.0: {} + psl@1.13.0: + dependencies: + punycode: 2.3.1 pstree.remy@1.1.8: {} @@ -23256,7 +23270,7 @@ snapshots: tough-cookie@4.1.4: dependencies: - psl: 1.9.0 + psl: 1.13.0 punycode: 2.3.1 universalify: 0.2.0 url-parse: 1.5.10