mirror of
https://activitypub.software/TransFem-org/Sharkey
synced 2024-12-22 08:20:10 +00:00
feat(server): Fetch icon url of an instance (#6591)
* feat(server): Fetch icon url of an instance Resolve #6589 * chore: Rename the function
This commit is contained in:
parent
cf9266eab9
commit
f1ef85b636
9 changed files with 183 additions and 80 deletions
14
migration/1595676934834-instance-icon-url.ts
Normal file
14
migration/1595676934834-instance-icon-url.ts
Normal file
|
@ -0,0 +1,14 @@
|
|||
import {MigrationInterface, QueryRunner} from "typeorm";
|
||||
|
||||
export class instanceIconUrl1595676934834 implements MigrationInterface {
|
||||
name = 'instanceIconUrl1595676934834'
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`ALTER TABLE "instance" ADD "iconUrl" character varying(256) DEFAULT null`);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "iconUrl"`);
|
||||
}
|
||||
|
||||
}
|
|
@ -21,8 +21,8 @@ export function getApLock(uri: string, timeout = 30 * 1000) {
|
|||
return lock(`ap-object:${uri}`, timeout);
|
||||
}
|
||||
|
||||
export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
|
||||
return lock(`nodeinfo:${host}`, timeout);
|
||||
export function getFetchInstanceMetadataLock(host: string, timeout = 30 * 1000) {
|
||||
return lock(`instance:${host}`, timeout);
|
||||
}
|
||||
|
||||
export function getChartInsertLock(lockKey: string, timeout = 30 * 1000) {
|
||||
|
|
|
@ -27,6 +27,27 @@ export async function getJson(url: string, accept = 'application/json, */*', tim
|
|||
return await res.json();
|
||||
}
|
||||
|
||||
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: HeadersInit) {
|
||||
const res = await fetch(url, {
|
||||
headers: Object.assign({
|
||||
'User-Agent': config.userAgent,
|
||||
Accept: accept
|
||||
}, headers || {}),
|
||||
timeout,
|
||||
agent: getAgentByUrl,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw {
|
||||
name: `StatusError`,
|
||||
statusCode: res.status,
|
||||
message: `${res.status} ${res.statusText}`,
|
||||
};
|
||||
}
|
||||
|
||||
return await res.text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get http non-proxy agent
|
||||
*/
|
||||
|
|
|
@ -158,6 +158,11 @@ export class Instance {
|
|||
})
|
||||
public maintainerEmail: string | null;
|
||||
|
||||
@Column('varchar', {
|
||||
length: 256, nullable: true, default: null,
|
||||
})
|
||||
public iconUrl: string | null;
|
||||
|
||||
@Column('timestamp with time zone', {
|
||||
nullable: true,
|
||||
})
|
||||
|
|
|
@ -4,7 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
|
|||
import Logger from '../../services/logger';
|
||||
import { Instances } from '../../models';
|
||||
import { instanceChart } from '../../services/chart';
|
||||
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
|
||||
import { fetchInstanceMetadata } from '../../services/fetch-instance-metadata';
|
||||
import { fetchMeta } from '../../misc/fetch-meta';
|
||||
import { toPuny } from '../../misc/convert-host';
|
||||
|
||||
|
@ -48,7 +48,7 @@ export default async (job: Bull.Job) => {
|
|||
isNotResponding: false
|
||||
});
|
||||
|
||||
fetchNodeinfo(i);
|
||||
fetchInstanceMetadata(i);
|
||||
|
||||
instanceChart.requestSent(i.host, true);
|
||||
});
|
||||
|
|
|
@ -8,7 +8,7 @@ import { instanceChart } from '../../services/chart';
|
|||
import { fetchMeta } from '../../misc/fetch-meta';
|
||||
import { toPuny, extractDbHost } from '../../misc/convert-host';
|
||||
import { getApId } from '../../remote/activitypub/type';
|
||||
import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
|
||||
import { fetchInstanceMetadata } from '../../services/fetch-instance-metadata';
|
||||
import { InboxJobData } from '..';
|
||||
import DbResolver from '../../remote/activitypub/db-resolver';
|
||||
import { resolvePerson } from '../../remote/activitypub/models/person';
|
||||
|
@ -126,7 +126,7 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
|
|||
isNotResponding: false
|
||||
});
|
||||
|
||||
fetchNodeinfo(i);
|
||||
fetchInstanceMetadata(i);
|
||||
|
||||
instanceChart.requestReceived(i.host);
|
||||
});
|
||||
|
|
|
@ -26,7 +26,7 @@ import { validActor } from '../../../remote/activitypub/type';
|
|||
import { getConnection } from 'typeorm';
|
||||
import { ensure } from '../../../prelude/ensure';
|
||||
import { toArray } from '../../../prelude/array';
|
||||
import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
|
||||
import { fetchInstanceMetadata } from '../../../services/fetch-instance-metadata';
|
||||
|
||||
const logger = apLogger;
|
||||
|
||||
|
@ -204,7 +204,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
|
|||
registerOrFetchInstanceDoc(host).then(i => {
|
||||
Instances.increment({ id: i.id }, 'usersCount', 1);
|
||||
instanceChart.newUser(i.host);
|
||||
fetchNodeinfo(i);
|
||||
fetchInstanceMetadata(i);
|
||||
});
|
||||
|
||||
usersChart.update(user!, true);
|
||||
|
|
135
src/services/fetch-instance-metadata.ts
Normal file
135
src/services/fetch-instance-metadata.ts
Normal file
|
@ -0,0 +1,135 @@
|
|||
import { JSDOM } from 'jsdom';
|
||||
import fetch from 'node-fetch';
|
||||
import { getJson, getHtml, getAgentByUrl } from '../misc/fetch';
|
||||
import { Instance } from '../models/entities/instance';
|
||||
import { Instances } from '../models';
|
||||
import { getFetchInstanceMetadataLock } from '../misc/app-lock';
|
||||
import Logger from './logger';
|
||||
import { URL } from 'url';
|
||||
|
||||
const logger = new Logger('metadata', 'cyan');
|
||||
|
||||
export async function fetchInstanceMetadata(instance: Instance): Promise<void> {
|
||||
const unlock = await getFetchInstanceMetadataLock(instance.host);
|
||||
|
||||
const _instance = await Instances.findOne({ host: instance.host });
|
||||
const now = Date.now();
|
||||
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
|
||||
unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Fetching metadata of ${instance.host} ...`);
|
||||
|
||||
try {
|
||||
const [info, icon] = await Promise.all([
|
||||
fetchNodeinfo(instance).catch(() => null),
|
||||
fetchIconUrl(instance).catch(() => null),
|
||||
]);
|
||||
|
||||
logger.succ(`Successfuly fetched metadata of ${instance.host}`);
|
||||
|
||||
const updates = {
|
||||
infoUpdatedAt: new Date(),
|
||||
} as Record<string, any>;
|
||||
|
||||
if (info) {
|
||||
updates.softwareName = info.software.name.toLowerCase();
|
||||
updates.softwareVersion = info.software.version;
|
||||
updates.openRegistrations = info.openRegistrations;
|
||||
updates.name = info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null;
|
||||
updates.description = info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null;
|
||||
updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null;
|
||||
updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null;
|
||||
}
|
||||
|
||||
if (icon) {
|
||||
updates.iconUrl = icon;
|
||||
}
|
||||
|
||||
await Instances.update(instance.id, updates);
|
||||
|
||||
logger.succ(`Successfuly updated metadata of ${instance.host}`);
|
||||
} catch (e) {
|
||||
logger.error(`Failed to update metadata of ${instance.host}: ${e}`);
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchNodeinfo(instance: Instance): Promise<Record<string, any>> {
|
||||
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
|
||||
|
||||
try {
|
||||
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
|
||||
.catch(e => {
|
||||
if (e.statusCode === 404) {
|
||||
throw 'No nodeinfo provided';
|
||||
} else {
|
||||
throw e.statusCode || e.message;
|
||||
}
|
||||
});
|
||||
|
||||
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
|
||||
throw 'No wellknown links';
|
||||
}
|
||||
|
||||
const links = wellknown.links as any[];
|
||||
|
||||
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
|
||||
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
|
||||
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
|
||||
const link = lnik2_1 || lnik2_0 || lnik1_0;
|
||||
|
||||
if (link == null) {
|
||||
throw 'No nodeinfo link provided';
|
||||
}
|
||||
|
||||
const info = await getJson(link.href)
|
||||
.catch(e => {
|
||||
throw e.statusCode || e.message;
|
||||
});
|
||||
|
||||
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
|
||||
|
||||
return info;
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
|
||||
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchIconUrl(instance: Instance): Promise<string | null> {
|
||||
logger.info(`Fetching icon URL of ${instance.host} ...`);
|
||||
|
||||
const url = 'https://' + instance.host;
|
||||
|
||||
const html = await getHtml(url);
|
||||
|
||||
const { window } = new JSDOM(html);
|
||||
const doc = window.document;
|
||||
|
||||
const hrefAppleTouchIconPrecomposed = doc.querySelector('link[rel="apple-touch-icon-precomposed"]')?.getAttribute('href');
|
||||
const hrefAppleTouchIcon = doc.querySelector('link[rel="apple-touch-icon"]')?.getAttribute('href');
|
||||
const hrefIcon = doc.querySelector('link[rel="icon"]')?.getAttribute('href');
|
||||
|
||||
const href = hrefAppleTouchIconPrecomposed || hrefAppleTouchIcon || hrefIcon;
|
||||
|
||||
if (href) {
|
||||
return (new URL(href, url)).href;
|
||||
}
|
||||
|
||||
const faviconUrl = url + '/favicon.ico';
|
||||
|
||||
const favicon = await fetch(faviconUrl, {
|
||||
timeout: 10000,
|
||||
agent: getAgentByUrl,
|
||||
});
|
||||
|
||||
if (favicon.ok) {
|
||||
return faviconUrl;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
|
@ -1,72 +0,0 @@
|
|||
import { getJson } from '../misc/fetch';
|
||||
import { Instance } from '../models/entities/instance';
|
||||
import { Instances } from '../models';
|
||||
import { getNodeinfoLock } from '../misc/app-lock';
|
||||
import Logger from '../services/logger';
|
||||
|
||||
export const logger = new Logger('nodeinfo', 'cyan');
|
||||
|
||||
export async function fetchNodeinfo(instance: Instance) {
|
||||
const unlock = await getNodeinfoLock(instance.host);
|
||||
|
||||
const _instance = await Instances.findOne({ host: instance.host });
|
||||
const now = Date.now();
|
||||
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
|
||||
unlock();
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Fetching nodeinfo of ${instance.host} ...`);
|
||||
|
||||
try {
|
||||
const wellknown = await getJson('https://' + instance.host + '/.well-known/nodeinfo')
|
||||
.catch(e => {
|
||||
if (e.statusCode === 404) {
|
||||
throw 'No nodeinfo provided';
|
||||
} else {
|
||||
throw e.statusCode || e.message;
|
||||
}
|
||||
});
|
||||
|
||||
if (wellknown.links == null || !Array.isArray(wellknown.links)) {
|
||||
throw 'No wellknown links';
|
||||
}
|
||||
|
||||
const links = wellknown.links as any[];
|
||||
|
||||
const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
|
||||
const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
|
||||
const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
|
||||
const link = lnik2_1 || lnik2_0 || lnik1_0;
|
||||
|
||||
if (link == null) {
|
||||
throw 'No nodeinfo link provided';
|
||||
}
|
||||
|
||||
const info = await getJson(link.href)
|
||||
.catch(e => {
|
||||
throw e.statusCode || e.message;
|
||||
});
|
||||
|
||||
await Instances.update(instance.id, {
|
||||
infoUpdatedAt: new Date(),
|
||||
softwareName: info.software.name.toLowerCase(),
|
||||
softwareVersion: info.software.version,
|
||||
openRegistrations: info.openRegistrations,
|
||||
name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
|
||||
description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
|
||||
maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
|
||||
maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
|
||||
});
|
||||
|
||||
logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
|
||||
} catch (e) {
|
||||
logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
|
||||
|
||||
await Instances.update(instance.id, {
|
||||
infoUpdatedAt: new Date(),
|
||||
});
|
||||
} finally {
|
||||
unlock();
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue