etc/Sharkey
0
0
Fork 0
mirror of https://activitypub.software/TransFem-org/Sharkey synced 2024-11-25 07:25:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

fix(backend): pagination with sinceId broken (#12586)

Browse source 
* fix(backend): pagination with sinceId broken

* fix(backend): pagination with sinceId broken for dbFallback
This commit is contained in:
anatawa12 2023-12-07 17:07:06 +09:00 committed by GitHub
parent e926411812
commit 1d3ef7b42f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
packages/backend/src/core/FanoutTimelineEndpointService.ts
View file

@ -60,11 +60,15 @@ export class FanoutTimelineEndpointService {
// 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える // 呼び出し元と以下の処理をシンプルにするためにdbFallbackを置き換える
if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]); if (!ps.useDbFallback) ps.dbFallback = () => Promise.resolve([]);
const shouldPrepend = ps.sinceId && !ps.untilId;
const idCompare: (a: string, b: string) => number = shouldPrepend ? (a, b) => a < b ? -1 : 1 : (a, b) => a > b ? -1 : 1;
const redisResult = await this.fanoutTimelineService.getMulti(ps.redisTimelines, ps.untilId, ps.sinceId); const redisResult = await this.fanoutTimelineService.getMulti(ps.redisTimelines, ps.untilId, ps.sinceId);
// TODO: いい感じにgetMulti内でソート済だからuniqするときにredisResultが全てソート済なのを利用して再ソートを避けたい
const redisResultIds = Array.from(new Set(redisResult.flat(1))); const redisResultIds = Array.from(new Set(redisResult.flat(1)));
redisResultIds.sort((a, b) => a > b ? -1 : 1); redisResultIds.sort(idCompare);
noteIds = redisResultIds.slice(0, ps.limit); noteIds = redisResultIds.slice(0, ps.limit);
shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0); shouldFallbackToDb = shouldFallbackToDb || (noteIds.length === 0);
@ -126,32 +130,43 @@ export class FanoutTimelineEndpointService {
const remainingToRead = ps.limit - redisTimeline.length; const remainingToRead = ps.limit - redisTimeline.length;
// DBからの取り直しを減らす初回と同じ割合以上で成功すると仮定するが、クエリの長さを考えて三倍まで // DBからの取り直しを減らす初回と同じ割合以上で成功すると仮定するが、クエリの長さを考えて三倍まで
const countToGet = remainingToRead * Math.ceil(Math.min(1.1 / lastSuccessfulRate, 3)); const countToGet = Math.ceil(remainingToRead * Math.min(1.1 / lastSuccessfulRate, 3));
noteIds = redisResultIds.slice(readFromRedis, readFromRedis + countToGet); noteIds = redisResultIds.slice(readFromRedis, readFromRedis + countToGet);
readFromRedis += noteIds.length; readFromRedis += noteIds.length;
const gotFromDb = await this.getAndFilterFromDb(noteIds, filter); const gotFromDb = await this.getAndFilterFromDb(noteIds, filter, idCompare);
redisTimeline.push(...gotFromDb); redisTimeline.push(...gotFromDb);
lastSuccessfulRate = gotFromDb.length / noteIds.length; lastSuccessfulRate = gotFromDb.length / noteIds.length;
if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) { if (ps.allowPartial ? redisTimeline.length !== 0 : redisTimeline.length >= ps.limit) {
// 十分Redisからとれた // 十分Redisからとれた
return redisTimeline.slice(0, ps.limit); const result = redisTimeline.slice(0, ps.limit);
if (shouldPrepend) result.reverse();
return result;
} }
} }
// まだ足りない分はDBにフォールバック // まだ足りない分はDBにフォールバック
const remainingToRead = ps.limit - redisTimeline.length; const remainingToRead = ps.limit - redisTimeline.length;
const gotFromDb = await ps.dbFallback(noteIds[noteIds.length - 1], ps.sinceId, remainingToRead); let dbUntil: string | null;
redisTimeline.push(...gotFromDb); let dbSince: string | null;
return redisTimeline; if (shouldPrepend) {
redisTimeline.reverse();
dbUntil = ps.untilId;
dbSince = noteIds[noteIds.length - 1];
} else {
dbUntil = noteIds[noteIds.length - 1];
dbSince = ps.sinceId;
}
const gotFromDb = await ps.dbFallback(dbUntil, dbSince, remainingToRead);
return shouldPrepend ? [...gotFromDb, ...redisTimeline] : [...redisTimeline, ...gotFromDb];
} }
return await ps.dbFallback(ps.untilId, ps.sinceId, ps.limit); return await ps.dbFallback(ps.untilId, ps.sinceId, ps.limit);
} }
private async getAndFilterFromDb(noteIds: string[], noteFilter: (note: MiNote) => boolean): Promise<MiNote[]> { private async getAndFilterFromDb(noteIds: string[], noteFilter: (note: MiNote) => boolean, idCompare: (a: string, b: string) => number): Promise<MiNote[]> {
const query = this.notesRepository.createQueryBuilder('note') const query = this.notesRepository.createQueryBuilder('note')
.where('note.id IN (:...noteIds)', { noteIds: noteIds }) .where('note.id IN (:...noteIds)', { noteIds: noteIds })
.innerJoinAndSelect('note.user', 'user') .innerJoinAndSelect('note.user', 'user')
@ -163,7 +178,7 @@ export class FanoutTimelineEndpointService {
const notes = (await query.getMany()).filter(noteFilter); const notes = (await query.getMany()).filter(noteFilter);
notes.sort((a, b) => a.id > b.id ? -1 : 1); notes.sort((a, b) => idCompare(a.id, b.id));
return notes; return notes;
} }