diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml
index 25c052bc42..493875af6e 100644
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@@ -1417,6 +1417,9 @@ admin/views/drive.vue:
unmark-as-sensitive: "閲覧注意を解除"
marked-as-sensitive: "閲覧注意に設定しました"
unmarked-as-sensitive: "閲覧注意を解除しました"
+ clean-remote-files: "リモートファイルのキャッシュを削除"
+ clean-remote-files-are-you-sure: "すべてのリモートファイルのキャッシュを削除してもよろしいですか?"
+ clean-up: "クリーンアップ"
admin/views/users.vue:
operation: "操作"
diff --git a/src/client/app/admin/views/drive.vue b/src/client/app/admin/views/drive.vue
index 22b3b7eb96..1152db2b91 100644
--- a/src/client/app/admin/views/drive.vue
+++ b/src/client/app/admin/views/drive.vue
@@ -14,6 +14,10 @@
{{ $t('lookup') }}
+
+ {{ $t('clean-up') }}
+ {{ $t('clean-remote-files') }}
+
@@ -227,6 +231,29 @@ export default Vue.extend({
});
});
},
+
+ cleanRemoteFiles() {
+ this.$root.dialog({
+ type: 'warning',
+ text: this.$t('clean-remote-files-are-you-sure'),
+ showCancelButton: true
+ }).then(({ canceled }) => {
+ if (canceled) return;
+ this.$root.api('admin/drive/clean-remote-files');
+ this.$root.dialog({
+ type: 'success',
+ splash: true
+ });
+ });
+ },
+
+ cleanUp() {
+ this.$root.api('admin/drive/cleanup');
+ this.$root.dialog({
+ type: 'success',
+ splash: true
+ });
+ }
}
});
diff --git a/src/queue/index.ts b/src/queue/index.ts
index a010004f15..70233d8318 100644
--- a/src/queue/index.ts
+++ b/src/queue/index.ts
@@ -8,6 +8,7 @@ import { program } from '../argv';
import processDeliver from './processors/deliver';
import processInbox from './processors/inbox';
import processDb from './processors/db';
+import procesObjectStorage from './processors/object-storage';
import { queueLogger } from './logger';
import { DriveFile } from '../models/entities/drive-file';
@@ -34,9 +35,12 @@ function renderError(e: Error): any {
export const deliverQueue = initializeQueue('deliver');
export const inboxQueue = initializeQueue('inbox');
export const dbQueue = initializeQueue('db');
+export const objectStorageQueue = initializeQueue('objectStorage');
const deliverLogger = queueLogger.createSubLogger('deliver');
const inboxLogger = queueLogger.createSubLogger('inbox');
+const dbLogger = queueLogger.createSubLogger('db');
+const objectStorageLogger = queueLogger.createSubLogger('objectStorage');
deliverQueue
.on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`))
@@ -54,6 +58,22 @@ inboxQueue
.on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) }))
.on('stalled', (job) => inboxLogger.warn(`stalled id=${job.id} activity=${job.data.activity ? job.data.activity.id : 'none'}`));
+dbQueue
+ .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`))
+ .on('active', (job) => dbLogger.debug(`active id=${job.id}`))
+ .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`))
+ .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
+ .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) }))
+ .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`));
+
+objectStorageQueue
+ .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`))
+ .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`))
+ .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`))
+ .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) }))
+ .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) }))
+ .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`));
+
export function deliver(user: ILocalUser, content: any, to: any) {
if (content == null) return null;
@@ -165,11 +185,21 @@ export function createImportUserListsJob(user: ILocalUser, fileId: DriveFile['id
});
}
+export function createDeleteObjectStorageFileJob(key: string) {
+ return objectStorageQueue.add('deleteFile', {
+ key: key
+ }, {
+ removeOnComplete: true,
+ removeOnFail: true
+ });
+}
+
export default function() {
if (!program.onlyServer) {
deliverQueue.process(128, processDeliver);
inboxQueue.process(128, processInbox);
processDb(dbQueue);
+ procesObjectStorage(objectStorageQueue);
}
}
diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts
index 4e4eab86b7..5ecee9139a 100644
--- a/src/queue/processors/db/delete-drive-files.ts
+++ b/src/queue/processors/db/delete-drive-files.ts
@@ -1,7 +1,7 @@
import * as Bull from 'bull';
import { queueLogger } from '../../logger';
-import deleteFile from '../../../services/drive/delete-file';
+import { deleteFile } from '../../../services/drive/delete-file';
import { Users, DriveFiles } from '../../../models';
import { MoreThan } from 'typeorm';
diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts
new file mode 100644
index 0000000000..8e6b5f959e
--- /dev/null
+++ b/src/queue/processors/object-storage/delete-file.ts
@@ -0,0 +1,22 @@
+import * as Bull from 'bull';
+import * as Minio from 'minio';
+import { fetchMeta } from '../../../misc/fetch-meta';
+
+export default async (job: Bull.Job) => {
+ const meta = await fetchMeta();
+
+ const minio = new Minio.Client({
+ endPoint: meta.objectStorageEndpoint!,
+ region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
+ port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
+ useSSL: meta.objectStorageUseSSL,
+ accessKey: meta.objectStorageAccessKey!,
+ secretKey: meta.objectStorageSecretKey!,
+ });
+
+ const key: string = job.data.key;
+
+ await minio.removeObject(meta.objectStorageBucket!, key);
+
+ return 'Success';
+};
diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts
new file mode 100644
index 0000000000..f2fc08f1ab
--- /dev/null
+++ b/src/queue/processors/object-storage/index.ts
@@ -0,0 +1,12 @@
+import * as Bull from 'bull';
+import deleteFile from './delete-file';
+
+const jobs = {
+ deleteFile,
+} as any;
+
+export default function(q: Bull.Queue) {
+ for (const [k, v] of Object.entries(jobs)) {
+ q.process(k, v as any);
+ }
+}
diff --git a/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts
index 84e9c363e1..276d1f37bd 100644
--- a/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts
+++ b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts
@@ -1,6 +1,6 @@
import $ from 'cafy';
import define from '../../define';
-import del from '../../../../services/drive/delete-file';
+import { deleteFile } from '../../../../services/drive/delete-file';
import { DriveFiles } from '../../../../models';
import { ID } from '../../../../misc/cafy-id';
@@ -27,6 +27,6 @@ export default define(meta, async (ps, me) => {
});
for (const file of files) {
- del(file);
+ deleteFile(file);
}
});
diff --git a/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/src/server/api/endpoints/admin/drive/clean-remote-files.ts
new file mode 100644
index 0000000000..a861052bbe
--- /dev/null
+++ b/src/server/api/endpoints/admin/drive/clean-remote-files.ts
@@ -0,0 +1,21 @@
+import { Not, IsNull } from 'typeorm';
+import define from '../../../define';
+import { deleteFile } from '../../../../../services/drive/delete-file';
+import { DriveFiles } from '../../../../../models';
+
+export const meta = {
+ tags: ['admin'],
+
+ requireCredential: true,
+ requireModerator: true,
+};
+
+export default define(meta, async (ps, me) => {
+ const files = await DriveFiles.find({
+ userHost: Not(IsNull())
+ });
+
+ for (const file of files) {
+ deleteFile(file, true);
+ }
+});
diff --git a/src/server/api/endpoints/admin/drive/cleanup.ts b/src/server/api/endpoints/admin/drive/cleanup.ts
new file mode 100644
index 0000000000..272416dee8
--- /dev/null
+++ b/src/server/api/endpoints/admin/drive/cleanup.ts
@@ -0,0 +1,21 @@
+import { IsNull } from 'typeorm';
+import define from '../../../define';
+import { deleteFile } from '../../../../../services/drive/delete-file';
+import { DriveFiles } from '../../../../../models';
+
+export const meta = {
+ tags: ['admin'],
+
+ requireCredential: true,
+ requireModerator: true,
+};
+
+export default define(meta, async (ps, me) => {
+ const files = await DriveFiles.find({
+ userId: IsNull()
+ });
+
+ for (const file of files) {
+ deleteFile(file);
+ }
+});
diff --git a/src/server/api/endpoints/admin/federation/delete-all-files.ts b/src/server/api/endpoints/admin/federation/delete-all-files.ts
index befb362264..24ad9ce808 100644
--- a/src/server/api/endpoints/admin/federation/delete-all-files.ts
+++ b/src/server/api/endpoints/admin/federation/delete-all-files.ts
@@ -1,6 +1,6 @@
import $ from 'cafy';
import define from '../../../define';
-import del from '../../../../../services/drive/delete-file';
+import { deleteFile } from '../../../../../services/drive/delete-file';
import { DriveFiles } from '../../../../../models';
export const meta = {
@@ -22,6 +22,6 @@ export default define(meta, async (ps, me) => {
});
for (const file of files) {
- del(file);
+ deleteFile(file);
}
});
diff --git a/src/server/api/endpoints/drive/files/delete.ts b/src/server/api/endpoints/drive/files/delete.ts
index d8cc5ec0a1..d6a20f352e 100644
--- a/src/server/api/endpoints/drive/files/delete.ts
+++ b/src/server/api/endpoints/drive/files/delete.ts
@@ -1,6 +1,6 @@
import $ from 'cafy';
import { ID } from '../../../../../misc/cafy-id';
-import del from '../../../../../services/drive/delete-file';
+import { deleteFile } from '../../../../../services/drive/delete-file';
import { publishDriveStream } from '../../../../../services/stream';
import define from '../../../define';
import { ApiError } from '../../../error';
@@ -57,7 +57,7 @@ export default define(meta, async (ps, user) => {
}
// Delete
- await del(file);
+ await deleteFile(file);
// Publish fileDeleted event
publishDriveStream(user.id, 'fileDeleted', file.id);
diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts
index eb0b0abaa0..d21c67d180 100644
--- a/src/services/drive/add-file.ts
+++ b/src/services/drive/add-file.ts
@@ -7,7 +7,7 @@ import * as uuid from 'uuid';
import * as sharp from 'sharp';
import { publishMainStream, publishDriveStream } from '../stream';
-import delFile from './delete-file';
+import { deleteFile } from './delete-file';
import { fetchMeta } from '../../misc/fetch-meta';
import { GenerateVideoThumbnail } from './generate-video-thumbnail';
import { driveLogger } from './logger';
@@ -233,7 +233,7 @@ async function deleteOldFile(user: IRemoteUser) {
const oldFile = await q.getOne();
if (oldFile) {
- delFile(oldFile, true);
+ deleteFile(oldFile, true);
}
}
diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts
index b429ca0db8..258c0853fb 100644
--- a/src/services/drive/delete-file.ts
+++ b/src/services/drive/delete-file.ts
@@ -1,11 +1,10 @@
-import * as Minio from 'minio';
import { DriveFile } from '../../models/entities/drive-file';
import { InternalStorage } from './internal-storage';
import { DriveFiles, Instances, Notes } from '../../models';
import { driveChart, perUserDriveChart, instanceChart } from '../chart';
-import { fetchMeta } from '../../misc/fetch-meta';
+import { createDeleteObjectStorageFileJob } from '../../queue';
-export default async function(file: DriveFile, isExpired = false) {
+export async function deleteFile(file: DriveFile, isExpired = false) {
if (file.storedInternal) {
InternalStorage.del(file.accessKey!);
@@ -17,25 +16,14 @@ export default async function(file: DriveFile, isExpired = false) {
InternalStorage.del(file.webpublicAccessKey!);
}
} else if (!file.isLink) {
- const meta = await fetchMeta();
-
- const minio = new Minio.Client({
- endPoint: meta.objectStorageEndpoint!,
- region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined,
- port: meta.objectStoragePort ? meta.objectStoragePort : undefined,
- useSSL: meta.objectStorageUseSSL,
- accessKey: meta.objectStorageAccessKey!,
- secretKey: meta.objectStorageSecretKey!,
- });
-
- await minio.removeObject(meta.objectStorageBucket!, file.accessKey!);
+ createDeleteObjectStorageFileJob(file.accessKey!);
if (file.thumbnailUrl) {
- await minio.removeObject(meta.objectStorageBucket!, file.thumbnailAccessKey!);
+ createDeleteObjectStorageFileJob(file.thumbnailAccessKey!);
}
if (file.webpublicUrl) {
- await minio.removeObject(meta.objectStorageBucket!, file.webpublicAccessKey!);
+ createDeleteObjectStorageFileJob(file.webpublicAccessKey!);
}
}
@@ -44,8 +32,8 @@ export default async function(file: DriveFile, isExpired = false) {
DriveFiles.update(file.id, {
isLink: true,
url: file.uri,
- thumbnailUrl: null,
- webpublicUrl: null
+ thumbnailUrl: file.uri,
+ webpublicUrl: file.uri
});
} else {
DriveFiles.delete(file.id);
diff --git a/src/tools/clean-remote-files.ts b/src/tools/clean-remote-files.ts
deleted file mode 100644
index 633a6fc04d..0000000000
--- a/src/tools/clean-remote-files.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import * as promiseLimit from 'promise-limit';
-import del from '../services/drive/delete-file';
-import { DriveFiles } from '../models';
-import { Not, IsNull } from 'typeorm';
-import { DriveFile } from '../models/entities/drive-file';
-import { ensure } from '../prelude/ensure';
-
-const limit = promiseLimit(16);
-
-DriveFiles.find({
- userHost: Not(IsNull())
-}).then(async files => {
- console.log(`there is ${files.length} files`);
-
- await Promise.all(files.map(file => limit(() => job(file))));
-
- console.log('ALL DONE');
-});
-
-async function job(file: DriveFile): Promise {
- file = await DriveFiles.findOne(file.id).then(ensure);
-
- await del(file, true);
-
- console.log('done', file.id);
-}