mirror of
https://codeberg.org/yeentown/barkey
synced 2024-11-21 22:35:11 +00:00
parent
050787238b
commit
658901a47f
6 changed files with 1267 additions and 164 deletions
|
@ -37,6 +37,9 @@
|
|||
"@tensorflow/tfjs-node": "4.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.294.0",
|
||||
"@aws-sdk/lib-storage": "^3.294.0",
|
||||
"@aws-sdk/node-http-handler": "^3.292.0",
|
||||
"@bull-board/api": "5.0.0",
|
||||
"@bull-board/fastify": "5.0.0",
|
||||
"@bull-board/ui": "5.0.0",
|
||||
|
@ -59,7 +62,6 @@
|
|||
"ajv": "8.12.0",
|
||||
"archiver": "5.3.1",
|
||||
"autwh": "0.1.0",
|
||||
"aws-sdk": "2.1318.0",
|
||||
"bcryptjs": "2.4.3",
|
||||
"blurhash": "2.0.5",
|
||||
"bull": "4.10.4",
|
||||
|
@ -190,6 +192,7 @@
|
|||
"@types/ws": "8.5.4",
|
||||
"@typescript-eslint/eslint-plugin": "5.54.1",
|
||||
"@typescript-eslint/parser": "5.54.1",
|
||||
"aws-sdk-client-mock": "^2.1.1",
|
||||
"cross-env": "7.0.3",
|
||||
"eslint": "8.35.0",
|
||||
"eslint-plugin-import": "2.27.5",
|
||||
|
|
|
@ -4,6 +4,7 @@ import { v4 as uuid } from 'uuid';
|
|||
import sharp from 'sharp';
|
||||
import { sharpBmp } from 'sharp-read-bmp';
|
||||
import { IsNull } from 'typeorm';
|
||||
import { DeleteObjectCommandInput, PutObjectCommandInput, NoSuchKey } from '@aws-sdk/client-s3';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type { DriveFilesRepository, UsersRepository, DriveFoldersRepository, UserProfilesRepository } from '@/models/index.js';
|
||||
import type { Config } from '@/config.js';
|
||||
|
@ -36,7 +37,6 @@ import { bindThis } from '@/decorators.js';
|
|||
import { RoleService } from '@/core/RoleService.js';
|
||||
import { correctFilename } from '@/misc/correct-filename.js';
|
||||
import { isMimeImage } from '@/misc/is-mime-image.js';
|
||||
import type S3 from 'aws-sdk/clients/s3.js';
|
||||
|
||||
type AddFileArgs = {
|
||||
/** User who wish to add file */
|
||||
|
@ -81,6 +81,7 @@ type UploadFromUrlArgs = {
|
|||
export class DriveService {
|
||||
private registerLogger: Logger;
|
||||
private downloaderLogger: Logger;
|
||||
private deleteLogger: Logger;
|
||||
|
||||
constructor(
|
||||
@Inject(DI.config)
|
||||
|
@ -118,6 +119,7 @@ export class DriveService {
|
|||
const logger = new Logger('drive', 'blue');
|
||||
this.registerLogger = logger.createSubLogger('register', 'yellow');
|
||||
this.downloaderLogger = logger.createSubLogger('downloader');
|
||||
this.deleteLogger = logger.createSubLogger('delete');
|
||||
}
|
||||
|
||||
/***
|
||||
|
@ -368,7 +370,7 @@ export class DriveService {
|
|||
Body: stream,
|
||||
ContentType: type,
|
||||
CacheControl: 'max-age=31536000, immutable',
|
||||
} as S3.PutObjectRequest;
|
||||
} as PutObjectCommandInput;
|
||||
|
||||
if (filename) params.ContentDisposition = contentDisposition(
|
||||
'inline',
|
||||
|
@ -378,21 +380,16 @@ export class DriveService {
|
|||
);
|
||||
if (meta.objectStorageSetPublicRead) params.ACL = 'public-read';
|
||||
|
||||
const s3 = this.s3Service.getS3(meta);
|
||||
|
||||
const upload = s3.upload(params, {
|
||||
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
|
||||
});
|
||||
|
||||
await upload.promise()
|
||||
await this.s3Service.upload(meta, params)
|
||||
.then(
|
||||
result => {
|
||||
if (result) {
|
||||
if ('Bucket' in result) { // CompleteMultipartUploadCommandOutput
|
||||
this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
|
||||
} else {
|
||||
this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`);
|
||||
} else { // AbortMultipartUploadCommandOutput
|
||||
this.registerLogger.error(`Upload Result Aborted: key = ${key}, filename = ${filename}`);
|
||||
}
|
||||
},
|
||||
})
|
||||
.catch(
|
||||
err => {
|
||||
this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
|
||||
},
|
||||
|
@ -528,10 +525,10 @@ export class DriveService {
|
|||
};
|
||||
|
||||
const properties: {
|
||||
width?: number;
|
||||
height?: number;
|
||||
orientation?: number;
|
||||
} = {};
|
||||
width?: number;
|
||||
height?: number;
|
||||
orientation?: number;
|
||||
} = {};
|
||||
|
||||
if (info.width) {
|
||||
properties['width'] = info.width;
|
||||
|
@ -720,22 +717,22 @@ export class DriveService {
|
|||
@bindThis
|
||||
public async deleteObjectStorageFile(key: string) {
|
||||
const meta = await this.metaService.fetch();
|
||||
|
||||
const s3 = this.s3Service.getS3(meta);
|
||||
|
||||
try {
|
||||
await s3.deleteObject({
|
||||
Bucket: meta.objectStorageBucket!,
|
||||
const param = {
|
||||
Bucket: meta.objectStorageBucket,
|
||||
Key: key,
|
||||
}).promise();
|
||||
} as DeleteObjectCommandInput;
|
||||
|
||||
await this.s3Service.delete(meta, param);
|
||||
} catch (err: any) {
|
||||
if (err.code === 'NoSuchKey') {
|
||||
console.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err);
|
||||
if (err.name === 'NoSuchKey') {
|
||||
this.deleteLogger.warn(`The object storage had no such key to delete: ${key}. Skipping this.`, err as Error);
|
||||
return;
|
||||
} else {
|
||||
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
throw new Error(`Failed to delete the file from the object storage with the given key: ${key}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1,11 +1,16 @@
|
|||
import { URL } from 'node:url';
|
||||
import * as http from 'node:http';
|
||||
import * as https from 'node:https';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import S3 from 'aws-sdk/clients/s3.js';
|
||||
import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
import { NodeHttpHandler, NodeHttpHandlerOptions } from '@aws-sdk/node-http-handler';
|
||||
import { DI } from '@/di-symbols.js';
|
||||
import type { Config } from '@/config.js';
|
||||
import type { Meta } from '@/models/entities/Meta.js';
|
||||
import { HttpRequestService } from '@/core/HttpRequestService.js';
|
||||
import { bindThis } from '@/decorators.js';
|
||||
import type { DeleteObjectCommandInput, PutObjectCommandInput } from '@aws-sdk/client-s3';
|
||||
|
||||
@Injectable()
|
||||
export class S3Service {
|
||||
|
@ -18,25 +23,47 @@ export class S3Service {
|
|||
}
|
||||
|
||||
@bindThis
|
||||
public getS3(meta: Meta) {
|
||||
public getS3Client(meta: Meta): S3Client {
|
||||
const u = meta.objectStorageEndpoint
|
||||
? `${meta.objectStorageUseSSL ? 'https://' : 'http://'}${meta.objectStorageEndpoint}`
|
||||
: `${meta.objectStorageUseSSL ? 'https://' : 'http://'}example.net`;
|
||||
? `${meta.objectStorageUseSSL ? 'https' : 'http'}://${meta.objectStorageEndpoint}`
|
||||
: `${meta.objectStorageUseSSL ? 'https' : 'http'}://example.net`; // dummy url to select http(s) agent
|
||||
|
||||
return new S3({
|
||||
endpoint: meta.objectStorageEndpoint && meta.objectStorageEndpoint.length > 0
|
||||
? meta.objectStorageEndpoint
|
||||
: undefined,
|
||||
accessKeyId: meta.objectStorageAccessKey!,
|
||||
secretAccessKey: meta.objectStorageSecretKey!,
|
||||
const agent = this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy);
|
||||
const handlerOption: NodeHttpHandlerOptions = {};
|
||||
if (meta.objectStorageUseSSL) {
|
||||
handlerOption.httpsAgent = agent as https.Agent;
|
||||
} else {
|
||||
handlerOption.httpAgent = agent as http.Agent;
|
||||
}
|
||||
|
||||
return new S3Client({
|
||||
endpoint: meta.objectStorageEndpoint ? u : undefined,
|
||||
credentials: (meta.objectStorageAccessKey !== null && meta.objectStorageSecretKey !== null) ? {
|
||||
accessKeyId: meta.objectStorageAccessKey,
|
||||
secretAccessKey: meta.objectStorageSecretKey,
|
||||
} : undefined,
|
||||
region: meta.objectStorageRegion ?? undefined,
|
||||
sslEnabled: meta.objectStorageUseSSL,
|
||||
s3ForcePathStyle: !meta.objectStorageEndpoint // AWS with endPoint omitted
|
||||
? false
|
||||
: meta.objectStorageS3ForcePathStyle,
|
||||
httpOptions: {
|
||||
agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
|
||||
},
|
||||
tls: meta.objectStorageUseSSL,
|
||||
forcePathStyle: meta.objectStorageEndpoint ? meta.objectStorageS3ForcePathStyle : false, // AWS with endPoint omitted
|
||||
requestHandler: new NodeHttpHandler(handlerOption),
|
||||
});
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public async upload(meta: Meta, input: PutObjectCommandInput) {
|
||||
const client = this.getS3Client(meta);
|
||||
return new Upload({
|
||||
client,
|
||||
params: input,
|
||||
partSize: (client.config.endpoint && (await client.config.endpoint()).hostname === 'storage.googleapis.com')
|
||||
? 500 * 1024 * 1024
|
||||
: 8 * 1024 * 1024,
|
||||
}).done();
|
||||
}
|
||||
|
||||
@bindThis
|
||||
public delete(meta: Meta, input: DeleteObjectCommandInput) {
|
||||
const client = this.getS3Client(meta);
|
||||
return client.send(new DeleteObjectCommand(input));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,55 +1,56 @@
|
|||
process.env.NODE_ENV = 'test';
|
||||
|
||||
import { jest } from '@jest/globals';
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { DeleteObjectCommandOutput, DeleteObjectCommand, NoSuchKey, InvalidObjectState, S3Client } from '@aws-sdk/client-s3';
|
||||
import { mockClient } from 'aws-sdk-client-mock';
|
||||
import { GlobalModule } from '@/GlobalModule.js';
|
||||
import { DriveService } from '@/core/DriveService.js';
|
||||
import { CoreModule } from '@/core/CoreModule.js';
|
||||
import { S3Service } from '@/core/S3Service';
|
||||
import type { Meta } from '@/models';
|
||||
import type { DeleteObjectOutput } from 'aws-sdk/clients/s3';
|
||||
import type { AWSError } from 'aws-sdk/lib/error';
|
||||
import type { PromiseResult, Request } from 'aws-sdk/lib/request';
|
||||
import type { TestingModule } from '@nestjs/testing';
|
||||
|
||||
describe('DriveService', () => {
|
||||
let app: TestingModule;
|
||||
let driveService: DriveService;
|
||||
const s3Mock = mockClient(S3Client);
|
||||
|
||||
beforeEach(async () => {
|
||||
beforeAll(async () => {
|
||||
app = await Test.createTestingModule({
|
||||
imports: [GlobalModule, CoreModule],
|
||||
providers: [DriveService, S3Service],
|
||||
providers: [DriveService],
|
||||
}).compile();
|
||||
app.enableShutdownHooks();
|
||||
driveService = app.get<DriveService>(DriveService);
|
||||
});
|
||||
|
||||
const s3Service = app.get<S3Service>(S3Service);
|
||||
const s3 = s3Service.getS3({} as Meta);
|
||||
beforeEach(async () => {
|
||||
s3Mock.reset();
|
||||
});
|
||||
|
||||
// new S3() surprisingly does not return an instance of class S3.
|
||||
// Let's use getPrototypeOf here to get a real prototype, since spying on S3.prototype doesn't work.
|
||||
// TODO: Use `aws-sdk-client-mock` package when upgrading to AWS SDK v3.
|
||||
jest.spyOn(Object.getPrototypeOf(s3), 'deleteObject').mockImplementation(() => {
|
||||
// Roughly mock AWS request object
|
||||
return {
|
||||
async promise(): Promise<PromiseResult<DeleteObjectOutput, AWSError>> {
|
||||
const err = new Error('mock') as AWSError;
|
||||
err.code = 'NoSuchKey';
|
||||
throw err;
|
||||
},
|
||||
} as Request<DeleteObjectOutput, AWSError>;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
describe('Object storage', () => {
|
||||
test('delete a file', async () => {
|
||||
s3Mock.on(DeleteObjectCommand)
|
||||
.resolves({} as DeleteObjectCommandOutput);
|
||||
|
||||
await driveService.deleteObjectStorageFile('peace of the world');
|
||||
});
|
||||
|
||||
test('delete a file then unexpected error', async () => {
|
||||
s3Mock.on(DeleteObjectCommand)
|
||||
.rejects(new InvalidObjectState({ $metadata: {}, message: '' }));
|
||||
|
||||
await expect(driveService.deleteObjectStorageFile('unexpected')).rejects.toThrowError(Error);
|
||||
});
|
||||
|
||||
test('delete a file with no valid key', async () => {
|
||||
try {
|
||||
await driveService.deleteObjectStorageFile('lol no way');
|
||||
} catch (err: any) {
|
||||
console.log(err.cause);
|
||||
throw err;
|
||||
}
|
||||
// Some S3 implementations returns 404 Not Found on deleting with a non-existent key
|
||||
s3Mock.on(DeleteObjectCommand)
|
||||
.rejects(new NoSuchKey({ $metadata: {}, message: 'allowed error.' }));
|
||||
|
||||
await driveService.deleteObjectStorageFile('lol no way');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
77
packages/backend/test/unit/S3Service.ts
Normal file
77
packages/backend/test/unit/S3Service.ts
Normal file
|
@ -0,0 +1,77 @@
|
|||
process.env.NODE_ENV = 'test';
|
||||
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { UploadPartCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { mockClient } from 'aws-sdk-client-mock';
|
||||
import { GlobalModule } from '@/GlobalModule.js';
|
||||
import { CoreModule } from '@/core/CoreModule.js';
|
||||
import { S3Service } from '@/core/S3Service';
|
||||
import { Meta } from '@/models';
|
||||
import type { TestingModule } from '@nestjs/testing';
|
||||
|
||||
describe('S3Service', () => {
|
||||
let app: TestingModule;
|
||||
let s3Service: S3Service;
|
||||
const s3Mock = mockClient(S3Client);
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await Test.createTestingModule({
|
||||
imports: [GlobalModule, CoreModule],
|
||||
providers: [S3Service],
|
||||
}).compile();
|
||||
app.enableShutdownHooks();
|
||||
s3Service = app.get<S3Service>(S3Service);
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
s3Mock.reset();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
describe('upload', () => {
|
||||
test('upload a file', async () => {
|
||||
s3Mock.on(PutObjectCommand).resolves({});
|
||||
|
||||
await s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
|
||||
Bucket: 'fake',
|
||||
Key: 'fake',
|
||||
Body: 'x',
|
||||
});
|
||||
});
|
||||
|
||||
test('upload a large file', async () => {
|
||||
s3Mock.on(CreateMultipartUploadCommand).resolves({ UploadId: '1' });
|
||||
s3Mock.on(UploadPartCommand).resolves({ ETag: '1' });
|
||||
s3Mock.on(CompleteMultipartUploadCommand).resolves({ Bucket: 'fake', Key: 'fake' });
|
||||
|
||||
await s3Service.upload({} as Meta, {
|
||||
Bucket: 'fake',
|
||||
Key: 'fake',
|
||||
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
|
||||
});
|
||||
});
|
||||
|
||||
test('upload a file error', async () => {
|
||||
s3Mock.on(PutObjectCommand).rejects({ name: 'Fake Error' });
|
||||
|
||||
await expect(s3Service.upload({ objectStorageRegion: 'us-east-1' } as Meta, {
|
||||
Bucket: 'fake',
|
||||
Key: 'fake',
|
||||
Body: 'x',
|
||||
})).rejects.toThrowError(Error);
|
||||
});
|
||||
|
||||
test('upload a large file error', async () => {
|
||||
s3Mock.on(UploadPartCommand).rejects();
|
||||
|
||||
await expect(s3Service.upload({} as Meta, {
|
||||
Bucket: 'fake',
|
||||
Key: 'fake',
|
||||
Body: 'x'.repeat(8 * 1024 * 1024 + 1), // デフォルトpartSizeにしている 8 * 1024 * 1024 を越えるサイズ
|
||||
})).rejects.toThrowError(Error);
|
||||
});
|
||||
});
|
||||
});
|
1176
pnpm-lock.yaml
1176
pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
Loading…
Reference in a new issue