enhance(backend): ビデオファイルにビデオトラックがあるかを確認するように (#13568)

* enhance(backend): ビデオファイルにビデオトラックがあるかを確認するように

(cherry picked from commit 23d38a2d6492a2b24e9b2c031d66c3e8a5d382ef)

* Update Changelog

* Update Changelog

* Revert "Update Changelog"

This reverts commit 93fd996932b87ef550c38b48bd0678060f3ed1af.

* fix(test) ffmpegをインストールするように

* 入れる方間違えた

* fix test

* 拡張子変わらなかったのでそのまま行く

* ログを出力するように

* msg

* remove unused import

* add log

* attempt to fix test error

* Revert "attempt to fix test error"

This reverts commit d9d6524cadd655e6d8e9398b26fdfef332f30f4d.

* Update FileInfoService.ts

* oggも検査の対象にする

.github/workflows/test-backend.yml

@@ -45,6 +45,8 @@ jobs:
       with:
         version: 8
         run_install: false
+    - name: Install FFmpeg
+      uses: FedericoCarboni/setup-ffmpeg@v3
     - name: Use Node.js ${{ matrix.node-version }}
       uses: actions/setup-node@v4.0.2
       with:

CHANGELOG.md

@@ -41,6 +41,7 @@
   (Cherry-picked from https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/440)
 - Fix: エンドポイント`notes/translate`のエラーを改善
 - Fix: CleanRemoteFilesProcessorService report progress from 100% (#13632)
+- Fix: 一部の音声ファイルが映像ファイルとして扱われる問題を修正
 
 ## 2024.3.1
 

packages/backend/src/core/FileInfoService.ts

@@ -14,11 +14,12 @@ import FFmpeg from 'fluent-ffmpeg';
 import isSvg from 'is-svg';
 import probeImageSize from 'probe-image-size';
 import { type predictionType } from 'nsfwjs';
-import sharp from 'sharp';
 import { sharpBmp } from '@misskey-dev/sharp-read-bmp';
 import { encode } from 'blurhash';
 import { createTempDir } from '@/misc/create-temp.js';
 import { AiService } from '@/core/AiService.js';
+import { LoggerService } from '@/core/LoggerService.js';
+import type Logger from '@/logger.js';
 import { bindThis } from '@/decorators.js';
 
 export type FileInfo = {
@@ -49,9 +50,13 @@ const TYPE_SVG = {
 
 @Injectable()
 export class FileInfoService {
+	private logger: Logger;
+
 	constructor(
 		private aiService: AiService,
+		private loggerService: LoggerService,
 	) {
+		this.logger = this.loggerService.getLogger('file-info');
 	}
 
 	/**
@@ -317,6 +322,34 @@ export class FileInfoService {
 		return mime;
 	}
 
+	/**
+	 * ビデオファイルにビデオトラックがあるかどうかチェック
+	 * （ない場合：m4a, webmなど）
+	 *
+	 * @param path ファイルパス
+	 * @returns ビデオトラックがあるかどうか（エラー発生時は常に`true`を返す）
+	 */
+	@bindThis
+	private hasVideoTrackOnVideoFile(path: string): Promise<boolean> {
+		const sublogger = this.logger.createSubLogger('ffprobe');
+		sublogger.info(`Checking the video file. File path: ${path}`);
+		return new Promise((resolve) => {
+			try {
+				FFmpeg.ffprobe(path, (err, metadata) => {
+					if (err) {
+						sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err);
+						resolve(true);
+						return;
+					}
+					resolve(metadata.streams.some((stream) => stream.codec_type === 'video'));
+				});
+			} catch (err) {
+				sublogger.warn(`Could not check the video file. Returns true. File path: ${path}`, err as Error);
+				resolve(true);
+			}
+		});
+	}
+
 	/**
 	 * Detect MIME Type and extension
 	 */
@@ -339,6 +372,20 @@ export class FileInfoService {
 				return TYPE_SVG;
 			}
 
+			if ((type.mime.startsWith('video') || type.mime === 'application/ogg') && !(await this.hasVideoTrackOnVideoFile(path))) {
+				const newMime = `audio/${type.mime.split('/')[1]}`;
+				if (newMime === 'audio/mp4') {
+					return {
+						mime: 'audio/mp4',
+						ext: 'm4a',
+					};
+				}
+				return {
+					mime: newMime,
+					ext: type.ext,
+				};
+			}
+
 			return {
 				mime: this.fixMime(type.mime),
 				ext: type.ext,

packages/backend/test/unit/FileInfoService.ts

@@ -15,6 +15,7 @@ import { GlobalModule } from '@/GlobalModule.js';
 import { FileInfoService } from '@/core/FileInfoService.js';
 //import { DI } from '@/di-symbols.js';
 import { AiService } from '@/core/AiService.js';
+import { LoggerService } from '@/core/LoggerService.js';
 import type { TestingModule } from '@nestjs/testing';
 import type { MockFunctionMetadata } from 'jest-mock';
 
@@ -35,6 +36,7 @@ describe('FileInfoService', () => {
 			],
 			providers: [
 				AiService,
+				LoggerService,
 				FileInfoService,
 			],
 		})
@@ -323,8 +325,26 @@ describe('FileInfoService', () => {
 			});
 		});
 
-		/*
-		 * video/webmとして検出されてしまう
+		test('MPEG-4 AUDIO (M4A)', async () => {
+			const path = `${resources}/kick_gaba7.m4a`;
+			const info = await fileInfoService.getFileInfo(path, { skipSensitiveDetection: true }) as any;
+			delete info.warnings;
+			delete info.blurhash;
+			delete info.sensitive;
+			delete info.porn;
+			delete info.width;
+			delete info.height;
+			delete info.orientation;
+			assert.deepStrictEqual(info, {
+				size: 9817,
+				md5: '74c9279a4abe98789565f1dc1a541a42',
+				type: {
+					mime: 'audio/mp4',
+					ext: 'm4a',
+				},
+			});
+		});
+
 		test('WEBM AUDIO', async () => {
 			const path = `${resources}/kick_gaba7.webm`;
 			const info = await fileInfoService.getFileInfo(path, { skipSensitiveDetection: true }) as any;
@@ -337,13 +357,12 @@ describe('FileInfoService', () => {
 			delete info.orientation;
 			assert.deepStrictEqual(info, {
 				size: 8879,
-				md5: '3350083dec312419cfdc06c16413aca7',
+				md5: '53bc1adcb6acbbda67ff9bd484896438',
 				type: {
 					mime: 'audio/webm',
 					ext: 'webm',
 				},
 			});
 		});
-		 */
 	});
 });