From ba23c63a3824328cb87de03e98da6db6a7114673 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Tue, 15 Aug 2017 21:51:32 +0000 Subject: [PATCH 01/61] chore(package): update @types/js-yaml to version 3.9.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 875ffc3c66..1632eed245 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "@types/inquirer": "0.0.34", "@types/is-root": "1.0.0", "@types/is-url": "1.2.28", - "@types/js-yaml": "3.9.0", + "@types/js-yaml": "3.9.1", "@types/mocha": "2.2.41", "@types/mongodb": "2.2.9", "@types/monk": "1.0.5", From cdee14faa262b284c6d10cb53d6bf4ccaf4da5db Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Mon, 21 Aug 2017 21:49:00 +0000 Subject: [PATCH 02/61] chore(package): update @types/bcryptjs to version 2.4.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 141e95c88b..ff3f6c79fe 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "test": "gulp test" }, "devDependencies": { - "@types/bcryptjs": "2.4.0", + "@types/bcryptjs": "2.4.1", "@types/body-parser": "1.16.4", "@types/chai": "4.0.3", "@types/chai-http": "3.0.2", From e2b359399fb38d423490e1c418ce5f6c2e43ea84 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Mon, 21 Aug 2017 23:28:05 +0000 Subject: [PATCH 03/61] chore(package): update @types/rimraf to version 2.0.2 Closes #697 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 47006ae5dd..f16c3c001e 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "@types/ratelimiter": "2.1.28", "@types/redis": "2.6.0", "@types/request": "2.0.1", - "@types/rimraf": "2.0.0", + "@types/rimraf": "2.0.2", "@types/riot": "3.6.0", "@types/serve-favicon": "2.2.28", "@types/uuid": "3.4.0", From 73c85a52be7e06c087237761e887620e5f2f9221 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Mon, 21 Aug 2017 23:28:23 +0000 Subject: [PATCH 04/61] chore(package): update @types/serve-favicon to version 2.2.29 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 47006ae5dd..639261cb99 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "@types/request": "2.0.1", "@types/rimraf": "2.0.0", "@types/riot": "3.6.0", - "@types/serve-favicon": "2.2.28", + "@types/serve-favicon": "2.2.29", "@types/uuid": "3.4.0", "@types/webpack": "3.0.9", "@types/webpack-stream": "3.2.7", From 98e6ce6d6a1ebf31ccb2b1748deae671f8222a70 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 28 Sep 2017 07:55:56 +0000 Subject: [PATCH 05/61] chore(package): update @types/elasticsearch to version 5.0.17 Closes #698 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0a437eefb4..4552041115 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@types/cors": "2.8.1", "@types/debug": "0.0.30", "@types/deep-equal": "1.0.1", - "@types/elasticsearch": "5.0.14", + "@types/elasticsearch": "5.0.17", "@types/event-stream": "3.3.32", "@types/express": "4.0.37", "@types/gm": "1.17.32", From 29edeb2dc08fdb7072e288177abe0aa26c4c2109 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Fri, 6 Oct 2017 03:32:55 +0000 Subject: [PATCH 06/61] chore(package): update mocha to version 4.0.1 Closes #810 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b56f97c797..4dc11b02f2 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "gulp-typescript": "3.2.2", "gulp-uglify": "3.0.0", "gulp-util": "3.0.8", - "mocha": "3.5.3", + "mocha": "4.0.1", "riot-tag-loader": "1.0.0", "string-replace-webpack-plugin": "0.1.3", "style-loader": "0.19.0", From c5b9e01288c42c6411f8c36aa042002e3d4199be Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Tue, 17 Oct 2017 00:15:25 +0000 Subject: [PATCH 07/61] chore(package): update @types/gm to version 1.17.33 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 688664fa66..0c6d3ff3d8 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@types/elasticsearch": "5.0.14", "@types/event-stream": "3.3.32", "@types/express": "4.0.37", - "@types/gm": "1.17.32", + "@types/gm": "1.17.33", "@types/gulp": "4.0.3", "@types/gulp-htmlmin": "1.3.30", "@types/gulp-mocha": "0.0.30", From 97bebddbc03b4a28aa0ba52e79e9fe67879e615c Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Tue, 17 Oct 2017 07:08:45 +0000 Subject: [PATCH 08/61] fix(package): update file-type to version 7.2.0 Closes #821 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 688664fa66..a7f52d0649 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "elasticsearch": "13.3.1", "escape-regexp": "0.0.1", "express": "4.15.4", - "file-type": "6.2.0", + "file-type": "7.2.0", "fuckadblock": "3.2.1", "gm": "1.23.0", "inquirer": "3.3.0", From 7dc81a3bda40fed661be414ee45ef6e161998977 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Tue, 17 Oct 2017 18:37:46 +0000 Subject: [PATCH 09/61] chore(package): update @types/uuid to version 3.4.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 688664fa66..daf3e3dc03 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "@types/rimraf": "2.0.0", "@types/riot": "3.6.0", "@types/serve-favicon": "2.2.28", - "@types/uuid": "3.4.2", + "@types/uuid": "3.4.3", "@types/webpack": "3.0.13", "@types/webpack-stream": "3.2.7", "@types/websocket": "0.0.34", From 73a209359fad3cfdb6b96064bc228686a4f2d4fa Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Fri, 20 Oct 2017 19:19:57 +0000 Subject: [PATCH 10/61] chore(package): update tslint to version 5.8.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dd1622a37b..09a8b7c889 100644 --- a/package.json +++ b/package.json @@ -88,7 +88,7 @@ "stylus": "0.54.5", "stylus-loader": "3.0.1", "swagger-jsdoc": "1.9.7", - "tslint": "5.7.0", + "tslint": "5.8.0", "uglify-es": "3.0.27", "uglify-es-webpack-plugin": "0.10.0", "uglify-js": "git+https://github.com/mishoo/UglifyJS2.git#harmony", From a39168b081eb4b977930e73c050801df97ea0845 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Mon, 23 Oct 2017 20:25:17 +0000 Subject: [PATCH 11/61] chore(package): update gulp-typescript to version 3.2.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb3cb451..aa98122de8 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "gulp-rename": "1.2.2", "gulp-replace": "0.6.1", "gulp-tslint": "8.1.2", - "gulp-typescript": "3.2.2", + "gulp-typescript": "3.2.3", "gulp-uglify": "3.0.0", "gulp-util": "3.0.8", "mocha": "3.5.3", From fdcff509d48081bdf58291f9c383b0ab8273c280 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Tue, 24 Oct 2017 02:40:07 +0000 Subject: [PATCH 12/61] chore(package): update uglifyjs-webpack-plugin to version 1.0.1 Closes #841 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb3cb451..0b990ea40f 100644 --- a/package.json +++ b/package.json @@ -91,7 +91,7 @@ "tslint": "5.7.0", "uglify-es": "3.0.27", "uglify-js": "git+https://github.com/mishoo/UglifyJS2.git#harmony", - "uglifyjs-webpack-plugin": "1.0.0-beta.2", + "uglifyjs-webpack-plugin": "1.0.1", "webpack": "3.8.1" }, "dependencies": { From 7d95989fa03021fbaf163d661afb144ba580161e Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Tue, 24 Oct 2017 04:31:04 +0000 Subject: [PATCH 13/61] fix(package): update chalk to version 2.3.0 Closes #833 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb3cb451..2284bb01b0 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "bcryptjs": "2.4.3", "body-parser": "1.18.2", "cafy": "3.0.0", - "chalk": "2.1.0", + "chalk": "2.3.0", "compression": "1.7.1", "cors": "2.8.4", "cropperjs": "1.1.3", From 51476ed4c89b0d4f507cd5aa92ef32cf5e4cfa5d Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 00:30:04 +0000 Subject: [PATCH 14/61] chore(package): update @types/gulp to version 4.0.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb3cb451..7be9e49d63 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "@types/event-stream": "3.3.32", "@types/express": "4.0.37", "@types/gm": "1.17.32", - "@types/gulp": "4.0.3", + "@types/gulp": "4.0.5", "@types/gulp-htmlmin": "1.3.30", "@types/gulp-mocha": "0.0.30", "@types/gulp-rename": "0.0.32", From 513ed7d90122069ee850cc6d830e5476d7f0b042 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 01:05:05 +0000 Subject: [PATCH 15/61] chore(package): update @types/gulp-uglify to version 3.0.3 Closes #664 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb3cb451..96d3973071 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "@types/gulp-replace": "0.0.30", "@types/gulp-tslint": "3.6.31", "@types/gulp-typescript": "2.13.0", - "@types/gulp-uglify": "0.0.30", + "@types/gulp-uglify": "3.0.3", "@types/gulp-util": "3.0.31", "@types/inquirer": "0.0.34", "@types/is-root": "1.0.0", From 58b097620a4e6638dc288dc9e1effb238baf913a Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 01:30:35 +0000 Subject: [PATCH 16/61] chore(package): update @types/mongodb to version 2.2.15 Closes #836 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb3cb451..5105ea7273 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "@types/is-url": "1.2.28", "@types/js-yaml": "3.9.0", "@types/mocha": "2.2.43", - "@types/mongodb": "2.2.13", + "@types/mongodb": "2.2.15", "@types/monk": "1.0.6", "@types/morgan": "1.7.33", "@types/ms": "0.7.30", From ee9ff7edd8234482c1f5f168169117647276e59d Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 02:13:31 +0000 Subject: [PATCH 17/61] chore(package): update @types/node to version 8.0.47 Closes #822 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ddb3cb451..0f63f1f0af 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@types/morgan": "1.7.33", "@types/ms": "0.7.30", "@types/multer": "1.3.2", - "@types/node": "8.0.33", + "@types/node": "8.0.47", "@types/ratelimiter": "2.1.28", "@types/redis": "2.6.0", "@types/request": "2.0.4", From e830dd71e1644b8627d3833cb663faa90af9aae2 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 16:48:59 +0000 Subject: [PATCH 18/61] chore(package): update @types/riot to version 3.6.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..1eb0e94cda 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "@types/redis": "2.6.0", "@types/request": "2.0.4", "@types/rimraf": "2.0.0", - "@types/riot": "3.6.0", + "@types/riot": "3.6.1", "@types/serve-favicon": "2.2.28", "@types/uuid": "3.4.2", "@types/webpack": "3.0.13", From 781ca218e7760c2d3ce33d5169779e303f2a658a Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 18:05:35 +0000 Subject: [PATCH 19/61] chore(package): update @types/webpack to version 3.0.14 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..d15c25a289 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "@types/riot": "3.6.0", "@types/serve-favicon": "2.2.28", "@types/uuid": "3.4.2", - "@types/webpack": "3.0.13", + "@types/webpack": "3.0.14", "@types/webpack-stream": "3.2.7", "@types/websocket": "0.0.34", "awesome-typescript-loader": "3.2.3", From c8c16c8190751589144f350d8658ab57adaa735a Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 19:26:49 +0000 Subject: [PATCH 20/61] chore(package): update @types/webpack-stream to version 3.2.8 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..f83f7ff0be 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "@types/serve-favicon": "2.2.28", "@types/uuid": "3.4.2", "@types/webpack": "3.0.13", - "@types/webpack-stream": "3.2.7", + "@types/webpack-stream": "3.2.8", "@types/websocket": "0.0.34", "awesome-typescript-loader": "3.2.3", "chai": "4.1.2", From e25ae9ad6e921f6a5c65b11553b26df924006bfe Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 25 Oct 2017 20:20:52 +0000 Subject: [PATCH 21/61] chore(package): update @types/request to version 2.0.7 Closes #827 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..473878e394 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "@types/node": "8.0.33", "@types/ratelimiter": "2.1.28", "@types/redis": "2.6.0", - "@types/request": "2.0.4", + "@types/request": "2.0.7", "@types/rimraf": "2.0.0", "@types/riot": "3.6.0", "@types/serve-favicon": "2.2.28", From 753df86030e6c503850333a1c0d7c6bd5334603c Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 26 Oct 2017 15:08:12 +0000 Subject: [PATCH 22/61] chore(package): update @types/gulp-util to version 3.0.33 Closes #846 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..1da36f5be9 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "@types/gulp-tslint": "3.6.31", "@types/gulp-typescript": "2.13.0", "@types/gulp-uglify": "0.0.30", - "@types/gulp-util": "3.0.31", + "@types/gulp-util": "3.0.33", "@types/inquirer": "0.0.34", "@types/is-root": "1.0.0", "@types/is-url": "1.2.28", From a2621f40d7f1f311089975c4b255f108fad0fdee Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 26 Oct 2017 15:09:38 +0000 Subject: [PATCH 23/61] chore(package): update @types/chalk to version 2.2.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..0deb7dbb07 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "@types/body-parser": "1.16.5", "@types/chai": "4.0.4", "@types/chai-http": "3.0.3", - "@types/chalk": "0.4.31", + "@types/chalk": "2.2.0", "@types/compression": "0.0.34", "@types/cors": "2.8.1", "@types/debug": "0.0.30", From cdff672a8354ee119a8b704075ff69b3ff760c25 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 26 Oct 2017 18:06:50 +0000 Subject: [PATCH 24/61] chore(package): update @types/mocha to version 2.2.44 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..163c2da574 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "@types/is-root": "1.0.0", "@types/is-url": "1.2.28", "@types/js-yaml": "3.9.0", - "@types/mocha": "2.2.43", + "@types/mocha": "2.2.44", "@types/mongodb": "2.2.13", "@types/monk": "1.0.6", "@types/morgan": "1.7.33", From 55c480cd7e43d5a272acc87f905f9b9f3bab6a48 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 26 Oct 2017 19:32:50 +0000 Subject: [PATCH 25/61] chore(package): update @types/body-parser to version 1.16.7 Closes #843 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..8b32db9a87 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ }, "devDependencies": { "@types/bcryptjs": "2.4.0", - "@types/body-parser": "1.16.5", + "@types/body-parser": "1.16.7", "@types/chai": "4.0.4", "@types/chai-http": "3.0.3", "@types/chalk": "0.4.31", From c4806958fc86c2e65351ce23bba2688ac81a2ddf Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 26 Oct 2017 19:34:24 +0000 Subject: [PATCH 26/61] chore(package): update @types/express to version 4.0.39 Closes #844 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..c6da419d2f 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "@types/deep-equal": "1.0.1", "@types/elasticsearch": "5.0.14", "@types/event-stream": "3.3.32", - "@types/express": "4.0.37", + "@types/express": "4.0.39", "@types/gm": "1.17.32", "@types/gulp": "4.0.3", "@types/gulp-htmlmin": "1.3.30", From cccf267f70bbe8c033c9b6cf3ab5053d4b3f9ddd Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 26 Oct 2017 19:35:00 +0000 Subject: [PATCH 27/61] chore(package): update @types/morgan to version 1.7.35 Closes #847 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..d2c8ac38a3 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "@types/mocha": "2.2.43", "@types/mongodb": "2.2.13", "@types/monk": "1.0.6", - "@types/morgan": "1.7.33", + "@types/morgan": "1.7.35", "@types/ms": "0.7.30", "@types/multer": "1.3.2", "@types/node": "8.0.33", From b6733c57d15d1767acbb40c1a9b0fc05618d3259 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Thu, 26 Oct 2017 19:35:25 +0000 Subject: [PATCH 28/61] chore(package): update @types/multer to version 1.3.5 Closes #725 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..7e1d481a4e 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "@types/monk": "1.0.6", "@types/morgan": "1.7.33", "@types/ms": "0.7.30", - "@types/multer": "1.3.2", + "@types/multer": "1.3.5", "@types/node": "8.0.33", "@types/ratelimiter": "2.1.28", "@types/redis": "2.6.0", From 4da5fd57c7e49111e9116f0990ad081fcd4d7963 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Sat, 28 Oct 2017 00:55:09 +0000 Subject: [PATCH 29/61] chore(package): update @types/redis to version 2.8.1 Closes #829 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43a0159619..e278cadd8a 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "@types/multer": "1.3.2", "@types/node": "8.0.33", "@types/ratelimiter": "2.1.28", - "@types/redis": "2.6.0", + "@types/redis": "2.8.1", "@types/request": "2.0.4", "@types/rimraf": "2.0.0", "@types/riot": "3.6.0", From 1ecc35ca6fa2e8a5b4a3df1b93893b31e192a9f4 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Tue, 31 Oct 2017 17:03:16 +0000 Subject: [PATCH 30/61] fix(package): update typescript to version 2.6.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7a81bed7a6..4c6cfb5f3a 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,7 @@ "tcp-port-used": "0.1.2", "textarea-caret": "3.0.2", "ts-node": "3.3.0", - "typescript": "2.5.3", + "typescript": "2.6.1", "uuid": "3.1.0", "vhost": "3.0.2", "websocket": "1.0.25", From fcdf2c4f89c4eb7bb666337d7e162e1c5e727e61 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Wed, 1 Nov 2017 06:55:03 +0000 Subject: [PATCH 31/61] chore(package): update awesome-typescript-loader to version 3.3.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 87db0c8e1e..dd5a57015a 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "@types/webpack": "3.0.13", "@types/webpack-stream": "3.2.7", "@types/websocket": "0.0.34", - "awesome-typescript-loader": "3.2.3", + "awesome-typescript-loader": "3.3.0", "chai": "4.1.2", "chai-http": "3.0.0", "css-loader": "0.28.7", From 0e8a4f36a24826909a2c0e864c4c131b56ea0e8a Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 20:55:51 +0900 Subject: [PATCH 32/61] start to improve serializers From 78487934c7e55b36b07f30b73127577ddec31f32 Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 21:11:16 +0900 Subject: [PATCH 33/61] selializers - posts: unneed async-await Promise.all resolves all Promise, and selializeDriveFile returns Promise. --- src/api/serializers/post.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index 7c3690ef79..b2c54e9df8 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -84,8 +84,8 @@ const self = ( // Populate media if (_post.media_ids) { - _post.media = await Promise.all(_post.media_ids.map(async fileId => - await serializeDriveFile(fileId) + _post.media = await Promise.all(_post.media_ids.map(fileId => + serializeDriveFile(fileId) )); } From 11190f56ad85a12faaa3653f8743dd75948ff11e Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 22:13:28 +0900 Subject: [PATCH 34/61] serializers/post - run promises in parallel now w/ opts.detail, returns my_reaction field as 'null' w/ no reaction (before: field appears w/ some reaction) --- package.json | 1 + src/api/serializers/post.ts | 122 ++++++++++++++++++++---------------- 2 files changed, 70 insertions(+), 53 deletions(-) diff --git a/package.json b/package.json index 051eb1cb83..1e6e8d8136 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,7 @@ "webpack": "3.8.1" }, "dependencies": { + "@prezzemolo/rap": "^0.1.0", "accesses": "2.5.0", "animejs": "2.2.0", "autwh": "0.0.1", diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index b2c54e9df8..352932acff 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -12,6 +12,7 @@ import serializeChannel from './channel'; import serializeUser from './user'; import serializeDriveFile from './drive-file'; import parse from '../common/text'; +import rap from '@prezzemolo/rap' /** * Serialize a post @@ -70,21 +71,21 @@ const self = ( } // Populate user - _post.user = await serializeUser(_post.user_id, meId); + _post.user = serializeUser(_post.user_id, meId); // Populate app if (_post.app_id) { - _post.app = await serializeApp(_post.app_id); + _post.app = serializeApp(_post.app_id); } // Populate channel if (_post.channel_id) { - _post.channel = await serializeChannel(_post.channel_id); + _post.channel = serializeChannel(_post.channel_id); } // Populate media if (_post.media_ids) { - _post.media = await Promise.all(_post.media_ids.map(fileId => + _post.media = Promise.all(_post.media_ids.map(fileId => serializeDriveFile(fileId) )); } @@ -92,82 +93,97 @@ const self = ( // When requested a detailed post data if (opts.detail) { // Get previous post info - const prev = await Post.findOne({ - user_id: _post.user_id, - _id: { - $lt: id - } - }, { - fields: { - _id: true - }, - sort: { - _id: -1 - } - }); - _post.prev = prev ? prev._id : null; + _post.prev = (async () => { + const prev = Post.findOne({ + user_id: _post.user_id, + _id: { + $lt: id + } + }, { + fields: { + _id: true + }, + sort: { + _id: -1 + } + }); + return prev ? prev._id : null; + })() // Get next post info - const next = await Post.findOne({ - user_id: _post.user_id, - _id: { - $gt: id - } - }, { - fields: { - _id: true - }, - sort: { - _id: 1 - } - }); - _post.next = next ? next._id : null; + _post.next = (async () => { + const next = await Post.findOne({ + user_id: _post.user_id, + _id: { + $gt: id + } + }, { + fields: { + _id: true + }, + sort: { + _id: 1 + } + }); + return next ? next._id : null; + })() if (_post.reply_id) { // Populate reply to post - _post.reply = await self(_post.reply_id, meId, { + _post.reply = self(_post.reply_id, meId, { detail: false }); } if (_post.repost_id) { // Populate repost - _post.repost = await self(_post.repost_id, meId, { + _post.repost = self(_post.repost_id, meId, { detail: _post.text == null }); } // Poll if (meId && _post.poll) { - const vote = await Vote - .findOne({ - user_id: meId, - post_id: id - }); + _post.poll = (async (poll) => { + const vote = await Vote + .findOne({ + user_id: meId, + post_id: id + }); - if (vote != null) { - const myChoice = _post.poll.choices - .filter(c => c.id == vote.choice)[0]; + if (vote != null) { + const myChoice = poll.choices + .filter(c => c.id == vote.choice)[0]; - myChoice.is_voted = true; - } + myChoice.is_voted = true; + } + + return poll + })(_post.poll) } // Fetch my reaction if (meId) { - const reaction = await Reaction - .findOne({ - user_id: meId, - post_id: id, - deleted_at: { $exists: false } - }); + _post.my_reaction = (async () => { + const reaction = await Reaction + .findOne({ + user_id: meId, + post_id: id, + deleted_at: { $exists: false } + }); - if (reaction) { - _post.my_reaction = reaction.reaction; - } + if (reaction) { + return reaction.reaction; + } + + return null + })(); } } + // resolve promises in _post object + _post = await rap(_post) + resolve(_post); }); From 5aa5e5cc7074003cec3417636ea1972b6d88150d Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 22:22:49 +0900 Subject: [PATCH 35/61] serializers - user: run promises in parallel as possible --- src/api/serializers/post.ts | 2 +- src/api/serializers/user.ts | 40 +++++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index 352932acff..99e9bb667c 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -12,7 +12,7 @@ import serializeChannel from './channel'; import serializeUser from './user'; import serializeDriveFile from './drive-file'; import parse from '../common/text'; -import rap from '@prezzemolo/rap' +import rap from '@prezzemolo/rap'; /** * Serialize a post diff --git a/src/api/serializers/user.ts b/src/api/serializers/user.ts index 3deff2d003..3527921ded 100644 --- a/src/api/serializers/user.ts +++ b/src/api/serializers/user.ts @@ -8,6 +8,7 @@ import serializePost from './post'; import Following from '../models/following'; import getFriends from '../common/get-friends'; import config from '../../conf'; +import rap from '@prezzemolo/rap'; /** * Serialize a user @@ -104,26 +105,30 @@ export default ( if (meId && !meId.equals(_user.id)) { // If the user is following - const follow = await Following.findOne({ - follower_id: meId, - followee_id: _user.id, - deleted_at: { $exists: false } - }); - _user.is_following = follow !== null; + _user.is_following = (async () => { + const follow = await Following.findOne({ + follower_id: meId, + followee_id: _user.id, + deleted_at: { $exists: false } + }); + return follow !== null; + })() // If the user is followed - const follow2 = await Following.findOne({ - follower_id: _user.id, - followee_id: meId, - deleted_at: { $exists: false } - }); - _user.is_followed = follow2 !== null; + _user.is_followed = (async () => { + const follow2 = await Following.findOne({ + follower_id: _user.id, + followee_id: meId, + deleted_at: { $exists: false } + }); + return follow2 !== null; + })() } if (opts.detail) { if (_user.pinned_post_id) { // Populate pinned post - _user.pinned_post = await serializePost(_user.pinned_post_id, meId, { + _user.pinned_post = serializePost(_user.pinned_post_id, meId, { detail: true }); } @@ -132,23 +137,24 @@ export default ( const myFollowingIds = await getFriends(meId); // Get following you know count - const followingYouKnowCount = await Following.count({ + _user.following_you_know_count = Following.count({ followee_id: { $in: myFollowingIds }, follower_id: _user.id, deleted_at: { $exists: false } }); - _user.following_you_know_count = followingYouKnowCount; // Get followers you know count - const followersYouKnowCount = await Following.count({ + _user.followers_you_know_count = Following.count({ followee_id: _user.id, follower_id: { $in: myFollowingIds }, deleted_at: { $exists: false } }); - _user.followers_you_know_count = followersYouKnowCount; } } + // resolve promises in _user object + _user = await rap(_user) + resolve(_user); }); /* From 7cd6b1c666605c7a256e4a8dd8db5edeb02da6db Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 22:26:16 +0900 Subject: [PATCH 36/61] follow lint --- src/api/serializers/post.ts | 12 ++++++------ src/api/serializers/user.ts | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index 99e9bb667c..e1ab784359 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -108,7 +108,7 @@ const self = ( } }); return prev ? prev._id : null; - })() + })(); // Get next post info _post.next = (async () => { @@ -126,7 +126,7 @@ const self = ( } }); return next ? next._id : null; - })() + })(); if (_post.reply_id) { // Populate reply to post @@ -158,8 +158,8 @@ const self = ( myChoice.is_voted = true; } - return poll - })(_post.poll) + return poll; + })(_post.poll); } // Fetch my reaction @@ -176,13 +176,13 @@ const self = ( return reaction.reaction; } - return null + return null; })(); } } // resolve promises in _post object - _post = await rap(_post) + _post = await rap(_post); resolve(_post); }); diff --git a/src/api/serializers/user.ts b/src/api/serializers/user.ts index 3527921ded..d00f073897 100644 --- a/src/api/serializers/user.ts +++ b/src/api/serializers/user.ts @@ -112,7 +112,7 @@ export default ( deleted_at: { $exists: false } }); return follow !== null; - })() + })(); // If the user is followed _user.is_followed = (async () => { @@ -122,7 +122,7 @@ export default ( deleted_at: { $exists: false } }); return follow2 !== null; - })() + })(); } if (opts.detail) { @@ -153,7 +153,7 @@ export default ( } // resolve promises in _user object - _user = await rap(_user) + _user = await rap(_user); resolve(_user); }); From 09baf205ead75eab3eaf0f3de82215665c2a3e73 Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 22:29:58 +0900 Subject: [PATCH 37/61] remove ^ from @prezzemolo/rap dependency --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 1e6e8d8136..c3a093420c 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "clean": "gulp clean", "cleanall": "gulp cleanall", "lint": "gulp lint", - "test": "gulp test" + "test": "gulp test" }, "devDependencies": { "@types/bcryptjs": "2.4.0", @@ -95,7 +95,7 @@ "webpack": "3.8.1" }, "dependencies": { - "@prezzemolo/rap": "^0.1.0", + "@prezzemolo/rap": "0.1.0", "accesses": "2.5.0", "animejs": "2.2.0", "autwh": "0.0.1", From 327d2705b4a3dad6ef8a8dfa8165c25a3a40d109 Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 22:37:00 +0900 Subject: [PATCH 38/61] update @prezzemolo/rap to 0.1.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c3a093420c..27e292cc1a 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "webpack": "3.8.1" }, "dependencies": { - "@prezzemolo/rap": "0.1.0", + "@prezzemolo/rap": "0.1.1", "accesses": "2.5.0", "animejs": "2.2.0", "autwh": "0.0.1", From ac2a0f46cd9ee877adda57bb939a1b31f7109911 Mon Sep 17 00:00:00 2001 From: otofune Date: Sun, 5 Nov 2017 22:47:04 +0900 Subject: [PATCH 39/61] update @prezzemolo/rap to 0.1.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 27e292cc1a..6ea91a7f53 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,7 @@ "webpack": "3.8.1" }, "dependencies": { - "@prezzemolo/rap": "0.1.1", + "@prezzemolo/rap": "0.1.2", "accesses": "2.5.0", "animejs": "2.2.0", "autwh": "0.0.1", From 55fc8de44d18deb6cd89c887895b6b6d30bcd229 Mon Sep 17 00:00:00 2001 From: "greenkeeper[bot]" Date: Sun, 5 Nov 2017 20:40:07 +0000 Subject: [PATCH 40/61] fix(package): update riot to version 3.7.4 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 051eb1cb83..7a7cd1c09c 100644 --- a/package.json +++ b/package.json @@ -140,7 +140,7 @@ "redis": "2.8.0", "request": "2.83.0", "rimraf": "2.6.2", - "riot": "3.7.3", + "riot": "3.7.4", "rndstr": "1.0.0", "s-age": "1.1.0", "serve-favicon": "2.4.5", From 7e81e0db6ac1289ae9504f7e3da5db6e56f41a51 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 14:37:00 +0900 Subject: [PATCH 41/61] support GridFS --- src/api/common/add-file-to-drive.ts | 37 ++++++++++++++++++----------- src/api/models/drive-file.ts | 15 ++++++++++-- src/db/mongodb.ts | 35 +++++++++++++++++++++++---- 3 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 714eeb520d..f48f0cbcf5 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -4,14 +4,27 @@ import * as gm from 'gm'; import * as debug from 'debug'; import fileType = require('file-type'); import prominence = require('prominence'); -import DriveFile from '../models/drive-file'; +import DriveFile, { getGridFSBucket } from '../models/drive-file'; import DriveFolder from '../models/drive-folder'; import serialize from '../serializers/drive-file'; import event from '../event'; import config from '../../conf'; +import { Duplex } from 'stream'; const log = debug('misskey:register-drive-file'); +const addToGridFS = (name, binary, metadata): Promise => new Promise(async (resolve, reject) => { + const dataStream = new Duplex() + dataStream.push(binary) + dataStream.push(null) + + const bucket = await getGridFSBucket() + const writeStream = bucket.openUploadStream(name, { metadata }) + writeStream.once('finish', (doc) => { resolve(doc) }) + writeStream.on('error', reject) + dataStream.pipe(writeStream) +}) + /** * Add file to drive * @@ -58,7 +71,7 @@ export default ( // Generate hash const hash = crypto - .createHash('sha256') + .createHash('md5') .update(data) .digest('hex') as string; @@ -67,8 +80,10 @@ export default ( if (!force) { // Check if there is a file with the same hash const much = await DriveFile.findOne({ - user_id: user._id, - hash: hash + md5: hash, + metadata: { + user_id: user._id + } }); if (much !== null) { @@ -82,13 +97,13 @@ export default ( // Calculate drive usage const usage = ((await DriveFile .aggregate([ - { $match: { user_id: user._id } }, + { $match: { metadata: { user_id: user._id } } }, { $project: { - datasize: true + length: true }}, { $group: { _id: null, - usage: { $sum: '$datasize' } + usage: { $sum: '$length' } }} ]))[0] || { usage: 0 @@ -131,21 +146,15 @@ export default ( } // Create DriveFile document - const file = await DriveFile.insert({ - created_at: new Date(), + const file = await addToGridFS(`${user._id}/${name}`, data, { user_id: user._id, folder_id: folder !== null ? folder._id : null, - data: data, - datasize: size, type: mime, name: name, comment: comment, - hash: hash, properties: properties }); - delete file.data; - log(`drive file has been created ${file._id}`); resolve(file); diff --git a/src/api/models/drive-file.ts b/src/api/models/drive-file.ts index 8d158cf563..79a87f6572 100644 --- a/src/api/models/drive-file.ts +++ b/src/api/models/drive-file.ts @@ -1,11 +1,22 @@ -import db from '../../db/mongodb'; +import * as mongodb from 'mongodb'; +import monkDb, { nativeDbConn } from '../../db/mongodb'; -const collection = db.get('drive_files'); +const collection = monkDb.get('drive_files.files'); (collection as any).createIndex('hash'); // fuck type definition export default collection as any; // fuck type definition +const getGridFSBucket = async (): Promise => { + const db = await nativeDbConn() + const bucket = new mongodb.GridFSBucket(db, { + bucketName: 'drive_files' + }) + return bucket +} + +export { getGridFSBucket } + export function validateFileName(name: string): boolean { return ( (name.trim().length > 0) && diff --git a/src/db/mongodb.ts b/src/db/mongodb.ts index 6ee7f4534f..75f1a1d3c6 100644 --- a/src/db/mongodb.ts +++ b/src/db/mongodb.ts @@ -1,11 +1,38 @@ -import * as mongo from 'monk'; - import config from '../conf'; const uri = config.mongodb.user && config.mongodb.pass - ? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}` - : `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`; +? `mongodb://${config.mongodb.user}:${config.mongodb.pass}@${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}` +: `mongodb://${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`; + +/** + * monk + */ +import * as mongo from 'monk'; const db = mongo(uri); export default db; + +/** + * MongoDB native module (officialy) + */ +import * as mongodb from 'mongodb' + +let mdb: mongodb.Db; + +const nativeDbConn = async (): Promise => { + if (mdb) return mdb; + + const db = await ((): Promise => new Promise((resolve, reject) => { + mongodb.MongoClient.connect(uri, (e, db) => { + if (e) return reject(e) + resolve(db) + }) + }))() + + mdb = db + + return db +} + +export { nativeDbConn } From 18b1ef29adc6166c2b1a327b378c3e159a18b80c Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:18:45 +0900 Subject: [PATCH 42/61] migration to GridFS's DriveFile --- src/api/common/add-file-to-drive.ts | 1 + src/api/endpoints/drive.ts | 6 ++-- src/api/endpoints/drive/files.ts | 9 +++-- src/api/endpoints/drive/files/find.ts | 10 +++--- src/api/endpoints/drive/files/show.ts | 6 ++-- src/api/endpoints/drive/files/update.ts | 31 +++++++++-------- .../endpoints/messaging/messages/create.ts | 6 ++-- src/api/endpoints/posts/create.ts | 6 ++-- src/api/endpoints/posts/timeline.ts | 24 +++++++------- src/api/serializers/drive-file.ts | 33 ++++++++----------- src/api/serializers/drive-folder.ts | 4 ++- 11 files changed, 66 insertions(+), 70 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index f48f0cbcf5..376c470e93 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -154,6 +154,7 @@ export default ( comment: comment, properties: properties }); + console.dir(file) log(`drive file has been created ${file._id}`); diff --git a/src/api/endpoints/drive.ts b/src/api/endpoints/drive.ts index 41ad6301d7..b9c4e3e506 100644 --- a/src/api/endpoints/drive.ts +++ b/src/api/endpoints/drive.ts @@ -14,16 +14,16 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Calculate drive usage const usage = ((await DriveFile .aggregate([ - { $match: { user_id: user._id } }, + { $match: { metadata: { user_id: user._id } } }, { $project: { - datasize: true + length: true } }, { $group: { _id: null, - usage: { $sum: '$datasize' } + usage: { $sum: '$length' } } } ]))[0] || { diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts index a68ae34817..eb0bfe6ba5 100644 --- a/src/api/endpoints/drive/files.ts +++ b/src/api/endpoints/drive/files.ts @@ -40,8 +40,10 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { _id: -1 }; const query = { - user_id: user._id, - folder_id: folderId + metadata: { + user_id: user._id, + folder_id: folderId + } } as any; if (sinceId) { sort._id = 1; @@ -57,9 +59,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { // Issue query const files = await DriveFile .find(query, { - fields: { - data: false - }, limit: limit, sort: sort }); diff --git a/src/api/endpoints/drive/files/find.ts b/src/api/endpoints/drive/files/find.ts index cd0b33f2ca..255faf94ec 100644 --- a/src/api/endpoints/drive/files/find.ts +++ b/src/api/endpoints/drive/files/find.ts @@ -24,12 +24,10 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Issue query const files = await DriveFile .find({ - name: name, - user_id: user._id, - folder_id: folderId - }, { - fields: { - data: false + metadata: { + name: name, + user_id: user._id, + folder_id: folderId } }); diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts index 8dbc297e4f..9135a04c57 100644 --- a/src/api/endpoints/drive/files/show.ts +++ b/src/api/endpoints/drive/files/show.ts @@ -21,10 +21,8 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const file = await DriveFile .findOne({ _id: fileId, - user_id: user._id - }, { - fields: { - data: false + metadata: { + user_id: user._id } }); diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts index 1cfbdd8f0b..c4d2673688 100644 --- a/src/api/endpoints/drive/files/update.ts +++ b/src/api/endpoints/drive/files/update.ts @@ -20,25 +20,29 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const [fileId, fileIdErr] = $(params.file_id).id().$; if (fileIdErr) return rej('invalid file_id param'); + console.dir(user) + // Fetch file const file = await DriveFile .findOne({ _id: fileId, - user_id: user._id - }, { - fields: { - data: false + metadata: { + user_id: user._id } }); + console.dir(file) + if (file === null) { return rej('file-not-found'); } + const updateQuery: any = {} + // Get 'name' parameter const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$; if (nameErr) return rej('invalid name param'); - if (name) file.name = name; + if (name) updateQuery.name = name; // Get 'folder_id' parameter const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$; @@ -46,7 +50,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { if (folderId !== undefined) { if (folderId === null) { - file.folder_id = null; + updateQuery.folder_id = null; } else { // Fetch folder const folder = await DriveFolder @@ -59,19 +63,20 @@ module.exports = (params, user) => new Promise(async (res, rej) => { return rej('folder-not-found'); } - file.folder_id = folder._id; + updateQuery.folder_id = folder._id; } } - DriveFile.update(file._id, { - $set: { - name: file.name, - folder_id: file.folder_id - } + const updated = await DriveFile.update(file._id, { + $set: { metadata: updateQuery } }); + console.dir(updated) + // Serialize - const fileObj = await serialize(file); + const fileObj = await serialize(updated); + + console.dir(fileObj) // Response res(fileObj); diff --git a/src/api/endpoints/messaging/messages/create.ts b/src/api/endpoints/messaging/messages/create.ts index 8af55d850c..1d186268fb 100644 --- a/src/api/endpoints/messaging/messages/create.ts +++ b/src/api/endpoints/messaging/messages/create.ts @@ -54,9 +54,9 @@ module.exports = (params, user) => new Promise(async (res, rej) => { if (fileId !== undefined) { file = await DriveFile.findOne({ _id: fileId, - user_id: user._id - }, { - data: false + metadata: { + user_id: user._id + } }); if (file === null) { diff --git a/src/api/endpoints/posts/create.ts b/src/api/endpoints/posts/create.ts index f982b9ee93..1507639776 100644 --- a/src/api/endpoints/posts/create.ts +++ b/src/api/endpoints/posts/create.ts @@ -44,9 +44,9 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => { // SELECT _id const entity = await DriveFile.findOne({ _id: mediaId, - user_id: user._id - }, { - _id: true + metadata: { + user_id: user._id + } }); if (entity === null) { diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts index aa5aff5ba5..496de62b69 100644 --- a/src/api/endpoints/posts/timeline.ts +++ b/src/api/endpoints/posts/timeline.ts @@ -2,6 +2,7 @@ * Module dependencies */ import $ from 'cafy'; +import rap from '@prezzemolo/rap'; import Post from '../../models/post'; import ChannelWatching from '../../models/channel-watching'; import getFriends from '../../common/get-friends'; @@ -33,14 +34,15 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { return rej('cannot set since_id and max_id'); } - // ID list of the user itself and other users who the user follows - const followingIds = await getFriends(user._id); - - // Watchしているチャンネルを取得 - const watches = await ChannelWatching.find({ - user_id: user._id, - // 削除されたドキュメントは除く - deleted_at: { $exists: false } + const { followingIds, watchChannelIds } = await rap({ + // ID list of the user itself and other users who the user follows + followingIds: getFriends(user._id), + // Watchしているチャンネルを取得 + watchChannelIds: ChannelWatching.find({ + user_id: user._id, + // 削除されたドキュメントは除く + deleted_at: { $exists: false } + }).then(watches => watches.map(w => w.channel_id)) }); //#region Construct query @@ -65,7 +67,7 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { }, { // Watchしているチャンネルへの投稿 channel_id: { - $in: watches.map(w => w.channel_id) + $in: watchChannelIds } }] } as any; @@ -90,7 +92,5 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { }); // Serialize - res(await Promise.all(timeline.map(async post => - await serialize(post, user) - ))); + res(Promise.all(timeline.map(post => serialize(post, user)))); }); diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts index b4e2ab064a..4c750f4c6b 100644 --- a/src/api/serializers/drive-file.ts +++ b/src/api/serializers/drive-file.ts @@ -31,44 +31,37 @@ export default ( if (mongo.ObjectID.prototype.isPrototypeOf(file)) { _file = await DriveFile.findOne({ _id: file - }, { - fields: { - data: false - } - }); + }); } else if (typeof file === 'string') { _file = await DriveFile.findOne({ _id: new mongo.ObjectID(file) - }, { - fields: { - data: false - } - }); + }); } else { _file = deepcopy(file); } - // Rename _id to id - _file.id = _file._id; - delete _file._id; + // rendered target + let _target: any = {}; - delete _file.data; + _target.id = _file._id; - _file.url = `${config.drive_url}/${_file.id}/${encodeURIComponent(_file.name)}`; + _target = Object.assign(_target, _file.metadata); - if (opts.detail && _file.folder_id) { + _target.url = `${config.drive_url}/${_target.id}/${encodeURIComponent(_target.name)}`; + + if (opts.detail && _target.folder_id) { // Populate folder - _file.folder = await serializeDriveFolder(_file.folder_id, { + _target.folder = await serializeDriveFolder(_target.folder_id, { detail: true }); } - if (opts.detail && _file.tags) { + if (opts.detail && _target.tags) { // Populate tags - _file.tags = await _file.tags.map(async (tag: any) => + _target.tags = await _target.tags.map(async (tag: any) => await serializeDriveTag(tag) ); } - resolve(_file); + resolve(_target); }); diff --git a/src/api/serializers/drive-folder.ts b/src/api/serializers/drive-folder.ts index a428464108..3b5f61aeed 100644 --- a/src/api/serializers/drive-folder.ts +++ b/src/api/serializers/drive-folder.ts @@ -44,7 +44,9 @@ const self = ( }); const childFilesCount = await DriveFile.count({ - folder_id: _folder.id + metadata: { + folder_id: _folder.id + } }); _folder.folders_count = childFoldersCount; From d0dab265f40a37cd715b7d4b64a364c78a7a35b9 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:27:16 +0900 Subject: [PATCH 43/61] serializers - drive-file: add created_at field by uploadedDate --- src/api/serializers/drive-file.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts index 4c750f4c6b..f98cdaa599 100644 --- a/src/api/serializers/drive-file.ts +++ b/src/api/serializers/drive-file.ts @@ -44,6 +44,7 @@ export default ( let _target: any = {}; _target.id = _file._id; + _target.created_at = _file.uploadDate _target = Object.assign(_target, _file.metadata); From a5160a1bbaa3dd75d7ef45b305a90020317e95a8 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:35:20 +0900 Subject: [PATCH 44/61] fileserver - support DriveFile w/ GridFS --- src/file/server.ts | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/file/server.ts b/src/file/server.ts index ee67cf7860..bd29e13c5c 100644 --- a/src/file/server.ts +++ b/src/file/server.ts @@ -9,7 +9,7 @@ import * as cors from 'cors'; import * as mongodb from 'mongodb'; import * as gm from 'gm'; -import File from '../api/models/drive-file'; +import DriveFile, { getGridFSBucket } from '../api/models/drive-file'; /** * Init app @@ -97,17 +97,28 @@ app.get('/:id', async (req, res) => { return; } - const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) }); + const fileId = new mongodb.ObjectID(req.params.id) + const file = await DriveFile.findOne({ _id: fileId }); if (file == null) { res.status(404).sendFile(`${__dirname} / assets / dummy.png`); return; - } else if (file.data == null) { - res.sendStatus(400); - return; } - send(file.data.buffer, file.type, req, res); + const bucket = await getGridFSBucket() + + const buffer = await ((id): Promise => new Promise((resolve, reject) => { + const chunks = [] + const readableStream = bucket.openDownloadStream(id) + readableStream.on('data', chunk => { + chunks.push(chunk); + }) + readableStream.on('end', () => { + resolve(Buffer.concat(chunks)) + }) + }))(fileId) + + send(buffer, file.metadata.type, req, res); }); app.get('/:id/:name', async (req, res) => { From 2ce3179d5000501391b020dd98385aab9fed8094 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:37:04 +0900 Subject: [PATCH 45/61] fileserver - fix dummy path --- src/file/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/file/server.ts b/src/file/server.ts index bd29e13c5c..068e88546b 100644 --- a/src/file/server.ts +++ b/src/file/server.ts @@ -101,7 +101,7 @@ app.get('/:id', async (req, res) => { const file = await DriveFile.findOne({ _id: fileId }); if (file == null) { - res.status(404).sendFile(`${__dirname} / assets / dummy.png`); + res.status(404).sendFile(`${__dirname}/assets/dummy.png`); return; } From 28a39bccf96549a35ef77c10dce5f90f9f8cc654 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:39:16 +0900 Subject: [PATCH 46/61] file-server - support new DriveFile w/ GridFS on '/:id/:name' --- src/file/server.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/file/server.ts b/src/file/server.ts index 068e88546b..f38599b89c 100644 --- a/src/file/server.ts +++ b/src/file/server.ts @@ -128,17 +128,28 @@ app.get('/:id/:name', async (req, res) => { return; } - const file = await File.findOne({ _id: new mongodb.ObjectID(req.params.id) }); + const fileId = new mongodb.ObjectID(req.params.id) + const file = await DriveFile.findOne({ _id: fileId }); if (file == null) { res.status(404).sendFile(`${__dirname}/assets/dummy.png`); return; - } else if (file.data == null) { - res.sendStatus(400); - return; } - send(file.data.buffer, file.type, req, res); + const bucket = await getGridFSBucket() + + const buffer = await ((id): Promise => new Promise((resolve, reject) => { + const chunks = [] + const readableStream = bucket.openDownloadStream(id) + readableStream.on('data', chunk => { + chunks.push(chunk); + }) + readableStream.on('end', () => { + resolve(Buffer.concat(chunks)) + }) + }))(fileId) + + send(buffer, file.metadata.type, req, res); }); module.exports = app; From 0ee6d6592113c5b2df071f4451cb1c1697b59d61 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:45:21 +0900 Subject: [PATCH 47/61] fix timeline --- src/api/endpoints/posts/timeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts index 496de62b69..19578e59b1 100644 --- a/src/api/endpoints/posts/timeline.ts +++ b/src/api/endpoints/posts/timeline.ts @@ -92,5 +92,5 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { }); // Serialize - res(Promise.all(timeline.map(post => serialize(post, user)))); + res(await Promise.all(timeline.map(post => serialize(post, user)))); }); From 7553c6dd38c6f8574894a009238d946d50c53477 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:52:09 +0900 Subject: [PATCH 48/61] serializers - posts: no need Promise wrapping --- src/api/serializers/post.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index e1ab784359..d1dcb66002 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -22,13 +22,13 @@ import rap from '@prezzemolo/rap'; * @param options? serialize options * @return response */ -const self = ( +const self = async ( post: string | mongo.ObjectID | IPost, me?: string | mongo.ObjectID | IUser, options?: { detail: boolean } -) => new Promise(async (resolve, reject) => { +) => { const opts = options || { detail: true, }; @@ -184,7 +184,7 @@ const self = ( // resolve promises in _post object _post = await rap(_post); - resolve(_post); -}); + return _post; +}; export default self; From 7b1fc2c5d62e229542e9411a29e078236a9d96db Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:55:47 +0900 Subject: [PATCH 49/61] api - endpoint:timeline: unneed promise wrapping --- src/api/endpoints/posts/timeline.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts index 19578e59b1..978825a109 100644 --- a/src/api/endpoints/posts/timeline.ts +++ b/src/api/endpoints/posts/timeline.ts @@ -16,22 +16,22 @@ import serialize from '../../serializers/post'; * @param {any} app * @return {Promise} */ -module.exports = (params, user, app) => new Promise(async (res, rej) => { +module.exports = async (params, user, app) => { // Get 'limit' parameter const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$; - if (limitErr) return rej('invalid limit param'); + if (limitErr) throw 'invalid limit param'; // Get 'since_id' parameter const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$; - if (sinceIdErr) return rej('invalid since_id param'); + if (sinceIdErr) throw 'invalid since_id param'; // Get 'max_id' parameter const [maxId, maxIdErr] = $(params.max_id).optional.id().$; - if (maxIdErr) return rej('invalid max_id param'); + if (maxIdErr) throw 'invalid max_id param'; // Check if both of since_id and max_id is specified if (sinceId && maxId) { - return rej('cannot set since_id and max_id'); + throw 'cannot set since_id and max_id'; } const { followingIds, watchChannelIds } = await rap({ @@ -92,5 +92,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { }); // Serialize - res(await Promise.all(timeline.map(post => serialize(post, user)))); -}); + const _timeline = await Promise.all(timeline.map(post => serialize(post, user))) + return _timeline +}; From b50813649afed671b75189551342b179d8cd60f7 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 15:58:39 +0900 Subject: [PATCH 50/61] serializers - posts: fix awaiting --- src/api/serializers/post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index d1dcb66002..5788b226f4 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -94,7 +94,7 @@ const self = async ( if (opts.detail) { // Get previous post info _post.prev = (async () => { - const prev = Post.findOne({ + const prev = await Post.findOne({ user_id: _post.user_id, _id: { $lt: id From 5279d062df205514f1f3cf95e3aab4fee425a3e4 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:09:51 +0900 Subject: [PATCH 51/61] fix --- src/api/endpoints/drive/files.ts | 18 +++++++++--------- src/api/endpoints/drive/files/show.ts | 14 ++++++++------ src/api/endpoints/drive/folders/find.ts | 3 +-- src/api/serializers/drive-file.ts | 2 ++ 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts index eb0bfe6ba5..41687c4993 100644 --- a/src/api/endpoints/drive/files.ts +++ b/src/api/endpoints/drive/files.ts @@ -13,27 +13,27 @@ import serialize from '../../serializers/drive-file'; * @param {any} app * @return {Promise} */ -module.exports = (params, user, app) => new Promise(async (res, rej) => { +module.exports = async (params, user, app) => { // Get 'limit' parameter const [limit = 10, limitErr] = $(params.limit).optional.number().range(1, 100).$; - if (limitErr) return rej('invalid limit param'); + if (limitErr) throw 'invalid limit param'; // Get 'since_id' parameter const [sinceId, sinceIdErr] = $(params.since_id).optional.id().$; - if (sinceIdErr) return rej('invalid since_id param'); + if (sinceIdErr) throw 'invalid since_id param'; // Get 'max_id' parameter const [maxId, maxIdErr] = $(params.max_id).optional.id().$; - if (maxIdErr) return rej('invalid max_id param'); + if (maxIdErr) throw 'invalid max_id param'; // Check if both of since_id and max_id is specified if (sinceId && maxId) { - return rej('cannot set since_id and max_id'); + throw 'cannot set since_id and max_id'; } // Get 'folder_id' parameter const [folderId = null, folderIdErr] = $(params.folder_id).optional.nullable.id().$; - if (folderIdErr) return rej('invalid folder_id param'); + if (folderIdErr) throw 'invalid folder_id param'; // Construct query const sort = { @@ -64,6 +64,6 @@ module.exports = (params, user, app) => new Promise(async (res, rej) => { }); // Serialize - res(await Promise.all(files.map(async file => - await serialize(file)))); -}); + const _files = await Promise.all(files.map(file => serialize(file))); + return _files +}; diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts index 9135a04c57..8830346008 100644 --- a/src/api/endpoints/drive/files/show.ts +++ b/src/api/endpoints/drive/files/show.ts @@ -12,10 +12,10 @@ import serialize from '../../../serializers/drive-file'; * @param {any} user * @return {Promise} */ -module.exports = (params, user) => new Promise(async (res, rej) => { +module.exports = async (params, user) => { // Get 'file_id' parameter const [fileId, fileIdErr] = $(params.file_id).id().$; - if (fileIdErr) return rej('invalid file_id param'); + if (fileIdErr) throw 'invalid file_id param'; // Fetch file const file = await DriveFile @@ -27,11 +27,13 @@ module.exports = (params, user) => new Promise(async (res, rej) => { }); if (file === null) { - return rej('file-not-found'); + throw 'file-not-found'; } // Serialize - res(await serialize(file, { + const _file = await serialize(file, { detail: true - })); -}); + }); + + return _file +}; diff --git a/src/api/endpoints/drive/folders/find.ts b/src/api/endpoints/drive/folders/find.ts index cdf055839a..a5eb8e015d 100644 --- a/src/api/endpoints/drive/folders/find.ts +++ b/src/api/endpoints/drive/folders/find.ts @@ -30,6 +30,5 @@ module.exports = (params, user) => new Promise(async (res, rej) => { }); // Serialize - res(await Promise.all(folders.map(async folder => - await serialize(folder)))); + res(await Promise.all(folders.map(folder => serialize(folder)))); }); diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts index f98cdaa599..9858c3b3c7 100644 --- a/src/api/serializers/drive-file.ts +++ b/src/api/serializers/drive-file.ts @@ -25,6 +25,8 @@ export default ( detail: false }, options); + if (!file) return reject('invalid file arg.') + let _file: any; // Populate the file if 'file' is ID From b266ed3e4f98ab16d95e52cff517d6519b78742a Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:11:24 +0900 Subject: [PATCH 52/61] fix --- src/api/serializers/drive-file.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts index 9858c3b3c7..e749f80387 100644 --- a/src/api/serializers/drive-file.ts +++ b/src/api/serializers/drive-file.ts @@ -25,8 +25,6 @@ export default ( detail: false }, options); - if (!file) return reject('invalid file arg.') - let _file: any; // Populate the file if 'file' is ID @@ -42,6 +40,8 @@ export default ( _file = deepcopy(file); } + if (!_file) return reject('invalid file arg.') + // rendered target let _target: any = {}; From 64be0d6deddef4b8caced377dc22f94425cc4358 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:22:18 +0900 Subject: [PATCH 53/61] =?UTF-8?q?MongoDB=E3=81=AE=E9=9A=8E=E5=B1=A4?= =?UTF-8?q?=E6=A7=8B=E9=80=A0=E6=A4=9C=E7=B4=A2=E3=81=AB=E9=96=A2=E3=81=99?= =?UTF-8?q?=E3=82=8B=E6=80=9D=E3=81=84=E9=81=95=E3=81=84=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/endpoints/drive.ts | 2 +- src/api/endpoints/drive/files.ts | 6 ++---- src/api/endpoints/drive/files/find.ts | 8 +++----- src/api/endpoints/drive/files/show.ts | 4 +--- src/api/endpoints/drive/files/update.ts | 19 +++++-------------- .../endpoints/messaging/messages/create.ts | 4 +--- src/api/endpoints/posts/create.ts | 4 +--- src/api/serializers/drive-folder.ts | 4 +--- 8 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/api/endpoints/drive.ts b/src/api/endpoints/drive.ts index b9c4e3e506..d92473633a 100644 --- a/src/api/endpoints/drive.ts +++ b/src/api/endpoints/drive.ts @@ -14,7 +14,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Calculate drive usage const usage = ((await DriveFile .aggregate([ - { $match: { metadata: { user_id: user._id } } }, + { $match: { 'metadata.user_id': user._id } }, { $project: { length: true diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts index 41687c4993..035916b309 100644 --- a/src/api/endpoints/drive/files.ts +++ b/src/api/endpoints/drive/files.ts @@ -40,10 +40,8 @@ module.exports = async (params, user, app) => { _id: -1 }; const query = { - metadata: { - user_id: user._id, - folder_id: folderId - } + 'metadata.user_id': user._id, + 'metadata.folder_id': folderId } as any; if (sinceId) { sort._id = 1; diff --git a/src/api/endpoints/drive/files/find.ts b/src/api/endpoints/drive/files/find.ts index 255faf94ec..1c818131d7 100644 --- a/src/api/endpoints/drive/files/find.ts +++ b/src/api/endpoints/drive/files/find.ts @@ -24,11 +24,9 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Issue query const files = await DriveFile .find({ - metadata: { - name: name, - user_id: user._id, - folder_id: folderId - } + 'metadata.name': name, + 'metadata.user_id': user._id, + 'metadata.folder_id': folderId }); // Serialize diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts index 8830346008..0a19b19939 100644 --- a/src/api/endpoints/drive/files/show.ts +++ b/src/api/endpoints/drive/files/show.ts @@ -21,9 +21,7 @@ module.exports = async (params, user) => { const file = await DriveFile .findOne({ _id: fileId, - metadata: { - user_id: user._id - } + 'metadata.user_id': user._id }); if (file === null) { diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts index c4d2673688..7a6d2562fb 100644 --- a/src/api/endpoints/drive/files/update.ts +++ b/src/api/endpoints/drive/files/update.ts @@ -20,19 +20,14 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const [fileId, fileIdErr] = $(params.file_id).id().$; if (fileIdErr) return rej('invalid file_id param'); - console.dir(user) // Fetch file const file = await DriveFile .findOne({ _id: fileId, - metadata: { - user_id: user._id - } + 'metadata.user_id': user._id }); - console.dir(file) - if (file === null) { return rej('file-not-found'); } @@ -42,7 +37,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { // Get 'name' parameter const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$; if (nameErr) return rej('invalid name param'); - if (name) updateQuery.name = name; + if (name) updateQuery['metadata.name'] = name; // Get 'folder_id' parameter const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$; @@ -50,7 +45,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { if (folderId !== undefined) { if (folderId === null) { - updateQuery.folder_id = null; + updateQuery['metadata.folder_id'] = null; } else { // Fetch folder const folder = await DriveFolder @@ -63,21 +58,17 @@ module.exports = (params, user) => new Promise(async (res, rej) => { return rej('folder-not-found'); } - updateQuery.folder_id = folder._id; + updateQuery['metadata.folder_id'] = folder._id; } } const updated = await DriveFile.update(file._id, { - $set: { metadata: updateQuery } + $set: { updateQuery } }); - console.dir(updated) - // Serialize const fileObj = await serialize(updated); - console.dir(fileObj) - // Response res(fileObj); diff --git a/src/api/endpoints/messaging/messages/create.ts b/src/api/endpoints/messaging/messages/create.ts index 1d186268fb..149852c093 100644 --- a/src/api/endpoints/messaging/messages/create.ts +++ b/src/api/endpoints/messaging/messages/create.ts @@ -54,9 +54,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { if (fileId !== undefined) { file = await DriveFile.findOne({ _id: fileId, - metadata: { - user_id: user._id - } + 'metadata.user_id': user._id }); if (file === null) { diff --git a/src/api/endpoints/posts/create.ts b/src/api/endpoints/posts/create.ts index 1507639776..4f4b7e2e83 100644 --- a/src/api/endpoints/posts/create.ts +++ b/src/api/endpoints/posts/create.ts @@ -44,9 +44,7 @@ module.exports = (params, user: IUser, app) => new Promise(async (res, rej) => { // SELECT _id const entity = await DriveFile.findOne({ _id: mediaId, - metadata: { - user_id: user._id - } + 'metadata.user_id': user._id }); if (entity === null) { diff --git a/src/api/serializers/drive-folder.ts b/src/api/serializers/drive-folder.ts index 3b5f61aeed..6ebf454a28 100644 --- a/src/api/serializers/drive-folder.ts +++ b/src/api/serializers/drive-folder.ts @@ -44,9 +44,7 @@ const self = ( }); const childFilesCount = await DriveFile.count({ - metadata: { - folder_id: _folder.id - } + 'metadata.folder_id': _folder.id }); _folder.folders_count = childFoldersCount; From 4c5a4d259738ba617bf29d2158d180cc5fa8401c Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:26:17 +0900 Subject: [PATCH 54/61] core - fix metadata searching --- src/api/common/add-file-to-drive.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 376c470e93..1f882389ac 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -81,9 +81,7 @@ export default ( // Check if there is a file with the same hash const much = await DriveFile.findOne({ md5: hash, - metadata: { - user_id: user._id - } + 'metadata.user_id': user._id }); if (much !== null) { @@ -97,7 +95,7 @@ export default ( // Calculate drive usage const usage = ((await DriveFile .aggregate([ - { $match: { metadata: { user_id: user._id } } }, + { $match: { 'metadata.user_id': user._id } }, { $project: { length: true }}, From 04648db1c235b0de14d3e0a2dc83f9346d0408f8 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:29:13 +0900 Subject: [PATCH 55/61] remove console --- src/api/common/add-file-to-drive.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index 1f882389ac..dff2d52356 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -152,7 +152,6 @@ export default ( comment: comment, properties: properties }); - console.dir(file) log(`drive file has been created ${file._id}`); From d5cc4cc9c28eb6a981ce37859def97cd7c57abc6 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:32:01 +0900 Subject: [PATCH 56/61] fix lint (automattic) --- src/api/common/add-file-to-drive.ts | 18 ++++++------- src/api/endpoints/drive/files.ts | 2 +- src/api/endpoints/drive/files/show.ts | 2 +- src/api/endpoints/drive/files/update.ts | 3 +-- src/api/endpoints/posts/timeline.ts | 4 +-- src/api/models/drive-file.ts | 10 +++---- src/api/serializers/drive-file.ts | 4 +-- src/db/mongodb.ts | 18 ++++++------- src/file/server.ts | 36 ++++++++++++------------- 9 files changed, 48 insertions(+), 49 deletions(-) diff --git a/src/api/common/add-file-to-drive.ts b/src/api/common/add-file-to-drive.ts index dff2d52356..f9c22ccacd 100644 --- a/src/api/common/add-file-to-drive.ts +++ b/src/api/common/add-file-to-drive.ts @@ -14,16 +14,16 @@ import { Duplex } from 'stream'; const log = debug('misskey:register-drive-file'); const addToGridFS = (name, binary, metadata): Promise => new Promise(async (resolve, reject) => { - const dataStream = new Duplex() - dataStream.push(binary) - dataStream.push(null) + const dataStream = new Duplex(); + dataStream.push(binary); + dataStream.push(null); - const bucket = await getGridFSBucket() - const writeStream = bucket.openUploadStream(name, { metadata }) - writeStream.once('finish', (doc) => { resolve(doc) }) - writeStream.on('error', reject) - dataStream.pipe(writeStream) -}) + const bucket = await getGridFSBucket(); + const writeStream = bucket.openUploadStream(name, { metadata }); + writeStream.once('finish', (doc) => { resolve(doc); }); + writeStream.on('error', reject); + dataStream.pipe(writeStream); +}); /** * Add file to drive diff --git a/src/api/endpoints/drive/files.ts b/src/api/endpoints/drive/files.ts index 035916b309..53b48a8bec 100644 --- a/src/api/endpoints/drive/files.ts +++ b/src/api/endpoints/drive/files.ts @@ -63,5 +63,5 @@ module.exports = async (params, user, app) => { // Serialize const _files = await Promise.all(files.map(file => serialize(file))); - return _files + return _files; }; diff --git a/src/api/endpoints/drive/files/show.ts b/src/api/endpoints/drive/files/show.ts index 0a19b19939..3c7cf774f9 100644 --- a/src/api/endpoints/drive/files/show.ts +++ b/src/api/endpoints/drive/files/show.ts @@ -33,5 +33,5 @@ module.exports = async (params, user) => { detail: true }); - return _file + return _file; }; diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts index 7a6d2562fb..4e56b30ace 100644 --- a/src/api/endpoints/drive/files/update.ts +++ b/src/api/endpoints/drive/files/update.ts @@ -20,7 +20,6 @@ module.exports = (params, user) => new Promise(async (res, rej) => { const [fileId, fileIdErr] = $(params.file_id).id().$; if (fileIdErr) return rej('invalid file_id param'); - // Fetch file const file = await DriveFile .findOne({ @@ -32,7 +31,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { return rej('file-not-found'); } - const updateQuery: any = {} + const updateQuery: any = {}; // Get 'name' parameter const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$; diff --git a/src/api/endpoints/posts/timeline.ts b/src/api/endpoints/posts/timeline.ts index 978825a109..203413e23a 100644 --- a/src/api/endpoints/posts/timeline.ts +++ b/src/api/endpoints/posts/timeline.ts @@ -92,6 +92,6 @@ module.exports = async (params, user, app) => { }); // Serialize - const _timeline = await Promise.all(timeline.map(post => serialize(post, user))) - return _timeline + const _timeline = await Promise.all(timeline.map(post => serialize(post, user))); + return _timeline; }; diff --git a/src/api/models/drive-file.ts b/src/api/models/drive-file.ts index 79a87f6572..8968d065cd 100644 --- a/src/api/models/drive-file.ts +++ b/src/api/models/drive-file.ts @@ -8,14 +8,14 @@ const collection = monkDb.get('drive_files.files'); export default collection as any; // fuck type definition const getGridFSBucket = async (): Promise => { - const db = await nativeDbConn() + const db = await nativeDbConn(); const bucket = new mongodb.GridFSBucket(db, { bucketName: 'drive_files' - }) - return bucket -} + }); + return bucket; +}; -export { getGridFSBucket } +export { getGridFSBucket }; export function validateFileName(name: string): boolean { return ( diff --git a/src/api/serializers/drive-file.ts b/src/api/serializers/drive-file.ts index e749f80387..2af7db5726 100644 --- a/src/api/serializers/drive-file.ts +++ b/src/api/serializers/drive-file.ts @@ -40,13 +40,13 @@ export default ( _file = deepcopy(file); } - if (!_file) return reject('invalid file arg.') + if (!_file) return reject('invalid file arg.'); // rendered target let _target: any = {}; _target.id = _file._id; - _target.created_at = _file.uploadDate + _target.created_at = _file.uploadDate; _target = Object.assign(_target, _file.metadata); diff --git a/src/db/mongodb.ts b/src/db/mongodb.ts index 75f1a1d3c6..c978e6460f 100644 --- a/src/db/mongodb.ts +++ b/src/db/mongodb.ts @@ -16,7 +16,7 @@ export default db; /** * MongoDB native module (officialy) */ -import * as mongodb from 'mongodb' +import * as mongodb from 'mongodb'; let mdb: mongodb.Db; @@ -25,14 +25,14 @@ const nativeDbConn = async (): Promise => { const db = await ((): Promise => new Promise((resolve, reject) => { mongodb.MongoClient.connect(uri, (e, db) => { - if (e) return reject(e) - resolve(db) - }) - }))() + if (e) return reject(e); + resolve(db); + }); + }))(); - mdb = db + mdb = db; - return db -} + return db; +}; -export { nativeDbConn } +export { nativeDbConn }; diff --git a/src/file/server.ts b/src/file/server.ts index f38599b89c..375f29487d 100644 --- a/src/file/server.ts +++ b/src/file/server.ts @@ -97,7 +97,7 @@ app.get('/:id', async (req, res) => { return; } - const fileId = new mongodb.ObjectID(req.params.id) + const fileId = new mongodb.ObjectID(req.params.id); const file = await DriveFile.findOne({ _id: fileId }); if (file == null) { @@ -105,18 +105,18 @@ app.get('/:id', async (req, res) => { return; } - const bucket = await getGridFSBucket() + const bucket = await getGridFSBucket(); const buffer = await ((id): Promise => new Promise((resolve, reject) => { - const chunks = [] - const readableStream = bucket.openDownloadStream(id) - readableStream.on('data', chunk => { + const chunks = []; + const readableStream = bucket.openDownloadStream(id); + readableStream.on('data', chunk => { chunks.push(chunk); - }) + }); readableStream.on('end', () => { - resolve(Buffer.concat(chunks)) - }) - }))(fileId) + resolve(Buffer.concat(chunks)); + }); + }))(fileId); send(buffer, file.metadata.type, req, res); }); @@ -128,7 +128,7 @@ app.get('/:id/:name', async (req, res) => { return; } - const fileId = new mongodb.ObjectID(req.params.id) + const fileId = new mongodb.ObjectID(req.params.id); const file = await DriveFile.findOne({ _id: fileId }); if (file == null) { @@ -136,18 +136,18 @@ app.get('/:id/:name', async (req, res) => { return; } - const bucket = await getGridFSBucket() + const bucket = await getGridFSBucket(); const buffer = await ((id): Promise => new Promise((resolve, reject) => { - const chunks = [] - const readableStream = bucket.openDownloadStream(id) - readableStream.on('data', chunk => { + const chunks = []; + const readableStream = bucket.openDownloadStream(id); + readableStream.on('data', chunk => { chunks.push(chunk); - }) + }); readableStream.on('end', () => { - resolve(Buffer.concat(chunks)) - }) - }))(fileId) + resolve(Buffer.concat(chunks)); + }); + }))(fileId); send(buffer, file.metadata.type, req, res); }); From 3be69a8cb7bacca181fa400f234fd77c1d1d5bde Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:49:07 +0900 Subject: [PATCH 57/61] /drive/files/update - return collectly value --- src/api/endpoints/drive/files/update.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/api/endpoints/drive/files/update.ts b/src/api/endpoints/drive/files/update.ts index 4e56b30ace..d7b858c2ba 100644 --- a/src/api/endpoints/drive/files/update.ts +++ b/src/api/endpoints/drive/files/update.ts @@ -31,12 +31,10 @@ module.exports = (params, user) => new Promise(async (res, rej) => { return rej('file-not-found'); } - const updateQuery: any = {}; - // Get 'name' parameter const [name, nameErr] = $(params.name).optional.string().pipe(validateFileName).$; if (nameErr) return rej('invalid name param'); - if (name) updateQuery['metadata.name'] = name; + if (name) file.metadata.name = name; // Get 'folder_id' parameter const [folderId, folderIdErr] = $(params.folder_id).optional.nullable.id().$; @@ -44,7 +42,7 @@ module.exports = (params, user) => new Promise(async (res, rej) => { if (folderId !== undefined) { if (folderId === null) { - updateQuery['metadata.folder_id'] = null; + file.metadata.folder_id = null; } else { // Fetch folder const folder = await DriveFolder @@ -57,16 +55,19 @@ module.exports = (params, user) => new Promise(async (res, rej) => { return rej('folder-not-found'); } - updateQuery['metadata.folder_id'] = folder._id; + file.metadata.folder_id = folder._id; } } - const updated = await DriveFile.update(file._id, { - $set: { updateQuery } + await DriveFile.update(file._id, { + $set: { + 'metadata.name': file.metadata.name, + 'metadata.folder_id': file.metadata.folder_id + } }); // Serialize - const fileObj = await serialize(updated); + const fileObj = await serialize(file); // Response res(fileObj); From 73bb81de8f17fe603dfde57ae70aa61669161bfc Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 16:59:09 +0900 Subject: [PATCH 58/61] update test for GridFS --- test/api.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/api.js b/test/api.js index b43eb7ff62..c0da9d6c5b 100644 --- a/test/api.js +++ b/test/api.js @@ -1152,9 +1152,12 @@ async function insertHimawari(opts) { } async function insertDriveFile(opts) { - return await db.get('drive_files').insert(Object.assign({ - name: 'strawberry-pasta.png' - }, opts)); + return await db.get('drive_files.files').insert({ + length: opts.datasize, + metadata: Object.assign({ + name: 'strawberry-pasta.png' + }, opts) + }); } async function insertDriveFolder(opts) { From 26602dcd209198dead66081f54b1800627e0bff8 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 17:57:03 +0900 Subject: [PATCH 59/61] migration - add GridFS migration --- tools/migration/use-gridfs.js | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tools/migration/use-gridfs.js diff --git a/tools/migration/use-gridfs.js b/tools/migration/use-gridfs.js new file mode 100644 index 0000000000..d41514416c --- /dev/null +++ b/tools/migration/use-gridfs.js @@ -0,0 +1,49 @@ +// for Node.js interpret + +const { default: db } = require('../../built/db/mongodb') +const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file') +const { Duplex } = require('stream') + +const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => { + const writeStream = bucket.openUploadStreamWithId(...rest) + + const dataStream = new Duplex() + dataStream.push(buffer) + dataStream.push(null) + + writeStream.once('finish', resolve) + writeStream.on('error', reject) + + dataStream.pipe(writeStream) +}) + +const migrateToGridFS = async (doc) => { + const id = doc._id + const buffer = doc.data.buffer + const created_at = doc.created_at + + delete doc._id + delete doc.created_at + delete doc.datasize + delete doc.hash + delete doc.data + + const bucket = await getGridFSBucket() + const added = await writeToGridFS(bucket, buffer, id, `${id}/${doc.name}`, { metadata: doc }) + + const result = await DriveFile.update(id, { + $set: { + uploadDate: created_at + } + }) + + return added && result.ok === 1 +} + +const main = async () => { + const docs = await db.get('drive_files').find() + const all = await Promise.all(docs.map(migrateToGridFS)) + return all +} + +main().then(console.dir).catch(console.error) From c1fc3b9f6ec176999932958a7856d160317b7762 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 18:30:49 +0900 Subject: [PATCH 60/61] add safety guard to serializers & fix importing uncorrect serializer --- src/api/endpoints/drive/folders/update.ts | 2 +- src/api/serializers/post.ts | 2 ++ src/api/serializers/user.ts | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/endpoints/drive/folders/update.ts b/src/api/endpoints/drive/folders/update.ts index eec2757878..4f2e3d2a7a 100644 --- a/src/api/endpoints/drive/folders/update.ts +++ b/src/api/endpoints/drive/folders/update.ts @@ -4,7 +4,7 @@ import $ from 'cafy'; import DriveFolder from '../../../models/drive-folder'; import { isValidFolderName } from '../../../models/drive-folder'; -import serialize from '../../../serializers/drive-file'; +import serialize from '../../../serializers/drive-folder'; import event from '../../../event'; /** diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index 5788b226f4..5a63384f0e 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -57,6 +57,8 @@ const self = async ( _post = deepcopy(post); } + if (!_post) throw 'invalid post arg.'; + const id = _post._id; // Rename _id to id diff --git a/src/api/serializers/user.ts b/src/api/serializers/user.ts index d00f073897..0d24d6cc04 100644 --- a/src/api/serializers/user.ts +++ b/src/api/serializers/user.ts @@ -56,6 +56,8 @@ export default ( _user = deepcopy(user); } + if (!_user) return reject('invalid user arg.'); + // Me const meId: mongo.ObjectID = me ? mongo.ObjectID.prototype.isPrototypeOf(me) From d7e1ffb0055f0786a707015350a14351b8a0fbf0 Mon Sep 17 00:00:00 2001 From: otofune Date: Mon, 6 Nov 2017 18:38:59 +0900 Subject: [PATCH 61/61] remove whitespace --- src/api/serializers/post.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/serializers/post.ts b/src/api/serializers/post.ts index 5a63384f0e..03fd120772 100644 --- a/src/api/serializers/post.ts +++ b/src/api/serializers/post.ts @@ -57,7 +57,7 @@ const self = async ( _post = deepcopy(post); } - if (!_post) throw 'invalid post arg.'; + if (!_post) throw 'invalid post arg.'; const id = _post._id;