feat(storage-service): use storage-service, replace old fs;

.vscode/settings.json

@@ -70,6 +70,7 @@
     "coll",
     "dockerode",
     "EJSON",
+    "entrypoint",
     "lafyun",
     "objs",
     "signin",

docker-compose.yml

@@ -31,10 +31,12 @@ services:
       SYS_SERVER_SECRET_SALT: Rewrite_Your_Own_Secret_Salt_abcdefg1234567
       SHARED_NETWORK: laf_shared_network
       LOG_LEVEL: debug
-      APP_SERVICE_IMAGE: lafyun/app-service:latest
       ACCOUNT_DEFAULT_APP_QUOTA: 5
+      APP_SERVICE_IMAGE: lafyun/app-service:latest
       APP_SERVICE_DEPLOY_HOST: local-dev.host:8080     # `*.local-dev.host` always resolved to 127.0.0.1, used to local development
       APP_SERVICE_DEPLOY_URL_SCHEMA: 'http'
+      STORAGE_SERVICE_API_ENTRYPOINT: 'http://storage_service:9010'
+      STORAGE_SERVICE_SECRET: Rewrite_Your_Own_Secret_Salt_abcdefg1234567
       DEBUG_BIND_HOST_APP_PATH: '${PWD}/packages/app-service'
     command: npx nodemon
     volumes:
@@ -72,28 +74,26 @@ services:
     networks:
       - laf_shared_network
 
-  # storage-service:
-  #   image: node:16-alpine
-  #   user: root
-  #   working_dir: /app
-  #   environment:
-  #     DB_URI : mongodb://root:password123@mongo:27017/laf-fs?authSource=admin&replicaSet=laf&writeConcern=majority
-  #     LOG_LEVEL : debug
-  #   command: npx nodemon
-  #   volumes:
-  #     - ./packages/storage-service:/app
-  #     - ./packages/database-proxy:/app/node_modules/database-proxy:ro
-  #     - ./packages/database-ql:/app/node_modules/database-ql:ro
-  #     - ./packages/database-ql:/app/node_modules/database-proxy/node_modules/database-ql:ro
-  #   ports:
-  #     - "9001"
-  #   depends_on:
-  #     - mongo
-  #   tmpfs:
-  #     - /tmp
-  #   restart: always
-  #   networks:
-  #     - laf_shared_network
+  storage_service:
+    image: node:16-alpine
+    user: root
+    working_dir: /app
+    environment:
+      DB_URI : mongodb://root:password123@mongo:27017/laf-fs?authSource=admin&replicaSet=laf&writeConcern=majority
+      LOG_LEVEL : debug
+      SERVER_SECRET_SALT: Rewrite_Your_Own_Secret_Salt_abcdefg1234567
+    command: npx nodemon
+    volumes:
+      - ./packages/storage-service:/app
+    ports:
+      - "9010"
+    depends_on:
+      - mongo
+    tmpfs:
+      - /tmp
+    restart: always
+    networks:
+      - laf_shared_network
 
 volumes:
   db-data:

packages/gateway/app.conf

@@ -42,6 +42,55 @@ server {
       }
    }
 
+   location /file {
+      # Allow CORS
+      add_header Access-Control-Allow-Origin *;
+      add_header Access-Control-Allow-Methods GET,POST,PUT,DELETE,PATCH,OPTIONS;
+      add_header Access-Control-Allow-Headers *;
+      add_header Access-Control-Expose-Headers *;
+      add_header Access-Control-Max-Age 3600;
+
+      if ($request_method = 'OPTIONS') {
+         return 204;
+      }
+
+      # Resolve app service
+      resolver 127.0.0.11;
+      if ($host ~* "(\w{8}(-\w{4}){3}-\w{12})\.(.+)$") {
+         set $appid $1;
+      }
+
+      if ($uri ~* "/file/([\w|\d]{1,32})/(.*)$") {
+         set $tmp _$1;
+         set $bucket $appid$tmp;
+         set $proxy_uri $bucket?path=/$2&$query_string;
+      }
+
+      # just for `/bucket` (no `/` in ends)
+      if ($uri ~* "/file/([\w|\d]{1,32})$") {
+         set $tmp _$1;
+         set $bucket $appid$tmp;
+         set $proxy_uri $bucket?path=/&$query_string;
+      }
+
+      if ($uri ~* "/file/([\w|\d]{1,32})/dir$") {
+         set $tmp _$1;
+         set $bucket $appid$tmp;
+         set $proxy_uri $bucket/dir$is_args$query_string;
+      }
+
+      proxy_pass http://storage_service:9010/$proxy_uri;
+      add_header appid $appid;
+      add_header bucket $bucket;
+      proxy_read_timeout 600s;
+      proxy_set_header Host $host;
+
+      # Logging
+      log_by_lua_block {
+         ngx.log(ngx.ERR, ngx.var.appid, '<storage service>', ngx.var.bucket)
+      }
+   }
+
    location /deploy/incoming {
       # Allow CORS
       add_header Access-Control-Allow-Origin *;

packages/storage-service/src/api/bucket.ts

@@ -24,7 +24,7 @@ export async function getBucketByName(name: string) {
  */
 export async function createBucket(bucket: BucketType) {
   assert.ok(bucket, 'empty bucket got')
-  assert.ok(bucket.mode, 'empty bucket mode got')
+  assert.ok(bucket.mode !== undefined, 'empty bucket mode got')
   assert.ok(bucket.name, 'empty bucket name got')
 
   const coll = DatabaseAgent.db.collection(Constants.cn.BUCKETS)
@@ -41,9 +41,19 @@ export async function createBucket(bucket: BucketType) {
 export async function deleteBucketByName(name: string) {
   assert.ok(name, 'empty bucket name got')
 
+  const c_files = name + '.files'
+  const c_chunks = name + '.chunks'
+
   const db = DatabaseAgent.db
-  await db.collection(name + '.files').drop()
-  await db.collection(name + '.chunks').drop()
+  const files_coll = await db.listCollections({ name: c_files }).toArray()
+  if (files_coll?.length) {
+    await db.collection(c_files).drop()
+  }
+
+  const chunks_coll = await db.listCollections({ name: c_chunks }).toArray()
+  if (chunks_coll?.length) {
+    await db.collection(c_chunks).drop()
+  }
 
   const coll = db.collection(Constants.cn.BUCKETS)
   const r = await coll.deleteOne({ name })
@@ -59,7 +69,7 @@ export async function deleteBucketByName(name: string) {
  */
 export async function updateBucketByName(name: string, mode: BucketMode, secret: string) {
   assert.ok(name, 'empty name got')
-  assert.ok(mode, 'empty mode got')
+  assert.ok(mode !== undefined, 'empty mode got')
 
   const data = {
     mode,
@@ -78,6 +88,32 @@ export async function updateBucketByName(name: string, mode: BucketMode, secret:
   return r.matchedCount > 0 ? true : false
 }
 
+/**
+ * Create bucket collections and indexes: [bucket].files, [bucket].chunks
+ * @param bucket bucket name
+ */
+export async function createBucketCollections(bucket: string) {
+  assert.ok(bucket, 'empty bucket got')
+
+  const db = DatabaseAgent.db
+  await db.createCollection(bucket + ".files")
+  await db.createCollection(bucket + ".chunks")
+
+  await db.collection(bucket + ".files").createIndexes([
+    {
+      key: { filename: 1, uploadDate: 1 },
+      unique: true
+    },
+    { key: { 'metadata.parent': 1 } }
+  ])
+
+  await db.collection(bucket + ".chunks").createIndexes([
+    {
+      key: { files_id: 1, n: 1 },
+      unique: true
+    }
+  ])
+}
 
 /**
  * Create the root path for a bucket
@@ -102,13 +138,6 @@ export async function createBucketRootPath(bucket: string) {
     },
   }
 
-  await coll.createIndexes([
-    {
-      key: { filename: 1, uploadDate: -1 },
-      unique: true
-    },
-    { key: { 'metadata.parent': 1 } }
-  ])
   const r = await coll.insertOne(doc)
   return r.insertedId
 }

packages/storage-service/src/api/file.ts

@@ -26,12 +26,17 @@ export async function getFileByName(bucket: string, filename: string) {
  * @param parent 
  * @returns 
  */
-export async function getFilesInDirectory(bucket: string, parent: string) {
+export async function getFilesInDirectory(bucket: string, parent: string, offset?: number, limit?: number) {
   assert.ok(bucket, 'empty bucket got')
   assert.ok(parent, 'empty parent got')
-
+  const options = {
+    skip: offset ?? 0
+  }
+  if (limit) {
+    options['limit'] = limit
+  }
   const coll = DatabaseAgent.db.collection<FileItemType>(bucket + ".files")
-  const files = await coll.find({ "metadata.parent": parent }).toArray()
+  const files = await coll.find({ "metadata.parent": parent }, options).toArray()
   return files
 }
 

packages/storage-service/src/config.ts

@@ -22,7 +22,7 @@ export default class Config {
    * the serving port, default is 9001
    */
   static get PORT(): number {
-    return (process.env.PORT ?? 9001) as number
+    return (process.env.PORT ?? 9010) as number
   }
 
   /**

packages/storage-service/src/constants.ts

@@ -7,7 +7,7 @@ export const Constants = {
    */
   cn: {
     /** buckets collection name */
-    BUCKETS: "laf-fs-buckets",
+    BUCKETS: "fs_buckets",
   },
   DIRECTORY_MIME_TYPE: 'application/x-directory'
 }

packages/storage-service/src/index.ts

@@ -12,19 +12,16 @@ server.use(
   })
 )
 
-/**
- * Allow CORS by default
- */
-server.all("*", function (_req, res, next) {
-  res.header("Access-Control-Allow-Origin", "*")
-  res.header("Access-Control-Allow-Headers", "Authorization, Content-Type")
-  res.header("Access-Control-Allow-Methods", "*")
-  res.header("X-Powered-By", "LaF Server")
-  next()
-})
-
 server.use(router)
 
 server.listen(Config.PORT, () =>
   logger.info(`server ${process.pid} listened on ${Config.PORT}`)
 )
+
+process.on('unhandledRejection', (reason, promise) => {
+  logger.error(`Caught unhandledRejection:`, reason, promise)
+})
+
+process.on('uncaughtException', err => {
+  logger.error(`Caught uncaughtException:`, err)
+})

packages/storage-service/src/router/bucket/create.ts

@@ -1,5 +1,5 @@
 import * as express from "express"
-import { createBucket, createBucketRootPath, getBucketByName } from "../../api/bucket"
+import { createBucket, createBucketCollections, createBucketRootPath, getBucketByName } from "../../api/bucket"
 import { logger } from "../../lib/logger"
 import { BucketMode, BucketType } from "../../lib/types"
 
@@ -35,6 +35,9 @@ export async function handleCreateBucket(req: express.Request, res: express.Resp
     // save into database
     const insertedId = await createBucket(data)
 
+    // create bucket collections
+    await createBucketCollections(name)
+
     // created root directory
     await createBucketRootPath(name)
 

packages/storage-service/src/router/bucket/delete.ts

@@ -19,21 +19,21 @@ export async function handleDeleteBucket(req: express.Request, res: express.Resp
     // check if bucket exists
     const bucket = await getBucketByName(name)
     if (!bucket) {
-      return res.status(400).send("bucket doesn't exist")
+      return res.status(200).send({ code: 'NOT_EXISTS', error: "bucket doesn't exist" })
     }
 
     // check if bucket empty
     const fileCount = await countFilesInDirectory(name, '/')
     if (fileCount > 0) {
-      return res.status(200).send({
+      return res.send({
         code: 'BUCKET_NOT_EMPTY',
         error: 'cannot delete none-empty-bucket'
       })
     }
 
     // delete this bucket
     const r = await deleteBucketByName(name)
-    return res.status(200).send({
+    return res.send({
       code: 0,
       data: r
     })

packages/storage-service/src/router/file/get.ts

@@ -1,7 +1,7 @@
 /*
  * @Author: Maslow<[email protected]>
  * @Date: 2021-08-19 16:10:27
- * @LastEditTime: 2021-10-27 18:02:21
+ * @LastEditTime: 2021-11-13 20:36:48
  * @Description:
  */
 
@@ -13,7 +13,7 @@ import { GridFSStorage } from "../../lib/gridfs-storage"
 import { DatabaseAgent } from "../../lib/database"
 import { BucketMode } from "../../lib/types"
 import { getBucketByName } from "../../api/bucket"
-import { getFileByName, getFilesInDirectory } from "../../api/file"
+import { countFilesInDirectory, getFileByName, getFilesInDirectory } from "../../api/file"
 
 /**
  * Downloads file by bucket name and filename
@@ -51,9 +51,13 @@ export async function handleGetFile(req: express.Request, res: express.Response)
     if (code) {
       return res.status(code).send(message)
     }
-    const files = await getFilesInDirectory(bucket_name, filename)
-    res.contentType(file.metadata.contentType)
-    return res.status(200).send({ type: 'directory', data: files })
+
+    const offset = (req.query?.offset ?? 0) as any
+    const limit = req.query?.limit as any
+    const files = await getFilesInDirectory(bucket_name, filename, offset, limit)
+    const total = await countFilesInDirectory(bucket_name, filename)
+    res.type('json')
+    return res.status(200).send({ type: 'directory', data: files, total })
   }
 
   try {