Fixed Minio key and filename mismatch by using file name

server-nest/src/common/IRecordUserMetadata.ts

@@ -1,3 +1,5 @@
+
+// Headers set in the Databrary client to persist values required in tasks
 export interface IRecordUserMetadata {
   'X-Amz-Meta-Asset-Id'?: number
   'X-Amz-Meta-Upload-Type'?: string

server-nest/src/dtos/record.dto.ts

@@ -1,11 +1,13 @@
 import { parse } from 'path'
 import { IRecordUserMetadata } from '../common/IRecordUserMetadata'
 
+// A record is the http request sent by Minio to the Databrary webhook on an upload event
 export class RecordDTO {
   readonly key: string
   readonly eTag: string
   readonly size: number
   readonly contentType: string
+  // Headers set in the client
   private userMetadata: Partial<IRecordUserMetadata>
 
   constructor (record: Partial<RecordDTO>) {

server-nest/src/minio/minio.controller.ts

@@ -43,7 +43,7 @@ export class MinioController {
         // Send signed url
         this.minioService.client.presignedPutObject(
           bucketName,
-          encodeURIComponent(filename),
+          filename, // Do not encode the file name
           1000,
           function (err, presignedUrl) {
             if (err != null) return console.log(err)

server-nest/src/minio/minio.service.ts

@@ -22,7 +22,9 @@ export class MinioService {
     try {
       return await this.client.bucketExists(bucket)
     } catch (error) {
-      console.error(error)
+      // Nothing to do
+      // bucketExists will throw an error if bucket does not exist
+      console.error(error.message)
     }
 
     return false
@@ -33,7 +35,11 @@ export class MinioService {
       const { etag } = await this.client.statObject(bucket, name)
       // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
       return !!etag
-    } catch (error) {}
+    } catch (error) {
+      // Nothing to do
+      // statObject will throw an error if file is not found
+      console.error(error.message)
+    }
 
     return false
   }
@@ -47,9 +53,8 @@ export class MinioService {
 
       return true
     } catch (error) {
-      console.error(error)
+      throw new Error(error.message)
     }
-    return false
   }
 
   async getObject (
@@ -60,7 +65,7 @@ export class MinioService {
     return await new Promise((resolve, reject) => {
       this.client.fGetObject(bucket, name, path, (err) => {
         // eslint-disable-next-line prefer-promise-reject-errors
-        if (err != null) reject(false)
+        if (err != null) return reject(false)
         resolve(true)
       })
     })
@@ -76,7 +81,7 @@ export class MinioService {
       this.client.fPutObject(bucket, name, path, metaData, (err, eTag) => {
         if (err != null) {
           // eslint-disable-next-line prefer-promise-reject-errors
-          reject(false)
+          return reject(false)
         }
         resolve(true)
       })
@@ -93,21 +98,20 @@ export class MinioService {
       const hashSha1 = createHash('sha1') // todo check this
       let size = 0
       // eslint-disable-next-line @typescript-eslint/space-before-function-paren
-      this.client.getObject(bucket, name, function(err, dataStream) {
+      this.client.getObject(bucket, name, (err, dataStream) => {
         if (err != null) {
-          reject(err)
+          return reject(err)
         }
-        console.log('bucket ', bucket)
-        console.log('Datastream ', dataStream)
-        dataStream.on('data', function (chunk) {
+
+        dataStream.on('data', (chunk) => {
           // eslint-disable-next-line @typescript-eslint/restrict-plus-operands
           size += chunk.length
           hashMd5.update(chunk)
           hashSha1.update(chunk)
           hashSha256.update(chunk)
         })
 
-        dataStream.on('end', function () {
+        dataStream.on('end', () => {
           const md5 = hashMd5.digest().toString('hex')
           const sha1 = hashSha1.digest().toString('hex')
           const sha256 = hashSha256.digest().toString('hex')
@@ -122,8 +126,8 @@ export class MinioService {
           )
         })
 
-        dataStream.on('error', function (err) {
-          reject(err)
+        dataStream.on('error', (err) => {
+          return reject(err)
         })
       })
     })

server-nest/src/task/task.processor.ts

@@ -21,7 +21,7 @@ import { ImageKey, Buckets } from '../common/types'
 import { TMP_DIR, AVATAR_SIZES, AVATAR_FORMAT } from '../common/constants'
 import { AssetService } from '../asset/asset.service'
 import { NotFoundException } from '@nestjs/common'
-import { AssetDTO } from 'src/dtos/asset.dto'
+import { AssetDTO } from '../dtos/asset.dto'
 
 type Location = 'MINIO' | 'LOCAL'
 
@@ -61,7 +61,7 @@ export class TaskProcessor {
             'MINIO',
             'uploads',
             'cas',
-            record.key,
+            record.fileName,
             record.size
           )
 
@@ -70,14 +70,14 @@ export class TaskProcessor {
           await this.minioService.copyObject(
             'cas',
             fileObject.sha256,
-            `/uploads/${record.key}`,
+            `/uploads/${record.fileName}`,
             record.eTag
           )
         } catch (error) {
-          console.error(error)
+          console.error(error.message)
 
           // Retry the Job
-          await job.retry()
+          // await job.retry()
         }
 
         break
@@ -95,16 +95,16 @@ export class TaskProcessor {
 
           record.assetId = asset.id
 
-          const originalFile = resolve(TMP_DIR, record.key)
+          const originalFile = resolve(TMP_DIR, record.fileName)
 
-          console.log(`Downloading avatar ${record.key}...`)
+          console.log(`Downloading avatar ${record.fileName}...`)
           const downloaded = await this.minioService.getObject(
             'uploads',
-            record.key,
+            record.fileName,
             originalFile
           )
 
-          if (!downloaded) { throw new NotFoundException(`Avatar ${record.key} download failed`) }
+          if (!downloaded) { throw new NotFoundException(`Avatar ${record.fileName} download failed`) }
 
           const fileObject: FileObjectDTO = await this.hashAndSizeAndCheckExists(
             'LOCAL',
@@ -116,15 +116,15 @@ export class TaskProcessor {
 
           if (fileObject == null) break
 
-          console.log(`Upload image ${record.key} as ${fileObject.sha256}...`)
+          console.log(`Upload image ${record.fileName} as ${fileObject.sha256}...`)
           const fileUploaded = await this.minioService.uploadObject(
             'public',
             fileObject.sha256,
             originalFile,
             { ...record.metaData }
           )
 
-          if (!fileUploaded) { throw new Error(`Avatar ${record.key} upload failed`) }
+          if (!fileUploaded) { throw new Error(`Avatar ${record.fileName} upload failed`) }
 
           for (const size of Object.values(AVATAR_SIZES)) {
             record.fileDimension = size
@@ -134,7 +134,7 @@ export class TaskProcessor {
             let targetPath = resolve(TMP_DIR, record.fileName)
 
             console.log(
-              `Resize image ${record.key} to ${record.fileDimension}...`
+              `Resize image ${record.fileName} to ${record.fileDimension}...`
             )
 
             const info = await this.fileService.resizePicture(
@@ -171,11 +171,11 @@ export class TaskProcessor {
             )
           }
         } catch (error) {
-          console.error(error)
+          console.error(error.message)
 
           // if (record.assetId != null) { await this.assetService.removeAsset(record.assetId) }
 
-          await job.retry()
+          // await job.retry()
         } finally {
           this.clearDir()
         }
@@ -252,11 +252,13 @@ export class TaskProcessor {
       }
 
       console.log('update user avatar')
-      await this.userService.updateUserAvatar(
-        record.userId,
-        record.assetId,
-        image
-      )
+      if (image != null) {
+        await this.userService.updateUserAvatar(
+          record.userId,
+          record.assetId,
+          image
+        )
+      }
     } catch (error) {
       console.error(error)
       if (file.fileobjectId != null) {
@@ -266,7 +268,7 @@ export class TaskProcessor {
       // Delete object from CAS
 
       // Retry the Job
-      await job.retry()
+      // await job.retry()
     }
   }
 
@@ -332,7 +334,9 @@ export class TaskProcessor {
         : await FileObjectDTO.hashAndSizeFile(originalBucket, file)
 
     if (size != null && size !== fileObject.size) {
-      console.error('Size mismatch')
+      console.error('File size mismatch, processing upload!')
+      // Should we through an error here
+      // throw new Error('File size mismatch')
     }
 
     const fileExistsInBucket = await this.minioService.fileExists(