Merge pull request #352 from ZIMkaRU/feature/add-support-for-pdf-reports

Add support for PDF reports
bitfinexcom · Feb 20, 2024 · 4b9ea88 · 4b9ea88
2 parents 65a37f7 + 8eb0204
commit 4b9ea88
Show file tree

Hide file tree

Showing 37 changed files with 954 additions and 350 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,9 +12,11 @@ logs/*
 db/*
 !db/.gitkeep
 csv/
+report-files/
 logs
 package-lock.json
 workers/loc.api/queue/temp/*.csv
+workers/loc.api/queue/temp/*.pdf
 workers/loc.api/queue/temp/*.zip
 bfx-report-ui/.env
 bfx-report-ui/src

diff --git a/README.md b/README.md
@@ -130,7 +130,7 @@ Options:
 During the setup process, the user will be asked some questions
 
 - if no Docker/docker-compose are found, the user will be prompted to install them
-- to remove all Log and DB and CSV files to setup the app from scratch
+- to remove all Log and DB and report files to setup the app from scratch
 - to choose syncing repository branch (master/staging), by default master
 - to set NGINX port, by default 80
 - to set NGINX host, by default localhost

diff --git a/config/service.report.json.example b/config/service.report.json.example
@@ -14,7 +14,7 @@
     "signedUrlExpireTime": 28800
   },
   "isSpamRestrictionMode": true,
-  "isAddedUniqueEndingToCsvName": true,
+  "isAddedUniqueEndingToReportFileName": true,
   "isСompress": true,
   "isLoggerDisabled": false,
   "dbDriver": "better-sqlite",

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -55,15 +55,15 @@ services:
     volumes:
       - temp:/home/node/bfx-reports-framework/temp
       - ./db:/home/node/bfx-reports-framework/db
-      - ./csv:/home/node/bfx-reports-framework/csv
+      - ./report-files:/home/node/bfx-reports-framework/report-files
       - ./logs:/home/node/bfx-reports-framework/logs
     environment:
       <<: *common-variables
       WORKER_API_PORT: "1337"
       WORKER_WS_PORT: "1455"
       TEMP_FOLDER: "/home/node/bfx-reports-framework/temp"
       DB_FOLDER: "/home/node/bfx-reports-framework/db"
-      CSV_FOLDER: "/home/node/bfx-reports-framework/csv"
+      REPORT_FOLDER: "/home/node/bfx-reports-framework/report-files"
       LOGS_FOLDER: "/home/node/bfx-reports-framework/logs"
       SECRET_KEY: ${SECRET_KEY} # Required
       SCHEDULER_RULE: ${SCHEDULER_RULE} # Non-required
@@ -122,10 +122,10 @@ services:
       - ./nginx-configs/nginx.conf:/etc/nginx/nginx.conf:ro
       - ./nginx-configs/templates:/etc/nginx/templates
       - ./logs:/var/log/nginx/bfx-reports-framework
-      - ./csv:/var/www/csv
+      - ./report-files:/var/www/report-files
     environment:
       <<: *common-variables
-      CSV_FOLDER: "/var/www/csv"
+      REPORT_FOLDER: "/var/www/report-files"
       MAINTENANCE_STATIC_FOLDER: "/var/www/maintenance"
       FRONT_BUILD_FOLDER: "/var/www/html"
       LOGS_FOLDER: "/var/log/nginx/bfx-reports-framework"

diff --git a/nginx-configs/autoindex-format.xslt b/nginx-configs/autoindex-format.xslt
@@ -242,7 +242,7 @@
       </head>
       <body>
         <main role="main" class="container">
-          <h1 class="mt-5 title">Bitfinex Report CSV</h1>
+          <h1 class="mt-5 title">Bitfinex Report</h1>
           <ol class="breadcrumb"><xsl:call-template name="breadcrumb"><xsl:with-param name="list" select="$path" /></xsl:call-template></ol>
           <div class="list">
             <table class="table" summary="Directory Listing" cellpadding="0" cellspacing="0">

diff --git a/nginx-configs/templates/default.conf.template b/nginx-configs/templates/default.conf.template
@@ -48,20 +48,20 @@ server {
     add_header Cache-Control "public";
   }
 
-  location /csv {
+  location /report-files {
     auth_request /api/auth;
 
     autoindex ${NGINX_AUTOINDEX};
     autoindex_format xml;
     xslt_stylesheet /etc/nginx/autoindex-format.xslt path='$uri';
 
     try_files $uri/ =404;
-    alias ${CSV_FOLDER};
+    alias ${REPORT_FOLDER};
 
     if ($arg_token ~ ".+") {
       add_header Set-Cookie "token=$arg_token; Max-Age=3600; SameSite=strict; HttpOnly";
     }
-    if ($request_filename ~ "^.*/(.+\.(zip|csv))$") {
+    if ($request_filename ~ "^.*/(.+\.(zip|csv|pdf))$") {
       set $fname $1;
       add_header Content-Disposition 'attachment; filename="$fname"';
     }

diff --git a/package.json b/package.json
@@ -12,6 +12,7 @@
     "bfx-svc-boot-js": "https://github.com/bitfinexcom/bfx-svc-boot-js.git",
     "csv": "5.5.3",
     "grenache-nodejs-ws": "git+https://github.com:bitfinexcom/grenache-nodejs-ws.git",
+    "html-pdf": "3.0.1",
     "inversify": "6.0.1",
     "lib-js-util-base": "git+https://github.com/bitfinexcom/lib-js-util-base.git",
     "lodash": "4.17.21",
@@ -20,14 +21,14 @@
     "yargs": "17.2.1"
   },
   "devDependencies": {
-    "bfx-report-express": "git+https://github.com/bitfinexcom/bfx-report-express.git",
     "bfx-api-mock-srv": "git+https://github.com/bitfinexcom/bfx-api-mock-srv.git",
+    "bfx-report-express": "git+https://github.com/bitfinexcom/bfx-report-express.git",
+    "chai": "4.3.4",
     "grenache-grape": "git+https://github.com/bitfinexcom/grenache-grape.git",
     "mocha": "10.1.0",
-    "chai": "4.3.4",
     "nodemon": "2.0.20",
-    "supertest": "6.3.2",
-    "standard": "17.0.0"
+    "standard": "17.0.0",
+    "supertest": "6.3.2"
   },
   "contributors": [
     "Paolo Ardoino <[email protected]>",

diff --git a/scripts/setup.sh b/scripts/setup.sh
@@ -194,13 +194,13 @@ fi
 
 if askUser "\
 The app will be setup from scratch!\n\
-All Log and DB and CSV files will be removed!\n\
+All Log and DB and report files will be removed!\n\
 Are you sure?\
 "; then
   rm -rf "$ROOT/logs"
-  rm -rf "$ROOT/csv"
+  rm -rf "$ROOT/report-files"
   mkdir "$ROOT/logs" 2>/dev/null
-  mkdir "$ROOT/csv" 2>/dev/null
+  mkdir "$ROOT/report-files" 2>/dev/null
 
   if [ $isDBFoldedRemoved == 1 ]; then
     find "$ROOT/db" ! -path "$ROOT/db/.gitkeep" -type f -exec rm -rf {} +

diff --git a/scripts/worker-entrypoint.sh b/scripts/worker-entrypoint.sh
@@ -18,7 +18,7 @@ set -- node "$@" \
   "--wsPort" "$WORKER_WS_PORT" \
   "--tempFolder" "$TEMP_FOLDER" \
   "--dbFolder" "$DB_FOLDER" \
-  "--csvFolder" "$CSV_FOLDER" \
+  "--reportFolder" "$REPORT_FOLDER" \
   "--logsFolder" "$LOGS_FOLDER" \
   "--grape" "http://$GRAPE_HOST:$GRAPE_APH" \
   "--secretKey" "$SECRET_KEY"
@@ -27,7 +27,7 @@ if [ -n "${SCHEDULER_RULE:-}" ]; then
   set -- "$@" "--schedulerRule" "$SCHEDULER_RULE"
 fi
 if [ "${NGINX_AUTOINDEX:-}" = "on" ]; then
-  set -- "$@" "--remoteCsvUrn" "csv/"
+  set -- "$@" "--remoteReportUrn" "report-files/"
 fi
 
 exec "$@"
diff --git a/test/1-api-sync-mode-sqlite.spec.js b/test/1-api-sync-mode-sqlite.spec.js
@@ -39,7 +39,7 @@ let mockRESTv2Srv = null
 const basePath = '/api'
 const tempDirPath = path.join(__dirname, '..', 'workers/loc.api/queue/temp')
 const dbDirPath = path.join(__dirname, '..', 'db')
-const csvDirPath = path.join(__dirname, '..', 'csv')
+const reportFolderPath = path.join(__dirname, '..', 'report-files')
 const date = new Date()
 const end = date.getTime()
 const start = (new Date()).setDate(date.getDate() - 90)
@@ -78,7 +78,7 @@ describe('Sync mode API with SQLite', () => {
 
     mockRESTv2Srv = createMockRESTv2SrvWithDate(start, end, 100)
 
-    await rmRf(csvDirPath)
+    await rmRf(reportFolderPath)
     await rmAllFiles(tempDirPath, ['README.md'])
     await rmDB(dbDirPath)
     const env = await startEnvironment(false, false, 1)
@@ -98,31 +98,53 @@ describe('Sync mode API with SQLite', () => {
     await stopEnvironment()
     await rmDB(dbDirPath)
     await rmAllFiles(tempDirPath, ['README.md'])
-    await rmRf(csvDirPath)
+    await rmRf(reportFolderPath)
 
     try {
       await mockRESTv2Srv.close()
     } catch (err) { }
   })
 
   describe('Use BFX API keys', () => {
-    signUpTestCase(agent, params)
-    apiSyncModeSqliteTestCases(agent, params)
-    signUpTestCase(agent, params)
-    removeUserTestCases(agent, params)
+    describe('CSV generation', () => {
+      params.isPDFRequired = false
+      signUpTestCase(agent, params)
+      apiSyncModeSqliteTestCases(agent, params)
+      signUpTestCase(agent, params)
+      removeUserTestCases(agent, params)
+    })
+
+    describe('PDF generation', () => {
+      params.isPDFRequired = true
+      signUpTestCase(agent, params)
+      apiSyncModeSqliteTestCases(agent, params)
+      signUpTestCase(agent, params)
+      removeUserTestCases(agent, params)
+    })
   })
 
   describe('Use BFX auth token', () => {
     before(async function () {
       this.timeout(20000)
 
-      await rmRf(csvDirPath)
+      await rmRf(reportFolderPath)
       await rmAllFiles(tempDirPath, ['README.md'])
     })
 
-    signUpTestCase(agent, paramsWithAuthToken)
-    apiSyncModeSqliteTestCases(agent, paramsWithAuthToken)
-    signUpTestCase(agent, paramsWithAuthToken)
-    removeUserTestCases(agent, paramsWithAuthToken)
+    describe('CSV generation', () => {
+      paramsWithAuthToken.isPDFRequired = false
+      signUpTestCase(agent, paramsWithAuthToken)
+      apiSyncModeSqliteTestCases(agent, paramsWithAuthToken)
+      signUpTestCase(agent, paramsWithAuthToken)
+      removeUserTestCases(agent, paramsWithAuthToken)
+    })
+
+    describe('PDF generation', () => {
+      paramsWithAuthToken.isPDFRequired = true
+      signUpTestCase(agent, paramsWithAuthToken)
+      apiSyncModeSqliteTestCases(agent, paramsWithAuthToken)
+      signUpTestCase(agent, paramsWithAuthToken)
+      removeUserTestCases(agent, paramsWithAuthToken)
+    })
   })
 })
diff --git a/test/2-additional-api-sync-mode-sqlite.spec.js b/test/2-additional-api-sync-mode-sqlite.spec.js
@@ -38,7 +38,7 @@ let mockRESTv2Srv = null
 const basePath = '/api'
 const tempDirPath = path.join(__dirname, '..', 'workers/loc.api/queue/temp')
 const dbDirPath = path.join(__dirname, '..', 'db')
-const csvDirPath = path.join(__dirname, '..', 'csv')
+const reportFolderPath = path.join(__dirname, '..', 'report-files')
 const date = new Date()
 const end = date.getTime()
 const start = (new Date()).setDate(date.getDate() - 90)
@@ -77,7 +77,7 @@ describe('Additional sync mode API with SQLite', () => {
 
     mockRESTv2Srv = createMockRESTv2SrvWithDate(start, end, 100)
 
-    await rmRf(csvDirPath)
+    await rmRf(reportFolderPath)
     await rmAllFiles(tempDirPath, ['README.md'])
     await rmDB(dbDirPath)
     const env = await startEnvironment(false, false, 1)
@@ -97,27 +97,45 @@ describe('Additional sync mode API with SQLite', () => {
     await stopEnvironment()
     await rmDB(dbDirPath)
     await rmAllFiles(tempDirPath, ['README.md'])
-    await rmRf(csvDirPath)
+    await rmRf(reportFolderPath)
 
     try {
       await mockRESTv2Srv.close()
     } catch (err) { }
   })
 
   describe('Use BFX API keys', () => {
-    signUpTestCase(agent, params)
-    additionalApiSyncModeSqliteTestCases(agent, params)
+    describe('CSV generation', () => {
+      params.isPDFRequired = false
+      signUpTestCase(agent, params)
+      additionalApiSyncModeSqliteTestCases(agent, params)
+    })
+
+    describe('PDF generation', () => {
+      params.isPDFRequired = true
+      signUpTestCase(agent, params)
+      additionalApiSyncModeSqliteTestCases(agent, params)
+    })
   })
 
   describe('Use BFX auth token', () => {
     before(async function () {
       this.timeout(20000)
 
-      await rmRf(csvDirPath)
+      await rmRf(reportFolderPath)
       await rmAllFiles(tempDirPath, ['README.md'])
     })
 
-    signUpTestCase(agent, paramsWithAuthToken)
-    additionalApiSyncModeSqliteTestCases(agent, paramsWithAuthToken)
+    describe('CSV generation', () => {
+      paramsWithAuthToken.isPDFRequired = false
+      signUpTestCase(agent, paramsWithAuthToken)
+      additionalApiSyncModeSqliteTestCases(agent, paramsWithAuthToken)
+    })
+
+    describe('PDF generation', () => {
+      paramsWithAuthToken.isPDFRequired = true
+      signUpTestCase(agent, paramsWithAuthToken)
+      additionalApiSyncModeSqliteTestCases(agent, paramsWithAuthToken)
+    })
   })
 })