refactor(ts-migration): ndjsonstream and process-decrypted-content

src/public/modules/forms/admin/controllers/view-responses.client.controller.js

@@ -1,6 +1,8 @@
 'use strict'
 
-const processDecryptedContent = require('../../helpers/process-decrypted-content')
+const {
+  processDecryptedContent,
+} = require('../../helpers/process-decrypted-content')
 const { triggerFileDownload } = require('../../helpers/util')
 
 const SHOW_PROGRESS_DELAY_MS = 3000

src/public/modules/forms/helpers/decryption.worker.js

@@ -8,7 +8,9 @@ const JSZip = require('jszip')
 const moment = require('moment-timezone')
 const { default: PQueue } = require('p-queue')
 
-const processDecryptedContent = require('../helpers/process-decrypted-content')
+const {
+  processDecryptedContent,
+} = require('../helpers/process-decrypted-content')
 const {
   TRANSACTION_EXPIRE_AFTER_SECONDS,
 } = require('../../../../shared/util/verification')

src/public/modules/forms/helpers/ndjsonStream.ts

@@ -0,0 +1,73 @@
+// Modified fork of https://github.com/canjs/can-ndjson-stream to enqueue
+// the string immediately without a JSON.parse() step, as the stream payload
+// is to be decrypted by the decryption worker.
+
+// Note that this code assumes a polyfill of TextDecoder is available to run in IE11.
+
+export const ndjsonStream = (
+  response: ReadableStream<Uint8Array>,
+): ReadableStream => {
+  // For cancellation
+  let maybeReader: ReadableStreamDefaultReader<Uint8Array> | undefined
+  let shouldCancel = false
+  return new ReadableStream({
+    start: function (controller) {
+      const reader = response.getReader()
+      maybeReader = reader
+      const decoder = new TextDecoder()
+      let data_buf = ''
+
+      return reader
+        .read()
+        .then(function processResult(result): Promise<void> | undefined | void {
+          if (result.done && shouldCancel) {
+            return
+          }
+
+          if (result.done) {
+            data_buf = data_buf.trim()
+            if (data_buf.length !== 0) {
+              try {
+                controller.enqueue(data_buf)
+              } catch (e) {
+                controller.error(e)
+                return
+              }
+            }
+            controller.close()
+            return
+          }
+
+          // Read the input in as a stream and split by newline and trim
+          data_buf += decoder.decode(result.value, { stream: true })
+          const lines = data_buf.split('\n')
+
+          // Reads in every line BUT the last
+          // Trims the line and queues it in the controller if there is content in the line
+          for (let i = 0; i < lines.length - 1; ++i) {
+            const l = lines[i].trim()
+            if (l.length > 0) {
+              try {
+                controller.enqueue(l)
+              } catch (e) {
+                controller.error(e)
+                shouldCancel = true
+                void reader.cancel()
+                return
+              }
+            }
+          }
+          data_buf = lines[lines.length - 1]
+
+          return reader.read().then(processResult)
+        })
+    },
+    cancel: function (reason) {
+      console.log('Cancel registered due to ', reason)
+      shouldCancel = true
+      if (maybeReader) {
+        void maybeReader.cancel()
+      }
+    },
+  })
+}

src/public/modules/forms/helpers/process-decrypted-content.ts

@@ -0,0 +1,91 @@
+import {
+  DecryptedContent,
+  FormField as VerifiedFormField,
+} from '@opengovsg/formsg-sdk/dist/types'
+import has from 'lodash/has'
+
+import {
+  CURRENT_VERIFIED_FIELDS,
+  VerifiedKeys,
+} from '../../../../shared/util/verified-content'
+import { BasicField, SPCPFieldTitle } from '../../../../types'
+
+/**
+ * Returns a response matching the given type containing the given value.
+ * @param type the field type to match
+ * @param value the value to insert into the response to be returned
+ * @returns the desired response object if type is valid. Else returns null.
+ */
+const getResponseFromVerifiedField = (
+  type: VerifiedKeys,
+  value: string,
+): VerifiedFormField | null => {
+  switch (type) {
+    case VerifiedKeys.SpUinFin:
+      return {
+        question: SPCPFieldTitle.SpNric,
+        fieldType: BasicField.Nric,
+        answer: value,
+        // Just a unique identifier for CSV header uniqueness
+        _id: SPCPFieldTitle.SpNric,
+      }
+
+    case VerifiedKeys.CpUen:
+      return {
+        question: SPCPFieldTitle.CpUen,
+        fieldType: BasicField.ShortText,
+        answer: value,
+        _id: SPCPFieldTitle.CpUen,
+      }
+
+    case VerifiedKeys.CpUid:
+      return {
+        question: SPCPFieldTitle.CpUid,
+        fieldType: BasicField.Nric,
+        answer: value,
+        _id: SPCPFieldTitle.CpUid,
+      }
+    default:
+      return null
+  }
+}
+
+/**
+ * Converts a decrypted verified object into an array with the same shape as the
+ * current decrypted content to be concatenated with the decrypted content.
+ * NOTE: This function assumes verifiedObj is an object with simple string
+ * key-value pairs.
+ * @param verifiedObj the object to convert
+ * @returns the converted array.
+ */
+const convertToResponseArray = (
+  verifiedObj: Record<string, string>,
+): VerifiedFormField[] => {
+  return CURRENT_VERIFIED_FIELDS.filter((fieldType) =>
+    has(verifiedObj, fieldType),
+  )
+    .map((fieldType) =>
+      getResponseFromVerifiedField(fieldType, verifiedObj[fieldType]),
+    )
+    .filter((field): field is VerifiedFormField => {
+      return !!field
+    })
+}
+
+/**
+ * Processes the decrypted content containing the previously encrypted responses
+ * and verified content, and combines them into a single response array.
+ * @param decrypted.responses the previously encrypted responses content
+ * @param decrypted.verified the previously encrypted verified content,if it exists
+ * @returns the processed content
+ */
+export const processDecryptedContent = (
+  decrypted: DecryptedContent,
+): VerifiedFormField[] => {
+  const { responses: displayedContent, verified } = decrypted
+  // Convert decrypted content into displayable object.
+
+  return verified
+    ? displayedContent.concat(convertToResponseArray(verified))
+    : displayedContent
+}

src/public/modules/forms/services/submissions.client.factory.js

@@ -3,7 +3,7 @@
 const CsvMHGenerator = require('../helpers/CsvMergedHeadersGenerator')
 const DecryptionWorker = require('../helpers/decryption.worker.js')
 const { fixParamsToUrl, triggerFileDownload } = require('../helpers/util')
-const ndjsonStream = require('../helpers/ndjsonStream')
+const { ndjsonStream } = require('../helpers/ndjsonStream')
 const fetchStream = require('fetch-readablestream')
 const { decode: decodeBase64 } = require('@stablelib/base64')
 const JSZip = require('jszip')