From 5bffa357b7b187c5418f28ccf20c85a86e25ee2d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Sun, 24 Jan 2021 19:57:34 -0800 Subject: [PATCH 1/9] refactor: Minor refactoring of the submissions client factory (#1040) --- .../services/submissions.client.factory.js | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/public/modules/forms/services/submissions.client.factory.js b/src/public/modules/forms/services/submissions.client.factory.js index c478459d0a..815560dcff 100644 --- a/src/public/modules/forms/services/submissions.client.factory.js +++ b/src/public/modules/forms/services/submissions.client.factory.js @@ -46,6 +46,17 @@ function SubmissionsFactory( const publicSubmitUrl = '/v2/submissions/:responseMode/:formId' const previewSubmitUrl = '/v2/submissions/:responseMode/preview/:formId' + const generateDownloadUrl = (params, downloadAttachments) => { + let resUrl = `${fixParamsToUrl(params, submitAdminUrl)}/download?` + if (params.startDate && params.endDate) { + resUrl += `startDate=${params.startDate}&endDate=${params.endDate}&` + } + if (downloadAttachments) { + resUrl += `downloadAttachments=true&` + } + return resUrl + } + /** * Creates form data and submits as multi-part * @param {String} resUrl @@ -277,14 +288,7 @@ function SubmissionsFactory( }) } - let resUrl = `${fixParamsToUrl(params, submitAdminUrl)}/download?` - if (params.startDate && params.endDate) { - resUrl += `startDate=${params.startDate}&endDate=${params.endDate}&` - } - if (downloadAttachments) { - resUrl += `downloadAttachments=true&` - } - + let resUrl = generateDownloadUrl(params, downloadAttachments) let experimentalCsvGenerator = new CsvMHGenerator( expectedNumResponses, NUM_OF_METADATA_ROWS, @@ -399,7 +403,7 @@ function SubmissionsFactory( 'Failed to download data, is there a network issue?', err, ) - workerPool.forEach((worker) => worker.terminate()) + killWorkers(workerPool) reject(err) }) .finally(() => { From 7bfd617ca1bdd0d8c04cac2b3fad4e667f8043fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 11:59:36 +0800 Subject: [PATCH 2/9] chore(deps-dev): bump webpack from 4.43.0 to 4.46.0 (#1035) Bumps [webpack](https://github.com/webpack/webpack) from 4.43.0 to 4.46.0. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v4.43.0...v4.46.0) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 186 ++++++++++++++++++++++++++++------------------ package.json | 2 +- 2 files changed, 113 insertions(+), 75 deletions(-) diff --git a/package-lock.json b/package-lock.json index daad4e0d01..6725715fd9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6503,34 +6503,34 @@ } }, "browserify-rsa": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", - "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", + "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", "dev": true, "requires": { - "bn.js": "^4.1.0", + "bn.js": "^5.0.0", "randombytes": "^2.0.1" }, "dependencies": { "bn.js": { - "version": "4.11.9", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz", + "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==", "dev": true } } }, "browserify-sign": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.0.tgz", - "integrity": "sha512-hEZC1KEeYuoHRqhGhTy6gWrpJA3ZDjFWv0DE61643ZnOXAKJb3u7yWcrU0mMc9SwAqK1n7myPGndkp0dFG7NFA==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", "dev": true, "requires": { "bn.js": "^5.1.1", "browserify-rsa": "^4.0.1", "create-hash": "^1.2.0", "create-hmac": "^1.1.7", - "elliptic": "^6.5.2", + "elliptic": "^6.5.3", "inherits": "^2.0.4", "parse-asn1": "^5.1.5", "readable-stream": "^3.6.0", @@ -6538,9 +6538,9 @@ }, "dependencies": { "bn.js": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.2.tgz", - "integrity": "sha512-40rZaf3bUNKTVYu9sIeeEGOg7g14Yvnj9kH7b50EiwX0Q7A6umbvfI5tvHaOERH0XigqKkfLkFQxzb4e6CIXnA==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz", + "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==", "dev": true }, "readable-stream": { @@ -7733,13 +7733,10 @@ } }, "console-browserify": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.1.0.tgz", - "integrity": "sha1-8CQcRXMKn8YyOyBtvzjtx0HQuxA=", - "dev": true, - "requires": { - "date-now": "^0.1.4" - } + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==", + "dev": true }, "console-control-strings": { "version": "1.1.0", @@ -8111,13 +8108,13 @@ } }, "create-ecdh": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz", - "integrity": "sha512-GbEHQPMOswGpKXM9kCWVrremUcBmjteUaQ01T9rkKCPDXfUHX0IoP9LpHYo2NPFampa4e+/pFDc3jQdxrxQLaw==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", "dev": true, "requires": { "bn.js": "^4.1.0", - "elliptic": "^6.0.0" + "elliptic": "^6.5.3" }, "dependencies": { "bn.js": { @@ -8582,12 +8579,6 @@ "integrity": "sha512-sAJVKx/FqrLYHAQeN7VpJrPhagZc9R4ImZIWYRFZaaohR3KzmuK88touwsSwSVT8Qcbd4zoDsnGfX4GFB4imyQ==", "dev": true }, - "date-now": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz", - "integrity": "sha1-6vQ5/U1ISK105cx9vvIAZyueNFs=", - "dev": true - }, "dateformat": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-1.0.12.tgz", @@ -16943,9 +16934,9 @@ } }, "events": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.1.0.tgz", - "integrity": "sha512-Rv+u8MLHNOdMjTAFeT3nCjHn2aGlx435FP/sDHNaRhDEMwyI/aB22Kj2qIN8R0cw3z28psEQLYwxVKLsKrMgWg==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz", + "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg==", "dev": true }, "inherits": { @@ -17653,28 +17644,28 @@ } }, "parse-asn1": { - "version": "5.1.5", - "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.5.tgz", - "integrity": "sha512-jkMYn1dcJqF6d5CpU689bq7w/b5ALS9ROVSpQDPrZsqqesUJii9qutvoT5ltGedNXMO2e16YUWIghG9KxaViTQ==", + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", "dev": true, "requires": { - "asn1.js": "^4.0.0", + "asn1.js": "^5.2.0", "browserify-aes": "^1.0.0", - "create-hash": "^1.1.0", "evp_bytestokey": "^1.0.0", "pbkdf2": "^3.0.3", "safe-buffer": "^5.1.1" }, "dependencies": { "asn1.js": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz", - "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==", + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", "dev": true, "requires": { "bn.js": "^4.0.0", "inherits": "^2.0.1", - "minimalistic-assert": "^1.0.0" + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" } }, "bn.js": { @@ -22792,9 +22783,9 @@ "dev": true }, "timers-browserify": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz", - "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==", + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", + "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", "dev": true, "requires": { "setimmediate": "^1.0.4" @@ -23786,21 +23777,57 @@ } }, "watchpack": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.2.tgz", - "integrity": "sha512-ymVbbQP40MFTp+cNMvpyBpBtygHnPzPkHqoIwRRj/0B8KhqQwV8LaKjtbaxF2lK4vl8zN9wCxS46IFCU5K4W0g==", + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.5.tgz", + "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==", "dev": true, "requires": { - "chokidar": "^3.4.0", + "chokidar": "^3.4.1", "graceful-fs": "^4.1.2", "neo-async": "^2.5.0", - "watchpack-chokidar2": "^2.0.0" + "watchpack-chokidar2": "^2.0.1" + }, + "dependencies": { + "chokidar": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", + "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", + "dev": true, + "optional": true, + "requires": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.3.1", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.5.0" + } + }, + "fsevents": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.1.tgz", + "integrity": "sha512-YR47Eg4hChJGAB1O3yEAOkGO+rlzutoICGqGo9EZ4lKWokzZRSyIW1QmTzqjtw8MJdj9srP869CuWw/hyzSiBw==", + "dev": true, + "optional": true + }, + "readdirp": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", + "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", + "dev": true, + "optional": true, + "requires": { + "picomatch": "^2.2.1" + } + } } }, "watchpack-chokidar2": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz", - "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz", + "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==", "dev": true, "optional": true, "requires": { @@ -24057,9 +24084,9 @@ "dev": true }, "webpack": { - "version": "4.43.0", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.43.0.tgz", - "integrity": "sha512-GW1LjnPipFW2Y78OOab8NJlCflB7EFskMih2AHdvjbpKMeDJqEgSx24cXXXiPS65+WSwVyxtDsJH6jGX2czy+g==", + "version": "4.46.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.46.0.tgz", + "integrity": "sha512-6jJuJjg8znb/xRItk7bkT0+Q7AHCYjjFnvKIWQPkNIOyRqoCGvkOs0ipeQzrqz4l5FtN5ZI/ukEHroeX/o1/5Q==", "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", @@ -24070,7 +24097,7 @@ "ajv": "^6.10.2", "ajv-keywords": "^3.4.1", "chrome-trace-event": "^1.0.2", - "enhanced-resolve": "^4.1.0", + "enhanced-resolve": "^4.5.0", "eslint-scope": "^4.0.3", "json-parse-better-errors": "^1.0.2", "loader-runner": "^2.4.0", @@ -24083,28 +24110,16 @@ "schema-utils": "^1.0.0", "tapable": "^1.1.3", "terser-webpack-plugin": "^1.4.3", - "watchpack": "^1.6.1", + "watchpack": "^1.7.4", "webpack-sources": "^1.4.1" }, "dependencies": { "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", "dev": true }, - "ajv": { - "version": "6.12.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", - "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, "braces": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", @@ -24134,6 +24149,29 @@ } } }, + "enhanced-resolve": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.5.0.tgz", + "integrity": "sha512-Nv9m36S/vxpsI+Hc4/ZGRs0n9mXqSWGGq49zxb/cJfPAQMbUtttJAlNPS4AQzaBdw/pKskw5bMbekT/Y7W/Wlg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "memory-fs": "^0.5.0", + "tapable": "^1.0.0" + }, + "dependencies": { + "memory-fs": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.5.0.tgz", + "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "dev": true, + "requires": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + } + } + }, "eslint-scope": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", diff --git a/package.json b/package.json index e1689c301e..0e2193321e 100644 --- a/package.json +++ b/package.json @@ -257,7 +257,7 @@ "type-fest": "^0.20.2", "typescript": "^4.1.3", "url-loader": "^1.1.2", - "webpack": "^4.43.0", + "webpack": "^4.46.0", "webpack-cli": "^3.3.12", "webpack-merge": "^4.1.3", "worker-loader": "^2.0.0" From 4eecc583d867b4cb4e129debc722635816aab18d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 12:00:47 +0800 Subject: [PATCH 3/9] fix(deps): bump angular-translate-loader-partial from 2.18.3 to 2.18.4 (#1037) Bumps [angular-translate-loader-partial](https://github.com/angular-translate/bower-angular-translate-loader-partial) from 2.18.3 to 2.18.4. - [Release notes](https://github.com/angular-translate/bower-angular-translate-loader-partial/releases) - [Commits](https://github.com/angular-translate/bower-angular-translate-loader-partial/compare/2.18.3...2.18.4) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6725715fd9..a7c0bc488f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5338,17 +5338,17 @@ } }, "angular-translate-loader-partial": { - "version": "2.18.3", - "resolved": "https://registry.npmjs.org/angular-translate-loader-partial/-/angular-translate-loader-partial-2.18.3.tgz", - "integrity": "sha512-Wi8x45aScHYcxLd2Pi761F0rDjPV3cKeAjuxUadYtOqov0pWWH18nBELUFMhaELt6qUi4fFxQ9o5rfqDNXqVoQ==", + "version": "2.18.4", + "resolved": "https://registry.npmjs.org/angular-translate-loader-partial/-/angular-translate-loader-partial-2.18.4.tgz", + "integrity": "sha512-bsjR+FbB0sdA2528E/ugwKdlPPQhA1looxLxI3otayBTFXBpED33besfSZhYAISLgNMSL038vSssfRUen9qD8w==", "requires": { - "angular-translate": "~2.18.3" + "angular-translate": "~2.18.4" }, "dependencies": { "angular-translate": { - "version": "2.18.3", - "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.3.tgz", - "integrity": "sha512-ziEi1nTaNIsdn3iKpcALaSFjM9CjwvoIWE/EKepajB/6qT5oB3K1IU5VcaH7Bd0scNRfaJpW+qUD6nsHWDEZ6A==", + "version": "2.18.4", + "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.18.4.tgz", + "integrity": "sha512-KohNrkH6J9PK+VW0L/nsRTcg5Fw70Ajwwe3Jbfm54Pf9u9Fd+wuingoKv+h45mKf38eT+Ouu51FPua8VmZNoCw==", "requires": { "angular": "^1.8.0" } diff --git a/package.json b/package.json index 0e2193321e..3350a42053 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "angular-resource": "^1.8.2", "angular-sanitize": "^1.8.2", "angular-translate": "^2.18.3", - "angular-translate-loader-partial": "^2.18.3", + "angular-translate-loader-partial": "^2.18.4", "angular-ui-bootstrap": "~2.5.6", "angular-ui-router": "~1.0.29", "await-to-js": "^2.1.1", From fc096ce305da3843d26a927b048e42e1f8173993 Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Tue, 26 Jan 2021 11:26:37 +0800 Subject: [PATCH 4/9] fix: upgrade mongoose from 5.11.9 to 5.11.10 (#1047) Snyk has created this PR to upgrade mongoose from 5.11.9 to 5.11.10. See this package in npm: https://www.npmjs.com/package/mongoose See this project in Snyk: https://app.snyk.io/org/open-government-products/project/4c4f5244-5ce5-491a-a42d-c295bc7f51fd?utm_source=github&utm_medium=upgrade-pr Co-authored-by: snyk-bot --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index a7c0bc488f..dc4c288a9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16465,16 +16465,16 @@ "integrity": "sha1-D3ca0W9IOuZfQoeWlCjp+8SqYYE=" }, "mongoose": { - "version": "5.11.9", - "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.11.9.tgz", - "integrity": "sha512-lmG6R64jtGGxqtn88BkkY+v470LUfGgyTKUyjswQ5c01GNgQvxA0kQd8h+tm0hZb639hKNRxL9ZBQlLleUpuIQ==", + "version": "5.11.10", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-5.11.10.tgz", + "integrity": "sha512-daE2L6VW7WNywv7tL2KUkBViWvODbzr50Of1kJpIbzW3w3N5/TYcgSmhCsEDWfYGQXbun2rdd7+sOdsEC8zQSQ==", "requires": { "@types/mongodb": "^3.5.27", "bson": "^1.1.4", "kareem": "2.3.2", "mongodb": "3.6.3", "mongoose-legacy-pluralize": "1.0.2", - "mpath": "0.8.1", + "mpath": "0.8.3", "mquery": "3.2.3", "ms": "2.1.2", "regexp-clone": "1.0.0", @@ -16570,9 +16570,9 @@ } }, "mpath": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.8.1.tgz", - "integrity": "sha512-norEinle9aFc05McBawVPwqgFZ7npkts9yu17ztIVLwPwO9rq0OTp89kGVTqvv5rNLMz96E5iWHpVORjI411vA==" + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.8.3.tgz", + "integrity": "sha512-eb9rRvhDltXVNL6Fxd2zM9D4vKBxjVVQNLNijlj7uoXUy19zNDsIif5zR+pWmPCWNKwAtqyo4JveQm4nfD5+eA==" }, "mquery": { "version": "3.2.3", diff --git a/package.json b/package.json index 3350a42053..47069cbd2a 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "lodash": "^4.17.20", "moment-timezone": "0.5.32", "mongodb-uri": "^0.9.7", - "mongoose": "^5.11.9", + "mongoose": "^5.11.10", "multiparty": ">=4.2.2", "neverthrow": "^3.2.0", "ng-infinite-scroll": "^1.3.0", From 640b6edea546169c8e5fb45de1204d8eacc7911b Mon Sep 17 00:00:00 2001 From: Yuan Ruo Date: Tue, 26 Jan 2021 11:31:49 +0800 Subject: [PATCH 5/9] refactor/submit-form: reduce possibility of race conditions in submit-form controller (#1045) The bug in #1044 was not reproduced. This is not a surprise as concurrency bugs are indeterministic by definition. However, based on the symptom and rarity of the bug, it is possible that the behavior is a result of edge cases caused by poor integration with AngularJS. Multiple code paths were thoroughly examined, and the appropriate refactor performed to improve code clarity and reduce the likelihood of concurrency issues. controllerState variable refactor The controllerState variable is used to store the cloned form object in the savedForm property before fields were locked/disabled. It was previously not attached to the directive scope, but simply as a const which makes it a possible candidate to resolve the concurrency bug as AngularJS would not be tracking changes to this variable during the digest cycle. submitFormMain function refactor The need to clone the form object before calling this function was removed by refactoring to avoid assigning to the form.attachment property (it is a very bad practice to mutate function arguments). This not only removed the need for an expensive clone operation, but also guarantees that there can be no divergence between the form object in the controller scope vs the one used in the submitForm function, since the call to submitForm(scope.form) is a reference to the same form object. Co-authored-by: Yuanruo Liang --- .../base/directives/submit-form.directive.js | 72 +++++-------------- .../modules/forms/viewmodels/Form.class.js | 65 +++++++++++++++-- 2 files changed, 79 insertions(+), 58 deletions(-) diff --git a/src/public/modules/forms/base/directives/submit-form.directive.js b/src/public/modules/forms/base/directives/submit-form.directive.js index 30eb428241..4e9eba4c4f 100644 --- a/src/public/modules/forms/base/directives/submit-form.directive.js +++ b/src/public/modules/forms/base/directives/submit-form.directive.js @@ -6,8 +6,6 @@ const { getLogicUnitPreventingSubmit, } = require('../../../../../shared/util/logic') -// The current encrypt version to assign to the encrypted submission. -const ENCRYPT_VERSION = 1 /** * @typedef {number} FormState */ @@ -83,9 +81,11 @@ function submitFormDirective( progressModal: null, submitPrevented: false, } + // Also used to store a backup of the form state during submission, the state // of the progress modal - const controllerState = {} + scope.controllerState = {} + scope.hasMyInfoFields = FormFields.containsMyInfoFields(scope.form) scope.formLogin = function (authType, rememberMe) { // Fire GA tracking event @@ -211,7 +211,7 @@ function submitFormDirective( } break case FORM_STATES.SUBMITTING: - controllerState.savedForm = cloneDeep(scope.form) + scope.controllerState.savedForm = cloneDeep(scope.form) scope.form.lockFields() scope.uiState.submitButtonClicked = true if (scope.form.hasAttachments()) { @@ -231,7 +231,7 @@ function submitFormDirective( } break case FORM_STATES.SUBMISSION_ERROR: - scope.form = controllerState.savedForm + scope.form = scope.controllerState.savedForm setFormState(FORM_STATES.DEFAULT) break case FORM_STATES.SUBMIT_PREVENTED: @@ -251,7 +251,7 @@ function submitFormDirective( * Opens the submission progress modal. */ const openProgressModal = function () { - controllerState.progressModal = $uibModal.open({ + scope.controllerState.progressModal = $uibModal.open({ animation: true, backdrop: 'static', keyboard: false, @@ -265,9 +265,9 @@ function submitFormDirective( * Closes the submission progress modal if it is currently open. */ const closeProgressModal = () => { - if (controllerState.progressModal) { - controllerState.progressModal.close() - controllerState.progressModal = null + if (scope.controllerState.progressModal) { + scope.controllerState.progressModal.close() + scope.controllerState.progressModal = null } } @@ -276,9 +276,8 @@ function submitFormDirective( * and handle any errors. */ scope.submitForm = () => { - let form = cloneDeep(scope.form) try { - submitFormMain(form) + submitFormMain(scope.form) } catch (error) { handleSubmitFailure(error, 'Please try again later.') } @@ -292,36 +291,21 @@ function submitFormDirective( // Disable UI and optionally open progress modal while processing setFormState(FORM_STATES.SUBMITTING) + // submissionContent is the POST body to backend when we submit the form + let submissionContent + try { - form.attachments = await form.getAttachments() + submissionContent = Object.assign( + { captchaResponse: captchaService.response }, + await form.getSubmissionContent(), + ) } catch (err) { return handleSubmitFailure( err, - 'Could not encrypt your attachments. Please contact the form administrator.', + 'Could not prepare your submission. Please contact the form administrator.', ) } - const responses = form.getResponses() - - // submissionContent is the POST body to backend when we submit the form - let submissionContent = { - attachments: form.attachments, - captchaResponse: captchaService.response, - isPreview: form.isPreview, - responses, - } - if (form.responseMode === responseModeEnum.ENCRYPT && form.publicKey) { - try { - encryptSubmissionContent(submissionContent, form.publicKey) - submissionContent.version = ENCRYPT_VERSION - } catch (err) { - return handleSubmitFailure( - err, - 'Could not encrypt your submission. Please contact the form administrator.', - ) - } - } - Submissions.post( { formId: scope.form._id, @@ -331,26 +315,6 @@ function submitFormDirective( ).then(handleSubmitSuccess, handleSubmitFailure) } - /** - * Encrypts submission content for end-to-end encryption. - * @param {Object} submissionContent Content to encrypt - * @param {String} publicKey Encryption public key - */ - const encryptSubmissionContent = (submissionContent, publicKey) => { - submissionContent.encryptedContent = FormSgSdk.crypto.encrypt( - submissionContent.responses, - publicKey, - ) - submissionContent.responses = submissionContent.responses - .filter((item) => ['mobile', 'email'].includes(item.fieldType)) - .map((item) => { - return _(item) - .pick(['fieldType', '_id', 'answer', 'signature']) - .omitBy(_.isNull) - .value() - }) - } - /** * Helper function for Google Analytics to check for persistent login. */ diff --git a/src/public/modules/forms/viewmodels/Form.class.js b/src/public/modules/forms/viewmodels/Form.class.js index af632387d6..35d77c37d4 100644 --- a/src/public/modules/forms/viewmodels/Form.class.js +++ b/src/public/modules/forms/viewmodels/Form.class.js @@ -1,3 +1,6 @@ +const _ = require('lodash') +const formsg = require('@opengovsg/formsg-sdk')() + const FieldFactory = require('../helpers/field-factory') const { getEncryptedAttachmentsMap, @@ -6,6 +9,11 @@ const { } = require('../helpers/attachments-map') const { NoAnswerField } = require('./Fields') +// The current encrypt version to assign to the encrypted submission. +// This is needed if we ever break backwards compatibility with +// end-to-end encryption +const ENCRYPT_VERSION = 1 + /** * Deserialises raw form object returned by backend and * manages form-wide operations. @@ -31,10 +39,11 @@ class Form { } /** - * Gets the submitted values of all fields. + * Internal helper function to get the submitted values of all fields + * that are supposed to have answers. * @returns {Array} Array of response objects. */ - getResponses() { + _getResponses() { return this.form_fields .filter((field) => !(field instanceof NoAnswerField)) .map((field) => { @@ -46,10 +55,12 @@ class Form { } /** - * Creates a map of field ID to attachment file. + * Internal helper function that creates a map of field ID to attachment file. + * The values of the map are encrypted for Storage Mode. + * forms. * @returns {Object} Map of { id: file } */ - getAttachments() { + _getAttachments() { const attachmentsMap = getAttachmentsMap(this.form_fields) if (this.responseMode === 'encrypt') { return getEncryptedAttachmentsMap(attachmentsMap, this.publicKey) @@ -65,6 +76,52 @@ class Form { hasAttachments() { return this.form_fields.some(fieldHasAttachment) } + + /** + * Gets the encrypted values of the responses. Only applicable to + * Storage Mode forms. + */ + _getEncryptedContent() { + if (this.responseMode === 'encrypt') { + return formsg.crypto.encrypt(this._getResponses(), this.publicKey) + } + return null + } + + /** + * Method to abstract away edge cases for submission responses in email vs encrypt mode + */ + _getResponsesForSubmission() { + if (this.responseMode === 'encrypt') { + // Edge case: We still send mobile and email fields to the server in plaintext + // even with end-to-end encryption in order to support SMS and email autoreplies + return this._getResponses() + .filter((item) => ['mobile', 'email'].includes(item.fieldType)) + .map((item) => { + return _(item) + .pick(['fieldType', '_id', 'answer', 'signature']) + .omitBy(_.isNull) + .value() + }) + } else return this._getResponses() + } + + /** + * Method to determine what to POST to the backend submission endpoint. + * Does not include captcha verification. + */ + async getSubmissionContent() { + const submissionContent = { + attachments: await this._getAttachments(), + isPreview: this.isPreview, + responses: this._getResponsesForSubmission(), + } + if (this.responseMode === 'encrypt') { + submissionContent.encryptedContent = this._getEncryptedContent() + submissionContent.version = ENCRYPT_VERSION + } + return submissionContent + } } module.exports = Form From 904afff55646dfc7e0824155ac1a304d9ba168b2 Mon Sep 17 00:00:00 2001 From: Yuan Ruo Date: Tue, 26 Jan 2021 14:32:43 +0800 Subject: [PATCH 6/9] refactor/polyfills: browser detection, other fixes for performance gains (#1039) * refactor: move polyfills used in main JS thread into a single file * doc: add explanations for each * feat(performance): only IE11 should receive existing polyfills * docs(browser-detection): add reminder that isInternetExplorer() is not safe to call from a WebWorker context * refactor(worker): only polyfill if Promise is unavailable. * fix: avoid transpiling polyfills since they pneed to monkey patch native APIs * fix(regenerator-runtime): babel transpilation expects library to be available in global namespace * refactor(ndjson-stream): remove polyfills and document code better * fix: remove misleading comments * refactor: convert from browser detection to feature detection Co-authored-by: Yuanruo Liang --- src/public/main.js | 11 ----- .../modules/forms/helpers/ndjsonStream.js | 19 +++------ src/public/polyfills.js | 40 +++++++++++++++++-- webpack.common.js | 1 + webpack.prod.js | 3 +- 5 files changed, 44 insertions(+), 30 deletions(-) diff --git a/src/public/main.js b/src/public/main.js index b3d4c4ec5a..d2ea90362e 100644 --- a/src/public/main.js +++ b/src/public/main.js @@ -1,21 +1,10 @@ 'use strict' -require('core-js/stable') -require('regenerator-runtime/runtime') require('./polyfills') -const textEncoding = require('text-encoding') const Sentry = require('@sentry/browser') const { Angular: AngularIntegration } = require('@sentry/integrations') -if (!window['TextDecoder']) { - window['TextDecoder'] = textEncoding.TextDecoder -} - -if (!window['TextEncoder']) { - window['TextEncoder'] = textEncoding.TextEncoder -} - // Define module dependencies (without ngSentry) const moduleDependencies = [ 'ui.select', diff --git a/src/public/modules/forms/helpers/ndjsonStream.js b/src/public/modules/forms/helpers/ndjsonStream.js index 4c3e929df5..4df867a980 100644 --- a/src/public/modules/forms/helpers/ndjsonStream.js +++ b/src/public/modules/forms/helpers/ndjsonStream.js @@ -1,26 +1,17 @@ /* eslint-disable camelcase */ 'use strict' -// Polyfills -const streams = require('web-streams-polyfill') -const textEncoding = require('text-encoding') +// Modified fork of https://github.com/canjs/can-ndjson-stream to enqueue +// the string immediately without a JSON.parse() step, as the stream payload +// is to be decrypted by the decryption worker. -// Use polyfill if it does not exist -if (!window['TextDecoder']) { - window['TextDecoder'] = textEncoding.TextDecoder -} - -// Modified fork of https://github.com/canjs/can-ndjson-stream to use polyfilled -// ReadableStream so it works in ie11. - -// Also modified to return the string immediately instead of parsing since the -// string is to be passed into a decryption worker. +// Note that this code assumes a polyfill of TextDecoder is available to run in IE11. const ndjsonStream = function (response) { // For cancellation var is_reader var cancellationRequest = false - return new streams.ReadableStream({ + return new ReadableStream({ start: function (controller) { var reader = response.getReader() is_reader = reader diff --git a/src/public/polyfills.js b/src/public/polyfills.js index 31177667ee..f97242e3e5 100644 --- a/src/public/polyfills.js +++ b/src/public/polyfills.js @@ -1,4 +1,36 @@ -require('web-streams-polyfill') -require('text-encoding') -require('whatwg-fetch') -require('abortcontroller-polyfill/dist/polyfill-patch-fetch') +/** + * Runtime polyfill injection with feature detection to make app work on older browsers. + */ + +// core-js polyfills crucial, commonly used functions expected to be in the standard library +// e.g. Promise, Object.values, Array.prototype.includes +// The stable option includes ES and web standards. See link for more info: +// https://github.com/zloirock/core-js/blob/master/docs/2019-03-19-core-js-3-babel-and-a-look-into-the-future.md +require('core-js/stable') + +// Runtime library for async and generator functions. Babel transpilation expects this to be in +// the target namespace, and it also relies on core-js, so this cannot be removed +// until IE11 support is dropped. +require('regenerator-runtime/runtime') + +// Web Streams API has varying levels of support amongst different browsers +// See https://caniuse.com/streams +if (!window.ReadableStream || !window.WritableStream) { + require('web-streams-polyfill') +} + +// For IE11, Opera Mini +// https://caniuse.com/?search=fetch +if (!window.fetch) { + require('whatwg-fetch') // fetch API + require('abortcontroller-polyfill/dist/polyfill-patch-fetch') +} + +// For IE11, Opera Mini +// https://caniuse.com/?search=TextEncoder +if (!window.TextDecoder || !window.TextEncoder) { + // TextEncoder and TextDecoder + const textEncoding = require('text-encoding') + window['TextDecoder'] = textEncoding.TextDecoder + window['TextEncoder'] = textEncoding.TextEncoder +} diff --git a/webpack.common.js b/webpack.common.js index 0babd2a907..c5dfee45f2 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -25,6 +25,7 @@ module.exports = [ }, { test: /\.worker\.js$/, + // Don't transpile polyfills exclude: /@babel(?:\/|\\{1,2})runtime|core-js/, use: [ { diff --git a/webpack.prod.js b/webpack.prod.js index 25acd45b6e..7120f80fb0 100644 --- a/webpack.prod.js +++ b/webpack.prod.js @@ -35,7 +35,8 @@ module.exports = [ rules: [ { test: /\.js$/, - exclude: /@babel(?:\/|\\{1,2})runtime|core-js/, + // Don't transpile polyfills + exclude: /@babel(?:\/|\\{1,2})runtime|core-js|web-streams-polyfill|whatwg-fetch|abortcontroller-polyfill|text-encoding/, use: { loader: 'babel-loader', }, From cdb7355a4ff694b07b6f4811f59059d9741f91b5 Mon Sep 17 00:00:00 2001 From: Antariksh Date: Tue, 26 Jan 2021 14:38:13 +0800 Subject: [PATCH 7/9] chore: bump version to 4.55.0 --- CHANGELOG.md | 13 +++++++++++++ package-lock.json | 2 +- package.json | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b98353505..60f4071381 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,20 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v4.55.0](https://github.com/opengovsg/FormSG/compare/v4.54.0...v4.55.0) + +- refactor/polyfills: browser detection, other fixes for performance gains [`#1039`](https://github.com/opengovsg/FormSG/pull/1039) +- refactor/submit-form: reduce possibility of race conditions in submit-form controller [`#1045`](https://github.com/opengovsg/FormSG/pull/1045) +- fix: upgrade mongoose from 5.11.9 to 5.11.10 [`#1047`](https://github.com/opengovsg/FormSG/pull/1047) +- fix(deps): bump angular-translate-loader-partial from 2.18.3 to 2.18.4 [`#1037`](https://github.com/opengovsg/FormSG/pull/1037) +- chore(deps-dev): bump webpack from 4.43.0 to 4.46.0 [`#1035`](https://github.com/opengovsg/FormSG/pull/1035) +- chore: merge 4.54.0 into develop [`#1032`](https://github.com/opengovsg/FormSG/pull/1032) +- refactor: Minor refactoring of the submissions client factory [`#1040`](https://github.com/opengovsg/FormSG/pull/1040) + #### [v4.54.0](https://github.com/opengovsg/FormSG/compare/v4.53.0...v4.54.0) +> 20 January 2021 + - fix(deps): bump @sentry/integrations from 5.29.2 to 5.30.0 [`#1028`](https://github.com/opengovsg/FormSG/pull/1028) - chore(deps-dev): bump csv-parse from 4.14.2 to 4.15.0 [`#1027`](https://github.com/opengovsg/FormSG/pull/1027) - build/typescript-4.1.3: TypeScript version upgrade [`#1029`](https://github.com/opengovsg/FormSG/pull/1029) @@ -29,6 +41,7 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - chore(deps-dev): bump husky from 4.3.6 to 4.3.7 [`#1006`](https://github.com/opengovsg/FormSG/pull/1006) - chore(deps-dev): bump @types/validator from 13.1.2 to 13.1.3 [`#1001`](https://github.com/opengovsg/FormSG/pull/1001) - chore: merge 4.53.0 back to develop [`#1005`](https://github.com/opengovsg/FormSG/pull/1005) +- chore: bump version to 4.54.0 [`bbd8e0b`](https://github.com/opengovsg/FormSG/commit/bbd8e0b8af934df8b4d5032f945f4fba7a36eacb) #### [v4.53.0](https://github.com/opengovsg/FormSG/compare/v4.52.1...v4.53.0) diff --git a/package-lock.json b/package-lock.json index dc4c288a9b..0af783e6dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "FormSG", - "version": "4.54.0", + "version": "4.55.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 47069cbd2a..7cd4b338ab 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "FormSG", "description": "Form Manager for Government", - "version": "4.54.0", + "version": "4.55.0", "homepage": "https://form.gov.sg", "authors": [ "FormSG " From 089fa17ff5135bd18357a58303cd70dc9244cf3a Mon Sep 17 00:00:00 2001 From: Antariksh Date: Tue, 26 Jan 2021 15:55:35 +0800 Subject: [PATCH 8/9] build: empty commit to trigger build From b9a26765b90d6cbfdf7abfdb6c01c6a1d81b009b Mon Sep 17 00:00:00 2001 From: Antariksh Mahajan Date: Tue, 26 Jan 2021 16:24:28 +0800 Subject: [PATCH 9/9] chore: log encrypt sub ID with request meta (#1049) --- .../encrypt-submissions.server.controller.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/app/controllers/encrypt-submissions.server.controller.js b/src/app/controllers/encrypt-submissions.server.controller.js index 639213f127..9614791b86 100644 --- a/src/app/controllers/encrypt-submissions.server.controller.js +++ b/src/app/controllers/encrypt-submissions.server.controller.js @@ -66,6 +66,11 @@ function onEncryptSubmissionFailure(err, req, res, submission) { */ exports.saveResponseToDb = function (req, res, next) { const { form, formData, attachmentData } = req + const logMeta = { + action: 'saveResponseToDb', + formId: form._id, + ...createReqMeta(req), + } const { verified } = res.locals let attachmentMetadata = new Map() let attachmentUploadPromises = [] @@ -95,10 +100,7 @@ exports.saveResponseToDb = function (req, res, next) { .catch((err) => { logger.error({ message: 'Attachment upload error', - meta: { - action: 'saveResponseToDb', - ...createReqMeta(req), - }, + meta: logMeta, error: err, }) return onEncryptSubmissionFailure(err, req, res, submission) @@ -121,6 +123,13 @@ exports.saveResponseToDb = function (req, res, next) { return submission.save() }) .then((savedSubmission) => { + logger.info({ + message: 'Saved submission to MongoDB', + meta: { + ...logMeta, + submissionId: savedSubmission._id, + }, + }) req.submission = savedSubmission return next() })