From c3ca99f1abeec986b5ba8a8063a43517ffb144e9 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 14 May 2021 00:23:41 -0700 Subject: [PATCH] add downloadAndDecryptAttachments for downloading and decrypting attachments (#62) * add downloadAndDecryptAttachments for downloading and decrypting attachments * add more tests to improve coverage * Update src/crypto.ts Co-authored-by: Antariksh Mahajan * update code per comments * add a backtick to readme.md * address comments by @seaerchin and @mantariksh * update types in README.md * remove extraneous http status check * remove type annotations in readme.md Co-authored-by: Antariksh Mahajan --- README.md | 60 +++++++++++++-- package-lock.json | 28 +++++++ package.json | 2 + spec/crypto.spec.ts | 176 ++++++++++++++++++++++++++++++++++++++++++++ src/crypto.ts | 64 ++++++++++++++++ src/types.ts | 26 +++++++ 6 files changed, 349 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index abab919..f9c206d 100644 --- a/README.md +++ b/README.md @@ -63,13 +63,6 @@ app.post( express.json(), // Decrypt the submission function (req, res, next) { - // `req.body.data` is an object fulfilling the DecryptParams interface. - // interface DecryptParams { - // encryptedContent: EncryptedContent - // version: number - // verifiedContent?: EncryptedContent - // } - /** @type {{responses: FormField[], verified?: Record}} */ const submission = formsg.crypto.decrypt( formSecretKey, // If `verifiedContent` is provided in `req.body.data`, the return object @@ -86,6 +79,42 @@ app.post( } ) +// Example for submissions with attachments +app.post( + '/submissions-attachment', + // Endpoint authentication by verifying signatures + function (req, res, next) { + try { + formsg.webhooks.authenticate(req.get('X-FormSG-Signature'), POST_URI) + // Continue processing the POST body + return next() + } catch (e) { + return res.status(401).send({ message: 'Unauthorized' }) + } + }, + // Parse JSON from raw request body + express.json(), + // Decrypt the submission and attachments + async function (req, res, next) { + const submission = formsg.crypto.decryptWithAttachments( + formSecretKey, + // If `verifiedContent` is provided in `req.body.data`, the return object + // will include a verified key. + req.body.data + ) + + // If the decryption failed at any point, submission will be `null`. + if (submission) { + // Continue processing the submission + + // processSubmission(submission.content) + // processAttachments(submission.attachments) + } else { + // Could not decrypt the submission + } + } +) + app.listen(8080, () => console.log('Running on port 8080')) ``` @@ -103,6 +132,7 @@ The underlying cryptosystem is `x25519-xsalsa20-poly1305` which is implemented b | submissionId | string | Unique response identifier, displayed as 'Response ID' to form respondents | | encryptedContent | string | The encrypted submission in base64. | | created | string | Creation timestamp. | +| attachmentDownloadUrls | Record | (Optional) Records containing field IDs and URLs where encrypted uploaded attachments can be downloaded. | ### Format of Decrypted Submissions @@ -158,6 +188,22 @@ If the decrypted content is the correct shape, then: verified content. **If the verification fails, `null` is returned, even if `decryptParams.encryptedContent` was successfully decrypted.** +### Processing Attachments + +`formsg.crypto.decryptWithAttachments(formSecretKey: string, decryptParams: DecryptParams)` behaves similarly except it will return a `Promise`. + +`DecryptedContentAndAttachments` is an object containing two fields: + - `content`: the standard form decrypted responses (same as the return type of `formsg.crypto.decrypt`) + - `attachments`: A `Record` containing a map of field ids of the attachment fields to a object containing the original user supplied filename and a `Uint8Array` containing the contents of the uploaded file. + +If the contents of any file fails to decrypt or there is a mismatch between the attachments and submission (e.g. the submission doesn't contain the original file name), then `null` will be returned. + +Attachments are downloaded using S3 pre-signed URLs, with a expiry time of *one hour*. You must call `decryptWithAttachments` within this time window, or else the URL to the encrypted files will become invalid. + +Attachments are end-to-end encrypted in the same way as normal form submissions, so any eavesdropper will not be able to view form attachments without your secret key. + +*Warning:* We do not have the ability to scan any attachments for malicious content (e.g. spyware or viruses), so careful handling is neeeded. + ## Verifying Signatures Manually You can use the following information to create a custom solution, although we recommend using this SDK. diff --git a/package-lock.json b/package-lock.json index 7ba0612..bc16074 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1759,6 +1759,14 @@ "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==", "dev": true }, + "axios": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, "babel-jest": { "version": "25.3.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-25.3.0.tgz", @@ -2765,6 +2773,11 @@ "locate-path": "^3.0.0" } }, + "follow-redirects": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.0.tgz", + "integrity": "sha512-0vRwd7RKQBTt+mgu87mtYeofLFZpTas2S9zY+jIeuLJMNvudIgF52nr19q40HOwH5RrhWIPuj9puybzSJiRrVg==" + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -4382,6 +4395,15 @@ "@jest/types": "^25.3.0" } }, + "jest-mock-axios": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/jest-mock-axios/-/jest-mock-axios-4.4.0.tgz", + "integrity": "sha512-MF5MbjIZcv2KCtO6oH/Fmy1sML1LxQoaGyIPRGArDdd9pcsfjWoCmFFUD12GgOTeJw8ChjuVYHN0s8QnhGriQA==", + "dev": true, + "requires": { + "synchronous-promise": "^2.0.15" + } + }, "jest-pnp-resolver": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.1.tgz", @@ -6111,6 +6133,12 @@ "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", "dev": true }, + "synchronous-promise": { + "version": "2.0.15", + "resolved": "https://registry.npmjs.org/synchronous-promise/-/synchronous-promise-2.0.15.tgz", + "integrity": "sha512-k8uzYIkIVwmT+TcglpdN50pS2y1BDcUnBPK9iJeGu0Pl1lOI8pD6wtzgw91Pjpe+RxtTncw32tLxs/R0yNL2Mg==", + "dev": true + }, "terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", diff --git a/package.json b/package.json index 8746c0e..7bd16a3 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "author": "Open Government Products (FormSG)", "license": "MIT", "dependencies": { + "axios": "^0.21.1", "tweetnacl": "^1.0.3", "tweetnacl-util": "^0.15.1" }, @@ -34,6 +35,7 @@ "@types/node": "^13.11.1", "coveralls": "^3.1.0", "jest": "^25.3.0", + "jest-mock-axios": "^4.4.0", "ts-jest": "^25.3.1", "typescript": "^3.8.3" } diff --git a/spec/crypto.spec.ts b/spec/crypto.spec.ts index 3e85857..7b325e7 100644 --- a/spec/crypto.spec.ts +++ b/spec/crypto.spec.ts @@ -1,6 +1,11 @@ +import mockAxios from 'jest-mock-axios' import Crypto from '../src/crypto' import { SIGNING_KEYS } from '../src/resource/signing-keys' +import { + encodeBase64, +} from 'tweetnacl-util' + import { plaintext, ciphertext, @@ -16,7 +21,11 @@ const encryptionPublicKey = SIGNING_KEYS.test.publicKey const signingSecretKey = SIGNING_KEYS.test.secretKey const testFileBuffer = new Uint8Array(Buffer.from('./resources/ogp.svg')) +jest.mock('axios', () => mockAxios) + describe('Crypto', function () { + afterEach(() => mockAxios.reset()) + const crypto = new Crypto({ signingPublicKey: encryptionPublicKey }) const mockVerifiedContent = { @@ -231,4 +240,171 @@ describe('Crypto', function () { }) expect(decryptResult).toBeNull() }) + + it('should be able to download and decrypt an attachment successfully', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + let attachmentPlaintext = plaintext.slice(0) + attachmentPlaintext.push({ + _id: '6e771c946b3c5100240368e5', + question: 'Random file', + fieldType: 'attachment', + answer: 'my-random-file.txt', + }) + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(attachmentPlaintext, publicKey) + + // Encrypt file + const encryptedFile = await crypto.encryptFile(testFileBuffer, publicKey) + const uploadedFile = { + submissionPublicKey: encryptedFile.submissionPublicKey, + nonce: encryptedFile.nonce, + binary: encodeBase64(encryptedFile.binary) + } + + // Act + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) + const decryptedContentWithAttachments = await decryptedFilesPromise + const decryptedFiles = decryptedContentWithAttachments!.attachments + + // Assert + expect(mockAxios.get).toHaveBeenCalledWith('https://some.s3.url/some/encrypted/file', { responseType: 'json' }) + expect(decryptedFiles).toHaveProperty('6e771c946b3c5100240368e5', { filename: 'my-random-file.txt', content: testFileBuffer }) + }) + + it('should be able to handle fields without attachmentDownloadUrls', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(plaintext, publicKey) + + // Act + const decryptedContentWithAttachments = await crypto.decryptWithAttachments(secretKey, { + encryptedContent: ciphertext, + version: INTERNAL_TEST_VERSION, + }) + const decryptedFiles = decryptedContentWithAttachments!.attachments + + // Assert + expect(decryptedFiles).toEqual({}) + }) + + it('should be able to handle corrupted encrypted content', async () => { + // Arrange + const { secretKey } = crypto.generate() + + // Act + const decryptedContents = await crypto.decryptWithAttachments(secretKey, { + encryptedContent: 'bad encrypted content', + version: INTERNAL_TEST_VERSION, + }) + + // Assert + expect(decryptedContents).toBe(null) + }) + + it('should be able to handle corrupted download', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + let attachmentPlaintext = plaintext.slice(0) + attachmentPlaintext.push({ + _id: '6e771c946b3c5100240368e5', + question: 'Random file', + fieldType: 'attachment', + answer: 'my-random-file.txt', + }) + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(attachmentPlaintext, publicKey) + + // Encrypt file + const encryptedFile = await crypto.encryptFile(testFileBuffer, publicKey) + const uploadedFile = { + submissionPublicKey: encryptedFile.submissionPublicKey, + nonce: encryptedFile.nonce, + binary: 'YmFkZW5jcnlwdGVkY29udGVudHM=', // invalid data + } + + // Act + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + mockAxios.mockResponse({ data: { encryptedFile: uploadedFile }}) + const decryptedContents = await decryptedFilesPromise + + // Assert + expect(decryptedContents).toBe(null) + }) + + it('should be able to handle decrypted submission without corresponding attachment field', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + // Encrypt content that is not signed + // Note that plaintext doesn't have any attachment fields + const ciphertext = crypto.encrypt(plaintext, publicKey) + + // Encrypt file + const encryptedFile = await crypto.encryptFile(testFileBuffer, publicKey) + const uploadedFile = { + submissionPublicKey: encryptedFile.submissionPublicKey, + nonce: encryptedFile.nonce, + binary: encodeBase64(encryptedFile.binary) + } + + // Act + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + const decryptedContents = await decryptedFilesPromise + + // Assert + expect(decryptedContents).toBe(null) + }) + + it('should be able to handle axios errors', async () => { + // Arrange + const { publicKey, secretKey } = crypto.generate() + + let attachmentPlaintext = plaintext.slice(0) + attachmentPlaintext.push({ + _id: '6e771c946b3c5100240368e5', + question: 'Random file', + fieldType: 'attachment', + answer: 'my-random-file.txt', + }) + + // Encrypt content that is not signed + const ciphertext = crypto.encrypt(attachmentPlaintext, publicKey) + + // Act + const decryptedFilesPromise = crypto.decryptWithAttachments(secretKey, { + encryptedContent: ciphertext, + attachmentDownloadUrls: { '6e771c946b3c5100240368e5': 'https://some.s3.url/some/encrypted/file' }, + version: INTERNAL_TEST_VERSION, + }) + mockAxios.mockResponse({ + data: {}, + status: 404, + statusText: 'Not Found', + }) + const decryptedContents = await decryptedFilesPromise + + // Assert + expect(mockAxios.get).toHaveBeenCalledWith('https://some.s3.url/some/encrypted/file', { responseType: 'json' }) + expect(decryptedContents).toBe(null) + }) }) diff --git a/src/crypto.ts b/src/crypto.ts index 54cac76..af774b1 100644 --- a/src/crypto.ts +++ b/src/crypto.ts @@ -1,8 +1,13 @@ +import axios from 'axios' import nacl from 'tweetnacl' import { DecryptParams, + DecryptedAttachments, DecryptedContent, + DecryptedContentAndAttachments, + EncryptedAttachmentContent, + EncryptedAttachmentRecords, EncryptedContent, EncryptedFileContent, FormField, @@ -195,4 +200,63 @@ export default class Crypto { decodeBase64(formSecretKey) ) } + + /** + * Decrypts an encrypted submission, and also download and decrypt any attachments alongside it. + * @param formSecretKey Secret key as a base-64 string + * @param decryptParams The params containing encrypted content and information. + * @returns A promise of the decrypted submission, including attachments (if any). Or else returns null if a decryption error decrypting any part of the submission. + * @throws {MissingPublicKeyError} if a public key is not provided when instantiating this class and is needed for verifying signed content. + */ + decryptWithAttachments = async ( + formSecretKey: string, + decryptParams: DecryptParams + ): Promise => { + const decryptedRecords: DecryptedAttachments = {} + const filenames: Record = {} + + const attachmentRecords: EncryptedAttachmentRecords = decryptParams.attachmentDownloadUrls ?? {} + const decryptedContent = this.decrypt(formSecretKey, decryptParams) + if (decryptedContent === null) return null + + // Retrieve all original filenames for attachments for easy lookup + decryptedContent.responses.forEach((response) => { + if (response.fieldType === 'attachment' && response.answer) { + filenames[response._id] = response.answer + } + }) + + const downloadPromises : Array> = [] + for (let fieldId in attachmentRecords) { + // Original name for the file is not found + if (filenames[fieldId] === undefined) return null + + downloadPromises.push( + axios.get(attachmentRecords[fieldId], { responseType: 'json' }) + .then((downloadResponse) => { + const encryptedAttachment: EncryptedAttachmentContent = downloadResponse.data + const encryptedFile: EncryptedFileContent = { + submissionPublicKey: encryptedAttachment.encryptedFile.submissionPublicKey, + nonce: encryptedAttachment.encryptedFile.nonce, + binary: decodeBase64(encryptedAttachment.encryptedFile.binary), + } + + return this.decryptFile(formSecretKey, encryptedFile) + }).then((decryptedFile) => { + if (decryptedFile === null) throw new Error("Attachment decryption failed") + decryptedRecords[fieldId] = { filename: filenames[fieldId], content: decryptedFile } + })) + } + + try { + await Promise.all(downloadPromises) + } catch (e) { + return null + } + + return { + content: decryptedContent, + attachments: decryptedRecords + } + } } diff --git a/src/types.ts b/src/types.ts index 09adafe..96b7e68 100644 --- a/src/types.ts +++ b/src/types.ts @@ -45,10 +45,15 @@ export type FormField = { // ;: export type EncryptedContent = string +// Records containing a map of field IDs to URLs where encrypted +// attachments can be downloaded. +export type EncryptedAttachmentRecords = Record + export interface DecryptParams { encryptedContent: EncryptedContent version: number verifiedContent?: EncryptedContent + attachmentDownloadUrls?: EncryptedAttachmentRecords } export type DecryptedContent = { @@ -56,12 +61,33 @@ export type DecryptedContent = { verified?: Record } +export type DecryptedFile = { + filename: string + content: Uint8Array +} + +// Records containing a map of field IDs to DecryptedFiles. +export type DecryptedAttachments = Record + +export type DecryptedContentAndAttachments = { + content: DecryptedContent + attachments: DecryptedAttachments +} + export type EncryptedFileContent = { submissionPublicKey: string nonce: string binary: Uint8Array } +export type EncryptedAttachmentContent = { + encryptedFile: { + submissionPublicKey: string + nonce: string + binary: string + } +} + // A base-64 encoded cryptographic keypair suitable for curve25519. export type Keypair = { publicKey: string