feat: enable retries for webhooks (#2093)

* build(webhook-retries): create SQS queue in dev mode (1) (#1938)

* feat(webhook-retries): add isRetryEnabled to model (2) (#1939)

* feat(webhook-retries): produce and consume retries (3) (#1940)

* test(webhook-retries): add unit tests for retrieveWebhookInfoById (5) (#1942)

* test(webhook-retries): add unit tests for webhook service (6) (#1943)

* test(webhook-retries): add unit tests for message, producer, consumer (7) (#1988)

* test: add tests for WebhookQueueMessage

* test: add tests for webhook consumer

* test: add tests for WebhookProducer

* test: add tests for important utils

* feat: increase jitter with increasing base interval

* fix: calculate next attempt from time of initial attempt

* feat(webhook-retries): enable toggling retries on frontend (4) (#1941)

* test: make Date.now consistent in producer tests

Dockerfile.development

@@ -26,9 +26,12 @@ RUN apk update && apk upgrade && \
     ttf-freefont \
     tini \
     # Localstack - these are necessary in order to initialise local S3 buckets
+    # jq is a package for easily parsing Localstack health endpoint's JSON output
+    jq \
     py-pip && \
     npm install --quiet node-gyp -g && \
-    pip install awscli-local
+    # [ver1] ensures that the underlying AWS CLI version is also installed
+    pip install awscli-local[ver1]
 
 # Chinese fonts
 RUN echo @edge http://nl.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories && apk add wqy-zenhei@edge
@@ -41,5 +44,5 @@ EXPOSE 5000
 # tini is the init process that will adopt orphaned zombie processes
 # e.g. chromium when launched to create a new PDF
 ENTRYPOINT [ "tini", "--" ]
-# Create local S3 buckets before building the app
-CMD npm run docker-dev
+# Create local AWS resources before building the app
+CMD sh init-localstack.sh && npm run docker-dev

docker-compose.yml

@@ -38,6 +38,7 @@ services:
       - MYINFO_CERT_PATH=./node_modules/@opengovsg/mockpass/static/certs/spcp.crt
       - MYINFO_CLIENT_ID=mockClientId
       - MYINFO_CLIENT_SECRET=mockClientSecret
+      - WEBHOOK_SQS_URL=http://localhost:4566/000000000000/local-webhooks-sqs-main
       - GA_TRACKING_ID
       - SENTRY_CONFIG_URL
       - TWILIO_ACCOUNT_SID
@@ -105,17 +106,11 @@ services:
     depends_on:
       - formsg
     environment:
-      - SERVICES=s3
+      - SERVICES=s3,sqs
       - DATA_DIR=/tmp/localstack/data
-      - ATTACHMENT_S3_BUCKET=local-attachment-bucket
-      - IMAGE_S3_BUCKET=local-image-bucket
-      - LOGO_S3_BUCKET=local-logo-bucket
     volumes:
       - './.localstack:/tmp/localstack'
       - '/var/run/docker.sock:/var/run/docker.sock'
-      # This is where we add scripts to initialise AWS resources.
-      # Docs: https://github.com/localstack/localstack#initializing-a-fresh-instance
-      - './docker-entrypoint-initaws.d:/docker-entrypoint-initaws.d'
     network_mode: 'service:formsg' # reuse formsg service's network stack so that it can resolve localhost:4566 to localstack:4566
 
   maildev:

init-localstack.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Wait for all Localstack services to be ready
+while [[ "$(curl -s -f http://localhost:4566/health | jq '[.services[] == "running"] | all')" != "true" ]]; do
+  sleep 5
+done
+
+# Create SQS queue for webhooks
+# First create dead-letter queue and get its ARN so it can be specified as the DLQ
+# for the main queue. Note that the DLQ name is not an environment variable
+# in the application, as this is configured from the AWS console in production.
+DLQ_NAME=local-webhooks-sqs-deadLetter
+DLQ_URL=$(awslocal sqs create-queue --queue-name $DLQ_NAME | jq --raw-output '.QueueUrl')
+DLQ_ARN=$(awslocal sqs get-queue-attributes --queue-url $DLQ_URL --attribute-names QueueArn | jq --raw-output '.Attributes.QueueArn')
+
+# Show output for all main resources created
+set -x
+
+# For main queue, extract queue name, which is the part of the queue URL after the final "/"
+awslocal sqs create-queue --queue-name ${WEBHOOK_SQS_URL##*/} --attributes '{
+  "ReceiveMessageWaitTimeSeconds": "20",
+  "RedrivePolicy": "{\"deadLetterTargetArn\":\"'"$DLQ_ARN"'\",\"maxReceiveCount\":1}"
+}'
+
+# Create S3 buckets
+awslocal s3 mb s3://$IMAGE_S3_BUCKET
+awslocal s3 mb s3://$LOGO_S3_BUCKET
+awslocal s3 mb s3://$ATTACHMENT_S3_BUCKET
+
+set +x

package.json

@@ -142,6 +142,8 @@
     "slick-carousel": "1.8.1",
     "sortablejs": "~1.13.0",
     "spark-md5": "^3.0.1",
+    "sqs-consumer": "^5.5.0",
+    "sqs-producer": "^2.1.0",
     "text-encoding": "^0.7.0",
     "toastr": "^2.1.4",
     "triple-beam": "^1.3.0",
@@ -154,7 +156,8 @@
     "web-streams-polyfill": "^3.0.3",
     "whatwg-fetch": "^3.6.2",
     "winston": "^3.3.3",
-    "winston-cloudwatch": "^2.5.2"
+    "winston-cloudwatch": "^2.5.2",
+    "zod": "^3.0.0"
   },
   "devDependencies": {
     "@babel/core": "^7.14.3",

src/app/config/feature-manager/types.ts

@@ -79,6 +79,7 @@ export interface IVerifiedFields {
 
 export interface IWebhookVerifiedContent {
   signingSecretKey: string
+  webhookQueueUrl: string
 }
 
 export interface IIntranet {

src/app/config/feature-manager/webhook-verified-content.config.ts

@@ -10,6 +10,12 @@ const webhookVerifiedContentFeature: RegisterableFeature<FeatureNames.WebhookVer
         default: null,
         env: 'SIGNING_SECRET_KEY',
       },
+      webhookQueueUrl: {
+        doc: 'URL of AWS SQS queue for webhook retries',
+        format: String,
+        default: '',
+        env: 'WEBHOOK_SQS_URL',
+      },
     },
   }
 

src/app/constants/timezone.ts

@@ -0,0 +1 @@
+export const TIMEZONE = 'Asia/Singapore'

src/app/models/__tests__/form.server.model.spec.ts

@@ -72,6 +72,7 @@ const FORM_DEFAULTS = {
   permissionList: [],
   webhook: {
     url: '',
+    isRetryEnabled: false,
   },
   status: 'PRIVATE',
   submissionLimit: null,