build(webhook-retries): create SQS queue in dev mode (1) (#1938)

Dockerfile.development

@@ -26,9 +26,12 @@ RUN apk update && apk upgrade && \
     ttf-freefont \
     tini \
     # Localstack - these are necessary in order to initialise local S3 buckets
+    # jq is a package for easily parsing Localstack health endpoint's JSON output
+    jq \
     py-pip && \
     npm install --quiet node-gyp -g && \
-    pip install awscli-local
+    # [ver1] ensures that the underlying AWS CLI version is also installed
+    pip install awscli-local[ver1]
 
 # Chinese fonts
 RUN echo @edge http://nl.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories && apk add wqy-zenhei@edge
@@ -41,5 +44,5 @@ EXPOSE 5000
 # tini is the init process that will adopt orphaned zombie processes
 # e.g. chromium when launched to create a new PDF
 ENTRYPOINT [ "tini", "--" ]
-# Create local S3 buckets before building the app
-CMD npm run docker-dev
+# Create local AWS resources before building the app
+CMD sh init-localstack.sh && npm run docker-dev

docker-compose.yml

@@ -38,6 +38,7 @@ services:
       - MYINFO_CERT_PATH=./node_modules/@opengovsg/mockpass/static/certs/spcp.crt
       - MYINFO_CLIENT_ID=mockClientId
       - MYINFO_CLIENT_SECRET=mockClientSecret
+      - WEBHOOK_SQS_URL=http://localhost:4566/000000000000/local-webhooks-sqs-main
       - GA_TRACKING_ID
       - SENTRY_CONFIG_URL
       - TWILIO_ACCOUNT_SID
@@ -105,17 +106,11 @@ services:
     depends_on:
       - formsg
     environment:
-      - SERVICES=s3
+      - SERVICES=s3,sqs
       - DATA_DIR=/tmp/localstack/data
-      - ATTACHMENT_S3_BUCKET=local-attachment-bucket
-      - IMAGE_S3_BUCKET=local-image-bucket
-      - LOGO_S3_BUCKET=local-logo-bucket
     volumes:
       - './.localstack:/tmp/localstack'
       - '/var/run/docker.sock:/var/run/docker.sock'
-      # This is where we add scripts to initialise AWS resources.
-      # Docs: https://github.com/localstack/localstack#initializing-a-fresh-instance
-      - './docker-entrypoint-initaws.d:/docker-entrypoint-initaws.d'
     network_mode: 'service:formsg' # reuse formsg service's network stack so that it can resolve localhost:4566 to localstack:4566
 
   maildev:

init-localstack.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Wait for all Localstack services to be ready
+while [[ "$(curl -s -f http://localhost:4566/health | jq '[.services[] == "running"] | all')" != "true" ]]; do
+  sleep 5
+done
+
+# Create SQS queue for webhooks
+# First create dead-letter queue and get its ARN so it can be specified as the DLQ
+# for the main queue. Note that the DLQ name is not an environment variable
+# in the application, as this is configured from the AWS console in production.
+DLQ_NAME=local-webhooks-sqs-deadLetter
+DLQ_URL=$(awslocal sqs create-queue --queue-name $DLQ_NAME | jq --raw-output '.QueueUrl')
+DLQ_ARN=$(awslocal sqs get-queue-attributes --queue-url $DLQ_URL --attribute-names QueueArn | jq --raw-output '.Attributes.QueueArn')
+
+# Show output for all main resources created
+set -x
+
+# For main queue, extract queue name, which is the part of the queue URL after the final "/"
+awslocal sqs create-queue --queue-name ${WEBHOOK_SQS_URL##*/} --attributes '{
+  "ReceiveMessageWaitTimeSeconds": "20",
+  "RedrivePolicy": "{\"deadLetterTargetArn\":\"'"$DLQ_ARN"'\",\"maxReceiveCount\":1}"
+}'
+
+# Create S3 buckets
+awslocal s3 mb s3://$IMAGE_S3_BUCKET
+awslocal s3 mb s3://$LOGO_S3_BUCKET
+awslocal s3 mb s3://$ATTACHMENT_S3_BUCKET
+
+set +x