feat: add example K8s deployment configurations (#68)

example-k8s-deployment/README.md

@@ -0,0 +1,17 @@
+# k8s deployment example
+
+## How to deploy
+
+- Deploy infrastructure {IAM+DB} using tooling included for each AWS environment such as Test, UAT, Prod. Code is in `infrastructure` folder
+- Then deploy k8s using Helm. This example doesn't impose opinion on how Helm is being run. Code is in `deployment` folder
+
+
+## Assumptions
+
+- K8s cluster is based on EKS
+- K8s deployment is using Helm
+- K8s pods can assume IAM role setup in this example (This is out of scope for this example though)
+
+## Credits
+
+It is created by Song.Jin at Xero.

example-k8s-deployment/deployment/README.md

@@ -0,0 +1,15 @@
+# README
+
+This folder contains config files for Helm Chart "Generic-services".
+
+When using, ensure to load both config files `common.yaml` and `[environment].yaml`. Common.yaml contains common configs across all envs, while [environment].yaml contains environment-specific configs.
+
+## Assumptions
+
+- Helm value `image.tag` and `image.repository` has been injected by your Helm tooling
+
+## Intentional omissions
+
+- This example doesn't impose opinion on how Docker image for Pact Broker has been built and published.
+
+- It is assumed that config `pact-broker-db-connection-string` has been setup in AWS Secret-Manager and been made available as env-var for k8s pod, following this format `postgres://pact_broker_user:pact_broker_password@pact_broker_db_host/pact_broker`. This example doesn't want to impose opinion on how to retrieve Secret-Manager secret `DB password` as part of secret-management scheme for k8s, as different organisations have their own secret-management plan already in place.

example-k8s-deployment/deployment/common.yaml

@@ -0,0 +1,50 @@
+autoscaling:
+  enabled: true
+  minReplicas: 2
+  maxReplicas: 10
+  targetCPUUtilizationPercentage: 50
+  targetMemoryUtilizationPercentage: 50
+replicaCount: 2
+maxUnavailable: 1
+# it should have been configured by build server in order to keep single-source-of-truth
+# image:
+#   tag: latest
+#   repository: foo.bar.com/pact-broker
+containerPort: 9292
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx
+  path: /
+healthcheck:
+  path: /diagnostic/status/heartbeat
+  initialDelaySeconds: 3
+  periodSeconds: 10
+resources:
+  requests:
+    cpu: 250m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+name: pact-broker
+
+# https://github.com/pact-foundation/pact_broker/blob/a727da80982603a73af5fe518afe58e6b994f4fe/docs/CONFIGURATION.md
+env:
+  # without this config, Puma the web server will confuse with application port config
+  # https://github.com/pact-foundation/pact-broker-docker/blob/ee2f93db03f04103378e83e6359a961c14a63f07/README.md#port
+  - name: PACT_BROKER_PORT_ENVIRONMENT_VARIABLE_NAME
+    value: PACT_BROKER_APPLICATION_PORT
+  - name: PACT_BROKER_APPLICATION_PORT
+    value: "9292"
+  - name: PACT_BROKER_DATABASE_ADAPTER
+    value: postgres
+  - name: PACT_BROKER_DATABASE_URL
+    value: "[SENSITIVE STRING]pact-broker-db-connection-string" # IMPLIES param should have been set by infra code when creating the DB. postgres://pact_broker_user:pact_broker_password@pact_broker_db_host/pact_broker
+  - name: PACT_BROKER_PUBLIC_HEARTBEAT
+    value: "true"
+
+# setting security context so that non-root user access AWS resources by reading AWS token file var/run/secrets/eks.amazonaws.com/serviceaccount/token
+podSecurityContext:
+  # user id for the existing user in Docker image
+  fsGroup: 101

example-k8s-deployment/deployment/test.yaml

@@ -0,0 +1,8 @@
+ingress:
+  hosts:
+    - pact-broker.payments.foo-test.com
+
+iamRole: 'arn:aws:iam::[test aws account]:role/pact-broker'
+
+extraEnvMap:
+  PACT_ENVIRONMENT: Test

example-k8s-deployment/infrastructure/README.md

@@ -0,0 +1,12 @@
+# README
+
+## How to deploy:
+
+Run `./scripts/deploy_cfn.sh iam $ENVIRONMENT && ./scripts/deploy_cfn.sh db $ENVIRONMENT`
+
+Script "deploy_cfn" utilises Docker-compose to run a Cloudformation tooling `Stackup`.
+
+## Assumptions:
+
+- K8s is based on AWS EKS service. (Hence the OIDC setup in IAM role
+- AWS assumed role session is being represented by 3 env-vars {AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN}

example-k8s-deployment/infrastructure/db/envs/common.yaml

@@ -0,0 +1 @@
+Uuid: 123

example-k8s-deployment/infrastructure/db/envs/prod.yaml

@@ -0,0 +1,13 @@
+dbKmsKey: "arn:aws:kms:us-east-1:123:alias/prod-rds-encryption"
+appDataTierSubnets: "subnet-1,subnet-2,subnet-3"
+tpzCIDR: "172.16.8.0/23"
+vpcID: "vpc-123"
+
+AllocatedStorage: "50"
+AvailabilityZone: "us-east-1c"
+DBInstanceClass: "db.r6g.large"
+EngineVersion: "13.3"
+MaxAllocatedStorage: "200"
+PreferredBackupWindow: "09:00-09:30"
+PreferredMaintenanceWindow: "Sat:10:30-Sat:11:00"
+DeletionProtection: "false"

example-k8s-deployment/infrastructure/db/envs/test.yaml

@@ -0,0 +1,13 @@
+dbKmsKey: "arn:aws:kms:us-east-1:123:alias/test-rds-encryption"
+appDataTierSubnets: "subnet-1,subnet-2,subnet-3"
+tpzCIDR: "172.16.8.0/23"
+vpcID: "vpc-123"
+
+AllocatedStorage: "50"
+AvailabilityZone: "us-east-1c"
+DBInstanceClass: "db.r6g.large"
+EngineVersion: "13.3"
+MaxAllocatedStorage: "200"
+PreferredBackupWindow: "09:00-09:30"
+PreferredMaintenanceWindow: "Sat:10:30-Sat:11:00"
+DeletionProtection: "false"

example-k8s-deployment/infrastructure/db/envs/uat.yaml

@@ -0,0 +1,13 @@
+dbKmsKey: "arn:aws:kms:us-east-1:123:alias/uat-rds-encryption"
+appDataTierSubnets: "subnet-1,subnet-2,subnet-3"
+tpzCIDR: "172.16.8.0/23"
+vpcID: "vpc-123"
+
+AllocatedStorage: "50"
+AvailabilityZone: "us-east-1c"
+DBInstanceClass: "db.r6g.large"
+EngineVersion: "13.3"
+MaxAllocatedStorage: "200"
+PreferredBackupWindow: "09:00-09:30"
+PreferredMaintenanceWindow: "Sat:10:30-Sat:11:00"
+DeletionProtection: "false"

example-k8s-deployment/infrastructure/db/template.yaml

@@ -0,0 +1,138 @@
+---
+AWSTemplateFormatVersion: "2010-09-09"
+Description: "This stack includes postgres DB that is supporting the Pact Broker application"
+
+Parameters:
+  dbKmsKey:
+    Description: "KMS key for encrypting the DB"
+    Type: String
+  ssmKmsKey:
+    Description: "KMS key for encrypting secrets"
+    Type: String
+  Uuid:
+    Description: "uuid"
+    Type: String
+  appDataTierSubnets:
+    Description: "subnets for running DB in foo bar corp"
+    Type: String
+  tpzCIDR:
+    Description: "CIDR range for TPZ. This CIDR allows traffic from k8s pod"
+    Type: String
+  vpcID:
+    Description: "vpc for SG"
+    Type: String
+  AllocatedStorage:
+    Type: String
+  AvailabilityZone:
+    Type: String
+  DBInstanceClass:
+    Type: String
+  DBName:
+    Type: String
+  EngineVersion:
+    Type: String
+  MaxAllocatedStorage:
+    Type: String
+  PreferredBackupWindow:
+    Type: String
+  PreferredMaintenanceWindow:
+    Type: String
+  DeletionProtection:
+    Type: String
+
+Resources:
+  pactBrokerSG:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: SG for PactBroker
+      GroupName: packBrokerSG
+      SecurityGroupIngress:
+      - IpProtocol: tcp
+        FromPort: 5432
+        ToPort: 5432
+        CidrIp: !Ref tpzCIDR
+      VpcId: !Ref vpcID
+
+  pactBrokerDbSubnetGrp:
+    Type: AWS::RDS::DBSubnetGroup
+    Properties:
+      DBSubnetGroupDescription: db subnet group for pact broker
+      DBSubnetGroupName: packBrokerDbSubnetGrp
+      SubnetIds: !Split [",",!Ref appDataTierSubnets]
+
+  #This is a Secret resource with a randomly generated password in its SecretString JSON.
+  MyRDSInstanceRotationSecret:
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Description: This is pact-broker db instance creds
+      Name: "pact-broker-db-creds"
+      GenerateSecretString:
+        SecretStringTemplate: '{"username": "postgres"}'
+        GenerateStringKey: password
+        PasswordLength: 16
+        ExcludeCharacters: "\"@/\\"
+        ExcludePunctuation: true
+      KmsKeyId: !Ref ssmKmsKey
+      Tags:
+        - Key: uuid
+          Value: !Ref Uuid
+
+  pactBrokerDb:
+    Type: AWS::RDS::DBInstance
+    Properties:
+      AllocatedStorage: !Ref AllocatedStorage
+      AllowMajorVersionUpgrade: false
+      AutoMinorVersionUpgrade: false
+      AvailabilityZone: !Ref AvailabilityZone
+      BackupRetentionPeriod: 7
+      CACertificateIdentifier: "rds-ca-2019"
+      CopyTagsToSnapshot: true
+      DBInstanceClass: !Ref DBInstanceClass
+      DBName: !Ref DBName
+      DBSubnetGroupName: !Ref pactBrokerDbSubnetGrp
+      DeleteAutomatedBackups: true
+      DeletionProtection: !Ref DeletionProtection
+      EnableCloudwatchLogsExports:
+        - postgresql
+        - upgrade
+      EnableIAMDatabaseAuthentication: false
+      EnablePerformanceInsights: true
+      Engine: postgres
+      EngineVersion: !Ref EngineVersion
+      # Iops: 500 # db size * 10, no IOPS whilst using gp2 type storage
+      KmsKeyId: !Ref dbKmsKey
+      MasterUsername: !Sub "{{resolve:secretsmanager:${MyRDSInstanceRotationSecret}::username}}"
+      MasterUserPassword: !Sub "{{resolve:secretsmanager:${MyRDSInstanceRotationSecret}::password}}"
+      MaxAllocatedStorage: !Ref MaxAllocatedStorage
+      MultiAZ: false
+      PerformanceInsightsKMSKeyId: !Ref dbKmsKey
+      PerformanceInsightsRetentionPeriod: 7
+      Port: 5432
+      PreferredBackupWindow: !Ref PreferredBackupWindow
+      PreferredMaintenanceWindow: !Ref PreferredMaintenanceWindow
+      PubliclyAccessible: false
+      StorageEncrypted: true
+      StorageType: GP2
+      Tags:
+        - Key: uuid
+          Value: !Ref Uuid
+      UseDefaultProcessorFeatures: true
+      VPCSecurityGroups:
+        - !Ref pactBrokerSG
+
+  RDSInstanceConnectionString:
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Description: "This is pact-broker db instance connection string. postgres://pact_broker_user:pact_broker_password@pact_broker_db_host:5432/pact_broker"
+      Name: "pact-broker-db-connection-string"
+      SecretString: !Sub
+        - "postgres://${pact_broker_user}:${pact_broker_password}@${pact_broker_db_host}:5432/${DBName}"
+        - pact_broker_user: !Sub "{{resolve:secretsmanager:${MyRDSInstanceRotationSecret}::username}}"
+          pact_broker_password: !Sub "{{resolve:secretsmanager:${MyRDSInstanceRotationSecret}::password}}"
+          pact_broker_db_host: !GetAtt pactBrokerDb.Endpoint.Address
+          DBName: !Ref DBName
+      KmsKeyId: !Ref ssmKmsKey
+      Tags:
+        - Key: uuid
+          Value: !Ref Uuid
+

example-k8s-deployment/infrastructure/deploy_cfn.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+# This is a templated script to be run by build server
+stack=$1
+env=$2
+
+docker-compose run --rm stackup-"$env" pact-broker-"$stack" up \
+  -t infrastructure/"$stack"/template.yaml \
+  -p infrastructure/"$stack"/envs/common.yaml \
+  -p infrastructure/"$stack"/envs/"$env".yaml

example-k8s-deployment/infrastructure/docker-compose.yaml

@@ -0,0 +1,28 @@
+version: "3.7"
+
+services:
+  stackup-base:
+    image: realestate/stackup:1.7.1
+    working_dir: /app
+    volumes:
+      - ".:/app"
+    entrypoint: stackup
+    environment:
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
+
+  stackup-test:
+    extends: stackup-base
+    environment:
+      - AWS_DEFAULT_REGION=ap-southeast-2
+
+  stackup-UAT:
+    extends: stackup-base
+    environment:
+      - AWS_DEFAULT_REGION=us-west-2
+
+  stackup-Prod:
+    extends: stackup-base
+    environment:
+      - AWS_DEFAULT_REGION=us-east-1

example-k8s-deployment/infrastructure/iam/envs/common.yaml

@@ -0,0 +1,4 @@
+k8sNamespace: "foobar"
+kotahiUuid: 123
+# name to the service account, this has to match app-name used for k8s service account
+serviceAccount: pact-broker

example-k8s-deployment/infrastructure/iam/envs/prod.yaml

@@ -0,0 +1,2 @@
+K8sIdpString: "123"
+ssmKmsKey: "arn:aws:kms:us-east-1:123:key/123"

example-k8s-deployment/infrastructure/iam/envs/test.yaml

@@ -0,0 +1,2 @@
+K8sIdpString: "123"
+ssmKmsKey: "arn:aws:kms:ap-southeast-2:123:key/123"

example-k8s-deployment/infrastructure/iam/envs/uat.yaml

@@ -0,0 +1,2 @@
+K8sIdpString: "123"
+ssmKmsKey: "arn:aws:kms:us-west-2:123:key/123"