[YOMA-16] Automate Deploy to Stage and Prod (#679)

* Tweak CICD * Don't trigger on tag * Trigger on Release * On Github Release - deploy to Prod * On Push to default branch - deploy to Stage * Otherwise, deploy to Dev * Rename `continuous-integration.yml` to `cicd.yml` * Clean up some Helmfile, tweak Deploy conditions * Remove unused Helmfile values * Yoma API Post Install doesn't need to be configured in Helmfile * Standardize refering to git ref in Yoma API Helm * `didx-xyz/yoma` is a public repo, we don't need to use Github PAT to download the repo in init-containers * Remove no-longer needed `Set GIT_REF` step in deploy job * Deploy everything on Release * Otherwise, if changes are detected * Otherwise, if action was manually triggered * Build and E2E on Release * Tweak Deploy Concurrency Group
didx-xyz · Apr 5, 2024 · 785f8cf · 785f8cf
1 parent 0eeeb19
commit 785f8cf
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 84 deletions.
diff --git a/.github/workflows/continuous-integration.yml → .github/workflows/cicd.yml b/.github/workflows/continuous-integration.yml → .github/workflows/cicd.yml
@@ -4,14 +4,13 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - "master"
-    tags:
-      - "v*"
+      - master
   pull_request:
     branches:
-      - "master"
+      - master
   release:
-    types: [released, prereleased]
+    types:
+      - released
 
 env:
   REGISTRY: ghcr.io/${{ github.repository_owner }}
@@ -148,10 +147,20 @@ jobs:
             changes:
               - '${{ matrix.context }}/**'
               - '${{ matrix.helm }}/**'
+      - name: Should build?
+        id: should-run
+        run: |-
+          if [ "${{ github.event_name }}" = "release" ] || \
+              [ "${{ steps.filter.outputs.changes }}" = "true" ] || \
+              [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo run=true >> $GITHUB_OUTPUT
+          else
+            echo run=false >> $GITHUB_OUTPUT
+          fi
       - uses: docker/setup-buildx-action@v3
-        if: steps.filter.outputs.changes == 'true' || github.event_name == 'workflow_dispatch'
+        if: steps.should-run.outputs.run == 'true'
       - uses: docker/login-action@v3
-        if: steps.filter.outputs.changes == 'true' || github.event_name == 'workflow_dispatch'
+        if: steps.should-run.outputs.run == 'true'
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -169,7 +178,7 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
       - uses: docker/build-push-action@v5
-        if: steps.filter.outputs.changes == 'true' || github.event_name == 'workflow_dispatch'
+        if: steps.should-run.outputs.run == 'true'
         with:
           platforms: linux/amd64 # linux/arm64/v8 is a little too slow right now
           context: ${{ matrix.context }}
@@ -219,11 +228,12 @@ jobs:
       - name: Should e2e run?
         id: should-run
         run: |-
-          if [ "${{ steps.filter.outputs.api }}" = "true" ] || \
+          if [ "${{ github.event_name }}" = "release" ] || \
+              [ "${{ steps.filter.outputs.api }}" = "true" ] || \
               [ "${{ steps.filter.outputs.web }}" = "true" ] || \
               [ "${{ steps.filter.outputs.keycloak }}" = "true" ] || \
               [ "${{ steps.filter.outputs.cypress }}" = "true" ] || \
-              [ "${{ github.event_name == 'workflow_dispatch' }}" = "true" ]; then
+              [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
              echo run=true >> $GITHUB_OUTPUT
           else
             echo run=false >> $GITHUB_OUTPUT
@@ -326,12 +336,12 @@ jobs:
       TAG: ${{ needs.build.outputs.image_version }}
 
     concurrency:
-      group: ${{ github.workflow }}-deploy-${{ matrix.project }}-${{ github.ref }}
+      group: ${{ github.workflow }}-deploy
       cancel-in-progress: false
     environment:
       name: ${{
-        (github.event_name == 'release' && github.event.action == 'released') && 'prod' ||
-        (github.event_name == 'release' && github.event.action == 'prereleased') && 'stage' ||
+        github.event_name == 'release' && 'prod' ||
+        (github.event_name == 'push' && github.event.repository.default_branch == github.ref_name) && 'stage' ||
         'dev'
         }}
       url: ${{ vars.PUBLIC_URL }}
@@ -360,7 +370,7 @@ jobs:
           if [ "${{ steps.filter.outputs.api }}" = "true" ] || \
               [ "${{ steps.filter.outputs.web }}" = "true" ] || \
               [ "${{ steps.filter.outputs.keycloak }}" = "true" ] || \
-              [ "${{ github.event_name == 'workflow_dispatch' }}" = "true" ]; then
+              [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
              echo run=true >> $GITHUB_OUTPUT
           else
             echo run=false >> $GITHUB_OUTPUT
@@ -383,55 +393,59 @@ jobs:
         with:
           authkey: ${{ secrets.TAILSCALE_AUTHKEY }}
           version: ${{ env.TAILSCALE_VERSION }}
-      - name: Set GIT_REF Environment Variable
-        run: |
-          if [ "${{ github.event_name }}" == "pull_request" ]; then
-            echo "GIT_REF=${{ github.head_ref }}" >> $GITHUB_ENV
-          else
-            echo "GIT_REF=$(echo ${{ github.ref }} | sed 's/refs\/heads\///')" >> $GITHUB_ENV
-          fi
       # Diff on PR draft, otherwise Apply
       - name: Helmfile Apply/Diff Keycloak
-        if: steps.filter.outputs.keycloak == 'true' || github.event_name == 'workflow_dispatch'
+        if: (
+          github.event_name == 'release' ||
+          steps.filter.outputs.keycloak == 'true' ||
+          github.event_name == 'workflow_dispatch'
+          )
         uses: helmfile/[email protected]
         with:
           helmfile-args: |
             ${{ (github.event_name == 'pull_request' && github.event.pull_request.draft) && 'diff' || 'apply' }} \
               --environment ${{ vars.ENVIRONMENT }} \
               --selector app=keycloak \
-              --set config-cli.init.ref=${{ github.sha }} \
-              --set keycloak.themes.ref=${{ github.sha }} \
-              --set postInstallHook.ref=${{ github.sha }}
+              --set config-cli.init.ref=${{ github.event_name == 'release' && github.ref_name || github.sha }} \
+              --set keycloak.themes.ref=${{ github.event_name == 'release' && github.ref_name || github.sha }} \
+              --set postInstallHook.ref=${{ github.event_name == 'release' && github.ref_name || github.sha }}
           helmfile-version: ${{ env.HELMFILE_VERSION }}
           helm-version: ${{ env.HELM_VERSION }}
           helm-plugins: |
             https://github.com/databus23/helm-diff,
             https://github.com/jkroepke/helm-secrets,
             https://github.com/aslafy-z/helm-git
       - name: Helmfile Apply/Diff API
-        if: steps.filter.outputs.api == 'true' || github.event_name == 'workflow_dispatch'
+        if: (
+          github.event_name == 'release' ||
+          steps.filter.outputs.api == 'true' ||
+          github.event_name == 'workflow_dispatch'
+          )
         uses: helmfile/[email protected]
         with:
           helmfile-args: |
             ${{ (github.event_name == 'pull_request' && github.event.pull_request.draft) && 'diff' || 'apply' }} \
               --environment ${{ vars.ENVIRONMENT }} \
               --selector app=yoma-api \
-              --state-values-set gitref=${{ env.GIT_REF }}
+              --set postInstallHook.ref=${{ github.event_name == 'release' && github.ref_name || github.sha }}
           helmfile-version: ${{ env.HELMFILE_VERSION }}
           helm-version: ${{ env.HELM_VERSION }}
           helm-plugins: |
             https://github.com/databus23/helm-diff,
             https://github.com/jkroepke/helm-secrets,
             https://github.com/aslafy-z/helm-git
       - name: Helmfile Apply/Diff Web
-        if: steps.filter.outputs.web == 'true' || github.event_name == 'workflow_dispatch'
+        if: (
+          github.event_name == 'release' ||
+          steps.filter.outputs.web == 'true' ||
+          github.event_name == 'workflow_dispatch'
+          )
         uses: helmfile/[email protected]
         with:
           helmfile-args: |
             ${{ (github.event_name == 'pull_request' && github.event.pull_request.draft) && 'diff' || 'apply' }} \
               --environment ${{ vars.ENVIRONMENT }} \
-              --selector app=yoma-web \
-              --state-values-set gitref=${{ env.GIT_REF }}
+              --selector app=yoma-web
           helmfile-version: ${{ env.HELMFILE_VERSION }}
           helm-version: ${{ env.HELM_VERSION }}
           helm-plugins: |

diff --git a/helm/keycloak/values.yaml b/helm/keycloak/values.yaml
@@ -122,17 +122,10 @@ postInstallHook:
         - |-
           cd /scripts
           wget -qO - \
-            --header="Authorization: token ${GITHUB_PAT}"\
             --header="Accept:application/vnd.github.v3.raw" \
             https://api.github.com/repos/didx-xyz/yoma/tarball/{{ .Values.postInstallHook.ref }} | tar xz
           cp -v ./didx-xyz-yoma-*/src/keycloak/scripts/users.sh /scripts/
           rm -rf ./didx-xyz-yoma-*
-      env:
-        - name: GITHUB_PAT
-          valueFrom:
-            secretKeyRef:
-              name: keycloak-github-pat
-              key: github-pat
       volumeMounts:
         - name: scripts
           mountPath: /scripts

diff --git a/helm/yoma-api/conf/dev/values.yaml b/helm/yoma-api/conf/dev/values.yaml
@@ -100,21 +100,12 @@ postInstallHook:
         - -c
         - |-
           cd /init
-          wget -O - \
-            --header="Authorization: token ${GITHUB_PAT}"\
+          wget -qO - \
             --header="Accept:application/vnd.github.v3.raw" \
-            https://api.github.com/repos/didx-xyz/yoma/tarball/${GIT_REF} | tar xz
+            https://api.github.com/repos/didx-xyz/yoma/tarball/{{ .Values.postInstallHook.ref }} | tar xz
           cp -v ./didx-xyz-yoma-*/src/api/cicd/scripts/postgressql-init/* /init/
           chmod +x /init/post.sh
           rm -rvf ./didx-xyz-yoma-*
-      env:
-        - name: GITHUB_PAT
-          valueFrom:
-            secretKeyRef:
-              name: keycloak-github-pat
-              key: github-pat
-        - name: GIT_REF
-          value: "master"
       volumeMounts:
         - name: init
           mountPath: /init

diff --git a/helm/yoma-api/templates/post-install.yaml b/helm/yoma-api/templates/post-install.yaml
@@ -21,7 +21,7 @@ spec:
       restartPolicy: Never
       {{- if .Values.postInstallHook.initContainers }}
       initContainers:
-        {{- toYaml .Values.postInstallHook.initContainers | nindent 8 }}
+        {{- tpl (toYaml .Values.postInstallHook.initContainers) $ | nindent 8 }}
       {{- end }}
       containers:
       - name: post-install-job

diff --git a/helm/yoma-api/values.yaml b/helm/yoma-api/values.yaml
@@ -136,6 +136,26 @@ persistence: {}
 
 postInstallHook:
   enabled: false
+  ref: master
+  initContainers:
+    - name: download-sql
+      image: docker.io/busybox:stable
+      imagePullPolicy: IfNotPresent
+      command:
+        - sh
+      args:
+        - -c
+        - |-
+          cd /init
+          wget -qO - \
+            --header="Accept:application/vnd.github.v3.raw" \
+            https://api.github.com/repos/didx-xyz/yoma/tarball/{{ .Values.postInstallHook.ref }} | tar xz
+          cp -v ./didx-xyz-yoma-*/src/api/cicd/scripts/postgressql-init/* /init/
+          chmod +x /init/post.sh
+          rm -rvf ./didx-xyz-yoma-*
+      volumeMounts:
+        - name: init
+          mountPath: /init
 
 readinessProbe:
   enabled: true

diff --git a/helmfile.yaml b/helmfile.yaml
@@ -6,31 +6,25 @@ environments:
     values:
       - namespace: yoma-v3-dev
         arch: amd64
-        lifecycle: spot
         tag: {{ env "TAG" | default "master" }}
         ddInjectEnabled: false
         ddProfilingEnabled: false
-        gitref: master
         postgresEnabled: true
   stage:
     values:
       - namespace: yoma-v3-stage
         arch: amd64
-        lifecycle: on-demand
         tag: {{ env "TAG" | default "master" }}
         ddInjectEnabled: true
         ddProfilingEnabled: false
-        gitref: master
         postgresEnabled: false
   prod:
     values:
       - namespace: yoma-v3-prod
         arch: amd64
-        lifecycle: on-demand
         tag: {{ env "TAG" | default "master" }}
         ddInjectEnabled: true
         ddProfilingEnabled: false
-        gitref: master
         postgresEnabled: false
 ---
 repositories:
@@ -130,35 +124,6 @@ releases:
         podLabels:
           admission.datadoghq.com/enabled: {{ quote .Values.ddInjectEnabled }}
           tags.datadoghq.com/env: v3-{{ .Environment.Name }}
-      - postInstallHook:
-          initContainers:
-            - name: download-sql
-              image: docker.io/busybox:stable
-              imagePullPolicy: IfNotPresent
-              command:
-                - sh
-              args:
-                - -c
-                - |-
-                  cd /init
-                  wget -O - \
-                    --header="Authorization: token ${GITHUB_PAT}"\
-                    --header="Accept:application/vnd.github.v3.raw" \
-                    https://api.github.com/repos/didx-xyz/yoma/tarball/${GIT_REF} | tar xz
-                  cp -v ./didx-xyz-yoma-*/src/api/cicd/scripts/postgressql-init/* /init/
-                  chmod +x /init/post.sh
-                  rm -rvf ./didx-xyz-yoma-*
-              env:
-                - name: GITHUB_PAT
-                  valueFrom:
-                    secretKeyRef:
-                      name: keycloak-github-pat
-                      key: github-pat
-                - name: GIT_REF
-                  value: {{ .Values.gitref }}
-              volumeMounts:
-                - name: init
-                  mountPath: /init
     secrets:
       - ./helm/yoma-api/conf/base/secrets.yaml
       - ./helm/yoma-api/conf/{{ .Environment.Name }}/secrets.yaml