diff --git a/.github/workflows/publish-image-backup-storage-gold.yml b/.github/workflows/publish-image-backup-storage-gold.yml index f273d798..fa9f6515 100644 --- a/.github/workflows/publish-image-backup-storage-gold.yml +++ b/.github/workflows/publish-image-backup-storage-gold.yml @@ -1,13 +1,7 @@ # https://github.com/bcgov/helm-charts/tree/master/charts/backup-storage#build-the-container-image-using-github-actions -name: Create and publish Backup Storage Docker image Gold +name: Create and publish a devevlopment Backup Storage Image -on: - workflow_dispatch: - inputs: - postgres_version: - description: 'The postgres version' - required: true - options: ['12', '13'] +on: workflow_dispatch env: GITHUB_REGISTRY: ghcr.io @@ -28,7 +22,6 @@ jobs: run: git clone https://github.com/BCDevOps/backup-container.git - name: Replace the dockerfile - if: ${{ github.event.inputs.postgres_version == '13'}} run: cp ./docker/backup-container/* ./backup-container/docker - name: Log in to the GitHub Container registry @@ -43,5 +36,5 @@ jobs: with: context: backup-container/docker push: true - tags: ${{ env.GITHUB_REGISTRY }}/${{ env.IMAGE_NAME }}:postgres-${{ github.event.inputs.postgres_version}} + tags: ${{ env.GITHUB_REGISTRY }}/${{ env.IMAGE_NAME }}:development labels: sso-keycloak-backup diff --git a/.github/workflows/publish-image-backup-storage-test.yml b/.github/workflows/publish-image-backup-storage-test.yml deleted file mode 100644 index 687fe967..00000000 --- a/.github/workflows/publish-image-backup-storage-test.yml +++ /dev/null @@ -1,61 +0,0 @@ -# https://github.com/bcgov/helm-charts/tree/master/charts/backup-storage#build-the-container-image-using-github-actions -name: Create and publish Backup Storage Docker image - -on: - workflow_dispatch: - inputs: - postgres_version: - description: 'The postgres version' - required: true - options: ['12', '13'] - -env: - GITHUB_REGISTRY: ghcr.io - IMAGE_NAME: thegentlemanphysicist/backup-storage - -jobs: - build-and-push-image: - runs-on: ubuntu-20.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout the sso-repos - uses: actions/checkout@v4 - - - name: Checkout backup storage repository - run: git clone https://github.com/BCDevOps/backup-container.git - - - name: Replace the dockerfile - if: ${{ github.event.inputs.postgres_version == '13'}} - run: cp ./docker/backup-container/Dockerfile ./backup-container/docker/Dockerfile - - - name: Log in to the GitHub Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # - name: Extract metadata (tags, labels) for Docker - # id: meta - # uses: docker/metadata-action@v5 - # with: - # images: ${{ env.GITHUB_REGISTRY }}/${{ env.IMAGE_NAME }} - - # - name: Print tags - # run: echo ${{ steps.meta.outputs.tags }} - - # - name: Print labels - # run: echo ${{ steps.meta.outputs.labels }} - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: backup-container/docker - push: true - tags: ${{ env.GITHUB_REGISTRY }}/${{ env.IMAGE_NAME }}:postgres-${{ github.event.inputs.postgres_version}} - # tags: ${{ steps.meta.outputs.tags }} - # labels: postgres-${{ github.event.inputs.postgres_version}} - labels: sso-keycloak-backup diff --git a/.github/workflows/publish-image-backup-storage.yml b/.github/workflows/publish-image-backup-storage.yml deleted file mode 100644 index 214a1091..00000000 --- a/.github/workflows/publish-image-backup-storage.yml +++ /dev/null @@ -1,48 +0,0 @@ -# https://github.com/bcgov/helm-charts/tree/master/charts/backup-storage#build-the-container-image-using-github-actions -name: Create and publish Backup Storage Docker image - -on: - push: - branches: - - main - - dev - tags: - - '*' - paths: - - '.github/workflows/publish-image-backup-storage.yml' - -env: - GITHUB_REGISTRY: ghcr.io - IMAGE_NAME: bcgov/backup-storage - -jobs: - build-and-push-image: - runs-on: ubuntu-20.04 - permissions: - contents: read - packages: write - - steps: - - name: Checkout backup storage repository - run: git clone https://github.com/BCDevOps/backup-container.git - - - name: Log in to the GitHub Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.GITHUB_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.GITHUB_REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: backup-container/docker - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/docker/backup-container/Dockerfile b/docker/backup-container/Dockerfile index 9bff3e44..278f47fb 100644 --- a/docker/backup-container/Dockerfile +++ b/docker/backup-container/Dockerfile @@ -9,7 +9,7 @@ WORKDIR / # Load the backup scripts into the container (must be executable). COPY backup.* / -COPY webhook-template.json / +# COPY webhook-template.json / # ======================================================================================================== # Install go-crond (from https://github.com/webdevops/go-crond) diff --git a/docs/bkp-and-restore-keycloak-db.md b/docs/bkp-and-restore-keycloak-db.md deleted file mode 100644 index 02c5cc6f..00000000 --- a/docs/bkp-and-restore-keycloak-db.md +++ /dev/null @@ -1,62 +0,0 @@ -# Backup and Restore Keycloak Database - -Restoring a db from a backup in the same namespace it was created is documented in the [Backup Container Documentation](https://developer.gov.bc.ca/Backup-Container). However if you need to restore a backup in a different namespace or cluster from it's source, the following approach will work. - -## Set the environment - -```sh -export SOURCE_NAMESPACE= -export DEST_NAMESPACE= -``` - -## Creating a backup - -```sh - oc -n $SOURCE_NAMESPACE exec $(oc -n $SOURCE_NAMESPACE get pod -l "app.kubernetes.io/name=sso-backup-storage" -o custom-columns=":metadata.name") -- ./backup.sh -s -``` - -## Restoring the database - -- Scale down the keycloak pods to 0 - - ```sh - oc scale --replicas=0 deployment sso-keycloak - ``` - -- Follow below steps to restore the database - - ```sh - # copy latest backup to your local folder - # Note: update YYYY-MON-DD_HOUR-MIN-SEC with latest date and time - oc -n $SOURCE_NAMESPACE cp $(oc -n $SOURCE_NAMESPACE get pod -l "app.kubernetes.io/name=sso-backup-storage" -o custom-columns=":metadata.name"):/backups/daily/YYYY-MON-DD/sso-patroni-ssokeycloak_YYYY-MON-DD_HOUR-MIN-SEC.sql.gz /sso-patroni-ssokeycloak.sql.gz - - # copy the latest backup from your local folder to master patroni pod /tmp/backup folder - oc -n $DEST_NAMESPACE cp ./ $(oc -n $DEST_NAMESPACE get pod -l "spilo-role=master" -o custom-columns=":metadata.name"):/tmp/backup - - # ssh to your master patroni pod - oc -n $DEST_NAMESPACE exec -ti $(oc -n $DEST_NAMESPACE get pod -l "spilo-role=master" -o custom-columns=":metadata.name") -- bash - - # extract sql file - gunzip /tmp/backup/sso-patroni-ssokeycloak.sql.gz - - # delete existing database - psql -c "drop database ssokeycloak" - - # create new database - psql -c "create database ssokeycloak" - - # run the sql file on the new database - psql -d ssokeycloak -f /tmp/backup/patroni-spilo-ssokeycloak.sql - ``` - -- Scale up keycloak pods - - ```sh - oc scale --replicas=5 deployment sso-keycloak - ``` - -- After restoration, update the `sso-keycloak-admin` secret in destination namespace using source namespace secret - -## References - -- https://developer.gov.bc.ca/Backup-Container diff --git a/helm/backup-storage/README.md b/helm/backup-storage/README.md index de18c8ac..d5e866ac 100644 --- a/helm/backup-storage/README.md +++ b/helm/backup-storage/README.md @@ -1,29 +1,43 @@ +# The sso-keycloak implementation of the backup container -## **BEFORE RUNNING HELM UPGRADE:** +As part of the backup restore process we can build and host backup-container images that our helm charts can use to deploy the container. However, currently we are using the backup container's hosted images. -Make sure to add the rocket chat webhook to production facing values files. Make sure not to commit this value. +The backup container repo we use is [here](https://github.com/BCDevOps/backup-container). -## Intro to backup container -The current patroni clusters are backud up using a modified version of the platform services backup container. This is due to an issue with the spilo patroni image. +## The verification and restore process -The backup container repo is [here](https://github.com/BCDevOps/backup-container). +Currently running backup restoration and verification againts the patroni cluster raises several errors, meaning the `-I` flag needs to be added to the verification config and restoration process. To verify a backup run: -The modified postgres plugin can be found [here](https://github.com/bcgov/sso-keycloak/blob/dev/docker/backup-container/backup.postgres.plugin). +`./backup.sh -I -v all` -A side effect of this conflict is that we cannot currently verify the daily backups with a test restoration. +To restore from a backup: -## Installing and upgrading backups +`./backup.sh -I -r sso-patroni:5432/ssokeycloak` -These charts can be upgraded using make commands: +Furhter documentation can be found in the backup container's [repos](https://github.com/BCDevOps/backup-container) -`make upgrade NAME=sso-backup NAMESPACE=<>` +## Building the image + +As of May 2024 we are using the backup-container's images directly in our deployments. However the action `.github/workflows/publish-image-backup-storage-gold.yml` allows us create a backup container image with files overridden by those in the folder `sso-keycloak/docker/backup-container/*`. We can tag the image and then use it in our helm chart via the yaml stanza: + +``` +image: + repository: ghcr.io/bcgov/backup-storage + tag: <> + pullPolicy: Always +``` -To restore from the most recent backup, follow the docs `sso-keycloak/docs/bkp-and-restore-keycloak-db.md` +## Deploying the charts +### **BEFORE RUNNING HELM UPGRADE:** -Deprecated method left as an example: rsh into the backup pod in the namespace in question and run: +Make sure to add the rocket chat webhook to production facing values files. Make sure not to commit this value. + +Make certain the image tag reflects the version of the backup container you intend to deploy. -**dev silver production**: +### Installing and upgrading backups -`./backup.sh -r postgres=sso-pgsql-dev-11-patroni:5432/rhsso` +These charts can be upgraded using make commands: + +`make upgrade NAME=sso-backup NAMESPACE=<>` diff --git a/helm/backup-storage/values-e4ca1d-dev-sso-backup.yaml b/helm/backup-storage/values-e4ca1d-dev-sso-backup.yaml new file mode 100644 index 00000000..9eb8d4ca --- /dev/null +++ b/helm/backup-storage/values-e4ca1d-dev-sso-backup.yaml @@ -0,0 +1,37 @@ +nameOverride: sso-backup-storage +fullnameOverride: sso-backup-storage + +image: + repository: bcgovimages/backup-container + tag: 2.8.1 + pullPolicy: Always + +backupConfig: | + sso-patroni:5432/ssokeycloak + 0 1 * * * default ./backup.sh -s + 0 4 * * * default ./backup.sh -I -s -v all + +db: + secretName: sso-patroni-appusers + usernameKey: username-appuser1 + passwordKey: password-appuser1 + +persistence: + backup: + size: 5Gi + +env: + ENVIRONMENT_FRIENDLY_NAME: + value: 'SSO Sandbox Client Dev Backup' + ENVIRONMENT_NAME: + value: e4ca1d-dev + WEBHOOK_URL: +# value: '<>' + secure: true + + DAILY_BACKUPS: + value: '3' + WEEKLY_BACKUPS: + value: '2' + MONTHLY_BACKUPS: + value: '1' diff --git a/helm/backup-storage/values-e4ca1d-prod-sso-backup.yaml b/helm/backup-storage/values-e4ca1d-prod-sso-backup.yaml new file mode 100644 index 00000000..e9c0f63a --- /dev/null +++ b/helm/backup-storage/values-e4ca1d-prod-sso-backup.yaml @@ -0,0 +1,37 @@ +nameOverride: sso-backup-storage +fullnameOverride: sso-backup-storage + +image: + repository: bcgovimages/backup-container + tag: 2.8.1 + pullPolicy: Always + +backupConfig: | + sso-patroni:5432/ssokeycloak + 0 1 * * * default ./backup.sh -s + 0 4 * * * default ./backup.sh -I -s -v all + +db: + secretName: sso-patroni-appusers + usernameKey: username-appuser1 + passwordKey: password-appuser1 + +persistence: + backup: + size: 5Gi + +env: + ENVIRONMENT_FRIENDLY_NAME: + value: 'SSO Sandbox Client Prod Backup' + ENVIRONMENT_NAME: + value: e4ca1d-prod + WEBHOOK_URL: + # value: '<>' + secure: true + + DAILY_BACKUPS: + value: '3' + WEEKLY_BACKUPS: + value: '2' + MONTHLY_BACKUPS: + value: '1' diff --git a/helm/backup-storage/values-e4ca1d-test-sso-backup.yaml b/helm/backup-storage/values-e4ca1d-test-sso-backup.yaml new file mode 100644 index 00000000..4c66bb68 --- /dev/null +++ b/helm/backup-storage/values-e4ca1d-test-sso-backup.yaml @@ -0,0 +1,37 @@ +nameOverride: sso-backup-storage +fullnameOverride: sso-backup-storage + +image: + repository: bcgovimages/backup-container + tag: 2.8.1 + pullPolicy: Always + +backupConfig: | + sso-patroni:5432/ssokeycloak + 0 1 * * * default ./backup.sh -s + 0 4 * * * default ./backup.sh -I -s -v all + +db: + secretName: sso-patroni-appusers + usernameKey: username-appuser1 + passwordKey: password-appuser1 + +persistence: + backup: + size: 5Gi + +env: + ENVIRONMENT_FRIENDLY_NAME: + value: 'SSO Sandbox Client Test Backup' + ENVIRONMENT_NAME: + value: e4ca1d-test + WEBHOOK_URL: + # value: '<>' + secure: true + + DAILY_BACKUPS: + value: '3' + WEEKLY_BACKUPS: + value: '2' + MONTHLY_BACKUPS: + value: '1' diff --git a/helm/backup-storage/values-eb75ad-dev-sso-backup.yaml b/helm/backup-storage/values-eb75ad-dev-sso-backup.yaml index ea71c3fc..c0b6009d 100644 --- a/helm/backup-storage/values-eb75ad-dev-sso-backup.yaml +++ b/helm/backup-storage/values-eb75ad-dev-sso-backup.yaml @@ -2,13 +2,14 @@ nameOverride: sso-backup-storage fullnameOverride: sso-backup-storage image: - repository: ghcr.io/bcgov/backup-storage - tag: v7.6.5-build.27 + repository: bcgovimages/backup-container + tag: 2.8.1 pullPolicy: Always backupConfig: | sso-patroni:5432/ssokeycloak 0 1 * * * default ./backup.sh -s + 0 4 * * * default ./backup.sh -I -s -v all db: secretName: sso-patroni-appusers @@ -17,7 +18,7 @@ db: persistence: backup: - size: 16Gi + size: 20Gi env: ENVIRONMENT_FRIENDLY_NAME: diff --git a/helm/backup-storage/values-eb75ad-prod-sso-backup.yaml b/helm/backup-storage/values-eb75ad-prod-sso-backup.yaml index 71cce65d..229477be 100644 --- a/helm/backup-storage/values-eb75ad-prod-sso-backup.yaml +++ b/helm/backup-storage/values-eb75ad-prod-sso-backup.yaml @@ -2,19 +2,24 @@ nameOverride: sso-backup-storage fullnameOverride: sso-backup-storage image: - repository: ghcr.io/bcgov/backup-storage - tag: v7.6.5-build.27 + repository: bcgovimages/backup-container + tag: 2.8.1 pullPolicy: Always backupConfig: | sso-patroni:5432/ssokeycloak 0 1 * * * default ./backup.sh -s + 0 4 * * * default ./backup.sh -I -s -v all db: secretName: sso-patroni-appusers usernameKey: username-appuser1 passwordKey: password-appuser1 +persistence: + backup: + size: 32Gi + env: ENVIRONMENT_FRIENDLY_NAME: value: 'SSO Gold Client Production Backup' diff --git a/helm/backup-storage/values-eb75ad-test-sso-backup.yaml b/helm/backup-storage/values-eb75ad-test-sso-backup.yaml index e6bd1861..e19e7149 100644 --- a/helm/backup-storage/values-eb75ad-test-sso-backup.yaml +++ b/helm/backup-storage/values-eb75ad-test-sso-backup.yaml @@ -2,19 +2,24 @@ nameOverride: sso-backup-storage fullnameOverride: sso-backup-storage image: - repository: ghcr.io/bcgov/backup-storage - tag: v7.6.5-build.27 + repository: bcgovimages/backup-container + tag: 2.8.1 pullPolicy: Always backupConfig: | sso-patroni:5432/ssokeycloak 0 1 * * * default ./backup.sh -s + 0 4 * * * default ./backup.sh -I -s -v all db: secretName: sso-patroni-appusers usernameKey: username-appuser1 passwordKey: password-appuser1 +persistence: + backup: + size: 5Gi + env: ENVIRONMENT_FRIENDLY_NAME: value: 'SSO Gold Client Test Backup'