Merge pull request #2019 from LBHackney-IT/di-447-amend-dq-tests-for-… #999
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Terraform ETL | |
| concurrency: deploy-etl | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'terraform/core/**' | |
| - 'terraform/networking/**' | |
| - 'terraform/backend-setup/**' | |
| workflow_dispatch: | |
| inputs: | |
| terraform_infra_import: | |
| description: "Infrastructure terraform import statements" | |
| required: false | |
| terraform_infra_remove: | |
| description: "Infrastructure terraform state rm statements" | |
| required: false | |
| jobs: | |
| test: | |
| uses: ./.github/workflows/test-python-and-lambda.yml | |
| validate: | |
| uses: ./.github/workflows/validate-and-lint-terraform.yml | |
| with: | |
| environment: "stg" | |
| automation_build_url: "https://github.com/LBHackney-IT/data-platform/actions/workflows/data_platform_stg.yml" | |
| build_path: "./terraform/etl" | |
| terraform_state_s3_key_prefix: "data-platform" | |
| terraform_state_file_name: "stg-terraform-etl.tfstate" | |
| secrets: | |
| GOOGLE_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID_STG }} | |
| AWS_DEPLOY_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_DATA_PLATFORM_STG }} | |
| INFRASTRUCTURE_PRIVATE_KEY: ${{ secrets.INFRASTRUCTURE_PRIVATE_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_MOSAIC_PROD_ACCOUNT_ID: ${{ secrets.AWS_MOSAIC_PROD_ACCOUNT_ID }} | |
| AWS_API_ACCOUNT_PROD: ${{ secrets.AWS_API_ACCOUNT_PROD }} | |
| AWS_HACKIT_ACCOUNT_ID: ${{ secrets.AWS_HACKIT_ACCOUNT_ID }} | |
| AWS_DATA_PLATFORM_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_DATA_PLATFORM_DEV }} | |
| AWS_SANDBOX_ACCOUNT_ID: ${{ secrets.AWS_SANDBOX_ACCOUNT_ID }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| AWS_API_VPC_ID: ${{ secrets.AWS_API_VPC_ID }} | |
| AWS_HOUSING_VPC_ID: ${{ secrets.AWS_HOUSING_VPC_ID }} | |
| AWS_MOSAIC_VPC_ID: ${{ secrets.AWS_MOSAIC_VPC_ID }} | |
| AWS_DP_VPC_ID: ${{ secrets.AWS_DP_DEV_VPC_ID }} | |
| GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS_STG }} | |
| COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE: ${{ secrets.COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE }} | |
| PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN: ${{ secrets.PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN }} | |
| TERRAFORM_SECRET_TOKEN: ${{ secrets.TERRAFORM_SECRET_TOKEN }} | |
| pre-production: | |
| needs: ["test", "validate"] | |
| uses: ./.github/workflows/deploy_terraform.yml | |
| with: | |
| terraform_infra_import: ${{ github.event.inputs.terraform_infra_import }} | |
| terraform_infra_remove: ${{ github.event.inputs.terraform_infra_remove }} | |
| environment: "stg" | |
| automation_build_url: "https://github.com/LBHackney-IT/data-platform/actions/workflows/data_platform_stg.yml" | |
| build_path: "./terraform/etl" | |
| terraform_state_s3_key_prefix: "data-platform" | |
| terraform_state_file_name: "stg-terraform-etl.tfstate" | |
| secrets: | |
| GOOGLE_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID_STG }} | |
| AWS_DEPLOY_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_DATA_PLATFORM_STG }} | |
| INFRASTRUCTURE_PRIVATE_KEY: ${{ secrets.INFRASTRUCTURE_PRIVATE_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_API_ACCOUNT_PROD: ${{ secrets.AWS_API_ACCOUNT_PROD }} | |
| AWS_MOSAIC_PROD_ACCOUNT_ID: ${{ secrets.AWS_MOSAIC_PROD_ACCOUNT_ID }} | |
| AWS_HACKIT_ACCOUNT_ID: ${{ secrets.AWS_HACKIT_ACCOUNT_ID }} | |
| AWS_DATA_PLATFORM_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_DATA_PLATFORM_DEV }} | |
| AWS_SANDBOX_ACCOUNT_ID: ${{ secrets.AWS_SANDBOX_ACCOUNT_ID }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| AWS_API_VPC_ID: ${{ secrets.AWS_API_VPC_ID }} | |
| AWS_HOUSING_VPC_ID: ${{ secrets.AWS_HOUSING_VPC_ID }} | |
| AWS_MOSAIC_VPC_ID: ${{ secrets.AWS_MOSAIC_VPC_ID }} | |
| AWS_DP_VPC_ID: ${{ secrets.AWS_DP_DEV_VPC_ID }} | |
| GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS_STG }} | |
| COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE: ${{ secrets.COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE }} | |
| PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN: ${{ secrets.PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN }} | |
| PRODUCTION_FIREWALL_IP: ${{ secrets.PRODUCTION_FIREWALL_IP }} | |
| TERRAFORM_SECRET_TOKEN: ${{ secrets.TERRAFORM_SECRET_TOKEN }} | |
| production: | |
| needs: [ "pre-production" ] | |
| uses: ./.github/workflows/deploy_terraform.yml | |
| with: | |
| terraform_infra_import: ${{ github.event.inputs.terraform_infra_import }} | |
| terraform_infra_remove: ${{ github.event.inputs.terraform_infra_remove }} | |
| environment: "prod" | |
| automation_build_url: "https://github.com/LBHackney-IT/data-platform/actions/workflows/data_platform_prod.yml" | |
| build_path: "./terraform/etl" | |
| terraform_state_s3_key_prefix: "data-platform" | |
| terraform_state_file_name: "prod-terraform-etl.tfstate" | |
| secrets: | |
| GOOGLE_PROJECT_ID: ${{ secrets.GOOGLE_PROJECT_ID_PROD }} | |
| AWS_DEPLOY_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_DATA_PLATFORM_PROD }} | |
| INFRASTRUCTURE_PRIVATE_KEY: ${{ secrets.INFRASTRUCTURE_PRIVATE_KEY }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_API_ACCOUNT_PROD: ${{ secrets.AWS_API_ACCOUNT_PROD }} | |
| AWS_MOSAIC_PROD_ACCOUNT_ID: ${{ secrets.AWS_MOSAIC_PROD_ACCOUNT_ID }} | |
| AWS_HACKIT_ACCOUNT_ID: ${{ secrets.AWS_HACKIT_ACCOUNT_ID }} | |
| AWS_DATA_PLATFORM_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_DATA_PLATFORM_STG }} | |
| AWS_SANDBOX_ACCOUNT_ID: ${{ secrets.AWS_SANDBOX_ACCOUNT_ID }} | |
| AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }} | |
| AWS_API_VPC_ID: ${{ secrets.AWS_API_VPC_ID }} | |
| AWS_HOUSING_VPC_ID: ${{ secrets.AWS_HOUSING_VPC_ID }} | |
| AWS_MOSAIC_VPC_ID: ${{ secrets.AWS_MOSAIC_VPC_ID }} | |
| AWS_DP_VPC_ID: ${{ secrets.AWS_DP_STG_VPC_ID }} | |
| GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS_PROD }} | |
| COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE: ${{ secrets.COPY_LIBERATOR_TO_PRE_PROD_LAMBDA_EXECUTION_ROLE }} | |
| PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN: ${{ secrets.PRE_PRODUCTION_LIBERATOR_DATA_STORAGE_KMS_KEY_ARN }} | |
| PRODUCTION_FIREWALL_IP: ${{ secrets.PRODUCTION_FIREWALL_IP }} | |
| TERRAFORM_SECRET_TOKEN: ${{ secrets.TERRAFORM_SECRET_TOKEN }} |