test-environment.yml

name: Create Environment
run-name: Creating ${{ github.event.inputs.deployment_name }} by @${{ github.actor }}

on:
  # Ability to execute on demand
  workflow_dispatch:
    inputs:
      deployment_name:
        type: string
        description: |
          Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123'
        required: true
      serverless_mode:
        description: "Deploy a serverless project instead of an ESS deployment"
        type: boolean
        required: true
        default: false
      elk-stack-version:
        required: true
        description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
        default: "8.13.3"
        type: string
      ess-region:
        required: true
        description: "Elastic Cloud deployment region"
        default: "gcp-us-west2"
        type: string
      docker-image-override:
        required: false
        description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
        type: string
      run-sanity-tests:
          description: "Run sanity tests after provision"
          default: false
          type: boolean
      run-ui-sanity-tests:
          description: "Run UI sanity tests after provision"
          default: false
          type: boolean
      kibana_ref:
        description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
        required: false
        default: "main"
        type: string
      expiration_days:
        description: "Number of days until environment expiration"
        required: false
        default: 5
        type: string
      ec-api-key:
        type: string
        description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
        required: false
  workflow_call:
    inputs:
      deployment_name:
        description: Name of the deployment to create
        type: string
        required: true
      serverless_mode:
        description: "Deploy a serverless project instead of an ESS deployment"
        type: boolean
        required: true
        default: false
      elk-stack-version:
        required: true
        description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
        default: "8.13.3"
        type: string
      ess-region:
        required: true
        description: "Elastic Cloud deployment region"
        default: "gcp-us-west2"
        type: string
      docker-image-override:
        required: false
        description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
        type: string
      run-sanity-tests:
        description: "Run sanity tests after provision"
        default: false
        type: boolean
      run-ui-sanity-tests:
          description: "Run UI sanity tests after provision"
          default: false
          type: boolean
      kibana_ref:
        description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
        required: false
        default: "main"
        type: string
      expiration_days:
        description: "Number of days until environment expiration"
        required: false
        default: 5
        type: string
      ec-api-key:
        type: string
        description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
        required: false
      infra-type:
        description: "Type of infrastructure to create"
        type: string
        required: false
        default: "cis"
    outputs:
      s3-bucket:
        description: "Terraform state s3 bucket folder"
        value: ${{ jobs.Deploy.outputs.deploy-s3-bucket }}
      cnvm-stack-name:
        description: "AWS CNVM integration stack name"
        value: ${{ jobs.Deploy.outputs.aws-cnvm-stack-name }}

env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_REGION: "eu-west-1"
  WORKING_DIR: deploy/test-environments
  INTEGRATIONS_SETUP_DIR: tests/integrations_setup
  AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
  GCP_ZONE: "us-central1-a"
  AZURE_DEFAULT_TAGS: "division=engineering org=security team=cloud-security-posture project=test-environments owner=${{ github.actor }}"
  TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
  TF_VAR_gcp_service_account_json: ${{ secrets.GCP_AGENT_CREDENTIALS }}

jobs:
  Deploy:
    runs-on: ubuntu-20.04
    timeout-minutes: 120
    defaults:
      run:
        working-directory: ${{ env.WORKING_DIR }}
    env:
      TF_VAR_ess_region: ${{ inputs.ess-region }}
      DEPLOYMENT_NAME: ${{ inputs.deployment_name }}
      TF_VAR_serverless_mode: ${{ inputs.serverless_mode }}
      TEST_AGENTLESS: ${{ inputs.serverless_mode }}
      S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
      S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra"
      DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }}
      CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack"
    # Add "id-token" with the intended permissions.
    permissions:
      contents: 'read'
      id-token: 'write'
    outputs:
      deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }}
      aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4

      - name: Init Hermit
        run: ./bin/hermit env -r >> $GITHUB_ENV
        working-directory: ./

      - name: Check Deployment Name
        run: |
          deployment_name="${{ inputs.deployment_name }}"

          # Check length
          if [ ${#deployment_name} -gt 20 ]; then
            echo "error: Deployment name is too long (max 20 characters)"
            exit 1
          fi

          # Check pattern required for cloud deployment
          if ! [[ $deployment_name =~ ^[a-z][-a-z0-9]*$ ]]; then
            echo "error: Deployment name doesn't match the required pattern [a-z][-a-z0-9]*"
            exit 1
          fi

      - name: Mask Sensitive Data
        if: inputs.ec-api-key != ''
        run: |
          ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
          echo "::add-mask::$ec_api_key"
          echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV

      - name: Process Stack Version
        id: remove-commit-hash
        run: |
          # Extract the stack version
          stack_version="${{ inputs.elk-stack-version }}"

          echo "TF_VAR_stack_version=$stack_version" >> $GITHUB_ENV
          echo "STACK_VERSION=$stack_version" >> $GITHUB_ENV

          # Handle BC versions with commit hash (e.g. 8.11.0-1234567890)
          if [[ $stack_version =~ -[a-f0-9]+ ]]; then
            cleaned_version=$(echo $stack_version | awk -F"-" '{print $1}')

            # Versions with commit hash are not allowed for EC regular deployments and should be modified
            # EC module resource:
            # ec_deployment.deployment.version is required attribute and should be in format 8.x.y | 8.x.y-SNAPSHOT
            # Therefore, we need to modify the version in the env variable
            echo "TF_VAR_stack_version=$cleaned_version" >> $GITHUB_ENV

            # env variable STACK_VERSION is used in sanity tests for findings validation
            # findings are saved with version without commit hash
            # therefore, we need to modify the version in the env variable
            echo "STACK_VERSION=$cleaned_version" >> $GITHUB_ENV

            # TF_VAR_pin_version is used to override stack docker images
            # for BC versions with commit hash
            # This version will be used to override the docker images
            # elasticsearch.config.docker_image
            # kibana.config.docker_image
            # integrations_server.config.docker_image
            echo "TF_VAR_pin_version=$stack_version" >> $GITHUB_ENV
          fi

      - name: Init Enrollment Token
        run: |
          enrollment_token="init"
          echo "::add-mask::$enrollment_token"
          echo "ENROLLMENT_TOKEN=$enrollment_token" >> $GITHUB_ENV

      - name: Init Infra Type
        id: init-infra-type
        env:
          INPUT_INFRA_TYPE: ${{ inputs.infra-type }}
        run: |
          if [[ -z "${INPUT_INFRA_TYPE}" ]]; then
            echo "INFRA_TYPE=cis" >> $GITHUB_ENV
          else
            echo "INFRA_TYPE=$INPUT_INFRA_TYPE" >> $GITHUB_ENV
          fi

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - name: Install Poetry
        run: |
          curl -sSL https://install.python-poetry.org | python3 -
          poetry --version

      - name: Install Fleet & Tests Dependencies
        id: fleet-and-tests-deps
        working-directory: ./tests
        run: |
          poetry install

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4.0.1
        with:
          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ env.AWS_REGION }}

      - id: azure-auth
        name: Azure login
        uses: azure/login@v2
        with:
          creds: ${{ secrets.AZURE_CREDENTIALS }}

      - id: google-auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}

      - name: Update Vars
        run: |
          echo "TF_VAR_gcp_project_id=$GCP_PROJECT" >> $GITHUB_ENV
          echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV

      - name: Deploy ELK Cloud Stack
        id: elk-stack
        uses: ./.github/actions/elk-stack
        with:
          deployment-name: ${{ env.DEPLOYMENT_NAME }}
          serverless-mode: ${{ env.TEST_AGENTLESS }}
          elk-stack-version: ${{ env.STACK_VERSION }}
          ess-region: ${{ env.TF_VAR_ess_region }}
          ec-api-key: ${{ env.TF_VAR_ec_api_key }}
          docker-image-version-override: ${{ env.TF_VAR_pin_version }}
          env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
          tag-project: ${{ github.actor }}
          tag-owner: ${{ github.actor }}

      - name: Upload environment info
        id: upload-state
        if: always()
        env:
          S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
          EXPIRATION_DAYS: ${{ inputs.expiration_days }}
        run: |
          echo "s3-bucket-folder=${S3_BUCKET}" >> $GITHUB_OUTPUT
          echo "aws-cnvm-stack=${CNVM_STACK_NAME}" >> $GITHUB_OUTPUT
          python3 ../../.ci/scripts/create_env_config.py
          aws s3 cp "./env_config.json" "${S3_BUCKET}/env_config.json"

      - name: Update Stack Vars
        env:
          STACK_ES_USER: ${{ steps.elk-stack.outputs.es-user }}
          STACK_ES_PASSWORD: ${{ steps.elk-stack.outputs.es-password }}
          STACK_KIBANA_URL: ${{ steps.elk-stack.outputs.kibana-url }}
          STACK_ES_URL: ${{ steps.elk-stack.outputs.es-url }}
        run: |
          echo "ES_USER=$STACK_ES_USER" >> $GITHUB_ENV
          echo "ES_PASSWORD=$STACK_ES_PASSWORD" >> $GITHUB_ENV
          echo "KIBANA_URL=$STACK_KIBANA_URL" >> $GITHUB_ENV
          echo "ES_URL=$STACK_ES_URL" >> $GITHUB_ENV

      - name: Summary
        if: success()
        run: |
          summary="Kibana URL: $KIBANA_URL"
          bucket_name="$S3_BASE_BUCKET"
          bucket_name="${bucket_name#s3://}"
          s3_bucket_link="[creds and keys](https://s3.console.aws.amazon.com/s3/buckets/$bucket_name)"
          summary=$(cat <<-EOF
          Kibana URL: [kibana]($KIBANA_URL)
          Environment Details: $s3_bucket_link
          EOF
          )
          echo "$summary" >> $GITHUB_STEP_SUMMARY
          echo "$summary" # Print the summary to the workflow log

      - name: Deploy CDR Integrations
        id: cdr-integrations
        if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cis' }}
        uses: ./.github/actions/cdr
        with:
          deployment-name: ${{ env.DEPLOYMENT_NAME }}
          aws-region: ${{ env.AWS_REGION }}
          gcp-project-id: ${{ env.GCP_PROJECT }}
          gcp-service-account-json: ${{ secrets.GCP_AGENT_CREDENTIALS }}
          aws-cloudtrail-s3-bucket: ${{ secrets.CLOUDTRAIL_S3 }}
          azure-eventhub-connection-string: ${{ secrets.AZURE_EVENTHUB_CONNECTION_STRING }}
          azure-storage-account-key: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
          env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
          es-user: ${{ steps.elk-stack.outputs.es-user }}
          es-password: ${{ steps.elk-stack.outputs.es-password }}
          kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
          elk-stack-version: ${{ env.STACK_VERSION }}
          azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
          tag-project: ${{ github.actor }}
          tag-owner: ${{ github.actor }}

      - name: Deploy CIS Integrations
        id: cis-integrations
        if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
        uses: ./.github/actions/cis
        with:
          deployment-name: ${{ env.DEPLOYMENT_NAME }}
          cnvm-stack-name: ${{ env.CNVM_STACK_NAME }}
          cspm-gcp-zone: ${{ env.GCP_ZONE }}
          cspm-azure-creds: ${{ secrets.AZURE_CREDENTIALS }}
          cspm-azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
          stack-enrollment-token: ${{ env.ENROLLMENT_TOKEN }}
          env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
          es-user: ${{ steps.elk-stack.outputs.es-user }}
          es-password: ${{ steps.elk-stack.outputs.es-password }}
          kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
          test-agentless: ${{ env.TEST_AGENTLESS }}
          tag-project: ${{ github.actor }}
          tag-owner: ${{ github.actor }}

      - name: Wait for agents to enroll
        id: wait-for-agents
        working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }}
        run: |
          poetry run python ./agents_enrolled.py

      - name: Run Sanity checks
        if: ${{ success() && inputs.run-sanity-tests == true && env.INFRA_TYPE != 'cdr' }}
        working-directory: ./tests
        run: |
          poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4

      - name: Run UI Sanity checks (Kibana)
        uses: ./.github/actions/kibana-ftr
        if: ${{ success() && inputs.run-ui-sanity-tests == true && env.INFRA_TYPE != 'cdr' }}
        with:
          test_kibana_url: ${{ steps.elk-stack.outputs.test-kibana-url }}
          test_es_url: ${{ steps.elk-stack.outputs.test-es-url }}
          es_version: ${{ env.STACK_VERSION }}
          kibana_ref: ${{ inputs.kibana_ref }}

      - name: Create Slack Payload
        if: always()
        id: prepare-slack-data
        working-directory: ./
        env:
          WORKFLOW: "${{ github.workflow }}"
          RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          GITHUB_ACTOR: "${{ github.actor }}"
          ESS_TYPE: ${{ inputs.serverless_mode }}
          JOB_STATUS: "${{ job.status }}"
          S3_BUCKET: "${{ env.S3_BUCKET_URL }}?region=${{ env.AWS_REGION }}&prefix=${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}/"
        run: |
          python3 ./.ci/scripts/prepare_slack_data.py

      - name: Send Slack Notification
        uses: ./.github/actions/slack-notification
        if: always()
        continue-on-error: true
        with:
          vault-url: ${{ secrets.VAULT_ADDR }}
          vault-role-id: ${{ secrets.CSP_VAULT_ROLE_ID }}
          vault-secret-id: ${{ secrets.CSP_VAULT_SECRET_ID }}
          slack-payload: ${{ steps.prepare-slack-data.outputs.payload }}