Skip to content

Creating ricky-qa-17bc2 by @animehart #755

Creating ricky-qa-17bc2 by @animehart

Creating ricky-qa-17bc2 by @animehart #755

name: Create Environment
run-name: Creating ${{ github.event.inputs.deployment_name }} by @${{ github.actor }}
on:
# Ability to execute on demand
workflow_dispatch:
inputs:
deployment_name:
type: string
description: |
Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123'
required: true
serverless_mode:
description: "Deploy a serverless project instead of an ESS deployment"
type: boolean
required: true
default: false
elk-stack-version:
required: true
description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
default: "8.13.3"
type: string
ess-region:
required: true
description: "Elastic Cloud deployment region"
default: "gcp-us-west2"
type: string
docker-image-override:
required: false
description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
type: string
run-sanity-tests:
description: "Run sanity tests after provision"
default: false
type: boolean
run-ui-sanity-tests:
description: "Run UI sanity tests after provision"
default: false
type: boolean
kibana_ref:
description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
required: false
default: "main"
type: string
expiration_days:
description: "Number of days until environment expiration"
required: false
default: 5
type: string
ec-api-key:
type: string
description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
required: false
workflow_call:
inputs:
deployment_name:
description: Name of the deployment to create
type: string
required: true
serverless_mode:
description: "Deploy a serverless project instead of an ESS deployment"
type: boolean
required: true
default: false
elk-stack-version:
required: true
description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
default: "8.13.3"
type: string
ess-region:
required: true
description: "Elastic Cloud deployment region"
default: "gcp-us-west2"
type: string
docker-image-override:
required: false
description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
type: string
run-sanity-tests:
description: "Run sanity tests after provision"
default: false
type: boolean
run-ui-sanity-tests:
description: "Run UI sanity tests after provision"
default: false
type: boolean
kibana_ref:
description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
required: false
default: "main"
type: string
expiration_days:
description: "Number of days until environment expiration"
required: false
default: 5
type: string
ec-api-key:
type: string
description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
required: false
infra-type:
description: "Type of infrastructure to create"
type: string
required: false
default: "cis"
outputs:
s3-bucket:
description: "Terraform state s3 bucket folder"
value: ${{ jobs.Deploy.outputs.deploy-s3-bucket }}
cnvm-stack-name:
description: "AWS CNVM integration stack name"
value: ${{ jobs.Deploy.outputs.aws-cnvm-stack-name }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
WORKING_DIR: deploy/test-environments
INTEGRATIONS_SETUP_DIR: tests/integrations_setup
AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
GCP_ZONE: "us-central1-a"
AZURE_DEFAULT_TAGS: "division=engineering org=security team=cloud-security-posture project=test-environments owner=${{ github.actor }}"
TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
TF_VAR_gcp_service_account_json: ${{ secrets.GCP_AGENT_CREDENTIALS }}
jobs:
Deploy:
runs-on: ubuntu-20.04
timeout-minutes: 120
defaults:
run:
working-directory: ${{ env.WORKING_DIR }}
env:
TF_VAR_ess_region: ${{ inputs.ess-region }}
DEPLOYMENT_NAME: ${{ inputs.deployment_name }}
TF_VAR_serverless_mode: ${{ inputs.serverless_mode }}
TEST_AGENTLESS: true
S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra"
DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }}
CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack"
# Add "id-token" with the intended permissions.
permissions:
contents: "read"
id-token: "write"
outputs:
deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }}
aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }}
steps:
- name: Check out the repo
uses: actions/checkout@v4
- name: Init Hermit
run: ./bin/hermit env -r >> $GITHUB_ENV
working-directory: ./
- name: Check Deployment Name
run: |
deployment_name="${{ inputs.deployment_name }}"
# Check length
if [ ${#deployment_name} -gt 20 ]; then
echo "error: Deployment name is too long (max 20 characters)"
exit 1
fi
# Check pattern required for cloud deployment
if ! [[ $deployment_name =~ ^[a-z][-a-z0-9]*$ ]]; then
echo "error: Deployment name doesn't match the required pattern [a-z][-a-z0-9]*"
exit 1
fi
- name: Mask Sensitive Data
if: inputs.ec-api-key != ''
run: |
ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
echo "::add-mask::$ec_api_key"
echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV
- name: Process Stack Version
id: remove-commit-hash
run: |
# Extract the stack version
stack_version="${{ inputs.elk-stack-version }}"
echo "TF_VAR_stack_version=$stack_version" >> $GITHUB_ENV
echo "STACK_VERSION=$stack_version" >> $GITHUB_ENV
# Handle BC versions with commit hash (e.g. 8.11.0-1234567890)
if [[ $stack_version =~ -[a-f0-9]+ ]]; then
cleaned_version=$(echo $stack_version | awk -F"-" '{print $1}')
# Versions with commit hash are not allowed for EC regular deployments and should be modified
# EC module resource:
# ec_deployment.deployment.version is required attribute and should be in format 8.x.y | 8.x.y-SNAPSHOT
# Therefore, we need to modify the version in the env variable
echo "TF_VAR_stack_version=$cleaned_version" >> $GITHUB_ENV
# env variable STACK_VERSION is used in sanity tests for findings validation
# findings are saved with version without commit hash
# therefore, we need to modify the version in the env variable
echo "STACK_VERSION=$cleaned_version" >> $GITHUB_ENV
# TF_VAR_pin_version is used to override stack docker images
# for BC versions with commit hash
# This version will be used to override the docker images
# elasticsearch.config.docker_image
# kibana.config.docker_image
# integrations_server.config.docker_image
echo "TF_VAR_pin_version=$stack_version" >> $GITHUB_ENV
fi
- name: Init Enrollment Token
run: |
enrollment_token="init"
echo "::add-mask::$enrollment_token"
echo "ENROLLMENT_TOKEN=$enrollment_token" >> $GITHUB_ENV
- name: Init Infra Type
id: init-infra-type
env:
INPUT_INFRA_TYPE: ${{ inputs.infra-type }}
run: |
if [[ -z "${INPUT_INFRA_TYPE}" ]]; then
echo "INFRA_TYPE=cis" >> $GITHUB_ENV
else
echo "INFRA_TYPE=$INPUT_INFRA_TYPE" >> $GITHUB_ENV
fi
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.9"
- name: Install Poetry
run: |
curl -sSL https://install.python-poetry.org | python3 -
poetry --version
- name: Install Fleet & Tests Dependencies
id: fleet-and-tests-deps
working-directory: ./tests
run: |
poetry install
- name: Configure AWS credentials
uses: aws-actions/[email protected]
with:
aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
- id: azure-auth
name: Azure login
uses: azure/login@v2
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- id: google-auth
name: Authenticate to Google Cloud
uses: google-github-actions/auth@v2
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Update Vars
run: |
echo "TF_VAR_gcp_project_id=$GCP_PROJECT" >> $GITHUB_ENV
echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV
- name: Deploy ELK Cloud Stack
id: elk-stack
uses: ./.github/actions/elk-stack
with:
deployment-name: ${{ env.DEPLOYMENT_NAME }}
serverless-mode: ${{ env.TF_VAR_serverless_mode }}
elk-stack-version: ${{ env.STACK_VERSION }}
ess-region: ${{ env.TF_VAR_ess_region }}
ec-api-key: ${{ env.TF_VAR_ec_api_key }}
docker-image-version-override: ${{ env.TF_VAR_pin_version }}
env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
tag-project: ${{ github.actor }}
tag-owner: ${{ github.actor }}
- name: Upload environment info
id: upload-state
if: always()
env:
S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
EXPIRATION_DAYS: ${{ inputs.expiration_days }}
run: |
echo "s3-bucket-folder=${S3_BUCKET}" >> $GITHUB_OUTPUT
echo "aws-cnvm-stack=${CNVM_STACK_NAME}" >> $GITHUB_OUTPUT
python3 ../../.ci/scripts/create_env_config.py
aws s3 cp "./env_config.json" "${S3_BUCKET}/env_config.json"
- name: Update Stack Vars
env:
STACK_ES_USER: ${{ steps.elk-stack.outputs.es-user }}
STACK_ES_PASSWORD: ${{ steps.elk-stack.outputs.es-password }}
STACK_KIBANA_URL: ${{ steps.elk-stack.outputs.kibana-url }}
STACK_ES_URL: ${{ steps.elk-stack.outputs.es-url }}
run: |
echo "ES_USER=$STACK_ES_USER" >> $GITHUB_ENV
echo "ES_PASSWORD=$STACK_ES_PASSWORD" >> $GITHUB_ENV
echo "KIBANA_URL=$STACK_KIBANA_URL" >> $GITHUB_ENV
echo "ES_URL=$STACK_ES_URL" >> $GITHUB_ENV
- name: Summary
if: success()
run: |
summary="Kibana URL: $KIBANA_URL"
bucket_name="$S3_BASE_BUCKET"
bucket_name="${bucket_name#s3://}"
s3_bucket_link="[creds and keys](https://s3.console.aws.amazon.com/s3/buckets/$bucket_name)"
summary=$(cat <<-EOF
Kibana URL: [kibana]($KIBANA_URL)
Environment Details: $s3_bucket_link
EOF
)
echo "$summary" >> $GITHUB_STEP_SUMMARY
echo "$summary" # Print the summary to the workflow log
- name: Deploy CDR Integrations
id: cdr-integrations
if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cis' }}
uses: ./.github/actions/cdr
with:
deployment-name: ${{ env.DEPLOYMENT_NAME }}
aws-region: ${{ env.AWS_REGION }}
gcp-project-id: ${{ env.GCP_PROJECT }}
gcp-service-account-json: ${{ secrets.GCP_AGENT_CREDENTIALS }}
aws-cloudtrail-s3-bucket: ${{ secrets.CLOUDTRAIL_S3 }}
azure-eventhub-connection-string: ${{ secrets.AZURE_EVENTHUB_CONNECTION_STRING }}
azure-storage-account-key: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
wiz-client-id: ${{ secrets.WIZ_CLIENT_ID }}
wiz-client-secret: ${{ secrets.WIZ_CLIENT_SECRET }}
wiz-endpoint-url: ${{ secrets.WIZ_ENDPOINT_URL }}
wiz-token-url: ${{ secrets.WIZ_TOKEN_URL }}
env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
es-user: ${{ steps.elk-stack.outputs.es-user }}
es-password: ${{ steps.elk-stack.outputs.es-password }}
kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
elk-stack-version: ${{ env.STACK_VERSION }}
azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
tag-project: ${{ github.actor }}
tag-owner: ${{ github.actor }}
- name: Deploy CIS Integrations
id: cis-integrations
if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
uses: ./.github/actions/cis
with:
deployment-name: ${{ env.DEPLOYMENT_NAME }}
cnvm-stack-name: ${{ env.CNVM_STACK_NAME }}
cspm-gcp-zone: ${{ env.GCP_ZONE }}
cspm-azure-creds: ${{ secrets.AZURE_CREDENTIALS }}
cspm-azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
stack-enrollment-token: ${{ env.ENROLLMENT_TOKEN }}
env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
es-user: ${{ steps.elk-stack.outputs.es-user }}
es-password: ${{ steps.elk-stack.outputs.es-password }}
kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
test-agentless: ${{ env.TEST_AGENTLESS }}
tag-project: ${{ github.actor }}
tag-owner: ${{ github.actor }}
- name: Wait for agents to enroll
id: wait-for-agents
working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }}
run: |
poetry run python ./agents_enrolled.py
- name: Run Sanity checks
if: ${{ success() && inputs.run-sanity-tests == true && env.INFRA_TYPE != 'cdr' }}
working-directory: ./tests
run: |
poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
- name: Run UI Sanity checks (Kibana)
uses: ./.github/actions/kibana-ftr
if: ${{ success() && inputs.run-ui-sanity-tests == true && env.INFRA_TYPE != 'cdr' }}
with:
test_kibana_url: ${{ steps.elk-stack.outputs.test-kibana-url }}
test_es_url: ${{ steps.elk-stack.outputs.test-es-url }}
es_version: ${{ env.STACK_VERSION }}
kibana_ref: ${{ inputs.kibana_ref }}
- name: Create Slack Payload
if: always()
id: prepare-slack-data
working-directory: ./
env:
WORKFLOW: "${{ github.workflow }}"
RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
GITHUB_ACTOR: "${{ github.actor }}"
ESS_TYPE: ${{ inputs.serverless_mode }}
JOB_STATUS: "${{ job.status }}"
S3_BUCKET: "${{ env.S3_BUCKET_URL }}?region=${{ env.AWS_REGION }}&prefix=${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}/"
run: |
python3 ./.ci/scripts/prepare_slack_data.py
- name: Send Slack Notification
uses: ./.github/actions/slack-notification
if: always()
continue-on-error: true
with:
vault-url: ${{ secrets.VAULT_ADDR }}
vault-role-id: ${{ secrets.CSP_VAULT_ROLE_ID }}
vault-secret-id: ${{ secrets.CSP_VAULT_SECRET_ID }}
slack-payload: ${{ steps.prepare-slack-data.outputs.payload }}