Creating dg-dep by @gurevichdmitry #751

Summary
Jobs
- Deploy
Run details
- Usage
- Workflow file

Workflow file for this run

.github/workflows/test-environment.yml at d6fedb3

	name: Create Environment
	run-name: Creating ${{ github.event.inputs.deployment_name }} by @${{ github.actor }}

	on:
	# Ability to execute on demand
	workflow_dispatch:
	inputs:
	deployment_name:
	type: string
	description: \|
	Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123'
	required: true
	serverless_mode:
	description: "Deploy a serverless project instead of an ESS deployment"
	type: boolean
	required: true
	default: false
	elk-stack-version:
	required: true
	description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
	default: "8.13.3"
	type: string
	ess-region:
	required: true
	description: "Elastic Cloud deployment region"
	default: "gcp-us-west2"
	type: string
	docker-image-override:
	required: false
	description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
	type: string
	run-sanity-tests:
	description: "Run sanity tests after provision"
	default: false
	type: boolean
	run-ui-sanity-tests:
	description: "Run UI sanity tests after provision"
	default: false
	type: boolean
	kibana_ref:
	description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
	required: false
	default: "main"
	type: string
	expiration_days:
	description: "Number of days until environment expiration"
	required: false
	default: 5
	type: string
	ec-api-key:
	type: string
	description: "Optional By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
	required: false
	workflow_call:
	inputs:
	deployment_name:
	description: Name of the deployment to create
	type: string
	required: true
	serverless_mode:
	description: "Deploy a serverless project instead of an ESS deployment"
	type: boolean
	required: true
	default: false
	elk-stack-version:
	required: true
	description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT"
	default: "8.13.3"
	type: string
	ess-region:
	required: true
	description: "Elastic Cloud deployment region"
	default: "gcp-us-west2"
	type: string
	docker-image-override:
	required: false
	description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
	type: string
	run-sanity-tests:
	description: "Run sanity tests after provision"
	default: false
	type: boolean
	run-ui-sanity-tests:
	description: "Run UI sanity tests after provision"
	default: false
	type: boolean
	kibana_ref:
	description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
	required: false
	default: "main"
	type: string
	expiration_days:
	description: "Number of days until environment expiration"
	required: false
	default: 5
	type: string
	ec-api-key:
	type: string
	description: "Optional By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key."
	required: false
	infra-type:
	description: "Type of infrastructure to create"
	type: string
	required: false
	default: "cis"
	outputs:
	s3-bucket:
	description: "Terraform state s3 bucket folder"
	value: ${{ jobs.Deploy.outputs.deploy-s3-bucket }}
	cnvm-stack-name:
	description: "AWS CNVM integration stack name"
	value: ${{ jobs.Deploy.outputs.aws-cnvm-stack-name }}

	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	AWS_REGION: "eu-west-1"
	WORKING_DIR: deploy/test-environments
	INTEGRATIONS_SETUP_DIR: tests/integrations_setup
	AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
	GCP_ZONE: "us-central1-a"
	AZURE_DEFAULT_TAGS: "division=engineering org=security team=cloud-security-posture project=test-environments owner=${{ github.actor }}"
	TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
	TF_VAR_gcp_service_account_json: ${{ secrets.GCP_AGENT_CREDENTIALS }}

	jobs:
	Deploy:
	runs-on: ubuntu-20.04
	timeout-minutes: 120
	defaults:
	run:
	working-directory: ${{ env.WORKING_DIR }}
	env:
	TF_VAR_ess_region: ${{ inputs.ess-region }}
	DEPLOYMENT_NAME: ${{ inputs.deployment_name }}
	TF_VAR_serverless_mode: ${{ inputs.serverless_mode }}
	TEST_AGENTLESS: true
	S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
	S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra"
	DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }}
	CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack"
	# Add "id-token" with the intended permissions.
	permissions:
	contents: "read"
	id-token: "write"
	outputs:
	deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }}
	aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }}
	steps:
	- name: Check out the repo
	uses: actions/checkout@v4

	- name: Init Hermit
	run: ./bin/hermit env -r >> $GITHUB_ENV
	working-directory: ./

	- name: Check Deployment Name
	run: \|
	deployment_name="${{ inputs.deployment_name }}"

	# Check length
	if [ ${#deployment_name} -gt 20 ]; then
	echo "error: Deployment name is too long (max 20 characters)"
	exit 1
	fi

	# Check pattern required for cloud deployment
	if ! [[ $deployment_name =~ ^[a-z][-a-z0-9]*$ ]]; then
	echo "error: Deployment name doesn't match the required pattern [a-z][-a-z0-9]*"
	exit 1
	fi

	- name: Mask Sensitive Data
	if: inputs.ec-api-key != ''
	run: \|
	ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
	echo "::add-mask::$ec_api_key"
	echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV

	- name: Process Stack Version
	id: remove-commit-hash
	run: \|
	# Extract the stack version
	stack_version="${{ inputs.elk-stack-version }}"

	echo "TF_VAR_stack_version=$stack_version" >> $GITHUB_ENV
	echo "STACK_VERSION=$stack_version" >> $GITHUB_ENV

	# Handle BC versions with commit hash (e.g. 8.11.0-1234567890)
	if [[ $stack_version =~ -[a-f0-9]+ ]]; then
	cleaned_version=$(echo $stack_version \| awk -F"-" '{print $1}')

	# Versions with commit hash are not allowed for EC regular deployments and should be modified
	# EC module resource:
	# ec_deployment.deployment.version is required attribute and should be in format 8.x.y \| 8.x.y-SNAPSHOT
	# Therefore, we need to modify the version in the env variable
	echo "TF_VAR_stack_version=$cleaned_version" >> $GITHUB_ENV

	# env variable STACK_VERSION is used in sanity tests for findings validation
	# findings are saved with version without commit hash
	# therefore, we need to modify the version in the env variable
	echo "STACK_VERSION=$cleaned_version" >> $GITHUB_ENV

	# TF_VAR_pin_version is used to override stack docker images
	# for BC versions with commit hash
	# This version will be used to override the docker images
	# elasticsearch.config.docker_image
	# kibana.config.docker_image
	# integrations_server.config.docker_image
	echo "TF_VAR_pin_version=$stack_version" >> $GITHUB_ENV
	fi

	- name: Init Enrollment Token
	run: \|
	enrollment_token="init"
	echo "::add-mask::$enrollment_token"
	echo "ENROLLMENT_TOKEN=$enrollment_token" >> $GITHUB_ENV

	- name: Init Infra Type
	id: init-infra-type
	env:
	INPUT_INFRA_TYPE: ${{ inputs.infra-type }}
	run: \|
	if [[ -z "${INPUT_INFRA_TYPE}" ]]; then
	echo "INFRA_TYPE=cis" >> $GITHUB_ENV
	else
	echo "INFRA_TYPE=$INPUT_INFRA_TYPE" >> $GITHUB_ENV
	fi

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.9"

	- name: Install Poetry
	run: \|
	curl -sSL https://install.python-poetry.org \| python3 -
	poetry --version

	- name: Install Fleet & Tests Dependencies
	id: fleet-and-tests-deps
	working-directory: ./tests
	run: \|
	poetry install

	- name: Configure AWS credentials
	uses: aws-actions/[email protected]
	with:
	aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
	aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
	aws-region: ${{ env.AWS_REGION }}

	- id: azure-auth
	name: Azure login
	uses: azure/login@v2
	with:
	creds: ${{ secrets.AZURE_CREDENTIALS }}

	- id: google-auth
	name: Authenticate to Google Cloud
	uses: google-github-actions/auth@v2
	with:
	workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
	service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}

	- name: Update Vars
	run: \|
	echo "TF_VAR_gcp_project_id=$GCP_PROJECT" >> $GITHUB_ENV
	echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV

	- name: Deploy ELK Cloud Stack
	id: elk-stack
	uses: ./.github/actions/elk-stack
	with:
	deployment-name: ${{ env.DEPLOYMENT_NAME }}
	serverless-mode: ${{ env.TF_VAR_serverless_mode }}
	elk-stack-version: ${{ env.STACK_VERSION }}
	ess-region: ${{ env.TF_VAR_ess_region }}
	ec-api-key: ${{ env.TF_VAR_ec_api_key }}
	docker-image-version-override: ${{ env.TF_VAR_pin_version }}
	env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
	tag-project: ${{ github.actor }}
	tag-owner: ${{ github.actor }}

	- name: Upload environment info
	id: upload-state
	if: always()
	env:
	S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
	EXPIRATION_DAYS: ${{ inputs.expiration_days }}
	run: \|
	echo "s3-bucket-folder=${S3_BUCKET}" >> $GITHUB_OUTPUT
	echo "aws-cnvm-stack=${CNVM_STACK_NAME}" >> $GITHUB_OUTPUT
	python3 ../../.ci/scripts/create_env_config.py
	aws s3 cp "./env_config.json" "${S3_BUCKET}/env_config.json"

	- name: Update Stack Vars
	env:
	STACK_ES_USER: ${{ steps.elk-stack.outputs.es-user }}
	STACK_ES_PASSWORD: ${{ steps.elk-stack.outputs.es-password }}
	STACK_KIBANA_URL: ${{ steps.elk-stack.outputs.kibana-url }}
	STACK_ES_URL: ${{ steps.elk-stack.outputs.es-url }}
	run: \|
	echo "ES_USER=$STACK_ES_USER" >> $GITHUB_ENV
	echo "ES_PASSWORD=$STACK_ES_PASSWORD" >> $GITHUB_ENV
	echo "KIBANA_URL=$STACK_KIBANA_URL" >> $GITHUB_ENV
	echo "ES_URL=$STACK_ES_URL" >> $GITHUB_ENV

	- name: Summary
	if: success()
	run: \|
	summary="Kibana URL: $KIBANA_URL"
	bucket_name="$S3_BASE_BUCKET"
	bucket_name="${bucket_name#s3://}"
	s3_bucket_link="[creds and keys](https://s3.console.aws.amazon.com/s3/buckets/$bucket_name)"
	summary=$(cat <<-EOF
	Kibana URL: [kibana]($KIBANA_URL)
	Environment Details: $s3_bucket_link
	EOF
	)
	echo "$summary" >> $GITHUB_STEP_SUMMARY
	echo "$summary" # Print the summary to the workflow log

	- name: Deploy CDR Integrations
	id: cdr-integrations
	if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cis' }}
	uses: ./.github/actions/cdr
	with:
	deployment-name: ${{ env.DEPLOYMENT_NAME }}
	aws-region: ${{ env.AWS_REGION }}
	gcp-project-id: ${{ env.GCP_PROJECT }}
	gcp-service-account-json: ${{ secrets.GCP_AGENT_CREDENTIALS }}
	aws-cloudtrail-s3-bucket: ${{ secrets.CLOUDTRAIL_S3 }}
	azure-eventhub-connection-string: ${{ secrets.AZURE_EVENTHUB_CONNECTION_STRING }}
	azure-storage-account-key: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
	wiz-client-id: ${{ secrets.WIZ_CLIENT_ID }}
	wiz-client-secret: ${{ secrets.WIZ_CLIENT_SECRET }}
	wiz-endpoint-url: ${{ secrets.WIZ_ENDPOINT_URL }}
	wiz-token-url: ${{ secrets.WIZ_TOKEN_URL }}
	env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
	es-user: ${{ steps.elk-stack.outputs.es-user }}
	es-password: ${{ steps.elk-stack.outputs.es-password }}
	kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
	elk-stack-version: ${{ env.STACK_VERSION }}
	azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
	tag-project: ${{ github.actor }}
	tag-owner: ${{ github.actor }}

	- name: Deploy CIS Integrations
	id: cis-integrations
	if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
	uses: ./.github/actions/cis
	with:
	deployment-name: ${{ env.DEPLOYMENT_NAME }}
	cnvm-stack-name: ${{ env.CNVM_STACK_NAME }}
	cspm-gcp-zone: ${{ env.GCP_ZONE }}
	cspm-azure-creds: ${{ secrets.AZURE_CREDENTIALS }}
	cspm-azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
	stack-enrollment-token: ${{ env.ENROLLMENT_TOKEN }}
	env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
	es-user: ${{ steps.elk-stack.outputs.es-user }}
	es-password: ${{ steps.elk-stack.outputs.es-password }}
	kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
	test-agentless: ${{ env.TEST_AGENTLESS }}
	tag-project: ${{ github.actor }}
	tag-owner: ${{ github.actor }}

	- name: Wait for agents to enroll
	id: wait-for-agents
	working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }}
	run: \|
	poetry run python ./agents_enrolled.py

	- name: Run Sanity checks
	if: ${{ success() && inputs.run-sanity-tests == true && env.INFRA_TYPE != 'cdr' }}
	working-directory: ./tests
	run: \|
	poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4

	- name: Run UI Sanity checks (Kibana)
	uses: ./.github/actions/kibana-ftr
	if: ${{ success() && inputs.run-ui-sanity-tests == true && env.INFRA_TYPE != 'cdr' }}
	with:
	test_kibana_url: ${{ steps.elk-stack.outputs.test-kibana-url }}
	test_es_url: ${{ steps.elk-stack.outputs.test-es-url }}
	es_version: ${{ env.STACK_VERSION }}
	kibana_ref: ${{ inputs.kibana_ref }}

	- name: Create Slack Payload
	if: always()
	id: prepare-slack-data
	working-directory: ./
	env:
	WORKFLOW: "${{ github.workflow }}"
	RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
	GITHUB_ACTOR: "${{ github.actor }}"
	ESS_TYPE: ${{ inputs.serverless_mode }}
	JOB_STATUS: "${{ job.status }}"
	S3_BUCKET: "${{ env.S3_BUCKET_URL }}?region=${{ env.AWS_REGION }}&prefix=${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}/"
	run: \|
	python3 ./.ci/scripts/prepare_slack_data.py

	- name: Send Slack Notification
	uses: ./.github/actions/slack-notification
	if: always()
	continue-on-error: true
	with:
	vault-url: ${{ secrets.VAULT_ADDR }}
	vault-role-id: ${{ secrets.CSP_VAULT_ROLE_ID }}
	vault-secret-id: ${{ secrets.CSP_VAULT_SECRET_ID }}
	slack-payload: ${{ steps.prepare-slack-data.outputs.payload }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Creating dg-dep by @gurevichdmitry #751

Workflow file

Creating dg-dep by @gurevichdmitry #751

Jobs

Run details

Workflow file for this run