Skip to content

Creating bc-8-10-1 by @CohenIdo #142

Creating bc-8-10-1 by @CohenIdo

Creating bc-8-10-1 by @CohenIdo #142

name: Create Environment
run-name: Creating ${{ github.event.inputs.deployment_name }} by @${{ github.actor }}
on:
# Ability to execute on demand
workflow_dispatch:
inputs:
deployment_name:
type: string
description: |
Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123'
required: true
ec-api-key:
type: string
description: "Elastic Cloud API KEY"
required: true
elk-stack-version:
required: true
description: "Stack version: For released/BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT"
default: "8.8.0"
ess-region:
required: true
description: "Elastic Cloud deployment region"
default: "gcp-us-west2"
docker-image-override:
required: false
description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
run-sanity-tests:
description: "Run sanity tests after provision"
default: false
type: boolean
cleanup-env:
description: "Cleanup resources after provision"
default: false
type: boolean
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
WORKING_DIR: deploy/test-environments
FLEET_API_DIR: fleet_api/src
AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments"
GCP_DEFAULT_TAGS: "division=engineering,org=security,team=cloud-security-posture,project=test-environments"
jobs:
Deploy:
runs-on: ubuntu-20.04
timeout-minutes: 120
defaults:
run:
working-directory: ${{ env.WORKING_DIR }}
env:
TF_VAR_stack_version: ${{ github.event.inputs.elk-stack-version }}
TF_VAR_ess_region: ${{ github.event.inputs.ess-region }}
DEPLOYMENT_NAME: ${{ github.event.inputs.deployment_name }}
S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
DOCKER_IMAGE_OVERRIDE: ${{ github.event.inputs.docker-image-override }}
STACK_VERSION: ${{ github.event.inputs.elk-stack-version }}
CNVM_STACK_NAME: "${{ github.event.inputs.deployment_name }}-cnvm-sanity-test-stack"
# Add "id-token" with the intended permissions.
permissions:
contents: 'read'
id-token: 'write'
steps:
- name: Check out the repo
uses: actions/checkout@v4
- name: Init Hermit
run: ./bin/hermit env -r >> $GITHUB_ENV
working-directory: ./
- name: Check Deployment Name
run: |
deployment_name="${{ github.event.inputs.deployment_name }}"
# Check length
if [ ${#deployment_name} -gt 20 ]; then
echo "error: Deployment name is too long (max 20 characters)"
exit 1
fi
# Check pattern required for CNVM deployment
if ! [[ $deployment_name =~ ^[a-zA-Z][-a-zA-Z0-9]*$ ]]; then
echo "error: Deployment name doesn't match the required pattern [a-zA-Z][-a-zA-Z0-9]*"
exit 1
fi
- name: Mask Sensitive Data
run: |
ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
echo "::add-mask::$ec_api_key"
echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV
enrollment_token="init"
echo "::add-mask::$enrollment_token"
echo "ENROLLMENT_TOKEN=$enrollment_token" >> $GITHUB_ENV
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Install Poetry
run: |
curl -sSL https://install.python-poetry.org | python3 -
poetry --version
- name: Install Fleet API dependencies
id: fleet-api-deps
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry install
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v2
with:
aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ env.AWS_REGION }}
- name: Set TF_STATE_FOLDER
run: |
echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV
- name: Terraform Init
run: terraform init
- name: Terraform Validate
run: terraform validate
- name: Provision Test Environment (EC + EC2 K8s + EC2 CSPM)
id: apply
if: success()
run: |
terraform apply --auto-approve -var="deployment_name=${{ env.DEPLOYMENT_NAME }}" -var="region=${{ env.AWS_REGION }}"
- name: Set Environment Output
id: env-output
run: |
echo "KIBANA_URL=$(terraform output -raw kibana_url)" >> $GITHUB_ENV
echo "ES_URL=$(terraform output -raw elasticsearch_url)" >> $GITHUB_ENV
echo "ES_USER=$(terraform output -raw elasticsearch_username)" >> $GITHUB_ENV
export ES_PASSWORD=$(terraform output -raw elasticsearch_password)
echo "::add-mask::$ES_PASSWORD"
echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV
export EC2_CSPM=$(terraform output -raw ec2_cspm_ssh_cmd)
echo "::add-mask::$EC2_CSPM"
echo "EC2_CSPM=$EC2_CSPM" >> $GITHUB_ENV
export EC2_KSPM=$(terraform output -raw ec2_kspm_ssh_cmd)
echo "::add-mask::$EC2_KSPM"
echo "EC2_KSPM=$EC2_KSPM" >> $GITHUB_ENV
export EC2_CSPM_KEY=$(terraform output -raw ec2_cspm_key)
echo "::add-mask::$EC2_CSPM_KEY"
echo "EC2_CSPM_KEY=$EC2_CSPM_KEY" >> $GITHUB_ENV
export EC2_KSPM_KEY=$(terraform output -raw ec2_kspm_key)
echo "::add-mask::$EC2_KSPM_KEY"
echo "EC2_KSPM_KEY=$EC2_KSPM_KEY" >> $GITHUB_ENV
export KSPM_PUBLIC_IP=$(terraform output -raw ec2_kspm_public_ip)
echo "::add-mask::$KSPM_PUBLIC_IP"
echo "KSPM_PUBLIC_IP=$KSPM_PUBLIC_IP" >> $GITHUB_ENV
export CSPM_PUBLIC_IP=$(terraform output -raw ec2_cspm_public_ip)
echo "::add-mask::$CSPM_PUBLIC_IP"
echo "CSPM_PUBLIC_IP=$CSPM_PUBLIC_IP" >> $GITHUB_ENV
- name: Upload tf state
if: always()
env:
S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
run: |
aws s3 cp "./terraform.tfstate" "${{ env.S3_BUCKET }}/terraform.tfstate"
aws s3 cp "${{ env.EC2_CSPM_KEY }}" "${{ env.S3_BUCKET }}/cspm.pem"
aws s3 cp "${{ env.EC2_KSPM_KEY }}" "${{ env.S3_BUCKET }}/kspm.pem"
- name: Summary
if: success()
run: |
kibana_url=$(terraform output -raw kibana_url)
summary="Kibana URL: $kibana_url"
bucket_name="${{ env.S3_BASE_BUCKET }}"
bucket_name="${bucket_name#s3://}"
s3_bucket_link="[creds and keys](https://s3.console.aws.amazon.com/s3/buckets/$bucket_name)"
summary=$(cat <<-EOF
Kibana URL: [kibana]($kibana_url)
Environment Details: $s3_bucket_link
EOF
)
echo "$summary" >> $GITHUB_STEP_SUMMARY
echo "$summary" # Print the summary to the workflow log
- name: Install CNVM integration
id: cnvm
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry run python src/install_cnvm_integration.py
- name: Deploy CNVM agent
env:
STACK_NAME: "${{ env.CNVM_STACK_NAME}}"
run: |
unset ENROLLMENT_TOKEN
just deploy-cloudformation
- name: Install CSPM GCP integration
id: cspm-gcp-integration
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry run python src/install_cspm_gcp_integration.py
- id: google-auth
name: Authenticate to Google Cloud
uses: google-github-actions/auth@v1
with:
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
- name: Deploy CSPM GCP agent
id: cspm-gcp-agent
working-directory: deploy/deployment-manager
env:
DEPLOYMENT_LABELS: ${{ env.GCP_DEFAULT_TAGS }}
run: |
. ./set_env.sh && ./deploy.sh
- name: Install D4C integration
id: kspm-d4c
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry run python src/install_d4c_integration.py
- name: Install KSPM EKS integration
id: kspm-eks
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry run python src/install_kspm_eks_integration.py
- name: Deploy KSPM EKS agent
env:
S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
run: |
aws eks --region ${{ env.AWS_REGION }} update-kubeconfig \
--name $(terraform output -raw deployment_name) --alias eks-config
echo 'KUBE_CONFIG_DATA=$(cat ~/.kube/config | base64)' >> $GITHUB_ENV
aws s3 cp ~/.kube/config "${{ env.S3_BUCKET }}/kubeconfig"
kubectl config use-context eks-config
kubectl apply -f ${{ env.FLEET_API_DIR}}/kspm_d4c.yaml
- name: Install KSPM Unmanaged integration
id: kspm-unmanaged
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry run python src/install_kspm_unmanaged_integration.py
- name: Deploy KSPM Unmanaged agent
run: |
chmod 600 ${{ env.EC2_KSPM_KEY }}
# Copy the manifest file to the EC2 instance
scp -o StrictHostKeyChecking=no -v -i ${{ env.EC2_KSPM_KEY }} ${{ env.FLEET_API_DIR}}/kspm_unmanaged.yaml "ubuntu@${{ env.KSPM_PUBLIC_IP }}:~/."
# Apply the manifest file
ssh -o StrictHostKeyChecking=no -v -i ${{ env.EC2_KSPM_KEY }} "ubuntu@${{ env.KSPM_PUBLIC_IP }}" "kubectl apply -f kspm_unmanaged.yaml"
- name: Install CSPM integration
id: cspm
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry run python src/install_cspm_integration.py
- name: Deploy CSPM agent
run: |
chmod 600 ${{ env.EC2_CSPM_KEY }}
# Copy the manifest file to the EC2 instance
scp -o StrictHostKeyChecking=no -v -i ${{ env.EC2_CSPM_KEY }} ${{ env.FLEET_API_DIR}}/cspm-linux.sh "ubuntu@${{ env.CSPM_PUBLIC_IP }}:~/."
# Apply the manifest file
ssh -o StrictHostKeyChecking=no -v -i ${{ env.EC2_CSPM_KEY }} "ubuntu@${{ env.CSPM_PUBLIC_IP }}" "chmod +x cspm-linux.sh && ./cspm-linux.sh"
- name: Upload Integrations data
if: always()
env:
S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
run: |
aws s3 cp "${{ env.FLEET_API_DIR}}/kspm_unmanaged.yaml" "${{ env.S3_BUCKET }}/kspm_unmanaged.yaml"
aws s3 cp "${{ env.FLEET_API_DIR}}/kspm_d4c.yaml" "${{ env.S3_BUCKET }}/kspm_d4c.yaml"
aws s3 cp "${{ env.FLEET_API_DIR}}/kspm_eks.yaml" "${{ env.S3_BUCKET }}/kspm_eks.yaml"
aws s3 cp "${{ env.FLEET_API_DIR}}/cspm-linux.sh" "${{ env.S3_BUCKET }}/cspm-linux.sh"
- name: Wait for agents to enroll
id: wait-for-agents
working-directory: ${{ env.WORKING_DIR }}/fleet_api
run: |
poetry run python src/agents_enrolled.py
- name: Run Sanity checks
if: ${{ success() && github.event.inputs.run-sanity-tests == 'true' }}
working-directory: ./tests
run: |
poetry install
poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
- name: Cleanup Environment
if: github.event.inputs.cleanup-env == 'true'
run: |
just delete-cloud-env ${{ env.DEPLOYMENT_NAME }} '' "false"
- name: Tag CNVM Instance
if: github.event.inputs.cleanup-env == 'false'
env:
STACK_NAME: "${{ env.CNVM_STACK_NAME}}"
run: just create-cnvm-stack-tags ${{ env.AWS_REGION}} ${{ env.STACK_NAME }} '${{ env.AWS_DEFAULT_TAGS }} Key=owner,Value=${{ github.actor }}'