Creating dg-upgrade-30 by @gurevichdmitry #36
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Upgrade Environment | |
| run-name: Creating ${{ github.event.inputs.deployment_name }} by @${{ github.actor }} | |
| on: | |
| # Ability to execute on demand | |
| workflow_dispatch: | |
| inputs: | |
| deployment_name: | |
| type: string | |
| description: | | |
| Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123' | |
| required: true | |
| target-elk-stack-version: | |
| required: true | |
| description: "Target version of the ELK stack: For BC version use 8.x.y, for SNAPSHOT use 8.x.y-SNAPSHOT" | |
| default: "8.11.0" | |
| type: string | |
| docker-image-override: | |
| required: false | |
| description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)" | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "eu-west-1" | |
| WORKING_DIR: deploy/test-environments | |
| FLEET_API_DIR: fleet_api/src | |
| TF_VAR_stack_version: ${{ inputs.target-elk-stack-version }} | |
| TF_VAR_ess_region: gcp-us-west2 | |
| TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} | |
| DOCKER_IMAGE: ${{ inputs.docker-image-override }} | |
| jobs: | |
| init: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| stack-version: ${{ steps.set-previous-version.outputs.PREVIOUS_VERSION }} | |
| ess-region: ${{ env.TF_VAR_ess_region }} | |
| steps: | |
| - name: Set Previous Version | |
| id: set-previous-version | |
| run: | | |
| VERSION="${{ inputs.target-elk-stack-version }}" | |
| # Extract the major and minor versions | |
| MAJOR_VERSION=$(echo $VERSION | cut -d'.' -f1) | |
| MINOR_VERSION=$(echo $VERSION | cut -d'.' -f2) | |
| # Calculate the previous version (assuming it's always X.(Y-1)) | |
| PREVIOUS_VERSION="$MAJOR_VERSION.$((MINOR_VERSION - 1))" | |
| URL="https://snapshots.elastic.co/latest/$PREVIOUS_VERSION.json" | |
| # Use curl to fetch the JSON data | |
| JSON_RESPONSE=$(curl -s "$URL") | |
| # Get latest snapshot version | |
| SNAPSHOT_VERSION=$(echo "$JSON_RESPONSE" | jq -r '.version') | |
| # Split the version into major, minor, and patch parts | |
| IFS='.-' read -ra PARTS <<< "$SNAPSHOT_VERSION" | |
| MAJOR="${PARTS[0]}" | |
| MINOR="${PARTS[1]}" | |
| PATCH="${PARTS[2]}" | |
| # Decrement the patch version by 1 | |
| PATCH=$((PATCH - 1)) | |
| # Format the previous version | |
| PREVIOUS_VERSION="$MAJOR.$MINOR.$PATCH" | |
| echo "Current Version: $VERSION" | |
| echo "Latest Released Version: $PREVIOUS_VERSION" | |
| echo "PREVIOUS_VERSION=$PREVIOUS_VERSION" >> $GITHUB_OUTPUT | |
| deploy: | |
| uses: ./.github/workflows/test-environment.yml | |
| needs: init | |
| with: | |
| deployment_name: ${{ inputs.deployment_name }} | |
| elk-stack-version: ${{ needs.init.outputs.stack-version }} | |
| ess-region: ${{ needs.init.outputs.ess-region }} | |
| run-sanity-tests: true | |
| secrets: inherit | |
| upgrade: | |
| runs-on: ubuntu-20.04 | |
| needs: [init, deploy] | |
| timeout-minutes: 120 | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKING_DIR }} | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Init Hermit | |
| run: ./bin/hermit env -r >> $GITHUB_ENV | |
| working-directory: ./ | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install Poetry | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| poetry --version | |
| - name: Install Fleet API dependencies | |
| id: fleet-api-deps | |
| working-directory: ${{ env.WORKING_DIR }}/fleet_api | |
| run: | | |
| poetry install | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::704479110758:role/Developer_eks | |
| role-session-name: github-ci | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Download tf state data | |
| env: | |
| S3_BUCKET: ${{ needs.deploy.outputs.s3-bucket }} | |
| run: | | |
| aws s3 cp "${{ env.S3_BUCKET }}/terraform.tfstate" "./terraform.tfstate" | |
| - name: Terraform Init | |
| run: terraform init | |
| - name: Terraform Validate | |
| run: terraform validate | |
| - name: Update ELK stack version | |
| id: apply | |
| if: success() | |
| run: | | |
| terraform apply --auto-approve -var="deployment_name=${{ inputs.deployment_name }}" -var="region=${{ env.AWS_REGION }}" | |
| - name: Set Environment Output | |
| id: env-output | |
| run: | | |
| echo "KIBANA_URL=$(terraform output -raw kibana_url)" >> $GITHUB_ENV | |
| echo "ES_URL=$(terraform output -raw elasticsearch_url)" >> $GITHUB_ENV | |
| echo "ES_USER=$(terraform output -raw elasticsearch_username)" >> $GITHUB_ENV | |
| export ES_PASSWORD=$(terraform output -raw elasticsearch_password) | |
| echo "::add-mask::$ES_PASSWORD" | |
| echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV | |
| export EC2_CSPM=$(terraform output -raw ec2_cspm_ssh_cmd) | |
| echo "::add-mask::$EC2_CSPM" | |
| echo "EC2_CSPM=$EC2_CSPM" >> $GITHUB_ENV | |
| export EC2_KSPM=$(terraform output -raw ec2_kspm_ssh_cmd) | |
| echo "::add-mask::$EC2_KSPM" | |
| echo "EC2_KSPM=$EC2_KSPM" >> $GITHUB_ENV | |
| export EC2_CSPM_KEY=$(terraform output -raw ec2_cspm_key) | |
| echo "::add-mask::$EC2_CSPM_KEY" | |
| echo "EC2_CSPM_KEY=$EC2_CSPM_KEY" >> $GITHUB_ENV | |
| export EC2_KSPM_KEY=$(terraform output -raw ec2_kspm_key) | |
| echo "::add-mask::$EC2_KSPM_KEY" | |
| echo "EC2_KSPM_KEY=$EC2_KSPM_KEY" >> $GITHUB_ENV | |
| export KSPM_PUBLIC_IP=$(terraform output -raw ec2_kspm_public_ip) | |
| echo "::add-mask::$KSPM_PUBLIC_IP" | |
| echo "KSPM_PUBLIC_IP=$KSPM_PUBLIC_IP" >> $GITHUB_ENV | |
| export CSPM_PUBLIC_IP=$(terraform output -raw ec2_cspm_public_ip) | |
| echo "::add-mask::$CSPM_PUBLIC_IP" | |
| echo "CSPM_PUBLIC_IP=$CSPM_PUBLIC_IP" >> $GITHUB_ENV | |
| - name: Run Sanity checks | |
| if: success() | |
| working-directory: ./tests | |
| env: | |
| USE_K8S: false | |
| STACK_VERSION: ${{ needs.init.outputs.stack-version }} | |
| run: | | |
| poetry install | |
| poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4 | |
| - name: Set Docker Image version | |
| if: ${{ ! inputs.docker-image-override }} | |
| env: | |
| VERSION: 'docker.elastic.co/beats/elastic-agent:${{ inputs.target-elk-stack-version }}' | |
| run: | | |
| echo "DOCKER_IMAGE=${{ env.VERSION }}" >> $GITHUB_ENV | |
| - name: Download Integrations data | |
| env: | |
| S3_BUCKET: ${{ needs.deploy.outputs.s3-bucket }} | |
| run: | | |
| aws s3 cp "${{ env.S3_BUCKET }}/kspm.pem" "${{ env.EC2_KSPM_KEY }}" | |
| aws s3 cp "${{ env.S3_BUCKET }}/state_data.json" "${{ env.FLEET_API_DIR }}/state_data.json" | |
| - name: Upgrade KSPM Unmanaged agent | |
| run: | | |
| chmod 600 ${{ env.EC2_KSPM_KEY }} | |
| # Update image | |
| ssh -o StrictHostKeyChecking=no -v -i ${{ env.EC2_KSPM_KEY }} "ubuntu@${{ env.KSPM_PUBLIC_IP }}" "kubectl set image daemonset elastic-agent -n kube-system elastic-agent=${{ env.DOCKER_IMAGE }}" | |
| - name: Upgrade KSPM EKS agent | |
| run: | | |
| aws eks --region ${{ env.AWS_REGION }} update-kubeconfig \ | |
| --name $(terraform output -raw deployment_name) --alias eks-config | |
| kubectl config use-context eks-config | |
| kubectl set image daemonset elastic-agent -n kube-system elastic-agent=${{ env.DOCKER_IMAGE }} | |
| - name: Upgrade Linux agents | |
| working-directory: ${{ env.WORKING_DIR }}/${{ env.FLEET_API_DIR }} | |
| env: | |
| CNVM_STACK_NAME: ${{ needs.deploy.outputs.cnvm-stack-name }} | |
| STACK_VERSION: ${{ inputs.target-elk-stack-version }} | |
| run: | | |
| poetry run python upgrade_agents.py | |
| - name: Run Upgrade Sanity checks | |
| if: success() | |
| working-directory: ./tests | |
| env: | |
| STACK_VERSION: ${{ inputs.target-elk-stack-version }} | |
| USE_K8S: false | |
| run: | | |
| poetry install | |
| poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4 |