Skip to content

Commit

Permalink
Integrate Docker for E2E testing with Makefile and Azure pipeline orc…
Browse files Browse the repository at this point in the history
…hestration
  • Loading branch information
shubhadapaithankar committed Oct 1, 2024
1 parent 817438f commit e9b6a5b
Show file tree
Hide file tree
Showing 6 changed files with 255 additions and 3 deletions.
92 changes: 91 additions & 1 deletion .pipelines/ci.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Azure DevOps Pipeline running CI
# Azure DevOps Pipeline running CI and E2E

trigger:
branches:
Expand Down Expand Up @@ -101,3 +101,93 @@ stages:
acrFQDN: 'arosvcdev.azurecr.io'
repository: 'aro'
pushLatest: true

- job: Lint_Az_ARO_Extension
pool:
name: 1es-aro-ci-pool
variables:
HOME: $(Agent.BuildDirectory)
steps:
- template: ./templates/template-checkout.yml
- script: |
set -xe
export AZDEV_CONFIG_DIR=$(Agent.BuildDirectory)/azdev-config
make test-python
[[ -z "$(git status -s)" ]]
target: python
# New E2E Stage with Docker Compose
- stage: E2E
dependsOn: Containerized
jobs:
- job: Run_E2E_Tests
pool:
name: 1es-aro-ci-pool
steps:
# Checkout the code
- template: ./templates/template-checkout.yml

# Install Docker Compose and pull the RP image
- template: ./templates/e2e-pipeline-template.yml
parameters:
rpImageACR: 'arosvcdev.azurecr.io'
acrCredentialsJSON: $(acr-credentials)

# Install OpenVPN (the command will depend on the agent OS)
- script: |
set -xe
sudo apt-get update && sudo apt-get install -y openvpn || \
sudo tdnf install -y openvpn || \
sudo yum install -y openvpn
displayName: Install OpenVPN
# AZ CLI Login using the existing secret as in the old pipeline
- template: ./templates/template-az-cli-login.yml
parameters:
azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)

- script: |
set -xe
az account set -s $AZURE_SUBSCRIPTION_ID
# Download secrets tarball containing the VPN certificates and config
az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz
# Extract the secrets tarball
sudo mkdir -p /etc/openvpn
sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn
# Ensure the extracted files contain the required certificates
ls /etc/openvpn
displayName: Download and Extract VPN Secrets
# Setup Azure and source secrets/env
- script: |
set -xe
export RP_IMAGE_ACR=arosvcdev.azurecr.io
export VERSION=${BUILD_BUILDID}
export E2E_FLAGS="--flag1 --flag2"
export E2E_LABEL="test-label"
export E2E_DELETE_CLUSTER="false"
echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env
echo "VERSION=$VERSION" >> .env
echo "E2E_FLAGS=$E2E_FLAGS" >> .env
echo "E2E_LABEL=$E2E_LABEL" >> .env
cat .env
sudo openvpn --config /etc/openvpn/vpn.conf &
docker-compose --env-file .env -f docker-compose.yml up -d
displayName: Start OpenVPN and Run Docker Compose for E2E Services
# Log the output from the e2e container in case of failure
- script: |
set -xe
docker-compose logs e2e
displayName: Log E2E Test Output
condition: failed()
# Clean up Docker Compose
- script: |
docker-compose down
displayName: Cleanup Docker Compose
condition: always()
44 changes: 44 additions & 0 deletions .pipelines/templates/e2e-pipeline-template.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# ./templates/e2e-pipeline-template.yml
parameters:
- name: rpImageACR
type: string
- name: acrCredentialsJSON
type: string

steps:
# Authenticate to ACR and Install Docker Compose
- task: AzureCLI@2
displayName: 'Authenticate to ACR and Install Docker Compose'
inputs:
azureSubscription: 'ado-pipeline-dev-image-push' # Replace with your service connection
scriptType: bash
scriptLocation: 'inlineScript'
inlineScript: |
set -xe
# Ensure RP_IMAGE_ACR is correctly passed as a parameter
if [ -z "${{ parameters.rpImageACR }}" ]; then
echo "Error: RP_IMAGE_ACR is not set"
exit 1
fi
ACR_FQDN="${{ parameters.rpImageACR }}"
REGISTRY_NAME=$(echo $ACR_FQDN | cut -d'.' -f1)
# Install Docker Compose
sudo apt-get update
sudo apt-get install -y docker-compose
# Login to ACR
az acr login --name $REGISTRY_NAME
# Pull the RP Docker image
- script: |
if [ -z "${{ parameters.rpImageACR }}" ]; then
echo "Error: RP_IMAGE_ACR is not set"
exit 1
fi
export RP_IMAGE_ACR=${{ parameters.rpImageACR }}
export VERSION=$(Build.BuildId)
docker pull ${RP_IMAGE_ACR}/aro:${VERSION}
displayName: Pull RP Docker Image
10 changes: 8 additions & 2 deletions .pipelines/templates/template-az-cli-login.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,13 @@ steps:
set -e
trap 'rm -f devops-spn.json' EXIT
base64 -d >devops-spn.json <<<${{ parameters.azureDevOpsJSONSPN }}
echo "${{ parameters.azureDevOpsJSONSPN }}" | base64 -d > devops-spn.json
az login --service-principal -u "$(jq -r .clientId <devops-spn.json)" -p "$(jq -r .clientSecret <devops-spn.json)" -t "$(jq -r .tenantId <devops-spn.json)" --allow-no-subscriptions >/dev/null
az login --service-principal \
-u "$(jq -r .clientId <devops-spn.json)" \
-p "$(jq -r .clientSecret <devops-spn.json)" \
-t "$(jq -r .tenantId <devops-spn.json)" --allow-no-subscriptions >/dev/null
# Cleanup
rm -f devops-spn.json
displayName: 🗝 AZ Login
11 changes: 11 additions & 0 deletions Dockerfile.vpn
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Use a Microsoft-approved image
FROM mcr.microsoft.com/azure-cli:2.61.0 AS base

# Install OpenVPN
RUN apk add --no-cache openvpn || tdnf install -y openvpn || dnf install -y openvpn

# Create the config directory and generate a basic vpn.conf file
RUN mkdir -p /etc/openvpn && echo "client\nremote vpn-server-address 1194\nproto udp\ndev tun\nresolv-retry infinite\nnobind\npersist-key\npersist-tun\nca ca.crt\ncert client.crt\nkey client.key\ncomp-lzo\nverb 3" > /etc/openvpn/vpn.conf

# Run OpenVPN when the container starts
CMD ["openvpn", "--config", "/etc/openvpn/vpn.conf"]
45 changes: 45 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,7 @@ LOCAL_ARO_PORTAL_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-portal-build
LOCAL_ARO_RP_BUILD_IMAGE ?= $(LOCAL_ARO_RP_IMAGE)-build
LOCAL_AZ_EXT_ARO_IMAGE ?= azext-aro
LOCAL_TUNNEL_IMAGE ?= aro-tunnel
LOCAL_VPN_IMAGE ?= vpn_image

###############################################################################
# Targets
Expand Down Expand Up @@ -539,3 +540,47 @@ run-rp: ci-rp podman-secrets
--secret proxy-client.crt,target=/app/secrets/proxy-client.crt \
--secret proxy.crt,target=/app/secrets/proxy.crt \
$(LOCAL_ARO_RP_IMAGE):$(VERSION) rp

# Run selenium using Docker
.PHONY: run-selenium
run-selenium:
docker run -d --name selenium-container selenium/standalone-chrome

# Run RP using Docker
.PHONY: run-rp-docker
run-rp: run-selenium
docker run -d --name rp-container $(ARO_IMAGE_BASE):$(VERSION)

# Run E2E Tests using Docker
.PHONY: run-e2e
run-e2e: e2e.test
docker-compose run --rm e2e /usr/local/bin/e2e.test $(E2E_FLAGS) --ginkgo.label-filter=$(E2E_LABEL)

# Clean up containers after E2E tests
.PHONY: e2e-cluster-clean
e2e-cluster-clean:
docker stop selenium-container rp-container e2e-container || true
docker rm selenium-container rp-container e2e-container || true

# Build the VPN Docker image
.PHONY: build-vpn
build-vpn:
@echo "Building VPN image with VERSION: $(VERSION)"
docker build . $(DOCKER_BUILD_CI_ARGS) \
-f Dockerfile.vpn \
-t $(LOCAL_VPN_IMAGE):$(VERSION)

# Push the VPN image to ACR
.PHONY: push-vpn
push-vpn: build-vpn
@echo "Pushing VPN image to ACR: $(RP_IMAGE_ACR)"
@echo "VERSION is: $(VERSION)"
if [ -z "$(RP_IMAGE_ACR)" ]; then \
echo "Error: RP_IMAGE_ACR is not set"; \
exit 1; \
fi
# Tag the VPN image with the ACR registry and version
docker tag $(LOCAL_VPN_IMAGE):$(VERSION) $(RP_IMAGE_ACR)/vpn_image:$(VERSION)
# Push the VPN image to ACR
docker push $(RP_IMAGE_ACR)/vpn_image:$(VERSION)

56 changes: 56 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
version: '3.8'

services:
vpn:
# Use an image with OpenVPN installed (or you can use the previous image and install OpenVPN at runtime)
image: kylemanna/openvpn
container_name: vpn-container
privileged: true # Required for OpenVPN
network_mode: host # Use host network
volumes:
- /dev/shm:/dev/shm # Shared memory mount
- /etc/openvpn:/etc/openvpn # Mount the extracted VPN secrets
devices:
- /dev/net/tun # Required for VPN to access tunnel
command: ["openvpn", "--config", "/etc/openvpn/vpn.conf"] # Start OpenVPN with existing config

selenium:
image: selenium/standalone-chrome
container_name: selenium-container
network_mode: host # Use host network (no need for port mappings)
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:4444"]
interval: 30s
timeout: 10s
retries: 3

rp:
image: ${RP_IMAGE_ACR}/aro:${VERSION}
container_name: rp-container
network_mode: host # Use host network
depends_on:
vpn:
condition: service_healthy
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8443/healthz"]
interval: 30s
timeout: 10s
retries: 3

e2e:
image: ${RP_IMAGE_ACR}/aro:${VERSION}
container_name: e2e-container
network_mode: host # Use host network
depends_on:
rp:
condition: service_healthy
environment:
- CI=true
- E2E_FLAGS=${E2E_FLAGS}
- E2E_LABEL=${E2E_LABEL}
entrypoint: ["/usr/local/bin/e2e.test"]
command: ["${E2E_FLAGS}", "--ginkgo.label-filter=${E2E_LABEL}"]

networks:
default:
external: true

0 comments on commit e9b6a5b

Please sign in to comment.