fixed the vpn issue

Azure · Oct 2, 2024 · 605e3ad · 605e3ad
1 parent e9b6a5b
commit 605e3ad
Show file tree

Hide file tree

Showing 2 changed files with 68 additions and 38 deletions.
diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml
@@ -116,78 +116,100 @@ stages:
               [[ -z "$(git status -s)" ]]
             target: python
 
-  # New E2E Stage with Docker Compose
+  # E2E Stage using Docker Compose
   - stage: E2E
     dependsOn: Containerized
     jobs:
       - job: Run_E2E_Tests
         pool:
           name: 1es-aro-ci-pool
         steps:
-          # Checkout the code
+          # Step 1: Checkout the code
           - template: ./templates/template-checkout.yml
 
-          # Install Docker Compose and pull the RP image
+          # Step 2: Pull the RP image from ACR
           - template: ./templates/e2e-pipeline-template.yml
             parameters:
               rpImageACR: 'arosvcdev.azurecr.io'
               acrCredentialsJSON: $(acr-credentials)
 
-          # Install OpenVPN (the command will depend on the agent OS)
+          # Step 3: Install Docker Compose and Docker
           - script: |
-                set -xe
-                sudo apt-get update && sudo apt-get install -y openvpn || \
-                sudo tdnf install -y openvpn || \
-                sudo yum install -y openvpn
-            displayName: Install OpenVPN
+              sudo apt-get update && sudo apt-get install -y openvpn docker.io docker-compose make
+              sudo systemctl start docker
+              sudo systemctl enable docker
+            displayName: Install OpenVPN, Docker, and Docker Compose
 
-          # AZ CLI Login using the existing secret as in the old pipeline
+          # Step 4: AZ CLI Login using the existing secret as in the old pipeline
           - template: ./templates/template-az-cli-login.yml
             parameters:
               azureDevOpsJSONSPN: $(aro-v4-e2e-devops-spn)
 
+          # Step 5: Download and extract VPN secrets with `make secrets`
           - script: |
-              set -xe
               az account set -s $AZURE_SUBSCRIPTION_ID
-
-              # Download secrets tarball containing the VPN certificates and config
-              az storage blob download --account-name $(SECRET_SA_ACCOUNT_NAME) --container-name secrets --name vpn-secrets.tar.gz --file vpn-secrets.tar.gz
-
-              # Extract the secrets tarball
+              
+              #command to download the secrets tarball and set the environment
+              SECRET_SA_ACCOUNT_NAME=$(SECRET_SA_ACCOUNT_NAME) make secrets
+              . secrets/env
               sudo mkdir -p /etc/openvpn
-              sudo tar -xzf vpn-secrets.tar.gz -C /etc/openvpn
-
-              # Ensure the extracted files contain the required certificates
+              tar -xzf secrets.tar.gz -C ./secrets
+              echo "Listing contents of extracted secrets:"
+              ls -R ./secrets
+              # Copy VPN secrets (certificates, config) to /etc/openvpn
+              sudo cp ./secrets/vpn-ca.crt /etc/openvpn/ca.crt
+              sudo cp ./secrets/vpn-client.crt /etc/openvpn/client.crt
+              sudo cp ./secrets/vpn-client.key /etc/openvpn/client.key
+              sudo cp ./secrets/vpn-eastus.ovpn /etc/openvpn/vpn.conf
+  
               ls /etc/openvpn
-            displayName: Download and Extract VPN Secrets
+            displayName: Download and Setup VPN Secrets
 
-          # Setup Azure and source secrets/env
+          # Step 6: Setup environment variables for Docker Compose
           - script: |
-              set -xe
               export RP_IMAGE_ACR=arosvcdev.azurecr.io
               export VERSION=${BUILD_BUILDID}
               export E2E_FLAGS="--flag1 --flag2"
               export E2E_LABEL="test-label"
-              export E2E_DELETE_CLUSTER="false"
               echo "RP_IMAGE_ACR=$RP_IMAGE_ACR" > .env
               echo "VERSION=$VERSION" >> .env
               echo "E2E_FLAGS=$E2E_FLAGS" >> .env
               echo "E2E_LABEL=$E2E_LABEL" >> .env
-
               cat .env
-              sudo openvpn --config /etc/openvpn/vpn.conf &
+            displayName: Set Up Environment Variables
+
+          # Step 7: Start services using Docker Compose
+          - script: |
               docker-compose --env-file .env -f docker-compose.yml up -d
-            displayName: Start OpenVPN and Run Docker Compose for E2E Services
+              docker ps
+            displayName: Start Services using Docker Compose
 
-          # Log the output from the e2e container in case of failure
+          # Step 8: Execute E2E tests
           - script: |
-              set -xe
-              docker-compose logs e2e
-            displayName: Log E2E Test Output
+              docker-compose exec e2e /usr/local/bin/e2e.test ${E2E_FLAGS} --ginkgo.label-filter=${E2E_LABEL}
+            displayName: Execute E2E Tests
+
+          # Step 9: Log the output from the services in case of failure
+          - script: |
+              docker-compose logs vpn
+              docker-compose logs selenium
+              docker-compose logs rp
+            displayName: Log Service Output
             condition: failed()
 
-          # Clean up Docker Compose
+          # Step 10: Clean up Docker Compose services
           - script: |
               docker-compose down
-            displayName: Cleanup Docker Compose
+            displayName: Cleanup Docker Compose Services
             condition: always()
+
+          # Step 11: Cleanup Azure resources
+          - script: |
+              export CI=true
+              . ./hack/e2e/run-rp-and-e2e.sh
+              clean_e2e_db
+            displayName: Cleanup (Azure)
+            condition: always()
+
+          # Step 12: AZ CLI Logout
+          - template: ./templates/template-az-cli-logout.yml
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -2,8 +2,7 @@ version: '3.8'
 
 services:
   vpn:
-    # Use an image with OpenVPN installed (or you can use the previous image and install OpenVPN at runtime)
-    image: kylemanna/openvpn
+    image: registry.access.redhat.com/ubi8/toolbox:8.8
     container_name: vpn-container
     privileged: true  # Required for OpenVPN
     network_mode: host  # Use host network
@@ -12,12 +11,21 @@ services:
       - /etc/openvpn:/etc/openvpn  # Mount the extracted VPN secrets
     devices:
       - /dev/net/tun  # Required for VPN to access tunnel
-    command: ["openvpn", "--config", "/etc/openvpn/vpn.conf"]  # Start OpenVPN with existing config
+    command: >
+      bash -c "
+      yum install -y openvpn &&
+      openvpn --config /etc/openvpn/vpn.conf"  # Start OpenVPN after installing it
+      openvpn --version && openvpn --config /etc/openvpn/vpn.conf
+    healthcheck:
+      test: ["CMD", "pgrep", "openvpn"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   selenium:
     image: selenium/standalone-chrome
     container_name: selenium-container
-    network_mode: host  # Use host network (no need for port mappings)
+    network_mode: host
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:4444"]
       interval: 30s
@@ -27,7 +35,7 @@ services:
   rp:
     image: ${RP_IMAGE_ACR}/aro:${VERSION}
     container_name: rp-container
-    network_mode: host  # Use host network
+    network_mode: host
     depends_on:
       vpn:
         condition: service_healthy
@@ -40,7 +48,7 @@ services:
   e2e:
     image: ${RP_IMAGE_ACR}/aro:${VERSION}
     container_name: e2e-container
-    network_mode: host  # Use host network
+    network_mode: host
     depends_on:
       rp:
         condition: service_healthy