From 2a6e391541d78095f69636999fff5426d598d0a6 Mon Sep 17 00:00:00 2001
From: Ciara Stacke <c.stacke@f5.com>
Date: Fri, 23 Feb 2024 16:55:48 +0000
Subject: [PATCH 1/3] Add NFR workflow

---
 .github/workflows/nfr.yml                | 196 +++++++++++++++++++++++
 tests/Makefile                           |  10 +-
 tests/README.md                          |  27 +++-
 tests/scripts/cleanup-gcp-pipeline.sh    |   8 +
 tests/scripts/cleanup-vm.sh              |   6 +-
 tests/scripts/create-and-setup-gcp-vm.sh |  14 +-
 tests/scripts/create-gke-cluster.sh      |  15 ++
 tests/scripts/delete-gke-cluster.sh      |   5 +
 tests/scripts/run-tests-gcp-vm.sh        |   6 +-
 tests/scripts/vars.env-example           |   1 +
 10 files changed, 271 insertions(+), 17 deletions(-)
 create mode 100644 .github/workflows/nfr.yml
 create mode 100644 tests/scripts/cleanup-gcp-pipeline.sh
 create mode 100644 tests/scripts/create-gke-cluster.sh
 create mode 100644 tests/scripts/delete-gke-cluster.sh

diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml
new file mode 100644
index 000000000..beea8cc0d
--- /dev/null
+++ b/.github/workflows/nfr.yml
@@ -0,0 +1,196 @@
+name: Non Functional Testing
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_label:
+        description: 'NFR test to run. Choose between performance, upgrade or all.'
+        required: true
+        default: 'all'
+        type: choice
+        options: [performance, upgrade, all]
+      version:
+        description: 'Version of NGF under test'
+        required: true
+        default: 'edge'
+      nginx-plus:
+        description: 'Run tests with NGINX Plus'
+        required: false
+        default: false
+        type: boolean
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.ref_name }}-ci
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  setup-and-run-tests:
+    name: Setup and Run NFR Tests
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write # needed for opening PR with the results files
+      pull-requests: write # needed for opening PR with the results files
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - name: Setup Golang Environment
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version: stable
+
+    - name: Set GOPATH
+      run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+
+    - name: Docker Buildx
+      uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+    - name: NGF Docker meta
+      id: ngf-meta
+      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+      with:
+        images: |
+          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric
+        tags: |
+          type=raw,value=${{ inputs.version }}-${{ github.run_id }}
+
+    - name: NGINX Docker meta
+      id: nginx-meta
+      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+      with:
+        images: |
+          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/${{ inputs.nginx-plus == true && 'nginxplus' || 'nginx' }}
+        tags: |
+          type=raw,value=${{ inputs.version }}-${{ github.run_id }}
+
+    - name: Build binary
+      uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
+      with:
+        version: latest
+        args: build --snapshot --clean
+
+    - name: Authenticate to Google Cloud
+      id: auth
+      uses: google-github-actions/auth@a6e2e39c0a0331da29f7fd2c2a20a427e8d3ad1f # v2.1.1
+      with:
+        token_format: access_token
+        workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY }}
+        service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
+      with:
+        project_id: '${{ secrets.GCP_PROJECT_ID }}'
+        install_components: 'kubectl'
+
+    - name: Login to GCR
+      run: gcloud auth configure-docker gcr.io -q
+
+    - name: Build NGF Docker Image
+      uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+      with:
+        file: build/Dockerfile
+        tags: ${{ steps.ngf-meta.outputs.tags }}
+        context: "."
+        provenance: false
+        platforms: linux/amd64
+        target: goreleaser
+        pull: true
+        push: true
+
+    - name: Build NGINX Docker Image
+      uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+      with:
+        file: build/Dockerfile${{ inputs.nginx-plus == true && '.nginxplus' || '.nginx'}}
+        tags: ${{ steps.nginx-meta.outputs.tags }}
+        context: "."
+        platforms: linux/amd64
+        provenance: false
+        pull: true
+        push: true
+        build-args: |
+          NJS_DIR=internal/mode/static/nginx/modules/src
+          NGINX_CONF_DIR=internal/mode/static/nginx/conf
+          BUILD_AGENT=gha
+        secrets: |
+          ${{ inputs.nginx-plus == true && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
+          ${{ inputs.nginx-plus == true && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }}
+
+    - name: 'Setup dotenv file'
+      working-directory: ./tests/scripts
+      env:
+        GKE_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+        SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+        NODES_SERVICE_ACCOUNT: ${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}
+      run: |
+        echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
+        echo "TAG=${{ inputs.version }}-${{ github.run_id }}" >> vars.env
+        echo "PREFIX=gcr.io/${GKE_PROJECT_ID}/ngf-nfr/nginx-gateway-fabric" >> vars.env
+        echo "NGINX_PREFIX=gcr.io/${GKE_PROJECT_ID}/ngf-nfr/nginx-gateway-fabric/nginx" >> vars.env
+        echo "NGINX_PLUS_PREFIX=gcr.io/${GKE_PROJECT_ID}/ngf-nfr/nginx-gateway-fabric/nginxplus" >> vars.env
+        echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
+        echo "GKE_CLUSTER_ZONE=us-east1-b" >> vars.env
+        echo "GKE_CLUSTER_REGION=us-east1" >> vars.env
+        echo "GKE_PROJECT=${GKE_PROJECT_ID}" >> vars.env
+        echo "GKE_SVC_ACCOUNT=${SERVICE_ACCOUNT}" >> vars.env
+        echo "GKE_NODES_SERVICE_ACCOUNT=${NODES_SERVICE_ACCOUNT}" >> vars.env
+        echo "IMAGE=projects/debian-cloud/global/images/debian-11-bullseye-v20240213" >> vars.env
+        echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}" >> vars.env
+        echo "NGF_REPO=nginxinc" >> vars.env
+        echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
+        echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
+        echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
+        echo "PLUS_ENABLED=${{ inputs.nginx-plus }}" >> vars.env
+        echo "GINKGO_LABEL=" >> vars.env
+
+    - name: Create GKE cluster
+      working-directory: ./tests
+      run:
+        make create-gke-cluster
+
+    - name: Setup VM and router
+      working-directory: ./tests
+      run:
+        make create-and-setup-vm
+
+    - name: Run Performance Tests
+      working-directory: ./tests
+      if: ${{ inputs.test_label == 'performance' || inputs.test_label == 'all' }}
+      run:
+        sed -i '/^GINKGO_LABEL=/s/=.*/="performance"/' "scripts/vars.env" && make run-tests-on-vm
+
+    - name: Run Upgrade Tests
+      working-directory: ./tests
+      if: ${{ inputs.test_label == 'upgrade' || inputs.test_label == 'all' }}
+      run:
+        sed -i '/^GINKGO_LABEL=/s/=.*/="upgrade"/' "scripts/vars.env" && make run-tests-on-vm
+
+    - name: Cleanup
+      working-directory: ./tests
+      if: always()
+      run:
+        bash scripts/cleanup-gcp-pipeline.sh && rm -rf scripts/vars.env
+
+    - name: Open a PR with the results
+      uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc # v6.0.1
+      with:
+        commit-message: NFR Test Results for NGF version ${{ inputs.version }}
+        author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com>
+        branch: tests/nfr-tests-${{ inputs.version }}
+        delete-branch: true
+        title: 'NFR Test Results for NGF version ${{ inputs.version }}'
+        body: |
+          Update with NFR test results for NGF version ${{ inputs.version }}
+          - Auto-generated by the NFR tests workflow run ${{ github.run_id }}
+        labels: |
+          tests
+        assignees: ${{ github.actor }}
+        draft: true
diff --git a/tests/Makefile b/tests/Makefile
index 031e51f99..245803e49 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -80,4 +80,12 @@ cleanup-router: ## Delete the GKE router
 setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the tests
 
 .PHONY: cleanup-gcp
-cleanup-gcp: cleanup-router cleanup-vm ## Cleanup all GCP resources
+cleanup-gcp: cleanup-router cleanup-vm delete-gke-cluster ## Cleanup all GCP resources
+
+.PHONY: create-gke-cluster
+create-gke-cluster: ## Create a GKE cluster
+	bash scripts/create-gke-cluster.sh
+
+.PHONY: delete-gke-cluster
+delete-gke-cluster: ## Delete the GKE cluster
+	bash scripts/delete-gke-cluster.sh
diff --git a/tests/README.md b/tests/README.md
index b5d01b10c..50dda8b2f 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -37,15 +37,19 @@ make
 ```
 
 ```text
+build-images-with-plus         Build NGF and NGINX Plus images
 build-images                   Build NGF and NGINX images
 cleanup-gcp                    Cleanup all GCP resources
 cleanup-router                 Delete the GKE router
 cleanup-vm                     Delete the test GCP VM and delete the firewall rule
 create-and-setup-vm            Create and setup a GCP VM for tests
+create-gke-cluster             Create a GKE cluster
 create-gke-router              Create a GKE router to allow egress traffic from private nodes (allows for external image pulls)
 create-kind-cluster            Create a kind cluster
+delete-gke-cluster             Delete the GKE cluster
 delete-kind-cluster            Delete kind cluster
 help                           Display this help
+load-images-with-plus          Load NGF and NGINX Plus images on configured kind cluster
 load-images                    Load NGF and NGINX images on configured kind cluster
 run-tests-on-vm                Run the tests on a GCP VM
 setup-gcp-and-run-tests        Create and setup a GKE router and GCP VM for tests and run the tests
@@ -71,7 +75,9 @@ test                           Run the system tests against your default k8s clu
 
 ## Step 1 - Create a Kubernetes cluster
 
-This can be done in a cloud provider of choice, or locally using `kind`:
+This can be done in a cloud provider of choice, or locally using `kind`.
+
+To create a local `kind` cluster:
 
 ```makefile
 make create-kind-cluster
@@ -84,6 +90,17 @@ make create-kind-cluster
 make create-kind-cluster KIND_IMAGE=kindest/node:v1.27.3
 ```
 
+To create a GKE cluster:
+
+Before running the below `make` command, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
+required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions,
+and `GKE_NODES_SERVICE_ACCOUNT` needs to be the name of a service account that has `Artifact Registry Reader`,
+`Kubernetes Engine Node Service Account` and `Monitoring Viewer` permissions.
+
+```makefile
+make create-gke-cluster
+```
+
 ## Step 2 - Build and Load Images
 
 Loading the images only applies to a `kind` cluster. If using a cloud provider, you will need to tag and push
@@ -115,7 +132,7 @@ make test TAG=$(whoami) PLUS_ENABLED=true
 
 ### 3b - Run the tests on a GKE cluster from a GCP VM
 
-This step only applies if you would like to run the tests from a GCP based VM.
+This step only applies if you would like to run the tests on a GKE cluster from a GCP based VM.
 
 Before running the below `make` command, copy the `scripts/vars.env-example` file to `scripts/vars.env` and populate the
 required env vars. `GKE_SVC_ACCOUNT` needs to be the name of a service account that has Kubernetes admin permissions.
@@ -205,7 +222,7 @@ For more information of filtering specs, see [the docs here](https://onsi.github
     make delete-kind-cluster
     ```
 
-2. Delete the GCP components (GKE router, VM, and firewall rule), if required
+2. Delete the GCP components (GKE cluster, GKE router, VM, and firewall rule), if required
 
     ```makefile
     make cleanup-gcp
@@ -220,3 +237,7 @@ For more information of filtering specs, see [the docs here](https://onsi.github
     ```makefile
     make cleanup-vm
     ```
+
+    ```makefile
+    make delete-gke-cluster
+    ```
diff --git a/tests/scripts/cleanup-gcp-pipeline.sh b/tests/scripts/cleanup-gcp-pipeline.sh
new file mode 100644
index 000000000..788d95d8e
--- /dev/null
+++ b/tests/scripts/cleanup-gcp-pipeline.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+source scripts/vars.env
+
+gcloud compute instances delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE}
+gcloud compute firewall-rules delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT}
+
+gcloud container clusters delete ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --project ${GKE_PROJECT} --quiet
diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh
index 517d98750..9864d84f2 100644
--- a/tests/scripts/cleanup-vm.sh
+++ b/tests/scripts/cleanup-vm.sh
@@ -4,11 +4,11 @@ source scripts/vars.env
 
 # Remove VM IP from GKE master control node access, if required
 if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then
-    EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \
+    EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \
                     --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
-    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} \
+    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} \
                             --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//')
-    gcloud container clusters update ${GKE_CLUSTER_NAME} --enable-master-authorized-networks --master-authorized-networks=${CURRENT_AUTH_NETWORK}
+    gcloud container clusters update ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${CURRENT_AUTH_NETWORK}
 fi
 
 gcloud compute instances delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE}
diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh
index a28166b2f..05b43a830 100644
--- a/tests/scripts/create-and-setup-gcp-vm.sh
+++ b/tests/scripts/create-and-setup-gcp-vm.sh
@@ -24,17 +24,17 @@ gcloud compute instances create ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone
 if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then
     EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \
                     --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
-    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} \
+    CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} \
                             --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//')
-    gcloud container clusters update ${GKE_CLUSTER_NAME} --enable-master-authorized-networks --master-authorized-networks=${CURRENT_AUTH_NETWORK},${EXTERNAL_IP}/32
+    gcloud container clusters update ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${EXTERNAL_IP}/32,${CURRENT_AUTH_NETWORK}
 fi
 
 # Poll for SSH connectivity
-MAX_RETRIES=30
-RETRY_INTERVAL=10
+MAX_RETRIES=10
+RETRY_INTERVAL=5
 for ((i=1; i<=MAX_RETRIES; i++)); do
     echo "Attempt $i to connect to the VM..."
-    gcloud compute ssh ${RESOURCE_NAME} --zone=${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --quiet --command="echo 'VM is ready'"
+    gcloud compute ssh username@${RESOURCE_NAME} --zone=${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --quiet --command="echo 'VM is ready'"
     if [ $? -eq 0 ]; then
         echo "SSH connection successful. VM is ready."
         break
@@ -43,6 +43,6 @@ for ((i=1; i<=MAX_RETRIES; i++)); do
     sleep ${RETRY_INTERVAL}
 done
 
-gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env ${RESOURCE_NAME}:~
+gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env username@${RESOURCE_NAME}:~
 
-gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/install-deps.sh
+gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} username@${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/install-deps.sh
diff --git a/tests/scripts/create-gke-cluster.sh b/tests/scripts/create-gke-cluster.sh
new file mode 100644
index 000000000..d048eda36
--- /dev/null
+++ b/tests/scripts/create-gke-cluster.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+source scripts/vars.env
+
+ip_random_digit=$((1 + $RANDOM % 250))
+
+gcloud container clusters create ${GKE_CLUSTER_NAME} \
+    --project ${GKE_PROJECT} \
+    --zone ${GKE_CLUSTER_ZONE} \
+    --enable-master-authorized-networks \
+    --enable-ip-alias \
+    --service-account ${GKE_NODES_SERVICE_ACCOUNT} \
+    --enable-private-nodes \
+    --master-ipv4-cidr 172.16.${ip_random_digit}.32/28 \
+    --metadata=block-project-ssh-keys=TRUE
diff --git a/tests/scripts/delete-gke-cluster.sh b/tests/scripts/delete-gke-cluster.sh
new file mode 100644
index 000000000..dedacbdb1
--- /dev/null
+++ b/tests/scripts/delete-gke-cluster.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+source scripts/vars.env
+
+gcloud container clusters delete ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --project ${GKE_PROJECT} --quiet
diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh
index 4a1872dab..2a407bfa1 100644
--- a/tests/scripts/run-tests-gcp-vm.sh
+++ b/tests/scripts/run-tests-gcp-vm.sh
@@ -4,8 +4,8 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 source scripts/vars.env
 
-gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env ${RESOURCE_NAME}:~
+gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env username@${RESOURCE_NAME}:~
 
-gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/run-tests.sh
+gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} username@${RESOURCE_NAME} --command="bash -s" < ${SCRIPT_DIR}/remote-scripts/run-tests.sh
 
-gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse ${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .
+gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse username@${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results .
diff --git a/tests/scripts/vars.env-example b/tests/scripts/vars.env-example
index 16cd2091a..196f9d0dc 100644
--- a/tests/scripts/vars.env-example
+++ b/tests/scripts/vars.env-example
@@ -9,6 +9,7 @@ GKE_CLUSTER_ZONE=<zone where GKE cluster is deployed>
 GKE_CLUSTER_REGION=<region where GKE cluster is deployed>
 GKE_PROJECT=<GCP project>
 GKE_SVC_ACCOUNT=<service account with k8s admin permissions>
+GKE_NODES_SERVICE_ACCOUNT=<service account for the GKE nodes to assume>
 IMAGE=projects/debian-cloud/global/images/debian-11-bullseye-v20231212
 NETWORK_TAGS=<network tags>
 NGF_REPO=nginxinc

From b026903ef8c4f4d81f237e86ba160dd1ff8037d4 Mon Sep 17 00:00:00 2001
From: Ciara Stacke <c.stacke@f5.com>
Date: Wed, 28 Feb 2024 16:12:34 +0000
Subject: [PATCH 2/3] Initial review feedback

---
 .github/workflows/nfr.yml             | 61 ++++++++++++---------------
 tests/scripts/cleanup-gcp-pipeline.sh |  8 ----
 tests/scripts/cleanup-vm.sh           |  4 +-
 3 files changed, 31 insertions(+), 42 deletions(-)
 delete mode 100644 tests/scripts/cleanup-gcp-pipeline.sh

diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml
index beea8cc0d..5423a8c31 100644
--- a/.github/workflows/nfr.yml
+++ b/.github/workflows/nfr.yml
@@ -4,17 +4,17 @@ on:
   workflow_dispatch:
     inputs:
       test_label:
-        description: 'NFR test to run. Choose between performance, upgrade or all.'
+        description: NFR test to run. Choose between performance, upgrade or all.
         required: true
-        default: 'all'
+        default: all
         type: choice
         options: [performance, upgrade, all]
       version:
-        description: 'Version of NGF under test'
+        description: Version of NGF under test
         required: true
-        default: 'edge'
+        default: edge
       nginx-plus:
-        description: 'Run tests with NGINX Plus'
+        description: Run tests with NGINX Plus
         required: false
         default: false
         type: boolean
@@ -24,7 +24,7 @@ defaults:
     shell: bash
 
 concurrency:
-  group: ${{ github.ref_name }}-ci
+  group: ${{ github.ref_name }}-nfr
   cancel-in-progress: true
 
 permissions:
@@ -67,7 +67,7 @@ jobs:
       uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
       with:
         images: |
-          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/${{ inputs.nginx-plus == true && 'nginxplus' || 'nginx' }}
+          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/${{ inputs.nginx-plus == true && 'nginx-plus' || 'nginx' }}
         tags: |
           type=raw,value=${{ inputs.version }}-${{ github.run_id }}
 
@@ -88,8 +88,8 @@ jobs:
     - name: Set up Cloud SDK
       uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
       with:
-        project_id: '${{ secrets.GCP_PROJECT_ID }}'
-        install_components: 'kubectl'
+        project_id: ${{ secrets.GCP_PROJECT_ID }}
+        install_components: kubectl
 
     - name: Login to GCR
       run: gcloud auth configure-docker gcr.io -q
@@ -124,24 +124,20 @@ jobs:
           ${{ inputs.nginx-plus == true && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
           ${{ inputs.nginx-plus == true && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }}
 
-    - name: 'Setup dotenv file'
+    - name: Setup dotenv file
       working-directory: ./tests/scripts
-      env:
-        GKE_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
-        SERVICE_ACCOUNT: ${{ secrets.GCP_SERVICE_ACCOUNT }}
-        NODES_SERVICE_ACCOUNT: ${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}
       run: |
         echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
         echo "TAG=${{ inputs.version }}-${{ github.run_id }}" >> vars.env
-        echo "PREFIX=gcr.io/${GKE_PROJECT_ID}/ngf-nfr/nginx-gateway-fabric" >> vars.env
-        echo "NGINX_PREFIX=gcr.io/${GKE_PROJECT_ID}/ngf-nfr/nginx-gateway-fabric/nginx" >> vars.env
-        echo "NGINX_PLUS_PREFIX=gcr.io/${GKE_PROJECT_ID}/ngf-nfr/nginx-gateway-fabric/nginxplus" >> vars.env
+        echo "PREFIX=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric" >> vars.env
+        echo "NGINX_PREFIX=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/nginx" >> vars.env
+        echo "NGINX_PLUS_PREFIX=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/nginx-plus" >> vars.env
         echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
         echo "GKE_CLUSTER_ZONE=us-east1-b" >> vars.env
         echo "GKE_CLUSTER_REGION=us-east1" >> vars.env
-        echo "GKE_PROJECT=${GKE_PROJECT_ID}" >> vars.env
-        echo "GKE_SVC_ACCOUNT=${SERVICE_ACCOUNT}" >> vars.env
-        echo "GKE_NODES_SERVICE_ACCOUNT=${NODES_SERVICE_ACCOUNT}" >> vars.env
+        echo "GKE_PROJECT=${{ secrets.GCP_PROJECT_ID }}" >> vars.env
+        echo "GKE_SVC_ACCOUNT=${{ secrets.GCP_SERVICE_ACCOUNT }}" >> vars.env
+        echo "GKE_NODES_SERVICE_ACCOUNT=${{ secrets.GKE_NODES_SERVICE_ACCOUNT }}" >> vars.env
         echo "IMAGE=projects/debian-cloud/global/images/debian-11-bullseye-v20240213" >> vars.env
         echo "NETWORK_TAGS=nfr-tests-${{ github.run_id }}" >> vars.env
         echo "NGF_REPO=nginxinc" >> vars.env
@@ -161,23 +157,22 @@ jobs:
       run:
         make create-and-setup-vm
 
-    - name: Run Performance Tests
+    - name: Run Tests
       working-directory: ./tests
-      if: ${{ inputs.test_label == 'performance' || inputs.test_label == 'all' }}
-      run:
-        sed -i '/^GINKGO_LABEL=/s/=.*/="performance"/' "scripts/vars.env" && make run-tests-on-vm
-
-    - name: Run Upgrade Tests
-      working-directory: ./tests
-      if: ${{ inputs.test_label == 'upgrade' || inputs.test_label == 'all' }}
-      run:
-        sed -i '/^GINKGO_LABEL=/s/=.*/="upgrade"/' "scripts/vars.env" && make run-tests-on-vm
+      run: |
+        if ${{ inputs.test_label != 'all' }}; then
+          sed -i '/^GINKGO_LABEL=/s/=.*/="${{ inputs.test_label }}"/' "scripts/vars.env" && make run-tests-on-vm;
+        else
+          make run-tests-on-vm;
+        fi
 
     - name: Cleanup
       working-directory: ./tests
       if: always()
-      run:
-        bash scripts/cleanup-gcp-pipeline.sh && rm -rf scripts/vars.env
+      run: |
+        bash scripts/cleanup-vm.sh true
+        make delete-gke-cluster
+        rm -rf scripts/vars.env
 
     - name: Open a PR with the results
       uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc # v6.0.1
@@ -186,7 +181,7 @@ jobs:
         author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com>
         branch: tests/nfr-tests-${{ inputs.version }}
         delete-branch: true
-        title: 'NFR Test Results for NGF version ${{ inputs.version }}'
+        title: NFR Test Results for NGF version ${{ inputs.version }}
         body: |
           Update with NFR test results for NGF version ${{ inputs.version }}
           - Auto-generated by the NFR tests workflow run ${{ github.run_id }}
diff --git a/tests/scripts/cleanup-gcp-pipeline.sh b/tests/scripts/cleanup-gcp-pipeline.sh
deleted file mode 100644
index 788d95d8e..000000000
--- a/tests/scripts/cleanup-gcp-pipeline.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-source scripts/vars.env
-
-gcloud compute instances delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE}
-gcloud compute firewall-rules delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT}
-
-gcloud container clusters delete ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --project ${GKE_PROJECT} --quiet
diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh
index 9864d84f2..f0265bc8d 100644
--- a/tests/scripts/cleanup-vm.sh
+++ b/tests/scripts/cleanup-vm.sh
@@ -2,8 +2,10 @@
 
 source scripts/vars.env
 
+skip_gke_master_control_node_access="${1:-false}"
+
 # Remove VM IP from GKE master control node access, if required
-if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then
+if [ ${ADD_VM_IP_AUTH_NETWORKS} = "true" ] && [ ${skip_gke_master_control_node_access} != "true" ]; then
     EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \
                     --format='value(networkInterfaces[0].accessConfigs[0].natIP)')
     CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} \

From 37c7ea4317351c4c113d3be4eb108b691ecf922c Mon Sep 17 00:00:00 2001
From: Ciara Stacke <c.stacke@f5.com>
Date: Thu, 29 Feb 2024 10:01:04 +0000
Subject: [PATCH 3/3] Fix typos, update docs, and remove image builds unless
 plus

---
 .github/workflows/nfr.yml                 | 65 +++++++++--------------
 docs/developer/release-process.md         |  4 +-
 tests/Makefile                            |  8 +--
 tests/scripts/remote-scripts/run-tests.sh |  2 +-
 tests/scripts/vars.env-example            |  1 +
 tests/suite/system_suite_test.go          |  5 +-
 6 files changed, 39 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml
index 5423a8c31..e809976e3 100644
--- a/.github/workflows/nfr.yml
+++ b/.github/workflows/nfr.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       test_label:
-        description: NFR test to run. Choose between performance, upgrade or all.
+        description: NFR test to run. Choose between performance, upgrade, or all
         required: true
         default: all
         type: choice
@@ -13,7 +13,11 @@ on:
         description: Version of NGF under test
         required: true
         default: edge
-      nginx-plus:
+      image_tag:
+        description: Tag of the NGF and NGINX Docker images
+        required: true
+        default: edge
+      nginx_plus:
         description: Run tests with NGINX Plus
         required: false
         default: false
@@ -51,31 +55,18 @@ jobs:
       run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
 
     - name: Docker Buildx
+      if: ${{ inputs.nginx_plus == true }}
       uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
 
-    - name: NGF Docker meta
-      id: ngf-meta
-      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
-      with:
-        images: |
-          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric
-        tags: |
-          type=raw,value=${{ inputs.version }}-${{ github.run_id }}
-
     - name: NGINX Docker meta
       id: nginx-meta
+      if: ${{ inputs.nginx_plus == true }}
       uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
       with:
         images: |
-          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/${{ inputs.nginx-plus == true && 'nginx-plus' || 'nginx' }}
+          name=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/nginx-plus
         tags: |
-          type=raw,value=${{ inputs.version }}-${{ github.run_id }}
-
-    - name: Build binary
-      uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
-      with:
-        version: latest
-        args: build --snapshot --clean
+          type=raw,value=${{ inputs.image_tag }}
 
     - name: Authenticate to Google Cloud
       id: auth
@@ -92,24 +83,14 @@ jobs:
         install_components: kubectl
 
     - name: Login to GCR
+      if: ${{ inputs.nginx_plus == true }}
       run: gcloud auth configure-docker gcr.io -q
 
-    - name: Build NGF Docker Image
-      uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
-      with:
-        file: build/Dockerfile
-        tags: ${{ steps.ngf-meta.outputs.tags }}
-        context: "."
-        provenance: false
-        platforms: linux/amd64
-        target: goreleaser
-        pull: true
-        push: true
-
-    - name: Build NGINX Docker Image
+    - name: Build NGINX Plus Docker Image
+      if: ${{ inputs.nginx_plus == true }}
       uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
       with:
-        file: build/Dockerfile${{ inputs.nginx-plus == true && '.nginxplus' || '.nginx'}}
+        file: build/Dockerfile.nginxplus
         tags: ${{ steps.nginx-meta.outputs.tags }}
         context: "."
         platforms: linux/amd64
@@ -121,16 +102,16 @@ jobs:
           NGINX_CONF_DIR=internal/mode/static/nginx/conf
           BUILD_AGENT=gha
         secrets: |
-          ${{ inputs.nginx-plus == true && format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) || '' }}
-          ${{ inputs.nginx-plus == true && format('"nginx-repo.key={0}"', secrets.NGINX_KEY) || '' }}
+          ${{ format('"nginx-repo.crt={0}"', secrets.NGINX_CRT) }}
+          ${{ format('"nginx-repo.key={0}"', secrets.NGINX_KEY) }}
 
     - name: Setup dotenv file
       working-directory: ./tests/scripts
       run: |
         echo "RESOURCE_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
-        echo "TAG=${{ inputs.version }}-${{ github.run_id }}" >> vars.env
-        echo "PREFIX=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric" >> vars.env
-        echo "NGINX_PREFIX=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/nginx" >> vars.env
+        echo "TAG=${{ inputs.image_tag }}" >> vars.env
+        echo "PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric" >> vars.env
+        echo "NGINX_PREFIX=ghcr.io/nginxinc/nginx-gateway-fabric/nginx" >> vars.env
         echo "NGINX_PLUS_PREFIX=gcr.io/${{ secrets.GCP_PROJECT_ID }}/ngf-nfr/nginx-gateway-fabric/nginx-plus" >> vars.env
         echo "GKE_CLUSTER_NAME=nfr-tests-${{ github.run_id }}" >> vars.env
         echo "GKE_CLUSTER_ZONE=us-east1-b" >> vars.env
@@ -144,15 +125,16 @@ jobs:
         echo "NGF_BRANCH=${{ github.ref_name }}" >> vars.env
         echo "SOURCE_IP_RANGE=$(curl -sS -4 icanhazip.com)/32" >> vars.env
         echo "ADD_VM_IP_AUTH_NETWORKS=true" >> vars.env
-        echo "PLUS_ENABLED=${{ inputs.nginx-plus }}" >> vars.env
+        echo "PLUS_ENABLED=${{ inputs.nginx_plus }}" >> vars.env
         echo "GINKGO_LABEL=" >> vars.env
+        echo "NGF_VERSION=${{ inputs.version }}" >> vars.env
 
     - name: Create GKE cluster
       working-directory: ./tests
       run:
         make create-gke-cluster
 
-    - name: Setup VM and router
+    - name: Create and setup VM
       working-directory: ./tests
       run:
         make create-and-setup-vm
@@ -185,6 +167,9 @@ jobs:
         body: |
           Update with NFR test results for NGF version ${{ inputs.version }}
           - Auto-generated by the NFR tests workflow run ${{ github.run_id }}
+          - Tests ran using Docker image tag ${{ inputs.image_tag }}
+          - ${{ inputs.test_label }} test(s) ran
+          - NGINX Plus enabled: ${{ inputs.nginx_plus }}
         labels: |
           tests
         assignees: ${{ github.actor }}
diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md
index 8b7c98377..16a969e00 100644
--- a/docs/developer/release-process.md
+++ b/docs/developer/release-process.md
@@ -28,7 +28,9 @@ To create a new release, follow these steps:
    format `Release X.Y.Z`.
 2. Stop merging any new work into the main branch.
 3. Test the main branch for release-readiness. For that, use the `edge` containers, which are built from the main
-   branch, and the [example applications](/examples).
+   branch, and run the following:
+   1. Run the [example applications](/examples) manually and verify their correctness.
+   2. Kick off the [NFR workflow](https://github.com/nginxinc/nginx-gateway-fabric/actions/nfr.yml) in the browser. For `image_tag`, use `edge`, and for  `version`, use the upcoming `X.Y.Z` NGF version. This will run all of the NFR tests which are automated and open a PR with the results files when it is complete. Review this PR and make any necessary changes before merging.
 4. If a problem is found, prepare a fix PR, merge it into the main branch and return to the previous step.
 5. If the supported Gateway API minor version has changed since the last release, test NGINX Gateway Fabric with the previous version of the Gateway API CRDs.
 6. If a compatibility issue is found, add a note to the release notes explaining that the previous version is not supported.
diff --git a/tests/Makefile b/tests/Makefile
index 245803e49..3770196a0 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -11,6 +11,7 @@ GW_SERVICE_TYPE=NodePort
 GW_SVC_GKE_INTERNAL=false
 GINKGO_LABEL=
 GINKGO_FLAGS=
+NGF_VERSION=
 
 ifneq ($(GINKGO_LABEL),)
     override GINKGO_FLAGS += -ginkgo.label-filter "$(GINKGO_LABEL)"
@@ -48,9 +49,10 @@ load-images-with-plus: ## Load NGF and NGINX Plus images on configured kind clus
 
 test: ## Run the system tests against your default k8s cluster
 	go test -v ./suite $(GINKGO_FLAGS) -args --gateway-api-version=$(GW_API_VERSION) \
-		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --plus-enabled=$(PLUS_ENABLED) \
-		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --pull-policy=$(PULL_POLICY) \
-		--k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) --is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
+		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
+		--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) \
+		--pull-policy=$(PULL_POLICY) --k8s-version=$(K8S_VERSION) --service-type=$(GW_SERVICE_TYPE) \
+		--is-gke-internal-lb=$(GW_SVC_GKE_INTERNAL)
 
 .PHONY: delete-kind-cluster
 delete-kind-cluster: ## Delete kind cluster
diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh
index 3bd3e47fb..67024ef56 100644
--- a/tests/scripts/remote-scripts/run-tests.sh
+++ b/tests/scripts/remote-scripts/run-tests.sh
@@ -6,4 +6,4 @@ source ~/vars.env
 
 echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.profile && . $HOME/.profile
 
-cd nginx-gateway-fabric/tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true
+cd nginx-gateway-fabric/tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION}
diff --git a/tests/scripts/vars.env-example b/tests/scripts/vars.env-example
index 196f9d0dc..4de2c4955 100644
--- a/tests/scripts/vars.env-example
+++ b/tests/scripts/vars.env-example
@@ -19,3 +19,4 @@ GINKGO_FLAGS=<optional flags to pass to the go test command>
 SOURCE_IP_RANGE=<IPs that should be allowed SSH to the VM, e.g. 1.2.3.4/32 or $(curl -sS -4 icanhazip.com)/32>
 ADD_VM_IP_AUTH_NETWORKS=<if set to true, the script will add the VM IP to the list of the GKE cluster master authorized networks>
 PLUS_ENABLED=<enable nginx plus>
+NGF_VERSION=<version of NGF being tested. Defaults to value of TAG if not set>
diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go
index 1a3e15718..8d2af38b5 100644
--- a/tests/suite/system_suite_test.go
+++ b/tests/suite/system_suite_test.go
@@ -49,6 +49,7 @@ var (
 	ngfImageRepository   = flag.String("ngf-image-repo", "", "Image repo for NGF control plane")
 	nginxImageRepository = flag.String("nginx-image-repo", "", "Image repo for NGF data plane")
 	imageTag             = flag.String("image-tag", "", "Image tag for NGF images")
+	versionUnderTest     = flag.String("version-under-test", "", "Version of NGF that is being tested")
 	imagePullPolicy      = flag.String("pull-policy", "", "Image pull policy for NGF images")
 	serviceType          = flag.String("service-type", "NodePort", "Type of service fronting NGF to be deployed")
 	isGKEInternalLB      = flag.Bool("is-gke-internal-lb", false, "Is the LB service GKE internal only")
@@ -130,7 +131,9 @@ func setup(cfg setupConfig, extraInstallArgs ...string) {
 		installCfg.ImagePullPolicy = *imagePullPolicy
 	}
 
-	if *imageTag != "" {
+	if *versionUnderTest != "" {
+		version = *versionUnderTest
+	} else if *imageTag != "" {
 		version = *imageTag
 	} else {
 		version = "edge"