[WIP] re-enable some e2e tests

.github/workflows/pr-kubernetes-tests.yaml

@@ -0,0 +1,139 @@
+name: Kubernetes Tests
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+env:
+  VERSION: '1.0.0-ci1'
+  GITHUB_TOKEN: ${{ github.token }}
+
+jobs:
+  # TODO consider adding back later if we want to handle skip directives.
+  # currently we don't have changelogs to add the skip directives to
+  # prepare_env:
+  #   name: Prepare Environment
+  #   runs-on: ubuntu-22.04
+  #   timeout-minutes: 5
+  #   outputs:
+  #     should-auto-succeed-kube-tests: ${{ steps.run-strategy.outputs.auto_succeed }}
+  #   steps:
+  #   - name: Cancel Previous Actions
+  #     uses: styfle/[email protected]
+  #     with:
+  #       access_token: ${{ github.token }}
+  #   - id: checkout-code
+  #     uses: actions/checkout@v4
+  #     with:
+  #       # We require gathering the branch and tag history since we rely on a `git diff`
+  #       # which compares the state of two branches
+  #       fetch-depth: 0
+  #   - id: process-skip-directives
+  #     uses: ./.github/actions/process-skip-directives
+  #     with:
+  #       base-ref: ${{ github.base_ref }}
+  #   - id: run-strategy
+  #     name: Determine Test Run Strategy
+  #     run: |
+  #       skip_kube_tests=${{ steps.process-skip-directives.outputs.skip-kube-tests }}
+  #       if [[ ! -z $skip_kube_tests && $skip_kube_tests = true ]] ; then
+  #         echo "auto_succeed=true" >> $GITHUB_OUTPUT
+  #       fi
+
+  end_to_end_tests:
+    name: End-to-End (${{ matrix.test.cluster-name }})
+    # needs: prepare_env
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    if: ${{ !github.event.pull_request.draft }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # We intentionally set an upper threshold of 30 minutes for our End-to-End tests
+        # Our goal is to load balance tests in a way that allows quick iteration on PRs
+        # If tests are exceeding the 30-minute limit, please see:
+        # /test/kubernetes/e2e/load_balancing_tests.md
+        #
+        # Above each test below, we document the latest date/time for the GitHub action step to run
+        # NOTE: We use the GitHub action step time (as opposed to the `go test` time), because it is easier to capture
+
+        test:
+        - cluster-name: 'cluster-one'
+          go-test-args: '-v -timeout=25m'
+          go-test-run-regex: '^TestK8sGateway$$/^RouteDelegation$$'
+        # Dec 4, 2024: 22 minutes
+        # - cluster-name: 'cluster-one'
+        #   go-test-args: '-v -timeout=25m'
+        #   go-test-run-regex: '^TestK8sGateway$$/^RouteDelegation$$|^TestGlooctlGlooGatewayEdgeGateway$$|^TestGlooctlK8sGateway$$'
+
+        # # Dec 4, 2024: 23 minutes
+        # - cluster-name: 'cluster-two'
+        #   go-test-args: '-v -timeout=25m'
+        #   go-test-run-regex: '^TestK8sGatewayIstioRevision$$|^TestRevisionIstioRegression$$|^TestK8sGateway$$/^Deployer$$|^TestK8sGateway$$/^RouteOptions$$|^TestK8sGateway$$/^VirtualHostOptions$$|^TestK8sGateway$$/^Upstreams$$|^TestK8sGateway$$/^HeadlessSvc$$|^TestK8sGateway$$/^PortRouting$$|^TestK8sGatewayMinimalDefaultGatewayParameters$$|^TestK8sGateway$$/^DirectResponse$$|^TestK8sGateway$$/^HttpListenerOptions$$|^TestK8sGateway$$/^ListenerOptions$$|^TestK8sGateway$$/^GlooAdminServer$$'
+
+        # # Dec 4, 2024: 24 minutes
+        # - cluster-name: 'cluster-three'
+        #   go-test-args: '-v -timeout=30m'
+        #   go-test-run-regex: '(^TestK8sGatewayIstioAutoMtls$$|^TestAutomtlsIstioEdgeApisGateway$$|^TestIstioEdgeApiGateway$$|^TestIstioRegression$$)'
+
+        # # Dec 4, 2024: 21 minutes
+        # - cluster-name: 'cluster-four'
+        #   go-test-args: '-v -timeout=30m'
+        #   go-test-run-regex: '(^TestK8sGatewayIstio$$|^TestGlooGatewayEdgeGateway$$|^TestGlooctlIstioInjectEdgeApiGateway$$)'
+
+        # # Dec 4, 2024: 24 minutes
+        # - cluster-name: 'cluster-five'
+        #   go-test-args: '-v -timeout=30m'
+        #   go-test-run-regex: '^TestFullEnvoyValidation$$|^TestValidationStrict$$|^TestValidationAlwaysAccept$$|^TestTransformationValidationDisabled$$'
+
+        # # Dec 4, 2024: 26 minutes
+        # - cluster-name: 'cluster-six'
+        #   go-test-args: '-v -timeout=30m'
+        #   go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestK8sGatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestK8sGatewayAws$$|^TestK8sGateway$$/^HTTPRouteServices$$|^TestK8sGateway$$/^TCPRouteServices$$|^TestZeroDowntimeRollout$$'
+
+        # # Dec 4, 2024: 13 minutes
+        # - cluster-name: 'cluster-seven'
+        #   go-test-args: '-v -timeout=25m'
+        #   go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestWatchNamespaceSelector$$'
+
+        # In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support
+        # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
+        version-files:
+          - file: './.github/workflows/.env/pr-tests/versions.env'
+    steps:
+    # - id: auto-succeed-tests
+    #   if: needs.prepare_env.outputs.should-auto-succeed-kube-tests == 'true'
+    #   run: |
+    #     echo "Regression tests auto-succeeded"
+    - uses: actions/checkout@v4
+    - name: Prep Go Runner
+      uses: ./.github/actions/prep-go-runner
+      # if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true'
+    # The dotenv action is used to load key-value pairs from files.
+    # In this case, the file is specified in the matrix and will contain the versions of the tools to use
+    - name: Dotenv Action
+      uses: falti/[email protected]
+      # if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true'
+      id: dotenv
+      with:
+        path: ${{ matrix.version-files.file }}
+        log-variables: true
+    - id: setup-kind-cluster
+      name: Setup KinD Cluster
+      # if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true'
+      uses: ./.github/actions/setup-kind-cluster
+      with:
+        cluster-name: ${{ matrix.test.cluster-name }}
+        kind-node-version: ${{ steps.dotenv.outputs.node_version }}
+        kind-version: ${{ steps.dotenv.outputs.kind_version }}
+        kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }}
+        helm-version: ${{ steps.dotenv.outputs.helm_version }}
+        istio-version: ${{ steps.dotenv.outputs.istio_version }}
+    - id: run-tests
+      # if: needs.prepare_env.outputs.should-auto-succeed-kube-tests != 'true'
+      uses: ./.github/actions/kubernetes-e2e-tests
+      with:
+        cluster-name: ${{ matrix.test.cluster-name }}
+        test-args: ${{ matrix.test.go-test-args }}
+        run-regex: ${{ matrix.test.go-test-run-regex }}
+        istio-version: ${{ steps.dotenv.outputs.istio_version }}
+        matrix-label: "pr"

test/kubernetes/e2e/features/admin_server/suite.go

@@ -77,10 +77,6 @@ func (s *testingSuite) TestGetInputSnapshotIncludesEdgeApiResources() {
 // TestGetInputSnapshotIncludesK8sGatewayApiResources verifies that we can query the /snapshots/input API and have it return K8s Gateway API
 // resources without an error
 func (s *testingSuite) TestGetInputSnapshotIncludesK8sGatewayApiResources() {
-	if !s.testInstallation.Metadata.K8sGatewayEnabled {
-		s.T().Skip("Installation of Gloo Gateway does not have K8s Gateway enabled, skipping test as there is nothing to test")
-	}
-
 	s.T().Cleanup(func() {
 		err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, gatewayParametersManifest)
 		s.NoError(err, "can delete manifest")

test/kubernetes/e2e/features/virtualhost_options/vhost_opt_suite.go

@@ -157,20 +157,12 @@ func (s *testingSuite) TestConfigureInvalidVirtualHostOptions() {
 	s.testInstallation.Assertions.ExpectObjectAdmitted(manifestVhoWebhookReject, err, output,
 		"Validating *v1.VirtualHostOption failed")
 
-	if !s.testInstallation.Metadata.ValidationAlwaysAccept {
-		s.testInstallation.Assertions.ExpectGlooObjectNotExist(
-			s.ctx,
-			s.getterForMeta(&vhoWebhookReject),
-			&vhoWebhookReject,
-		)
-	} else {
-		// Check status is rejected on bad VirtualHostOption
-		s.testInstallation.Assertions.EventuallyResourceStatusMatchesState(
-			s.getterForMeta(&vhoWebhookReject),
-			core.Status_Rejected,
-			defaults.KubeGatewayReporter,
-		)
-	}
+	// Check status is rejected on bad VirtualHostOption
+	s.testInstallation.Assertions.EventuallyResourceStatusMatchesState(
+		s.getterForMeta(&vhoWebhookReject),
+		core.Status_Rejected,
+		defaults.KubeGatewayReporter,
+	)
 
 	// Check healthy response with no x-bar header
 	s.testInstallation.Assertions.AssertEventualCurlResponse(

test/kubernetes/e2e/suite.go

@@ -1,5 +1,3 @@
-//go:build ignore
-
 package e2e
 
 import (

test/kubernetes/e2e/test.go

@@ -1,5 +1,3 @@
-//go:build ignore
-
 package e2e
 
 import (
@@ -13,14 +11,14 @@
 	"testing"
 	"time"
 
 	"github.com/kgateway-dev/kgateway/test/helpers"
 	"github.com/kgateway-dev/kgateway/test/kubernetes/testutils/actions"
 	"github.com/kgateway-dev/kgateway/test/kubernetes/testutils/assertions"
 	"github.com/kgateway-dev/kgateway/test/kubernetes/testutils/cluster"
 	"github.com/kgateway-dev/kgateway/test/kubernetes/testutils/gloogateway"
 	"github.com/kgateway-dev/kgateway/test/kubernetes/testutils/helper"
 	testruntime "github.com/kgateway-dev/kgateway/test/kubernetes/testutils/runtime"
 	"github.com/kgateway-dev/kgateway/test/testutils"
 )
 
 // MustTestHelper returns the SoloTestHelper used for e2e tests
@@ -185,7 +183,7 @@
 			//	1. First, we define the common recommendations. These are values which we want all tests to inherit by default.
 			//	2. Next, we define a "profile", which is a standard collection of values for a type of installation
 			//	3. Finally, we define test specific values, so that a test can have the final say over which values to define.
 			helper.WithExtraArgs("--values", CommonRecommendationManifest),
 			helper.WithExtraArgs("--values", i.Metadata.ProfileValuesManifestFile),
 			helper.WithExtraArgs("--values", i.Metadata.ValuesManifestFile),
 		)

test/kubernetes/e2e/tests/k8s_gw_aws_test.go

@@ -26,7 +26,6 @@ func TestK8sGatewayAws(t *testing.T) {
 			InstallNamespace:          installNs,
 			ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
 			ValuesManifestFile:        e2e.ManifestPath("aws-lambda-helm.yaml"),
-			K8sGatewayEnabled:         true,
 			// these should correspond to the `settings.aws.*` values in the aws-lambda-helm.yaml manifest
 			AwsOptions: &gloogateway.AwsOptions{
 				EnableServiceAccountCredentials: true,

test/kubernetes/e2e/tests/k8s_gw_istio_test.go

@@ -25,7 +25,6 @@ func TestK8sGatewayIstio(t *testing.T) {
 			InstallNamespace:          installNs,
 			ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
 			ValuesManifestFile:        e2e.ManifestPath("istio-automtls-disabled-helm.yaml"),
-			K8sGatewayEnabled:         true,
 		},
 	)
 

test/kubernetes/e2e/tests/k8s_gw_minimal_default_gatewayparameters_test.go

@@ -26,8 +26,6 @@ func TestK8sGatewayMinimalDefaultGatewayParameters(t *testing.T) {
 			InstallNamespace:          installNs,
 			ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
 			ValuesManifestFile:        e2e.ManifestPath("k8s-gateway-minimal-default-gatewayparameters-test-helm.yaml"),
-			ValidationAlwaysAccept:    false,
-			K8sGatewayEnabled:         true,
 		},
 	)
 

test/kubernetes/e2e/tests/k8s_gw_no_validation_test.go

@@ -25,8 +25,6 @@ func TestK8sGatewayNoValidation(t *testing.T) {
 			InstallNamespace:          installNs,
 			ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
 			ValuesManifestFile:        e2e.ManifestPath("k8s-gateway-no-webhook-validation-test-helm.yaml"),
-			ValidationAlwaysAccept:    true,
-			K8sGatewayEnabled:         true,
 		},
 	)
 

test/kubernetes/e2e/tests/k8s_gw_test.go

@@ -1,5 +1,3 @@
-//go:build ignore
-
 package tests_test
 
 import (
@@ -11,7 +9,7 @@
 	"github.com/kgateway-dev/kgateway/pkg/utils/envutils"
 	"github.com/kgateway-dev/kgateway/test/kubernetes/e2e"
 	. "github.com/kgateway-dev/kgateway/test/kubernetes/e2e/tests"
 	"github.com/kgateway-dev/kgateway/test/kubernetes/testutils/gloogateway"
 	"github.com/kgateway-dev/kgateway/test/testutils"
 )
 
@@ -25,8 +23,6 @@
 			InstallNamespace:          installNs,
 			ProfileValuesManifestFile: e2e.KubernetesGatewayProfilePath,
 			ValuesManifestFile:        e2e.EmptyValuesManifestPath,
-			ValidationAlwaysAccept:    false,
-			K8sGatewayEnabled:         true,
 		},
 	)