feat: use upstream gateway-api conformance tests

- make all implementations use upstream gateway-api conformance tests - patch gateway-api to inject test and feature information in user-agent
apisnoop · Jul 15, 2024 · 23a2424 · 23a2424
1 parent 871006b
commit 23a2424
Show file tree

Hide file tree

Showing 11 changed files with 156 additions and 290 deletions.
diff --git a/generate-data.sh b/generate-data.sh
diff --git a/implementation/cilium.sh b/implementation/cilium.sh
@@ -39,18 +39,14 @@ EOF
 run::cilium::conformance() {
   echo "run::cilium::conformance"
 
-  if [[ ! -d ${IMPLEMENTATION_REPO_PATH} ]]; then
-    echo "IMPLEMENTATION_REPO_PATH: ${IMPLEMENTATION_REPO_PATH}"
-    echo "Repo for \"${IMPLEMENTATION}\" doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/cilium/cilium.git
-    cd - || exit
-  fi
-  pushd repos/cilium || exit 1
-
-  git checkout ${IMPLEMENTATION_VERSION}
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   case ${GATEWAY_API_VERSION} in
 
@@ -75,20 +71,20 @@ run::cilium::conformance() {
 
   GATEWAY_API_CONFORMANCE_TESTS=1 go test \
     -p 4 \
-    -v ./operator/pkg/gateway-api \
+    ./conformance/ \
     --gateway-class cilium \
+    --supported-features "${SUPPORTED_FEATURES:-}" \
+    --exempt-features="${EXEMPT_FEATURES:-""}" \
+    --conformance-profiles="${CONFORMANCE_PROFILES:-""}" \
     --report-output=${REPORT} \
     --organization=cilium \
     --project=cilium \
-    --url=https://github.com/cilium/cilium \
-    --version="$IMPLEMENTATION_VERSION" \
-    --contact='https://github.com/cilium/community/blob/main/roles/Maintainers.md' \
-    --supported-features="${SUPPORTED_FEATURES:-""}" \
-    --exempt-features="${EXEMPT_FEATURES:-""}" \
-    --conformance-profiles="${CONFORMANCE_PROFILES:-""}" \
+    --url=https://cilium.io/ \
+    --version="${IMPLEMENTATION_VERSION}" \
+    --contact='@cilium/maintainers' \
     -test.run "TestConformance" \
-    -test.skip "${SKIP_TESTS:-""}" \
-    ${ALL_FEATURES:-""}
+    -test.skip "${SKIP_TESTS:-}" \
+    -test.v 10
 
   popd || exit
 

diff --git a/implementation/contour.sh b/implementation/contour.sh
@@ -42,19 +42,14 @@ run::contour::conformance() {
 run::contour::gateway-api-conformance() {
   echo "run::contour::gateway-api-conformance"
 
-  export GATEWAY_API_REPO_PATH=${GATEWAY_API_REPO_PATH:-"${PWD}/repos/gateway-api}"}
-  if [[ ! -d ${GATEWAY_API_REPO_PATH} ]]; then
-    echo "GATEWAY_API_REPO_PATH: ${GATEWAY_API_REPO_PATH}"
-    echo "Repo doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/kubernetes-sigs/gateway-api.git || true
-    cd - || exit
-  fi
-  pushd repos/gateway-api/conformance || exit 1
-
-  git checkout ${GATEWAY_API_VERSION}
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   SUPPORTED_FEATURES="Gateway,HTTPRoute"
   SKIP_TESTS="Mesh"
@@ -63,6 +58,7 @@ run::contour::gateway-api-conformance() {
 
   GATEWAY_API_CONFORMANCE_TESTS=1 go test \
     -p 4 \
+    ./conformance/ \
     --gateway-class contour \
     --supported-features "${SUPPORTED_FEATURES}" \
     --report-output=${REPORT} \
@@ -72,7 +68,7 @@ run::contour::gateway-api-conformance() {
     --version=v1.29.0 \
     --contact='@projectcontour/maintainers' \
     -test.run "TestConformance" \
-    -test.skip "${SKIP_TESTS}" \
+    -test.skip "${SKIP_TESTS:-}" \
     -test.v 10
 
   popd || exit

diff --git a/implementation/envoy-gateway.sh b/implementation/envoy-gateway.sh
@@ -21,19 +21,14 @@ run::envoy-gateway::conformance() {
 run::envoy-gateway::gateway-api-conformance() {
   echo "run::envoy-gateway::gateway-api-conformance"
 
-  export GATEWAY_API_REPO_PATH=${GATEWAY_API_REPO_PATH:-"${PWD}/repos/gateway-api}"}
-  if [[ ! -d ${GATEWAY_API_REPO_PATH} ]]; then
-    echo "GATEWAY_API_REPO_PATH: ${GATEWAY_API_REPO_PATH}"
-    echo "Repo doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/kubernetes-sigs/gateway-api.git
-    cd - || exit
-  fi
-  pushd repos/gateway-api/conformance || exit 1
-
-  git checkout ${GATEWAY_API_VERSION}
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   SUPPORTED_FEATURES="Gateway,HTTPRoute"
   SKIP_TESTS="Mesh"
@@ -42,6 +37,7 @@ run::envoy-gateway::gateway-api-conformance() {
 
   GATEWAY_API_CONFORMANCE_TESTS=1 go test \
     -p 4 \
+    ./conformance \
     --gateway-class eg \
     --supported-features "${SUPPORTED_FEATURES}" \
     --report-output=${REPORT} \
@@ -51,7 +47,7 @@ run::envoy-gateway::gateway-api-conformance() {
     --version=v1.0.1 \
     --contact=https://github.com/envoyproxy/gateway/blob/main/GOVERNANCE.md \
     -test.run "TestConformance" \
-    -test.skip "${SKIP_TESTS}" \
+    -test.skip "${SKIP_TESTS:-}" \
     -test.v 10
 
   popd || exit

diff --git a/implementation/istio.sh b/implementation/istio.sh
@@ -21,39 +21,34 @@ EOF
 run::istio::conformance() {
   echo "run::istio::conformance"
 
-  if [[ ! -d ${IMPLEMENTATION_REPO_PATH} ]]; then
-    echo "IMPLEMENTATION_REPO_PATH: ${IMPLEMENTATION_REPO_PATH}"
-    echo "Repo for \"${IMPLEMENTATION}\" doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/istio/istio.git
-    cd - || exit
-  fi
-  pushd repos/istio || exit 1
-  git fetch --tags -a
-  git checkout "$IMPLEMENTATION_VERSION"
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   SUPPORTED_FEATURES="Gateway,HTTPRoute,HTTPRouteDestinationPortMatching,HTTPRouteHostRewrite,HTTPRouteMethodMatching,HTTPRoutePathRedirect,HTTPRoutePathRewrite,HTTPRoutePortRedirect,HTTPRouteQueryParamMatching,HTTPRouteRequestMirror,HTTPRouteRequestMultipleMirrors,HTTPRouteResponseHeaderModification,HTTPRouteSchemeRedirect,ReferenceGrant,TLSRoute"
   SKIP_TESTS=""
   CURRENT_DATE_TIME=$(date +"%Y%m%d-%H%M")
   REPORT="/tmp/conformance-suite-report-${CURRENT_DATE_TIME}-istio.yaml"
 
-  go test \
-    -v -timeout 60m \
-    -run TestGatewayConformance \
-    -skip "$SKIP_TESTS" \
-    ./tests/integration/pilot \
-    -tags=integ \
-    --gateway-class=istio \
-    --all-features \
-    --contact='@istio/maintainers' \
-    --version="$IMPLEMENTATION_VERSION" \
-    --report-output="$REPORT" \
-    --url="https://istio.io" \
-    --project=istio \
+  GATEWAY_API_CONFORMANCE_TESTS=1 go test \
+    -p 4 \
+    ./conformance/ \
+    --gateway-class istio \
+    --supported-features "${SUPPORTED_FEATURES:-}" \
+    --report-output=${REPORT} \
     --organization=istio \
-    --supported-features="$SUPPORTED_FEATURES"
+    --project=istio \
+    --url=https://istio.io/ \
+    --version="$IMPLEMENTATION_VERSION" \
+    --contact='@istio/maintainers' \
+    -test.run "TestConformance" \
+    -test.skip "${SKIP_TESTS:-}" \
+    -test.v 10
 
   popd || exit
 

diff --git a/lib/209_endpoint_hit_by_new_test.sql b/lib/209_endpoint_hit_by_new_test.sql
diff --git a/lib/210_projected_change_in_coverage.sql b/lib/210_projected_change_in_coverage.sql
diff --git a/lib/gateway-api-v1.0.0-useragent.patch b/lib/gateway-api-v1.0.0-useragent.patch
@@ -0,0 +1,52 @@
+diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go
+index bbd65634..de2e5222 100644
+--- a/conformance/conformance_test.go
++++ b/conformance/conformance_test.go
+@@ -38,6 +38,7 @@ func TestConformance(t *testing.T) {
+ 	if err != nil {
+ 		t.Fatalf("Error loading Kubernetes config: %v", err)
+ 	}
++	cfg.UserAgent = "gateway-api-conformance.test::v1.0.0::::"
+ 	client, err := client.New(cfg, client.Options{})
+ 	if err != nil {
+ 		t.Fatalf("Error initializing Kubernetes client: %v", err)
+diff --git a/conformance/utils/suite/suite.go b/conformance/utils/suite/suite.go
+index 58a55624..c852ac5c 100644
+--- a/conformance/utils/suite/suite.go
++++ b/conformance/utils/suite/suite.go
+@@ -21,10 +21,12 @@ import (
+ 	"strings"
+ 	"testing"
+
++	"github.com/stretchr/testify/require"
+ 	"k8s.io/apimachinery/pkg/util/sets"
+ 	clientset "k8s.io/client-go/kubernetes"
+ 	"k8s.io/client-go/rest"
+ 	"sigs.k8s.io/controller-runtime/pkg/client"
++	clicfg "sigs.k8s.io/controller-runtime/pkg/client/config"
+
+ 	"sigs.k8s.io/gateway-api/apis/v1beta1"
+ 	"sigs.k8s.io/gateway-api/conformance"
+@@ -210,6 +212,22 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) {
+ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
+ 	for _, test := range tests {
+ 		t.Run(test.ShortName, func(t *testing.T) {
++			cfg, err := clicfg.GetConfig()
++			require.NoError(t, err, "error getting Kubernetes config")
++			featureNames := []string{}
++			for _, v := range test.Features {
++				featureNames = append(featureNames, string(v))
++			}
++			cfg.UserAgent = "gateway-api-conformance.test::" +
++				"v1.0.0" + "::" + test.ShortName + "::" + strings.Join(featureNames, ",")
++			require.NoError(t, err, "error loading Kubernetes config")
++			client, err := client.New(cfg, client.Options{})
++			require.NoError(t, err, "error initializing Kubernetes client")
++			clientset, err := clientset.NewForConfig(cfg)
++			require.NoError(t, err, "error initializing Kubernetes clientset")
++			suite.Client = client
++			suite.Clientset = clientset
++
+ 			test.Run(t, suite)
+ 		})
+ 	}