feat: add generation of gateway-api coverage data

implementation/cilium.sh

@@ -39,18 +39,14 @@ EOF
 run::cilium::conformance() {
   echo "run::cilium::conformance"
 
-  if [[ ! -d ${IMPLEMENTATION_REPO_PATH} ]]; then
-    echo "IMPLEMENTATION_REPO_PATH: ${IMPLEMENTATION_REPO_PATH}"
-    echo "Repo for \"${IMPLEMENTATION}\" doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/cilium/cilium.git
-    cd - || exit
-  fi
-  pushd repos/cilium || exit 1
-
-  git checkout ${IMPLEMENTATION_VERSION}
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   case ${GATEWAY_API_VERSION} in
 
@@ -60,8 +56,8 @@ run::cilium::conformance() {
       ;;
 
     "v1.1.0")
-      CONFORMANCE_PROFILES="HTTP,TLS"
-      SUPPORTED_FEATURES="ReferenceGrant,HTTPRoute,TLSRoute,HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,GatewayClassObservedGenerationBump,HTTPRouteHostRewrite,HTTPRoutePathRewrite,HTTPRouteSchemeRedirect,HTTPRoutePathRedirect,HTTPRoutePortRedirect,HTTPRouteRequestMirror,HTTPRouteRequestMultipleMirrors"
+      CONFORMANCE_PROFILES="GATEWAY-HTTP,GATEWAY-TLS"
+      EXEMPT_FEATURES="GatewayStaticAddresses,HTTPRouteParentRefPort,MeshConsumerRoute"
       ;;
 
     *)
@@ -75,20 +71,20 @@ run::cilium::conformance() {
 
   GATEWAY_API_CONFORMANCE_TESTS=1 go test \
     -p 4 \
-    -v ./operator/pkg/gateway-api \
+    ./conformance/ \
     --gateway-class cilium \
+    --supported-features "${SUPPORTED_FEATURES:-}" \
+    --exempt-features="${EXEMPT_FEATURES:-""}" \
+    --conformance-profiles="${CONFORMANCE_PROFILES:-""}" \
     --report-output=${REPORT} \
     --organization=cilium \
     --project=cilium \
-    --url=https://github.com/cilium/cilium \
-    --version="$IMPLEMENTATION_VERSION" \
-    --contact='https://github.com/cilium/community/blob/main/roles/Maintainers.md' \
-    --supported-features="${SUPPORTED_FEATURES:-""}" \
-    --exempt-features="${EXEMPT_FEATURES:-""}" \
-    --conformance-profiles="${CONFORMANCE_PROFILES:-""}" \
+    --url=https://cilium.io/ \
+    --version="${IMPLEMENTATION_VERSION}" \
+    --contact='@cilium/maintainers' \
     -test.run "TestConformance" \
-    -test.skip "${SKIP_TESTS:-""}" \
-    ${ALL_FEATURES:-""}
+    -test.skip "${SKIP_TESTS:-}" \
+    -test.v 10
 
   popd || exit
 

implementation/contour.sh

@@ -42,19 +42,14 @@ run::contour::conformance() {
 run::contour::gateway-api-conformance() {
   echo "run::contour::gateway-api-conformance"
 
-  export GATEWAY_API_REPO_PATH=${GATEWAY_API_REPO_PATH:-"${PWD}/repos/gateway-api}"}
-  if [[ ! -d ${GATEWAY_API_REPO_PATH} ]]; then
-    echo "GATEWAY_API_REPO_PATH: ${GATEWAY_API_REPO_PATH}"
-    echo "Repo doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/kubernetes-sigs/gateway-api.git || true
-    cd - || exit
-  fi
-  pushd repos/gateway-api/conformance || exit 1
-
-  git checkout ${GATEWAY_API_VERSION}
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   SUPPORTED_FEATURES="Gateway,HTTPRoute"
   SKIP_TESTS="Mesh"
@@ -63,6 +58,7 @@ run::contour::gateway-api-conformance() {
 
   GATEWAY_API_CONFORMANCE_TESTS=1 go test \
     -p 4 \
+    ./conformance/ \
     --gateway-class contour \
     --supported-features "${SUPPORTED_FEATURES}" \
     --report-output=${REPORT} \
@@ -72,7 +68,7 @@ run::contour::gateway-api-conformance() {
     --version=v1.29.0 \
     --contact='@projectcontour/maintainers' \
     -test.run "TestConformance" \
-    -test.skip "${SKIP_TESTS}" \
+    -test.skip "${SKIP_TESTS:-}" \
     -test.v 10
 
   popd || exit

implementation/envoy-gateway.sh

@@ -21,19 +21,14 @@ run::envoy-gateway::conformance() {
 run::envoy-gateway::gateway-api-conformance() {
   echo "run::envoy-gateway::gateway-api-conformance"
 
-  export GATEWAY_API_REPO_PATH=${GATEWAY_API_REPO_PATH:-"${PWD}/repos/gateway-api}"}
-  if [[ ! -d ${GATEWAY_API_REPO_PATH} ]]; then
-    echo "GATEWAY_API_REPO_PATH: ${GATEWAY_API_REPO_PATH}"
-    echo "Repo doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/kubernetes-sigs/gateway-api.git
-    cd - || exit
-  fi
-  pushd repos/gateway-api/conformance || exit 1
-
-  git checkout ${GATEWAY_API_VERSION}
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   SUPPORTED_FEATURES="Gateway,HTTPRoute"
   SKIP_TESTS="Mesh"
@@ -42,6 +37,7 @@ run::envoy-gateway::gateway-api-conformance() {
 
   GATEWAY_API_CONFORMANCE_TESTS=1 go test \
     -p 4 \
+    ./conformance \
     --gateway-class eg \
     --supported-features "${SUPPORTED_FEATURES}" \
     --report-output=${REPORT} \
@@ -51,7 +47,7 @@ run::envoy-gateway::gateway-api-conformance() {
     --version=v1.0.1 \
     --contact=https://github.com/envoyproxy/gateway/blob/main/GOVERNANCE.md \
     -test.run "TestConformance" \
-    -test.skip "${SKIP_TESTS}" \
+    -test.skip "${SKIP_TESTS:-}" \
     -test.v 10
 
   popd || exit

implementation/istio.sh

@@ -28,39 +28,34 @@ EOF
 run::istio::conformance() {
   echo "run::istio::conformance"
 
-  if [[ ! -d ${IMPLEMENTATION_REPO_PATH} ]]; then
-    echo "IMPLEMENTATION_REPO_PATH: ${IMPLEMENTATION_REPO_PATH}"
-    echo "Repo for \"${IMPLEMENTATION}\" doesn't exists, cloning repo"
-
-    mkdir -p repos
-    cd $_ || exit
-    git clone https://github.com/istio/istio.git
-    cd - || exit
-  fi
-  pushd repos/istio || exit 1
-  git fetch --tags -a
-  git checkout "$IMPLEMENTATION_VERSION"
+  mkdir -p repos
+  cd $_ || exit
+  [ -d "gateway-api-${GATEWAY_API_VERSION}" ] || git clone https://github.com/kubernetes-sigs/gateway-api.git gateway-api-"${GATEWAY_API_VERSION}"
+  cd - || exit
+  pushd repos/gateway-api-"${GATEWAY_API_VERSION}" || exit 1
+  git reset --hard HEAD
+  git checkout "${GATEWAY_API_VERSION}"
+  git apply ../../lib/gateway-api-"${GATEWAY_API_VERSION}"-useragent.patch
 
   SUPPORTED_FEATURES="Gateway,HTTPRoute,HTTPRouteDestinationPortMatching,HTTPRouteHostRewrite,HTTPRouteMethodMatching,HTTPRoutePathRedirect,HTTPRoutePathRewrite,HTTPRoutePortRedirect,HTTPRouteQueryParamMatching,HTTPRouteRequestMirror,HTTPRouteRequestMultipleMirrors,HTTPRouteResponseHeaderModification,HTTPRouteSchemeRedirect,ReferenceGrant,TLSRoute"
   SKIP_TESTS=""
   CURRENT_DATE_TIME=$(date +"%Y%m%d-%H%M")
   REPORT="/tmp/conformance-suite-report-${CURRENT_DATE_TIME}-istio.yaml"
 
-  go test \
-    -v -timeout 60m \
-    -run TestGatewayConformance \
-    -skip "$SKIP_TESTS" \
-    ./tests/integration/pilot \
-    -tags=integ \
-    --gateway-class=istio \
-    --all-features \
-    --contact='@istio/maintainers' \
-    --version="$IMPLEMENTATION_VERSION" \
-    --report-output="$REPORT" \
-    --url="https://istio.io" \
-    --project=istio \
+  GATEWAY_API_CONFORMANCE_TESTS=1 go test \
+    -p 4 \
+    ./conformance/ \
+    --gateway-class istio \
+    --supported-features "${SUPPORTED_FEATURES:-}" \
+    --report-output=${REPORT} \
     --organization=istio \
-    --supported-features="$SUPPORTED_FEATURES"
+    --project=istio \
+    --url=https://istio.io/ \
+    --version="$IMPLEMENTATION_VERSION" \
+    --contact='@istio/maintainers' \
+    -test.run "TestConformance" \
+    -test.skip "${SKIP_TESTS:-}" \
+    -test.v 10
 
   popd || exit
 

lib/gateway-api-v1.0.0-useragent.patch

@@ -0,0 +1,86 @@
+diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go
+index bbd65634..9ec86970 100644
+--- a/conformance/conformance_test.go
++++ b/conformance/conformance_test.go
+@@ -38,6 +38,7 @@ func TestConformance(t *testing.T) {
+ 	if err != nil {
+ 		t.Fatalf("Error loading Kubernetes config: %v", err)
+ 	}
++	cfg.UserAgent = "gateway-api-conformance.test::v1.0.0::unknownTest::unknownFeature"
+ 	client, err := client.New(cfg, client.Options{})
+ 	if err != nil {
+ 		t.Fatalf("Error initializing Kubernetes client: %v", err)
+diff --git a/conformance/utils/suite/suite.go b/conformance/utils/suite/suite.go
+index 58a55624..be8e1c8f 100644
+--- a/conformance/utils/suite/suite.go
++++ b/conformance/utils/suite/suite.go
+@@ -21,10 +21,12 @@ import (
+ 	"strings"
+ 	"testing"
+
++	"github.com/stretchr/testify/require"
+ 	"k8s.io/apimachinery/pkg/util/sets"
+ 	clientset "k8s.io/client-go/kubernetes"
+ 	"k8s.io/client-go/rest"
+ 	"sigs.k8s.io/controller-runtime/pkg/client"
++	clicfg "sigs.k8s.io/controller-runtime/pkg/client/config"
+
+ 	"sigs.k8s.io/gateway-api/apis/v1beta1"
+ 	"sigs.k8s.io/gateway-api/conformance"
+@@ -157,6 +159,10 @@ func New(s Options) *ConformanceTestSuite {
+ 	return suite
+ }
+
++const (
++	testSuiteUserAgentPrefix = "gateway-api-conformance.test"
++)
++
+ // Setup ensures the base resources required for conformance tests are installed
+ // in the cluster. It also ensures that all relevant resources are ready.
+ func (suite *ConformanceTestSuite) Setup(t *testing.T) {
+@@ -206,10 +212,45 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) {
+ 	}
+ }
+
++func (suite *ConformanceTestSuite) setClientsetForTest(test ConformanceTest) error {
++	cfg, err := clicfg.GetConfig()
++	if err != nil {
++		return err
++	}
++	featureNames := []string{}
++	for _, v := range test.Features {
++		featureNames = append(featureNames, string(v))
++	}
++	if len(test.Features) == 0 {
++		featureNames = []string{"unknownFeature"}
++	}
++	cfg.UserAgent = strings.Join(
++		[]string{
++			testSuiteUserAgentPrefix,
++			"v1.0.0", // NOTE version not exported in v1.0.0
++			test.ShortName,
++			strings.Join(featureNames, ","),
++		},
++		"::")
++	client, err := client.New(cfg, client.Options{})
++	if err != nil {
++		return err
++	}
++	clientset, err := clientset.NewForConfig(cfg)
++	if err != nil {
++		return err
++	}
++	suite.Client = client
++	suite.Clientset = clientset
++	return nil
++}
++
+ // Run runs the provided set of conformance tests.
+ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
+ 	for _, test := range tests {
+ 		t.Run(test.ShortName, func(t *testing.T) {
++			err := suite.setClientsetForTest(test)
++			require.NoError(t, err, "failed to create new clientset for test")
+ 			test.Run(t, suite)
+ 		})
+ 	}

lib/gateway-api-v1.1.0-useragent.patch

@@ -0,0 +1,72 @@
+diff --git a/conformance/utils/suite/suite.go b/conformance/utils/suite/suite.go
+index 3e6e1a07..0c937ebb 100644
+--- a/conformance/utils/suite/suite.go
++++ b/conformance/utils/suite/suite.go
+@@ -35,6 +35,7 @@ import (
+ 	clientset "k8s.io/client-go/kubernetes"
+ 	"k8s.io/client-go/rest"
+ 	"sigs.k8s.io/controller-runtime/pkg/client"
++	clicfg "sigs.k8s.io/controller-runtime/pkg/client/config"
+
+ 	"sigs.k8s.io/gateway-api/apis/v1beta1"
+ 	confv1 "sigs.k8s.io/gateway-api/conformance/apis/v1"
+@@ -307,6 +308,10 @@ func NewConformanceTestSuite(options ConformanceOptions) (*ConformanceTestSuite,
+ // Conformance Test Suite - Public Methods
+ // -----------------------------------------------------------------------------
+
++const (
++	testSuiteUserAgentPrefix = "gateway-api-conformance.test"
++)
++
+ // Setup ensures the base resources required for conformance tests are installed
+ // in the cluster. It also ensures that all relevant resources are ready.
+ func (suite *ConformanceTestSuite) Setup(t *testing.T, tests []ConformanceTest) {
+@@ -374,6 +379,39 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T, tests []ConformanceTest)
+ 	}
+ }
+
++func (suite *ConformanceTestSuite) setClientsetForTest(test ConformanceTest) error {
++	cfg, err := clicfg.GetConfig()
++	if err != nil {
++		return err
++	}
++	featureNames := []string{}
++	for _, v := range test.Features {
++		featureNames = append(featureNames, string(v))
++	}
++	if len(test.Features) == 0 {
++		featureNames = []string{"unknownFeature"}
++	}
++	cfg.UserAgent = strings.Join(
++		[]string{
++			testSuiteUserAgentPrefix,
++			suite.apiVersion,
++			test.ShortName,
++			strings.Join(featureNames, ","),
++		},
++		"::")
++	client, err := client.New(cfg, client.Options{})
++	if err != nil {
++		return err
++	}
++	clientset, err := clientset.NewForConfig(cfg)
++	if err != nil {
++		return err
++	}
++	suite.Client = client
++	suite.Clientset = clientset
++	return nil
++}
++
+ // Run runs the provided set of conformance tests.
+ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) error {
+ 	// verify that the test suite isn't already running, don't start a new run
+@@ -394,6 +432,8 @@ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) er
+ 	results := make(map[string]testResult)
+ 	for _, test := range tests {
+ 		succeeded := t.Run(test.ShortName, func(t *testing.T) {
++			err := suite.setClientsetForTest(test)
++			require.NoError(t, err, "failed to create new clientset for test")
+ 			test.Run(t, suite)
+ 		})
+ 		res := testSucceeded