Add composite provider to support multiple network providers

.github/workflows/e2e-multi-network-provider.yaml

@@ -0,0 +1,85 @@
+name: E2E-Multiple-NetworkProvider
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+env:
+  # Common versions
+  GO_VERSION: '1.17'
+  KIND_IMAGE: 'kindest/node:v1.23.3'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+
+  rollout:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/[email protected]
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise Rollout
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "1" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-rollout -o yaml
+          set -e
+          if [ "$PODS" -eq "1" ]; then
+            echo "Wait for kruise-rollout ready successfully"
+          else
+            echo "Timeout to wait for kruise-rollout ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          kubectl apply -f ./test/e2e/test_data/customNetworkProvider/istio_crd.yaml
+          kubectl apply -f ./test/e2e/test_data/customNetworkProvider/lua_script_configmap.yaml
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='Canary rollout with multiple network providers' test/e2e
+          retVal=$?
+          # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
+          restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-rollout has not restarted"
+          else
+              kubectl get pod -n kruise-rollout --no-headers
+              echo "Kruise-rollout has restarted, abort!!!"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
+              exit 1
+          fi
+          exit $retVal

lua_configuration/trafficrouting_ingress/nginx.lua

@@ -26,19 +26,21 @@ end
 -- headers & cookie apis
 -- traverse matches
 for _,match in ipairs(obj.matches) do
-    local header = match.headers[1]
-    -- cookie
-    if ( header.name == "canary-by-cookie" )
-    then
-        annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
-    else
-        annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
-        -- if regular expression
-        if ( header.type == "RegularExpression" )
+    if match.headers and next(match.headers) ~= nil then
+        local header = match.headers[1]
+        -- cookie
+        if ( header.name == "canary-by-cookie" )
         then
-            annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
+            annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
         else
-            annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
+            annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
+            -- if regular expression
+            if ( header.type == "RegularExpression" )
+            then
+                annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
+            else
+                annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
+            end
         end
     end
 end

pkg/trafficrouting/manager.go

@@ -336,8 +336,9 @@ func (m *Manager) PatchStableService(c *TrafficRoutingContext) (bool, error) {
 
 func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, cService string) (network.NetworkProvider, error) {
 	trafficRouting := con.ObjectRef[0]
+	networkProviders := make([]network.NetworkProvider, 0, 3)
 	if trafficRouting.CustomNetworkRefs != nil {
-		return custom.NewCustomController(c, custom.Config{
+		np, innerErr := custom.NewCustomController(c, custom.Config{
 			Key:           con.Key,
 			RolloutNs:     con.Namespace,
 			CanaryService: cService,
@@ -347,27 +348,44 @@ func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, c
 			//only set for CustomController, never work for Ingress and Gateway
 			DisableGenerateCanaryService: con.DisableGenerateCanaryService,
 		})
+		if innerErr != nil {
+			return nil, innerErr
+		}
+		networkProviders = append(networkProviders, np)
 	}
 	if trafficRouting.Ingress != nil {
-		return ingress.NewIngressTrafficRouting(c, ingress.Config{
+		np, innerErr := ingress.NewIngressTrafficRouting(c, ingress.Config{
 			Key:           con.Key,
 			Namespace:     con.Namespace,
 			CanaryService: cService,
 			StableService: sService,
 			TrafficConf:   trafficRouting.Ingress,
 			OwnerRef:      con.OwnerRef,
 		})
+		if innerErr != nil {
+			return nil, innerErr
+		}
+		networkProviders = append(networkProviders, np)
 	}
 	if trafficRouting.Gateway != nil {
-		return gateway.NewGatewayTrafficRouting(c, gateway.Config{
+		np, innerErr := gateway.NewGatewayTrafficRouting(c, gateway.Config{
 			Key:           con.Key,
 			Namespace:     con.Namespace,
 			CanaryService: cService,
 			StableService: sService,
 			TrafficConf:   trafficRouting.Gateway,
 		})
+		if innerErr != nil {
+			return nil, innerErr
+		}
+		networkProviders = append(networkProviders, np)
+	}
+	if len(networkProviders) == 0 {
+		return nil, fmt.Errorf("TrafficRouting current only supports Ingress, Gateway API and CustomNetworkRefs")
+	} else if len(networkProviders) == 1 {
+		return networkProviders[0], nil
 	}
-	return nil, fmt.Errorf("TrafficRouting current only support Ingress or Gateway API")
+	return network.CompositeController(networkProviders), nil
 }
 
 func (m *Manager) createCanaryService(c *TrafficRoutingContext, cService string, spec corev1.ServiceSpec) (*corev1.Service, error) {

pkg/trafficrouting/network/composite.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2024 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/util/validation/field"
+
+	"github.com/openkruise/rollouts/api/v1beta1"
+)
+
+var (
+	_ NetworkProvider = (CompositeController)(nil)
+)
+
+// CompositeController is a set of NetworkProvider
+type CompositeController []NetworkProvider
+
+func (c CompositeController) Initialize(ctx context.Context) error {
+	for _, provider := range c {
+		if err := provider.Initialize(ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c CompositeController) EnsureRoutes(ctx context.Context, strategy *v1beta1.TrafficRoutingStrategy) (bool, error) {
+	done := true
+	for _, provider := range c {
+		if innerDone, innerErr := provider.EnsureRoutes(ctx, strategy); innerErr != nil {
+			return false, innerErr
+		} else if !innerDone {
+			done = false
+		}
+	}
+	return done, nil
+}
+
+func (c CompositeController) Finalise(ctx context.Context) (bool, error) {
+	modified := false
+	errList := field.ErrorList{}
+	for _, provider := range c {
+		if updated, innerErr := provider.Finalise(ctx); innerErr != nil {
+			errList = append(errList, field.InternalError(field.NewPath("FinalizeChildNetworkProvider"), innerErr))
+		} else if updated {
+			modified = true
+		}
+	}
+	return modified, errList.ToAggregate()
+}

pkg/trafficrouting/network/interface.go

@@ -27,12 +27,11 @@ type NetworkProvider interface {
 	// Initialize only determine if the network resources(ingress & gateway api) exist.
 	// If error is nil, then the network resources exist.
 	Initialize(ctx context.Context) error
-	// EnsureRoutes check and set canary weight and matches.
-	// weight indicates percentage of traffic to canary service, and range of values[0,100]
-	// matches indicates A/B Testing release for headers, cookies
-	// 1. check if canary has been set desired weight.
-	// 2. If not, set canary desired weight
-	// When the first set weight is returned false, mainly to give the provider some time to process, only when again ensure, will return true
+	// EnsureRoutes check and set routes, e.g. canary weight and match conditions.
+	// 1. Canary weight specifies the relative proportion of traffic to be forwarded to the canary service within the range of [0,100]
+	// 2. Match conditions indicates rules to be satisfied for A/B testing scenarios, such as header, cookie, queryParams etc.
+	// Return true if and only if the route resources have been correctly updated and does not change in this round of reconciliation.
+	// Otherwise, return false to wait for the eventual consistency.
 	EnsureRoutes(ctx context.Context, strategy *v1beta1.TrafficRoutingStrategy) (bool, error)
 	// Finalise will do some cleanup work after the canary rollout complete, such as delete canary ingress.
 	// if error is nil, the return bool value means if the resources are modified