Support non-default namespaced gateway (#217)

* Allow to add non default namespaced gateway

* Allow reconcile service network which for non-default namespace

* Add some examples for non-default namespaced gateway

* Add logic to only delete servicenetwork if not used by all namespaces

* Update debug msg

* Add unit test for non-default namespace gateways modelbuid

* Add unit test on synthesize gateway

* Remove stale code and address CR comments

controllers/gateway_controller.go

@@ -123,13 +123,6 @@ func (r *GatewayReconciler) reconcile(ctx context.Context, req ctrl.Request) err
 		return client.IgnoreNotFound(err)
 	}
 
-	if !r.isDefaultNameSpace(gw.Namespace) {
-		errmsg := "VPC lattice do not support no-default namespace gateway111"
-		glog.V(2).Infof(errmsg)
-		r.updateBadStatus(ctx, errmsg, gw)
-		return nil
-	}
-
 	gwClass := &gateway_api.GatewayClass{}
 	gwClassName := types.NamespacedName{
 		Namespace: defaultNameSpace,
@@ -156,7 +149,7 @@ func (r *GatewayReconciler) reconcile(ctx context.Context, req ctrl.Request) err
 					continue
 				}
 				gwName := types.NamespacedName{
-					Namespace: defaultNameSpace,
+					Namespace: httpRoute.Namespace,
 					Name:      string(httpRoute.Spec.ParentRefs[0].Name),
 				}
 
@@ -166,7 +159,7 @@ func (r *GatewayReconciler) reconcile(ctx context.Context, req ctrl.Request) err
 					continue
 				}
 
-				if httpGW.Name == gw.Name {
+				if httpGW.Name == gw.Name && httpGW.Namespace == gw.Namespace {
 
 					gwLog.Info("Can not delete because it is referenced by some HTTPRoutes")
 					return errors.New("retry later, since it is referenced by some HTTPRoutes")

controllers/httproute_controller.go

@@ -159,12 +159,15 @@ func (r *HTTPRouteReconciler) isHTTPRouteRelevant(ctx context.Context, httpRoute
 		return false
 	}
 
-	// TODO,  gatway are defined in default namespace for now
-	// TODO create a sim for need to handle namespaced gateway
 	gw := &gateway_api.Gateway{}
 
+	// TODO handle multiple parentRefs
+	gwNamespace := "default"
+	if httpRoute.Spec.ParentRefs[0].Namespace != nil {
+		gwNamespace = string(*httpRoute.Spec.ParentRefs[0].Namespace)
+	}
 	gwName := types.NamespacedName{
-		Namespace: "default",
+		Namespace: gwNamespace,
 		// TODO assume one parent for now and point to service network
 		Name: string(httpRoute.Spec.ParentRefs[0].Name),
 	}

examples/gateway-infra-1-namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gw-infra-1

examples/inventory-route-gw-infra-1.yaml

@@ -0,0 +1,17 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: inventory
+  namespace: gw-infra-1
+spec:
+  parentRefs:
+  - name: my-hotel
+    namespace: gw-infra-1
+    sectionName: http 
+  rules:
+  - backendRefs:  
+    - name: inventory-v1
+      namespace: gw-infra-1
+      kind: Service
+      port: 8090
+      weight: 10

examples/inventory-ver1-gw-infra-1.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: inventory-v1
+  namespace: gw-infra-1
+  labels:
+    app: inventory-v1
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: inventory-v1
+  template:
+    metadata:
+      labels:
+        app: inventory-v1
+    spec:
+      containers:
+      - name: inventory-v1
+        image: public.ecr.aws/x2j8p8w7/http-server:latest
+        env:
+        - name: PodName
+          value: "inventory-v1 handler pod" 
+
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: inventory-v1
+  namespace: gw-infra-1
+spec:
+  selector:
+    app: inventory-v1
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8090

examples/my-hotel-gateway-infra-1-ns.yaml

@@ -0,0 +1,13 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: Gateway
+metadata:
+  name: my-hotel
+  namespace: gw-infra-1 
+  annotations:
+    application-networking.k8s.aws/lattice-vpc-association: "true"
+spec:
+  gatewayClassName: amazon-vpc-lattice
+  listeners:
+  - name: http
+    protocol: HTTP
+    port: 80

pkg/deploy/lattice/service_network_synthesizer.go

@@ -8,7 +8,6 @@ import (
 	"github.com/aws/aws-application-networking-k8s/pkg/latticestore"
 	"github.com/aws/aws-application-networking-k8s/pkg/model/core"
 	latticemodel "github.com/aws/aws-application-networking-k8s/pkg/model/lattice"
-	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gateway_api "sigs.k8s.io/gateway-api/apis/v1beta1"
 )
@@ -65,6 +64,27 @@ func (s *serviceNetworkSynthesizer) synthesizeTriggeredGateways(ctx context.Cont
 	for _, resServiceNetwork := range resServiceNetworks {
 		if resServiceNetwork.Spec.IsDeleted {
 			glog.V(6).Infof("Synthersing Gateway: Del %v\n", resServiceNetwork.Spec.Name)
+
+			// TODO need to check if servicenetwork is referenced by gateway in other namespace
+			gwList := &gateway_api.GatewayList{}
+			s.Client.List(context.TODO(), gwList)
+			snUsedByGateway := false
+			for _, gw := range gwList.Items {
+				if gw.Name == resServiceNetwork.Spec.Name &&
+					gw.Namespace != resServiceNetwork.Spec.Namespace {
+					glog.V(6).Infof("Skip deleting gw %v, namespace %v, since it is still used", gw.Name, gw.Namespace)
+					snUsedByGateway = true
+					break
+				}
+			}
+
+			if snUsedByGateway {
+				glog.V(6).Infof("Skiping deleting gw: %v since it is still used by gateway(s)",
+					resServiceNetwork.Spec.Name)
+
+				continue
+			}
+
 			err := s.serviceNetworkManager.Delete(ctx, resServiceNetwork.Spec.Name)
 			if err != nil {
 				ret = LATTICE_RETRY
@@ -108,25 +128,27 @@ func (s *serviceNetworkSynthesizer) synthesizeSDKServiceNetworks(ctx context.Con
 		return err
 	}
 	glog.V(6).Infof("SDK List: %v \n", sdkServiceNetworks)
+
 	// handling delete those gateway in lattice DB, but not in K8S DB
+	// check local K8S cache
+	gwList := &gateway_api.GatewayList{}
+	s.Client.List(context.TODO(), gwList)
+
 	for _, sdkServiceNetwork := range sdkServiceNetworks {
 		glog.V(6).Infof("Synthersizing Gateway: checking if sdkServiceNetwork %v needed to be deleted \n", sdkServiceNetwork)
 
 		toBeDeleted := false
 
-		if !toBeDeleted {
-			// check local K8S cache
-			gw := &gateway_api.Gateway{}
-			gwName := types.NamespacedName{
-				Namespace: "default",
-				Name:      sdkServiceNetwork,
+		snUsedByGateway := false
+		for _, gw := range gwList.Items {
+			if gw.Name == sdkServiceNetwork {
+				snUsedByGateway = true
+				break
 			}
+		}
 
-			if err := s.Client.Get(ctx, gwName, gw); err != nil {
-				glog.V(6).Infof("Synthesizing Gateway %v not found in K8S cache\n", gwName)
-				toBeDeleted = true
-
-			}
+		if !snUsedByGateway {
+			toBeDeleted = true
 		}
 
 		if toBeDeleted {
@@ -138,6 +160,10 @@ func (s *serviceNetworkSynthesizer) synthesizeSDKServiceNetworks(ctx context.Con
 				glog.V(6).Infof("Need to retry synthesizing err %v", err)
 				ret = LATTICE_RETRY
 			}
+		} else {
+			glog.V(6).Infof("Skip deleting sdkServiceNetwork %v since some gateway(s) still reference it",
+				sdkServiceNetwork)
+
 		}
 
 	}

pkg/deploy/lattice/service_network_synthesizer_test.go

@@ -25,6 +25,7 @@ func Test_SynthesizeTriggeredGateways(t *testing.T) {
 	tests := []struct {
 		name                string
 		gw                  *gateway_api.Gateway
+		gwUsedByOtherNS     bool
 		meshManagerErr      error
 		wantSynthesizerErr  error
 		wantDataStoreErr    error
@@ -65,10 +66,26 @@ func Test_SynthesizeTriggeredGateways(t *testing.T) {
 				},
 			},
 			meshManagerErr:      nil,
+			gwUsedByOtherNS:     false,
 			wantSynthesizerErr:  nil,
 			wantDataStoreErr:    errors.New(latticestore.DATASTORE_SERVICE_NETWORK_NOT_EXIST),
 			wantDataStoreStatus: latticestore.DATASTORE_SERVICE_NETWORK_NOT_EXIST,
 		},
+		{
+			name: "Deleting Mesh Skipped due to other NS still uses it",
+			gw: &gateway_api.Gateway{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "mesh3",
+					Finalizers:        []string{"gateway.k8s.aws/resources"},
+					DeletionTimestamp: &now,
+				},
+			},
+			meshManagerErr:      nil,
+			gwUsedByOtherNS:     true,
+			wantSynthesizerErr:  nil,
+			wantDataStoreErr:    nil,
+			wantDataStoreStatus: "",
+		},
 		{
 			name: "Deleting Mesh Successfully in progress",
 			gw: &gateway_api.Gateway{
@@ -87,6 +104,7 @@ func Test_SynthesizeTriggeredGateways(t *testing.T) {
 
 	for _, tt := range tests {
 
+		fmt.Printf("Testing >>>>> %v\n", tt.name)
 		c := gomock.NewController(t)
 		defer c.Finish()
 		ctx := context.TODO()
@@ -101,18 +119,55 @@ func Test_SynthesizeTriggeredGateways(t *testing.T) {
 
 		mockMeshManager := NewMockServiceNetworkManager(c)
 
+		// testing deleting staled mesh (gateway)
+		mock_client := mock_client.NewMockClient(c)
+
 		// testing add or delete of triggered gateway(mesh)
 		if !tt.gw.DeletionTimestamp.IsZero() {
 			// testing delete
 			// insert the record in cache and verify it will be deleted later
 			ds.AddServiceNetwork(tt.gw.Name, config.AccountID, "ARN", "id", latticestore.DATASTORE_SERVICE_NETWORK_CREATED)
-			mockMeshManager.EXPECT().Delete(ctx, tt.gw.Name).Return(tt.meshManagerErr)
+
+			gwList := &gateway_api.GatewayList{}
+
+			gwList.Items = append(gwList.Items,
+				gateway_api.Gateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: tt.gw.GetObjectMeta().GetName(),
+					},
+				})
+			if tt.gwUsedByOtherNS {
+				gwList.Items = append(gwList.Items,
+					gateway_api.Gateway{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      tt.gw.GetObjectMeta().GetName(),
+							Namespace: "non-default",
+						},
+					},
+				)
+			}
+
+			mock_client.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn(
+				func(ctx context.Context, retGWList *gateway_api.GatewayList, arg3 ...interface{}) error {
+					// return empty gatway
+					for _, gw := range gwList.Items {
+						fmt.Printf("liwwu>>> test append %v\n", gw)
+						retGWList.Items = append(retGWList.Items, gw)
+					}
+					return nil
+
+				},
+			)
+
+			if !tt.gwUsedByOtherNS {
+				mockMeshManager.EXPECT().Delete(ctx, tt.gw.Name).Return(tt.meshManagerErr)
+			}
 		} else {
 			meshStatus = latticemodel.ServiceNetworkStatus{ServiceNetworkARN: "testing arn", ServiceNetworkID: "87654321"}
 			mockMeshManager.EXPECT().Create(ctx, mesh).Return(meshStatus, tt.meshManagerErr)
 		}
 
-		meshMeshSynthesizer := NewServiceNetworkSynthesizer(nil, mockMeshManager, stack, ds)
+		meshMeshSynthesizer := NewServiceNetworkSynthesizer(mock_client, mockMeshManager, stack, ds)
 		err := meshMeshSynthesizer.synthesizeTriggeredGateways(ctx)
 
 		assert.Equal(t, tt.wantSynthesizerErr, err)
@@ -177,6 +232,7 @@ func Test_SythesizeSDKMeshs(t *testing.T) {
 	}
 
 	for _, tt := range tests {
+		fmt.Printf("Testing >>>>> %v\n", tt.name)
 		c := gomock.NewController(t)
 		defer c.Finish()
 		ctx := context.TODO()
@@ -192,18 +248,40 @@ func Test_SythesizeSDKMeshs(t *testing.T) {
 		if len(tt.sdkMeshes) > 0 {
 			fmt.Printf("Testing deleting non-existing SDK mesh")
 
+			gwList := &gateway_api.GatewayList{}
+
 			for _, sdkMesh := range tt.sdkMeshes {
 				fmt.Printf("sdkMesh %v\n", sdkMesh)
 				sdkMeshsReturned = append(sdkMeshsReturned, sdkMesh.name)
 				fmt.Printf("sdkMeshsReturned --loop %v\n", sdkMeshsReturned)
 				ds.AddServiceNetwork(sdkMesh.name, config.AccountID, "staleMeshARN", "staleMeshId", latticestore.DATASTORE_SERVICE_NETWORK_CREATED)
+				if !sdkMesh.isStale {
+					gwList.Items = append(gwList.Items,
+						gateway_api.Gateway{
+							ObjectMeta: metav1.ObjectMeta{
+								Name: sdkMesh.name,
+							},
+						})
+
+				}
+			}
+
+			mock_client.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn(
+				func(ctx context.Context, retGWList *gateway_api.GatewayList, arg3 ...interface{}) error {
+					for _, gw := range gwList.Items {
+						retGWList.Items = append(retGWList.Items, gw)
+
+					}
+					return nil
+
+				},
+			)
+
+			for _, sdkMesh := range tt.sdkMeshes {
 				if sdkMesh.isStale {
 					// first add this datastore and see if it can be deleted byy business logic
-					mock_client.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errors.New("K8S not found"))
 					mockMeshManager.EXPECT().Delete(ctx, sdkMesh.name).Return(sdkMesh.meshManagerErr)
 
-				} else {
-					mock_client.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(nil)
 				}
 			}
 		}