From 1ebd452384512bf9d49c286d5e650983a8a54d94 Mon Sep 17 00:00:00 2001 From: abaguas Date: Thu, 25 Apr 2024 16:13:52 -1000 Subject: [PATCH 1/5] Decouple gslb from the kubernetes Ingress resource This change makes the GSLB resource independent of a Kubernetes Ingress. This is a first step to allow integrations with other ingress Resources (e.g. istio virtual services for Istio Gateway, HTTP routes for Gateway API) #552 The change is fully backwards compatible, embedding an Ingress resources will still behave the same way it worked before. In addition to the code refacting a new ResourceRef field is introduced. This field allows referencing resources that are not embedded in the GSLB definition and opens the gates to reference any resource types. As an example, the configuration bellow allows a GSLB resource to load balance the application defined in the Ingress resource on the same namespace with labels `app: demo` ``` spec: resourceRef: ingress: matchLabels: app: demo ``` --- Implementation details A couple of functions crucial for business logic, namely `GslbIngressExposedIPs` and `getServiceHealthStatus`, need to read configuration present in the Ingress resource. Since the code would become too complicated once new ways to configure ingress are integrated, the format of the data they depend on was generalized from the Kubernetes Ingress resource to an ingress agnostic format. The processing of the data looks as follows: A new `GslbReferenceResolver` interface was created. An implementation of this interface is capable of understanding a type of ingress configuration (e.g.: kubernetes' ingress, istio's virtual service, gateway API's http route) and implements two functions: `GetServers` and `GetGslbExposedIPs`. These functions extract the backends of the applications and the IP addresses they are exposed on, respectively. Once a reconciliation operation is triggered a new `GslbReferenceResolver` is instatiated. Then, the list of servers and the exposed IPs are read and stored in the status of the GSLB resource. Finally, the rest of the logic remains the same, with the difference that functions implementing business logic read the configuration from the status instead of looking up the Kubernetes Ingress resource. --- Points for discussion: * Should the list of servers and exposed IPs be stored in the status of GSLB resource? An internal data structure would also work, however we would need to pass it as an argument to numerous functions. * There is already a `depresolver` interface. Even though the names look similar `depresolver` resolves startup configuration, while `refresolver` resolves runtime configuration. In addition, logging is useful to communicate with the users but a logger cannot be instantiated in the `depresolver` package because it would lead to circular dependencies. For these reasons, a new package was created instead of adding this logic to the `depresolver` package. Naming of the package can also be discussed, the proposal `refresolver` comes from the fact that it resolves references to other resources. Signed-off-by: abaguas --- Makefile | 1 + api/v1beta1/gslb_types.go | 42 +++- api/v1beta1/zz_generated.deepcopy.go | 90 +++++++ chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml | 96 +++++++- controllers/dnsupdate.go | 5 +- controllers/gslb_controller_reconciliation.go | 40 +++- .../gslb_controller_reconciliation_test.go | 6 +- controllers/gslb_controller_setup.go | 6 +- controllers/gslb_controller_weight_test.go | 1 - controllers/mocks/assistant_mock.go | 16 -- controllers/mocks/provider_mock.go | 15 -- controllers/mocks/refresolver_mock.go | 90 +++++++ controllers/providers/assistant/assistant.go | 3 - controllers/providers/assistant/gslb.go | 40 ---- controllers/providers/dns/dns.go | 2 - controllers/providers/dns/empty.go | 4 - controllers/providers/dns/external.go | 6 +- controllers/providers/dns/external_test.go | 13 +- controllers/providers/dns/infoblox.go | 6 +- controllers/providers/dns/infoblox_test.go | 5 +- controllers/refresolver/ingress.go | 196 ++++++++++++++++ controllers/refresolver/ingress_test.go | 222 ++++++++++++++++++ controllers/refresolver/refresolver.go | 66 ++++++ .../refresolver/testdata/gslb_embedded.yaml | 23 ++ .../refresolver/testdata/gslb_referenced.yaml | 14 ++ .../gslb_referenced_and_embedded.yaml | 27 +++ .../testdata/ingress_embedded.yaml | 19 ++ .../testdata/ingress_multiple_servers.yaml | 46 ++++ .../testdata/ingress_referenced.yaml | 24 ++ controllers/status.go | 32 +-- controllers/utils/yaml.go | 18 ++ 31 files changed, 1031 insertions(+), 143 deletions(-) create mode 100644 controllers/mocks/refresolver_mock.go create mode 100644 controllers/refresolver/ingress.go create mode 100644 controllers/refresolver/ingress_test.go create mode 100644 controllers/refresolver/refresolver.go create mode 100644 controllers/refresolver/testdata/gslb_embedded.yaml create mode 100644 controllers/refresolver/testdata/gslb_referenced.yaml create mode 100644 controllers/refresolver/testdata/gslb_referenced_and_embedded.yaml create mode 100644 controllers/refresolver/testdata/ingress_embedded.yaml create mode 100644 controllers/refresolver/testdata/ingress_multiple_servers.yaml create mode 100644 controllers/refresolver/testdata/ingress_referenced.yaml diff --git a/Makefile b/Makefile index baca4a51b9..affa78fa96 100644 --- a/Makefile +++ b/Makefile @@ -395,6 +395,7 @@ mocks: mockgen -package=mocks -destination=controllers/mocks/manager_mock.go sigs.k8s.io/controller-runtime/pkg/manager Manager mockgen -package=mocks -destination=controllers/mocks/client_mock.go sigs.k8s.io/controller-runtime/pkg/client Client mockgen -package=mocks -destination=controllers/mocks/resolver_mock.go -source=controllers/depresolver/resolver.go GslbResolver + mockgen -package=mocks -destination=controllers/mocks/refresolver_mock.go -source=controllers/refresolver/refresolver.go GslbRefResolver mockgen -package=mocks -destination=controllers/mocks/provider_mock.go -source=controllers/providers/dns/dns.go Provider $(call golic) diff --git a/api/v1beta1/gslb_types.go b/api/v1beta1/gslb_types.go index c083f52afe..2d7ff79f43 100644 --- a/api/v1beta1/gslb_types.go +++ b/api/v1beta1/gslb_types.go @@ -40,13 +40,47 @@ type Strategy struct { SplitBrainThresholdSeconds int `json:"splitBrainThresholdSeconds,omitempty"` } +// ResourceRef selects a resource defining the GSLB's load balancer and server +// +k8s:openapi-gen=true +type ResourceRef struct { + // Ingress selects a kubernetes.networking.k8s.io/v1.Ingress resource + Ingress metav1.LabelSelector `json:"ingress,omitempty"` +} + // GslbSpec defines the desired state of Gslb // +k8s:openapi-gen=true type GslbSpec struct { // Gslb-enabled Ingress Spec - Ingress IngressSpec `json:"ingress"` + Ingress IngressSpec `json:"ingress,omitempty"` // Gslb Strategy spec Strategy Strategy `json:"strategy"` + // ResourceRef spec + ResourceRef ResourceRef `json:"resourceRef,omitempty"` +} + +// LoadBalancer holds the GSLB's load balancer configuration +// +k8s:openapi-gen=true +type LoadBalancer struct { + // ExposedIPs on the local Load Balancer. This information is extracted automatically from the 'ingress' or 'resourceRef' configuration (optional) + ExposedIPs []string `json:"exposedIps,omitempty"` +} + +// Servers holds the GSLB's servers' configuration +// +k8s:openapi-gen=true +type Server struct { + // Hostname exposed by the GSLB. This information is extracted automatically from the 'ingress' or 'resourceRef' configuration (optional) + Host string `json:"host,omitempty"` + // Kubernetes Services backing the load balanced application under the hostname. This information is extracted automatically from the 'ingress' or 'resourceRef' configuration (optional) + Services []*NamespacedName `json:"services,omitempty"` +} + +// NamespacedName holds a reference to a k8s resource +// +k8s:openapi-gen=true +type NamespacedName struct { + // Namespace where the resource can be found + Namespace string `json:"namespace"` + // Name of the resource + Name string `json:"name"` } // GslbStatus defines the observed state of Gslb @@ -57,8 +91,12 @@ type GslbStatus struct { HealthyRecords map[string][]string `json:"healthyRecords"` // Cluster Geo Tag GeoTag string `json:"geoTag"` - // Comma-separated list of hosts. Duplicating the value from range .spec.ingress.rules[*].host for printer column + // Comma-separated list of hosts Hosts string `json:"hosts,omitempty"` + // LoadBalancer configuration + LoadBalancer LoadBalancer `json:"loadBalancer"` + // Servers configuration + Servers []*Server `json:"servers"` } // +kubebuilder:object:root=true diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 64a55df33e..2bfaabbcee 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -90,6 +90,7 @@ func (in *GslbSpec) DeepCopyInto(out *GslbSpec) { *out = *in in.Ingress.DeepCopyInto(&out.Ingress) in.Strategy.DeepCopyInto(&out.Strategy) + in.ResourceRef.DeepCopyInto(&out.ResourceRef) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GslbSpec. @@ -128,6 +129,18 @@ func (in *GslbStatus) DeepCopyInto(out *GslbStatus) { (*out)[key] = outVal } } + in.LoadBalancer.DeepCopyInto(&out.LoadBalancer) + if in.Servers != nil { + in, out := &in.Servers, &out.Servers + *out = make([]*Server, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Server) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GslbStatus. @@ -170,6 +183,83 @@ func (in *IngressSpec) DeepCopy() *IngressSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoadBalancer) DeepCopyInto(out *LoadBalancer) { + *out = *in + if in.ExposedIPs != nil { + in, out := &in.ExposedIPs, &out.ExposedIPs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoadBalancer. +func (in *LoadBalancer) DeepCopy() *LoadBalancer { + if in == nil { + return nil + } + out := new(LoadBalancer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespacedName) DeepCopyInto(out *NamespacedName) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespacedName. +func (in *NamespacedName) DeepCopy() *NamespacedName { + if in == nil { + return nil + } + out := new(NamespacedName) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceRef) DeepCopyInto(out *ResourceRef) { + *out = *in + in.Ingress.DeepCopyInto(&out.Ingress) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceRef. +func (in *ResourceRef) DeepCopy() *ResourceRef { + if in == nil { + return nil + } + out := new(ResourceRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Server) DeepCopyInto(out *Server) { + *out = *in + if in.Services != nil { + in, out := &in.Services, &out.Services + *out = make([]*NamespacedName, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(NamespacedName) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Server. +func (in *Server) DeepCopy() *Server { + if in == nil { + return nil + } + out := new(Server) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Strategy) DeepCopyInto(out *Strategy) { *out = *in diff --git a/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml b/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml index 8f2fe5fae3..848b4d44a3 100644 --- a/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml +++ b/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml @@ -313,6 +313,55 @@ spec: type: object type: array type: object + resourceRef: + description: ResourceRef spec + properties: + ingress: + description: Ingress selects a kubernetes.networking.k8s.io/v1.Ingress + resource + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + type: object strategy: description: Gslb Strategy spec properties: @@ -337,7 +386,6 @@ spec: - type type: object required: - - ingress - strategy type: object status: @@ -354,9 +402,49 @@ spec: description: Current Healthy DNS record structure type: object hosts: - description: Comma-separated list of hosts. Duplicating the value - from range .spec.ingress.rules[*].host for printer column + description: Comma-separated list of hosts type: string + loadBalancer: + description: LoadBalancer configuration + properties: + exposedIps: + description: ExposedIPs on the local Load Balancer. This information + is extracted automatically from the 'ingress' or 'resourceRef' + configuration (optional) + items: + type: string + type: array + type: object + servers: + description: Servers configuration + items: + description: Servers holds the GSLB's servers' configuration + properties: + host: + description: Hostname exposed by the GSLB. This information + is extracted automatically from the 'ingress' or 'resourceRef' + configuration (optional) + type: string + services: + description: Kubernetes Services backing the load balanced application + under the hostname. This information is extracted automatically + from the 'ingress' or 'resourceRef' configuration (optional) + items: + description: NamespacedName holds a reference to a k8s resource + properties: + name: + description: Name of the resource + type: string + namespace: + description: Namespace where the resource can be found + type: string + required: + - name + - namespace + type: object + type: array + type: object + type: array serviceHealth: additionalProperties: type: string @@ -365,6 +453,8 @@ spec: required: - geoTag - healthyRecords + - loadBalancer + - servers - serviceHealth type: object type: object diff --git a/controllers/dnsupdate.go b/controllers/dnsupdate.go index 524726af6e..1705838846 100644 --- a/controllers/dnsupdate.go +++ b/controllers/dnsupdate.go @@ -43,10 +43,7 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D return nil, err } - localTargets, err := r.DNSProvider.GslbIngressExposedIPs(gslb) - if err != nil { - return nil, err - } + localTargets := gslb.Status.LoadBalancer.ExposedIPs for host, health := range serviceHealth { var finalTargets = assistant.NewTargets() diff --git a/controllers/gslb_controller_reconciliation.go b/controllers/gslb_controller_reconciliation.go index e39f1da3d0..2936622518 100644 --- a/controllers/gslb_controller_reconciliation.go +++ b/controllers/gslb_controller_reconciliation.go @@ -21,10 +21,12 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic import ( "context" "fmt" + "reflect" "github.com/k8gb-io/k8gb/controllers/utils" "github.com/k8gb-io/k8gb/controllers/providers/metrics" + "github.com/k8gb-io/k8gb/controllers/refresolver" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" "github.com/k8gb-io/k8gb/controllers/depresolver" @@ -33,6 +35,7 @@ import ( "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -63,7 +66,7 @@ var m = metrics.Metrics() // +kubebuilder:rbac:groups=k8gb.absa.oss,resources=gslbs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8gb.absa.oss,resources=gslbs/status,verbs=get;update;patch -// Reconcile runs main reconiliation loop +// Reconcile runs main reconciliation loop func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { ctx, span := r.Tracer.Start(ctx, "Reconcile") defer span.End() @@ -138,17 +141,44 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } // == Ingress ========== - ingress, err := r.gslbIngress(gslb) + if reflect.DeepEqual(gslb.Spec.ResourceRef.Ingress, metav1.LabelSelector{}) { + ingress, err := r.gslbIngress(gslb) + if err != nil { + m.IncrementError(gslb) + return result.RequeueError(err) + } + + err = r.saveIngress(gslb, ingress) + if err != nil { + m.IncrementError(gslb) + return result.RequeueError(err) + } + } + + // == Reference resolution == + refResolver, err := refresolver.New(gslb, r.Client) if err != nil { m.IncrementError(gslb) - return result.RequeueError(err) + return result.RequeueError(fmt.Errorf("error resolving references to the refresolver object (%s)", err)) + } + servers, err := refResolver.GetServers() + if err != nil { + m.IncrementError(gslb) + return result.RequeueError(fmt.Errorf("getting GSLB servers (%s)", err)) } + gslb.Status.Servers = servers + fmt.Printf("got servers: %v\n", servers) - err = r.saveIngress(gslb, ingress) + loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb, r.Client, r.Config.EdgeDNSServers) if err != nil { m.IncrementError(gslb) - return result.RequeueError(err) + return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err)) } + gslb.Status.LoadBalancer.ExposedIPs = loadBalancerExposedIPs + + log.Debug(). + Str("gslb", gslb.Name). + Msg("Resolved LoadBalancer and Server configuration referenced by Ingress") // == external-dns dnsendpoints CRs == dnsEndpoint, err := r.gslbDNSEndpoint(gslb) diff --git a/controllers/gslb_controller_reconciliation_test.go b/controllers/gslb_controller_reconciliation_test.go index 8ce815a733..f3d044b687 100644 --- a/controllers/gslb_controller_reconciliation_test.go +++ b/controllers/gslb_controller_reconciliation_test.go @@ -392,21 +392,19 @@ func TestGslbErrorsIncrement(t *testing.T) { var label = prometheus.Labels{"namespace": settings.gslb.Namespace, "name": settings.gslb.Name} m := mocks.NewMockProvider(ctrl) cnt := testutil.ToFloat64(metrics.Metrics().Get(metrics.K8gbGslbErrorsTotal).AsCounterVec().With(label)) - m.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return([]string{}, nil).Times(1) m.EXPECT().SaveDNSEndpoint(gomock.Any(), gomock.Any()).Return(fmt.Errorf("save DNS error")).Times(1) m.EXPECT().GetExternalTargets(gomock.Any()).Return(assistant.Targets{}).AnyTimes() - m.EXPECT().CreateZoneDelegationForExternalDNS(gomock.Any()).Return(nil).AnyTimes() settings.reconciler.DNSProvider = m // act _, err := settings.reconciler.Reconcile(context.TODO(), settings.request) require.Error(t, err) // let's break it on different place m.EXPECT().SaveDNSEndpoint(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - m.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return([]string{}, fmt.Errorf("exposed IP's error")).AnyTimes() + m.EXPECT().CreateZoneDelegationForExternalDNS(gomock.Any()).Return(fmt.Errorf("zone delegation error")).AnyTimes() _, err = settings.reconciler.Reconcile(context.TODO(), settings.request) cnt2 := testutil.ToFloat64(metrics.Metrics().Get(metrics.K8gbGslbErrorsTotal).AsCounterVec().With(label)) // assert - assert.Error(t, err) + assert.NoError(t, err) assert.Equal(t, cnt+2, cnt2) }) } diff --git a/controllers/gslb_controller_setup.go b/controllers/gslb_controller_setup.go index 0c5e8d85a0..00dcb3f3f9 100644 --- a/controllers/gslb_controller_setup.go +++ b/controllers/gslb_controller_setup.go @@ -58,9 +58,9 @@ func (r *GslbReconciler) SetupWithManager(mgr ctrl.Manager) error { reconcileRequests := []reconcile.Request{} GslbLoop: for _, gslb := range gslbList.Items { - for _, rule := range gslb.Spec.Ingress.Rules { - for _, path := range rule.HTTP.Paths { - if path.Backend.Service != nil && path.Backend.Service.Name == a.GetName() { + for _, server := range gslb.Status.Servers { + for _, service := range server.Services { + if service.Name == a.GetName() { reconcileRequests = append(reconcileRequests, reconcile.Request{ NamespacedName: types.NamespacedName{ Name: gslb.Name, diff --git a/controllers/gslb_controller_weight_test.go b/controllers/gslb_controller_weight_test.go index 89a21902a0..d496e9a2a8 100644 --- a/controllers/gslb_controller_weight_test.go +++ b/controllers/gslb_controller_weight_test.go @@ -172,7 +172,6 @@ func TestWeight(t *testing.T) { // settings := provideSettings(t, predefinedConfig) m := mocks.NewMockProvider(ctrl) r := mocks.NewMockGslbResolver(ctrl) - m.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return([]string{}, nil).Times(1) m.EXPECT().SaveDNSEndpoint(gomock.Any(), gomock.Any()).Do(assertAnnotation).Return(fmt.Errorf("save DNS error")).Times(1) m.EXPECT().CreateZoneDelegationForExternalDNS(gomock.Any()).Return(nil).AnyTimes() r.EXPECT().ResolveGslbSpec(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(injectWeight).AnyTimes() diff --git a/controllers/mocks/assistant_mock.go b/controllers/mocks/assistant_mock.go index e4deac1890..ffa837c62c 100644 --- a/controllers/mocks/assistant_mock.go +++ b/controllers/mocks/assistant_mock.go @@ -31,7 +31,6 @@ import ( reflect "reflect" time "time" - v1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" assistant "github.com/k8gb-io/k8gb/controllers/providers/assistant" gomock "go.uber.org/mock/gomock" endpoint "sigs.k8s.io/external-dns/endpoint" @@ -89,21 +88,6 @@ func (mr *MockAssistantMockRecorder) GetExternalTargets(host, extClusterNsNames return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExternalTargets", reflect.TypeOf((*MockAssistant)(nil).GetExternalTargets), host, extClusterNsNames) } -// GslbIngressExposedIPs mocks base method. -func (m *MockAssistant) GslbIngressExposedIPs(gslb *v1beta1.Gslb) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GslbIngressExposedIPs", gslb) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GslbIngressExposedIPs indicates an expected call of GslbIngressExposedIPs. -func (mr *MockAssistantMockRecorder) GslbIngressExposedIPs(gslb any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GslbIngressExposedIPs", reflect.TypeOf((*MockAssistant)(nil).GslbIngressExposedIPs), gslb) -} - // InspectTXTThreshold mocks base method. func (m *MockAssistant) InspectTXTThreshold(fqdn string, splitBrainThreshold time.Duration) error { m.ctrl.T.Helper() diff --git a/controllers/mocks/provider_mock.go b/controllers/mocks/provider_mock.go index 8dd784b1a5..71f074a1be 100644 --- a/controllers/mocks/provider_mock.go +++ b/controllers/mocks/provider_mock.go @@ -101,21 +101,6 @@ func (mr *MockProviderMockRecorder) GetExternalTargets(arg0 any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExternalTargets", reflect.TypeOf((*MockProvider)(nil).GetExternalTargets), arg0) } -// GslbIngressExposedIPs mocks base method. -func (m *MockProvider) GslbIngressExposedIPs(arg0 *v1beta1.Gslb) ([]string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GslbIngressExposedIPs", arg0) - ret0, _ := ret[0].([]string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GslbIngressExposedIPs indicates an expected call of GslbIngressExposedIPs. -func (mr *MockProviderMockRecorder) GslbIngressExposedIPs(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GslbIngressExposedIPs", reflect.TypeOf((*MockProvider)(nil).GslbIngressExposedIPs), arg0) -} - // SaveDNSEndpoint mocks base method. func (m *MockProvider) SaveDNSEndpoint(arg0 *v1beta1.Gslb, arg1 *endpoint.DNSEndpoint) error { m.ctrl.T.Helper() diff --git a/controllers/mocks/refresolver_mock.go b/controllers/mocks/refresolver_mock.go new file mode 100644 index 0000000000..dacf7c1970 --- /dev/null +++ b/controllers/mocks/refresolver_mock.go @@ -0,0 +1,90 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: controllers/refresolver/refresolver.go +// +// Generated by this command: +// +// mockgen -package=mocks -destination=controllers/mocks/refresolver_mock.go -source=controllers/refresolver/refresolver.go GslbRefResolver +// + +// Package mocks is a generated GoMock package. +package mocks + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + reflect "reflect" + + v1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + utils "github.com/k8gb-io/k8gb/controllers/internal/utils" + gomock "go.uber.org/mock/gomock" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockGslbReferenceResolver is a mock of GslbReferenceResolver interface. +type MockGslbReferenceResolver struct { + ctrl *gomock.Controller + recorder *MockGslbReferenceResolverMockRecorder +} + +// MockGslbReferenceResolverMockRecorder is the mock recorder for MockGslbReferenceResolver. +type MockGslbReferenceResolverMockRecorder struct { + mock *MockGslbReferenceResolver +} + +// NewMockGslbReferenceResolver creates a new mock instance. +func NewMockGslbReferenceResolver(ctrl *gomock.Controller) *MockGslbReferenceResolver { + mock := &MockGslbReferenceResolver{ctrl: ctrl} + mock.recorder = &MockGslbReferenceResolverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGslbReferenceResolver) EXPECT() *MockGslbReferenceResolverMockRecorder { + return m.recorder +} + +// GetGslbExposedIPs mocks base method. +func (m *MockGslbReferenceResolver) GetGslbExposedIPs(arg0 *v1beta1.Gslb, arg1 client.Client, arg2 utils.DNSList) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetGslbExposedIPs", arg0, arg1, arg2) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetGslbExposedIPs indicates an expected call of GetGslbExposedIPs. +func (mr *MockGslbReferenceResolverMockRecorder) GetGslbExposedIPs(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGslbExposedIPs", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetGslbExposedIPs), arg0, arg1, arg2) +} + +// GetServers mocks base method. +func (m *MockGslbReferenceResolver) GetServers() ([]*v1beta1.Server, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetServers") + ret0, _ := ret[0].([]*v1beta1.Server) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetServers indicates an expected call of GetServers. +func (mr *MockGslbReferenceResolverMockRecorder) GetServers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetServers", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetServers)) +} diff --git a/controllers/providers/assistant/assistant.go b/controllers/providers/assistant/assistant.go index 1471a33560..196555c050 100644 --- a/controllers/providers/assistant/assistant.go +++ b/controllers/providers/assistant/assistant.go @@ -21,15 +21,12 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic import ( "time" - k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" externaldns "sigs.k8s.io/external-dns/endpoint" ) type Assistant interface { // CoreDNSExposedIPs retrieves list of exposed IP by CoreDNS CoreDNSExposedIPs() ([]string, error) - // GslbIngressExposedIPs retrieves list of IP's exposed by all GSLB ingresses - GslbIngressExposedIPs(gslb *k8gbv1beta1.Gslb) ([]string, error) // GetExternalTargets retrieves slice of targets from external clusters GetExternalTargets(host string, extClusterNsNames map[string]string) (targets Targets) // SaveDNSEndpoint update DNS endpoint or create new one if doesnt exist diff --git a/controllers/providers/assistant/gslb.go b/controllers/providers/assistant/gslb.go index 6eab754eb4..1332a2afb8 100644 --- a/controllers/providers/assistant/gslb.go +++ b/controllers/providers/assistant/gslb.go @@ -27,12 +27,10 @@ import ( "github.com/k8gb-io/k8gb/controllers/utils" - k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" "github.com/k8gb-io/k8gb/controllers/logging" "github.com/miekg/dns" corev1 "k8s.io/api/core/v1" - netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -121,44 +119,6 @@ func extractIPFromLB(lb corev1.LoadBalancerIngress, ns utils.DNSList) (ips []str return nil, nil } -// GslbIngressExposedIPs retrieves list of IP's exposed by all GSLB ingresses -func (r *Gslb) GslbIngressExposedIPs(gslb *k8gbv1beta1.Gslb) ([]string, error) { - nn := types.NamespacedName{ - Name: gslb.Name, - Namespace: gslb.Namespace, - } - - gslbIngress := &netv1.Ingress{} - - err := r.client.Get(context.TODO(), nn, gslbIngress) - if err != nil { - if errors.IsNotFound(err) { - log.Info(). - Str("gslb", gslb.Name). - Msg("Can't find gslb Ingress") - } - return nil, err - } - - var gslbIngressIPs []string - - for _, ip := range gslbIngress.Status.LoadBalancer.Ingress { - if len(ip.IP) > 0 { - gslbIngressIPs = append(gslbIngressIPs, ip.IP) - } - if len(ip.Hostname) > 0 { - IPs, err := utils.Dig(ip.Hostname, r.edgeDNSServers...) - if err != nil { - log.Warn().Err(err).Msg("Dig error") - return nil, err - } - gslbIngressIPs = append(gslbIngressIPs, IPs...) - } - } - - return gslbIngressIPs, nil -} - // SaveDNSEndpoint update DNS endpoint or create new one if doesnt exist func (r *Gslb) SaveDNSEndpoint(namespace string, i *externaldns.DNSEndpoint) error { found := &externaldns.DNSEndpoint{} diff --git a/controllers/providers/dns/dns.go b/controllers/providers/dns/dns.go index 55f90b5bf3..dfa282bf35 100644 --- a/controllers/providers/dns/dns.go +++ b/controllers/providers/dns/dns.go @@ -27,8 +27,6 @@ import ( type Provider interface { // CreateZoneDelegationForExternalDNS handles delegated zone in Edge DNS CreateZoneDelegationForExternalDNS(*k8gbv1beta1.Gslb) error - // GslbIngressExposedIPs retrieves list of IP's exposed by all GSLB ingresses - GslbIngressExposedIPs(*k8gbv1beta1.Gslb) ([]string, error) // GetExternalTargets retrieves list of external targets for specified host GetExternalTargets(string) assistant.Targets // SaveDNSEndpoint update DNS endpoint in gslb or create new one if doesn't exist diff --git a/controllers/providers/dns/empty.go b/controllers/providers/dns/empty.go index 0a3c67d395..fd2e7dafa6 100644 --- a/controllers/providers/dns/empty.go +++ b/controllers/providers/dns/empty.go @@ -42,10 +42,6 @@ func (p *EmptyDNSProvider) CreateZoneDelegationForExternalDNS(*k8gbv1beta1.Gslb) return } -func (p *EmptyDNSProvider) GslbIngressExposedIPs(gslb *k8gbv1beta1.Gslb) (r []string, err error) { - return p.assistant.GslbIngressExposedIPs(gslb) -} - func (p *EmptyDNSProvider) GetExternalTargets(host string) (targets assistant.Targets) { return p.assistant.GetExternalTargets(host, p.config.GetExternalClusterNSNames()) } diff --git a/controllers/providers/dns/external.go b/controllers/providers/dns/external.go index 66673cb62c..31823fb0b2 100644 --- a/controllers/providers/dns/external.go +++ b/controllers/providers/dns/external.go @@ -66,7 +66,7 @@ func (p *ExternalDNSProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1bet if p.config.CoreDNSExposed { NSServerIPs, err = p.assistant.CoreDNSExposedIPs() } else { - NSServerIPs, err = p.assistant.GslbIngressExposedIPs(gslb) + NSServerIPs = gslb.Status.LoadBalancer.ExposedIPs } if err != nil { return err @@ -109,10 +109,6 @@ func (p *ExternalDNSProvider) GetExternalTargets(host string) (targets assistant return p.assistant.GetExternalTargets(host, p.config.GetExternalClusterNSNames()) } -func (p *ExternalDNSProvider) GslbIngressExposedIPs(gslb *k8gbv1beta1.Gslb) ([]string, error) { - return p.assistant.GslbIngressExposedIPs(gslb) -} - func (p *ExternalDNSProvider) SaveDNSEndpoint(gslb *k8gbv1beta1.Gslb, i *externaldns.DNSEndpoint) error { return p.assistant.SaveDNSEndpoint(gslb.Namespace, i) } diff --git a/controllers/providers/dns/external_test.go b/controllers/providers/dns/external_test.go index f26af5132f..3ca197cdd8 100644 --- a/controllers/providers/dns/external_test.go +++ b/controllers/providers/dns/external_test.go @@ -46,6 +46,11 @@ import ( ) // test data +var targetIPs = []string{ + "10.0.1.38", + "10.0.1.40", + "10.0.1.39", +} var a = struct { Config depresolver.Config Gslb *k8gbv1beta1.Gslb @@ -70,13 +75,10 @@ var a = struct { var crSampleYaml = "../../../deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml" gslbYaml, _ := os.ReadFile(crSampleYaml) gslb, _ := utils2.YamlToGslb(gslbYaml) + gslb.Status.LoadBalancer.ExposedIPs = targetIPs return gslb }(), - TargetIPs: []string{ - "10.0.1.38", - "10.0.1.40", - "10.0.1.39", - }, + TargetIPs: targetIPs, TargetNSNamesSorted: []string{ "gslb-ns-eu-cloud.example.com", "gslb-ns-us-cloud.example.com", @@ -114,7 +116,6 @@ func TestCreateZoneDelegationOnExternalDNS(t *testing.T) { defer ctrl.Finish() m := mocks.NewMockAssistant(ctrl) p := NewExternalDNS(a.Config, m) - m.EXPECT().GslbIngressExposedIPs(a.Gslb).Return(a.TargetIPs, nil).Times(1) m.EXPECT().SaveDNSEndpoint(a.Config.K8gbNamespace, gomock.Eq(expectedDNSEndpoint)).Return(nil).Times(1). Do(func(ns string, ep *externaldns.DNSEndpoint) { require.True(t, reflect.DeepEqual(ep, expectedDNSEndpoint)) diff --git a/controllers/providers/dns/infoblox.go b/controllers/providers/dns/infoblox.go index c5eab7de31..1565492ed4 100644 --- a/controllers/providers/dns/infoblox.go +++ b/controllers/providers/dns/infoblox.go @@ -71,7 +71,7 @@ func (p *InfobloxProvider) CreateZoneDelegationForExternalDNS(gslb *k8gbv1beta1. if p.config.CoreDNSExposed { addresses, err = p.assistant.CoreDNSExposedIPs() } else { - addresses, err = p.assistant.GslbIngressExposedIPs(gslb) + addresses = gslb.Status.LoadBalancer.ExposedIPs } if err != nil { m.InfobloxIncrementZoneUpdateError(gslb) @@ -206,10 +206,6 @@ func (p *InfobloxProvider) GetExternalTargets(host string) (targets assistant.Ta return p.assistant.GetExternalTargets(host, p.config.GetExternalClusterNSNames()) } -func (p *InfobloxProvider) GslbIngressExposedIPs(gslb *k8gbv1beta1.Gslb) ([]string, error) { - return p.assistant.GslbIngressExposedIPs(gslb) -} - func (p *InfobloxProvider) SaveDNSEndpoint(gslb *k8gbv1beta1.Gslb, i *externaldns.DNSEndpoint) error { return p.assistant.SaveDNSEndpoint(gslb.Namespace, i) } diff --git a/controllers/providers/dns/infoblox_test.go b/controllers/providers/dns/infoblox_test.go index 5f7b79ad47..da1eb44e81 100644 --- a/controllers/providers/dns/infoblox_test.go +++ b/controllers/providers/dns/infoblox_test.go @@ -169,7 +169,6 @@ func TestInfobloxCreateZoneDelegationForExternalDNS(t *testing.T) { a := mocks.NewMockAssistant(ctrl) cl := mocks.NewMockInfobloxClient(ctrl) con := mocks.NewMockIBConnector(ctrl) - a.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return(ipRange, nil).Times(1) con.EXPECT().CreateObject(gomock.Any()).Return(ref, nil).AnyTimes() con.EXPECT().UpdateObject(gomock.Any(), gomock.Any()).Return(ref, nil).Times(1) con.EXPECT().GetObject(gomock.Any(), gomock.Any(), gomock.Any()).SetArg(2, []ibclient.ZoneDelegated{defaultDelegatedZone}).Return(nil) @@ -190,8 +189,7 @@ func TestInfobloxCreateZoneDelegationForExternalDNSWithSplitBrainEnabled(t *test a := mocks.NewMockAssistant(ctrl) cl := mocks.NewMockInfobloxClient(ctrl) con := mocks.NewMockIBConnector(ctrl) - a.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return(ipRange, nil).Times(1) - a.EXPECT().InspectTXTThreshold(gomock.Any(), gomock.Any()).Do(func(fqdn string, _ interface{}) { + a.EXPECT().InspectTXTThreshold(gomock.Any(), gomock.Any()).Do(func(fqdn string, arg1 interface{}) { require.Equal(t, "test-gslb-heartbeat-us-east-1.example.com", fqdn) }).Return(nil).Times(1) con.EXPECT().CreateObject(gomock.Any()).Return(ref, nil).AnyTimes() @@ -219,7 +217,6 @@ func TestInfobloxCreateZoneDelegationForExternalDNSWithSplitBrainEnabledCreating a := mocks.NewMockAssistant(ctrl) cl := mocks.NewMockInfobloxClient(ctrl) con := mocks.NewMockIBConnector(ctrl) - a.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return(ipRange, nil).Times(1) a.EXPECT().InspectTXTThreshold(gomock.Any(), gomock.Any()).Return(nil).Times(1) con.EXPECT().CreateObject(gomock.Any()).Return(ref, nil).AnyTimes() con.EXPECT().UpdateObject(gomock.Any(), gomock.Any()).Return(ref, nil).Times(1) diff --git a/controllers/refresolver/ingress.go b/controllers/refresolver/ingress.go new file mode 100644 index 0000000000..c81ecf2e76 --- /dev/null +++ b/controllers/refresolver/ingress.go @@ -0,0 +1,196 @@ +package refresolver + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +/* +Copyright 2024 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "context" + "fmt" + "reflect" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/internal/utils" + netv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type IngressReferenceResolver struct { + ingress *netv1.Ingress +} + +// NewIngressReferenceResolver creates a reference resolver capable of understanding ingresses.networking.k8s.io resources +func NewIngressReferenceResolver(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (*IngressReferenceResolver, error) { + ingressList, err := getGslbIngressRef(gslb, k8sClient) + if err != nil { + return nil, err + } + ingressEmbedded, err := getGslbIngressEmbedded(gslb, k8sClient) + if err != nil { + return nil, err + } + if ingressEmbedded != nil { + ingressList = append(ingressList, *ingressEmbedded) + } + + for _, ingress := range ingressList { + log.Info(). + Str("IngressName", ingress.Name). + Msg("Found Ingress") + } + + if len(ingressList) != 1 { + return nil, fmt.Errorf("exactly one Ingress resource expected but %d were found", len(ingressList)) + } + + return &IngressReferenceResolver{ + ingress: &ingressList[0], + }, nil +} + +// getGslbIngressRef resolves a Kubernetes Ingress resource referenced by the Gslb spec +func getGslbIngressRef(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) ([]netv1.Ingress, error) { + ingressList := &netv1.IngressList{} + if reflect.DeepEqual(gslb.Spec.ResourceRef.Ingress, metav1.LabelSelector{}) { + log.Info(). + Str("gslb", gslb.Name). + Msg("No configuration for referenced Ingress resource") + return ingressList.Items, nil + } + + selector, err := metav1.LabelSelectorAsSelector(&gslb.Spec.ResourceRef.Ingress) + if err != nil { + return nil, err + } + opts := &client.ListOptions{ + LabelSelector: selector, + } + + err = k8sClient.List(context.TODO(), ingressList, opts) + if err != nil { + if errors.IsNotFound(err) { + log.Info(). + Str("gslb", gslb.Name). + Msg("Can't find referenced Ingress resource") + } + return nil, err + } + + return ingressList.Items, err +} + +// getGslbIngressEmbedded resolves a Kubernetes Ingress resource embedded in the Gslb spec +func getGslbIngressEmbedded(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (*netv1.Ingress, error) { + if reflect.DeepEqual(gslb.Spec.Ingress, k8gbv1beta1.IngressSpec{}) { + log.Info(). + Str("gslb", gslb.Name). + Msg("No configuration for embedded Ingress resource") + return nil, nil + } + + nn := types.NamespacedName{ + Name: gslb.Name, + Namespace: gslb.Namespace, + } + ingress := &netv1.Ingress{} + err := k8sClient.Get(context.TODO(), nn, ingress) + if err != nil { + if errors.IsNotFound(err) { + log.Warn(). + Str("gslb", gslb.Name). + Msg("Can't find gslb Ingress") + } + return nil, err + } + + return ingress, nil +} + +// GetServers retrieves the backend servers referenced by the GSLB +func (irr *IngressReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { + servers := []*k8gbv1beta1.Server{} + + for _, rule := range irr.ingress.Spec.Rules { + server := &k8gbv1beta1.Server{ + Host: rule.Host, + Services: []*k8gbv1beta1.NamespacedName{}, + } + for _, path := range rule.HTTP.Paths { + if path.Backend.Service == nil || path.Backend.Service.Name == "" { + log.Warn(). + Str("ingress", irr.ingress.Name). + Interface("service", path.Backend.Service). + Msg("Malformed service definition") + continue + } + + server.Services = append(server.Services, &k8gbv1beta1.NamespacedName{ + Name: path.Backend.Service.Name, + Namespace: irr.ingress.Namespace, + }) + } + servers = append(servers, server) + } + + return servers, nil +} + +// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB +func (irr *IngressReferenceResolver) GetGslbExposedIPs(gslb *k8gbv1beta1.Gslb, _ client.Client, edgeDNSServers utils.DNSList) ([]string, error) { + gslbIngressIPs := []string{} + + for _, ip := range irr.ingress.Status.LoadBalancer.Ingress { + if len(ip.IP) > 0 { + gslbIngressIPs = append(gslbIngressIPs, ip.IP) + } + if len(ip.Hostname) > 0 { + IPs, err := utils.Dig(ip.Hostname, edgeDNSServers...) + if err != nil { + log.Warn().Err(err).Msg("Dig error") + return nil, err + } + gslbIngressIPs = append(gslbIngressIPs, IPs...) + } + } + + return gslbIngressIPs, nil +} + +func (irr *IngressReferenceResolver) getIngress() *netv1.Ingress { + return irr.ingress +} diff --git a/controllers/refresolver/ingress_test.go b/controllers/refresolver/ingress_test.go new file mode 100644 index 0000000000..0d824a8a1b --- /dev/null +++ b/controllers/refresolver/ingress_test.go @@ -0,0 +1,222 @@ +package refresolver + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +/* +Copyright 2024 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "fmt" + "os" + "testing" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/internal/utils" + "github.com/stretchr/testify/assert" + netv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestGetIngress(t *testing.T) { + var tests = []struct { + name string + gslbYaml string + expectedIngressYaml string + expectedError error + }{ + { + name: "embedded ingress", + gslbYaml: "./testdata/gslb_embedded.yaml", + expectedIngressYaml: "./testdata/ingress_embedded.yaml", + expectedError: nil, + }, + { + name: "referenced ingress", + gslbYaml: "./testdata/gslb_referenced.yaml", + expectedIngressYaml: "./testdata/ingress_referenced.yaml", + expectedError: nil, + }, + { + name: "referenced and embedded ingress", + gslbYaml: "./testdata/gslb_referenced_and_embedded.yaml", + expectedIngressYaml: "", + expectedError: fmt.Errorf("exactly one Ingress resource expected but 2 were found"), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + cl, gslb := getTestContext(test.gslbYaml) + + // act + resolver, err := New(gslb, cl) + assert.Equal(t, test.expectedError, err) + + ingressResolver, ok := resolver.(*IngressReferenceResolver) + assert.True(t, ok, "referenced resolver is of type Ingress") + + // assert + if test.expectedIngressYaml != "" { + expectedIngress := fileToIngress(test.expectedIngressYaml) + assert.Equal(t, expectedIngress, ingressResolver.getIngress()) + } + }) + } +} + +func TestGetServers(t *testing.T) { + var tests = []struct { + name string + ingressYaml string + expectedServers []*k8gbv1beta1.Server + }{ + { + name: "multiple servers", + ingressYaml: "./testdata/ingress_multiple_servers.yaml", + expectedServers: []*k8gbv1beta1.Server{ + { + Host: "h1.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "s1", + Namespace: "test-gslb", + }, + }, + }, + { + Host: "h2.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "ss1", + Namespace: "test-gslb", + }, + { + Name: "ss2", + Namespace: "test-gslb", + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + ingress := fileToIngress(test.ingressYaml) + resolver := IngressReferenceResolver{ + ingress: ingress, + } + + // act + servers, err := resolver.GetServers() + assert.NoError(t, err) + + // assert + assert.Equal(t, test.expectedServers, servers) + }) + } +} + +func TestGetGslbExposedIPs(t *testing.T) { + var tests = []struct { + name string + ingressYaml string + expectedIPs []string + }{ + { + name: "multiple exposed IPs", + ingressYaml: "./testdata/ingress_multiple_servers.yaml", + expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, + }, + { + name: "no exposed IPs", + ingressYaml: "./testdata/ingress_referenced.yaml", + expectedIPs: []string{}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + ingress := fileToIngress(test.ingressYaml) + resolver := IngressReferenceResolver{ + ingress: ingress, + } + + // act + IPs, err := resolver.GetGslbExposedIPs(&k8gbv1beta1.Gslb{}, nil, []utils.DNSServer{}) + assert.NoError(t, err) + + // assert + assert.Equal(t, test.expectedIPs, IPs) + }) + } +} + +func getTestContext(gslbData string) (client.Client, *k8gbv1beta1.Gslb) { + gslbYaml, err := os.ReadFile(gslbData) + if err != nil { + panic(fmt.Errorf("can't open example CR file: %s", gslbData)) + } + gslb, err := utils.YamlToGslb(gslbYaml) + if err != nil { + panic(err) + } + + objs := []runtime.Object{ + gslb, + fileToIngress("./testdata/ingress_embedded.yaml"), + fileToIngress("./testdata/ingress_referenced.yaml"), + } + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(k8gbv1beta1.GroupVersion, gslb) + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build() + + return cl, gslb +} + +func fileToIngress(ingressData string) *netv1.Ingress { + ingressYaml, err := os.ReadFile(ingressData) + if err != nil { + panic(fmt.Errorf("can't open example CR file: %s", ingressData)) + } + ingress, err := utils.YamlToIngress(ingressYaml) + if err != nil { + panic(err) + } + return ingress +} diff --git a/controllers/refresolver/refresolver.go b/controllers/refresolver/refresolver.go new file mode 100644 index 0000000000..dadc868796 --- /dev/null +++ b/controllers/refresolver/refresolver.go @@ -0,0 +1,66 @@ +package refresolver + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +/* +Copyright 2024 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "fmt" + "reflect" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/internal/utils" + "github.com/k8gb-io/k8gb/controllers/logging" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var log = logging.Logger() + +// GslbReferenceResolver resolves references to other kubernetes resources concerning ingress configuration +type GslbReferenceResolver interface { + // GetServers retrieves the server configuration of the GSLB object + GetServers() ([]*k8gbv1beta1.Server, error) + // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB + GetGslbExposedIPs(*k8gbv1beta1.Gslb, client.Client, utils.DNSList) ([]string, error) +} + +// New creates a new GSLBReferenceResolver +func New(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (GslbReferenceResolver, error) { + if !reflect.DeepEqual(gslb.Spec.Ingress, k8gbv1beta1.IngressSpec{}) || !reflect.DeepEqual(gslb.Spec.ResourceRef.Ingress, metav1.LabelSelector{}) { + return NewIngressReferenceResolver(gslb, k8sClient) + } + return nil, fmt.Errorf("incomplete gslb configuration, no ingress found") +} diff --git a/controllers/refresolver/testdata/gslb_embedded.yaml b/controllers/refresolver/testdata/gslb_embedded.yaml new file mode 100644 index 0000000000..b199886d6c --- /dev/null +++ b/controllers/refresolver/testdata/gslb_embedded.yaml @@ -0,0 +1,23 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: embedded + namespace: test-gslb +spec: + ingress: + ingressClassName: nginx + rules: + - host: embedded.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: embedded + port: + name: http + strategy: + type: roundRobin + splitBrainThresholdSeconds: 300 + dnsTtlSeconds: 30 diff --git a/controllers/refresolver/testdata/gslb_referenced.yaml b/controllers/refresolver/testdata/gslb_referenced.yaml new file mode 100644 index 0000000000..271a00ebee --- /dev/null +++ b/controllers/refresolver/testdata/gslb_referenced.yaml @@ -0,0 +1,14 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: referenced + namespace: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: referenced + strategy: + type: roundRobin + splitBrainThresholdSeconds: 300 + dnsTtlSeconds: 30 diff --git a/controllers/refresolver/testdata/gslb_referenced_and_embedded.yaml b/controllers/refresolver/testdata/gslb_referenced_and_embedded.yaml new file mode 100644 index 0000000000..1073d464dd --- /dev/null +++ b/controllers/refresolver/testdata/gslb_referenced_and_embedded.yaml @@ -0,0 +1,27 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: embedded + namespace: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: referenced + ingress: + ingressClassName: nginx + rules: + - host: embedded.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: embedded + port: + name: http + strategy: + type: roundRobin + splitBrainThresholdSeconds: 300 + dnsTtlSeconds: 30 diff --git a/controllers/refresolver/testdata/ingress_embedded.yaml b/controllers/refresolver/testdata/ingress_embedded.yaml new file mode 100644 index 0000000000..7b939fe47d --- /dev/null +++ b/controllers/refresolver/testdata/ingress_embedded.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: embedded + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: embedded.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: embedded + port: + name: http diff --git a/controllers/refresolver/testdata/ingress_multiple_servers.yaml b/controllers/refresolver/testdata/ingress_multiple_servers.yaml new file mode 100644 index 0000000000..5066d8e995 --- /dev/null +++ b/controllers/refresolver/testdata/ingress_multiple_servers.yaml @@ -0,0 +1,46 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: multiple-servers + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: h1.cloud.example.com + http: + paths: + - path: /malformed + pathType: Prefix + backend: + service: + name: "" + - path: / + pathType: Prefix + backend: + service: + name: s1 + port: + name: http + - host: h2.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ss1 + port: + name: http + - path: /debug + pathType: Prefix + backend: + service: + name: ss2 + port: + name: http +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 + - ip: 10.0.0.2 diff --git a/controllers/refresolver/testdata/ingress_referenced.yaml b/controllers/refresolver/testdata/ingress_referenced.yaml new file mode 100644 index 0000000000..e4b4e8a61b --- /dev/null +++ b/controllers/refresolver/testdata/ingress_referenced.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + app: referenced + name: referenced + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: referenced.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: referenced + port: + name: http +status: + loadBalancer: + ingress: diff --git a/controllers/status.go b/controllers/status.go index d7b6d29527..897818c927 100644 --- a/controllers/status.go +++ b/controllers/status.go @@ -59,47 +59,37 @@ func (r *GslbReconciler) updateGslbStatus(gslb *k8gbv1beta1.Gslb, ep *externaldn func (r *GslbReconciler) getServiceHealthStatus(gslb *k8gbv1beta1.Gslb) (map[string]k8gbv1beta1.HealthStatus, error) { serviceHealth := make(map[string]k8gbv1beta1.HealthStatus) - for _, rule := range gslb.Spec.Ingress.Rules { - for _, path := range rule.HTTP.Paths { - if path.Backend.Service == nil || path.Backend.Service.Name == "" { - log.Warn(). - Str("gslb", gslb.Name). - Interface("service", path.Backend.Service). - Msg("Malformed service definition") - serviceHealth[rule.Host] = k8gbv1beta1.NotFound - continue - } + for _, server := range gslb.Status.Servers { + serviceHealth[server.Host] = k8gbv1beta1.NotFound + for _, svc := range server.Services { service := &corev1.Service{} finder := client.ObjectKey{ - Namespace: gslb.Namespace, - Name: path.Backend.Service.Name, + Namespace: svc.Namespace, + Name: svc.Name, } err := r.Get(context.TODO(), finder, service) if err != nil { if errors.IsNotFound(err) { - serviceHealth[rule.Host] = k8gbv1beta1.NotFound continue } return serviceHealth, err } endpoints := &corev1.Endpoints{} - nn := types.NamespacedName{ - Name: path.Backend.Service.Name, - Namespace: gslb.Namespace, + Name: svc.Name, + Namespace: svc.Namespace, } - err = r.Get(context.TODO(), nn, endpoints) if err != nil { return serviceHealth, err } - serviceHealth[rule.Host] = k8gbv1beta1.Unhealthy + serviceHealth[server.Host] = k8gbv1beta1.Unhealthy if len(endpoints.Subsets) > 0 { for _, subset := range endpoints.Subsets { if len(subset.Addresses) > 0 { - serviceHealth[rule.Host] = k8gbv1beta1.Healthy + serviceHealth[server.Host] = k8gbv1beta1.Healthy } } } @@ -139,8 +129,8 @@ func (r *GslbReconciler) getHealthyRecords(gslb *k8gbv1beta1.Gslb) (map[string][ func (r *GslbReconciler) hostsToCSV(gslb *k8gbv1beta1.Gslb) string { var hosts []string - for _, r := range gslb.Spec.Ingress.Rules { - hosts = append(hosts, r.Host) + for _, server := range gslb.Status.Servers { + hosts = append(hosts, server.Host) } return strings.Join(hosts, ", ") } diff --git a/controllers/utils/yaml.go b/controllers/utils/yaml.go index 1866e6b88f..1d91d79096 100644 --- a/controllers/utils/yaml.go +++ b/controllers/utils/yaml.go @@ -24,6 +24,7 @@ import ( yamlConv "github.com/ghodss/yaml" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + netv1 "k8s.io/api/networking/v1" ) // YamlToGslb takes yaml and returns Gslb object @@ -42,3 +43,20 @@ func YamlToGslb(yaml []byte) (*k8gbv1beta1.Gslb, error) { } return gslb, nil } + +// YamlToIngress takes yaml and returns Gslb object +func YamlToIngress(yaml []byte) (*netv1.Ingress, error) { + // yamlBytes contains a []byte of my yaml job spec + // convert the yaml to json + jsonBytes, err := yamlConv.YAMLToJSON(yaml) + if err != nil { + return &netv1.Ingress{}, err + } + // unmarshal the json into the kube struct + ingress := &netv1.Ingress{} + err = json.Unmarshal(jsonBytes, &ingress) + if err != nil { + return &netv1.Ingress{}, err + } + return ingress, nil +} From a3daa0871d420df6d440fc9fc507111b0e3cfb9c Mon Sep 17 00:00:00 2001 From: abaguas Date: Mon, 13 May 2024 15:11:20 -1000 Subject: [PATCH 2/5] go lint Signed-off-by: abaguas --- controllers/providers/dns/infoblox_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/providers/dns/infoblox_test.go b/controllers/providers/dns/infoblox_test.go index da1eb44e81..1bdb8d33ad 100644 --- a/controllers/providers/dns/infoblox_test.go +++ b/controllers/providers/dns/infoblox_test.go @@ -71,7 +71,6 @@ var ( } defaultGslb = new(k8gbv1beta1.Gslb) - ipRange = []string{"10.0.0.1", "10.0.0.2"} ) func TestCanFilterOutDelegatedZoneEntryAccordingFQDNProvided(t *testing.T) { From 3a4ee87fe715edd3563e91e6f7c8cf712005a74c Mon Sep 17 00:00:00 2001 From: abaguas Date: Mon, 13 May 2024 15:19:41 -1000 Subject: [PATCH 3/5] go lint Signed-off-by: abaguas --- api/v1beta1/gslb_types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/v1beta1/gslb_types.go b/api/v1beta1/gslb_types.go index 2d7ff79f43..2a913fc26b 100644 --- a/api/v1beta1/gslb_types.go +++ b/api/v1beta1/gslb_types.go @@ -61,16 +61,16 @@ type GslbSpec struct { // LoadBalancer holds the GSLB's load balancer configuration // +k8s:openapi-gen=true type LoadBalancer struct { - // ExposedIPs on the local Load Balancer. This information is extracted automatically from the 'ingress' or 'resourceRef' configuration (optional) + // ExposedIPs on the local Load Balancer ExposedIPs []string `json:"exposedIps,omitempty"` } // Servers holds the GSLB's servers' configuration // +k8s:openapi-gen=true type Server struct { - // Hostname exposed by the GSLB. This information is extracted automatically from the 'ingress' or 'resourceRef' configuration (optional) + // Hostname exposed by the GSLB Host string `json:"host,omitempty"` - // Kubernetes Services backing the load balanced application under the hostname. This information is extracted automatically from the 'ingress' or 'resourceRef' configuration (optional) + // Kubernetes Services backing the load balanced application Services []*NamespacedName `json:"services,omitempty"` } From dc6db01813035507536517ff3e3d492b200e2289 Mon Sep 17 00:00:00 2001 From: Andre Baptista Aguas Date: Sat, 8 Jun 2024 21:48:13 +0200 Subject: [PATCH 4/5] terratest roundrobin Signed-off-by: abaguas --- Makefile | 2 +- chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml | 42 +++++++-------- controllers/gslb_controller_reconciliation.go | 2 +- controllers/mocks/refresolver_mock.go | 10 ++-- controllers/providers/dns/infoblox_test.go | 2 +- controllers/refresolver/ingress.go | 4 +- controllers/refresolver/ingress_test.go | 4 +- controllers/refresolver/refresolver.go | 4 +- .../failover-playground-ref-gslb.yaml | 13 +++++ .../failover-playground-ref-ingress.yaml | 19 +++++++ terratest/examples/failover-ref-gslb.yaml | 13 +++++ terratest/examples/failover-ref-ingress.yaml | 19 +++++++ .../examples/roundrobin-weight1-ref-gslb.yaml | 15 ++++++ .../roundrobin-weight1-ref-ingress.yaml | 39 ++++++++++++++ ...n_weight1.yaml => roundrobin-weight1.yaml} | 0 terratest/examples/roundrobin2-ref-gslb.yaml | 12 +++++ .../examples/roundrobin2-ref-ingress.yaml | 19 +++++++ .../k8gb_abstract_full_roundrobin_test.go | 21 ++++---- .../test/k8gb_failover_playground_test.go | 51 ++++++++++++++---- terratest/test/k8gb_full_failover_test.go | 54 ++++++++++++++----- terratest/test/k8gb_full_roundrobin_test.go | 21 +++++++- terratest/test/k8gb_weight_test.go | 40 +++++++++++--- terratest/utils/extensions.go | 35 ++++++++---- 23 files changed, 353 insertions(+), 88 deletions(-) create mode 100644 terratest/examples/failover-playground-ref-gslb.yaml create mode 100644 terratest/examples/failover-playground-ref-ingress.yaml create mode 100644 terratest/examples/failover-ref-gslb.yaml create mode 100644 terratest/examples/failover-ref-ingress.yaml create mode 100644 terratest/examples/roundrobin-weight1-ref-gslb.yaml create mode 100644 terratest/examples/roundrobin-weight1-ref-ingress.yaml rename terratest/examples/{roundrobin_weight1.yaml => roundrobin-weight1.yaml} (100%) create mode 100644 terratest/examples/roundrobin2-ref-gslb.yaml create mode 100644 terratest/examples/roundrobin2-ref-ingress.yaml diff --git a/Makefile b/Makefile index affa78fa96..db598f7193 100644 --- a/Makefile +++ b/Makefile @@ -452,7 +452,7 @@ terratest: # Run terratest suite echo -e "$(RED)Make sure you run the tests against at least two running clusters$(NC)" ;\ exit 1;\ fi - cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 15m -parallel=12 --tags=$(TEST_TAGS) + cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 20m -parallel=12 --tags=$(TEST_TAGS) .PHONY: website website: diff --git a/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml b/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml index 848b4d44a3..a0e8de3fbb 100644 --- a/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml +++ b/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml @@ -324,25 +324,25 @@ spec: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that relates - the key and values. + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. properties: key: description: key is the label key that the selector applies to. type: string operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If - the operator is In or NotIn, the values array must - be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced - during a strategic merge patch. + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. items: type: string type: array @@ -354,13 +354,13 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A - single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is "key", - the operator is "In", and the values array contains only - "value". The requirements are ANDed. + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. type: object type: object + x-kubernetes-map-type: atomic type: object strategy: description: Gslb Strategy spec @@ -408,9 +408,7 @@ spec: description: LoadBalancer configuration properties: exposedIps: - description: ExposedIPs on the local Load Balancer. This information - is extracted automatically from the 'ingress' or 'resourceRef' - configuration (optional) + description: ExposedIPs on the local Load Balancer items: type: string type: array @@ -421,14 +419,10 @@ spec: description: Servers holds the GSLB's servers' configuration properties: host: - description: Hostname exposed by the GSLB. This information - is extracted automatically from the 'ingress' or 'resourceRef' - configuration (optional) + description: Hostname exposed by the GSLB type: string services: description: Kubernetes Services backing the load balanced application - under the hostname. This information is extracted automatically - from the 'ingress' or 'resourceRef' configuration (optional) items: description: NamespacedName holds a reference to a k8s resource properties: diff --git a/controllers/gslb_controller_reconciliation.go b/controllers/gslb_controller_reconciliation.go index 2936622518..e998b98cd4 100644 --- a/controllers/gslb_controller_reconciliation.go +++ b/controllers/gslb_controller_reconciliation.go @@ -169,7 +169,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. gslb.Status.Servers = servers fmt.Printf("got servers: %v\n", servers) - loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb, r.Client, r.Config.EdgeDNSServers) + loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Client, r.Config.EdgeDNSServers) if err != nil { m.IncrementError(gslb) return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err)) diff --git a/controllers/mocks/refresolver_mock.go b/controllers/mocks/refresolver_mock.go index dacf7c1970..99a3db8a33 100644 --- a/controllers/mocks/refresolver_mock.go +++ b/controllers/mocks/refresolver_mock.go @@ -31,7 +31,7 @@ import ( reflect "reflect" v1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" - utils "github.com/k8gb-io/k8gb/controllers/internal/utils" + utils "github.com/k8gb-io/k8gb/controllers/utils" gomock "go.uber.org/mock/gomock" client "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -60,18 +60,18 @@ func (m *MockGslbReferenceResolver) EXPECT() *MockGslbReferenceResolverMockRecor } // GetGslbExposedIPs mocks base method. -func (m *MockGslbReferenceResolver) GetGslbExposedIPs(arg0 *v1beta1.Gslb, arg1 client.Client, arg2 utils.DNSList) ([]string, error) { +func (m *MockGslbReferenceResolver) GetGslbExposedIPs(arg0 client.Client, arg1 utils.DNSList) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetGslbExposedIPs", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "GetGslbExposedIPs", arg0, arg1) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetGslbExposedIPs indicates an expected call of GetGslbExposedIPs. -func (mr *MockGslbReferenceResolverMockRecorder) GetGslbExposedIPs(arg0, arg1, arg2 any) *gomock.Call { +func (mr *MockGslbReferenceResolverMockRecorder) GetGslbExposedIPs(arg0, arg1 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGslbExposedIPs", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetGslbExposedIPs), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGslbExposedIPs", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetGslbExposedIPs), arg0, arg1) } // GetServers mocks base method. diff --git a/controllers/providers/dns/infoblox_test.go b/controllers/providers/dns/infoblox_test.go index 1bdb8d33ad..82cfa264a3 100644 --- a/controllers/providers/dns/infoblox_test.go +++ b/controllers/providers/dns/infoblox_test.go @@ -188,7 +188,7 @@ func TestInfobloxCreateZoneDelegationForExternalDNSWithSplitBrainEnabled(t *test a := mocks.NewMockAssistant(ctrl) cl := mocks.NewMockInfobloxClient(ctrl) con := mocks.NewMockIBConnector(ctrl) - a.EXPECT().InspectTXTThreshold(gomock.Any(), gomock.Any()).Do(func(fqdn string, arg1 interface{}) { + a.EXPECT().InspectTXTThreshold(gomock.Any(), gomock.Any()).Do(func(fqdn string, _ interface{}) { require.Equal(t, "test-gslb-heartbeat-us-east-1.example.com", fqdn) }).Return(nil).Times(1) con.EXPECT().CreateObject(gomock.Any()).Return(ref, nil).AnyTimes() diff --git a/controllers/refresolver/ingress.go b/controllers/refresolver/ingress.go index c81ecf2e76..d8ba333805 100644 --- a/controllers/refresolver/ingress.go +++ b/controllers/refresolver/ingress.go @@ -42,7 +42,7 @@ import ( "reflect" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" - "github.com/k8gb-io/k8gb/controllers/internal/utils" + "github.com/k8gb-io/k8gb/controllers/utils" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -171,7 +171,7 @@ func (irr *IngressReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) } // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB -func (irr *IngressReferenceResolver) GetGslbExposedIPs(gslb *k8gbv1beta1.Gslb, _ client.Client, edgeDNSServers utils.DNSList) ([]string, error) { +func (irr *IngressReferenceResolver) GetGslbExposedIPs(_ client.Client, edgeDNSServers utils.DNSList) ([]string, error) { gslbIngressIPs := []string{} for _, ip := range irr.ingress.Status.LoadBalancer.Ingress { diff --git a/controllers/refresolver/ingress_test.go b/controllers/refresolver/ingress_test.go index 0d824a8a1b..62a7e94c30 100644 --- a/controllers/refresolver/ingress_test.go +++ b/controllers/refresolver/ingress_test.go @@ -42,7 +42,7 @@ import ( "testing" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" - "github.com/k8gb-io/k8gb/controllers/internal/utils" + "github.com/k8gb-io/k8gb/controllers/utils" "github.com/stretchr/testify/assert" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/runtime" @@ -177,7 +177,7 @@ func TestGetGslbExposedIPs(t *testing.T) { } // act - IPs, err := resolver.GetGslbExposedIPs(&k8gbv1beta1.Gslb{}, nil, []utils.DNSServer{}) + IPs, err := resolver.GetGslbExposedIPs(nil, []utils.DNSServer{}) assert.NoError(t, err) // assert diff --git a/controllers/refresolver/refresolver.go b/controllers/refresolver/refresolver.go index dadc868796..b63c19c453 100644 --- a/controllers/refresolver/refresolver.go +++ b/controllers/refresolver/refresolver.go @@ -41,8 +41,8 @@ import ( "reflect" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" - "github.com/k8gb-io/k8gb/controllers/internal/utils" "github.com/k8gb-io/k8gb/controllers/logging" + "github.com/k8gb-io/k8gb/controllers/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -54,7 +54,7 @@ type GslbReferenceResolver interface { // GetServers retrieves the server configuration of the GSLB object GetServers() ([]*k8gbv1beta1.Server, error) // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB - GetGslbExposedIPs(*k8gbv1beta1.Gslb, client.Client, utils.DNSList) ([]string, error) + GetGslbExposedIPs(client.Client, utils.DNSList) ([]string, error) } // New creates a new GSLBReferenceResolver diff --git a/terratest/examples/failover-playground-ref-gslb.yaml b/terratest/examples/failover-playground-ref-gslb.yaml new file mode 100644 index 0000000000..fa77797c58 --- /dev/null +++ b/terratest/examples/failover-playground-ref-gslb.yaml @@ -0,0 +1,13 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: test-gslb + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" diff --git a/terratest/examples/failover-playground-ref-ingress.yaml b/terratest/examples/failover-playground-ref-ingress.yaml new file mode 100644 index 0000000000..509542e81e --- /dev/null +++ b/terratest/examples/failover-playground-ref-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-gslb + labels: + app: test-gslb +spec: + ingressClassName: nginx + rules: + - host: playground-failover.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/examples/failover-ref-gslb.yaml b/terratest/examples/failover-ref-gslb.yaml new file mode 100644 index 0000000000..fa77797c58 --- /dev/null +++ b/terratest/examples/failover-ref-gslb.yaml @@ -0,0 +1,13 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: test-gslb + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" diff --git a/terratest/examples/failover-ref-ingress.yaml b/terratest/examples/failover-ref-ingress.yaml new file mode 100644 index 0000000000..99b276c2cf --- /dev/null +++ b/terratest/examples/failover-ref-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-gslb + labels: + app: test-gslb +spec: + ingressClassName: nginx + rules: + - host: terratest-failover.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/examples/roundrobin-weight1-ref-gslb.yaml b/terratest/examples/roundrobin-weight1-ref-gslb.yaml new file mode 100644 index 0000000000..0ddb8d10a0 --- /dev/null +++ b/terratest/examples/roundrobin-weight1-ref-gslb.yaml @@ -0,0 +1,15 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: test-gslb + strategy: + type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too + dnsTtlSeconds: 5 + weight: + eu: 5 + us: 5 diff --git a/terratest/examples/roundrobin-weight1-ref-ingress.yaml b/terratest/examples/roundrobin-weight1-ref-ingress.yaml new file mode 100644 index 0000000000..98fd48ef92 --- /dev/null +++ b/terratest/examples/roundrobin-weight1-ref-ingress.yaml @@ -0,0 +1,39 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-gslb + labels: + app: test-gslb +spec: + ingressClassName: nginx + rules: + - host: terratest-notfound.cloud.example.com # This is the GSLB enabled host that clients would use + http: # This section mirrors the same structure as that of an Ingress resource and will be used verbatim when creating the corresponding Ingress resource that will match the GSLB host + paths: + - path: / + pathType: Prefix + backend: + service: + name: non-existing-app # Gslb should reflect NotFound status + port: + name: http + - host: terratest-unhealthy.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: unhealthy-app # Gslb should reflect Unhealthy status + port: + name: http + - host: terratest-roundrobin.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/examples/roundrobin_weight1.yaml b/terratest/examples/roundrobin-weight1.yaml similarity index 100% rename from terratest/examples/roundrobin_weight1.yaml rename to terratest/examples/roundrobin-weight1.yaml diff --git a/terratest/examples/roundrobin2-ref-gslb.yaml b/terratest/examples/roundrobin2-ref-gslb.yaml new file mode 100644 index 0000000000..84e0794c81 --- /dev/null +++ b/terratest/examples/roundrobin2-ref-gslb.yaml @@ -0,0 +1,12 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: roundrobin-test-gslb +spec: + resourceRef: + ingress: + matchLabels: + app: roundrobin-test-gslb + strategy: + type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too + dnsTtlSeconds: 5 diff --git a/terratest/examples/roundrobin2-ref-ingress.yaml b/terratest/examples/roundrobin2-ref-ingress.yaml new file mode 100644 index 0000000000..209ef8e28d --- /dev/null +++ b/terratest/examples/roundrobin2-ref-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: roundrobin-test-gslb + labels: + app: roundrobin-test-gslb +spec: + ingressClassName: nginx + rules: + - host: roundrobin-test.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http diff --git a/terratest/test/k8gb_abstract_full_roundrobin_test.go b/terratest/test/k8gb_abstract_full_roundrobin_test.go index 81d3cfd92d..33a12c127f 100644 --- a/terratest/test/k8gb_abstract_full_roundrobin_test.go +++ b/terratest/test/k8gb_abstract_full_roundrobin_test.go @@ -26,7 +26,7 @@ import ( "github.com/stretchr/testify/require" ) -func abstractTestFullRoundRobin(t *testing.T, n int) { +func abstractTestFullRoundRobin(t *testing.T, n int, testPrefix string, gslbPath string, ingressPath string) { if n < 2 || n > 8 { t.Logf("Use value of n that represents the number of clusters from interval [2,8]") t.FailNow() @@ -36,20 +36,23 @@ func abstractTestFullRoundRobin(t *testing.T, n int) { var instances []*utils.Instance const host = "roundrobin-test.cloud.example.com" - const gslbPath = "../examples/roundrobin2.yaml" // start all the test apps on all the clusters for i := 0; i < n; i += 1 { - instance, er := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i). + workflow := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i). WithGslb(gslbPath, host). - WithTestApp(tags[i]). - Start() + WithTestApp(tags[i]) + if ingressPath != "" { + workflow = workflow.WithIngress(ingressPath) + } + + instance, er := workflow.Start() require.NoError(t, er) instances = append(instances, instance) defer instance.Kill() } var err error - t.Run(fmt.Sprintf("round-robin on %d concurrent clusters with podinfo running", n), func(t *testing.T) { + t.Run(fmt.Sprintf("%s round-robin on %d concurrent clusters with podinfo running", testPrefix, n), func(t *testing.T) { for _, ins := range instances { err = ins.WaitForAppIsRunning() require.NoError(t, err) @@ -61,13 +64,13 @@ func abstractTestFullRoundRobin(t *testing.T, n int) { for _, ins := range instances { workingTargets = append(workingTargets, ins.GetLocalTargets()...) } - t.Run(fmt.Sprintf("all %d clusters should be interconnected", n), func(t *testing.T) { + t.Run(fmt.Sprintf("%s all %d clusters should be interconnected", testPrefix, n), func(t *testing.T) { allShouldExpectTheseTargets(t, instances, workingTargets) }) // kill the apps on clusters one by one and expect less and less targets to be available for i, instance := range instances { - t.Run(fmt.Sprintf("kill podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) { + t.Run(fmt.Sprintf("%s kill podinfo on cluster %d (%s)", testPrefix, i+1, tags[i]), func(t *testing.T) { workingTargets = distinct(subtract(workingTargets, instance.GetLocalTargets())) t.Logf("New expected targets: %v", workingTargets) instance.StopTestApp() @@ -77,7 +80,7 @@ func abstractTestFullRoundRobin(t *testing.T, n int) { // start the test apps again on each cluster and check if the targets start appearing for i, instance := range instances { - t.Run(fmt.Sprintf("start podinfo on cluster %d (%s)", i+1, tags[i]), func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo on cluster %d (%s)", testPrefix, i+1, tags[i]), func(t *testing.T) { instance.StartTestApp() workingTargets = distinct(append(workingTargets, instance.GetLocalTargets()...)) t.Logf("New expected targets: %v", workingTargets) diff --git a/terratest/test/k8gb_failover_playground_test.go b/terratest/test/k8gb_failover_playground_test.go index e5e3d19817..4839172c0f 100644 --- a/terratest/test/k8gb_failover_playground_test.go +++ b/terratest/test/k8gb_failover_playground_test.go @@ -22,6 +22,7 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ import ( + "fmt" "k8gbterratest/utils" "testing" @@ -32,21 +33,51 @@ import ( // TestFailoverPlayground is equal to k8gb failover test running on local playground. // see: https://github.com/k8gb-io/k8gb/blob/master/docs/local.md#failover func TestFailoverPlayground(t *testing.T) { + tests := []struct { + name string + gslbPath string + ingressPath string + }{ + { + name: "embedded ingress", + gslbPath: "../examples/failover-playground.yaml", + ingressPath: "", + }, + { + name: "referenced ingress", + gslbPath: "../examples/failover-playground-ref-gslb.yaml", + ingressPath: "../examples/failover-playground-ref-ingress.yaml", + }, + } + + for _, test := range tests { + abstractTestFailoverPlayground(t, test.name, test.gslbPath, test.ingressPath) + } +} + +func abstractTestFailoverPlayground(t *testing.T, testPrefix string, gslbPath string, ingressPath string) { const host = "playground-failover.cloud.example.com" - const gslbPath = "../examples/failover-playground.yaml" + const euGeoTag = "eu" const usGeoTag = "us" - instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp(euGeoTag). - Start() + WithTestApp(euGeoTag) + if ingressPath != "" { + workflowEU = workflowEU.WithIngress(ingressPath) + } + instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp(usGeoTag). - Start() + WithTestApp(usGeoTag) + if ingressPath != "" { + workflowUS = workflowUS.WithIngress(ingressPath) + } + instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() @@ -63,7 +94,7 @@ func TestFailoverPlayground(t *testing.T) { assert.Equal(t, geoTag, httpResult.Message) } - t.Run("failover on two concurrent clusters with TestApp running", func(t *testing.T) { + t.Run(fmt.Sprintf("%s failover on two concurrent clusters with TestApp running", testPrefix), func(t *testing.T) { err = instanceEU.WaitForAppIsRunning() require.NoError(t, err) err = instanceUS.WaitForAppIsRunning() @@ -73,13 +104,13 @@ func TestFailoverPlayground(t *testing.T) { euLocalTargets := instanceEU.GetLocalTargets() usLocalTargets := instanceUS.GetLocalTargets() - t.Run("stop podinfo on eu cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s stop podinfo on eu cluster", testPrefix), func(t *testing.T) { instanceEU.StopTestApp() require.NoError(t, instanceEU.WaitForAppIsStopped()) actAndAssert(t.Name(), usGeoTag, usLocalTargets) }) - t.Run("start podinfo again on eu cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo again on eu cluster", testPrefix), func(t *testing.T) { instanceEU.StartTestApp() require.NoError(t, instanceEU.WaitForAppIsRunning()) actAndAssert(t.Name(), euGeoTag, euLocalTargets) diff --git a/terratest/test/k8gb_full_failover_test.go b/terratest/test/k8gb_full_failover_test.go index 0ad028f084..dd065e4055 100644 --- a/terratest/test/k8gb_full_failover_test.go +++ b/terratest/test/k8gb_full_failover_test.go @@ -22,6 +22,7 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ import ( + "fmt" "k8gbterratest/utils" "testing" @@ -29,23 +30,52 @@ import ( ) func TestFullFailover(t *testing.T) { + tests := []struct { + name string + gslbPath string + ingressPath string + }{ + { + name: "embedded ingress", + gslbPath: "../examples/failover.yaml", + ingressPath: "", + }, + { + name: "referenced ingress", + gslbPath: "../examples/failover-ref-gslb.yaml", + ingressPath: "../examples/failover-ref-ingress.yaml", + }, + } + + for _, test := range tests { + abstractTestFullFailover(t, test.name, test.gslbPath, test.ingressPath) + } +} + +func abstractTestFullFailover(t *testing.T, testPrefix string, gslbPath string, ingressPath string) { const host = "terratest-failover.cloud.example.com" - const gslbPath = "../examples/failover.yaml" - instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp("eu"). - Start() + WithTestApp("eu") + if ingressPath != "" { + workflowEU = workflowEU.WithIngress(ingressPath) + } + instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp("us"). - Start() + WithTestApp("us") + if ingressPath != "" { + workflowUS = workflowUS.WithIngress(ingressPath) + } + instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() - t.Run("failover on two concurrent clusters with podinfo running", func(t *testing.T) { + t.Run(fmt.Sprintf("%s failover on two concurrent clusters with podinfo running", testPrefix), func(t *testing.T) { err = instanceEU.WaitForAppIsRunning() require.NoError(t, err) err = instanceUS.WaitForAppIsRunning() @@ -55,7 +85,7 @@ func TestFullFailover(t *testing.T) { euLocalTargets := instanceEU.GetLocalTargets() usLocalTargets := instanceUS.GetLocalTargets() - t.Run("kill podinfo on the second cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s kill podinfo on the second cluster", testPrefix), func(t *testing.T) { instanceUS.StopTestApp() err = instanceUS.WaitForExpected(euLocalTargets) require.NoError(t, err) @@ -63,7 +93,7 @@ func TestFullFailover(t *testing.T) { require.NoError(t, err) }) - t.Run("kill podinfo on the first cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s kill podinfo on the first cluster", testPrefix), func(t *testing.T) { instanceEU.StopTestApp() err = instanceEU.WaitForExpected([]string{}) require.NoError(t, err) @@ -71,7 +101,7 @@ func TestFullFailover(t *testing.T) { require.NoError(t, err) }) - t.Run("start podinfo on the second cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo on the second cluster", testPrefix), func(t *testing.T) { instanceUS.StartTestApp() err = instanceUS.WaitForExpected(usLocalTargets) require.NoError(t, err) @@ -79,7 +109,7 @@ func TestFullFailover(t *testing.T) { require.NoError(t, err) }) - t.Run("start podinfo on the first cluster", func(t *testing.T) { + t.Run(fmt.Sprintf("%s start podinfo on the first cluster", testPrefix), func(t *testing.T) { instanceEU.StartTestApp() err = instanceEU.WaitForExpected(euLocalTargets) require.NoError(t, err) diff --git a/terratest/test/k8gb_full_roundrobin_test.go b/terratest/test/k8gb_full_roundrobin_test.go index 317d2a634c..bbb5d60f36 100644 --- a/terratest/test/k8gb_full_roundrobin_test.go +++ b/terratest/test/k8gb_full_roundrobin_test.go @@ -26,5 +26,24 @@ import ( ) func TestFullRoundRobin(t *testing.T) { - abstractTestFullRoundRobin(t, settings.ClustersNumber) + tests := []struct { + name string + gslbPath string + ingressPath string + }{ + { + name: "embedded ingress", + gslbPath: "../examples/roundrobin2.yaml", + ingressPath: "", + }, + { + name: "referenced ingress", + gslbPath: "../examples/roundrobin2-ref-gslb.yaml", + ingressPath: "../examples/roundrobin2-ref-ingress.yaml", + }, + } + + for test := range tests { + abstractTestFullRoundRobin(t, settings.ClusterNumber, test.name, test.gslbPath, test.ingressPath) + } } diff --git a/terratest/test/k8gb_weight_test.go b/terratest/test/k8gb_weight_test.go index dbeb260bdc..17cfc2b845 100644 --- a/terratest/test/k8gb_weight_test.go +++ b/terratest/test/k8gb_weight_test.go @@ -28,21 +28,47 @@ import ( func TestWeightsExistsInLocalDNSEndpoint(t *testing.T) { t.Parallel() + + tests := []struct { + gslbPath string + ingressPath string + }{ + { + gslbPath: "../examples/roundrobin-weight1.yaml", + ingressPath: "", + }, + { + gslbPath: "../examples/roundrobin-weight1-ref-gslb.yaml", + ingressPath: "../examples/roundrobin-weight1-ref-ingress.yaml", + }, + } + + for _, test := range tests { + abstractTestWeightsExistsInLocalDNSEndpoint(t, test.gslbPath, test.ingressPath) + } +} + +func abstractTestWeightsExistsInLocalDNSEndpoint(t *testing.T, gslbPath string, ingressPath string) { const host = "terratest-roundrobin.cloud.example.com" const endpointDNSNameEU = "gslb-ns-eu-cloud.example.com" const endpointDNSNameUS = "gslb-ns-us-cloud.example.com" - const gslbPath = "../examples/roundrobin_weight1.yaml" - instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). - WithTestApp("eu"). - Start() + WithTestApp("eu") + if ingressPath != "" { + workflowEU.WithIngress(ingressPath) + } + instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - instanceUS, err := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). WithGslb(gslbPath, host). - WithTestApp("us"). - Start() + WithTestApp("us") + if ingressPath != "" { + workflowUS.WithIngress(ingressPath) + } + instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() diff --git a/terratest/utils/extensions.go b/terratest/utils/extensions.go index 629d7a5166..20fb2e511b 100644 --- a/terratest/utils/extensions.go +++ b/terratest/utils/extensions.go @@ -125,7 +125,15 @@ func (w *Workflow) WithIngress(path string) *Workflow { if path == "" { w.error = fmt.Errorf("empty ingress resource path") } - w.settings.ingressResourcePath = path + var err error + w.settings.ingressResourcePath, err = filepath.Abs(path) + if err != nil { + w.error = fmt.Errorf("reading %s; %s", path, err) + } + w.settings.ingressName, err = w.getManifestName(w.settings.ingressResourcePath) + if err != nil { + w.error = err + } return w } @@ -148,9 +156,6 @@ func (w *Workflow) WithGslb(path, host string) *Workflow { w.error = err } w.state.gslb.host = host - if err != nil { - w.error = err - } return w } @@ -211,12 +216,20 @@ func (w *Workflow) Start() (*Instance, error) { // gslb if w.settings.gslbResourcePath != "" { - w.t.Logf("Create ingress %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) - k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) - k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.state.gslb.name, 100, 1*time.Second) - ingress := k8s.GetIngress(w.t, w.k8sOptions, w.state.gslb.name) - require.Equal(w.t, ingress.Name, w.state.gslb.name) - w.settings.ingressName = w.state.gslb.name + if w.settings.ingressResourcePath == "" { + w.t.Logf("Create ingress %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) + k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.state.gslb.name, 100, 5*time.Second) + ingress := k8s.GetIngress(w.t, w.k8sOptions, w.state.gslb.name) + require.Equal(w.t, ingress.Name, w.state.gslb.name) + w.settings.ingressName = w.state.gslb.name + } else { + w.t.Logf("Create ingress %s from %s", w.settings.ingressName, w.settings.ingressResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.ingressResourcePath) + k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.settings.ingressName, 100, 5*time.Second) + w.t.Logf("Create gslb %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) + } } return &Instance{ @@ -258,7 +271,7 @@ func (i *Instance) ReapplyIngress(path string) { require.NoError(i.w.t, err) k8s.KubectlApply(i.w.t, i.w.k8sOptions, i.w.settings.ingressResourcePath) // modifying inner state.gslb.name and ingress.Name has nothing to do with reading these values dynamically afterwards. - k8s.WaitUntilIngressAvailable(i.w.t, i.w.k8sOptions, i.w.state.gslb.name, 60, 1*time.Second) + k8s.WaitUntilIngressAvailable(i.w.t, i.w.k8sOptions, i.w.state.gslb.name, 60, 5*time.Second) ingress := k8s.GetIngress(i.w.t, i.w.k8sOptions, i.w.state.gslb.name) require.Equal(i.w.t, ingress.Name, i.w.state.gslb.name) i.w.settings.ingressName = i.w.state.gslb.name From 67c01602f058b776bed1f8abcc7860bcf341b0c1 Mon Sep 17 00:00:00 2001 From: abaguas Date: Wed, 26 Jun 2024 11:29:56 +0200 Subject: [PATCH 5/5] fix namespace isolation Signed-off-by: abaguas --- controllers/refresolver/ingress.go | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/refresolver/ingress.go b/controllers/refresolver/ingress.go index d8ba333805..1ca30e6977 100644 --- a/controllers/refresolver/ingress.go +++ b/controllers/refresolver/ingress.go @@ -99,6 +99,7 @@ func getGslbIngressRef(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) ([]netv1 } opts := &client.ListOptions{ LabelSelector: selector, + Namespace: gslb.Namespace, } err = k8sClient.List(context.TODO(), ingressList, opts)