From 7878e35471fe9b4b2e416d633f00590d21e375ca Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Thu, 15 Sep 2022 16:08:10 +0100 Subject: [PATCH 01/19] Initial support for TS external name --- internal/configs/configurator.go | 68 ++++----- internal/configs/transportserver.go | 33 +++-- internal/configs/transportserver_test.go | 134 ++++++++++++++++-- internal/configs/version1/nginx-plus.tmpl | 5 + .../version2/nginx-plus.transportserver.tmpl | 2 +- internal/configs/version2/stream.go | 1 + internal/configs/version2/templates_test.go | 55 +++++++ internal/k8s/controller.go | 32 +++-- 8 files changed, 256 insertions(+), 74 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 8ce08941a1..e72c29db61 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -573,29 +573,25 @@ func (cnf *Configurator) deleteTransportServerMetricsLabels(key string) { // AddOrUpdateTransportServer adds or updates NGINX configuration for the TransportServer resource. // It is a responsibility of the caller to check that the TransportServer references an existing listener. -func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) error { - err := cnf.addOrUpdateTransportServer(transportServerEx) +func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { + warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) if err != nil { - return fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) + return warnings, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } - if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return fmt.Errorf("Error reloading NGINX for TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) + return warnings, fmt.Errorf("error reloading NGINX for TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } - - return nil + return warnings, nil } -func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) error { +func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { name := getFileNameForTransportServer(transportServerEx.TransportServer) - - tsCfg := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus) + tsCfg, warnings := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus, cnf.IsResolverConfigured()) content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(tsCfg) if err != nil { - return fmt.Errorf("Error generating TransportServer config %v: %w", name, err) + return warnings, fmt.Errorf("error generating TransportServer config %v: %w", name, err) } - if cnf.isPlus && cnf.isPrometheusEnabled { cnf.updateTransportServerMetricsLabels(transportServerEx, tsCfg.Upstreams) } @@ -610,11 +606,9 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport Host: transportServerEx.TransportServer.Spec.Host, UnixSocket: generateUnixSocket(transportServerEx), } - - return cnf.updateTLSPassthroughHostsConfig() + return warnings, cnf.updateTLSPassthroughHostsConfig() } - - return nil + return warnings, nil } // GetVirtualServerRoutesForVirtualServer returns the virtualServerRoutes that a virtualServer @@ -697,16 +691,16 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn } for _, tsEx := range resources.TransportServerExes { - err := cnf.addOrUpdateTransportServer(tsEx) + warnings, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) + return allWarnings, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } + allWarnings.Add(warnings) } if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { return allWarnings, fmt.Errorf("Error when reloading NGINX when updating resources: %w", err) } - return allWarnings, nil } @@ -939,11 +933,11 @@ func (cnf *Configurator) UpdateEndpointsForTransportServers(transportServerExes reloadPlus := false for _, tsEx := range transportServerExes { - err := cnf.addOrUpdateTransportServer(tsEx) + // Ignore warnings here as no new warnings should appear when updating Endpoints for TransportServers + _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { - return fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) + return fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } - if cnf.isPlus { err := cnf.updatePlusEndpointsForTransportServer(tsEx) if err != nil { @@ -957,11 +951,9 @@ func (cnf *Configurator) UpdateEndpointsForTransportServers(transportServerExes glog.V(3).Info("No need to reload nginx") return nil } - if err := cnf.reload(nginx.ReloadForEndpointsUpdate); err != nil { - return fmt.Errorf("Error reloading NGINX when updating endpoints: %w", err) + return fmt.Errorf("error reloading NGINX when updating endpoints: %w", err) } - return nil } @@ -1069,6 +1061,7 @@ func (cnf *Configurator) updateStreamServersInPlus(upstream string, servers []st } // UpdateConfig updates NGINX configuration parameters. +// //gocyclo:ignore func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources ExtendedResources) (Warnings, error) { cnf.cfgParams = cfgParams @@ -1077,7 +1070,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende if cnf.cfgParams.MainServerSSLDHParamFileContent != nil { fileName, err := cnf.nginxManager.CreateDHParam(*cnf.cfgParams.MainServerSSLDHParamFileContent) if err != nil { - return allWarnings, fmt.Errorf("Error when updating dhparams: %w", err) + return allWarnings, fmt.Errorf("error when updating dhparams: %w", err) } cfgParams.MainServerSSLDHParam = fileName } @@ -1085,28 +1078,28 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende if cfgParams.MainTemplate != nil { err := cnf.templateExecutor.UpdateMainTemplate(cfgParams.MainTemplate) if err != nil { - return allWarnings, fmt.Errorf("Error when parsing the main template: %w", err) + return allWarnings, fmt.Errorf("error when parsing the main template: %w", err) } } if cfgParams.IngressTemplate != nil { err := cnf.templateExecutor.UpdateIngressTemplate(cfgParams.IngressTemplate) if err != nil { - return allWarnings, fmt.Errorf("Error when parsing the ingress template: %w", err) + return allWarnings, fmt.Errorf("error when parsing the ingress template: %w", err) } } if cfgParams.VirtualServerTemplate != nil { err := cnf.templateExecutorV2.UpdateVirtualServerTemplate(cfgParams.VirtualServerTemplate) if err != nil { - return allWarnings, fmt.Errorf("Error when parsing the VirtualServer template: %w", err) + return allWarnings, fmt.Errorf("error when parsing the VirtualServer template: %w", err) } } mainCfg := GenerateNginxMainConfig(cnf.staticCfgParams, cfgParams) mainCfgContent, err := cnf.templateExecutor.ExecuteMainConfigTemplate(mainCfg) if err != nil { - return allWarnings, fmt.Errorf("Error when writing main Config") + return allWarnings, fmt.Errorf("error when writing main Config") } cnf.nginxManager.CreateMainConfig(mainCfgContent) @@ -1137,15 +1130,24 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende // (2) addOrUpdateTransportServer doesn't return any warnings that we need to propagate to the caller. // if (1) and (2) is no longer the case, we need to generate the config for TransportServers + // Since we are adding support for resolving external names we need to generate TS config. + for _, tsEx := range resources.TransportServerExes { + warnings, err := cnf.addOrUpdateTransportServer(tsEx) + if err != nil { + return allWarnings, err + } + allWarnings.Add(warnings) + } + if mainCfg.OpenTracingLoadModule { if err := cnf.addOrUpdateOpenTracingTracerConfig(mainCfg.OpenTracingTracerConfig); err != nil { - return allWarnings, fmt.Errorf("Error when updating OpenTracing tracer config: %w", err) + return allWarnings, fmt.Errorf("error when updating OpenTracing tracer config: %w", err) } } cnf.nginxManager.SetOpenTracing(mainCfg.OpenTracingLoadModule) if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return allWarnings, fmt.Errorf("Error when updating config from ConfigMap: %w", err) + return allWarnings, fmt.Errorf("error when updating config from ConfigMap: %w", err) } return allWarnings, nil @@ -1154,7 +1156,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende // UpdateTransportServers updates TransportServers. func (cnf *Configurator) UpdateTransportServers(updatedTSExes []*TransportServerEx, deletedKeys []string) error { for _, tsEx := range updatedTSExes { - err := cnf.addOrUpdateTransportServer(tsEx) + _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { return fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index e0cb1ea5d1..8704da9360 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -12,10 +12,11 @@ const nginxNonExistingUnixSocket = "unix:/var/lib/nginx/non-existing-unix-socket // TransportServerEx holds a TransportServer along with the resources referenced by it. type TransportServerEx struct { - ListenerPort int - TransportServer *conf_v1alpha1.TransportServer - Endpoints map[string][]string - PodsByIP map[string]string + ListenerPort int + TransportServer *conf_v1alpha1.TransportServer + Endpoints map[string][]string + PodsByIP map[string]string + ExternalNameSvcs map[string]bool } func (tsEx *TransportServerEx) String() string { @@ -31,10 +32,10 @@ func (tsEx *TransportServerEx) String() string { } // generateTransportServerConfig generates a full configuration for a TransportServer. -func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool) *version2.TransportServerConfig { +func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool, isResolverConfigured bool) (*version2.TransportServerConfig, Warnings) { upstreamNamer := newUpstreamNamerForTransportServer(transportServerEx.TransportServer) - upstreams := generateStreamUpstreams(transportServerEx, upstreamNamer, isPlus) + upstreams, warnings := generateStreamUpstreams(transportServerEx, upstreamNamer, isPlus, isResolverConfigured) healthCheck, match := generateTransportServerHealthCheck(transportServerEx.TransportServer.Spec.Action.Pass, upstreamNamer.GetNameForUpstream(transportServerEx.TransportServer.Spec.Action.Pass), @@ -95,8 +96,7 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene Upstreams: upstreams, StreamSnippets: streamSnippets, } - - return tsConfig + return tsConfig, warnings } func generateUnixSocket(transportServerEx *TransportServerEx) string { @@ -107,17 +107,25 @@ func generateUnixSocket(transportServerEx *TransportServerEx) string { return "" } -func generateStreamUpstreams(transportServerEx *TransportServerEx, upstreamNamer *upstreamNamer, isPlus bool) []version2.StreamUpstream { +func generateStreamUpstreams(transportServerEx *TransportServerEx, upstreamNamer *upstreamNamer, isPlus bool, isResolverConfigured bool) ([]version2.StreamUpstream, Warnings) { + warnings := newWarnings() var upstreams []version2.StreamUpstream for _, u := range transportServerEx.TransportServer.Spec.Upstreams { - // subselector is not supported yet in TransportServer upstreams. That's why we pass "nil" here endpointsKey := GenerateEndpointsKey(transportServerEx.TransportServer.Namespace, u.Service, nil, uint16(u.Port)) + externalNameSvcKey := GenerateExternalNameSvcKey(transportServerEx.TransportServer.Namespace, u.Service) endpoints := transportServerEx.Endpoints[endpointsKey] - ups := generateStreamUpstream(u, upstreamNamer, endpoints, isPlus) + _, isExternalNameSvc := transportServerEx.ExternalNameSvcs[externalNameSvcKey] + if isExternalNameSvc && !isResolverConfigured { + msgFmt := "Type ExternalName service %v in upstream %v will be ignored. To use ExternalName services, a resolver must be configured in the ConfigMap" + warnings.AddWarningf(transportServerEx.TransportServer, msgFmt, u.Service, u.Name) + endpoints = []string{} + } + ups := generateStreamUpstream(u, upstreamNamer, endpoints, isPlus) + ups.Resolve = isExternalNameSvc ups.UpstreamLabels.Service = u.Service ups.UpstreamLabels.ResourceType = "transportserver" ups.UpstreamLabels.ResourceName = transportServerEx.TransportServer.Name @@ -125,8 +133,7 @@ func generateStreamUpstreams(transportServerEx *TransportServerEx, upstreamNamer upstreams = append(upstreams, ups) } - - return upstreams + return upstreams, warnings } func generateTransportServerHealthCheck(upstreamName string, generatedUpstreamName string, upstreams []conf_v1alpha1.Upstream) (*version2.StreamHealthCheck, *version2.Match) { diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index 7232e88527..c5e0ffcb9c 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -137,9 +137,12 @@ func TestGenerateTransportServerConfigForTCPSnippets(t *testing.T) { StreamSnippets: []string{"limit_conn_zone $binary_remote_addr zone=addr:10m;"}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -224,9 +227,12 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -313,9 +319,12 @@ func TestGenerateTransportServerConfigForTCPMaxConnections(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -400,9 +409,12 @@ func TestGenerateTransportServerConfigForTLSPassthrough(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -492,9 +504,101 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) + } +} + +func TestGenerateTransportServerConfig_ProducesValidConfigOnValidInputForExternalNameServiceAndConfiguredResolver(t *testing.T) { + t.Parallel() + transportServerEx := TransportServerEx{ + TransportServer: &conf_v1alpha1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "tcp-server", + Namespace: "default", + }, + Spec: conf_v1alpha1.TransportServerSpec{ + Listener: conf_v1alpha1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + Upstreams: []conf_v1alpha1.Upstream{ + { + Name: "tcp-app", + Service: "tcp-app-svc", + Port: 5001, + MaxFails: intPointer(3), + FailTimeout: "40s", + }, + }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1alpha1.SessionParameters{ + Timeout: "50s", + }, + Action: &conf_v1alpha1.Action{ + Pass: "tcp-app", + }, + }, + }, + Endpoints: map[string][]string{ + "default/tcp-app-svc:5001": { + "10.0.0.20:5001", + }, + }, + ExternalNameSvcs: map[string]bool{"default/tcp-app-svc": true}, + } + expected := &version2.TransportServerConfig{ + Upstreams: []version2.StreamUpstream{ + { + Name: "ts_default_tcp-server_tcp-app", + Servers: []version2.StreamUpstreamServer{ + { + Address: "10.0.0.20:5001", + MaxFails: 3, + FailTimeout: "40s", + }, + }, + UpstreamLabels: version2.UpstreamLabels{ + ResourceName: "tcp-server", + ResourceType: "transportserver", + ResourceNamespace: "default", + Service: "tcp-app-svc", + }, + LoadBalancingMethod: "random two least_conn", + Resolve: true, + }, + }, + Server: version2.StreamServer{ + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0s", + ProxyTimeout: "50s", + HealthCheck: nil, + ServerSnippets: []string{}, + }, + StreamSnippets: []string{}, + } + + result, warnings := generateTransportServerConfig(&transportServerEx, 2020, true, true) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Error(cmp.Diff(expected, result)) } } diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 814b8ac778..9eda364219 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -296,6 +296,11 @@ stream { include /etc/nginx/tls-passthrough-hosts.conf; } + {{if .ResolverAddresses}} + resolver {{range $resolver := .ResolverAddresses}}{{$resolver}}{{end}}{{if .ResolverValid}} valid={{.ResolverValid}}{{end}}{{if not .ResolverIPV6}} ipv6=off{{end}}; + {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} + {{end}} + server { listen 443{{if .ProxyProtocol}} proxy_protocol{{end}}; listen [::]:443{{if .ProxyProtocol}} proxy_protocol{{end}}; diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 755ff445a5..10141b05a4 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -7,7 +7,7 @@ upstream {{ $u.Name }} { {{ end }} {{ range $s := $u.Servers }} - server {{ $s.Address }} max_fails={{ $s.MaxFails }} fail_timeout={{ $s.FailTimeout }} max_conns={{ $s.MaxConnections }}; + server {{ $s.Address }} max_fails={{ $s.MaxFails }} fail_timeout={{ $s.FailTimeout }} max_conns={{ $s.MaxConnections }}{{ if $u.Resolve }} resolve{{ end }};; {{ end }} } {{ end }} diff --git a/internal/configs/version2/stream.go b/internal/configs/version2/stream.go index 3ef93c5f8c..76278f3a91 100644 --- a/internal/configs/version2/stream.go +++ b/internal/configs/version2/stream.go @@ -14,6 +14,7 @@ type StreamUpstream struct { Servers []StreamUpstreamServer UpstreamLabels UpstreamLabels LoadBalancingMethod string + Resolve bool } // StreamUpstreamServer defines a stream upstream server. diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index 37cef3ba81..6c1f9aad84 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -398,6 +398,49 @@ var transportServerCfg = TransportServerConfig{ }, } +var transportServerCfgWithResolver = TransportServerConfig{ + Upstreams: []StreamUpstream{ + { + Name: "udp-upstream", + Servers: []StreamUpstreamServer{ + { + Address: "10.0.0.20:5001", + }, + }, + Resolve: true, + }, + }, + Match: &Match{ + Name: "match_udp-upstream", + Send: `GET / HTTP/1.0\r\nHost: localhost\r\n\r\n`, + ExpectRegexModifier: "~*", + Expect: "200 OK", + }, + Server: StreamServer{ + Port: 1234, + UDP: true, + StatusZone: "udp-app", + ProxyRequests: createPointerFromInt(1), + ProxyResponses: createPointerFromInt(2), + ProxyPass: "udp-upstream", + ProxyTimeout: "10s", + ProxyConnectTimeout: "10s", + ProxyNextUpstream: true, + ProxyNextUpstreamTimeout: "10s", + ProxyNextUpstreamTries: 5, + HealthCheck: &StreamHealthCheck{ + Enabled: false, + Timeout: "5s", + Jitter: "0", + Port: 8080, + Interval: "5s", + Passes: 1, + Fails: 1, + Match: "match_udp-upstream", + }, + }, +} + func createPointerFromInt(n int) *int { return &n } @@ -447,6 +490,18 @@ func TestTransportServerForNginxPlus(t *testing.T) { t.Log(string(data)) } +func TestExecuteTemplateForTransportServerWithResolver(t *testing.T) { + t.Parallel() + executor, err := NewTemplateExecutor(nginxPlusVirtualServerTmpl, nginxPlusTransportServerTmpl) + if err != nil { + t.Fatal(err) + } + _, err = executor.ExecuteTransportServerTemplate(&transportServerCfgWithResolver) + if err != nil { + t.Errorf("Failed to execute template: %v", err) + } +} + func TestTransportServerForNginx(t *testing.T) { t.Parallel() executor, err := NewTemplateExecutor(nginxVirtualServerTmpl, nginxTransportServerTmpl) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 5b39932f91..1b2e5312b5 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -1219,8 +1219,8 @@ func (lbc *LoadBalancerController) processChanges(changes []ResourceChange) { case *TransportServerConfiguration: tsEx := lbc.createTransportServerEx(impl.TransportServer, impl.ListenerPort) - addOrUpdateErr := lbc.configurator.AddOrUpdateTransportServer(tsEx) - lbc.updateTransportServerStatusAndEvents(impl, addOrUpdateErr) + warnings, addOrUpdateErr := lbc.configurator.AddOrUpdateTransportServer(tsEx) + lbc.updateTransportServerStatusAndEvents(impl, warnings, addOrUpdateErr) } } else if c.Op == Delete { switch impl := c.Resource.(type) { @@ -1631,7 +1631,7 @@ func (lbc *LoadBalancerController) updateResourcesStatusAndEvents(resources []Re lbc.updateRegularIngressStatusAndEvents(impl, warnings, operationErr) } case *TransportServerConfiguration: - lbc.updateTransportServerStatusAndEvents(impl, operationErr) + lbc.updateTransportServerStatusAndEvents(impl, warnings, operationErr) } } } @@ -1755,7 +1755,7 @@ func (lbc *LoadBalancerController) updateRegularIngressStatusAndEvents(ingConfig } } -func (lbc *LoadBalancerController) updateTransportServerStatusAndEvents(tsConfig *TransportServerConfiguration, operationErr error) { +func (lbc *LoadBalancerController) updateTransportServerStatusAndEvents(tsConfig *TransportServerConfiguration, warnings configs.Warnings, operationErr error) { eventTitle := "AddedOrUpdated" eventType := api_v1.EventTypeNormal eventWarningMessage := "" @@ -1768,6 +1768,13 @@ func (lbc *LoadBalancerController) updateTransportServerStatusAndEvents(tsConfig state = conf_v1.StateWarning } + if messages, ok := warnings[tsConfig.TransportServer]; ok { + eventType = api_v1.EventTypeWarning + eventTitle = "AddedOrUpdatedWithWarning" + eventWarningMessage = fmt.Sprintf("with warning(s): %s", formatWarningMessages(messages)) + state = conf_v1.StateWarning + } + if operationErr != nil { eventType = api_v1.EventTypeWarning eventTitle = "AddedOrUpdatedWithError" @@ -3161,18 +3168,18 @@ func isMatchingResourceRef(ownerNs, resRef, key string) bool { func (lbc *LoadBalancerController) createTransportServerEx(transportServer *conf_v1alpha1.TransportServer, listenerPort int) *configs.TransportServerEx { endpoints := make(map[string][]string) + externalNameSvcs := make(map[string]bool) podsByIP := make(map[string]string) for _, u := range transportServer.Spec.Upstreams { podEndps, external, err := lbc.getEndpointsForUpstream(transportServer.Namespace, u.Service, uint16(u.Port)) + if err == nil && external && lbc.isNginxPlus { + externalNameSvcs[configs.GenerateExternalNameSvcKey(transportServer.Namespace, u.Service)] = true + } if err != nil { glog.Warningf("Error getting Endpoints for Upstream %v: %v", u.Name, err) } - if external { - glog.Warningf("ExternalName services are not yet supported in TransportServer upstreams") - } - // subselector is not supported yet in TransportServer upstreams. That's why we pass "nil" here endpointsKey := configs.GenerateEndpointsKey(transportServer.Namespace, u.Service, nil, uint16(u.Port)) @@ -3187,10 +3194,11 @@ func (lbc *LoadBalancerController) createTransportServerEx(transportServer *conf } return &configs.TransportServerEx{ - ListenerPort: listenerPort, - TransportServer: transportServer, - Endpoints: endpoints, - PodsByIP: podsByIP, + ListenerPort: listenerPort, + TransportServer: transportServer, + Endpoints: endpoints, + PodsByIP: podsByIP, + ExternalNameSvcs: externalNameSvcs, } } From e3f888ad4b6b3c72596208ce993fa47e928722f4 Mon Sep 17 00:00:00 2001 From: Venktesh Date: Wed, 14 Sep 2022 13:11:31 +0100 Subject: [PATCH 02/19] add example files for externalsvc vs example --- .../external-svc-dep.yaml | 39 +++++++++++++++++++ .../externalname-svc.yaml | 7 ++++ .../externalname-services/nginx-config.yaml | 7 ++++ 3 files changed, 53 insertions(+) create mode 100644 examples/custom-resources/externalname-services/external-svc-dep.yaml create mode 100644 examples/custom-resources/externalname-services/externalname-svc.yaml create mode 100644 examples/custom-resources/externalname-services/nginx-config.yaml diff --git a/examples/custom-resources/externalname-services/external-svc-dep.yaml b/examples/custom-resources/externalname-services/external-svc-dep.yaml new file mode 100644 index 0000000000..832b2e454a --- /dev/null +++ b/examples/custom-resources/externalname-services/external-svc-dep.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-ns +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: external-backend + namespace: external-ns +spec: + replicas: 1 + selector: + matchLabels: + app: external-backend + template: + metadata: + labels: + app: external-backend + spec: + containers: + - name: external-backend + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: external-backend-svc + namespace: external-ns +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: external-backend \ No newline at end of file diff --git a/examples/custom-resources/externalname-services/externalname-svc.yaml b/examples/custom-resources/externalname-services/externalname-svc.yaml new file mode 100644 index 0000000000..5901eaeba7 --- /dev/null +++ b/examples/custom-resources/externalname-services/externalname-svc.yaml @@ -0,0 +1,7 @@ +kind: Service +apiVersion: v1 +metadata: + name: externalname-service +spec: + type: ExternalName + externalName: external-backend-svc.external-ns.svc.cluster.local \ No newline at end of file diff --git a/examples/custom-resources/externalname-services/nginx-config.yaml b/examples/custom-resources/externalname-services/nginx-config.yaml new file mode 100644 index 0000000000..b272bb4932 --- /dev/null +++ b/examples/custom-resources/externalname-services/nginx-config.yaml @@ -0,0 +1,7 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + resolver-addresses: "kube-dns.kube-system.svc.cluster.local" \ No newline at end of file From 11cadc1e54fce966877b2a480c2f543ca0926294 Mon Sep 17 00:00:00 2001 From: Venktesh Date: Wed, 14 Sep 2022 15:28:05 +0100 Subject: [PATCH 03/19] add ts secure-app for externalName --- .../transport-server/secure-app-external.yaml | 84 +++++++++++++++++++ .../virtual-server/external-svc-dep.yaml | 39 +++++++++ 2 files changed, 123 insertions(+) create mode 100644 examples/custom-resources/externalname-services/transport-server/secure-app-external.yaml create mode 100644 examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml diff --git a/examples/custom-resources/externalname-services/transport-server/secure-app-external.yaml b/examples/custom-resources/externalname-services/transport-server/secure-app-external.yaml new file mode 100644 index 0000000000..20f72f6b01 --- /dev/null +++ b/examples/custom-resources/externalname-services/transport-server/secure-app-external.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-ns +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: secure-app-external-backend + namespace: external-ns +spec: + replicas: 1 + selector: + matchLabels: + app: secure-app-external-backend + template: + metadata: + labels: + app: secure-app-external-backend + spec: + containers: + - name: secure-app-external-backend + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8443 + volumeMounts: + - name: secret + mountPath: /etc/nginx/ssl + readOnly: true + - name: config-volume + mountPath: /etc/nginx/conf.d + volumes: + - name: secret + secret: + secretName: app-tls-secret + - name: config-volume + configMap: + name: secure-config +--- +apiVersion: v1 +kind: Service +metadata: + name: secure-app-external-backend-svc + namespace: external-ns +spec: + ports: + - port: 8443 + targetPort: 8443 + protocol: TCP + name: https + selector: + app: secure-app-external-backend +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: secure-config + namespace: external-ns +data: + app.conf: |- + server { + listen 8443 ssl; + listen [::]:8443 ssl; + + server_name app.example.com; + + ssl_certificate /etc/nginx/ssl/tls.crt; + ssl_certificate_key /etc/nginx/ssl/tls.key; + + default_type text/plain; + + location / { + return 200 "hello from pod $hostname\n"; + } + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: app-tls-secret + namespace: external-ns +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ3Q0NRQ3EzQWxhdnJiaWpqQU5CZ2txaGtpRzl3MEJBUXNGQURCTU1Rc3dDUVlEVlFRR0V3SlYKVXpFTE1Ba0dBMVVFQ0F3Q1EwRXhGakFVQmdOVkJBY01EVk5oYmlCR2NtRnVZMmx6WTI4eEdEQVdCZ05WQkFNTQpEMkZ3Y0M1bGVHRnRjR3hsTG1OdmJUQWVGdzB5TURBek1qTXlNekl3TkROYUZ3MHlNekF6TWpNeU16SXdORE5hCk1Fd3hDekFKQmdOVkJBWVRBbFZUTVFzd0NRWURWUVFJREFKRFFURVdNQlFHQTFVRUJ3d05VMkZ1SUVaeVlXNWoKYVhOamJ6RVlNQllHQTFVRUF3d1BZWEJ3TG1WNFlXMXdiR1V1WTI5dE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRgpBQU9DQVE4QU1JSUJDZ0tDQVFFQTJCRXhZR1JPRkhoN2VPMVlxeCtWRHMzRzMrVEhyTEZULzdEUFFEQlkza3pDCi9oZlprWCt3OW1NNkQ1RU9uK2lpVlNhUWlQMm1aNFA3N29pR0dmd3JrNjJ0eEQ5cHphODM5NC9aSjF5Q0dXZ1QKK2NWUEVZbkxjQktzSTRMcktJZ21oWVIwUjNzWWRjR1JkSXJWUFZlNUVUQlk1Z1U0RGhhMDZOUEIraitmK0krWgphWGIvMlRBekJhNHozMWpIQzg2amVQeTFMdklGazFiY3I2cSsxRGR5eklxcWxkRDYvU3Q4Q2t3cDlOaDFCUGFhCktZZ1ZVd010UVBib2s1cFFmbVMrdDg4NHdSM0dTTEU4VkxRbzgyYnJhNUR3emhIamlzOTlJRGhzbUt0U3lWOXMKaWNJbXp5dHBnSXlhTS9zWEhRQU9KbVFJblFteWgyekd1WFhTQ0lkRGtRSURBUUFCTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0tsVkhOZ1k5VHZLaW9Xb0tvdllCdnNRMmYrcmFOOEJwdWNDcnRvRm15NUczcGIzU2lPTndaCkF2cnhtSm4vR3lsa3JKTHBpQVA1eUNBNGI2Y2lYMnRGa3pQRmhJVFZKRTVBeDlpaEF2WWZwTUFSdWVqM29HN2UKd0xwQk1iUnlGbHJYV29NWUVBMGxsV0JueHRQQXZYS2Y4SVZGYTRSSDhzV1JJSDB4M2hFdjVtQ3VUZjJTRTg0QwpiNnNjS3Z3MW9CQU5VWGxXRVZVYTFmei9rWWZBa1lrdHZyV2JUcTZTWGxodXRJYWY4WEYzSUMrL2x1b3gzZThMCjBBcEFQVE5sZ0JwOTkvcXMrOG9PMWthSmQ1TmV6TnlJeXhSdUtJMzlDWkxuQm9OYmkzdlFYY1NzRCtYU2lYT0cKcEVnTjNtci8xRms4OVZMSENhTnkyKzBqMjZ0eWpiclcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRFlFVEZnWkU0VWVIdDQKN1Zpckg1VU96Y2JmNU1lc3NWUC9zTTlBTUZqZVRNTCtGOW1SZjdEMll6b1BrUTZmNktKVkpwQ0kvYVpuZy92dQppSVlaL0N1VHJhM0VQMm5OcnpmM2o5a25YSUlaYUJQNXhVOFJpY3R3RXF3amd1c29pQ2FGaEhSSGV4aDF3WkYwCml0VTlWN2tSTUZqbUJUZ09GclRvMDhINlA1LzRqNWxwZHYvWk1ETUZyalBmV01jTHpxTjQvTFV1OGdXVFZ0eXYKcXI3VU4zTE1pcXFWMFByOUszd0tUQ24wMkhVRTlwb3BpQlZUQXkxQTl1aVRtbEIrWkw2M3p6akJIY1pJc1R4VQp0Q2p6WnV0cmtQRE9FZU9LejMwZ09HeVlxMUxKWDJ5SndpYlBLMm1Bakpveit4Y2RBQTRtWkFpZENiS0hiTWE1CmRkSUloME9SQWdNQkFBRUNnZ0VCQUxYaW16ODZrT1A0bkhBcTFPYVEyb2l3dndhQTczbTNlUytZSm84eFk4NFcKcmxyNXRzUWR5dGxPcEhTd05yQjBSQnNNTU1XeFNPQ0JJWlltUlVVZ200cGd2Uk9rRWl2OG9VOThQMkE4SnFTKwprWHBFRjVCNi84K2pXRmM0Z1Q4SWhlMEZtR0VJQllvelhYL08wejBsV0h4WXg2MHluWUoycU9vS1FKT3A5YjlsCmpiUVBkaC9mN2ErRWF0RzZNUFlrNG5xSEY3a0FzcmNsRXo2SGUvaEx6NmRkSTJ1N2RMRjB6QlN0QjM5WDFRZysKZ1JzTittOXg1S1FVTXYxMktvajdLc2hEelozOG5hSjd5bDgycGhBV1lGZzBOZHlzRlBRbmt0WmlNSUxOblFjNwpOeUt0cHNQaUxIRE9ha05hdEZLU2lOaUJrUk1lY1ZUMlJNMzMzUG54bFVFQ2dZRUEvYTY5MEEralU4VFJNbVZyCk4vRnlYWkxYa1c5b2NxVjBRbTA0TDMrSExybFNCTlRWSzk2U1pVT203VjViTzIxNmd4S2dJK3IwYm5kdE5GTUQKLzFncDhsdlJNcUlIeGZTeUo4SHpsSzViT0lnaUpxRGhzK3BKWTZmLytIVzZ1QkZyN3NGS3lxbVlIQlA0SC9BdApsT3lLeEVjMHFXazFlT2tCMWNNSGx0WDRwemtDZ1lFQTJncDhDVDVYWjNMSWRQN2M1SHpDS1YwczBYS1hGNmYyCkxzclhPVlZaTmJCN1NIS1NsOTBIU2VWVGx3czdqSnNxcC9yWFY2aHF0eUdEaTg4aTFZekthcEF6dXl3b0U3TnEKMUJpd2ZYSURQeTlPNUdGNXFYNXFUeENzSWNIcmo2Z21XMEZVQWhoS1lQcDRxd1JMdzFMZkJsd3U1VmhuN3I3ego0SkZBTEFpdlp4a0NnWUJicnpuKzVvZjdFSmtqQTdDYWlYTHlDczVLUzkrTi8rcGl6NktNMkNSOWFKRVNHZkhwClp3bTErNXRyRXIwYVgxajE0bGRxWTlKdjBrM3ZxVWs2a2h5bThUUk1mbThjeG5GVkdTMzF3SVpMaWpmOWlndkkKd0paQnBFaEkvaE83enVBWmJGYWhwR1hMVUJSUFJyalNxQ01IQ1UwcEpWTWtIZUtCNVhqcXRPNm5VUUtCZ0NJUAp6VHlzYm44TW9XQVZpSEJ4Uk91dFVKa1BxNmJZYUU3N0JSQkIwd1BlSkFRM1VjdERqaVh2RzFYWFBXQkR4VEFrCnNZdFNGZ214eEprTXJNWnJqaHVEbDNFLy9xckZOb1VYcmtxS2l4Tk4wcWMreXdDOWJPSVpHcXJUWG5jOHIzRkcKRFZlZWI5QWlrTU0ya3BkYTFOaHJnaS8xMVphb1lmVE0vQmRrNi9IUkFvR0JBSnFzTmFZYzE2clVzYzAzUEwybApXUGNzRnZxZGI3SEJyakVSRkhFdzQ0Vkt2MVlxK0ZWYnNNN1FTQVZ1V1llcGxGQUpDYzcrSEt1YjRsa1hRM1RkCndSajJLK2pOUzJtUXp1Y2hOQnlBZ1hXVnYveHhMZEE3NnpuWmJYdjl5cXhnTVVjTVZwZGRuSkxVZm9QVVZ1dTcKS0tlVVU3TTNIblRKUStrcldtbUxraUlSCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml b/examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml new file mode 100644 index 0000000000..832b2e454a --- /dev/null +++ b/examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-ns +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: external-backend + namespace: external-ns +spec: + replicas: 1 + selector: + matchLabels: + app: external-backend + template: + metadata: + labels: + app: external-backend + spec: + containers: + - name: external-backend + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: external-backend-svc + namespace: external-ns +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: external-backend \ No newline at end of file From 9ed8919e631c70de2c303d8c25c275a212f9dc54 Mon Sep 17 00:00:00 2001 From: Venktesh Date: Thu, 15 Sep 2022 17:04:36 +0100 Subject: [PATCH 04/19] Remove duplicate --- .../external-svc-dep.yaml | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 examples/custom-resources/externalname-services/external-svc-dep.yaml diff --git a/examples/custom-resources/externalname-services/external-svc-dep.yaml b/examples/custom-resources/externalname-services/external-svc-dep.yaml deleted file mode 100644 index 832b2e454a..0000000000 --- a/examples/custom-resources/externalname-services/external-svc-dep.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: external-ns ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-backend - namespace: external-ns -spec: - replicas: 1 - selector: - matchLabels: - app: external-backend - template: - metadata: - labels: - app: external-backend - spec: - containers: - - name: external-backend - image: nginxdemos/nginx-hello:plain-text - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: external-backend-svc - namespace: external-ns -spec: - ports: - - port: 80 - targetPort: 8080 - protocol: TCP - name: http - selector: - app: external-backend \ No newline at end of file From 07cccded60da755dbf1d7ee3eba798df25b1baac Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Fri, 16 Sep 2022 10:22:25 +0100 Subject: [PATCH 05/19] Update TS template --- internal/configs/version2/nginx-plus.transportserver.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index 10141b05a4..b2a42af30e 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -7,7 +7,7 @@ upstream {{ $u.Name }} { {{ end }} {{ range $s := $u.Servers }} - server {{ $s.Address }} max_fails={{ $s.MaxFails }} fail_timeout={{ $s.FailTimeout }} max_conns={{ $s.MaxConnections }}{{ if $u.Resolve }} resolve{{ end }};; + server {{ $s.Address }} max_fails={{ $s.MaxFails }} fail_timeout={{ $s.FailTimeout }} max_conns={{ $s.MaxConnections }}{{ if $u.Resolve }} resolve{{ end }}; {{ end }} } {{ end }} From 89ebc2b2e22e45c18270f76a56ee616a4ee1b417 Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Fri, 16 Sep 2022 12:57:18 +0100 Subject: [PATCH 06/19] Remove obsolete comments --- internal/configs/configurator.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index e72c29db61..d8636a3e25 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -1125,12 +1125,6 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende allWarnings.Add(warnings) } - // we don't need to regenerate config for TransportServers, because: - // (1) Changes to the ConfigMap don't affect TransportServer configs directly - // (2) addOrUpdateTransportServer doesn't return any warnings that we need to propagate to the caller. - // if (1) and (2) is no longer the case, we need to generate the config for TransportServers - - // Since we are adding support for resolving external names we need to generate TS config. for _, tsEx := range resources.TransportServerExes { warnings, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { From f0bf34c3037bbc7c0ea0598dcea7771a108c077a Mon Sep 17 00:00:00 2001 From: Venktesh Date: Fri, 16 Sep 2022 15:57:34 +0100 Subject: [PATCH 07/19] move resolver addresses outside tls passthough --- internal/configs/version1/nginx-plus.tmpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 9eda364219..18625ece02 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -290,17 +290,17 @@ stream { {{range $value := .StreamSnippets}} {{$value}}{{end}} + {{if .ResolverAddresses}} + resolver {{range $resolver := .ResolverAddresses}}{{$resolver}}{{end}}{{if .ResolverValid}} valid={{.ResolverValid}}{{end}}{{if not .ResolverIPV6}} ipv6=off{{end}}; + {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} + {{end}} + {{if .TLSPassthrough}} map $ssl_preread_server_name $dest_internal_passthrough { default unix:/var/lib/nginx/passthrough-https.sock; include /etc/nginx/tls-passthrough-hosts.conf; } - {{if .ResolverAddresses}} - resolver {{range $resolver := .ResolverAddresses}}{{$resolver}}{{end}}{{if .ResolverValid}} valid={{.ResolverValid}}{{end}}{{if not .ResolverIPV6}} ipv6=off{{end}}; - {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} - {{end}} - server { listen 443{{if .ProxyProtocol}} proxy_protocol{{end}}; listen [::]:443{{if .ProxyProtocol}} proxy_protocol{{end}}; From f4e3422655827f79e370e0fa02af62ab874c6954 Mon Sep 17 00:00:00 2001 From: Venktesh Date: Fri, 16 Sep 2022 17:11:56 +0100 Subject: [PATCH 08/19] Add automated tests --- .../external-svc-deployment.yaml | 64 ++++++++ .../externalname-svc.yaml | 7 + .../nginx-config.yaml | 6 + .../standard/global-configuration.yaml | 12 ++ .../standard/service_deployment.yaml | 61 ++++++++ .../standard/transport-server.yaml | 14 ++ .../test_transport_server_external_name.py | 139 ++++++++++++++++++ 7 files changed, 303 insertions(+) create mode 100644 tests/data/transport-server-externalname/external-svc-deployment.yaml create mode 100644 tests/data/transport-server-externalname/externalname-svc.yaml create mode 100644 tests/data/transport-server-externalname/nginx-config.yaml create mode 100644 tests/data/transport-server-externalname/standard/global-configuration.yaml create mode 100644 tests/data/transport-server-externalname/standard/service_deployment.yaml create mode 100644 tests/data/transport-server-externalname/standard/transport-server.yaml create mode 100644 tests/suite/test_transport_server_external_name.py diff --git a/tests/data/transport-server-externalname/external-svc-deployment.yaml b/tests/data/transport-server-externalname/external-svc-deployment.yaml new file mode 100644 index 0000000000..51413aeeee --- /dev/null +++ b/tests/data/transport-server-externalname/external-svc-deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-external-config +data: + Corefile: | + .:5353 { + forward . 8.8.8.8:53 + log + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns-external-backend +spec: + replicas: 1 + selector: + matchLabels: + app: coredns-external-backend + template: + metadata: + labels: + app: coredns-external-backend + spec: + containers: + - name: coredns-external-backend + image: coredns/coredns:1.8.6 + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 5353 + name: dns + protocol: UDP + - containerPort: 5353 + name: dns-tcp + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + volumes: + - name: config-volume + configMap: + name: coredns-external-config + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns-external-backend-svc +spec: + selector: + app: coredns-external-backend + ports: + - name: dns + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-externalname/externalname-svc.yaml b/tests/data/transport-server-externalname/externalname-svc.yaml new file mode 100644 index 0000000000..c78d8f6b0b --- /dev/null +++ b/tests/data/transport-server-externalname/externalname-svc.yaml @@ -0,0 +1,7 @@ +kind: Service +apiVersion: v1 +metadata: + name: externalname-service +spec: + type: ExternalName + externalName: core-dns-external-backend-svc.external-ns.svc.cluster.local \ No newline at end of file diff --git a/tests/data/transport-server-externalname/nginx-config.yaml b/tests/data/transport-server-externalname/nginx-config.yaml new file mode 100644 index 0000000000..5a68ee3f55 --- /dev/null +++ b/tests/data/transport-server-externalname/nginx-config.yaml @@ -0,0 +1,6 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config +data: + resolver-addresses: "kube-dns.kube-system.svc.cluster.local" \ No newline at end of file diff --git a/tests/data/transport-server-externalname/standard/global-configuration.yaml b/tests/data/transport-server-externalname/standard/global-configuration.yaml new file mode 100644 index 0000000000..63a5213668 --- /dev/null +++ b/tests/data/transport-server-externalname/standard/global-configuration.yaml @@ -0,0 +1,12 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: GlobalConfiguration +metadata: + name: nginx-configuration +spec: + listeners: + - name: dns-udp + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-externalname/standard/service_deployment.yaml b/tests/data/transport-server-externalname/standard/service_deployment.yaml new file mode 100644 index 0000000000..6a89847ba1 --- /dev/null +++ b/tests/data/transport-server-externalname/standard/service_deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns +data: + Corefile: | + .:5353 { + forward . 8.8.8.8:53 + log + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns +spec: + replicas: 2 + selector: + matchLabels: + app: coredns + template: + metadata: + labels: + app: coredns + spec: + containers: + - name: coredns + image: coredns/coredns:1.8.6 + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 5353 + name: dns-tcp + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns +spec: + selector: + app: coredns + ports: + - name: dns + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-externalname/standard/transport-server.yaml b/tests/data/transport-server-externalname/standard/transport-server.yaml new file mode 100644 index 0000000000..1aff825167 --- /dev/null +++ b/tests/data/transport-server-externalname/standard/transport-server.yaml @@ -0,0 +1,14 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: TransportServer +metadata: + name: transport-server +spec: + listener: + name: dns-tcp + protocol: TCP + upstreams: + - name: dns-app + service: externalname-service + port: 5353 + action: + pass: dns-app diff --git a/tests/suite/test_transport_server_external_name.py b/tests/suite/test_transport_server_external_name.py new file mode 100644 index 0000000000..eb3c85763a --- /dev/null +++ b/tests/suite/test_transport_server_external_name.py @@ -0,0 +1,139 @@ +import pytest +from settings import DEPLOYMENTS, TEST_DATA +from suite.custom_assertions import assert_event +from suite.resources_utils import ( + create_items_from_yaml, + create_namespace_with_name_from_yaml, + create_service_from_yaml, + delete_namespace, + delete_service, + get_events, + get_file_contents, + get_first_pod_name, + replace_configmap_from_yaml, + wait_before_test, +) + + +class ExternalNameSetup: + """Encapsulate ExternalName example details. + + Attributes: + ic_pod_name: + external_host: external service host + """ + + def __init__(self, ic_pod_name, external_svc, external_host): + self.ic_pod_name = ic_pod_name + self.external_svc = external_svc + self.external_host = external_host + + +@pytest.fixture(scope="class") +def ts_externalname_setup( + request, kube_apis, ingress_controller_prerequisites, transport_server_setup +) -> ExternalNameSetup: + print( + "------------------------- Deploy external namespace with backend, configmap and service -----------------------------------" + ) + external_app_src = f"{TEST_DATA}/transport-server-externalname/external-svc-deployment.yaml" + external_ns = create_namespace_with_name_from_yaml(kube_apis.v1, "external-ns", f"{TEST_DATA}/common/ns.yaml") + create_items_from_yaml(kube_apis, external_app_src, external_ns) + + print("------------------------- ExternalName service Setup -----------------------------------") + config_map_name = ingress_controller_prerequisites.config_map["metadata"]["name"] + replace_configmap_from_yaml( + kube_apis.v1, + config_map_name, + ingress_controller_prerequisites.namespace, + f"{TEST_DATA}/transport-server-externalname/nginx-config.yaml", + ) + external_svc_src = f"{TEST_DATA}/transport-server-externalname/externalname-svc.yaml" + external_host = f"core-dns-external-backend-svc.external-ns.svc.cluster.local" + + external_svc = create_service_from_yaml(kube_apis.v1, transport_server_setup.namespace, external_svc_src) + wait_before_test() + ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace) + + def fin(): + print("Clean up ExternalName Setup:") + delete_service(kube_apis.v1, external_svc, transport_server_setup.namespace) + delete_namespace(kube_apis.v1, external_ns) + replace_configmap_from_yaml( + kube_apis.v1, + config_map_name, + ingress_controller_prerequisites.namespace, + f"{DEPLOYMENTS}/common/nginx-config.yaml", + ) + + request.addfinalizer(fin) + + return ExternalNameSetup(ic_pod_name, external_svc, external_host) + + +@pytest.mark.test +@pytest.mark.parametrize( + "crd_ingress_controller, transport_server_setup", + [ + ( + { + "type": "complete", + "extra_args": [ + "-global-configuration=nginx-ingress/nginx-configuration", + "-enable-leader-election=false", + ], + }, + {"example": "transport-server-externalname", "app_type": "simple"}, + ) + ], + indirect=True, +) +class TestTransportServerStatus: + def test_template_config( + self, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + transport_server_setup, + ts_externalname_setup, + ): + wait_before_test() + + nginx_file_path = f"/etc/nginx/nginx.conf" + nginx_conf = get_file_contents( + kube_apis.v1, nginx_file_path, ts_externalname_setup.ic_pod_name, ingress_controller_prerequisites.namespace + ) + resolver_count = nginx_conf.count("resolver kube-dns.kube-system.svc.cluster.local;") + + ts_file_path = ( + f"/etc/nginx/stream-conf.d/ts_{transport_server_setup.namespace}_{transport_server_setup.name}.conf" + ) + ts_conf = get_file_contents( + kube_apis.v1, ts_file_path, ts_externalname_setup.ic_pod_name, ingress_controller_prerequisites.namespace + ) + + assert resolver_count == 2 # one for http and other for stream context + assert ( + f"server {ts_externalname_setup.external_host}:5353 max_fails=1 fail_timeout=10s max_conns=0 resolve;" + in ts_conf + ) + + def test_event_warning( + self, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + transport_server_setup, + ts_externalname_setup, + ): + text = f"{transport_server_setup.namespace}/{transport_server_setup.name}" + event_text = f"Configuration for {text} was added or updated with warning(s): Type ExternalName service {ts_externalname_setup.external_svc} in upstream dns-app will be ignored. To use ExternalName services, a resolver must be configured in the ConfigMap" + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + f"{DEPLOYMENTS}/common/nginx-config.yaml", + ) + wait_before_test(5) + events = get_events(kube_apis.v1, transport_server_setup.namespace) + assert_event(event_text, events) From 0de1dfb4b6a9467d16eab0ce680c64a333a6256c Mon Sep 17 00:00:00 2001 From: Venktesh Date: Fri, 16 Sep 2022 17:13:31 +0100 Subject: [PATCH 09/19] skip test for OSS --- tests/suite/test_transport_server_external_name.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/suite/test_transport_server_external_name.py b/tests/suite/test_transport_server_external_name.py index eb3c85763a..3154d5d20a 100644 --- a/tests/suite/test_transport_server_external_name.py +++ b/tests/suite/test_transport_server_external_name.py @@ -71,7 +71,8 @@ def fin(): return ExternalNameSetup(ic_pod_name, external_svc, external_host) -@pytest.mark.test +@pytest.mark.ts +@pytest.mark.skip_for_nginx_oss @pytest.mark.parametrize( "crd_ingress_controller, transport_server_setup", [ From 488ce0cf957e854b5c9bf61c7921dca9a1818722 Mon Sep 17 00:00:00 2001 From: Venktesh Date: Fri, 16 Sep 2022 17:18:46 +0100 Subject: [PATCH 10/19] run linter --- tests/suite/test_transport_server_external_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/suite/test_transport_server_external_name.py b/tests/suite/test_transport_server_external_name.py index 3154d5d20a..1b4a689917 100644 --- a/tests/suite/test_transport_server_external_name.py +++ b/tests/suite/test_transport_server_external_name.py @@ -113,7 +113,7 @@ def test_template_config( kube_apis.v1, ts_file_path, ts_externalname_setup.ic_pod_name, ingress_controller_prerequisites.namespace ) - assert resolver_count == 2 # one for http and other for stream context + assert resolver_count == 2 # one for http and other for stream context assert ( f"server {ts_externalname_setup.external_host}:5353 max_fails=1 fail_timeout=10s max_conns=0 resolve;" in ts_conf From c95564cfdae0fa33a8ab708beaa09e253f32265b Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Mon, 19 Sep 2022 16:37:55 +0100 Subject: [PATCH 11/19] Update TS docs for ExternalName and ConfigMaps --- docs/content/configuration/transportserver-resource.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/configuration/transportserver-resource.md b/docs/content/configuration/transportserver-resource.md index 8448aa4589..8ae95b8e4c 100644 --- a/docs/content/configuration/transportserver-resource.md +++ b/docs/content/configuration/transportserver-resource.md @@ -372,4 +372,4 @@ Note how the events section includes a Warning event with the Rejected reason. ## Customization via ConfigMap -The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets` and `stream-log-format`) do not affect TransportServer resources. +The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. From 5bb47ccfe6d9b617fcc55cd82b558208316d4ddb Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Wed, 21 Sep 2022 16:16:49 +0100 Subject: [PATCH 12/19] Add a readme for TS example --- .../transport-server/README.md | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 examples/custom-resources/externalname-services/transport-server/README.md diff --git a/examples/custom-resources/externalname-services/transport-server/README.md b/examples/custom-resources/externalname-services/transport-server/README.md new file mode 100644 index 0000000000..5232275148 --- /dev/null +++ b/examples/custom-resources/externalname-services/transport-server/README.md @@ -0,0 +1,92 @@ +# Support for Type ExternalName Services +The Ingress Controller supports routing requests to services of the type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). + +An ExternalName service is defined by an external DNS name that is resolved into the IP addresses, typically external to the cluster. This enables to use the Ingress Controller to route requests to the destinations outside of the cluster. + +**Note:** This feature is only available in NGINX Plus. + +# Prerequisites: + +For the illustration purpose we will run NGINX Ingress Controller with the ```- -watch-namespace=nginx-ingress,default``` option. The option enables NIC to watch selected namespaces. + +We will use the tls-passthrough application example as our backend app that will be responding to requests. + +## Steps + +- Deploy the backend application as described in the ```examples/custom-resources/tls-passthrough```, and make sure it is working as described. + +- Navigate to the external-name example ```examples/custom-resources/externalname-services``` + +- Deploy backend application to the ```external-ns```. Note that the namespace is not being watched by ```KIC``` + +```bash +kubectl apply -f transport-server/secure-app-external.yaml +``` + +- Refer the newly created service in the external name svc ```secure-app-external-backend-svc``` in the spec section + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: externalname-service +spec: + type: ExternalName + externalName: secure-app-external-backend-svc.external-ns.svc.cluster.local +``` + +- Create the service + +```bash +kubectl apply -f externalname-svc.yaml +``` + +- Update config map ```nginx-config.yaml``` with the resolver address. + +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + resolver-addresses: "kube-dns.kube-system.svc.cluster.local" +``` + +- Apply the change + +```bash +kubectl apply -f nginx-config.yaml +``` + +- Add the ```externalname-service``` to the TransportServer deployed in the tls-passthrough example + +```yaml +apiVersion: k8s.nginx.org/v1alpha1 +kind: TransportServer +metadata: + name: secure-app +spec: + listener: + name: tls-passthrough + protocol: TLS_PASSTHROUGH + host: app.example.com + upstreams: + - name: secure-app + service: externalname-service + port: 8443 + action: + pass: secure-app +``` + +- Apply the change + +```bash +kubectl apply -f transport-server-passthrough.yaml +``` + +- Send a request to verify the response is comming from the "external" pod (refer to to the tls-passthrough example) + +```bash +curl --resolve app.example.com:$IC_HTTPS_PORT:$IC_IP https://app.example.com:$IC_HTTPS_PORT --insecure +``` From 21e980e5a83f53a34de95d46e900bebe787166bb Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Thu, 22 Sep 2022 12:41:46 +0100 Subject: [PATCH 13/19] Update example docs --- .../transport-server/README.md | 179 +++++++++--------- 1 file changed, 94 insertions(+), 85 deletions(-) diff --git a/examples/custom-resources/externalname-services/transport-server/README.md b/examples/custom-resources/externalname-services/transport-server/README.md index 5232275148..5b47e8aac3 100644 --- a/examples/custom-resources/externalname-services/transport-server/README.md +++ b/examples/custom-resources/externalname-services/transport-server/README.md @@ -1,92 +1,101 @@ # Support for Type ExternalName Services + The Ingress Controller supports routing requests to services of the type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). An ExternalName service is defined by an external DNS name that is resolved into the IP addresses, typically external to the cluster. This enables to use the Ingress Controller to route requests to the destinations outside of the cluster. **Note:** This feature is only available in NGINX Plus. -# Prerequisites: - -For the illustration purpose we will run NGINX Ingress Controller with the ```- -watch-namespace=nginx-ingress,default``` option. The option enables NIC to watch selected namespaces. - -We will use the tls-passthrough application example as our backend app that will be responding to requests. - -## Steps - -- Deploy the backend application as described in the ```examples/custom-resources/tls-passthrough```, and make sure it is working as described. - -- Navigate to the external-name example ```examples/custom-resources/externalname-services``` - -- Deploy backend application to the ```external-ns```. Note that the namespace is not being watched by ```KIC``` - -```bash -kubectl apply -f transport-server/secure-app-external.yaml -``` - -- Refer the newly created service in the external name svc ```secure-app-external-backend-svc``` in the spec section - -```yaml -kind: Service -apiVersion: v1 -metadata: - name: externalname-service -spec: - type: ExternalName - externalName: secure-app-external-backend-svc.external-ns.svc.cluster.local -``` - -- Create the service - -```bash -kubectl apply -f externalname-svc.yaml -``` - -- Update config map ```nginx-config.yaml``` with the resolver address. - -```yaml -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-config - namespace: nginx-ingress -data: - resolver-addresses: "kube-dns.kube-system.svc.cluster.local" -``` - -- Apply the change - -```bash -kubectl apply -f nginx-config.yaml -``` - -- Add the ```externalname-service``` to the TransportServer deployed in the tls-passthrough example - -```yaml -apiVersion: k8s.nginx.org/v1alpha1 -kind: TransportServer -metadata: - name: secure-app -spec: - listener: - name: tls-passthrough - protocol: TLS_PASSTHROUGH - host: app.example.com - upstreams: - - name: secure-app - service: externalname-service - port: 8443 - action: - pass: secure-app -``` - -- Apply the change - -```bash -kubectl apply -f transport-server-passthrough.yaml -``` - -- Send a request to verify the response is comming from the "external" pod (refer to to the tls-passthrough example) - -```bash -curl --resolve app.example.com:$IC_HTTPS_PORT:$IC_IP https://app.example.com:$IC_HTTPS_PORT --insecure -``` +# Prerequisites + + +For the illustration purpose we will run NGINX Ingress Controller (refered as NIC in the examples) with the ```-watch-namespace=nginx-ingress,default``` option. The option enables NIC to watch selected namespaces. + +Any application deployed in other namespaces will be treated as an external service. + +We will use the [tls-passthrough](../../tls-passthrough/README.md) application example as our backend app that will be responding to requests. + +# Example + +## 1. Deploy the tls-passthrough application + +1. Deploy the backend application as described in the [tls-passthrough example](../../tls-passthrough/README.md), and make sure it is working as described. + +## 2. Deploy external service to external namespace + +1. Navigate to the [external-name example](../../../custom-resources/externalname-services/README.md) + +2. Deploy external namespace (```external-ns```) and the backend application. Note that the namespace is not being watched by ```NIC``` + ``` + $ kubectl apply -f transport-server/secure-app-external.yaml + ``` + +## 3. Setup ExternalName service + +1. Refer the newly created service in the file [externalname-svc.yaml](../../../custom-resources/externalname-services/externalname-svc.yaml) in the spec section + ```yaml + kind: Service + apiVersion: v1 + metadata: + name: externalname-service + spec: + type: ExternalName + externalName: secure-app-external-backend-svc.external-ns.svc.cluster.local + ``` + +2. Create the service of type ```ExternalName``` + ``` + $ kubectl apply -f externalname-svc.yaml + ``` + +3. Update config map [nginx-config.yaml](../../../custom-resources/externalname-services/nginx-config.yaml) with the resolver address + ```yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: nginx-config + namespace: nginx-ingress + data: + resolver-addresses: "kube-dns.kube-system.svc.cluster.local" + ``` + +4. Apply the change + ```bash + $ kubectl apply -f nginx-config.yaml + ``` + +## 4. Change the TS to point to the ExternalName and verify if it is working correctly + +1. Navigate to the example [tls-passthrough](../../../custom-resources/tls-passthrough/README.md) and open the ```transport-server.yaml``` file. + +2. Replace the service name ```secure-app``` with ```externalname-service``` and apply the change. + ```yaml + apiVersion: k8s.nginx.org/v1alpha1 + kind: TransportServer + metadata: + name: secure-app + spec: + listener: + name: tls-passthrough + protocol: TLS_PASSTHROUGH + host: app.example.com + upstreams: + - name: secure-app + service: externalname-service + port: 8443 + action: + pass: secure-app + ``` + + ``` + $ kubectl apply -f transport-server-passthrough.yaml + ``` + +3. Verify if the application is working by sending a request and check if the response is coming from the "external backend pod" (refer to to the tls-passthrough example) + ```bash + $ curl --resolve app.example.com:$IC_HTTPS_PORT:$IC_IP https://app.example.com:$IC_HTTPS_PORT --insecure + ``` + Response + ``` + hello from pod secure-app-external-backend-5fbf4fb494-x7bkl + ``` From 41be7fcee6a7a201f96c6463dfe01a4a3c596dfd Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Thu, 22 Sep 2022 14:32:43 +0100 Subject: [PATCH 14/19] Increase test coverage --- internal/configs/transportserver_test.go | 168 +++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index c5e0ffcb9c..a0ea2f328e 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -602,6 +602,174 @@ func TestGenerateTransportServerConfig_ProducesValidConfigOnValidInputForExterna } } +func TestGenerateTransportServerConfig_GeneratesWarningOnNotConfiguredResolver(t *testing.T) { + t.Parallel() + transportServerEx := TransportServerEx{ + TransportServer: &conf_v1alpha1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "tcp-server", + Namespace: "default", + }, + Spec: conf_v1alpha1.TransportServerSpec{ + Listener: conf_v1alpha1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + Upstreams: []conf_v1alpha1.Upstream{ + { + Name: "tcp-app", + Service: "tcp-app-svc", + Port: 5001, + MaxFails: intPointer(3), + FailTimeout: "40s", + }, + }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1alpha1.SessionParameters{ + Timeout: "50s", + }, + Action: &conf_v1alpha1.Action{ + Pass: "tcp-app", + }, + }, + }, + Endpoints: map[string][]string{ + "default/tcp-app-svc:5001": { + "10.0.0.20:5001", + }, + }, + ExternalNameSvcs: map[string]bool{"default/tcp-app-svc": true}, + } + expected := &version2.TransportServerConfig{ + Upstreams: []version2.StreamUpstream{ + { + Name: "ts_default_tcp-server_tcp-app", + Servers: nil, + UpstreamLabels: version2.UpstreamLabels{ + ResourceName: "tcp-server", + ResourceType: "transportserver", + ResourceNamespace: "default", + Service: "tcp-app-svc", + }, + LoadBalancingMethod: "random two least_conn", + Resolve: true, + }, + }, + Server: version2.StreamServer{ + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0s", + ProxyTimeout: "50s", + HealthCheck: nil, + ServerSnippets: []string{}, + }, + StreamSnippets: []string{}, + } + + result, warnings := generateTransportServerConfig(&transportServerEx, 2020, true, false) + if len(warnings) == 0 { + t.Errorf("want warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Error(cmp.Diff(expected, result)) + } +} + +func TestGenerateTransportServerConfig_UsesNotExistignSocketOnNotPlusAndNoEndpoints(t *testing.T) { + t.Parallel() + transportServerEx := TransportServerEx{ + TransportServer: &conf_v1alpha1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "tcp-server", + Namespace: "default", + }, + Spec: conf_v1alpha1.TransportServerSpec{ + Listener: conf_v1alpha1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + Upstreams: []conf_v1alpha1.Upstream{ + { + Name: "tcp-app", + Service: "tcp-app-svc", + Port: 5001, + MaxFails: intPointer(3), + FailTimeout: "40s", + }, + }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1alpha1.SessionParameters{ + Timeout: "50s", + }, + Action: &conf_v1alpha1.Action{ + Pass: "tcp-app", + }, + }, + }, + Endpoints: map[string][]string{}, + ExternalNameSvcs: map[string]bool{"default/tcp-app-svc": true}, + } + expected := &version2.TransportServerConfig{ + Upstreams: []version2.StreamUpstream{ + { + Name: "ts_default_tcp-server_tcp-app", + Servers: []version2.StreamUpstreamServer{ + { + Address: nginxNonExistingUnixSocket, + MaxFails: 3, + FailTimeout: "40s", + }, + }, + UpstreamLabels: version2.UpstreamLabels{ + ResourceName: "tcp-server", + ResourceType: "transportserver", + ResourceNamespace: "default", + Service: "tcp-app-svc", + }, + LoadBalancingMethod: "random two least_conn", + Resolve: true, + }, + }, + Server: version2.StreamServer{ + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0s", + ProxyTimeout: "50s", + HealthCheck: nil, + ServerSnippets: []string{}, + }, + StreamSnippets: []string{}, + } + + result, warnings := generateTransportServerConfig(&transportServerEx, 2020, false, true) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Error(cmp.Diff(expected, result)) + } +} + func TestGenerateUnixSocket(t *testing.T) { t.Parallel() transportServerEx := &TransportServerEx{ From 8a3ab167cad50d7cde02a8b3b12a223ea71e910b Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Thu, 22 Sep 2022 17:40:09 +0100 Subject: [PATCH 15/19] Update README for TS ExternalName --- .../externalname-services/transport-server/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/custom-resources/externalname-services/transport-server/README.md b/examples/custom-resources/externalname-services/transport-server/README.md index 5b47e8aac3..b3b294de21 100644 --- a/examples/custom-resources/externalname-services/transport-server/README.md +++ b/examples/custom-resources/externalname-services/transport-server/README.md @@ -13,17 +13,17 @@ For the illustration purpose we will run NGINX Ingress Controller (refered as NI Any application deployed in other namespaces will be treated as an external service. -We will use the [tls-passthrough](../../tls-passthrough/README.md) application example as our backend app that will be responding to requests. +We will use the ```examples/custom-resources/tls-passthrough``` application example as our backend app that will be responding to requests. # Example ## 1. Deploy the tls-passthrough application -1. Deploy the backend application as described in the [tls-passthrough example](../../tls-passthrough/README.md), and make sure it is working as described. +1. Deploy the backend application as described in the ```examples/custom-resources/tls-passthrough``` example, and make sure it is working as described. ## 2. Deploy external service to external namespace -1. Navigate to the [external-name example](../../../custom-resources/externalname-services/README.md) +1. Navigate to the external-name example ```examples/custom-resources/externalname-services/transport-server``` 2. Deploy external namespace (```external-ns```) and the backend application. Note that the namespace is not being watched by ```NIC``` ``` @@ -32,7 +32,7 @@ We will use the [tls-passthrough](../../tls-passthrough/README.md) application e ## 3. Setup ExternalName service -1. Refer the newly created service in the file [externalname-svc.yaml](../../../custom-resources/externalname-services/externalname-svc.yaml) in the spec section +1. Refer the newly created service in the file ```examples/custom-resources/externalname-services/externalname-svc.yaml``` in the spec section ```yaml kind: Service apiVersion: v1 @@ -48,7 +48,7 @@ We will use the [tls-passthrough](../../tls-passthrough/README.md) application e $ kubectl apply -f externalname-svc.yaml ``` -3. Update config map [nginx-config.yaml](../../../custom-resources/externalname-services/nginx-config.yaml) with the resolver address +3. Update config map ```examples/custom-resources/externalname-services/nginx-config.yaml``` with the resolver address ```yaml kind: ConfigMap apiVersion: v1 @@ -66,7 +66,7 @@ We will use the [tls-passthrough](../../tls-passthrough/README.md) application e ## 4. Change the TS to point to the ExternalName and verify if it is working correctly -1. Navigate to the example [tls-passthrough](../../../custom-resources/tls-passthrough/README.md) and open the ```transport-server.yaml``` file. +1. Navigate to the tls-passthrough example ```examples/custom-resources/tls-passthrough``` and open the ```transport-server-passthrough.yaml``` file. 2. Replace the service name ```secure-app``` with ```externalname-service``` and apply the change. ```yaml From e49d503f2c8f2a36f1734d586330273a8c6fcd52 Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Fri, 23 Sep 2022 11:33:11 +0100 Subject: [PATCH 16/19] Move files for TransportServer example to one folder --- .../transport-server/README.md | 39 ++++--------------- .../externalname-svc.yaml | 0 .../{ => transport-server}/nginx-config.yaml | 0 3 files changed, 7 insertions(+), 32 deletions(-) rename examples/custom-resources/externalname-services/{ => transport-server}/externalname-svc.yaml (100%) rename examples/custom-resources/externalname-services/{ => transport-server}/nginx-config.yaml (100%) diff --git a/examples/custom-resources/externalname-services/transport-server/README.md b/examples/custom-resources/externalname-services/transport-server/README.md index b3b294de21..7446f325d5 100644 --- a/examples/custom-resources/externalname-services/transport-server/README.md +++ b/examples/custom-resources/externalname-services/transport-server/README.md @@ -1,4 +1,4 @@ -# Support for Type ExternalName Services +# Support for Type ExternalName Services in Transport Server The Ingress Controller supports routing requests to services of the type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). @@ -8,7 +8,6 @@ An ExternalName service is defined by an external DNS name that is resolved into # Prerequisites - For the illustration purpose we will run NGINX Ingress Controller (refered as NIC in the examples) with the ```-watch-namespace=nginx-ingress,default``` option. The option enables NIC to watch selected namespaces. Any application deployed in other namespaces will be treated as an external service. @@ -23,48 +22,24 @@ We will use the ```examples/custom-resources/tls-passthrough``` application exam ## 2. Deploy external service to external namespace -1. Navigate to the external-name example ```examples/custom-resources/externalname-services/transport-server``` - -2. Deploy external namespace (```external-ns```) and the backend application. Note that the namespace is not being watched by ```NIC``` - ``` - $ kubectl apply -f transport-server/secure-app-external.yaml +1. Deploy backend application to external namespace (```external-ns```). Note that the namespace is not being watched by ```NIC```. + ```bash + $ kubectl apply -f secure-app-external.yaml ``` ## 3. Setup ExternalName service -1. Refer the newly created service in the file ```examples/custom-resources/externalname-services/externalname-svc.yaml``` in the spec section - ```yaml - kind: Service - apiVersion: v1 - metadata: - name: externalname-service - spec: - type: ExternalName - externalName: secure-app-external-backend-svc.external-ns.svc.cluster.local - ``` - -2. Create the service of type ```ExternalName``` +1. Create the service of type ```ExternalName``` ``` $ kubectl apply -f externalname-svc.yaml ``` -3. Update config map ```examples/custom-resources/externalname-services/nginx-config.yaml``` with the resolver address - ```yaml - kind: ConfigMap - apiVersion: v1 - metadata: - name: nginx-config - namespace: nginx-ingress - data: - resolver-addresses: "kube-dns.kube-system.svc.cluster.local" - ``` - -4. Apply the change +2. Apply the config map ```bash $ kubectl apply -f nginx-config.yaml ``` -## 4. Change the TS to point to the ExternalName and verify if it is working correctly +## 4. Change the Transport Server to point to the ExternalName and verify if it is working correctly 1. Navigate to the tls-passthrough example ```examples/custom-resources/tls-passthrough``` and open the ```transport-server-passthrough.yaml``` file. diff --git a/examples/custom-resources/externalname-services/externalname-svc.yaml b/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml similarity index 100% rename from examples/custom-resources/externalname-services/externalname-svc.yaml rename to examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml diff --git a/examples/custom-resources/externalname-services/nginx-config.yaml b/examples/custom-resources/externalname-services/transport-server/nginx-config.yaml similarity index 100% rename from examples/custom-resources/externalname-services/nginx-config.yaml rename to examples/custom-resources/externalname-services/transport-server/nginx-config.yaml From 4bc26c92d176705bb0427de76dae4db9d12be778 Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Fri, 23 Sep 2022 21:03:52 +0100 Subject: [PATCH 17/19] Fix error handling, update docs --- .../virtual-server/external-svc-dep.yaml | 39 ------------------- internal/configs/configurator.go | 22 ++++++----- internal/configs/transportserver.go | 2 +- internal/configs/transportserver_test.go | 9 +++-- 4 files changed, 20 insertions(+), 52 deletions(-) delete mode 100644 examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml diff --git a/examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml b/examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml deleted file mode 100644 index 832b2e454a..0000000000 --- a/examples/custom-resources/externalname-services/virtual-server/external-svc-dep.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: external-ns ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: external-backend - namespace: external-ns -spec: - replicas: 1 - selector: - matchLabels: - app: external-backend - template: - metadata: - labels: - app: external-backend - spec: - containers: - - name: external-backend - image: nginxdemos/nginx-hello:plain-text - ports: - - containerPort: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: external-backend-svc - namespace: external-ns -spec: - ports: - - port: 80 - targetPort: 8080 - protocol: TCP - name: http - selector: - app: external-backend \ No newline at end of file diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index d8636a3e25..93514286f6 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -576,10 +576,10 @@ func (cnf *Configurator) deleteTransportServerMetricsLabels(key string) { func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) if err != nil { - return warnings, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) + return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return warnings, fmt.Errorf("error reloading NGINX for TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) + return nil, fmt.Errorf("error reloading NGINX for TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } return warnings, nil } @@ -590,7 +590,7 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(tsCfg) if err != nil { - return warnings, fmt.Errorf("error generating TransportServer config %v: %w", name, err) + return nil, fmt.Errorf("error generating TransportServer config %v: %w", name, err) } if cnf.isPlus && cnf.isPrometheusEnabled { cnf.updateTransportServerMetricsLabels(transportServerEx, tsCfg.Upstreams) @@ -606,7 +606,11 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport Host: transportServerEx.TransportServer.Spec.Host, UnixSocket: generateUnixSocket(transportServerEx), } - return warnings, cnf.updateTLSPassthroughHostsConfig() + err := cnf.updateTLSPassthroughHostsConfig() + if err != nil { + return nil, err + } + return warnings, nil } return warnings, nil } @@ -669,7 +673,7 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn for _, ingEx := range resources.IngressExes { warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) + return nil, fmt.Errorf("Error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } allWarnings.Add(warnings) } @@ -677,7 +681,7 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn for _, m := range resources.MergeableIngresses { warnings, err := cnf.addOrUpdateMergeableIngress(m) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) + return nil, fmt.Errorf("Error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) } allWarnings.Add(warnings) } @@ -685,7 +689,7 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn for _, vsEx := range resources.VirtualServerExes { warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err) + return nil, fmt.Errorf("Error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err) } allWarnings.Add(warnings) } @@ -693,13 +697,13 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn for _, tsEx := range resources.TransportServerExes { warnings, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { - return allWarnings, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) + return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } allWarnings.Add(warnings) } if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return allWarnings, fmt.Errorf("Error when reloading NGINX when updating resources: %w", err) + return nil, fmt.Errorf("Error when reloading NGINX when updating resources: %w", err) } return allWarnings, nil } diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index 7c6873a88a..5d12f5f8b8 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -17,7 +17,7 @@ type TransportServerEx struct { Endpoints map[string][]string PodsByIP map[string]string ExternalNameSvcs map[string]bool - DisableIPV6 bool + DisableIPV6 bool } func (tsEx *TransportServerEx) String() string { diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index a6ce87383f..ac389357d6 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -222,9 +222,12 @@ func TestGenerateTransportServerConfigForIPV6Disabled(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfigForIPV6Disabled() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfigForIPV6Disabled() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } From 0186b91bddf02cd4414ca79501b7b767cfbe227a Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Fri, 23 Sep 2022 21:09:41 +0100 Subject: [PATCH 18/19] Make linter happy again --- internal/k8s/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 0d3cfd8e10..ba98a80b0d 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -3204,7 +3204,7 @@ func (lbc *LoadBalancerController) createTransportServerEx(transportServer *conf Endpoints: endpoints, PodsByIP: podsByIP, ExternalNameSvcs: externalNameSvcs, - DisableIPV6: disableIPV6, + DisableIPV6: disableIPV6, } } From b3736b01fa5c00c083ff65fd903e933c1bd1ccd7 Mon Sep 17 00:00:00 2001 From: Jakub Jarosz Date: Mon, 26 Sep 2022 13:51:46 +0100 Subject: [PATCH 19/19] Update TS ExternalName example --- .../transport-server/externalname-svc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml b/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml index 5901eaeba7..06905441bf 100644 --- a/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml +++ b/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml @@ -4,4 +4,4 @@ metadata: name: externalname-service spec: type: ExternalName - externalName: external-backend-svc.external-ns.svc.cluster.local \ No newline at end of file + externalName: secure-app-external-backend-svc.external-ns.svc.cluster.local \ No newline at end of file