diff --git a/docs/content/configuration/transportserver-resource.md b/docs/content/configuration/transportserver-resource.md index 27277150c1..64d7539b4b 100644 --- a/docs/content/configuration/transportserver-resource.md +++ b/docs/content/configuration/transportserver-resource.md @@ -372,4 +372,4 @@ Note how the events section includes a Warning event with the Rejected reason. ## Customization via ConfigMap -The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets` and `stream-log-format`) do not affect TransportServer resources. +The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. diff --git a/examples/custom-resources/externalname-services/transport-server/README.md b/examples/custom-resources/externalname-services/transport-server/README.md new file mode 100644 index 0000000000..7446f325d5 --- /dev/null +++ b/examples/custom-resources/externalname-services/transport-server/README.md @@ -0,0 +1,76 @@ +# Support for Type ExternalName Services in Transport Server + +The Ingress Controller supports routing requests to services of the type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). + +An ExternalName service is defined by an external DNS name that is resolved into the IP addresses, typically external to the cluster. This enables to use the Ingress Controller to route requests to the destinations outside of the cluster. + +**Note:** This feature is only available in NGINX Plus. + +# Prerequisites + +For the illustration purpose we will run NGINX Ingress Controller (refered as NIC in the examples) with the ```-watch-namespace=nginx-ingress,default``` option. The option enables NIC to watch selected namespaces. + +Any application deployed in other namespaces will be treated as an external service. + +We will use the ```examples/custom-resources/tls-passthrough``` application example as our backend app that will be responding to requests. + +# Example + +## 1. Deploy the tls-passthrough application + +1. Deploy the backend application as described in the ```examples/custom-resources/tls-passthrough``` example, and make sure it is working as described. + +## 2. Deploy external service to external namespace + +1. Deploy backend application to external namespace (```external-ns```). Note that the namespace is not being watched by ```NIC```. + ```bash + $ kubectl apply -f secure-app-external.yaml + ``` + +## 3. Setup ExternalName service + +1. Create the service of type ```ExternalName``` + ``` + $ kubectl apply -f externalname-svc.yaml + ``` + +2. Apply the config map + ```bash + $ kubectl apply -f nginx-config.yaml + ``` + +## 4. Change the Transport Server to point to the ExternalName and verify if it is working correctly + +1. Navigate to the tls-passthrough example ```examples/custom-resources/tls-passthrough``` and open the ```transport-server-passthrough.yaml``` file. + +2. Replace the service name ```secure-app``` with ```externalname-service``` and apply the change. + ```yaml + apiVersion: k8s.nginx.org/v1alpha1 + kind: TransportServer + metadata: + name: secure-app + spec: + listener: + name: tls-passthrough + protocol: TLS_PASSTHROUGH + host: app.example.com + upstreams: + - name: secure-app + service: externalname-service + port: 8443 + action: + pass: secure-app + ``` + + ``` + $ kubectl apply -f transport-server-passthrough.yaml + ``` + +3. Verify if the application is working by sending a request and check if the response is coming from the "external backend pod" (refer to to the tls-passthrough example) + ```bash + $ curl --resolve app.example.com:$IC_HTTPS_PORT:$IC_IP https://app.example.com:$IC_HTTPS_PORT --insecure + ``` + Response + ``` + hello from pod secure-app-external-backend-5fbf4fb494-x7bkl + ``` diff --git a/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml b/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml new file mode 100644 index 0000000000..06905441bf --- /dev/null +++ b/examples/custom-resources/externalname-services/transport-server/externalname-svc.yaml @@ -0,0 +1,7 @@ +kind: Service +apiVersion: v1 +metadata: + name: externalname-service +spec: + type: ExternalName + externalName: secure-app-external-backend-svc.external-ns.svc.cluster.local \ No newline at end of file diff --git a/examples/custom-resources/externalname-services/transport-server/nginx-config.yaml b/examples/custom-resources/externalname-services/transport-server/nginx-config.yaml new file mode 100644 index 0000000000..b272bb4932 --- /dev/null +++ b/examples/custom-resources/externalname-services/transport-server/nginx-config.yaml @@ -0,0 +1,7 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + resolver-addresses: "kube-dns.kube-system.svc.cluster.local" \ No newline at end of file diff --git a/examples/custom-resources/externalname-services/transport-server/secure-app-external.yaml b/examples/custom-resources/externalname-services/transport-server/secure-app-external.yaml new file mode 100644 index 0000000000..20f72f6b01 --- /dev/null +++ b/examples/custom-resources/externalname-services/transport-server/secure-app-external.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: external-ns +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: secure-app-external-backend + namespace: external-ns +spec: + replicas: 1 + selector: + matchLabels: + app: secure-app-external-backend + template: + metadata: + labels: + app: secure-app-external-backend + spec: + containers: + - name: secure-app-external-backend + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8443 + volumeMounts: + - name: secret + mountPath: /etc/nginx/ssl + readOnly: true + - name: config-volume + mountPath: /etc/nginx/conf.d + volumes: + - name: secret + secret: + secretName: app-tls-secret + - name: config-volume + configMap: + name: secure-config +--- +apiVersion: v1 +kind: Service +metadata: + name: secure-app-external-backend-svc + namespace: external-ns +spec: + ports: + - port: 8443 + targetPort: 8443 + protocol: TCP + name: https + selector: + app: secure-app-external-backend +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: secure-config + namespace: external-ns +data: + app.conf: |- + server { + listen 8443 ssl; + listen [::]:8443 ssl; + + server_name app.example.com; + + ssl_certificate /etc/nginx/ssl/tls.crt; + ssl_certificate_key /etc/nginx/ssl/tls.key; + + default_type text/plain; + + location / { + return 200 "hello from pod $hostname\n"; + } + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: app-tls-secret + namespace: external-ns +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ3Q0NRQ3EzQWxhdnJiaWpqQU5CZ2txaGtpRzl3MEJBUXNGQURCTU1Rc3dDUVlEVlFRR0V3SlYKVXpFTE1Ba0dBMVVFQ0F3Q1EwRXhGakFVQmdOVkJBY01EVk5oYmlCR2NtRnVZMmx6WTI4eEdEQVdCZ05WQkFNTQpEMkZ3Y0M1bGVHRnRjR3hsTG1OdmJUQWVGdzB5TURBek1qTXlNekl3TkROYUZ3MHlNekF6TWpNeU16SXdORE5hCk1Fd3hDekFKQmdOVkJBWVRBbFZUTVFzd0NRWURWUVFJREFKRFFURVdNQlFHQTFVRUJ3d05VMkZ1SUVaeVlXNWoKYVhOamJ6RVlNQllHQTFVRUF3d1BZWEJ3TG1WNFlXMXdiR1V1WTI5dE1JSUJJakFOQmdrcWhraUc5dzBCQVFFRgpBQU9DQVE4QU1JSUJDZ0tDQVFFQTJCRXhZR1JPRkhoN2VPMVlxeCtWRHMzRzMrVEhyTEZULzdEUFFEQlkza3pDCi9oZlprWCt3OW1NNkQ1RU9uK2lpVlNhUWlQMm1aNFA3N29pR0dmd3JrNjJ0eEQ5cHphODM5NC9aSjF5Q0dXZ1QKK2NWUEVZbkxjQktzSTRMcktJZ21oWVIwUjNzWWRjR1JkSXJWUFZlNUVUQlk1Z1U0RGhhMDZOUEIraitmK0krWgphWGIvMlRBekJhNHozMWpIQzg2amVQeTFMdklGazFiY3I2cSsxRGR5eklxcWxkRDYvU3Q4Q2t3cDlOaDFCUGFhCktZZ1ZVd010UVBib2s1cFFmbVMrdDg4NHdSM0dTTEU4VkxRbzgyYnJhNUR3emhIamlzOTlJRGhzbUt0U3lWOXMKaWNJbXp5dHBnSXlhTS9zWEhRQU9KbVFJblFteWgyekd1WFhTQ0lkRGtRSURBUUFCTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0tsVkhOZ1k5VHZLaW9Xb0tvdllCdnNRMmYrcmFOOEJwdWNDcnRvRm15NUczcGIzU2lPTndaCkF2cnhtSm4vR3lsa3JKTHBpQVA1eUNBNGI2Y2lYMnRGa3pQRmhJVFZKRTVBeDlpaEF2WWZwTUFSdWVqM29HN2UKd0xwQk1iUnlGbHJYV29NWUVBMGxsV0JueHRQQXZYS2Y4SVZGYTRSSDhzV1JJSDB4M2hFdjVtQ3VUZjJTRTg0QwpiNnNjS3Z3MW9CQU5VWGxXRVZVYTFmei9rWWZBa1lrdHZyV2JUcTZTWGxodXRJYWY4WEYzSUMrL2x1b3gzZThMCjBBcEFQVE5sZ0JwOTkvcXMrOG9PMWthSmQ1TmV6TnlJeXhSdUtJMzlDWkxuQm9OYmkzdlFYY1NzRCtYU2lYT0cKcEVnTjNtci8xRms4OVZMSENhTnkyKzBqMjZ0eWpiclcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2Z0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktnd2dnU2tBZ0VBQW9JQkFRRFlFVEZnWkU0VWVIdDQKN1Zpckg1VU96Y2JmNU1lc3NWUC9zTTlBTUZqZVRNTCtGOW1SZjdEMll6b1BrUTZmNktKVkpwQ0kvYVpuZy92dQppSVlaL0N1VHJhM0VQMm5OcnpmM2o5a25YSUlaYUJQNXhVOFJpY3R3RXF3amd1c29pQ2FGaEhSSGV4aDF3WkYwCml0VTlWN2tSTUZqbUJUZ09GclRvMDhINlA1LzRqNWxwZHYvWk1ETUZyalBmV01jTHpxTjQvTFV1OGdXVFZ0eXYKcXI3VU4zTE1pcXFWMFByOUszd0tUQ24wMkhVRTlwb3BpQlZUQXkxQTl1aVRtbEIrWkw2M3p6akJIY1pJc1R4VQp0Q2p6WnV0cmtQRE9FZU9LejMwZ09HeVlxMUxKWDJ5SndpYlBLMm1Bakpveit4Y2RBQTRtWkFpZENiS0hiTWE1CmRkSUloME9SQWdNQkFBRUNnZ0VCQUxYaW16ODZrT1A0bkhBcTFPYVEyb2l3dndhQTczbTNlUytZSm84eFk4NFcKcmxyNXRzUWR5dGxPcEhTd05yQjBSQnNNTU1XeFNPQ0JJWlltUlVVZ200cGd2Uk9rRWl2OG9VOThQMkE4SnFTKwprWHBFRjVCNi84K2pXRmM0Z1Q4SWhlMEZtR0VJQllvelhYL08wejBsV0h4WXg2MHluWUoycU9vS1FKT3A5YjlsCmpiUVBkaC9mN2ErRWF0RzZNUFlrNG5xSEY3a0FzcmNsRXo2SGUvaEx6NmRkSTJ1N2RMRjB6QlN0QjM5WDFRZysKZ1JzTittOXg1S1FVTXYxMktvajdLc2hEelozOG5hSjd5bDgycGhBV1lGZzBOZHlzRlBRbmt0WmlNSUxOblFjNwpOeUt0cHNQaUxIRE9ha05hdEZLU2lOaUJrUk1lY1ZUMlJNMzMzUG54bFVFQ2dZRUEvYTY5MEEralU4VFJNbVZyCk4vRnlYWkxYa1c5b2NxVjBRbTA0TDMrSExybFNCTlRWSzk2U1pVT203VjViTzIxNmd4S2dJK3IwYm5kdE5GTUQKLzFncDhsdlJNcUlIeGZTeUo4SHpsSzViT0lnaUpxRGhzK3BKWTZmLytIVzZ1QkZyN3NGS3lxbVlIQlA0SC9BdApsT3lLeEVjMHFXazFlT2tCMWNNSGx0WDRwemtDZ1lFQTJncDhDVDVYWjNMSWRQN2M1SHpDS1YwczBYS1hGNmYyCkxzclhPVlZaTmJCN1NIS1NsOTBIU2VWVGx3czdqSnNxcC9yWFY2aHF0eUdEaTg4aTFZekthcEF6dXl3b0U3TnEKMUJpd2ZYSURQeTlPNUdGNXFYNXFUeENzSWNIcmo2Z21XMEZVQWhoS1lQcDRxd1JMdzFMZkJsd3U1VmhuN3I3ego0SkZBTEFpdlp4a0NnWUJicnpuKzVvZjdFSmtqQTdDYWlYTHlDczVLUzkrTi8rcGl6NktNMkNSOWFKRVNHZkhwClp3bTErNXRyRXIwYVgxajE0bGRxWTlKdjBrM3ZxVWs2a2h5bThUUk1mbThjeG5GVkdTMzF3SVpMaWpmOWlndkkKd0paQnBFaEkvaE83enVBWmJGYWhwR1hMVUJSUFJyalNxQ01IQ1UwcEpWTWtIZUtCNVhqcXRPNm5VUUtCZ0NJUAp6VHlzYm44TW9XQVZpSEJ4Uk91dFVKa1BxNmJZYUU3N0JSQkIwd1BlSkFRM1VjdERqaVh2RzFYWFBXQkR4VEFrCnNZdFNGZ214eEprTXJNWnJqaHVEbDNFLy9xckZOb1VYcmtxS2l4Tk4wcWMreXdDOWJPSVpHcXJUWG5jOHIzRkcKRFZlZWI5QWlrTU0ya3BkYTFOaHJnaS8xMVphb1lmVE0vQmRrNi9IUkFvR0JBSnFzTmFZYzE2clVzYzAzUEwybApXUGNzRnZxZGI3SEJyakVSRkhFdzQ0Vkt2MVlxK0ZWYnNNN1FTQVZ1V1llcGxGQUpDYzcrSEt1YjRsa1hRM1RkCndSajJLK2pOUzJtUXp1Y2hOQnlBZ1hXVnYveHhMZEE3NnpuWmJYdjl5cXhnTVVjTVZwZGRuSkxVZm9QVVZ1dTcKS0tlVVU3TTNIblRKUStrcldtbUxraUlSCi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 8ce08941a1..93514286f6 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -573,29 +573,25 @@ func (cnf *Configurator) deleteTransportServerMetricsLabels(key string) { // AddOrUpdateTransportServer adds or updates NGINX configuration for the TransportServer resource. // It is a responsibility of the caller to check that the TransportServer references an existing listener. -func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) error { - err := cnf.addOrUpdateTransportServer(transportServerEx) +func (cnf *Configurator) AddOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { + warnings, err := cnf.addOrUpdateTransportServer(transportServerEx) if err != nil { - return fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) + return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } - if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return fmt.Errorf("Error reloading NGINX for TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) + return nil, fmt.Errorf("error reloading NGINX for TransportServer %v/%v: %w", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name, err) } - - return nil + return warnings, nil } -func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) error { +func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *TransportServerEx) (Warnings, error) { name := getFileNameForTransportServer(transportServerEx.TransportServer) - - tsCfg := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus) + tsCfg, warnings := generateTransportServerConfig(transportServerEx, transportServerEx.ListenerPort, cnf.isPlus, cnf.IsResolverConfigured()) content, err := cnf.templateExecutorV2.ExecuteTransportServerTemplate(tsCfg) if err != nil { - return fmt.Errorf("Error generating TransportServer config %v: %w", name, err) + return nil, fmt.Errorf("error generating TransportServer config %v: %w", name, err) } - if cnf.isPlus && cnf.isPrometheusEnabled { cnf.updateTransportServerMetricsLabels(transportServerEx, tsCfg.Upstreams) } @@ -610,11 +606,13 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport Host: transportServerEx.TransportServer.Spec.Host, UnixSocket: generateUnixSocket(transportServerEx), } - - return cnf.updateTLSPassthroughHostsConfig() + err := cnf.updateTLSPassthroughHostsConfig() + if err != nil { + return nil, err + } + return warnings, nil } - - return nil + return warnings, nil } // GetVirtualServerRoutesForVirtualServer returns the virtualServerRoutes that a virtualServer @@ -675,7 +673,7 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn for _, ingEx := range resources.IngressExes { warnings, err := cnf.addOrUpdateIngress(ingEx) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) + return nil, fmt.Errorf("Error adding or updating ingress %v/%v: %w", ingEx.Ingress.Namespace, ingEx.Ingress.Name, err) } allWarnings.Add(warnings) } @@ -683,7 +681,7 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn for _, m := range resources.MergeableIngresses { warnings, err := cnf.addOrUpdateMergeableIngress(m) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) + return nil, fmt.Errorf("Error adding or updating mergeableIngress %v/%v: %w", m.Master.Ingress.Namespace, m.Master.Ingress.Name, err) } allWarnings.Add(warnings) } @@ -691,22 +689,22 @@ func (cnf *Configurator) AddOrUpdateResources(resources ExtendedResources) (Warn for _, vsEx := range resources.VirtualServerExes { warnings, err := cnf.addOrUpdateVirtualServer(vsEx) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err) + return nil, fmt.Errorf("Error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err) } allWarnings.Add(warnings) } for _, tsEx := range resources.TransportServerExes { - err := cnf.addOrUpdateTransportServer(tsEx) + warnings, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { - return allWarnings, fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) + return nil, fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } + allWarnings.Add(warnings) } if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return allWarnings, fmt.Errorf("Error when reloading NGINX when updating resources: %w", err) + return nil, fmt.Errorf("Error when reloading NGINX when updating resources: %w", err) } - return allWarnings, nil } @@ -939,11 +937,11 @@ func (cnf *Configurator) UpdateEndpointsForTransportServers(transportServerExes reloadPlus := false for _, tsEx := range transportServerExes { - err := cnf.addOrUpdateTransportServer(tsEx) + // Ignore warnings here as no new warnings should appear when updating Endpoints for TransportServers + _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { - return fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) + return fmt.Errorf("error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } - if cnf.isPlus { err := cnf.updatePlusEndpointsForTransportServer(tsEx) if err != nil { @@ -957,11 +955,9 @@ func (cnf *Configurator) UpdateEndpointsForTransportServers(transportServerExes glog.V(3).Info("No need to reload nginx") return nil } - if err := cnf.reload(nginx.ReloadForEndpointsUpdate); err != nil { - return fmt.Errorf("Error reloading NGINX when updating endpoints: %w", err) + return fmt.Errorf("error reloading NGINX when updating endpoints: %w", err) } - return nil } @@ -1069,6 +1065,7 @@ func (cnf *Configurator) updateStreamServersInPlus(upstream string, servers []st } // UpdateConfig updates NGINX configuration parameters. +// //gocyclo:ignore func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources ExtendedResources) (Warnings, error) { cnf.cfgParams = cfgParams @@ -1077,7 +1074,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende if cnf.cfgParams.MainServerSSLDHParamFileContent != nil { fileName, err := cnf.nginxManager.CreateDHParam(*cnf.cfgParams.MainServerSSLDHParamFileContent) if err != nil { - return allWarnings, fmt.Errorf("Error when updating dhparams: %w", err) + return allWarnings, fmt.Errorf("error when updating dhparams: %w", err) } cfgParams.MainServerSSLDHParam = fileName } @@ -1085,28 +1082,28 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende if cfgParams.MainTemplate != nil { err := cnf.templateExecutor.UpdateMainTemplate(cfgParams.MainTemplate) if err != nil { - return allWarnings, fmt.Errorf("Error when parsing the main template: %w", err) + return allWarnings, fmt.Errorf("error when parsing the main template: %w", err) } } if cfgParams.IngressTemplate != nil { err := cnf.templateExecutor.UpdateIngressTemplate(cfgParams.IngressTemplate) if err != nil { - return allWarnings, fmt.Errorf("Error when parsing the ingress template: %w", err) + return allWarnings, fmt.Errorf("error when parsing the ingress template: %w", err) } } if cfgParams.VirtualServerTemplate != nil { err := cnf.templateExecutorV2.UpdateVirtualServerTemplate(cfgParams.VirtualServerTemplate) if err != nil { - return allWarnings, fmt.Errorf("Error when parsing the VirtualServer template: %w", err) + return allWarnings, fmt.Errorf("error when parsing the VirtualServer template: %w", err) } } mainCfg := GenerateNginxMainConfig(cnf.staticCfgParams, cfgParams) mainCfgContent, err := cnf.templateExecutor.ExecuteMainConfigTemplate(mainCfg) if err != nil { - return allWarnings, fmt.Errorf("Error when writing main Config") + return allWarnings, fmt.Errorf("error when writing main Config") } cnf.nginxManager.CreateMainConfig(mainCfgContent) @@ -1132,20 +1129,23 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende allWarnings.Add(warnings) } - // we don't need to regenerate config for TransportServers, because: - // (1) Changes to the ConfigMap don't affect TransportServer configs directly - // (2) addOrUpdateTransportServer doesn't return any warnings that we need to propagate to the caller. - // if (1) and (2) is no longer the case, we need to generate the config for TransportServers + for _, tsEx := range resources.TransportServerExes { + warnings, err := cnf.addOrUpdateTransportServer(tsEx) + if err != nil { + return allWarnings, err + } + allWarnings.Add(warnings) + } if mainCfg.OpenTracingLoadModule { if err := cnf.addOrUpdateOpenTracingTracerConfig(mainCfg.OpenTracingTracerConfig); err != nil { - return allWarnings, fmt.Errorf("Error when updating OpenTracing tracer config: %w", err) + return allWarnings, fmt.Errorf("error when updating OpenTracing tracer config: %w", err) } } cnf.nginxManager.SetOpenTracing(mainCfg.OpenTracingLoadModule) if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { - return allWarnings, fmt.Errorf("Error when updating config from ConfigMap: %w", err) + return allWarnings, fmt.Errorf("error when updating config from ConfigMap: %w", err) } return allWarnings, nil @@ -1154,7 +1154,7 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende // UpdateTransportServers updates TransportServers. func (cnf *Configurator) UpdateTransportServers(updatedTSExes []*TransportServerEx, deletedKeys []string) error { for _, tsEx := range updatedTSExes { - err := cnf.addOrUpdateTransportServer(tsEx) + _, err := cnf.addOrUpdateTransportServer(tsEx) if err != nil { return fmt.Errorf("Error adding or updating TransportServer %v/%v: %w", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name, err) } diff --git a/internal/configs/transportserver.go b/internal/configs/transportserver.go index 4ec31f00d5..5d12f5f8b8 100644 --- a/internal/configs/transportserver.go +++ b/internal/configs/transportserver.go @@ -12,11 +12,12 @@ const nginxNonExistingUnixSocket = "unix:/var/lib/nginx/non-existing-unix-socket // TransportServerEx holds a TransportServer along with the resources referenced by it. type TransportServerEx struct { - ListenerPort int - TransportServer *conf_v1alpha1.TransportServer - Endpoints map[string][]string - PodsByIP map[string]string - DisableIPV6 bool + ListenerPort int + TransportServer *conf_v1alpha1.TransportServer + Endpoints map[string][]string + PodsByIP map[string]string + ExternalNameSvcs map[string]bool + DisableIPV6 bool } func (tsEx *TransportServerEx) String() string { @@ -32,10 +33,10 @@ func (tsEx *TransportServerEx) String() string { } // generateTransportServerConfig generates a full configuration for a TransportServer. -func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool) *version2.TransportServerConfig { +func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool, isResolverConfigured bool) (*version2.TransportServerConfig, Warnings) { upstreamNamer := newUpstreamNamerForTransportServer(transportServerEx.TransportServer) - upstreams := generateStreamUpstreams(transportServerEx, upstreamNamer, isPlus) + upstreams, warnings := generateStreamUpstreams(transportServerEx, upstreamNamer, isPlus, isResolverConfigured) healthCheck, match := generateTransportServerHealthCheck(transportServerEx.TransportServer.Spec.Action.Pass, upstreamNamer.GetNameForUpstream(transportServerEx.TransportServer.Spec.Action.Pass), @@ -97,8 +98,7 @@ func generateTransportServerConfig(transportServerEx *TransportServerEx, listene Upstreams: upstreams, StreamSnippets: streamSnippets, } - - return tsConfig + return tsConfig, warnings } func generateUnixSocket(transportServerEx *TransportServerEx) string { @@ -109,17 +109,25 @@ func generateUnixSocket(transportServerEx *TransportServerEx) string { return "" } -func generateStreamUpstreams(transportServerEx *TransportServerEx, upstreamNamer *upstreamNamer, isPlus bool) []version2.StreamUpstream { +func generateStreamUpstreams(transportServerEx *TransportServerEx, upstreamNamer *upstreamNamer, isPlus bool, isResolverConfigured bool) ([]version2.StreamUpstream, Warnings) { + warnings := newWarnings() var upstreams []version2.StreamUpstream for _, u := range transportServerEx.TransportServer.Spec.Upstreams { - // subselector is not supported yet in TransportServer upstreams. That's why we pass "nil" here endpointsKey := GenerateEndpointsKey(transportServerEx.TransportServer.Namespace, u.Service, nil, uint16(u.Port)) + externalNameSvcKey := GenerateExternalNameSvcKey(transportServerEx.TransportServer.Namespace, u.Service) endpoints := transportServerEx.Endpoints[endpointsKey] - ups := generateStreamUpstream(u, upstreamNamer, endpoints, isPlus) + _, isExternalNameSvc := transportServerEx.ExternalNameSvcs[externalNameSvcKey] + if isExternalNameSvc && !isResolverConfigured { + msgFmt := "Type ExternalName service %v in upstream %v will be ignored. To use ExternalName services, a resolver must be configured in the ConfigMap" + warnings.AddWarningf(transportServerEx.TransportServer, msgFmt, u.Service, u.Name) + endpoints = []string{} + } + ups := generateStreamUpstream(u, upstreamNamer, endpoints, isPlus) + ups.Resolve = isExternalNameSvc ups.UpstreamLabels.Service = u.Service ups.UpstreamLabels.ResourceType = "transportserver" ups.UpstreamLabels.ResourceName = transportServerEx.TransportServer.Name @@ -127,8 +135,7 @@ func generateStreamUpstreams(transportServerEx *TransportServerEx, upstreamNamer upstreams = append(upstreams, ups) } - - return upstreams + return upstreams, warnings } func generateTransportServerHealthCheck(upstreamName string, generatedUpstreamName string, upstreams []conf_v1alpha1.Upstream) (*version2.StreamHealthCheck, *version2.Match) { diff --git a/internal/configs/transportserver_test.go b/internal/configs/transportserver_test.go index 6b140d5ec3..ac389357d6 100644 --- a/internal/configs/transportserver_test.go +++ b/internal/configs/transportserver_test.go @@ -139,9 +139,12 @@ func TestGenerateTransportServerConfigForTCPSnippets(t *testing.T) { StreamSnippets: []string{"limit_conn_zone $binary_remote_addr zone=addr:10m;"}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -219,9 +222,12 @@ func TestGenerateTransportServerConfigForIPV6Disabled(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfigForIPV6Disabled() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfigForIPV6Disabled() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -307,9 +313,12 @@ func TestGenerateTransportServerConfigForTCP(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -398,9 +407,12 @@ func TestGenerateTransportServerConfigForTCPMaxConnections(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -487,9 +499,12 @@ func TestGenerateTransportServerConfigForTLSPassthrough(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) } } @@ -581,9 +596,269 @@ func TestGenerateTransportServerConfigForUDP(t *testing.T) { StreamSnippets: []string{}, } - result := generateTransportServerConfig(&transportServerEx, listenerPort, true) - if diff := cmp.Diff(expected, result); diff != "" { - t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", diff) + result, warnings := generateTransportServerConfig(&transportServerEx, listenerPort, true, false) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Errorf("generateTransportServerConfig() mismatch (-want +got):\n%s", cmp.Diff(expected, result)) + } +} + +func TestGenerateTransportServerConfig_ProducesValidConfigOnValidInputForExternalNameServiceAndConfiguredResolver(t *testing.T) { + t.Parallel() + transportServerEx := TransportServerEx{ + TransportServer: &conf_v1alpha1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "tcp-server", + Namespace: "default", + }, + Spec: conf_v1alpha1.TransportServerSpec{ + Listener: conf_v1alpha1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + Upstreams: []conf_v1alpha1.Upstream{ + { + Name: "tcp-app", + Service: "tcp-app-svc", + Port: 5001, + MaxFails: intPointer(3), + FailTimeout: "40s", + }, + }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1alpha1.SessionParameters{ + Timeout: "50s", + }, + Action: &conf_v1alpha1.Action{ + Pass: "tcp-app", + }, + }, + }, + Endpoints: map[string][]string{ + "default/tcp-app-svc:5001": { + "10.0.0.20:5001", + }, + }, + ExternalNameSvcs: map[string]bool{"default/tcp-app-svc": true}, + } + expected := &version2.TransportServerConfig{ + Upstreams: []version2.StreamUpstream{ + { + Name: "ts_default_tcp-server_tcp-app", + Servers: []version2.StreamUpstreamServer{ + { + Address: "10.0.0.20:5001", + MaxFails: 3, + FailTimeout: "40s", + }, + }, + UpstreamLabels: version2.UpstreamLabels{ + ResourceName: "tcp-server", + ResourceType: "transportserver", + ResourceNamespace: "default", + Service: "tcp-app-svc", + }, + LoadBalancingMethod: "random two least_conn", + Resolve: true, + }, + }, + Server: version2.StreamServer{ + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0s", + ProxyTimeout: "50s", + HealthCheck: nil, + ServerSnippets: []string{}, + }, + StreamSnippets: []string{}, + } + + result, warnings := generateTransportServerConfig(&transportServerEx, 2020, true, true) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Error(cmp.Diff(expected, result)) + } +} + +func TestGenerateTransportServerConfig_GeneratesWarningOnNotConfiguredResolver(t *testing.T) { + t.Parallel() + transportServerEx := TransportServerEx{ + TransportServer: &conf_v1alpha1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "tcp-server", + Namespace: "default", + }, + Spec: conf_v1alpha1.TransportServerSpec{ + Listener: conf_v1alpha1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + Upstreams: []conf_v1alpha1.Upstream{ + { + Name: "tcp-app", + Service: "tcp-app-svc", + Port: 5001, + MaxFails: intPointer(3), + FailTimeout: "40s", + }, + }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1alpha1.SessionParameters{ + Timeout: "50s", + }, + Action: &conf_v1alpha1.Action{ + Pass: "tcp-app", + }, + }, + }, + Endpoints: map[string][]string{ + "default/tcp-app-svc:5001": { + "10.0.0.20:5001", + }, + }, + ExternalNameSvcs: map[string]bool{"default/tcp-app-svc": true}, + } + expected := &version2.TransportServerConfig{ + Upstreams: []version2.StreamUpstream{ + { + Name: "ts_default_tcp-server_tcp-app", + Servers: nil, + UpstreamLabels: version2.UpstreamLabels{ + ResourceName: "tcp-server", + ResourceType: "transportserver", + ResourceNamespace: "default", + Service: "tcp-app-svc", + }, + LoadBalancingMethod: "random two least_conn", + Resolve: true, + }, + }, + Server: version2.StreamServer{ + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0s", + ProxyTimeout: "50s", + HealthCheck: nil, + ServerSnippets: []string{}, + }, + StreamSnippets: []string{}, + } + + result, warnings := generateTransportServerConfig(&transportServerEx, 2020, true, false) + if len(warnings) == 0 { + t.Errorf("want warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Error(cmp.Diff(expected, result)) + } +} + +func TestGenerateTransportServerConfig_UsesNotExistignSocketOnNotPlusAndNoEndpoints(t *testing.T) { + t.Parallel() + transportServerEx := TransportServerEx{ + TransportServer: &conf_v1alpha1.TransportServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "tcp-server", + Namespace: "default", + }, + Spec: conf_v1alpha1.TransportServerSpec{ + Listener: conf_v1alpha1.TransportServerListener{ + Name: "tcp-listener", + Protocol: "TCP", + }, + Upstreams: []conf_v1alpha1.Upstream{ + { + Name: "tcp-app", + Service: "tcp-app-svc", + Port: 5001, + MaxFails: intPointer(3), + FailTimeout: "40s", + }, + }, + UpstreamParameters: &conf_v1alpha1.UpstreamParameters{ + ConnectTimeout: "30s", + NextUpstream: false, + }, + SessionParameters: &conf_v1alpha1.SessionParameters{ + Timeout: "50s", + }, + Action: &conf_v1alpha1.Action{ + Pass: "tcp-app", + }, + }, + }, + Endpoints: map[string][]string{}, + ExternalNameSvcs: map[string]bool{"default/tcp-app-svc": true}, + } + expected := &version2.TransportServerConfig{ + Upstreams: []version2.StreamUpstream{ + { + Name: "ts_default_tcp-server_tcp-app", + Servers: []version2.StreamUpstreamServer{ + { + Address: nginxNonExistingUnixSocket, + MaxFails: 3, + FailTimeout: "40s", + }, + }, + UpstreamLabels: version2.UpstreamLabels{ + ResourceName: "tcp-server", + ResourceType: "transportserver", + ResourceNamespace: "default", + Service: "tcp-app-svc", + }, + LoadBalancingMethod: "random two least_conn", + Resolve: true, + }, + }, + Server: version2.StreamServer{ + Port: 2020, + UDP: false, + StatusZone: "tcp-listener", + ProxyPass: "ts_default_tcp-server_tcp-app", + Name: "tcp-server", + Namespace: "default", + ProxyConnectTimeout: "30s", + ProxyNextUpstream: false, + ProxyNextUpstreamTries: 0, + ProxyNextUpstreamTimeout: "0s", + ProxyTimeout: "50s", + HealthCheck: nil, + ServerSnippets: []string{}, + }, + StreamSnippets: []string{}, + } + + result, warnings := generateTransportServerConfig(&transportServerEx, 2020, false, true) + if len(warnings) != 0 { + t.Errorf("want no warnings, got %v", warnings) + } + if !cmp.Equal(expected, result) { + t.Error(cmp.Diff(expected, result)) } } diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 162a3bb20d..772cd19ff9 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -290,6 +290,11 @@ stream { {{range $value := .StreamSnippets}} {{$value}}{{end}} + {{if .ResolverAddresses}} + resolver {{range $resolver := .ResolverAddresses}}{{$resolver}}{{end}}{{if .ResolverValid}} valid={{.ResolverValid}}{{end}}{{if not .ResolverIPV6}} ipv6=off{{end}}; + {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} + {{end}} + {{if .TLSPassthrough}} map $ssl_preread_server_name $dest_internal_passthrough { default unix:/var/lib/nginx/passthrough-https.sock; diff --git a/internal/configs/version2/nginx-plus.transportserver.tmpl b/internal/configs/version2/nginx-plus.transportserver.tmpl index f886e349c7..30d5f3c578 100644 --- a/internal/configs/version2/nginx-plus.transportserver.tmpl +++ b/internal/configs/version2/nginx-plus.transportserver.tmpl @@ -7,7 +7,7 @@ upstream {{ $u.Name }} { {{ end }} {{ range $s := $u.Servers }} - server {{ $s.Address }} max_fails={{ $s.MaxFails }} fail_timeout={{ $s.FailTimeout }} max_conns={{ $s.MaxConnections }}; + server {{ $s.Address }} max_fails={{ $s.MaxFails }} fail_timeout={{ $s.FailTimeout }} max_conns={{ $s.MaxConnections }}{{ if $u.Resolve }} resolve{{ end }}; {{ end }} } {{ end }} diff --git a/internal/configs/version2/stream.go b/internal/configs/version2/stream.go index 70b3e88738..15a868bf73 100644 --- a/internal/configs/version2/stream.go +++ b/internal/configs/version2/stream.go @@ -15,6 +15,7 @@ type StreamUpstream struct { Servers []StreamUpstreamServer UpstreamLabels UpstreamLabels LoadBalancingMethod string + Resolve bool } // StreamUpstreamServer defines a stream upstream server. diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index 37cef3ba81..6c1f9aad84 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -398,6 +398,49 @@ var transportServerCfg = TransportServerConfig{ }, } +var transportServerCfgWithResolver = TransportServerConfig{ + Upstreams: []StreamUpstream{ + { + Name: "udp-upstream", + Servers: []StreamUpstreamServer{ + { + Address: "10.0.0.20:5001", + }, + }, + Resolve: true, + }, + }, + Match: &Match{ + Name: "match_udp-upstream", + Send: `GET / HTTP/1.0\r\nHost: localhost\r\n\r\n`, + ExpectRegexModifier: "~*", + Expect: "200 OK", + }, + Server: StreamServer{ + Port: 1234, + UDP: true, + StatusZone: "udp-app", + ProxyRequests: createPointerFromInt(1), + ProxyResponses: createPointerFromInt(2), + ProxyPass: "udp-upstream", + ProxyTimeout: "10s", + ProxyConnectTimeout: "10s", + ProxyNextUpstream: true, + ProxyNextUpstreamTimeout: "10s", + ProxyNextUpstreamTries: 5, + HealthCheck: &StreamHealthCheck{ + Enabled: false, + Timeout: "5s", + Jitter: "0", + Port: 8080, + Interval: "5s", + Passes: 1, + Fails: 1, + Match: "match_udp-upstream", + }, + }, +} + func createPointerFromInt(n int) *int { return &n } @@ -447,6 +490,18 @@ func TestTransportServerForNginxPlus(t *testing.T) { t.Log(string(data)) } +func TestExecuteTemplateForTransportServerWithResolver(t *testing.T) { + t.Parallel() + executor, err := NewTemplateExecutor(nginxPlusVirtualServerTmpl, nginxPlusTransportServerTmpl) + if err != nil { + t.Fatal(err) + } + _, err = executor.ExecuteTransportServerTemplate(&transportServerCfgWithResolver) + if err != nil { + t.Errorf("Failed to execute template: %v", err) + } +} + func TestTransportServerForNginx(t *testing.T) { t.Parallel() executor, err := NewTemplateExecutor(nginxVirtualServerTmpl, nginxTransportServerTmpl) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 6e0140f828..75e2eec488 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -1250,8 +1250,8 @@ func (lbc *LoadBalancerController) processChanges(changes []ResourceChange) { case *TransportServerConfiguration: tsEx := lbc.createTransportServerEx(impl.TransportServer, impl.ListenerPort) - addOrUpdateErr := lbc.configurator.AddOrUpdateTransportServer(tsEx) - lbc.updateTransportServerStatusAndEvents(impl, addOrUpdateErr) + warnings, addOrUpdateErr := lbc.configurator.AddOrUpdateTransportServer(tsEx) + lbc.updateTransportServerStatusAndEvents(impl, warnings, addOrUpdateErr) } } else if c.Op == Delete { switch impl := c.Resource.(type) { @@ -1662,7 +1662,7 @@ func (lbc *LoadBalancerController) updateResourcesStatusAndEvents(resources []Re lbc.updateRegularIngressStatusAndEvents(impl, warnings, operationErr) } case *TransportServerConfiguration: - lbc.updateTransportServerStatusAndEvents(impl, operationErr) + lbc.updateTransportServerStatusAndEvents(impl, warnings, operationErr) } } } @@ -1786,7 +1786,7 @@ func (lbc *LoadBalancerController) updateRegularIngressStatusAndEvents(ingConfig } } -func (lbc *LoadBalancerController) updateTransportServerStatusAndEvents(tsConfig *TransportServerConfiguration, operationErr error) { +func (lbc *LoadBalancerController) updateTransportServerStatusAndEvents(tsConfig *TransportServerConfiguration, warnings configs.Warnings, operationErr error) { eventTitle := "AddedOrUpdated" eventType := api_v1.EventTypeNormal eventWarningMessage := "" @@ -1799,6 +1799,13 @@ func (lbc *LoadBalancerController) updateTransportServerStatusAndEvents(tsConfig state = conf_v1.StateWarning } + if messages, ok := warnings[tsConfig.TransportServer]; ok { + eventType = api_v1.EventTypeWarning + eventTitle = "AddedOrUpdatedWithWarning" + eventWarningMessage = fmt.Sprintf("with warning(s): %s", formatWarningMessages(messages)) + state = conf_v1.StateWarning + } + if operationErr != nil { eventType = api_v1.EventTypeWarning eventTitle = "AddedOrUpdatedWithError" @@ -3192,19 +3199,19 @@ func isMatchingResourceRef(ownerNs, resRef, key string) bool { func (lbc *LoadBalancerController) createTransportServerEx(transportServer *conf_v1alpha1.TransportServer, listenerPort int) *configs.TransportServerEx { endpoints := make(map[string][]string) + externalNameSvcs := make(map[string]bool) podsByIP := make(map[string]string) disableIPV6 := lbc.configuration.isIPV6Disabled for _, u := range transportServer.Spec.Upstreams { podEndps, external, err := lbc.getEndpointsForUpstream(transportServer.Namespace, u.Service, uint16(u.Port)) + if err == nil && external && lbc.isNginxPlus { + externalNameSvcs[configs.GenerateExternalNameSvcKey(transportServer.Namespace, u.Service)] = true + } if err != nil { glog.Warningf("Error getting Endpoints for Upstream %v: %v", u.Name, err) } - if external { - glog.Warningf("ExternalName services are not yet supported in TransportServer upstreams") - } - // subselector is not supported yet in TransportServer upstreams. That's why we pass "nil" here endpointsKey := configs.GenerateEndpointsKey(transportServer.Namespace, u.Service, nil, uint16(u.Port)) @@ -3219,11 +3226,12 @@ func (lbc *LoadBalancerController) createTransportServerEx(transportServer *conf } return &configs.TransportServerEx{ - ListenerPort: listenerPort, - TransportServer: transportServer, - Endpoints: endpoints, - PodsByIP: podsByIP, - DisableIPV6: disableIPV6, + ListenerPort: listenerPort, + TransportServer: transportServer, + Endpoints: endpoints, + PodsByIP: podsByIP, + ExternalNameSvcs: externalNameSvcs, + DisableIPV6: disableIPV6, } } diff --git a/tests/data/transport-server-externalname/external-svc-deployment.yaml b/tests/data/transport-server-externalname/external-svc-deployment.yaml new file mode 100644 index 0000000000..51413aeeee --- /dev/null +++ b/tests/data/transport-server-externalname/external-svc-deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns-external-config +data: + Corefile: | + .:5353 { + forward . 8.8.8.8:53 + log + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns-external-backend +spec: + replicas: 1 + selector: + matchLabels: + app: coredns-external-backend + template: + metadata: + labels: + app: coredns-external-backend + spec: + containers: + - name: coredns-external-backend + image: coredns/coredns:1.8.6 + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 5353 + name: dns + protocol: UDP + - containerPort: 5353 + name: dns-tcp + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + volumes: + - name: config-volume + configMap: + name: coredns-external-config + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns-external-backend-svc +spec: + selector: + app: coredns-external-backend + ports: + - name: dns + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-externalname/externalname-svc.yaml b/tests/data/transport-server-externalname/externalname-svc.yaml new file mode 100644 index 0000000000..c78d8f6b0b --- /dev/null +++ b/tests/data/transport-server-externalname/externalname-svc.yaml @@ -0,0 +1,7 @@ +kind: Service +apiVersion: v1 +metadata: + name: externalname-service +spec: + type: ExternalName + externalName: core-dns-external-backend-svc.external-ns.svc.cluster.local \ No newline at end of file diff --git a/tests/data/transport-server-externalname/nginx-config.yaml b/tests/data/transport-server-externalname/nginx-config.yaml new file mode 100644 index 0000000000..5a68ee3f55 --- /dev/null +++ b/tests/data/transport-server-externalname/nginx-config.yaml @@ -0,0 +1,6 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config +data: + resolver-addresses: "kube-dns.kube-system.svc.cluster.local" \ No newline at end of file diff --git a/tests/data/transport-server-externalname/standard/global-configuration.yaml b/tests/data/transport-server-externalname/standard/global-configuration.yaml new file mode 100644 index 0000000000..63a5213668 --- /dev/null +++ b/tests/data/transport-server-externalname/standard/global-configuration.yaml @@ -0,0 +1,12 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: GlobalConfiguration +metadata: + name: nginx-configuration +spec: + listeners: + - name: dns-udp + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-externalname/standard/service_deployment.yaml b/tests/data/transport-server-externalname/standard/service_deployment.yaml new file mode 100644 index 0000000000..6a89847ba1 --- /dev/null +++ b/tests/data/transport-server-externalname/standard/service_deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns +data: + Corefile: | + .:5353 { + forward . 8.8.8.8:53 + log + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns +spec: + replicas: 2 + selector: + matchLabels: + app: coredns + template: + metadata: + labels: + app: coredns + spec: + containers: + - name: coredns + image: coredns/coredns:1.8.6 + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 5353 + name: dns-tcp + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile +--- +apiVersion: v1 +kind: Service +metadata: + name: coredns +spec: + selector: + app: coredns + ports: + - name: dns + port: 5353 + protocol: UDP + - name: dns-tcp + port: 5353 + protocol: TCP diff --git a/tests/data/transport-server-externalname/standard/transport-server.yaml b/tests/data/transport-server-externalname/standard/transport-server.yaml new file mode 100644 index 0000000000..1aff825167 --- /dev/null +++ b/tests/data/transport-server-externalname/standard/transport-server.yaml @@ -0,0 +1,14 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: TransportServer +metadata: + name: transport-server +spec: + listener: + name: dns-tcp + protocol: TCP + upstreams: + - name: dns-app + service: externalname-service + port: 5353 + action: + pass: dns-app diff --git a/tests/suite/test_transport_server_external_name.py b/tests/suite/test_transport_server_external_name.py new file mode 100644 index 0000000000..1b4a689917 --- /dev/null +++ b/tests/suite/test_transport_server_external_name.py @@ -0,0 +1,140 @@ +import pytest +from settings import DEPLOYMENTS, TEST_DATA +from suite.custom_assertions import assert_event +from suite.resources_utils import ( + create_items_from_yaml, + create_namespace_with_name_from_yaml, + create_service_from_yaml, + delete_namespace, + delete_service, + get_events, + get_file_contents, + get_first_pod_name, + replace_configmap_from_yaml, + wait_before_test, +) + + +class ExternalNameSetup: + """Encapsulate ExternalName example details. + + Attributes: + ic_pod_name: + external_host: external service host + """ + + def __init__(self, ic_pod_name, external_svc, external_host): + self.ic_pod_name = ic_pod_name + self.external_svc = external_svc + self.external_host = external_host + + +@pytest.fixture(scope="class") +def ts_externalname_setup( + request, kube_apis, ingress_controller_prerequisites, transport_server_setup +) -> ExternalNameSetup: + print( + "------------------------- Deploy external namespace with backend, configmap and service -----------------------------------" + ) + external_app_src = f"{TEST_DATA}/transport-server-externalname/external-svc-deployment.yaml" + external_ns = create_namespace_with_name_from_yaml(kube_apis.v1, "external-ns", f"{TEST_DATA}/common/ns.yaml") + create_items_from_yaml(kube_apis, external_app_src, external_ns) + + print("------------------------- ExternalName service Setup -----------------------------------") + config_map_name = ingress_controller_prerequisites.config_map["metadata"]["name"] + replace_configmap_from_yaml( + kube_apis.v1, + config_map_name, + ingress_controller_prerequisites.namespace, + f"{TEST_DATA}/transport-server-externalname/nginx-config.yaml", + ) + external_svc_src = f"{TEST_DATA}/transport-server-externalname/externalname-svc.yaml" + external_host = f"core-dns-external-backend-svc.external-ns.svc.cluster.local" + + external_svc = create_service_from_yaml(kube_apis.v1, transport_server_setup.namespace, external_svc_src) + wait_before_test() + ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace) + + def fin(): + print("Clean up ExternalName Setup:") + delete_service(kube_apis.v1, external_svc, transport_server_setup.namespace) + delete_namespace(kube_apis.v1, external_ns) + replace_configmap_from_yaml( + kube_apis.v1, + config_map_name, + ingress_controller_prerequisites.namespace, + f"{DEPLOYMENTS}/common/nginx-config.yaml", + ) + + request.addfinalizer(fin) + + return ExternalNameSetup(ic_pod_name, external_svc, external_host) + + +@pytest.mark.ts +@pytest.mark.skip_for_nginx_oss +@pytest.mark.parametrize( + "crd_ingress_controller, transport_server_setup", + [ + ( + { + "type": "complete", + "extra_args": [ + "-global-configuration=nginx-ingress/nginx-configuration", + "-enable-leader-election=false", + ], + }, + {"example": "transport-server-externalname", "app_type": "simple"}, + ) + ], + indirect=True, +) +class TestTransportServerStatus: + def test_template_config( + self, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + transport_server_setup, + ts_externalname_setup, + ): + wait_before_test() + + nginx_file_path = f"/etc/nginx/nginx.conf" + nginx_conf = get_file_contents( + kube_apis.v1, nginx_file_path, ts_externalname_setup.ic_pod_name, ingress_controller_prerequisites.namespace + ) + resolver_count = nginx_conf.count("resolver kube-dns.kube-system.svc.cluster.local;") + + ts_file_path = ( + f"/etc/nginx/stream-conf.d/ts_{transport_server_setup.namespace}_{transport_server_setup.name}.conf" + ) + ts_conf = get_file_contents( + kube_apis.v1, ts_file_path, ts_externalname_setup.ic_pod_name, ingress_controller_prerequisites.namespace + ) + + assert resolver_count == 2 # one for http and other for stream context + assert ( + f"server {ts_externalname_setup.external_host}:5353 max_fails=1 fail_timeout=10s max_conns=0 resolve;" + in ts_conf + ) + + def test_event_warning( + self, + kube_apis, + ingress_controller_prerequisites, + crd_ingress_controller, + transport_server_setup, + ts_externalname_setup, + ): + text = f"{transport_server_setup.namespace}/{transport_server_setup.name}" + event_text = f"Configuration for {text} was added or updated with warning(s): Type ExternalName service {ts_externalname_setup.external_svc} in upstream dns-app will be ignored. To use ExternalName services, a resolver must be configured in the ConfigMap" + replace_configmap_from_yaml( + kube_apis.v1, + ingress_controller_prerequisites.config_map["metadata"]["name"], + ingress_controller_prerequisites.namespace, + f"{DEPLOYMENTS}/common/nginx-config.yaml", + ) + wait_before_test(5) + events = get_events(kube_apis.v1, transport_server_setup.namespace) + assert_event(event_text, events)