Add healthcheck for transport server (#3361)

Add healthcheck for Transport Server

docs/content/logging-and-monitoring/service-insight.md

@@ -11,7 +11,7 @@ docs: "DOCS-000"
 ---
 
 
-The Ingress Controller exposes an endpoint and provides host statistics for Virtual Servers (VS).
+The Ingress Controller exposes an endpoint and provides host statistics for Virtual Servers (VS) and Transport Servers (TS).
 It exposes data in JSON format and returns HTTP status codes.
 The response body holds information about the total, down and the unhealthy number of
 upstreams associated with the hostname.
@@ -20,14 +20,14 @@ Returned HTTP codes indicate the health of the upstreams (service).
 The service is not healthy (HTTP response code different than 200 OK) if all upstreams are unhealthy.
 The service is healthy if at least one upstream is healthy. In this case, the endpoint returns HTTP code 200 OK.
 
-
-
 ## Enabling Service Insight Endpoint
 
 If you're using *Kubernetes manifests* (Deployment or DaemonSet) to install the Ingress Controller, to enable the Service Insight endpoint:
-1. Run the Ingress Controller with the `-enable-service-insight` [command-line argument](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments). This will expose the Ingress Controller endpoint via the path `/probe/{hostname}` on port `9114` (customizable with the `-service-insight-listen-port` command-line argument).
+
+1. Run the Ingress Controller with the `-enable-service-insight` [command-line argument](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments). This will expose the Ingress Controller endpoint via paths `/probe/{hostname}` for Virtual Servers, and `/probe/ts/{service_name}` for Transport Servers on port `9114` (customizable with the `-service-insight-listen-port` command-line argument). The `service_name` parameter refers to the name of the deployed service (the service specified under `upstreams` in the transport server).
 1. To enable TLS for the Service Insight endpoint, configure the `-service-insight-tls-secret` cli argument with the namespace and name of a TLS Secret.
 1. Add the Service Insight port to the list of the ports of the Ingress Controller container in the template of the Ingress Controller pod:
+
     ```yaml
     - name: service-insight
       containerPort: 9114
@@ -39,9 +39,9 @@ If you're using *Helm* to install the Ingress Controller, to enable Service Insi
 
 The Service Insight provides the following statistics:
 
-* Total number of VS
-* Number of VS in 'Down' state
-* Number of VS in 'Healthy' state
+* Total number of VS and TS
+* Number of VS and TS in 'Up' state
+* Number of VS and TS in 'Unhealthy' state
 
 These statistics are returned as JSON:
 
@@ -52,7 +52,7 @@ These statistics are returned as JSON:
 Response codes:
 
 * HTTP 200 OK - Service is healthy
-* HTTP 404 - No upstreams/VS found for the requested hostname
-* HTTP 503 Service Unavailable - The service is down (All upstreams/VS are "Unhealthy")
+* HTTP 404 Not Found - No upstreams/VS/TS found for the requested hostname/name
+* HTTP 418 I'm a teapot - The service is down (All upstreams/VS/TS are "Unhealthy")
 
 **Note**: wildcards in hostnames are not supported at the moment.

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.18.7
 	github.com/aws/aws-sdk-go-v2/service/marketplacemetering v1.13.26
 	github.com/cert-manager/cert-manager v1.10.1
-	github.com/go-chi/chi v1.5.4
+	github.com/go-chi/chi/v5 v5.0.8
 	github.com/golang-jwt/jwt/v4 v4.4.3
 	github.com/golang/glog v1.0.0
 	github.com/google/go-cmp v0.5.9

go.sum

@@ -108,8 +108,8 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
 github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs=
-github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg=
+github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
+github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=

internal/configs/configurator.go

@@ -111,6 +111,7 @@ type Configurator struct {
 	ingresses               map[string]*IngressEx
 	minions                 map[string]map[string]bool
 	virtualServers          map[string]*VirtualServerEx
+	transportServers        map[string]*TransportServerEx
 	tlsPassthroughPairs     map[string]tlsPassthroughPair
 	isWildcardEnabled       bool
 	isPlus                  bool
@@ -145,6 +146,7 @@ func NewConfigurator(nginxManager nginx.Manager, staticCfgParams *StaticConfigPa
 		cfgParams:               config,
 		ingresses:               make(map[string]*IngressEx),
 		virtualServers:          make(map[string]*VirtualServerEx),
+		transportServers:        make(map[string]*TransportServerEx),
 		templateExecutor:        templateExecutor,
 		templateExecutorV2:      templateExecutorV2,
 		minions:                 make(map[string]map[string]bool),
@@ -265,8 +267,8 @@ func (cnf *Configurator) AddOrUpdateIngress(ingEx *IngressEx) (Warnings, error)
 	return warnings, nil
 }
 
-// GetVirtualServerForHost takes a hostname and returns a VS for the given hostname.
-func (cnf *Configurator) GetVirtualServerForHost(hostname string) *conf_v1.VirtualServer {
+// virtualServerForHost takes a hostname and returns a VS for the given hostname.
+func (cnf *Configurator) virtualServerForHost(hostname string) *conf_v1.VirtualServer {
 	for _, vsEx := range cnf.virtualServers {
 		if vsEx.VirtualServer.Spec.Host == hostname {
 			return vsEx.VirtualServer
@@ -275,8 +277,8 @@ func (cnf *Configurator) GetVirtualServerForHost(hostname string) *conf_v1.Virtu
 	return nil
 }
 
-// GetUpstreamsforVirtualServer takes VS and returns a slice of upstreams.
-func (cnf *Configurator) GetUpstreamsforVirtualServer(vs *conf_v1.VirtualServer) []string {
+// upstreamsForVirtualServer takes VirtualServer and returns a list of associated upstreams.
+func (cnf *Configurator) upstreamsForVirtualServer(vs *conf_v1.VirtualServer) []string {
 	glog.V(3).Infof("Get upstreamName for vs: %s", vs.Spec.Host)
 	upstreamNames := make([]string, 0, len(vs.Spec.Upstreams))
 
@@ -290,18 +292,52 @@ func (cnf *Configurator) GetUpstreamsforVirtualServer(vs *conf_v1.VirtualServer)
 	return upstreamNames
 }
 
-// GetUpstreamsforHost takes a hostname and returns a slice of upstreams
-// for the given hostname.
-func (cnf *Configurator) GetUpstreamsforHost(hostname string) []string {
+// UpstreamsForHost takes a hostname and returns upstreams for the given hostname.
+func (cnf *Configurator) UpstreamsForHost(hostname string) []string {
 	glog.V(3).Infof("Get upstream for host: %s", hostname)
-	vs := cnf.GetVirtualServerForHost(hostname)
-
+	vs := cnf.virtualServerForHost(hostname)
 	if vs != nil {
-		return cnf.GetUpstreamsforVirtualServer(vs)
+		return cnf.upstreamsForVirtualServer(vs)
+	}
+	return nil
+}
+
+// StreamUpstreamsForName takes a name and returns stream upstreams
+// associated with this name. The name represents TS's
+// (TransportServer) action name.
+func (cnf *Configurator) StreamUpstreamsForName(name string) []string {
+	glog.V(3).Infof("Get stream upstreams for name: '%s'", name)
+	ts := cnf.transportServerForActionName(name)
+	if ts != nil {
+		return cnf.streamUpstreamsForTransportServer(ts)
+	}
+	return nil
+}
+
+// transportServerForActionName takes an action name and returns
+// Transport Server obj associated with that name.
+func (cnf *Configurator) transportServerForActionName(name string) *conf_v1alpha1.TransportServer {
+	for _, tsEx := range cnf.transportServers {
+		glog.V(3).Infof("Check ts action '%s' for requested name: '%s'", tsEx.TransportServer.Spec.Action.Pass, name)
+		if tsEx.TransportServer.Spec.Action.Pass == name {
+			return tsEx.TransportServer
+		}
 	}
 	return nil
 }
 
+// streamUpstreamsForTransportServer takes TransportServer obj and returns
+// a list of stream upstreams associated with this TransportServer.
+func (cnf *Configurator) streamUpstreamsForTransportServer(ts *conf_v1alpha1.TransportServer) []string {
+	upstreamNames := make([]string, 0, len(ts.Spec.Upstreams))
+	n := newUpstreamNamerForTransportServer(ts)
+	for _, u := range ts.Spec.Upstreams {
+		un := n.GetNameForUpstream(u.Name)
+		upstreamNames = append(upstreamNames, un)
+	}
+	return upstreamNames
+}
+
 func (cnf *Configurator) addOrUpdateIngress(ingEx *IngressEx) (Warnings, error) {
 	apResources := cnf.updateApResources(ingEx)
 
@@ -489,8 +525,7 @@ func (cnf *Configurator) AddOrUpdateVirtualServer(virtualServerEx *VirtualServer
 }
 
 func (cnf *Configurator) addOrUpdateOpenTracingTracerConfig(content string) error {
-	err := cnf.nginxManager.CreateOpenTracingTracerConfig(content)
-	return err
+	return cnf.nginxManager.CreateOpenTracingTracerConfig(content)
 }
 
 func (cnf *Configurator) addOrUpdateVirtualServer(virtualServerEx *VirtualServerEx) (Warnings, error) {
@@ -632,9 +667,10 @@ func (cnf *Configurator) addOrUpdateTransportServer(transportServerEx *Transport
 	if cnf.isPlus && cnf.isPrometheusEnabled {
 		cnf.updateTransportServerMetricsLabels(transportServerEx, tsCfg.Upstreams)
 	}
-
 	cnf.nginxManager.CreateStreamConfig(name, content)
 
+	cnf.transportServers[name] = transportServerEx
+
 	// update TLS Passthrough Hosts config in case we have a TLS Passthrough TransportServer
 	// only TLS Passthrough TransportServers have non-empty hosts
 	if transportServerEx.TransportServer.Spec.Host != "" {

internal/configs/configurator_test.go

@@ -1281,3 +1281,121 @@ func TestUpdateApResourcesForVs(t *testing.T) {
 		}
 	}
 }
+
+func TestUpstreamsForHost_ReturnsNilForNoVirtualServers(t *testing.T) {
+	t.Parallel()
+
+	tcnf := createTestConfigurator(t)
+	tcnf.virtualServers = map[string]*VirtualServerEx{
+		"vs": invalidVirtualServerEx,
+	}
+
+	got := tcnf.UpstreamsForHost("tea.example.com")
+	if got != nil {
+		t.Errorf("want nil, got %+v", got)
+	}
+}
+
+func TestUpstreamsForHost_DoesNotReturnUpstreamsOnBogusHostname(t *testing.T) {
+	t.Parallel()
+
+	tcnf := createTestConfigurator(t)
+	tcnf.virtualServers = map[string]*VirtualServerEx{
+		"vs": validVirtualServerExWithUpstreams,
+	}
+
+	got := tcnf.UpstreamsForHost("bogus.host.org")
+	if got != nil {
+		t.Errorf("want nil, got %+v", got)
+	}
+}
+
+func TestUpstreamsForHost_ReturnsUpstreamsNamesForValidHostname(t *testing.T) {
+	t.Parallel()
+	tcnf := createTestConfigurator(t)
+	tcnf.virtualServers = map[string]*VirtualServerEx{
+		"vs": validVirtualServerExWithUpstreams,
+	}
+
+	want := []string{"vs_default_test-vs_tea-app"}
+	got := tcnf.UpstreamsForHost("tea.example.com")
+	if !cmp.Equal(want, got) {
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
+func TestStreamUpstreamsForName_DoesNotReturnUpstreamsForBogusName(t *testing.T) {
+	t.Parallel()
+
+	tcnf := createTestConfigurator(t)
+	tcnf.transportServers = map[string]*TransportServerEx{
+		"ts": validTransportServerExWithUpstreams,
+	}
+
+	got := tcnf.StreamUpstreamsForName("bogus-service-name")
+	if got != nil {
+		t.Errorf("want nil, got %+v", got)
+	}
+}
+
+func TestStreamUpstreamsForName_ReturnsStreamUpstreamsNamesOnValidServiceName(t *testing.T) {
+	t.Parallel()
+
+	tcnf := createTestConfigurator(t)
+	tcnf.transportServers = map[string]*TransportServerEx{
+		"ts": validTransportServerExWithUpstreams,
+	}
+
+	want := []string{"ts_default_secure-app_secure-app"}
+	got := tcnf.StreamUpstreamsForName("secure-app")
+	if !cmp.Equal(want, got) {
+		t.Error(cmp.Diff(want, got))
+	}
+}
+
+var (
+	invalidVirtualServerEx = &VirtualServerEx{
+		VirtualServer: &conf_v1.VirtualServer{},
+	}
+	validVirtualServerExWithUpstreams = &VirtualServerEx{
+		VirtualServer: &conf_v1.VirtualServer{
+			ObjectMeta: meta_v1.ObjectMeta{
+				Name:      "test-vs",
+				Namespace: "default",
+			},
+			Spec: conf_v1.VirtualServerSpec{
+				Host: "tea.example.com",
+				Upstreams: []conf_v1.Upstream{
+					{
+						Name: "tea-app",
+					},
+				},
+			},
+		},
+	}
+	validTransportServerExWithUpstreams = &TransportServerEx{
+		TransportServer: &conf_v1alpha1.TransportServer{
+			ObjectMeta: meta_v1.ObjectMeta{
+				Name:      "secure-app",
+				Namespace: "default",
+			},
+			Spec: conf_v1alpha1.TransportServerSpec{
+				Listener: conf_v1alpha1.TransportServerListener{
+					Name:     "tls-passthrough",
+					Protocol: "TLS_PASSTHROUGH",
+				},
+				Host: "example.com",
+				Upstreams: []conf_v1alpha1.Upstream{
+					{
+						Name:    "secure-app",
+						Service: "secure-app",
+						Port:    8443,
+					},
+				},
+				Action: &conf_v1alpha1.Action{
+					Pass: "secure-app",
+				},
+			},
+		},
+	}
+)

internal/configs/transportserver.go

@@ -24,14 +24,18 @@ func (tsEx *TransportServerEx) String() string {
 	if tsEx == nil {
 		return "<nil>"
 	}
-
 	if tsEx.TransportServer == nil {
 		return "TransportServerEx has no TransportServer"
 	}
-
 	return fmt.Sprintf("%s/%s", tsEx.TransportServer.Namespace, tsEx.TransportServer.Name)
 }
 
+func newUpstreamNamerForTransportServer(transportServer *conf_v1alpha1.TransportServer) *upstreamNamer {
+	return &upstreamNamer{
+		prefix: fmt.Sprintf("ts_%s_%s", transportServer.Namespace, transportServer.Name),
+	}
+}
+
 // generateTransportServerConfig generates a full configuration for a TransportServer.
 func generateTransportServerConfig(transportServerEx *TransportServerEx, listenerPort int, isPlus bool, isResolverConfigured bool) (*version2.TransportServerConfig, Warnings) {
 	upstreamNamer := newUpstreamNamerForTransportServer(transportServerEx.TransportServer)
@@ -105,7 +109,6 @@ func generateUnixSocket(transportServerEx *TransportServerEx) string {
 	if transportServerEx.TransportServer.Spec.Listener.Name == conf_v1alpha1.TLSPassthroughListenerName {
 		return fmt.Sprintf("unix:/var/lib/nginx/passthrough-%s_%s.sock", transportServerEx.TransportServer.Namespace, transportServerEx.TransportServer.Name)
 	}
-
 	return ""
 }
 
@@ -172,7 +175,6 @@ func generateTransportServerHealthCheck(upstreamName string, generatedUpstreamNa
 			break
 		}
 	}
-
 	return hc, match
 }
 

internal/configs/virtualserver.go

@@ -8,7 +8,6 @@ import (
 	"github.com/golang/glog"
 	"github.com/nginxinc/kubernetes-ingress/internal/k8s/secrets"
 	"github.com/nginxinc/kubernetes-ingress/internal/nginx"
-	conf_v1alpha1 "github.com/nginxinc/kubernetes-ingress/pkg/apis/configuration/v1alpha1"
 	api_v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
@@ -158,12 +157,6 @@ func NewUpstreamNamerForVirtualServerRoute(virtualServer *conf_v1.VirtualServer,
 	}
 }
 
-func newUpstreamNamerForTransportServer(transportServer *conf_v1alpha1.TransportServer) *upstreamNamer {
-	return &upstreamNamer{
-		prefix: fmt.Sprintf("ts_%s_%s", transportServer.Namespace, transportServer.Name),
-	}
-}
-
 func (namer *upstreamNamer) GetNameForUpstreamFromAction(action *conf_v1.Action) string {
 	var upstream string
 	if action.Proxy != nil && action.Proxy.Upstream != "" {