Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add metric about total number of TransportServers #2149

Merged
merged 2 commits into from
Nov 8, 2021
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add metric about total number of TransportServers
pleshakov committed Nov 8, 2021
commit 032d4836c0fa4ca62bd4eb497fd5206d41d06897
1 change: 1 addition & 0 deletions docs/content/logging-and-monitoring/prometheus.md
Original file line number Diff line number Diff line change
@@ -49,6 +49,7 @@ The Ingress Controller exports the following metrics:
* `controller_ingress_resources_total`. Number of handled Ingress resources. This metric includes the label type, that groups the Ingress resources by their type (regular, [minion or master](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration)). **Note**: The metric doesn't count minions without a master.
* `controller_virtualserver_resources_total`. Number of handled VirtualServer resources.
* `controller_virtualserverroute_resources_total`. Number of handled VirtualServerRoute resources. **Note**: The metric counts only VirtualServerRoutes that have a reference from a VirtualServer.
* `controller_transportserver_resources_total`. Number of handled TransportServer resources. This metric includes the label type, that groups the TransportServer resources by their type (passthrough, tcp or udp).
* Workqueue metrics. **Note**: the workqueue is a queue used by the Ingress Controller to process changes to the relevant resources in the cluster like Ingress resources. The Ingress Controller uses only one queue. The metrics for that queue will have the label `name="taskQueue"`
* `workqueue_depth`. Current depth of the workqueue.
* `workqueue_queue_duration_second`. How long in seconds an item stays in the workqueue before being requested.
31 changes: 31 additions & 0 deletions internal/k8s/configuration.go
Original file line number Diff line number Diff line change
@@ -312,6 +312,13 @@ func compareObjectMetasWithAnnotations(meta1 *metav1.ObjectMeta, meta2 *metav1.O
return compareObjectMetas(meta1, meta2) && reflect.DeepEqual(meta1.Annotations, meta2.Annotations)
}

// TransportServerMetrics holds metrics about TransportServer resources
type TransportServerMetrics struct {
TotalTLSPassthrough int
TotalTCP int
TotalUDP int
}

// Configuration represents the configuration of the Ingress Controller - a collection of configuration objects
// (Ingresses, VirtualServers, VirtualServerRoutes) ready to be transformed into NGINX config.
// It holds the latest valid state of those objects.
@@ -1442,6 +1449,30 @@ func (c *Configuration) buildVirtualServerRoutes(vs *conf_v1.VirtualServer) ([]*
return vsrs, warnings
}

// GetTransportServerMetrics returns metrics about TransportServers
func (c *Configuration) GetTransportServerMetrics() *TransportServerMetrics {
var metrics TransportServerMetrics

if c.isTLSPassthroughEnabled {
for _, resource := range c.hosts {
_, ok := resource.(*TransportServerConfiguration)
if ok {
metrics.TotalTLSPassthrough++
}
}
}

for _, tsConfig := range c.listeners {
if tsConfig.TransportServer.Spec.Listener.Protocol == "TCP" {
metrics.TotalTCP++
} else {
metrics.TotalUDP++
soneillf5 marked this conversation as resolved.
Show resolved Hide resolved
}
}

return &metrics
}

func getSortedIngressKeys(m map[string]*networking.Ingress) []string {
var keys []string

98 changes: 98 additions & 0 deletions internal/k8s/configuration_test.go
Original file line number Diff line number Diff line change
@@ -3256,6 +3256,104 @@ func TestGetResources(t *testing.T) {
}
}

func TestGetTransportServerMetrics(t *testing.T) {
tsPass := createTestTLSPassthroughTransportServer("transportserver", "abc.example.com")
tsTCP := createTestTransportServer("transportserver-tcp", "tcp-7777", "TCP")
tsUDP := createTestTransportServer("transportserver-udp", "udp-7777", "UDP")

tests := []struct {
tses []*conf_v1alpha1.TransportServer
expected *TransportServerMetrics
msg string
}{
{
tses: nil,
expected: &TransportServerMetrics{
TotalTLSPassthrough: 0,
TotalTCP: 0,
TotalUDP: 0,
},
msg: "no TransportServers",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsPass,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 1,
TotalTCP: 0,
TotalUDP: 0,
},
msg: "one TLSPassthrough TransportServer",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsTCP,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 0,
TotalTCP: 1,
TotalUDP: 0,
},
msg: "one TCP TransportServer",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsUDP,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 0,
TotalTCP: 0,
TotalUDP: 1,
},
msg: "one UDP TransportServer",
},
{
tses: []*conf_v1alpha1.TransportServer{
tsPass, tsTCP, tsUDP,
},
expected: &TransportServerMetrics{
TotalTLSPassthrough: 1,
TotalTCP: 1,
TotalUDP: 1,
},
msg: "TLSPasstrough, TCP and UDP TransportServers",
},
}

listeners := []conf_v1alpha1.Listener{
{
Name: "tcp-7777",
Port: 7777,
Protocol: "TCP",
},
{
Name: "udp-7777",
Port: 7777,
Protocol: "UDP",
},
}
gc := createTestGlobalConfiguration(listeners)

for _, test := range tests {
configuration := createTestConfiguration()

_, _, err := configuration.AddOrUpdateGlobalConfiguration(gc)
if err != nil {
t.Fatalf("AddOrUpdateGlobalConfiguration() returned unexpected error %v", err)
}

for _, ts := range test.tses {
configuration.AddOrUpdateTransportServer(ts)
}

result := configuration.GetTransportServerMetrics()
if diff := cmp.Diff(test.expected, result); diff != "" {
t.Errorf("GetTransportServerMetrics() returned unexpected result for the case of %s (-want +got):\n%s", test.msg, diff)
}
}
}

func TestIsEqualForIngressConfigurationes(t *testing.T) {
regularIng := createTestIngress("regular-ingress", "foo.example.com")

14 changes: 13 additions & 1 deletion internal/k8s/controller.go
Original file line number Diff line number Diff line change
@@ -315,7 +315,6 @@ func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalanc

lbc.secretStore = secrets.NewLocalSecretStore(lbc.configurator)

lbc.updateIngressMetrics()
return lbc
}

@@ -712,6 +711,7 @@ func (lbc *LoadBalancerController) sync(task task) {
case ingress:
lbc.syncIngress(task)
lbc.updateIngressMetrics()
lbc.updateTransportServerMetrics()
case configMap:
lbc.syncConfigMap(task)
case endpoints:
@@ -723,13 +723,16 @@ func (lbc *LoadBalancerController) sync(task task) {
case virtualserver:
lbc.syncVirtualServer(task)
lbc.updateVirtualServerMetrics()
lbc.updateTransportServerMetrics()
case virtualServerRoute:
lbc.syncVirtualServerRoute(task)
lbc.updateVirtualServerMetrics()
case globalConfiguration:
lbc.syncGlobalConfiguration(task)
lbc.updateTransportServerMetrics()
case transportserver:
lbc.syncTransportServer(task)
lbc.updateTransportServerMetrics()
case policy:
lbc.syncPolicy(task)
case appProtectPolicy:
@@ -1638,6 +1641,15 @@ func (lbc *LoadBalancerController) updateVirtualServerMetrics() {
lbc.metricsCollector.SetVirtualServerRoutes(vsrCount)
}

func (lbc *LoadBalancerController) updateTransportServerMetrics() {
if !lbc.areCustomResourcesEnabled {
return
}

metrics := lbc.configuration.GetTransportServerMetrics()
lbc.metricsCollector.SetTransportServers(metrics.TotalTLSPassthrough, metrics.TotalTCP, metrics.TotalUDP)
}

func (lbc *LoadBalancerController) syncService(task task) {
key := task.Key
glog.V(3).Infof("Syncing service %v", key)
82 changes: 60 additions & 22 deletions internal/metrics/collectors/controller.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ type ControllerCollector interface {
SetIngresses(ingressType string, count int)
SetVirtualServers(count int)
SetVirtualServerRoutes(count int)
SetTransportServers(tlsPassthroughCount, tcpCount, udpCount int)
Register(registry *prometheus.Registry) error
}

@@ -18,6 +19,7 @@ type ControllerMetricsCollector struct {
ingressesTotal *prometheus.GaugeVec
virtualServersTotal prometheus.Gauge
virtualServerRoutesTotal prometheus.Gauge
transportServersTotal *prometheus.GaugeVec
}

// NewControllerMetricsCollector creates a new ControllerMetricsCollector
@@ -32,34 +34,58 @@ func NewControllerMetricsCollector(crdsEnabled bool, constLabels map[string]stri
labelNamesController,
)

if !crdsEnabled {
return &ControllerMetricsCollector{ingressesTotal: ingResTotal}
var vsResTotal, vsrResTotal prometheus.Gauge
var tsResTotal *prometheus.GaugeVec

if crdsEnabled {
vsResTotal = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserver_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServer resources",
ConstLabels: constLabels,
},
)

vsrResTotal = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserverroute_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServerRoute resources",
ConstLabels: constLabels,
},
)

tsResTotal = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "transportserver_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled TransportServer resources",
ConstLabels: constLabels,
},
labelNamesController,
)
}

vsResTotal := prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserver_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServer resources",
ConstLabels: constLabels,
},
)

vsrResTotal := prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "virtualserverroute_resources_total",
Namespace: metricsNamespace,
Help: "Number of handled VirtualServerRoute resources",
ConstLabels: constLabels,
},
)

return &ControllerMetricsCollector{
crdsEnabled: true,
c := &ControllerMetricsCollector{
crdsEnabled: crdsEnabled,
ingressesTotal: ingResTotal,
virtualServersTotal: vsResTotal,
virtualServerRoutesTotal: vsrResTotal,
transportServersTotal: tsResTotal,
}

// if we don't set to 0 metrics with the label type, the metrics will not be created initially

c.SetIngresses("regular", 0)
c.SetIngresses("master", 0)
c.SetIngresses("minion", 0)

if crdsEnabled {
c.SetTransportServers(0, 0, 0)
}

return c
}

// SetIngresses sets the value of the ingress resources gauge for a given type
@@ -77,12 +103,20 @@ func (cc *ControllerMetricsCollector) SetVirtualServerRoutes(count int) {
cc.virtualServerRoutesTotal.Set(float64(count))
}

// SetTransportServers sets the value of the TransportServer resources gauge
func (cc *ControllerMetricsCollector) SetTransportServers(tlsPassthroughCount, tcpCount, udpCount int) {
cc.transportServersTotal.WithLabelValues("passthrough").Set(float64(tlsPassthroughCount))
cc.transportServersTotal.WithLabelValues("tcp").Set(float64(tcpCount))
cc.transportServersTotal.WithLabelValues("udp").Set(float64(udpCount))
}

// Describe implements prometheus.Collector interface Describe method
func (cc *ControllerMetricsCollector) Describe(ch chan<- *prometheus.Desc) {
cc.ingressesTotal.Describe(ch)
if cc.crdsEnabled {
cc.virtualServersTotal.Describe(ch)
cc.virtualServerRoutesTotal.Describe(ch)
cc.transportServersTotal.Describe(ch)
}
}

@@ -92,6 +126,7 @@ func (cc *ControllerMetricsCollector) Collect(ch chan<- prometheus.Metric) {
if cc.crdsEnabled {
cc.virtualServersTotal.Collect(ch)
cc.virtualServerRoutesTotal.Collect(ch)
cc.transportServersTotal.Collect(ch)
}
}

@@ -119,3 +154,6 @@ func (cc *ControllerFakeCollector) SetVirtualServers(count int) {}

// SetVirtualServerRoutes implements a fake SetVirtualServerRoutes
func (cc *ControllerFakeCollector) SetVirtualServerRoutes(count int) {}

// SetTransportServers implements a fake SetTransportServers
func (cc *ControllerFakeCollector) SetTransportServers(int, int, int) {}