Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support kube_state_metrics v2.0.0 #27552

Merged
merged 8 commits into from
Aug 26, 2021
Merged

Support kube_state_metrics v2.0.0 #27552

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
84fc11d
Support kube_pod_container_resource_requests and kube_pod_container_r…
MichaelKatsoulis Aug 23, 2021
bd17339
Add kube_node_status_capacity and kube_node_status_allocatable support
MichaelKatsoulis Aug 23, 2021
5521546
Update changelog
MichaelKatsoulis Aug 23, 2021
14a9c79
Add cpu requests and limits nanocores in test files
MichaelKatsoulis Aug 24, 2021
daba6cd
Review update
MichaelKatsoulis Aug 25, 2021
eda32e1
Add tests fro kube-state-metrics v2.0.0
MichaelKatsoulis Aug 25, 2021
827ae31
Add tests for ksm v2 for all state data streams
MichaelKatsoulis Aug 26, 2021
27fec71
Add deleted lines
MichaelKatsoulis Aug 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add state_job metricset to Kubernetes module{pull}26479[26479]
- Bump AWS SDK version to v0.24.0 for WebIdentity authentication flow {issue}19393[19393] {pull}27126[27126]
- Add Linux pressure metricset {pull}27355[27355]
- Add support for kube-state-metrics v2.0.0 {pull}27552[27552]

*Packetbeat*

Expand Down
6 changes: 5 additions & 1 deletion metricbeat/helper/prometheus/metric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,11 @@ type opFilterMap struct {
func (o opFilterMap) Process(field string, value interface{}, labels common.MapStr) (string, interface{}, common.MapStr) {
for k, v := range o.filterMap {
if labels[o.label] == k {
return fmt.Sprintf("%v.%v", field, v), value, labels
if field == "" {
return v, value, labels
} else {
return fmt.Sprintf("%v.%v", field, v), value, labels
}
}
}
return "", nil, nil
Expand Down
35 changes: 23 additions & 12 deletions metricbeat/module/kubernetes/state_container/_meta/test/ksm.v1.3.0.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.15
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also I think that we need to cover ksm.v2.0.0 in our tests, right?

"cores": 0.15,
"nanocores": 150000000
}
},
"id": "docker://e9560bbace13ca19de4b3771023198e8568f6b5ed6af3a949f10a5b8137b5be9",
Expand Down Expand Up @@ -70,7 +71,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.25
"cores": 0.25,
"nanocores": 250000000
}
},
"id": "docker://e9568dfef1dd249cabac4bf09e6bf4a239fe738ae20eba072b6516676fce4bf6",
Expand Down Expand Up @@ -153,7 +155,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.2
"cores": 0.2,
"nanocores": 200000000
}
},
"id": "docker://4beb9aab887ca162c9cb3534c4826156636241052cd548153eaa2a170b6d102f",
Expand Down Expand Up @@ -197,10 +200,12 @@
"MetricSetFields": {
"cpu": {
"limit": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
},
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://948c4ebd8ca4fdf352e7fbf7f5c5d381af7e615ced435dc42fde0c1d25851320",
Expand Down Expand Up @@ -252,7 +257,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://eadcbd54ba914dff6475ae64805887967cfb973aeb9b07364c94372658a71d11",
Expand Down Expand Up @@ -296,7 +302,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://1958e71d048065d38ce83dafda567c5fa9d0c1278cd7292d55b9f1d80b0a67f9",
Expand Down Expand Up @@ -387,7 +394,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.005
"cores": 0.005,
"nanocores": 5000000
}
},
"id": "docker://ab382dbe8f8265f88ee9fec7de142f778da4a5fd9fe0334e3bdb6fe851124c08",
Expand Down Expand Up @@ -436,10 +444,12 @@
"MetricSetFields": {
"cpu": {
"limit": {
"cores": 0.101
"cores": 0.101,
"nanocores": 101000000
},
"request": {
"cores": 0.101
"cores": 0.101,
"nanocores": 101000000
}
},
"id": "docker://88951e0178ea5131fa3e2d7cafacb3a7e63700795dd6fa0d40ed2e4ac1f52f9c",
Expand Down Expand Up @@ -530,7 +540,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.01
"cores": 0.01,
"nanocores": 10000000
}
},
"id": "docker://aad0addd205dc72dc7abc8f9d02a1b429a2f2e1df3acc60431ca6b79746c093b",
Expand Down Expand Up @@ -598,4 +609,4 @@
"Period": 0,
"DisableTimeSeries": false
}
]
]
20 changes: 13 additions & 7 deletions metricbeat/module/kubernetes/state_container/_meta/test/ksm.v1.8.0.expected
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.005
"cores": 0.005,
"nanocores": 5000000
}
},
"id": "docker://f13c53a3ed0f3626b33b3c588d6913257320f65714eff28f25ead8f7663dc93b",
Expand Down Expand Up @@ -70,7 +71,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://0ea0cef8a79c7643474a736e5da14c254d9411d87167028fa07c96d09748c83a",
Expand Down Expand Up @@ -153,7 +155,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.2
"cores": 0.2,
"nanocores": 200000000
}
},
"id": "docker://465ebffafd7fc238a2fa2e764255efcbff88d5513f4c68f57d70932985428d12",
Expand Down Expand Up @@ -237,7 +240,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://f8fe5be1dbb1931d702c89235c79965730cbcced7b0ced9895f6c54c1ae8e5c3",
Expand Down Expand Up @@ -367,7 +371,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://15ada7864628d1c8007c01420e5887a501590d3bc9c25628a4770172ad615112",
Expand Down Expand Up @@ -458,7 +463,8 @@
"MetricSetFields": {
"cpu": {
"request": {
"cores": 0.25
"cores": 0.25,
"nanocores": 250000000
}
},
"id": "docker://cdaefb4df2f2add498f884fdc717a6ca8d2681c1636934747de600e6427e0c0d",
Expand Down Expand Up @@ -558,4 +564,4 @@
"Period": 0,
"DisableTimeSeries": false
}
]
]
32 changes: 21 additions & 11 deletions metricbeat/module/kubernetes/state_container/_meta/testdata/ksm-v1_3_0.plain-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,10 +89,12 @@
"container": {
"cpu": {
"limit": {
"cores": 0.2
"cores": 0.2,
"nanocores": 200000000
},
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"memory": {
Expand Down Expand Up @@ -136,10 +138,12 @@
"container": {
"cpu": {
"limit": {
"cores": 0.2
"cores": 0.2,
"nanocores": 200000000
},
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://973cbe45982c5126a5caf8c58d964c0ab1d5bb2c165ccc59715fcc1ebd58ab3d",
Expand Down Expand Up @@ -192,7 +196,8 @@
"container": {
"cpu": {
"request": {
"cores": 0.2
"cores": 0.2,
"nanocores": 200000000
}
},
"id": "docker://e2ee1c2c7b8d4e5fd8c834b83cba8377d6b0e39da18157688ccc1a06b7c53117",
Expand Down Expand Up @@ -242,7 +247,8 @@
"container": {
"cpu": {
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://fa3d83f648de42492b38fa3e8501d109376f391c50f2bd210c895c8477ae4b62",
Expand Down Expand Up @@ -295,7 +301,8 @@
"container": {
"cpu": {
"request": {
"cores": 0.1
"cores": 0.1,
"nanocores": 100000000
}
},
"id": "docker://4fa227874ee68536bf902394fb662f07b99099798ca9cd5c1506b79075acc065",
Expand Down Expand Up @@ -345,7 +352,8 @@
"container": {
"cpu": {
"request": {
"cores": 0.2
"cores": 0.2,
"nanocores": 200000000
}
},
"id": "docker://fa3d83f648de42492b38fa3e8501d109376f391c50f2bd210c895c8477ae4b62-test",
Expand Down Expand Up @@ -398,7 +406,8 @@
"container": {
"cpu": {
"request": {
"cores": 0.005
"cores": 0.005,
"nanocores": 5000000
}
},
"id": "docker://91fdd43f6b1b4c3dd133cfca53e0b1210bc557c2ae56006026b5ccdb5f52826f",
Expand Down Expand Up @@ -448,7 +457,8 @@
"container": {
"cpu": {
"request": {
"cores": 0.01
"cores": 0.01,
"nanocores": 10000000
}
},
"id": "docker://52fa55e051dc5b68e44c027588685b7edd85aaa03b07f7216d399249ff4fc821",
Expand Down Expand Up @@ -524,4 +534,4 @@
"type": "kubernetes"
}
}
]
]
47 changes: 35 additions & 12 deletions metricbeat/module/kubernetes/state_container/state_container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,20 @@ var (
// Mapping of state metrics
mapping = &p.MetricsMapping{
Metrics: map[string]p.MetricMap{
"kube_pod_info": p.InfoMetric(),
"kube_pod_container_info": p.InfoMetric(),
"kube_pod_info": p.InfoMetric(),
"kube_pod_container_info": p.InfoMetric(),
"kube_pod_container_resource_requests": p.Metric("", p.OpFilterMap(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great that you were able to re-use our existing OpFilter to handle this!

"resource", map[string]string{
"cpu": "cpu.request.cores",
"memory": "memory.request.bytes",
},
)),
"kube_pod_container_resource_limits": p.Metric("", p.OpFilterMap(
"resource", map[string]string{
"cpu": "cpu.limit.cores",
"memory": "memory.limit.bytes",
},
)),
"kube_pod_container_resource_limits_cpu_cores": p.Metric("cpu.limit.cores"),
"kube_pod_container_resource_requests_cpu_cores": p.Metric("cpu.request.cores"),
"kube_pod_container_resource_limits_memory_bytes": p.Metric("memory.limit.bytes"),
Expand Down Expand Up @@ -130,18 +142,29 @@ func (m *MetricSet) Fetch(reporter mb.ReporterV2) error {
}

m.enricher.Enrich(events)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: consider not removing lines if there is no specific reason

//m.Logger().Infof("Events are %+v", events)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

leftover?

// Calculate deprecated nanocores values
for _, event := range events {
if request, ok := event["cpu.request.cores"]; ok {
if requestCores, ok := request.(float64); ok {
event["cpu.request.nanocores"] = requestCores * nanocores
}
}

if limit, ok := event["cpu.limit.cores"]; ok {
if limitCores, ok := limit.(float64); ok {
event["cpu.limit.nanocores"] = limitCores * nanocores
if cpuFields, ok := event["cpu"]; ok {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not to convert the event to common.MapStr and then access the cpu.limit.cores field directly, so as to avoid all these ifs? With the methods that MapStr provides you should be ok.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. event is already a common.MapStr

if cpuFieldsMapStr, ok := cpuFields.(common.MapStr); ok {
if request, ok := cpuFieldsMapStr["request"]; ok {
if requestFieldsMapStr, ok := request.(common.MapStr); ok {
if cores, ok := requestFieldsMapStr["cores"]; ok {
if requestCores, ok := cores.(float64); ok {
event.Put("cpu.request.nanocores", requestCores*nanocores)
}
}
}
}
if limit, ok := cpuFieldsMapStr["limit"]; ok {
if limitFieldsMapStr, ok := limit.(common.MapStr); ok {
if cores, ok := limitFieldsMapStr["cores"]; ok {
if limitCores, ok := cores.(float64); ok {
event.Put("cpu.limit.nanocores", limitCores*nanocores)
}
}
}
}
}
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: same here, do not remove lines if it's not intentional

Expand Down
18 changes: 16 additions & 2 deletions metricbeat/module/kubernetes/state_node/state_node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,22 @@ var (
"kube_node_status_allocatable_memory_bytes": p.Metric("memory.allocatable.bytes"),
"kube_node_status_capacity_cpu_cores": p.Metric("cpu.capacity.cores"),
"kube_node_status_allocatable_cpu_cores": p.Metric("cpu.allocatable.cores"),
"kube_node_spec_unschedulable": p.BooleanMetric("status.unschedulable"),
"kube_node_status_ready": p.LabelMetric("status.ready", "condition"),
"kube_node_status_capacity": p.Metric("", p.OpFilterMap(
"resource", map[string]string{
"pods": "pod.capacity.total",
"cpu": "cpu.capacity.cores",
"memory": "memory.capacity.bytes",
},
)),
"kube_node_status_allocatable": p.Metric("", p.OpFilterMap(
"resource", map[string]string{
"pods": "pod.allocatable.total",
"cpu": "cpu.allocatable.cores",
"memory": "memory.allocatable.bytes",
},
)),
"kube_node_spec_unschedulable": p.BooleanMetric("status.unschedulable"),
"kube_node_status_ready": p.LabelMetric("status.ready", "condition"),
"kube_node_status_condition": p.LabelMetric("status", "status", p.OpFilterMap(
"condition", map[string]string{
"Ready": "ready",
Expand Down