Add e2e test for NPD

The first test is a very simple test. It installs NPD on a VM, and then
verifies that NPD reports metric host_uptime in Prometheus format.

.gitignore

@@ -3,3 +3,4 @@
 /*.tar.gz
 ci.env
 pr.env
+junit*.xml

Makefile

@@ -75,7 +75,6 @@ ifneq ($(BUILD_TAGS), "")
 	BUILD_TAGS:=-tags "$(BUILD_TAGS)"
 endif
 
-
 vet:
 	GO111MODULE=on go list -mod vendor $(BUILD_TAGS) ./... | \
 		grep -v "./vendor/*" | \
@@ -107,15 +106,22 @@ Dockerfile: Dockerfile.in
 	sed -e 's|@BASEIMAGE@|$(BASEIMAGE)|g' $< >$@
 
 test: vet fmt
-	GO111MODULE=on go test -mod vendor -timeout=1m -v -race $(BUILD_TAGS) ./...
+	GO111MODULE=on go test -mod vendor -timeout=1m -v -race -short $(BUILD_TAGS) ./...
+
+e2e-test: vet fmt build-tar
+	GO111MODULE=on go test -mod vendor -timeout=10m -v $(BUILD_TAGS) \
+	./test/e2e/metriconly/... \
+	-project=$(PROJECT) -zone=$(ZONE) \
+	-image=$(VM_IMAGE) -image-project=$(IMAGE_PROJECT) \
+	-ssh-user=$(SSH_USER) -ssh-key=$(SSH_KEY) -npd-build-tar=`pwd`/$(TARBALL)
 
 build-binaries: ./bin/node-problem-detector ./bin/log-counter
 
 build-container: build-binaries Dockerfile
 	docker build -t $(IMAGE) .
 
 build-tar: ./bin/node-problem-detector ./bin/log-counter
-	tar -zcvf $(TARBALL) bin/ config/
+	tar -zcvf $(TARBALL) bin/ config/ test/e2e-install.sh
 	sha1sum $(TARBALL)
 	md5sum $(TARBALL)
 

config/systemd/node-problem-detector-metric-only.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Node problem detector
+Wants=local-fs.target
+After=local-fs.target
+
+[Service]
+Restart=always
+RestartSec=10
+ExecStart=/home/kubernetes/bin/node-problem-detector --v=2 --logtostderr --enable-k8s-exporter=false \
+          --config.system-log-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor.json,/home/kubernetes/node-problem-detector/config/docker-monitor.json,/home/kubernetes/node-problem-detector/config/systemd-monitor.json \
+          --config.custom-plugin-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor-counter.json,/home/kubernetes/node-problem-detector/config/systemd-monitor-counter.json \
+          --config.system-stats-monitor=/home/kubernetes/node-problem-detector/config/system-stats-monitor.json          
+
+[Install]
+WantedBy=multi-user.target

pkg/util/metrics/fakes.go

@@ -21,17 +21,6 @@ import (
 	"reflect"
 )
 
-// Int64MetricRepresentation represents a snapshot of an int64 metrics.
-// This is used for inspecting fake metrics.
-type Int64MetricRepresentation struct {
-	// Name is the metric name.
-	Name string
-	// Labels contains all metric labels in key-value pair format.
-	Labels map[string]string
-	// Value is the value of the metric.
-	Value int64
-}
-
 // Int64MetricInterface is used to create test double for Int64Metric.
 type Int64MetricInterface interface {
 	// Record records a measurement for the metric, with provided tags as metric labels.

pkg/util/metrics/helpers.go

@@ -18,8 +18,11 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 
+	pcm "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/tag"
@@ -34,12 +37,6 @@ func init() {
 	tagMapMutex.Unlock()
 }
 
-// Int64Metric represents an int64 metric.
-type Int64Metric struct {
-	name    string
-	measure *stats.Int64Measure
-}
-
 // Aggregation defines how measurements should be aggregated into data points.
 type Aggregation string
 
@@ -50,6 +47,23 @@ const (
 	Sum Aggregation = "Sum"
 )
 
+// Int64MetricRepresentation represents a snapshot of an int64 metrics.
+// This is used for inspecting metric internals.
+type Int64MetricRepresentation struct {
+	// Name is the metric name.
+	Name string
+	// Labels contains all metric labels in key-value pair format.
+	Labels map[string]string
+	// Value is the value of the metric.
+	Value int64
+}
+
+// Int64Metric represents an int64 metric.
+type Int64Metric struct {
+	name    string
+	measure *stats.Int64Measure
+}
+
 // NewInt64Metric create a Int64Metric metric, returns nil when name is empty.
 func NewInt64Metric(name string, description string, unit string, aggregation Aggregation, tagNames []string) (*Int64Metric, error) {
 	if name == "" {
@@ -106,6 +120,17 @@ func (metric *Int64Metric) Record(tags map[string]string, measurement int64) err
 		metric.measure.M(measurement))
 }
 
+// Float64MetricRepresentation represents a snapshot of a float64 metrics.
+// This is used for inspecting metric internals.
+type Float64MetricRepresentation struct {
+	// Name is the metric name.
+	Name string
+	// Labels contains all metric labels in key-value pair format.
+	Labels map[string]string
+	// Value is the value of the metric.
+	Value float64
+}
+
 // Float64Metric represents an float64 metric.
 type Float64Metric struct {
 	name    string
@@ -187,3 +212,66 @@ func getTagKeysFromNames(tagNames []string) ([]tag.Key, error) {
 	}
 	return tagKeys, nil
 }
+
+// ParsePrometheusMetrics parses Prometheus formatted metrics into metrics under Float64MetricRepresentation.
+//
+// Note: Prometheus's go library stores all counter/gauge-typed metric values under float64.
+func ParsePrometheusMetrics(metricsText string) ([]Float64MetricRepresentation, error) {
+	var metrics []Float64MetricRepresentation
+
+	var textParser expfmt.TextParser
+	metricFamilies, err := textParser.TextToMetricFamilies(strings.NewReader(metricsText))
+	if err != nil {
+		return metrics, err
+	}
+
+	for _, metricFamily := range metricFamilies {
+		for _, metric := range metricFamily.Metric {
+			labels := make(map[string]string)
+			for _, labelPair := range metric.Label {
+				labels[*labelPair.Name] = *labelPair.Value
+			}
+
+			var value float64
+			if *metricFamily.Type == pcm.MetricType_COUNTER {
+				value = *metric.Counter.Value
+			} else if *metricFamily.Type == pcm.MetricType_GAUGE {
+				value = *metric.Gauge.Value
+			} else {
+				return metrics, fmt.Errorf("unexpected MetricType %s for metric %s",
+					pcm.MetricType_name[int32(*metricFamily.Type)], *metricFamily.Name)
+			}
+
+			metrics = append(metrics, Float64MetricRepresentation{*metricFamily.Name, labels, value})
+		}
+	}
+
+	return metrics, nil
+}
+
+// GetFloat64Metric finds the metric matching provided name and labels.
+// When strictLabelMatching is set to true, the founded metric labels are identical to the provided labels;
+// when strictLabelMatching is set to false, the founded metric labels are a superset of the provided labels.
+func GetFloat64Metric(metrics []Float64MetricRepresentation, name string, labels map[string]string,
+	strictLabelMatching bool) (Float64MetricRepresentation, error) {
+	for _, metric := range metrics {
+		if metric.Name != name {
+			continue
+		}
+		if strictLabelMatching && len(metric.Labels) != len(labels) {
+			continue
+		}
+		sameLabels := true
+		for key, value := range labels {
+			if metric.Labels[key] != value {
+				sameLabels = false
+				break
+			}
+		}
+		if !sameLabels {
+			continue
+		}
+		return metric, nil
+	}
+	return Float64MetricRepresentation{}, fmt.Errorf("no matching metric found")
+}

pkg/util/metrics/helpers_test.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package metrics
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+// TestPrometheusMetricsParsingAndMatching verifies the behavior of ParsePrometheusMetrics() and GetFloat64Metric().
+func TestPrometheusMetricsParsingAndMatching(t *testing.T) {
+	testCases := []struct {
+		name                string
+		metricsTextPath     string
+		expectedMetrics     []Float64MetricRepresentation
+		notExpectedMetrics  []Float64MetricRepresentation
+		strictLabelMatching bool
+	}{
+		{
+			name:            "Relaxed label matching",
+			metricsTextPath: "testdata/sample_metrics.txt",
+			expectedMetrics: []Float64MetricRepresentation{
+				// Metric with no label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{},
+				},
+				// Metric with partial label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{"kernel_version": "4.14.127+"},
+				},
+				{
+					Name:   "disk_avg_queue_len",
+					Labels: map[string]string{"device": "sda1"},
+				},
+				{
+					Name:   "disk_avg_queue_len",
+					Labels: map[string]string{"device": "sda8"},
+				},
+			},
+			notExpectedMetrics: []Float64MetricRepresentation{
+				// Metric with non-existant label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{"non-existant-version": "0.0.1"},
+				},
+				// Metric with incorrect label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{"kernel_version": "mismatched-version"},
+				},
+				// Non-exsistant metric.
+				{
+					Name:   "host_downtime",
+					Labels: map[string]string{},
+				},
+			},
+			strictLabelMatching: false,
+		},
+		{
+			name:            "Strict label matching",
+			metricsTextPath: "testdata/sample_metrics.txt",
+			expectedMetrics: []Float64MetricRepresentation{
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{"kernel_version": "4.14.127+", "os_version": "cos 73-11647.217.0"},
+				},
+				{
+					Name:   "problem_counter",
+					Labels: map[string]string{"reason": "DockerHung"},
+				},
+				{
+					Name:   "problem_counter",
+					Labels: map[string]string{"reason": "OOMKilling"},
+				},
+			},
+			notExpectedMetrics: []Float64MetricRepresentation{
+				// Metric with incomplete label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{"kernel_version": "4.14.127+"},
+				},
+				// Metric with missing label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{},
+				},
+				// Metric with non-existant label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{"non-existant-version": "0.0.1"},
+				},
+				// Metric with incorrect label.
+				{
+					Name:   "host_uptime",
+					Labels: map[string]string{"kernel_version": "mismatched-version"},
+				},
+				// Non-exsistant metric.
+				{
+					Name:   "host_downtime",
+					Labels: map[string]string{},
+				},
+			},
+			strictLabelMatching: true,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			b, err := ioutil.ReadFile(test.metricsTextPath)
+			if err != nil {
+				t.Errorf("Unexpected error reading file %s: %v", test.metricsTextPath, err)
+			}
+			metricsText := string(b)
+
+			metrics, err := ParsePrometheusMetrics(metricsText)
+			if err != nil {
+				t.Errorf("Unexpected error parsing NPD metrics: %v\nMetrics text: %s\n", err, metricsText)
+			}
+
+			for _, expectedMetric := range test.expectedMetrics {
+				_, err = GetFloat64Metric(metrics, expectedMetric.Name, expectedMetric.Labels, test.strictLabelMatching)
+				if err != nil {
+					t.Errorf("Failed to find metric %v in these metrics %v.\nMetrics text: %s\n",
+						expectedMetric, metrics, metricsText)
+				}
+			}
+
+			for _, notExpectedMetric := range test.notExpectedMetrics {
+				_, err = GetFloat64Metric(metrics, notExpectedMetric.Name, notExpectedMetric.Labels, test.strictLabelMatching)
+				if err == nil {
+					t.Errorf("Unexpected metric %v found in these metrics %v.\nMetrics text: %s\n",
+						notExpectedMetric, metrics, metricsText)
+				}
+			}
+		})
+	}
+}