diff --git a/.gitignore b/.gitignore index 3524906ad..7dd2bdfe0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /*.tar.gz ci.env pr.env +junit*.xml diff --git a/Makefile b/Makefile index 646c09a40..62c3d9e53 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,6 @@ ifneq ($(BUILD_TAGS), "") BUILD_TAGS:=-tags "$(BUILD_TAGS)" endif - vet: GO111MODULE=on go list -mod vendor $(BUILD_TAGS) ./... | \ grep -v "./vendor/*" | \ @@ -107,7 +106,14 @@ Dockerfile: Dockerfile.in sed -e 's|@BASEIMAGE@|$(BASEIMAGE)|g' $< >$@ test: vet fmt - GO111MODULE=on go test -mod vendor -timeout=1m -v -race $(BUILD_TAGS) ./... + GO111MODULE=on go test -mod vendor -timeout=1m -v -race -short $(BUILD_TAGS) ./... + +e2e-test: vet fmt build-tar + GO111MODULE=on go test -mod vendor -timeout=10m -v $(BUILD_TAGS) \ + ./test/e2e/metriconly/... \ + -project=$(PROJECT) -zone=$(ZONE) \ + -image=$(VM_IMAGE) -image-project=$(IMAGE_PROJECT) \ + -ssh-user=$(SSH_USER) -ssh-key=$(SSH_KEY) -npd-build-tar=`pwd`/$(TARBALL) build-binaries: ./bin/node-problem-detector ./bin/log-counter @@ -115,7 +121,7 @@ build-container: build-binaries Dockerfile docker build -t $(IMAGE) . build-tar: ./bin/node-problem-detector ./bin/log-counter - tar -zcvf $(TARBALL) bin/ config/ + tar -zcvf $(TARBALL) bin/ config/ test/e2e-install.sh sha1sum $(TARBALL) md5sum $(TARBALL) diff --git a/config/systemd/node-problem-detector-metric-only.service b/config/systemd/node-problem-detector-metric-only.service new file mode 100644 index 000000000..142fee176 --- /dev/null +++ b/config/systemd/node-problem-detector-metric-only.service @@ -0,0 +1,15 @@ +[Unit] +Description=Node problem detector +Wants=local-fs.target +After=local-fs.target + +[Service] +Restart=always +RestartSec=10 +ExecStart=/home/kubernetes/bin/node-problem-detector --v=2 --logtostderr --enable-k8s-exporter=false \ + --config.system-log-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor.json,/home/kubernetes/node-problem-detector/config/docker-monitor.json,/home/kubernetes/node-problem-detector/config/systemd-monitor.json \ + --config.custom-plugin-monitor=/home/kubernetes/node-problem-detector/config/kernel-monitor-counter.json,/home/kubernetes/node-problem-detector/config/systemd-monitor-counter.json \ + --config.system-stats-monitor=/home/kubernetes/node-problem-detector/config/system-stats-monitor.json + +[Install] +WantedBy=multi-user.target diff --git a/pkg/util/metrics/fakes.go b/pkg/util/metrics/fakes.go index b98420303..53b3beea5 100644 --- a/pkg/util/metrics/fakes.go +++ b/pkg/util/metrics/fakes.go @@ -21,17 +21,6 @@ import ( "reflect" ) -// Int64MetricRepresentation represents a snapshot of an int64 metrics. -// This is used for inspecting fake metrics. -type Int64MetricRepresentation struct { - // Name is the metric name. - Name string - // Labels contains all metric labels in key-value pair format. - Labels map[string]string - // Value is the value of the metric. - Value int64 -} - // Int64MetricInterface is used to create test double for Int64Metric. type Int64MetricInterface interface { // Record records a measurement for the metric, with provided tags as metric labels. diff --git a/pkg/util/metrics/helpers.go b/pkg/util/metrics/helpers.go index 14e6f5d64..3cc6952ff 100644 --- a/pkg/util/metrics/helpers.go +++ b/pkg/util/metrics/helpers.go @@ -18,8 +18,11 @@ package metrics import ( "context" "fmt" + "strings" "sync" + pcm "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" @@ -34,12 +37,6 @@ func init() { tagMapMutex.Unlock() } -// Int64Metric represents an int64 metric. -type Int64Metric struct { - name string - measure *stats.Int64Measure -} - // Aggregation defines how measurements should be aggregated into data points. type Aggregation string @@ -50,6 +47,23 @@ const ( Sum Aggregation = "Sum" ) +// Int64MetricRepresentation represents a snapshot of an int64 metrics. +// This is used for inspecting metric internals. +type Int64MetricRepresentation struct { + // Name is the metric name. + Name string + // Labels contains all metric labels in key-value pair format. + Labels map[string]string + // Value is the value of the metric. + Value int64 +} + +// Int64Metric represents an int64 metric. +type Int64Metric struct { + name string + measure *stats.Int64Measure +} + // NewInt64Metric create a Int64Metric metric, returns nil when name is empty. func NewInt64Metric(name string, description string, unit string, aggregation Aggregation, tagNames []string) (*Int64Metric, error) { if name == "" { @@ -106,6 +120,17 @@ func (metric *Int64Metric) Record(tags map[string]string, measurement int64) err metric.measure.M(measurement)) } +// Float64MetricRepresentation represents a snapshot of a float64 metrics. +// This is used for inspecting metric internals. +type Float64MetricRepresentation struct { + // Name is the metric name. + Name string + // Labels contains all metric labels in key-value pair format. + Labels map[string]string + // Value is the value of the metric. + Value float64 +} + // Float64Metric represents an float64 metric. type Float64Metric struct { name string @@ -187,3 +212,66 @@ func getTagKeysFromNames(tagNames []string) ([]tag.Key, error) { } return tagKeys, nil } + +// ParsePrometheusMetrics parses Prometheus formatted metrics into metrics under Float64MetricRepresentation. +// +// Note: Prometheus's go library stores all counter/gauge-typed metric values under float64. +func ParsePrometheusMetrics(metricsText string) ([]Float64MetricRepresentation, error) { + var metrics []Float64MetricRepresentation + + var textParser expfmt.TextParser + metricFamilies, err := textParser.TextToMetricFamilies(strings.NewReader(metricsText)) + if err != nil { + return metrics, err + } + + for _, metricFamily := range metricFamilies { + for _, metric := range metricFamily.Metric { + labels := make(map[string]string) + for _, labelPair := range metric.Label { + labels[*labelPair.Name] = *labelPair.Value + } + + var value float64 + if *metricFamily.Type == pcm.MetricType_COUNTER { + value = *metric.Counter.Value + } else if *metricFamily.Type == pcm.MetricType_GAUGE { + value = *metric.Gauge.Value + } else { + return metrics, fmt.Errorf("unexpected MetricType %s for metric %s", + pcm.MetricType_name[int32(*metricFamily.Type)], *metricFamily.Name) + } + + metrics = append(metrics, Float64MetricRepresentation{*metricFamily.Name, labels, value}) + } + } + + return metrics, nil +} + +// GetFloat64Metric finds the metric matching provided name and labels. +// When strictLabelMatching is set to true, the founded metric labels are identical to the provided labels; +// when strictLabelMatching is set to false, the founded metric labels are a superset of the provided labels. +func GetFloat64Metric(metrics []Float64MetricRepresentation, name string, labels map[string]string, + strictLabelMatching bool) (Float64MetricRepresentation, error) { + for _, metric := range metrics { + if metric.Name != name { + continue + } + if strictLabelMatching && len(metric.Labels) != len(labels) { + continue + } + sameLabels := true + for key, value := range labels { + if metric.Labels[key] != value { + sameLabels = false + break + } + } + if !sameLabels { + continue + } + return metric, nil + } + return Float64MetricRepresentation{}, fmt.Errorf("no matching metric found") +} diff --git a/test/e2e-install.sh b/test/e2e-install.sh new file mode 100755 index 000000000..ee5d29a4f --- /dev/null +++ b/test/e2e-install.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is for installing node problem detector (NPD) on a running node +# in metric-only mode, as a setup for NPD e2e tests. + +set -o errexit +set -o nounset +set -o pipefail + +readonly BIN_DIR=/home/kubernetes/bin +readonly CONFIG_DIR=/home/kubernetes/node-problem-detector/config + +function print-help() { + echo "Usage: e2e-install.sh [flags] [command]" + echo + echo "Available flags:" + echo " -t [TARBALL] Specify the path of the NPD tarball (generated by 'make build-tar')." + echo + echo "Available commands:" + echo " help Print this help message" + echo " install Installs NPD to the this machine" + echo + echo "Examples:" + echo " e2e-install.sh help" + echo " e2e-install.sh -t /tmp/npd.tar.gz install" +} + +function install-npd() { + if [[ -z "${TARBALL}" ]]; then + echo "ERROR: tarball flag is missing." + exit 1 + fi + + readonly workdir=$(mktemp -d) + tar -xf "${TARBALL}" --directory "${workdir}" + + echo "Preparing NPD binary directory." + mkdir -p "${BIN_DIR}" + mount --bind "${BIN_DIR}" "${BIN_DIR}" + # Below remount is to work around COS's noexec mount on /home. + mount -o remount,exec "${BIN_DIR}" + + echo "Installing NPD binary." + cp "${workdir}"/bin/node-problem-detector "${BIN_DIR}" + + echo "Installing log-counter binary." + cp "${workdir}"/bin/log-counter "${BIN_DIR}" + + echo "Installing NPD configurations." + mkdir -p "${CONFIG_DIR}" + cp -r "${workdir}"/config/* "${CONFIG_DIR}" + + echo "Installing NPD systemd service." + cp "${workdir}"/config/systemd/node-problem-detector-metric-only.service /etc/systemd/system/node-problem-detector.service + + rm -rf "${workdir}" + + # Start systemd service. + echo "Starting NPD systemd service." + systemctl daemon-reload + systemctl stop node-problem-detector.service || true + systemctl start node-problem-detector.service +} + +function main() { + case ${1:-} in + help) print-help;; + install) install-npd;; + *) print-help;; + esac +} + +TARBALL="" + +while getopts "t:" opt; do + case ${opt} in + t) TARBALL="${OPTARG}";; + esac +done +shift "$((OPTIND-1))" + + +main "${@}" \ No newline at end of file diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..2c9e23dab --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,25 @@ +# Node Problem Detector End-To-End tests + +NPD e2e tests are meant for testing the NPD on a VM environment. + +Currently the tests only support Google Compute Engine (GCE) environment. Support for other vendors can be added in future. + +## Prerequisites + +1. Setup [Google Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials), which is [required for authentication](https://godoc.org/google.golang.org/api/compute/v1#hdr-Creating_a_client) by the Compute Engine API. +2. Setup a [project-wide SSH key](https://cloud.google.com/compute/docs/instances/adding-removing-ssh-keys#project-wide) that can be used to SSH into the GCE VMs. + +## Running tests + +From the node-problem-detector base directory, run: + +``` +export GOOGLE_APPLICATION_CREDENTIALS=[YOUR_ADC_PATH:~/.config/gcloud/application_default_credentials.json] +export ZONE=[ANY_GCE_ZONE:us-central1-a] +export PROJECT=[YOUR_PROJECT_ID] +export VM_IMAGE=[TESTED_OS_IMAGE:cos-73-11647-217-0] +export IMAGE_PROJECT=[TESTED_OS_IMAGE_PROJECT:cos-cloud] +export SSH_USER=${USER} +export SSH_KEY=~/.ssh/id_rsa +make e2e-test +``` diff --git a/test/e2e/lib/gce/gce.go b/test/e2e/lib/gce/gce.go new file mode 100644 index 000000000..7c1a99c4e --- /dev/null +++ b/test/e2e/lib/gce/gce.go @@ -0,0 +1,55 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "net/http" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + compute "google.golang.org/api/compute/v1" +) + +// GetComputeClient creates a GCE client with a 1 minute deadline. +func GetComputeClient() (*compute.Service, error) { + const retries = 10 + const backoff = time.Second * 6 + + // Setup the gce client for provisioning instances + // Getting credentials on gce jenkins is flaky, so try a couple times + var err error + var cs *compute.Service + for i := 0; i < retries; i++ { + if i > 0 { + time.Sleep(backoff) + } + + var client *http.Client + client, err = google.DefaultClient(oauth2.NoContext, compute.ComputeScope) + if err != nil { + continue + } + + cs, err = compute.New(client) + if err != nil { + continue + } + return cs, nil + } + return nil, err +} diff --git a/test/e2e/lib/gce/instance.go b/test/e2e/lib/gce/instance.go new file mode 100644 index 000000000..6aa4a4662 --- /dev/null +++ b/test/e2e/lib/gce/instance.go @@ -0,0 +1,166 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "fmt" + "os/exec" + "time" + + "k8s.io/node-problem-detector/test/e2e/lib/ssh" + + compute "google.golang.org/api/compute/v1" +) + +// Instance represents a GCE instance. +type Instance struct { + Name string + Zone string + Project string + MachineType string + ExternalIP string + SshKey string + SshUser string + ComputeService *compute.Service +} + +// CreateInstance creates a GCE instance with provided spec. +func CreateInstance(instance Instance, imageName string, imageProject string) (Instance, error) { + if instance.MachineType == "" { + instance.MachineType = "n1-standard-1" + } + + p, err := instance.ComputeService.Projects.Get(instance.Project).Do() + if err != nil { + return instance, fmt.Errorf("failed to get project info %q", instance.Project) + } + + i := &compute.Instance{ + Name: instance.Name, + MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", instance.Zone, instance.MachineType), + NetworkInterfaces: []*compute.NetworkInterface{ + { + AccessConfigs: []*compute.AccessConfig{ + { + Type: "ONE_TO_ONE_NAT", + Name: "External NAT", + }, + }}, + }, + Disks: []*compute.AttachedDisk{ + { + AutoDelete: true, + Boot: true, + Type: "PERSISTENT", + InitializeParams: &compute.AttachedDiskInitializeParams{ + SourceImage: fmt.Sprintf("projects/%s/global/images/%s", imageProject, imageName), + DiskSizeGb: 20, + }, + }, + }, + ServiceAccounts: []*compute.ServiceAccount{ + { + Email: p.DefaultServiceAccount, + Scopes: []string{ + "https://www.googleapis.com/auth/cloud-platform", + }, + }, + }, + } + + if _, err := instance.ComputeService.Instances.Get(instance.Project, instance.Zone, instance.Name).Do(); err != nil { + op, err := instance.ComputeService.Instances.Insert(instance.Project, instance.Zone, i).Do() + if err != nil { + ret := fmt.Sprintf("could not create instance %s: API error: %v", instance.Name, err) + if op != nil { + ret = fmt.Sprintf("%s: %v", ret, op.Error) + } + return instance, fmt.Errorf(ret) + } else if op.Error != nil { + return instance, fmt.Errorf("could not create instance %s: %+v", instance.Name, op.Error) + } + } + + instanceRunning := false + for i := 0; i < 30 && !instanceRunning; i++ { + if i > 0 { + time.Sleep(time.Second * 20) + } + if instance.ExternalIP == "" { + instance.populateExternalIP() + } + + result := ssh.Run("pwd", instance.ExternalIP, instance.SshUser, instance.SshKey) + if result.SSHError != nil { + err = fmt.Errorf("SSH to instance %s failed: %s", instance.Name, result.SSHError) + continue + } + instanceRunning = true + } + + if !instanceRunning { + return instance, err + } + return instance, nil +} + +func (ins *Instance) populateExternalIP() { + gceInstance, err := ins.ComputeService.Instances.Get(ins.Project, ins.Zone, ins.Name).Do() + if err != nil { + return + } + + for i := range gceInstance.NetworkInterfaces { + ni := gceInstance.NetworkInterfaces[i] + for j := range ni.AccessConfigs { + ac := ni.AccessConfigs[j] + if len(ac.NatIP) > 0 { + ins.ExternalIP = ac.NatIP + return + } + } + } +} + +// RunCommand runs a command on the GCE instance and returns the command result. +func (ins *Instance) RunCommand(cmd string) ssh.Result { + if ins.ExternalIP == "" { + ins.populateExternalIP() + } + return ssh.Run(cmd, ins.ExternalIP, ins.SshUser, ins.SshKey) +} + +// PushFile pushs a local file to a GCE instance. +func (ins *Instance) PushFile(srcPath, destPath string) error { + if ins.ExternalIP == "" { + ins.populateExternalIP() + } + return exec.Command("scp", "-o", "StrictHostKeyChecking no", + "-i", ins.SshKey, + srcPath, fmt.Sprintf("%s@%s:%s", ins.SshUser, ins.ExternalIP, destPath)).Run() +} + +// DeleteInstance deletes a GCE instance. +func (ins *Instance) DeleteInstance() error { + if _, err := ins.ComputeService.Instances.Get(ins.Project, ins.Zone, ins.Name).Do(); err != nil { + return err + } + if _, err := ins.ComputeService.Instances.Delete(ins.Project, ins.Zone, ins.Name).Do(); err != nil { + return err + } + return nil +} diff --git a/test/e2e/lib/npd/npd.go b/test/e2e/lib/npd/npd.go new file mode 100644 index 000000000..4337b4889 --- /dev/null +++ b/test/e2e/lib/npd/npd.go @@ -0,0 +1,84 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package npd + +import ( + "fmt" + "strings" + + "k8s.io/node-problem-detector/pkg/util/metrics" + "k8s.io/node-problem-detector/test/e2e/lib/gce" +) + +// SetupNPD installs NPD from the test tarball onto the provided GCE instance. +// +// Here is how it works: +// 1. SetupNPD will SCP the NPD build tarball onto the VM. +// 2. SetupNPD will extract the tarball in the VM, to expose the test/e2e-install.sh on the VM. +// 3. SetupNPD will then call the e2e-install.sh script, and feed the NPD build tarball as input. +// 4. Finally, the e2e-install.sh script will do the heavy lifting of installing NPD (setting up +// binary/config directories, setting up systemd config file, etc). +func SetupNPD(ins gce.Instance, npdBuildTar string) error { + tmpDirCmd := ins.RunCommand("mktemp -d") + if tmpDirCmd.SSHError != nil || tmpDirCmd.Code != 0 { + return fmt.Errorf("error creating temporary directory to hold NPD tarball: %v", tmpDirCmd) + } + + tmpDir := strings.TrimSuffix(tmpDirCmd.Stdout, "\n") + npdTarVMPath := tmpDir + "/npd.tar.gz" + npdExtractDir := tmpDir + "/npd" + + err := ins.PushFile(npdBuildTar, npdTarVMPath) + if err != nil { + return fmt.Errorf("error pushing local NPD build tarball %s to VM at %s: %v", npdBuildTar, npdTarVMPath, err) + } + + mkdirCmd := ins.RunCommand(fmt.Sprintf("mkdir -p %s", npdExtractDir)) + if mkdirCmd.SSHError != nil || mkdirCmd.Code != 0 { + return fmt.Errorf("error creating directory to extract NPD tarball into: %v", mkdirCmd) + } + + extractCmd := ins.RunCommand(fmt.Sprintf("tar -xf %s --directory %s", npdTarVMPath, npdExtractDir)) + if extractCmd.SSHError != nil || extractCmd.Code != 0 { + return fmt.Errorf("error extracting NPD build tarball: %v", extractCmd) + } + + installCmd := ins.RunCommand(fmt.Sprintf("sudo bash %s/test/e2e-install.sh -t %s install", npdExtractDir, npdTarVMPath)) + if installCmd.SSHError != nil || installCmd.Code != 0 { + return fmt.Errorf("error installing NPD: %v", installCmd) + } + + return nil +} + +// FetchNPDMetrics fetches and parses metrics reported by NPD on the provided GCE instance. +func FetchNPDMetrics(ins gce.Instance) ([]metrics.Float64MetricRepresentation, error) { + var npdMetrics []metrics.Float64MetricRepresentation + var err error + + curlCmd := ins.RunCommand("curl http://localhost:20257/metrics") + if curlCmd.SSHError != nil || curlCmd.Code != 0 { + return npdMetrics, fmt.Errorf("error fetching NPD metrics: %v", curlCmd) + } + + npdMetrics, err = metrics.ParsePrometheusMetrics(curlCmd.Stdout) + if err != nil { + return npdMetrics, fmt.Errorf("error parsing NPD metrics: %v", err) + } + + return npdMetrics, nil +} diff --git a/test/e2e/lib/ssh/ssh.go b/test/e2e/lib/ssh/ssh.go new file mode 100644 index 000000000..579e6ecf6 --- /dev/null +++ b/test/e2e/lib/ssh/ssh.go @@ -0,0 +1,64 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ssh + +import ( + "fmt" + "net" + "os" + + k8s_ssh "k8s.io/kubernetes/pkg/ssh" +) + +// Result holds the execution result of SSH command. +type Result struct { + User string + Host string + Cmd string + Stdout string + Stderr string + Code int + SSHError error +} + +// Run synchronously SSHs to a machine and runs cmd. +func Run(cmd, ip, sshUser, sshKey string) Result { + result := Result{User: sshUser, Host: ip, Cmd: cmd} + + if result.User == "" { + result.User = os.Getenv("USER") + } + + if ip == "" { + result.SSHError = fmt.Errorf("empty IP address") + return result + } + + signer, err := k8s_ssh.MakePrivateKeySignerFromFile(sshKey) + if err != nil { + result.SSHError = fmt.Errorf("error getting signer from key file %s: %v", sshKey, err) + return result + } + + stdout, stderr, code, err := k8s_ssh.RunSSHCommand(cmd, result.User, net.JoinHostPort(ip, "22"), signer) + result.Stdout = stdout + result.Stderr = stderr + result.Code = code + result.SSHError = err + + return result +} diff --git a/test/e2e/metriconly/e2e_npd_test.go b/test/e2e/metriconly/e2e_npd_test.go new file mode 100644 index 000000000..5eabdd31c --- /dev/null +++ b/test/e2e/metriconly/e2e_npd_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_metric_only + +import ( + "flag" + "fmt" + "os" + "path" + "testing" + + "k8s.io/node-problem-detector/test/e2e/lib/gce" + + "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/reporters" + compute "google.golang.org/api/compute/v1" +) + +const junitFileName = "junit.xml" +const artifactsDirEnv = "ARTIFACTS" + +var zone = flag.String("zone", "", "gce zone the hosts live in") +var project = flag.String("project", "", "gce project the hosts live in") +var image = flag.String("image", "", "image to test") +var imageProject = flag.String("image-project", "", "gce project of the OS image") +var sshKey = flag.String("ssh-key", "", "path to ssh private key.") +var sshUser = flag.String("ssh-user", "", "use predefined user for ssh.") +var npdBuildTar = flag.String("npd-build-tar", "", "tarball containing NPD to be tested.") + +var computeService *compute.Service + +func TestNPD(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + // The junit formatted result output is for showing test results on testgrid. + artifactsDir := os.Getenv(artifactsDirEnv) + var junitPath string + if artifactsDir != "" { + junitPath = path.Join(artifactsDir, junitFileName) + } else { + junitPath = junitFileName + } + + junitReporter := reporters.NewJUnitReporter(junitPath) + ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "NPD Metric-only Suite", []ginkgo.Reporter{junitReporter}) +} + +func TestMain(m *testing.M) { + flag.Parse() + + var err error + computeService, err = gce.GetComputeClient() + if err != nil { + panic(fmt.Sprintf("Unable to create gcloud compute service using defaults. Make sure you are authenticated. %v", err)) + } + + os.Exit(m.Run()) +} diff --git a/test/e2e/metriconly/metrics_test.go b/test/e2e/metriconly/metrics_test.go new file mode 100644 index 000000000..fa3139d8f --- /dev/null +++ b/test/e2e/metriconly/metrics_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_metric_only + +import ( + "fmt" + + "k8s.io/node-problem-detector/pkg/util/metrics" + "k8s.io/node-problem-detector/test/e2e/lib/gce" + "k8s.io/node-problem-detector/test/e2e/lib/npd" + + "github.com/onsi/ginkgo" + "github.com/pborman/uuid" +) + +var _ = ginkgo.Describe("NPD should export Prometheus metrics for detected problems", func() { + var instance gce.Instance + + ginkgo.BeforeEach(func() { + var err error + // TODO(xueweiz): Creating instance for each test case is slow. We should either reuse the instance + // between tests, or have a way to run these tests in parallel. + instance, err = gce.CreateInstance( + gce.Instance{ + Name: "npd-metrics-" + *image + "-" + uuid.NewUUID().String()[:8], + Zone: *zone, + Project: *project, + SshKey: *sshKey, + SshUser: *sshUser, + ComputeService: computeService, + }, + *image, + *imageProject) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Unable to create test instance: %v", err)) + } + + err = npd.SetupNPD(instance, *npdBuildTar) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Unable to setup NPD: %v", err)) + } + }) + + ginkgo.Context("when started on a clean instance,", func() { + + ginkgo.It("NPD should export Prometheus metrics for uptime", func() { + gotMetrics, err := npd.FetchNPDMetrics(instance) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Error fetching NPD metrics: %v", err)) + } + + _, err = metrics.GetFloat64Metric(gotMetrics, "host_uptime", map[string]string{}, false) + if err != nil { + ginkgo.Fail(fmt.Sprintf("Failed to find uptime metric: %v.\nHere is all NPD exported metrics: %v", + err, gotMetrics)) + } + }) + }) + + ginkgo.AfterEach(func() { + npdLog := instance.RunCommand("sudo journalctl -u node-problem-detector.service") + if npdLog.SSHError != nil || npdLog.Code != 0 { + fmt.Printf("Error fetching NPD logs: %v\n", npdLog) + } else { + fmt.Printf("NPD logs: %v\n", npdLog.Stdout) + } + + err := instance.DeleteInstance() + if err != nil { + ginkgo.Fail(fmt.Sprintf("Failed to clean up the test VM: %v", err)) + } + }) +})