diff --git a/etcdserver/membership/cluster.go b/etcdserver/membership/cluster.go index 4f0b1572ef6..ba5c5411d5f 100644 --- a/etcdserver/membership/cluster.go +++ b/etcdserver/membership/cluster.go @@ -36,6 +36,7 @@ import ( "github.com/coreos/etcd/version" "github.com/coreos/go-semver/semver" + "github.com/prometheus/client_golang/prometheus" ) // RaftCluster is a list of Members that belong to the same raft cluster @@ -368,6 +369,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version } else { plog.Noticef("set the initial cluster version to %v", version.Cluster(ver.String())) } + oldVer := c.version c.version = ver mustDetectDowngrade(c.version) if c.store != nil { @@ -376,6 +378,10 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*semver.Version if c.be != nil { mustSaveClusterVersionToBackend(c.be, ver) } + if oldVer != nil { + ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0) + } + ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1) onSet(ver) } diff --git a/etcdserver/membership/metrics.go b/etcdserver/membership/metrics.go new file mode 100644 index 00000000000..b3212bc80cd --- /dev/null +++ b/etcdserver/membership/metrics.go @@ -0,0 +1,31 @@ +// Copyright 2018 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package membership + +import "github.com/prometheus/client_golang/prometheus" + +var ( + ClusterVersionMetrics = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "etcd", + Subsystem: "cluster", + Name: "version", + Help: "Which version is running. 1 for 'cluster_version' label with current cluster version", + }, + []string{"cluster_version"}) +) + +func init() { + prometheus.MustRegister(ClusterVersionMetrics) +} diff --git a/etcdserver/server.go b/etcdserver/server.go index f46e91d3469..d2027a78745 100644 --- a/etcdserver/server.go +++ b/etcdserver/server.go @@ -602,6 +602,7 @@ func (s *EtcdServer) start() { s.leaderChanged = make(chan struct{}) if s.ClusterVersion() != nil { plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String())) + membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1) } else { plog.Infof("starting server... [version: %v, cluster version: to_be_decided]", version.Version) } diff --git a/tests/e2e/etcd_release_upgrade_test.go b/tests/e2e/etcd_release_upgrade_test.go index 4b65dad3b22..a5665e10e0f 100644 --- a/tests/e2e/etcd_release_upgrade_test.go +++ b/tests/e2e/etcd_release_upgrade_test.go @@ -103,6 +103,11 @@ func TestReleaseUpgrade(t *testing.T) { } } } + + // expect upgraded cluster version + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } } func TestReleaseUpgradeWithRestart(t *testing.T) { diff --git a/tests/e2e/metrics_test.go b/tests/e2e/metrics_test.go new file mode 100644 index 00000000000..aade2718f15 --- /dev/null +++ b/tests/e2e/metrics_test.go @@ -0,0 +1,59 @@ +// Copyright 2017 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "strings" + "testing" + + "github.com/coreos/etcd/version" +) + +func TestV3MetricsSecure(t *testing.T) { + cfg := configTLS + cfg.clusterSize = 1 + cfg.metricsURLScheme = "https" + testCtl(t, metricsTest) +} + +func TestV3MetricsInsecure(t *testing.T) { + cfg := configTLS + cfg.clusterSize = 1 + cfg.metricsURLScheme = "http" + testCtl(t, metricsTest) +} + +func metricsTest(cx ctlCtx) { + if err := ctlV3Put(cx, "k", "v", ""); err != nil { + cx.t.Fatal(err) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: `etcd_debugging_mvcc_keys_total 1`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_server_version{server_version="%s"} 1`, version.Version), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } + ver := version.Version + if strings.HasSuffix(ver, "+git") { + ver = strings.Replace(ver, "+git", "", 1) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/metrics", expected: fmt.Sprintf(`etcd_cluster_version{cluster_version="%s"} 1`, version.Cluster(version.Version)), metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } + if err := cURLGet(cx.epc, cURLReq{endpoint: "/health", expected: `{"health":"true"}`, metricsURLScheme: cx.cfg.metricsURLScheme}); err != nil { + cx.t.Fatalf("failed get with curl (%v)", err) + } +}