From 07377dc761b226909f9cc14f549ff840ecb26785 Mon Sep 17 00:00:00 2001 From: lalalalatt Date: Sat, 6 Jul 2024 19:03:47 +0800 Subject: [PATCH 1/8] feat: add last resolve time metric to kuberesolver --- builder.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/builder.go b/builder.go index 6a8a915..41cb78b 100644 --- a/builder.go +++ b/builder.go @@ -36,6 +36,13 @@ var ( }, []string{"target"}, ) + lastResolveMetric = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kuberesolver_last_resolve_timestamp", + Help: "Time in seconds since the last resolve request to the peer list", + }, + []string{"target"}, + ) ) type targetInfo struct { @@ -158,8 +165,9 @@ func (b *kubeBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts t: time.NewTimer(defaultFreq), freq: defaultFreq, - endpoints: endpointsForTarget.WithLabelValues(ti.String()), - addresses: addressesForTarget.WithLabelValues(ti.String()), + endpoints: endpointsForTarget.WithLabelValues(ti.String()), + addresses: addressesForTarget.WithLabelValues(ti.String()), + lastResolve: lastResolveMetric.WithLabelValues(ti.String()), } go until(func() { r.wg.Add(1) @@ -190,8 +198,11 @@ type kResolver struct { t *time.Timer freq time.Duration - endpoints prometheus.Gauge - addresses prometheus.Gauge + lastResolveTime time.Time + + endpoints prometheus.Gauge + addresses prometheus.Gauge + lastResolve prometheus.Gauge } // ResolveNow will be called by gRPC to try to resolve the target name again. @@ -247,6 +258,8 @@ func (k *kResolver) handle(e Endpoints) { k.cc.UpdateState(resolver.State{ Addresses: addrs, }) + k.lastResolveTime = time.Now() + k.lastResolve.Set(0) } k.endpoints.Set(float64(len(e.Subsets))) @@ -283,6 +296,7 @@ func (k *kResolver) watch() error { if hasMore { k.handle(up.Object) } else { + k.lastResolve.Set(float64(time.Since(k.lastResolveTime).Seconds())) return nil } } From 2d977bf0ab169a0361a2b6f4d00d946368ebbe68 Mon Sep 17 00:00:00 2001 From: lalalalatt Date: Mon, 8 Jul 2024 11:53:52 +0800 Subject: [PATCH 2/8] feat: add ticker for regular resolution updates in kuberesolver --- builder.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builder.go b/builder.go index 41cb78b..e3b37ac 100644 --- a/builder.go +++ b/builder.go @@ -284,6 +284,7 @@ func (k *kResolver) watch() error { if err != nil { return err } + ticker := time.NewTicker(time.Second * 30) for { select { case <-k.ctx.Done(): @@ -296,9 +297,11 @@ func (k *kResolver) watch() error { if hasMore { k.handle(up.Object) } else { - k.lastResolve.Set(float64(time.Since(k.lastResolveTime).Seconds())) return nil } + case <-ticker.C: + k.lastResolve.Set(float64(time.Since(k.lastResolveTime).Seconds())) + ticker.Reset(time.Second * 30) } } } From db81dd77e44d67eb148cf047d577a3555cc186b9 Mon Sep 17 00:00:00 2001 From: lalalalatt Date: Mon, 8 Jul 2024 11:57:34 +0800 Subject: [PATCH 3/8] chore: rename `lastResolve` to `resolveLag` --- builder.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/builder.go b/builder.go index e3b37ac..f10e78c 100644 --- a/builder.go +++ b/builder.go @@ -36,9 +36,9 @@ var ( }, []string{"target"}, ) - lastResolveMetric = promauto.NewGaugeVec( + clientResolveLag = promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "kuberesolver_last_resolve_timestamp", + Name: "kuberesolver_last_resolve_seconds", Help: "Time in seconds since the last resolve request to the peer list", }, []string{"target"}, @@ -165,9 +165,9 @@ func (b *kubeBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts t: time.NewTimer(defaultFreq), freq: defaultFreq, - endpoints: endpointsForTarget.WithLabelValues(ti.String()), - addresses: addressesForTarget.WithLabelValues(ti.String()), - lastResolve: lastResolveMetric.WithLabelValues(ti.String()), + endpoints: endpointsForTarget.WithLabelValues(ti.String()), + addresses: addressesForTarget.WithLabelValues(ti.String()), + resolveLag: clientResolveLag.WithLabelValues(ti.String()), } go until(func() { r.wg.Add(1) @@ -200,9 +200,9 @@ type kResolver struct { lastResolveTime time.Time - endpoints prometheus.Gauge - addresses prometheus.Gauge - lastResolve prometheus.Gauge + endpoints prometheus.Gauge + addresses prometheus.Gauge + resolveLag prometheus.Gauge } // ResolveNow will be called by gRPC to try to resolve the target name again. @@ -259,7 +259,7 @@ func (k *kResolver) handle(e Endpoints) { Addresses: addrs, }) k.lastResolveTime = time.Now() - k.lastResolve.Set(0) + k.resolveLag.Set(0) } k.endpoints.Set(float64(len(e.Subsets))) @@ -300,7 +300,7 @@ func (k *kResolver) watch() error { return nil } case <-ticker.C: - k.lastResolve.Set(float64(time.Since(k.lastResolveTime).Seconds())) + k.resolveLag.Set(float64(time.Since(k.lastResolveTime).Seconds())) ticker.Reset(time.Second * 30) } } From 5d975ad0fe6725865e9668c28a467d35278b7029 Mon Sep 17 00:00:00 2001 From: lalalalatt Date: Mon, 8 Jul 2024 12:22:27 +0800 Subject: [PATCH 4/8] feat: add ticker for regular resolution updates in kuberesolver --- builder.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/builder.go b/builder.go index f10e78c..165721d 100644 --- a/builder.go +++ b/builder.go @@ -165,6 +165,8 @@ func (b *kubeBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts t: time.NewTimer(defaultFreq), freq: defaultFreq, + tickerFreq: time.Second * 5, + endpoints: endpointsForTarget.WithLabelValues(ti.String()), addresses: addressesForTarget.WithLabelValues(ti.String()), resolveLag: clientResolveLag.WithLabelValues(ti.String()), @@ -199,6 +201,7 @@ type kResolver struct { freq time.Duration lastResolveTime time.Time + tickerFreq time.Duration endpoints prometheus.Gauge addresses prometheus.Gauge @@ -284,7 +287,7 @@ func (k *kResolver) watch() error { if err != nil { return err } - ticker := time.NewTicker(time.Second * 30) + ticker := time.NewTicker(k.tickerFreq) for { select { case <-k.ctx.Done(): @@ -301,7 +304,7 @@ func (k *kResolver) watch() error { } case <-ticker.C: k.resolveLag.Set(float64(time.Since(k.lastResolveTime).Seconds())) - ticker.Reset(time.Second * 30) + ticker.Reset(k.tickerFreq) } } } From 9c57070c8fd229e379fad9eb315218a6a7daec66 Mon Sep 17 00:00:00 2001 From: lalalalatt Date: Mon, 8 Jul 2024 12:23:02 +0800 Subject: [PATCH 5/8] feat: add test for resolving lag in kuberesolver --- builder_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/builder_test.go b/builder_test.go index fd38afc..325a532 100644 --- a/builder_test.go +++ b/builder_test.go @@ -6,7 +6,10 @@ import ( "net/url" "strings" "testing" + "time" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" ) @@ -69,6 +72,31 @@ func TestBuilder(t *testing.T) { } +func TestResolveLag(t *testing.T) { + bl := newTestBuilder() + fc := &fakeConn{ + cmp: make(chan struct{}), + } + rs, err := bl.Build(parseTarget("kubernetes://kube-dns.kube-system:53"), fc, resolver.BuildOptions{}) + if err != nil { + t.Fatal(err) + } + <-fc.cmp + if len(fc.found) == 0 { + t.Fatal("could not found endpoints") + } + time.Sleep(6 * time.Second) // wait for the ticker to update the resolveLag + + kresolver := rs.(*kResolver) + resolveLagValue := testutil.ToFloat64(kresolver.resolveLag) + assert.Greater(t, resolveLagValue, 0.0) + t.Logf("resolver lag: %v s", resolveLagValue) + + // rs.ResolveNow(resolver.ResolveNowOptions{}) + // <-fc.cmp + // assert.Greater(t, resolveLagValue, testutil.ToFloat64(kresolver.resolveLag)) +} + // copied from grpc package to test parsing endpoints // split2 returns the values from strings.SplitN(s, sep, 2). From 5e57ecaa1d869ea1ed963959e515e5dd6c6bb01f Mon Sep 17 00:00:00 2001 From: lalalalatt Date: Mon, 8 Jul 2024 12:23:45 +0800 Subject: [PATCH 6/8] chore: add `prometheus/testutil` in go mod --- go.mod | 4 ++++ go.sum | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/go.mod b/go.mod index 90131e9..b1afc4b 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,15 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/sys v0.7.0 // indirect google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d563d08..bc43c80 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -13,6 +15,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= @@ -21,6 +25,8 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= @@ -32,3 +38,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 7cd61cb4288d00da15e2ab5f9f69e5df5c60d327 Mon Sep 17 00:00:00 2001 From: lalalalatt Date: Mon, 8 Jul 2024 12:34:15 +0800 Subject: [PATCH 7/8] fix: fix accidentally go mod change --- go.mod | 6 ++++-- go.sum | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index b56c966..87ea7ba 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,14 @@ module github.com/sercand/kuberesolver/v5 -go 1.18 +go 1.21 + +toolchain go1.22.1 require ( github.com/fsnotify/fsnotify v1.6.0 github.com/prometheus/client_golang v1.15.1 github.com/stretchr/testify v1.9.0 - google.golang.org/grpc v1.56.3 + google.golang.org/grpc v1.65.0 ) require ( diff --git a/go.sum b/go.sum index 1976acb..dada16d 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,10 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -31,15 +33,22 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From f349307a5661df8d104303a12a014ebde3bf1b25 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Tue, 9 Jul 2024 11:12:54 +0800 Subject: [PATCH 8/8] feat: update kuberesolver to track the last update time of the resolver client --- builder.go | 31 +++++++++++-------------------- builder_test.go | 12 ++++-------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/builder.go b/builder.go index 66590dd..8e55d24 100644 --- a/builder.go +++ b/builder.go @@ -36,10 +36,10 @@ var ( }, []string{"target"}, ) - clientResolveLag = promauto.NewGaugeVec( + clientLastUpdate = promauto.NewGaugeVec( prometheus.GaugeOpts{ - Name: "kuberesolver_last_resolve_seconds", - Help: "Time in seconds since the last resolve request to the peer list", + Name: "kuberesolver_client_last_update", + Help: "The last time the resolver client was updated", }, []string{"target"}, ) @@ -164,11 +164,9 @@ func (b *kubeBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts t: time.NewTimer(defaultFreq), freq: defaultFreq, - tickerFreq: time.Second * 5, - - endpoints: endpointsForTarget.WithLabelValues(ti.String()), - addresses: addressesForTarget.WithLabelValues(ti.String()), - resolveLag: clientResolveLag.WithLabelValues(ti.String()), + endpoints: endpointsForTarget.WithLabelValues(ti.String()), + addresses: addressesForTarget.WithLabelValues(ti.String()), + lastUpdateUnix: clientLastUpdate.WithLabelValues(ti.String()), } go until(func() { r.wg.Add(1) @@ -197,12 +195,10 @@ type kResolver struct { t *time.Timer freq time.Duration - lastResolveTime time.Time - tickerFreq time.Duration - - endpoints prometheus.Gauge - addresses prometheus.Gauge - resolveLag prometheus.Gauge + endpoints prometheus.Gauge + addresses prometheus.Gauge + // lastUpdateUnix is the timestamp of the last successful update to the resolver client + lastUpdateUnix prometheus.Gauge } // ResolveNow will be called by gRPC to try to resolve the target name again. @@ -255,8 +251,7 @@ func (k *kResolver) handle(e Endpoints) { k.cc.UpdateState(resolver.State{ Addresses: addrs, }) - k.lastResolveTime = time.Now() - k.resolveLag.Set(0) + k.lastUpdateUnix.Set(float64(time.Now().Unix())) } k.endpoints.Set(float64(len(e.Subsets))) @@ -281,7 +276,6 @@ func (k *kResolver) watch() error { if err != nil { return err } - ticker := time.NewTicker(k.tickerFreq) for { select { case <-k.ctx.Done(): @@ -294,9 +288,6 @@ func (k *kResolver) watch() error { } else { return nil } - case <-ticker.C: - k.resolveLag.Set(float64(time.Since(k.lastResolveTime).Seconds())) - ticker.Reset(k.tickerFreq) } } } diff --git a/builder_test.go b/builder_test.go index 325a532..6b9e50a 100644 --- a/builder_test.go +++ b/builder_test.go @@ -85,16 +85,12 @@ func TestResolveLag(t *testing.T) { if len(fc.found) == 0 { t.Fatal("could not found endpoints") } - time.Sleep(6 * time.Second) // wait for the ticker to update the resolveLag + time.Sleep(2 * time.Second) kresolver := rs.(*kResolver) - resolveLagValue := testutil.ToFloat64(kresolver.resolveLag) - assert.Greater(t, resolveLagValue, 0.0) - t.Logf("resolver lag: %v s", resolveLagValue) - - // rs.ResolveNow(resolver.ResolveNowOptions{}) - // <-fc.cmp - // assert.Greater(t, resolveLagValue, testutil.ToFloat64(kresolver.resolveLag)) + clientResolveLag := testutil.ToFloat64(kresolver.lastUpdateUnix) - float64(time.Now().Unix()) + assert.Less(t, clientResolveLag, 0.0) + t.Logf("client resolver lag: %v s", -clientResolveLag) } // copied from grpc package to test parsing endpoints