From f53dba45d51bd6750197e2394b5eaece11e8e067 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Wed, 30 Mar 2022 10:58:58 -0700 Subject: [PATCH 1/4] metrics: emit stats for vault token next_renewal & last_renewal #5222 --- nomad/vault.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/nomad/vault.go b/nomad/vault.go index c5012fa9b07..be050f8bd43 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -151,6 +151,14 @@ type VaultStats struct { // TokenExpiry is the recorded expiry time of the current token TokenExpiry time.Time + + // LastRenewalTime is the time since the token was last renewed + LastRenewalTime time.Time + TimeFromLastRenewal time.Duration + + // NextRenewalTime is the time the token will attempt to renew + NextRenewalTime time.Time + TimeToNextRenewal time.Duration } // PurgeVaultAccessorFn is called to remove VaultAccessors from the system. If @@ -232,6 +240,9 @@ type vaultClient struct { // currentExpiration is the time the current token lease expires currentExpiration time.Time currentExpirationLock sync.Mutex + lastRenewalTime time.Time + nextRenewalTime time.Time + nextRenewalTimeLock sync.Mutex tomb *tomb.Tomb logger log.Logger @@ -557,6 +568,9 @@ func (v *vaultClient) renewalLoop() { if err == nil { // Attempt to renew the token at half the expiration time durationUntilRenew := time.Until(currentExpiration) / 2 + v.nextRenewalTimeLock.Lock() + v.nextRenewalTime = time.Now().Add(durationUntilRenew) + v.nextRenewalTimeLock.Unlock() v.logger.Info("successfully renewed token", "next_renewal", durationUntilRenew) authRenewTimer.Reset(durationUntilRenew) @@ -587,6 +601,9 @@ func (v *vaultClient) renewalLoop() { } durationUntilRetry := time.Duration(backoff) * time.Second + v.nextRenewalTimeLock.Lock() + v.nextRenewalTime = time.Now().Add(durationUntilRetry) + v.nextRenewalTimeLock.Unlock() v.logger.Info("backing off renewal", "retry", durationUntilRetry) authRenewTimer.Reset(durationUntilRetry) @@ -1391,15 +1408,27 @@ func (v *vaultClient) Stats() map[string]string { stat := v.stats() expireTimeStr := "" - if !stat.TokenExpiry.IsZero() { expireTimeStr = stat.TokenExpiry.Format(time.RFC3339) } + lastRenewTimeStr := "" + if !stat.LastRenewalTime.IsZero() { + lastRenewTimeStr = stat.LastRenewalTime.Format(time.RFC3339) + } + + nextRenewTimeStr := "" + if !stat.NextRenewalTime.IsZero() { + nextRenewTimeStr = stat.NextRenewalTime.Format(time.RFC3339) + } + + return map[string]string{ "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), "token_ttl": stat.TokenTTL.Round(time.Second).String(), "token_expire_time": expireTimeStr, + "last_renewal_time": lastRenewTimeStr, + "next_renewal_time": nextRenewTimeStr, } } @@ -1413,12 +1442,24 @@ func (v *vaultClient) stats() *VaultStats { v.currentExpirationLock.Lock() stats.TokenExpiry = v.currentExpiration + stats.LastRenewalTime = v.lastRenewalTime v.currentExpirationLock.Unlock() + v.nextRenewalTimeLock.Lock() + stats.NextRenewalTime = v.nextRenewalTime + v.nextRenewalTimeLock.Unlock() + if !stats.TokenExpiry.IsZero() { stats.TokenTTL = time.Until(stats.TokenExpiry) } + if !stats.LastRenewalTime.IsZero() { + stats.TimeFromLastRenewal = time.Since(stats.LastRenewalTime) + } + if !stats.NextRenewalTime.IsZero() { + stats.TimeToNextRenewal = time.Until(stats.NextRenewalTime) + } + return stats } @@ -1435,6 +1476,8 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) { stats := v.stats() metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke)) metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(stats.TokenTTL/time.Millisecond)) + metrics.SetGauge([]string{"nomad", "vault", "last_renewal"}, float32(stats.TimeFromLastRenewal/time.Millisecond)) + metrics.SetGauge([]string{"nomad", "vault", "next_renewal"}, float32(stats.TimeToNextRenewal/time.Millisecond)) case <-stopCh: return @@ -1445,7 +1488,9 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) { // extendExpiration sets the current auth token expiration record to ttLSeconds seconds from now func (v *vaultClient) extendExpiration(ttlSeconds int) { v.currentExpirationLock.Lock() - v.currentExpiration = time.Now().Add(time.Duration(ttlSeconds) * time.Second) + now := time.Now() + v.currentExpiration = now.Add(time.Duration(ttlSeconds) * time.Second) + v.lastRenewalTime = now v.currentExpirationLock.Unlock() } From a7d9cd39463bb6d452c3bec67cdb62b79247c370 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Tue, 5 Apr 2022 16:50:41 -0700 Subject: [PATCH 2/4] vault: change token renewal metrics names --- nomad/vault.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nomad/vault.go b/nomad/vault.go index be050f8bd43..b7964141bfd 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -1427,8 +1427,8 @@ func (v *vaultClient) Stats() map[string]string { "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), "token_ttl": stat.TokenTTL.Round(time.Second).String(), "token_expire_time": expireTimeStr, - "last_renewal_time": lastRenewTimeStr, - "next_renewal_time": nextRenewTimeStr, + "token_last_renewal_time": lastRenewTimeStr, + "token_next_renewal_time": nextRenewTimeStr, } } @@ -1476,8 +1476,8 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) { stats := v.stats() metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke)) metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(stats.TokenTTL/time.Millisecond)) - metrics.SetGauge([]string{"nomad", "vault", "last_renewal"}, float32(stats.TimeFromLastRenewal/time.Millisecond)) - metrics.SetGauge([]string{"nomad", "vault", "next_renewal"}, float32(stats.TimeToNextRenewal/time.Millisecond)) + metrics.SetGauge([]string{"nomad", "vault", "token_last_renewal"}, float32(stats.TimeFromLastRenewal/time.Millisecond)) + metrics.SetGauge([]string{"nomad", "vault", "token_next_renewal"}, float32(stats.TimeToNextRenewal/time.Millisecond)) case <-stopCh: return From 750b126c7b512e7fd7d2cf04e3949fb68c8fd5c2 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Tue, 5 Apr 2022 16:52:51 -0700 Subject: [PATCH 3/4] gofmt -s --- nomad/vault.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/nomad/vault.go b/nomad/vault.go index b7964141bfd..cff0fd2947f 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -153,11 +153,11 @@ type VaultStats struct { TokenExpiry time.Time // LastRenewalTime is the time since the token was last renewed - LastRenewalTime time.Time + LastRenewalTime time.Time TimeFromLastRenewal time.Duration // NextRenewalTime is the time the token will attempt to renew - NextRenewalTime time.Time + NextRenewalTime time.Time TimeToNextRenewal time.Duration } @@ -240,9 +240,9 @@ type vaultClient struct { // currentExpiration is the time the current token lease expires currentExpiration time.Time currentExpirationLock sync.Mutex - lastRenewalTime time.Time - nextRenewalTime time.Time - nextRenewalTimeLock sync.Mutex + lastRenewalTime time.Time + nextRenewalTime time.Time + nextRenewalTimeLock sync.Mutex tomb *tomb.Tomb logger log.Logger @@ -1422,13 +1422,12 @@ func (v *vaultClient) Stats() map[string]string { nextRenewTimeStr = stat.NextRenewalTime.Format(time.RFC3339) } - return map[string]string{ - "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), - "token_ttl": stat.TokenTTL.Round(time.Second).String(), - "token_expire_time": expireTimeStr, - "token_last_renewal_time": lastRenewTimeStr, - "token_next_renewal_time": nextRenewTimeStr, + "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), + "token_ttl": stat.TokenTTL.Round(time.Second).String(), + "token_expire_time": expireTimeStr, + "token_last_renewal_time": lastRenewTimeStr, + "token_next_renewal_time": nextRenewTimeStr, } } From 81956e67697c4d7d17f7bdf728b382f37bf56fb9 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Wed, 6 Apr 2022 08:45:27 -0700 Subject: [PATCH 4/4] vault: move lastRenewal to renewLoop --- nomad/vault.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nomad/vault.go b/nomad/vault.go index cff0fd2947f..f900860c4af 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -242,7 +242,7 @@ type vaultClient struct { currentExpirationLock sync.Mutex lastRenewalTime time.Time nextRenewalTime time.Time - nextRenewalTimeLock sync.Mutex + renewalTimeLock sync.Mutex tomb *tomb.Tomb logger log.Logger @@ -568,9 +568,11 @@ func (v *vaultClient) renewalLoop() { if err == nil { // Attempt to renew the token at half the expiration time durationUntilRenew := time.Until(currentExpiration) / 2 - v.nextRenewalTimeLock.Lock() - v.nextRenewalTime = time.Now().Add(durationUntilRenew) - v.nextRenewalTimeLock.Unlock() + v.renewalTimeLock.Lock() + now := time.Now() + v.lastRenewalTime = now + v.nextRenewalTime = now.Add(durationUntilRenew) + v.renewalTimeLock.Unlock() v.logger.Info("successfully renewed token", "next_renewal", durationUntilRenew) authRenewTimer.Reset(durationUntilRenew) @@ -601,9 +603,9 @@ func (v *vaultClient) renewalLoop() { } durationUntilRetry := time.Duration(backoff) * time.Second - v.nextRenewalTimeLock.Lock() + v.renewalTimeLock.Lock() v.nextRenewalTime = time.Now().Add(durationUntilRetry) - v.nextRenewalTimeLock.Unlock() + v.renewalTimeLock.Unlock() v.logger.Info("backing off renewal", "retry", durationUntilRetry) authRenewTimer.Reset(durationUntilRetry) @@ -1441,12 +1443,12 @@ func (v *vaultClient) stats() *VaultStats { v.currentExpirationLock.Lock() stats.TokenExpiry = v.currentExpiration - stats.LastRenewalTime = v.lastRenewalTime v.currentExpirationLock.Unlock() - v.nextRenewalTimeLock.Lock() + v.renewalTimeLock.Lock() stats.NextRenewalTime = v.nextRenewalTime - v.nextRenewalTimeLock.Unlock() + stats.LastRenewalTime = v.lastRenewalTime + v.renewalTimeLock.Unlock() if !stats.TokenExpiry.IsZero() { stats.TokenTTL = time.Until(stats.TokenExpiry) @@ -1487,9 +1489,7 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) { // extendExpiration sets the current auth token expiration record to ttLSeconds seconds from now func (v *vaultClient) extendExpiration(ttlSeconds int) { v.currentExpirationLock.Lock() - now := time.Now() - v.currentExpiration = now.Add(time.Duration(ttlSeconds) * time.Second) - v.lastRenewalTime = now + v.currentExpiration = time.Now().Add(time.Duration(ttlSeconds) * time.Second) v.currentExpirationLock.Unlock() }