Skip to content

Commit

Permalink
Merge pull request #7278 from Johnny-Three/fixlocaltopobrokeshowdatab…
Browse files Browse the repository at this point in the history
…asehang

fix hang when local topo was down
  • Loading branch information
deepthi authored Jan 21, 2021
2 parents 4264302 + ebd8c60 commit 3ddbad4
Showing 1 changed file with 7 additions and 5 deletions.
12 changes: 7 additions & 5 deletions go/vt/srvtopo/resilient_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ var (
// setting the watch fails, we will use the last known value until
// srv_topo_cache_ttl elapses and we only try to re-establish the watch
// once every srv_topo_cache_refresh interval.
srvTopoTimeout = flag.Duration("srv_topo_timeout", 1*time.Second, "topo server timeout")
srvTopoCacheTTL = flag.Duration("srv_topo_cache_ttl", 1*time.Second, "how long to use cached entries for topology")
srvTopoCacheRefresh = flag.Duration("srv_topo_cache_refresh", 1*time.Second, "how frequently to refresh the topology for cached entries")
)
Expand Down Expand Up @@ -288,9 +289,9 @@ func (server *ResilientServer) GetSrvKeyspaceNames(ctx context.Context, cell str
log.Errorf("GetSrvKeyspaceNames uncaught panic, cell :%v, err :%v)", cell, err)
}
}()

result, err := server.topoServer.GetSrvKeyspaceNames(ctx, cell)

newCtx, cancel := context.WithTimeout(ctx, *srvTopoTimeout)
defer cancel()
result, err := server.topoServer.GetSrvKeyspaceNames(newCtx, cell)
entry.mutex.Lock()
defer func() {
close(entry.refreshingChan)
Expand All @@ -308,7 +309,8 @@ func (server *ResilientServer) GetSrvKeyspaceNames(ctx context.Context, cell str
server.counts.Add(errorCategory, 1)
if entry.insertionTime.IsZero() {
log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (no cached value, caching and returning error)", ctx, cell, err)

} else if newCtx.Err() == context.DeadlineExceeded {
log.Errorf("GetSrvKeyspaceNames(%v, %v) failed: %v (request timeout), (keeping cached value: %v)", ctx, cell, err, entry.value)
} else if entry.value != nil && time.Since(entry.insertionTime) < server.cacheTTL {
server.counts.Add(cachedCategory, 1)
log.Warningf("GetSrvKeyspaceNames(%v, %v) failed: %v (keeping cached value: %v)", ctx, cell, err, entry.value)
Expand All @@ -320,7 +322,7 @@ func (server *ResilientServer) GetSrvKeyspaceNames(ctx context.Context, cell str
}

entry.lastError = err
entry.lastErrorCtx = ctx
entry.lastErrorCtx = newCtx
}()
}

Expand Down

0 comments on commit 3ddbad4

Please sign in to comment.