Skip to content

Commit

Permalink
Cache default config as map in API component
Browse files Browse the repository at this point in the history
  • Loading branch information
@chases2
chases2 committed Dec 1, 2022
1 parent 493210f commit 2fba485
Show file tree
Hide file tree
Showing 9 changed files with 423 additions and 149 deletions.
2 changes: 2 additions & 0 deletions config/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ import (
"github.com/GoogleCloudPlatform/testgrid/util/gcs"
)

// TODO(slchase): remove
// This "eventual consistency" isn't as useful for the utilities that are using Read() or ReadGCS(). Complex cases are handled by the snapshot library.
var (
cache map[string]Config
cacheLock sync.RWMutex
Expand Down
1 change: 1 addition & 0 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func (e ValidationError) Error() string {
}

// Normalize lowercases, and removes all non-alphanumeric characters from a string.
// WARNING: Unless you are validating config or sanitizing API input, avoid using normalization. Bare names are acceptable keys.
func Normalize(s string) string {
regex := regexp.MustCompile("[^a-zA-Z0-9]+")
s = regex.ReplaceAllString(s, "")
Expand Down
4 changes: 4 additions & 0 deletions pkg/api/v1/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go_library(
name = "go_default_library",
srcs = [
"config.go",
"config_cache.go",
"json.go",
"server.go",
"server_fake.go",
Expand Down Expand Up @@ -44,17 +45,20 @@ filegroup(
go_test(
name = "go_default_test",
srcs = [
"config_cache_test.go",
"config_http_test.go",
"config_test.go",
"state_test.go",
],
embed = [":go_default_library"],
deps = [
"//config/snapshot:go_default_library",
"//pb/api/v1:go_default_library",
"//pb/config:go_default_library",
"//pb/state:go_default_library",
"//util/gcs:go_default_library",
"@com_github_google_go_cmp//cmp:go_default_library",
"@com_github_google_go_cmp//cmp/cmpopts:go_default_library",
"@org_golang_google_protobuf//testing/protocmp:go_default_library",
"@org_golang_google_protobuf//types/known/timestamppb:go_default_library",
],
Expand Down
144 changes: 60 additions & 84 deletions pkg/api/v1/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package v1

import (
"context"
"errors"
"fmt"
"net/http"
"sort"
Expand All @@ -28,25 +27,11 @@ import (
"github.com/sirupsen/logrus"

"github.com/GoogleCloudPlatform/testgrid/config"
"github.com/GoogleCloudPlatform/testgrid/config/snapshot"
apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1"
"github.com/GoogleCloudPlatform/testgrid/util/gcs"
)

const scopeParam = "scope"

func (s *Server) configPath(scope string) (path *gcs.Path, isDefault bool, err error) {
if scope != "" {
path, err = gcs.NewPath(fmt.Sprintf("%s/%s", scope, "config"))
return path, false, err
}
if s.DefaultBucket != "" {
path, err = gcs.NewPath(fmt.Sprintf("%s/%s", s.DefaultBucket, "config"))
return path, true, err
}
return nil, false, errors.New("no testgrid scope")
}

// queryParams returns only the query parameters in the request that need to be passed through to links
func queryParams(scope string) string {
if scope != "" {
Expand All @@ -55,37 +40,17 @@ func queryParams(scope string) string {
return ""
}

// getConfig will return a config or an error.
// Does not expose wrapped errors to the user, instead logging them to the console.
func (s *Server) getConfig(ctx context.Context, log *logrus.Entry, scope string) (*snapshot.Config, error) {
configPath, isDefault, err := s.configPath(scope)
if err != nil || configPath == nil {
return nil, errors.New("Scope not specified")
}

// TODO(chases2): If is default, keep a cached snapshot that we are regularly observing.
// In addition, keep a map of normalized inputs -> real name to santize user input
cfgChan, err := snapshot.Observe(ctx, log, s.Client, *configPath, nil)
if err != nil {
// Only log an error if we set and use a default scope, but can't access it.
// Otherwise, invalid requests will write useless logs.
if isDefault {
log.WithError(err).Errorf("Can't read default config at %q; check permissions", configPath.String())
}
return nil, fmt.Errorf("Could not read config at %q", configPath.String())
}
return <-cfgChan, nil
}

// ListDashboardGroup returns every dashboard group in TestGrid
func (s *Server) ListDashboardGroup(ctx context.Context, req *apipb.ListDashboardGroupRequest) (*apipb.ListDashboardGroupResponse, error) {
cfg, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
c, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
if err != nil {
return nil, err
}
c.Mutex.RLock()
defer c.Mutex.RUnlock()

var resp apipb.ListDashboardGroupResponse
for name := range cfg.DashboardGroups {
for name := range c.Config.DashboardGroups {
rsc := apipb.Resource{
Name: name,
Link: fmt.Sprintf("/dashboard-groups/%s%s", config.Normalize(name), queryParams(req.GetScope())),
Expand Down Expand Up @@ -118,28 +83,31 @@ func (s Server) ListDashboardGroupHTTP(w http.ResponseWriter, r *http.Request) {

// GetDashboardGroup returns a given dashboard group
func (s *Server) GetDashboardGroup(ctx context.Context, req *apipb.GetDashboardGroupRequest) (*apipb.GetDashboardGroupResponse, error) {
cfg, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
c, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
if err != nil {
return nil, err
}

dashboardGroupKey := config.Normalize(req.GetDashboardGroup())
for _, group := range cfg.DashboardGroups {
if config.Normalize(group.Name) == dashboardGroupKey {
result := apipb.GetDashboardGroupResponse{}
for _, dash := range group.DashboardNames {
rsc := apipb.Resource{
Name: dash,
Link: fmt.Sprintf("/dashboards/%s%s", config.Normalize(dash), queryParams(req.GetScope())),
}
result.Dashboards = append(result.Dashboards, &rsc)
c.Mutex.RLock()
defer c.Mutex.RUnlock()

dashGroupName := c.NormalDashboardGroup[config.Normalize(req.GetDashboardGroup())]
group := c.Config.DashboardGroups[dashGroupName]

if group != nil {
result := apipb.GetDashboardGroupResponse{}
for _, dash := range group.DashboardNames {
rsc := apipb.Resource{
Name: dash,
Link: fmt.Sprintf("/dashboards/%s%s", config.Normalize(dash), queryParams(req.GetScope())),
}
sort.SliceStable(result.Dashboards, func(i, j int) bool {
return result.Dashboards[i].Name < result.Dashboards[j].Name
})
return &result, nil
result.Dashboards = append(result.Dashboards, &rsc)
}
sort.SliceStable(result.Dashboards, func(i, j int) bool {
return result.Dashboards[i].Name < result.Dashboards[j].Name
})
return &result, nil
}

return nil, fmt.Errorf("Dashboard group %q not found", req.GetDashboardGroup())
}

Expand All @@ -162,13 +130,15 @@ func (s Server) GetDashboardGroupHTTP(w http.ResponseWriter, r *http.Request) {

// ListDashboard returns every dashboard in TestGrid
func (s *Server) ListDashboard(ctx context.Context, req *apipb.ListDashboardRequest) (*apipb.ListDashboardResponse, error) {
cfg, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
c, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
if err != nil {
return nil, err
}
c.Mutex.RLock()
defer c.Mutex.RUnlock()

var resp apipb.ListDashboardResponse
for name := range cfg.Dashboards {
for name := range c.Config.Dashboards {
rsc := apipb.Resource{
Name: name,
Link: fmt.Sprintf("/dashboards/%s%s", config.Normalize(name), queryParams(req.GetScope())),
Expand Down Expand Up @@ -200,23 +170,26 @@ func (s Server) ListDashboardsHTTP(w http.ResponseWriter, r *http.Request) {

// GetDashboard returns a given dashboard
func (s *Server) GetDashboard(ctx context.Context, req *apipb.GetDashboardRequest) (*apipb.GetDashboardResponse, error) {
cfg, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
c, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
if err != nil {
return nil, err
}

dashboardKey := config.Normalize(req.GetDashboard())
for _, dashboard := range cfg.Dashboards {
if config.Normalize(dashboard.Name) == dashboardKey {
result := apipb.GetDashboardResponse{
DefaultTab: dashboard.DefaultTab,
HighlightToday: dashboard.HighlightToday,
SuppressFailingTabs: dashboard.DownplayFailingTabs,
Notifications: dashboard.Notifications,
}
return &result, nil
c.Mutex.RLock()
defer c.Mutex.RUnlock()

dashboardName := c.NormalDashboard[config.Normalize(req.GetDashboard())]
dashboard := c.Config.Dashboards[dashboardName]

if dashboard != nil {
result := apipb.GetDashboardResponse{
DefaultTab: dashboard.DefaultTab,
HighlightToday: dashboard.HighlightToday,
SuppressFailingTabs: dashboard.DownplayFailingTabs,
Notifications: dashboard.Notifications,
}
return &result, nil
}

return nil, fmt.Errorf("Dashboard %q not found", req.GetDashboard())
}

Expand All @@ -239,27 +212,30 @@ func (s Server) GetDashboardHTTP(w http.ResponseWriter, r *http.Request) {

// ListDashboardTabs returns a given dashboard tabs
func (s *Server) ListDashboardTabs(ctx context.Context, req *apipb.ListDashboardTabsRequest) (*apipb.ListDashboardTabsResponse, error) {
cfg, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
c, err := s.getConfig(ctx, logrus.WithContext(ctx), req.GetScope())
if err != nil {
return nil, err
}
c.Mutex.RLock()
defer c.Mutex.RUnlock()

dashboardName := c.NormalDashboard[config.Normalize((req.GetDashboard()))]
dashboard := c.Config.Dashboards[dashboardName]

if dashboard != nil {
var dashboardTabsResponse apipb.ListDashboardTabsResponse

dashboardKey := config.Normalize((req.GetDashboard()))
var dashboardTabsResponse apipb.ListDashboardTabsResponse
for _, dashboard := range cfg.Dashboards {
if config.Normalize(dashboard.Name) == dashboardKey {
for _, tab := range dashboard.DashboardTab {
rsc := apipb.Resource{
Name: tab.Name,
Link: fmt.Sprintf("/dashboards/%s/tabs/%s%s", config.Normalize(dashboard.Name), config.Normalize(tab.Name), queryParams(req.GetScope())),
}
dashboardTabsResponse.DashboardTabs = append(dashboardTabsResponse.DashboardTabs, &rsc)
for _, tab := range dashboard.DashboardTab {
rsc := apipb.Resource{
Name: tab.Name,
Link: fmt.Sprintf("/dashboards/%s/tabs/%s%s", config.Normalize(dashboard.Name), config.Normalize(tab.Name), queryParams(req.GetScope())),
}
sort.SliceStable(dashboardTabsResponse.DashboardTabs, func(i, j int) bool {
return dashboardTabsResponse.DashboardTabs[i].Name < dashboardTabsResponse.DashboardTabs[j].Name
})
return &dashboardTabsResponse, nil
dashboardTabsResponse.DashboardTabs = append(dashboardTabsResponse.DashboardTabs, &rsc)
}
sort.SliceStable(dashboardTabsResponse.DashboardTabs, func(i, j int) bool {
return dashboardTabsResponse.DashboardTabs[i].Name < dashboardTabsResponse.DashboardTabs[j].Name
})
return &dashboardTabsResponse, nil
}
return nil, fmt.Errorf("Dashboard %q not found", req.GetDashboard())
}
Expand Down
Loading

0 comments on commit 2fba485

Please sign in to comment.