Skip to content

Commit

Permalink
Separate HTTP Instrumentation for Querier (#2708)
Browse files Browse the repository at this point in the history 
* separate instrumentation for the querier

Signed-off-by: Jacob Lisi <[email protected]>

* check for prefix in integration tests

Signed-off-by: Jacob Lisi <[email protected]>

* fix linting issues

Signed-off-by: Jacob Lisi <[email protected]>

* refactor per PR comments

Signed-off-by: Jacob Lisi <[email protected]>

* fix liniting errors

Signed-off-by: Jacob Lisi <[email protected]>

* update the changelog

Signed-off-by: Jacob Lisi <[email protected]>

* make registerQueryAPI private

Signed-off-by: Jacob Lisi <[email protected]>
  • Loading branch information
@jtlisi
jtlisi authored Jun 17, 2020
1 parent 34b45d1 commit e3221b2
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 27 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
* [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502
* [FEATURE] The flag `-frontend.max-cache-freshness` is now supported within the limits overrides, to specify per-tenant max cache freshness values. The corresponding YAML config parameter has been changed from `results_cache.max_freshness` to `limits_config.max_cache_freshness`. The legacy YAML config parameter (`results_cache.max_freshness`) will continue to be supported till Cortex release `v1.4.0`. #2609
* [FEATURE] Experimental gRPC Store: Added support to 3rd parties index and chunk stores using gRPC client/server plugin mechanism. #2220
* [ENHANCEMENT] Querier: Added metric `cortex_querier_request_duration_seconds` for all requests to the querier. #2708
* [ENHANCEMENT] Experimental TSDB: added the following metrics to the ingester: #2580 #2583 #2589 #2654
* `cortex_ingester_tsdb_appender_add_duration_seconds`
* `cortex_ingester_tsdb_appender_commit_duration_seconds`
Expand Down
2 changes: 1 addition & 1 deletion integration/asserts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ var (
serviceMetricsPrefixes = map[ServiceType][]string{
Distributor: {},
Ingester: {"!cortex_ingester_client", "cortex_ingester"}, // The metrics prefix cortex_ingester_client may be used by other components so we ignore it.
Querier: {},
Querier: {"cortex_querier"},
QueryFrontend: {"cortex_frontend", "cortex_query_frontend"},
TableManager: {},
AlertManager: {"cortex_alertmanager"},
Expand Down
66 changes: 41 additions & 25 deletions pkg/api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (

"github.com/go-kit/kit/log"
"github.com/go-kit/kit/log/level"
"github.com/gorilla/mux"
"github.com/prometheus/common/route"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/promql"
Expand All @@ -22,8 +23,6 @@ import (
"github.com/weaveworks/common/middleware"
"github.com/weaveworks/common/server"

"github.com/gorilla/mux"

"github.com/cortexproject/cortex/pkg/alertmanager"
"github.com/cortexproject/cortex/pkg/chunk/purger"
"github.com/cortexproject/cortex/pkg/compactor"
Expand Down Expand Up @@ -252,7 +251,14 @@ func (a *API) RegisterCompactor(c *compactor.Compactor) {
// RegisterQuerier registers the Prometheus routes supported by the
// Cortex querier service. Currently this can not be registered simultaneously
// with the QueryFrontend.
func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine, distributor *distributor.Distributor, registerRoutesExternally bool, tombstonesLoader *purger.TombstonesLoader) http.Handler {
func (a *API) RegisterQuerier(
queryable storage.Queryable,
engine *promql.Engine,
distributor *distributor.Distributor,
registerRoutesExternally bool,
tombstonesLoader *purger.TombstonesLoader,
querierRequestDuration *prometheus.HistogramVec,
) http.Handler {
api := v1.NewAPI(
engine,
queryable,
Expand Down Expand Up @@ -288,10 +294,17 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine
router = a.server.HTTP
}

// Use a separate metric for the querier in order to differentiate requests from the query-frontend when
// running Cortex as a single binary.
inst := middleware.Instrument{
Duration: querierRequestDuration,
RouteMatcher: router,
}

promRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.PrometheusHTTPPrefix + "/api/v1")
api.Register(promRouter)
cacheGenHeaderMiddleware := getHTTPCacheGenNumberHeaderSetterMiddleware(tombstonesLoader)
promHandler := fakeRemoteAddr(cacheGenHeaderMiddleware.Wrap(promRouter))
promHandler := fakeRemoteAddr(inst.Wrap(cacheGenHeaderMiddleware.Wrap(promRouter)))

a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "POST")
a.registerRouteWithRouter(router, a.cfg.PrometheusHTTPPrefix+"/api/v1/query", promHandler, true, "GET", "POST")
Expand All @@ -305,7 +318,7 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine

legacyPromRouter := route.New().WithPrefix(a.cfg.ServerPrefix + a.cfg.LegacyHTTPPrefix + "/api/v1")
api.Register(legacyPromRouter)
legacyPromHandler := fakeRemoteAddr(cacheGenHeaderMiddleware.Wrap(legacyPromRouter))
legacyPromHandler := fakeRemoteAddr(inst.Wrap(cacheGenHeaderMiddleware.Wrap(legacyPromRouter)))

a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/read", querier.RemoteReadHandler(queryable), true, "POST")
a.registerRouteWithRouter(router, a.cfg.LegacyHTTPPrefix+"/api/v1/query", legacyPromHandler, true, "GET", "POST")
Expand All @@ -331,31 +344,34 @@ func (a *API) RegisterQuerier(queryable storage.Queryable, engine *promql.Engine
}))
}

// registerQueryAPI registers the Prometheus routes supported by the
// Cortex querier service. Currently this can not be registered simultaneously
// with the Querier.
func (a *API) registerQueryAPI(handler http.Handler) {
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/read", handler, true, "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query", handler, true, "GET", "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", handler, true, "GET", "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", handler, true, "GET", "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", handler, true, "GET")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/series", handler, true, "GET", "POST", "DELETE")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", handler, true, "GET")

// Register Legacy Routers
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/read", handler, true, "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query", handler, true, "GET", "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", handler, true, "GET", "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/labels", handler, true, "GET", "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", handler, true, "GET")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/series", handler, true, "GET", "POST", "DELETE")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", handler, true, "GET")
}

// RegisterQueryFrontend registers the Prometheus routes supported by the
// Cortex querier service. Currently this can not be registered simultaneously
// with the Querier.
func (a *API) RegisterQueryFrontend(f *frontend.Frontend) {
frontend.RegisterFrontendServer(a.server.GRPC, f)

// Previously the frontend handled all calls to the provided prefix. Instead explicit
// routing is used since it will be required to enable the frontend to be run as part
// of a single binary in the future.
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/read", f.Handler(), true, "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query", f.Handler(), true, "GET", "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/query_range", f.Handler(), true, "GET", "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/labels", f.Handler(), true, "GET", "POST")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/label/{name}/values", f.Handler(), true, "GET")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/series", f.Handler(), true, "GET", "POST", "DELETE")
a.RegisterRoute(a.cfg.PrometheusHTTPPrefix+"/api/v1/metadata", f.Handler(), true, "GET")

// Register Legacy Routers
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/read", f.Handler(), true, "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query", f.Handler(), true, "GET", "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/query_range", f.Handler(), true, "GET", "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/labels", f.Handler(), true, "GET", "POST")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/label/{name}/values", f.Handler(), true, "GET")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/series", f.Handler(), true, "GET", "POST", "DELETE")
a.RegisterRoute(a.cfg.LegacyHTTPPrefix+"/api/v1/metadata", f.Handler(), true, "GET")
a.registerQueryAPI(f.Handler())
}

// RegisterServiceMapHandler registers the Cortex structs service handler
Expand Down
12 changes: 11 additions & 1 deletion pkg/cortex/modules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import (

"github.com/go-kit/kit/log/level"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/prometheus/promql"
httpgrpc_server "github.com/weaveworks/common/httpgrpc/server"
"github.com/weaveworks/common/instrument"
"github.com/weaveworks/common/server"

"github.com/cortexproject/cortex/pkg/alertmanager"
Expand Down Expand Up @@ -162,9 +164,17 @@ func (t *Cortex) initDistributor() (serv services.Service, err error) {
func (t *Cortex) initQuerier() (serv services.Service, err error) {
queryable, engine := querier.New(t.Cfg.Querier, t.Distributor, t.StoreQueryable, t.TombstonesLoader, prometheus.DefaultRegisterer)

// Prometheus histograms for requests to the querier.
querierRequestDuration := promauto.With(prometheus.DefaultRegisterer).NewHistogramVec(prometheus.HistogramOpts{
Namespace: "cortex",
Name: "querier_request_duration_seconds",
Help: "Time (in seconds) spent serving HTTP requests to the querier.",
Buckets: instrument.DefBuckets,
}, []string{"method", "route", "status_code", "ws"})

// if we are not configured for single binary mode then the querier needs to register its paths externally
registerExternally := t.Cfg.Target != All
handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader)
handler := t.API.RegisterQuerier(queryable, engine, t.Distributor, registerExternally, t.TombstonesLoader, querierRequestDuration)

// single binary mode requires a properly configured worker. if the operator did not attempt to configure the
// worker we will attempt an automatic configuration here
Expand Down

0 comments on commit e3221b2

Please sign in to comment.