diff --git a/scripts/docker-integration-tests/query_fanout/restrict.go b/scripts/docker-integration-tests/query_fanout/restrict.go new file mode 100644 index 0000000000..39527839a9 --- /dev/null +++ b/scripts/docker-integration-tests/query_fanout/restrict.go @@ -0,0 +1,225 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net/http" + "os" + "runtime" + + "github.com/m3db/m3/src/query/api/v1/handler" + "github.com/m3db/m3/src/query/api/v1/handler/prometheus" + "github.com/m3db/m3/src/query/models" + + "github.com/stretchr/testify/require" +) + +func main() { + var ts int + flag.IntVar(&ts, "t", -1, "metric name to search") + flag.Parse() + + require.True(t, ts > 0, "no timestamp supplied") + name = fmt.Sprintf("foo_%d", ts) + instant := fmt.Sprintf("http://0.0.0.0:7201/api/v1/query?query=%s", name) + rnge := fmt.Sprintf("http://0.0.0.0:7201/api/v1/query_range?query=%s"+ + "&start=%d&end=%d&step=100", name, ts/100*100, (ts/100+1)*100) + + for _, url := range []string{instant, rnge} { + singleClusterDefaultStrip(url) + bothClusterCustomStrip(url) + bothClusterDefaultStrip(url) + bothClusterNoStrip(url) + bothClusterMultiStrip(url) + } +} + +func queryWithHeader(url string, h string) (prometheus.Response, error) { + var result prometheus.Response + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return result, err + } + + req.Header.Add(handler.RestrictByTagsJSONHeader, h) + client := http.DefaultClient + resp, err := client.Do(req) + if err != nil { + return result, err + } + + if resp.StatusCode != http.StatusOK { + return result, fmt.Errorf("response failed with code %s", resp.Status) + } + + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return result, err + } + + json.Unmarshal(data, &result) + return result, err +} + +func mustMatcher(t models.MatchType, n string, v string) models.Matcher { + m, err := models.NewMatcher(models.MatchEqual, []byte("val"), []byte("1")) + if err != nil { + panic(err) + } + + return m +} + +type tester struct{} + +// Ensure tester is a TestingT and set a global `t`. +var t require.TestingT = &tester{} + +// name is global and set on startup. +var name string + +func (t *tester) Errorf(format string, args ...interface{}) { + _, fn, line, _ := runtime.Caller(4) + args[2] = fmt.Sprintf(" at %s:%d:\n%v", fn, line, args[2]) + fmt.Printf(format, args...) +} + +func (t *tester) FailNow() { + os.Exit(1) +} + +func mustParseOpts(o handler.StringTagOptions) string { + m, err := json.Marshal(o) + require.NoError(t, err, "cannot marshal to json") + return string(m) +} + +func bothClusterDefaultStrip(url string) { + m := mustParseOpts(handler.StringTagOptions{ + Restrict: []handler.StringMatch{ + handler.StringMatch{Name: "val", Type: "EQUAL", Value: "1"}, + }, + }) + + resp, err := queryWithHeader(url, string(m)) + require.NoError(t, err, "failed to query") + + data := resp.Data.Result + data.Sort() + require.Equal(t, len(data), 2) + clusters := []string{"coordinator-cluster-a", "coordinator-cluster-b"} + for i, d := range data { + require.Equal(t, 2, len(d.Metric)) + require.Equal(t, name, d.Metric["__name__"]) + require.Equal(t, clusters[i], d.Metric["cluster"]) + } +} + +func bothClusterCustomStrip(url string) { + m := mustParseOpts(handler.StringTagOptions{ + Restrict: []handler.StringMatch{ + handler.StringMatch{Name: "val", Type: "EQUAL", Value: "1"}, + }, + Strip: []string{"__name__"}, + }) + + resp, err := queryWithHeader(url, string(m)) + require.NoError(t, err, "failed to query") + + data := resp.Data.Result + data.Sort() + require.Equal(t, len(data), 2) + clusters := []string{"coordinator-cluster-a", "coordinator-cluster-b"} + for i, d := range data { + require.Equal(t, 2, len(d.Metric)) + require.Equal(t, clusters[i], d.Metric["cluster"]) + require.Equal(t, "1", d.Metric["val"]) + } +} + +func bothClusterNoStrip(url string) { + m := mustParseOpts(handler.StringTagOptions{ + Restrict: []handler.StringMatch{ + handler.StringMatch{Name: "val", Type: "EQUAL", Value: "1"}, + }, + Strip: []string{}, + }) + + resp, err := queryWithHeader(url, string(m)) + require.NoError(t, err, "failed to query") + + data := resp.Data.Result + data.Sort() + require.Equal(t, len(data), 2) + clusters := []string{"coordinator-cluster-a", "coordinator-cluster-b"} + for i, d := range data { + require.Equal(t, 3, len(d.Metric)) + require.Equal(t, name, d.Metric["__name__"]) + require.Equal(t, clusters[i], d.Metric["cluster"]) + require.Equal(t, "1", d.Metric["val"]) + } +} + +func bothClusterMultiStrip(url string) { + m := mustParseOpts(handler.StringTagOptions{ + Restrict: []handler.StringMatch{ + handler.StringMatch{Name: "val", Type: "EQUAL", Value: "1"}, + }, + Strip: []string{"val", "__name__"}, + }) + + resp, err := queryWithHeader(url, string(m)) + require.NoError(t, err, "failed to query") + + data := resp.Data.Result + data.Sort() + require.Equal(t, len(data), 2) + clusters := []string{"coordinator-cluster-a", "coordinator-cluster-b"} + for i, d := range data { + require.Equal(t, 1, len(d.Metric)) + require.Equal(t, clusters[i], d.Metric["cluster"]) + } +} + +// NB: cluster 1 is expected to have metrics with vals in range: [1,5] +// and cluster 2 is expected to have metrics with vals in range: [1,10] +// so setting the value to be in (5..10] should hit only a single metric. +func singleClusterDefaultStrip(url string) { + m := mustParseOpts(handler.StringTagOptions{ + Restrict: []handler.StringMatch{ + handler.StringMatch{Name: "val", Type: "EQUAL", Value: "9"}, + }, + }) + + resp, err := queryWithHeader(url, string(m)) + require.NoError(t, err, "failed to query") + + data := resp.Data.Result + require.Equal(t, len(data), 1, url) + require.Equal(t, 2, len(data[0].Metric)) + require.Equal(t, name, data[0].Metric["__name__"], "single") + require.Equal(t, "coordinator-cluster-b", data[0].Metric["cluster"]) +} diff --git a/scripts/docker-integration-tests/query_fanout/restrict.sh b/scripts/docker-integration-tests/query_fanout/restrict.sh new file mode 100755 index 0000000000..e648c572c7 --- /dev/null +++ b/scripts/docker-integration-tests/query_fanout/restrict.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ex +TEST_PATH=$GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests +FANOUT_PATH=$TEST_PATH/query_fanout +source $TEST_PATH/common.sh +source $FANOUT_PATH/warning.sh + +function test_restrictions { + t=$(date +%s) + METRIC_NAME="foo_$t" + # # write 5 metrics to cluster a + write_metrics coordinator-cluster-a 5 + # write 10 metrics to cluster b + write_metrics coordinator-cluster-b 10 + + # unlimited query against cluster a has no header + ATTEMPTS=3 TIMEOUT=1 retry_with_backoff go run $FANOUT_PATH/restrict.go -t $t +} diff --git a/scripts/docker-integration-tests/query_fanout/test.sh b/scripts/docker-integration-tests/query_fanout/test.sh index 5d1f3ef905..5a4321e2dc 100755 --- a/scripts/docker-integration-tests/query_fanout/test.sh +++ b/scripts/docker-integration-tests/query_fanout/test.sh @@ -2,8 +2,11 @@ set -xe -source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/common.sh -source $GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/query_fanout/warning.sh +TEST_PATH=$GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests +FANOUT_PATH=$TEST_PATH/query_fanout +source $TEST_PATH/common.sh +source $FANOUT_PATH/warning.sh +source $FANOUT_PATH/restrict.sh REVISION=$(git rev-parse HEAD) COMPOSE_FILE=$GOPATH/src/github.com/m3db/m3/scripts/docker-integration-tests/query_fanout/docker-compose.yml @@ -216,3 +219,6 @@ ATTEMPTS=5 TIMEOUT=1 retry_with_backoff complete_tags echo "running fanout warning tests" test_fanout_warnings + +echo "running restrict tests" +test_restrictions diff --git a/scripts/docker-integration-tests/query_fanout/warning.sh b/scripts/docker-integration-tests/query_fanout/warning.sh index c2ed7dc358..dc12a5a398 100755 --- a/scripts/docker-integration-tests/query_fanout/warning.sh +++ b/scripts/docker-integration-tests/query_fanout/warning.sh @@ -337,6 +337,8 @@ function test_fanout_warning_missing_zone { ATTEMPTS=3 TIMEOUT=1 retry_with_backoff find_carbon 16 remote_store_cluster-c_complete_tags_warning ATTEMPTS=3 TIMEOUT=1 retry_with_backoff find_carbon 9 max_fetch_series_limit_applied,remote_store_cluster-c_complete_tags_warning + + docker-compose -f ${COMPOSE_FILE} start coordinator-cluster-c } function test_fanout_warnings { diff --git a/src/query/api/v1/handler/fetch_options.go b/src/query/api/v1/handler/fetch_options.go index 2248381320..d36ab9df01 100644 --- a/src/query/api/v1/handler/fetch_options.go +++ b/src/query/api/v1/handler/fetch_options.go @@ -21,6 +21,7 @@ package handler import ( + "encoding/json" "fmt" "math" "net/http" @@ -103,6 +104,7 @@ func (b fetchOptionsBuilder) NewFetchOptions( if err != nil { return nil, xhttp.NewParseError(err, http.StatusBadRequest) } + fetchOpts.Limit = limit if str := req.Header.Get(MetricsTypeHeader); str != "" { mt, err := storage.ParseMetricsType(str) @@ -111,9 +113,13 @@ func (b fetchOptionsBuilder) NewFetchOptions( "could not parse metrics type: input=%s, err=%v", str, err) return nil, xhttp.NewParseError(err, http.StatusBadRequest) } - fetchOpts.RestrictFetchOptions = newOrExistingRestrictFetchOptions(fetchOpts) - fetchOpts.RestrictFetchOptions.MetricsType = mt + + fetchOpts.RestrictQueryOptions = newOrExistingRestrictQueryOptions(fetchOpts) + fetchOpts.RestrictQueryOptions.RestrictByType = + newOrExistingRestrictQueryOptionsRestrictByType(fetchOpts) + fetchOpts.RestrictQueryOptions.RestrictByType.MetricsType = mt } + if str := req.Header.Get(MetricsStoragePolicyHeader); str != "" { sp, err := policy.ParseStoragePolicy(str) if err != nil { @@ -121,10 +127,29 @@ func (b fetchOptionsBuilder) NewFetchOptions( "could not parse storage policy: input=%s, err=%v", str, err) return nil, xhttp.NewParseError(err, http.StatusBadRequest) } - fetchOpts.RestrictFetchOptions = newOrExistingRestrictFetchOptions(fetchOpts) - fetchOpts.RestrictFetchOptions.StoragePolicy = sp + + fetchOpts.RestrictQueryOptions = newOrExistingRestrictQueryOptions(fetchOpts) + fetchOpts.RestrictQueryOptions.RestrictByType = + newOrExistingRestrictQueryOptionsRestrictByType(fetchOpts) + fetchOpts.RestrictQueryOptions.RestrictByType.StoragePolicy = sp } - if restrict := fetchOpts.RestrictFetchOptions; restrict != nil { + + if str := req.Header.Get(RestrictByTagsJSONHeader); str != "" { + var opts StringTagOptions + if err := json.Unmarshal([]byte(str), &opts); err != nil { + return nil, xhttp.NewParseError(err, http.StatusBadRequest) + } + + tagOpts, err := opts.toOptions() + if err != nil { + return nil, xhttp.NewParseError(err, http.StatusBadRequest) + } + + fetchOpts.RestrictQueryOptions = newOrExistingRestrictQueryOptions(fetchOpts) + fetchOpts.RestrictQueryOptions.RestrictByTag = tagOpts + } + + if restrict := fetchOpts.RestrictQueryOptions; restrict != nil { if err := restrict.Validate(); err != nil { err = fmt.Errorf( "could not validate restrict options: err=%v", err) @@ -152,13 +177,22 @@ func (b fetchOptionsBuilder) NewFetchOptions( return fetchOpts, nil } -func newOrExistingRestrictFetchOptions( +func newOrExistingRestrictQueryOptions( + fetchOpts *storage.FetchOptions, +) *storage.RestrictQueryOptions { + if v := fetchOpts.RestrictQueryOptions; v != nil { + return v + } + return &storage.RestrictQueryOptions{} +} + +func newOrExistingRestrictQueryOptionsRestrictByType( fetchOpts *storage.FetchOptions, -) *storage.RestrictFetchOptions { - if v := fetchOpts.RestrictFetchOptions; v != nil { +) *storage.RestrictByType { + if v := fetchOpts.RestrictQueryOptions.RestrictByType; v != nil { return v } - return &storage.RestrictFetchOptions{} + return &storage.RestrictByType{} } // ParseStep parses the step duration for an HTTP request. diff --git a/src/query/api/v1/handler/fetch_options_test.go b/src/query/api/v1/handler/fetch_options_test.go index d3acd770ae..175c284782 100644 --- a/src/query/api/v1/handler/fetch_options_test.go +++ b/src/query/api/v1/handler/fetch_options_test.go @@ -30,6 +30,7 @@ import ( "time" "github.com/m3db/m3/src/metrics/policy" + "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage" "github.com/stretchr/testify/assert" @@ -47,7 +48,7 @@ func TestFetchOptionsBuilder(t *testing.T) { headers map[string]string query string expectedLimit int - expectedRestrict *storage.RestrictFetchOptions + expectedRestrict *storage.RestrictQueryOptions expectedLookback *expectedLookback expectedErr bool }{ @@ -78,8 +79,10 @@ func TestFetchOptionsBuilder(t *testing.T) { headers: map[string]string{ MetricsTypeHeader: storage.UnaggregatedMetricsType.String(), }, - expectedRestrict: &storage.RestrictFetchOptions{ - MetricsType: storage.UnaggregatedMetricsType, + expectedRestrict: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.UnaggregatedMetricsType, + }, }, }, { @@ -88,9 +91,11 @@ func TestFetchOptionsBuilder(t *testing.T) { MetricsTypeHeader: storage.AggregatedMetricsType.String(), MetricsStoragePolicyHeader: "1m:14d", }, - expectedRestrict: &storage.RestrictFetchOptions{ - MetricsType: storage.AggregatedMetricsType, - StoragePolicy: policy.MustParseStoragePolicy("1m:14d"), + expectedRestrict: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.MustParseStoragePolicy("1m:14d"), + }, }, }, { @@ -167,10 +172,10 @@ func TestFetchOptionsBuilder(t *testing.T) { require.NoError(t, err) require.Equal(t, test.expectedLimit, opts.Limit) if test.expectedRestrict == nil { - require.Nil(t, opts.RestrictFetchOptions) + require.Nil(t, opts.RestrictQueryOptions) } else { - require.NotNil(t, opts.RestrictFetchOptions) - require.Equal(t, *test.expectedRestrict, *opts.RestrictFetchOptions) + require.NotNil(t, opts.RestrictQueryOptions) + require.Equal(t, *test.expectedRestrict, *opts.RestrictQueryOptions) } if test.expectedLookback == nil { require.Nil(t, opts.LookbackDuration) @@ -269,3 +274,54 @@ func TestParseDurationOverflowError(t *testing.T) { _, err = ParseDuration(r, StepParam) assert.Error(t, err) } + +func TestFetchOptionsWithHeader(t *testing.T) { + type expectedLookback struct { + value time.Duration + } + + headers := map[string]string{ + MetricsTypeHeader: storage.AggregatedMetricsType.String(), + MetricsStoragePolicyHeader: "1m:14d", + RestrictByTagsJSONHeader: `{ + "match":[ + {"name":"a", "value":"b", "type":"EQUAL"}, + {"name":"c", "value":"d", "type":"NOTEQUAL"}, + {"name":"e", "value":"f", "type":"REGEXP"}, + {"name":"g", "value":"h", "type":"NOTREGEXP"}, + {"name":"i", "value":"j", "type":"EXISTS"}, + {"name":"k", "value":"l", "type":"NOTEXISTS"} + ], + "strip":["foo"] + }`, + } + + builder := NewFetchOptionsBuilder(FetchOptionsBuilderOptions{Limit: 5}) + req := httptest.NewRequest("GET", "/", nil) + for k, v := range headers { + req.Header.Add(k, v) + } + + opts, err := builder.NewFetchOptions(req) + require.NoError(t, err) + require.NotNil(t, opts.RestrictQueryOptions) + ex := &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.MustParseStoragePolicy("1m:14d"), + }, + RestrictByTag: &storage.RestrictByTag{ + Restrict: models.Matchers{ + mustMatcher("a", "b", models.MatchEqual), + mustMatcher("c", "d", models.MatchNotEqual), + mustMatcher("e", "f", models.MatchRegexp), + mustMatcher("g", "h", models.MatchNotRegexp), + mustMatcher("i", "j", models.MatchField), + mustMatcher("k", "l", models.MatchNotField), + }, + Strip: toStrip("foo"), + }, + } + + require.Equal(t, ex, opts.RestrictQueryOptions) +} diff --git a/src/query/api/v1/handler/headers.go b/src/query/api/v1/handler/headers.go index 7af42d0508..d0829e23f0 100644 --- a/src/query/api/v1/handler/headers.go +++ b/src/query/api/v1/handler/headers.go @@ -56,6 +56,10 @@ const ( // metrics type. MetricsStoragePolicyHeader = "M3-Storage-Policy" + // RestrictByTagsJSONHeader provides tag options to enforces on queries, + // in JSON format. See `handler.stringTagOptions` for definitions.` + RestrictByTagsJSONHeader = "M3-Restrict-By-Tags-JSON" + // UnaggregatedStoragePolicy specifies the unaggregated storage policy. UnaggregatedStoragePolicy = "unaggregated" diff --git a/src/query/api/v1/handler/prometheus/common.go b/src/query/api/v1/handler/prometheus/common.go index 59b17d3ba8..a93a33c065 100644 --- a/src/query/api/v1/handler/prometheus/common.go +++ b/src/query/api/v1/handler/prometheus/common.go @@ -36,6 +36,7 @@ import ( "github.com/m3db/m3/src/query/models" xpromql "github.com/m3db/m3/src/query/parser/promql" "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/query/ts" "github.com/m3db/m3/src/query/util" "github.com/m3db/m3/src/query/util/json" xhttp "github.com/m3db/m3/src/x/net/http" @@ -451,9 +452,10 @@ func (r results) Less(i, j int) bool { // Swap swaps the elements with indexes i and j. func (r results) Swap(i, j int) { r[i], r[j] = r[j], r[i] } -func (r results) sort() { - for _, result := range r { - result.genID() +// Sort sorts the results. +func (r results) Sort() { + for i, result := range r { + r[i] = result.genID() } sort.Sort(r) @@ -477,18 +479,22 @@ type Values []Value // Value is a single value for Prometheus result. type Value []interface{} -func (r *Result) genID() { +func (r *Result) genID() Result { + tags := make(sort.StringSlice, len(r.Metric)) + for k, v := range r.Metric { + tags = append(tags, fmt.Sprintf("%s:%s,", k, v)) + } + + sort.Sort(tags) var sb strings.Builder // NB: this may clash but exact tag values are also checked, and this is a // validation endpoint so there's less concern over correctness. - for k, v := range r.Metric { - sb.WriteString(k) - sb.WriteString(`:"`) - sb.WriteString(v) - sb.WriteString(`",`) + for _, t := range tags { + sb.WriteString(t) } r.id = sb.String() + return *r } // MatchInformation describes how well two responses match. @@ -533,8 +539,8 @@ func (r results) matches(other results) (MatchInformation, error) { }, err } - r.sort() - other.sort() + r.Sort() + other.Sort() for i, result := range r { if err := result.matches(other[i]); err != nil { return MatchInformation{ @@ -632,3 +638,22 @@ type PromDebug struct { Input Response `json:"input"` Results Response `json:"results"` } + +// FilterSeriesByOptions removes series tags based on options. +func FilterSeriesByOptions( + series []*ts.Series, + opts *storage.FetchOptions, +) []*ts.Series { + if opts == nil { + return series + } + + keys := opts.RestrictQueryOptions.GetRestrictByTag().GetFilterByNames() + if len(keys) > 0 { + for i, s := range series { + series[i].Tags = s.Tags.TagsWithoutKeys(keys) + } + } + + return series +} diff --git a/src/query/api/v1/handler/prometheus/native/read.go b/src/query/api/v1/handler/prometheus/native/read.go index a4aa942a65..aabd3b78d8 100644 --- a/src/query/api/v1/handler/prometheus/native/read.go +++ b/src/query/api/v1/handler/prometheus/native/read.go @@ -146,7 +146,9 @@ func (h *PromReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { QueryContextOptions: models.QueryContextOptions{ LimitMaxTimeseries: fetchOpts.Limit, }} - if restrictOpts := fetchOpts.RestrictFetchOptions; restrictOpts != nil { + + restrictOpts := fetchOpts.RestrictQueryOptions.GetRestrictByType() + if restrictOpts != nil { restrict := &models.RestrictFetchTypeQueryContextOptions{ MetricsType: uint(restrictOpts.MetricsType), StoragePolicy: restrictOpts.StoragePolicy, diff --git a/src/query/api/v1/handler/prometheus/native/read_common.go b/src/query/api/v1/handler/prometheus/native/read_common.go index b58ab51861..3638f658f8 100644 --- a/src/query/api/v1/handler/prometheus/native/read_common.go +++ b/src/query/api/v1/handler/prometheus/native/read_common.go @@ -28,6 +28,7 @@ import ( "sort" "github.com/m3db/m3/src/query/api/v1/handler" + "github.com/m3db/m3/src/query/api/v1/handler/prometheus" "github.com/m3db/m3/src/query/block" "github.com/m3db/m3/src/query/executor" "github.com/m3db/m3/src/query/models" @@ -150,6 +151,7 @@ func read( return emptyResult, err } + series = prometheus.FilterSeriesByOptions(series, fetchOpts) return readResult{ series: series, meta: meta, diff --git a/src/query/api/v1/handler/prometheus/native/read_instantaneous.go b/src/query/api/v1/handler/prometheus/native/read_instantaneous.go index a8a13094f8..154b249256 100644 --- a/src/query/api/v1/handler/prometheus/native/read_instantaneous.go +++ b/src/query/api/v1/handler/prometheus/native/read_instantaneous.go @@ -96,7 +96,9 @@ func (h *PromReadInstantHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques QueryContextOptions: models.QueryContextOptions{ LimitMaxTimeseries: fetchOpts.Limit, }} - if restrictOpts := fetchOpts.RestrictFetchOptions; restrictOpts != nil { + + restrictOpts := fetchOpts.RestrictQueryOptions.GetRestrictByType() + if restrictOpts != nil { restrict := &models.RestrictFetchTypeQueryContextOptions{ MetricsType: uint(restrictOpts.MetricsType), StoragePolicy: restrictOpts.StoragePolicy, diff --git a/src/query/api/v1/handler/prometheus/remote/read.go b/src/query/api/v1/handler/prometheus/remote/read.go index a4280bd4ff..b8121c9fd9 100644 --- a/src/query/api/v1/handler/prometheus/remote/read.go +++ b/src/query/api/v1/handler/prometheus/remote/read.go @@ -232,6 +232,10 @@ func (h *PromReadHandler) read( mu.Lock() meta = meta.CombineMetadata(result.Metadata) mu.Unlock() + result.SeriesList = prometheus.FilterSeriesByOptions( + result.SeriesList, + fetchOpts, + ) promRes := storage.FetchResultToPromResult(result, h.keepEmpty) promResults[i] = promRes }() diff --git a/src/query/api/v1/handler/prometheus/validator/handler_test.go b/src/query/api/v1/handler/prometheus/validator/handler_test.go index 4c5254dc72..13e6494df6 100644 --- a/src/query/api/v1/handler/prometheus/validator/handler_test.go +++ b/src/query/api/v1/handler/prometheus/validator/handler_test.go @@ -22,7 +22,6 @@ package validator import ( "encoding/json" - "fmt" "io" "net/http" "net/http/httptest" @@ -335,7 +334,6 @@ func TestValidateEndpoint(t *testing.T) { recorder := httptest.NewRecorder() debugHandler.ServeHTTP(recorder, req) - fmt.Println(recorder.Body.String()) var mismatches MismatchesJSON require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &mismatches)) assert.False(t, mismatches.Correct) diff --git a/src/query/api/v1/handler/tag_options.go b/src/query/api/v1/handler/tag_options.go new file mode 100644 index 0000000000..664376aeb7 --- /dev/null +++ b/src/query/api/v1/handler/tag_options.go @@ -0,0 +1,113 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package handler + +import ( + "errors" + "fmt" + + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/storage" +) + +func matchByName(name string) (models.MatchType, error) { + t := models.MatchEqual + switch name { + case "EQUAL": + // noop + case "NOTEQUAL": + t = models.MatchNotEqual + case "REGEXP": + t = models.MatchRegexp + case "NOTREGEXP": + t = models.MatchNotRegexp + case "EXISTS": + t = models.MatchField + case "NOTEXISTS": + t = models.MatchNotField + case "ALL": + return t, errors.New("ALL type not supported as a tag matcher restriction") + default: + return t, fmt.Errorf("matcher type %s not recognized", name) + } + + return t, nil +} + +func (m StringMatch) toMatcher() (models.Matcher, error) { + t, err := matchByName(m.Type) + if err != nil { + return models.Matcher{}, err + } + + return models.NewMatcher(t, []byte(m.Name), []byte(m.Value)) +} + +func (o *StringTagOptions) toOptions() (*storage.RestrictByTag, error) { + if len(o.Restrict) == 0 && len(o.Strip) == 0 { + return nil, nil + } + + opts := &storage.RestrictByTag{} + if len(o.Restrict) > 0 { + opts.Restrict = make(models.Matchers, 0, len(o.Restrict)) + for _, m := range o.Restrict { + r, err := m.toMatcher() + if err != nil { + return nil, err + } + + opts.Restrict = append(opts.Restrict, r) + } + } + + if o.Strip != nil { + opts.Strip = make([][]byte, 0, len(o.Strip)) + for _, s := range o.Strip { + opts.Strip = append(opts.Strip, []byte(s)) + } + } else { + // If strip not explicitly set, strip tag names from + // the restricted matchers. + opts.Strip = make([][]byte, 0, len(opts.Restrict)) + for _, s := range opts.Restrict { + opts.Strip = append(opts.Strip, s.Name) + } + } + + return opts, nil +} + +// StringMatch is an easy to use JSON representation of models.Matcher that +// allows plaintext fields rather than forcing base64 encoded values. +type StringMatch struct { + Name string `json:"name"` + Type string `json:"type"` + Value string `json:"value"` +} + +// StringTagOptions is an easy to use JSON representation of +// storage.RestrictByTag that allows plaintext string fields rather than +// forcing base64 encoded values. +type StringTagOptions struct { + Restrict []StringMatch `json:"match"` + Strip []string `json:"strip"` +} diff --git a/src/query/api/v1/handler/tag_options_test.go b/src/query/api/v1/handler/tag_options_test.go new file mode 100644 index 0000000000..71204704a4 --- /dev/null +++ b/src/query/api/v1/handler/tag_options_test.go @@ -0,0 +1,144 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package handler + +import ( + "encoding/json" + "testing" + + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/storage" + + "github.com/stretchr/testify/require" +) + +func toStrip(strs ...string) [][]byte { + b := make([][]byte, 0, len(strs)) + for _, s := range strs { + b = append(b, []byte(s)) + } + + return b +} + +func mustMatcher(n string, v string, t models.MatchType) models.Matcher { + m, err := models.NewMatcher(t, []byte(n), []byte(v)) + if err != nil { + panic(err) + } + + return m +} + +func TestParse(t *testing.T) { + tests := []struct { + json string + expected *storage.RestrictByTag + expectedError bool + }{ + {"{}", nil, false}, + { + `{ + "match":[ + {"name":"a", "value":"b", "type":"EQUAL"}, + {"name":"c", "value":"d", "type":"NOTEQUAL"}, + {"name":"e", "value":"f", "type":"REGEXP"}, + {"name":"g", "value":"h", "type":"NOTREGEXP"}, + {"name":"i", "type":"EXISTS"}, + {"name":"j", "type":"NOTEXISTS"} + ] + }`, + &storage.RestrictByTag{ + Restrict: models.Matchers{ + mustMatcher("a", "b", models.MatchEqual), + mustMatcher("c", "d", models.MatchNotEqual), + mustMatcher("e", "f", models.MatchRegexp), + mustMatcher("g", "h", models.MatchNotRegexp), + mustMatcher("i", "", models.MatchField), + mustMatcher("j", "", models.MatchNotField), + }, + Strip: toStrip("a", "c", "e", "g", "i", "j"), + }, + false, + }, + { + `{ + "match":[ + {"name":"a", "value":"b", "type":"EQUAL"}, + {"name":"c", "value":"d", "type":"NOTEQUAL"}, + {"name":"e", "value":"f", "type":"REGEXP"}, + {"name":"g", "value":"h", "type":"NOTREGEXP"}, + {"name":"i", "value":"j", "type":"EXISTS"}, + {"name":"k", "value":"l", "type":"NOTEXISTS"} + ], + "strip":["foo"] + }`, + &storage.RestrictByTag{ + Restrict: models.Matchers{ + mustMatcher("a", "b", models.MatchEqual), + mustMatcher("c", "d", models.MatchNotEqual), + mustMatcher("e", "f", models.MatchRegexp), + mustMatcher("g", "h", models.MatchNotRegexp), + mustMatcher("i", "j", models.MatchField), + mustMatcher("k", "l", models.MatchNotField), + }, + Strip: toStrip("foo"), + }, + false, + }, + { + `{"match":[{"name":"i", "value":"j", "type":"EXISTS"}],"strip":[]}`, + &storage.RestrictByTag{ + Restrict: models.Matchers{ + mustMatcher("i", "j", models.MatchField), + }, + Strip: [][]byte{}, + }, + false, + }, + { + `{"strip":["foo"]}`, + &storage.RestrictByTag{ + Strip: toStrip("foo"), + }, + false, + }, + {`{"match":[{}]}`, nil, true}, + {`{"match":[{"type":"ALL"}]}`, nil, true}, + {`{"match":[{"type":"invalid"}]}`, nil, true}, + {`{"match":[{"name":"a","type":"EQUAL"}]}`, nil, true}, + } + + for _, tt := range tests { + var opts StringTagOptions + err := json.Unmarshal([]byte(tt.json), &opts) + require.NoError(t, err) + + a, err := opts.toOptions() + if tt.expectedError { + require.Error(t, err) + require.Nil(t, a) + } else { + require.NoError(t, err) + require.Equal(t, tt.expected, a) + } + } +} diff --git a/src/query/api/v1/handler/types.go b/src/query/api/v1/handler/types.go index c1dab5d06a..54a6a605ae 100644 --- a/src/query/api/v1/handler/types.go +++ b/src/query/api/v1/handler/types.go @@ -20,16 +20,16 @@ package handler -// HeaderKeyType is the type for the header key +// HeaderKeyType is the type for the header key. type HeaderKeyType int const ( - // HeaderKey is the key which headers will be added to in the request context + // HeaderKey is the key which headers will be added to in the request context. HeaderKey HeaderKeyType = iota - // RoutePrefixV1 is the v1 prefix for all coordinator routes + // RoutePrefixV1 is the v1 prefix for all coordinator routes. RoutePrefixV1 = "/api/v1" - // RoutePrefixExperimenta is the experimental prefix for all coordinator routes + // RoutePrefixExperimental is the experimental prefix for all coordinator routes. RoutePrefixExperimental = "/api/experimental" ) diff --git a/src/query/executor/engine.go b/src/query/executor/engine.go index 079638bbba..d4e1e1c0c1 100644 --- a/src/query/executor/engine.go +++ b/src/query/executor/engine.go @@ -115,7 +115,8 @@ func (e *engine) Execute( opts *QueryOptions, fetchOpts *storage.FetchOptions, ) (*storage.FetchResult, error) { - return e.opts.Store().Fetch(ctx, query, fetchOpts) + result, err := e.opts.Store().Fetch(ctx, query, fetchOpts) + return result, err } func (e *engine) ExecuteExpr( diff --git a/src/query/functions/fetch_test.go b/src/query/functions/fetch_test.go index e1832950bd..e7783aa74d 100644 --- a/src/query/functions/fetch_test.go +++ b/src/query/functions/fetch_test.go @@ -146,7 +146,8 @@ func TestFetchWithRestrictFetch(t *testing.T) { assert.Equal(t, expected, sink.Values) fetchOpts := mockStorage.LastFetchOptions() - require.NotNil(t, fetchOpts.RestrictFetchOptions) - assert.Equal(t, storage.AggregatedMetricsType, storage.MetricsType(fetchOpts.RestrictFetchOptions.MetricsType)) - assert.Equal(t, "10s:42d", fetchOpts.RestrictFetchOptions.StoragePolicy.String()) + restrictByType := fetchOpts.RestrictQueryOptions.GetRestrictByType() + require.NotNil(t, restrictByType) + assert.Equal(t, storage.AggregatedMetricsType, storage.MetricsType(restrictByType.MetricsType)) + assert.Equal(t, "10s:42d", restrictByType.StoragePolicy.String()) } diff --git a/src/query/generated/proto/rpcpb/query.pb.go b/src/query/generated/proto/rpcpb/query.pb.go index cac72df3d1..ae904e0ef8 100644 --- a/src/query/generated/proto/rpcpb/query.pb.go +++ b/src/query/generated/proto/rpcpb/query.pb.go @@ -34,7 +34,9 @@ TagMatchers TagMatcher FetchOptions - RestrictFetchOptions + RestrictQueryOptions + RestrictFetchType + RestrictFetchTags FetchResponse Series SeriesMetadata @@ -394,7 +396,7 @@ func (m *TagMatcher) GetType() MatcherType { type FetchOptions struct { Limit int64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` - Restrict *RestrictFetchOptions `protobuf:"bytes,2,opt,name=restrict" json:"restrict,omitempty"` + Restrict *RestrictQueryOptions `protobuf:"bytes,2,opt,name=restrict" json:"restrict,omitempty"` LookbackDuration int64 `protobuf:"varint,3,opt,name=lookbackDuration,proto3" json:"lookbackDuration,omitempty"` Unaggregated FanoutOption `protobuf:"varint,4,opt,name=unaggregated,proto3,enum=rpc.FanoutOption" json:"unaggregated,omitempty"` Aggregated FanoutOption `protobuf:"varint,5,opt,name=aggregated,proto3,enum=rpc.FanoutOption" json:"aggregated,omitempty"` @@ -414,7 +416,7 @@ func (m *FetchOptions) GetLimit() int64 { return 0 } -func (m *FetchOptions) GetRestrict() *RestrictFetchOptions { +func (m *FetchOptions) GetRestrict() *RestrictQueryOptions { if m != nil { return m.Restrict } @@ -456,30 +458,78 @@ func (m *FetchOptions) GetIncludeResolution() bool { return false } -type RestrictFetchOptions struct { +type RestrictQueryOptions struct { + RestrictFetchType *RestrictFetchType `protobuf:"bytes,3,opt,name=RestrictFetchType" json:"RestrictFetchType,omitempty"` + RestrictFetchTags *RestrictFetchTags `protobuf:"bytes,4,opt,name=RestrictFetchTags" json:"RestrictFetchTags,omitempty"` +} + +func (m *RestrictQueryOptions) Reset() { *m = RestrictQueryOptions{} } +func (m *RestrictQueryOptions) String() string { return proto.CompactTextString(m) } +func (*RestrictQueryOptions) ProtoMessage() {} +func (*RestrictQueryOptions) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{6} } + +func (m *RestrictQueryOptions) GetRestrictFetchType() *RestrictFetchType { + if m != nil { + return m.RestrictFetchType + } + return nil +} + +func (m *RestrictQueryOptions) GetRestrictFetchTags() *RestrictFetchTags { + if m != nil { + return m.RestrictFetchTags + } + return nil +} + +type RestrictFetchType struct { MetricsType MetricsType `protobuf:"varint,1,opt,name=metricsType,proto3,enum=rpc.MetricsType" json:"metricsType,omitempty"` MetricsStoragePolicy *policypb.StoragePolicy `protobuf:"bytes,2,opt,name=metricsStoragePolicy" json:"metricsStoragePolicy,omitempty"` } -func (m *RestrictFetchOptions) Reset() { *m = RestrictFetchOptions{} } -func (m *RestrictFetchOptions) String() string { return proto.CompactTextString(m) } -func (*RestrictFetchOptions) ProtoMessage() {} -func (*RestrictFetchOptions) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{6} } +func (m *RestrictFetchType) Reset() { *m = RestrictFetchType{} } +func (m *RestrictFetchType) String() string { return proto.CompactTextString(m) } +func (*RestrictFetchType) ProtoMessage() {} +func (*RestrictFetchType) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{7} } -func (m *RestrictFetchOptions) GetMetricsType() MetricsType { +func (m *RestrictFetchType) GetMetricsType() MetricsType { if m != nil { return m.MetricsType } return MetricsType_UNKNOWN_METRICS_TYPE } -func (m *RestrictFetchOptions) GetMetricsStoragePolicy() *policypb.StoragePolicy { +func (m *RestrictFetchType) GetMetricsStoragePolicy() *policypb.StoragePolicy { if m != nil { return m.MetricsStoragePolicy } return nil } +type RestrictFetchTags struct { + Restrict *TagMatchers `protobuf:"bytes,1,opt,name=restrict" json:"restrict,omitempty"` + Strip [][]byte `protobuf:"bytes,2,rep,name=strip" json:"strip,omitempty"` +} + +func (m *RestrictFetchTags) Reset() { *m = RestrictFetchTags{} } +func (m *RestrictFetchTags) String() string { return proto.CompactTextString(m) } +func (*RestrictFetchTags) ProtoMessage() {} +func (*RestrictFetchTags) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{8} } + +func (m *RestrictFetchTags) GetRestrict() *TagMatchers { + if m != nil { + return m.Restrict + } + return nil +} + +func (m *RestrictFetchTags) GetStrip() [][]byte { + if m != nil { + return m.Strip + } + return nil +} + type FetchResponse struct { Series []*Series `protobuf:"bytes,1,rep,name=series" json:"series,omitempty"` Meta *ResultMetadata `protobuf:"bytes,2,opt,name=meta" json:"meta,omitempty"` @@ -488,7 +538,7 @@ type FetchResponse struct { func (m *FetchResponse) Reset() { *m = FetchResponse{} } func (m *FetchResponse) String() string { return proto.CompactTextString(m) } func (*FetchResponse) ProtoMessage() {} -func (*FetchResponse) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{7} } +func (*FetchResponse) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{9} } func (m *FetchResponse) GetSeries() []*Series { if m != nil { @@ -515,7 +565,7 @@ type Series struct { func (m *Series) Reset() { *m = Series{} } func (m *Series) String() string { return proto.CompactTextString(m) } func (*Series) ProtoMessage() {} -func (*Series) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{8} } +func (*Series) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{10} } type isSeries_Value interface { isSeries_Value() @@ -644,7 +694,7 @@ type SeriesMetadata struct { func (m *SeriesMetadata) Reset() { *m = SeriesMetadata{} } func (m *SeriesMetadata) String() string { return proto.CompactTextString(m) } func (*SeriesMetadata) ProtoMessage() {} -func (*SeriesMetadata) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{9} } +func (*SeriesMetadata) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{11} } func (m *SeriesMetadata) GetId() []byte { if m != nil { @@ -675,7 +725,7 @@ type DecompressedSeries struct { func (m *DecompressedSeries) Reset() { *m = DecompressedSeries{} } func (m *DecompressedSeries) String() string { return proto.CompactTextString(m) } func (*DecompressedSeries) ProtoMessage() {} -func (*DecompressedSeries) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{10} } +func (*DecompressedSeries) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{12} } func (m *DecompressedSeries) GetDatapoints() []*Datapoint { if m != nil { @@ -699,7 +749,7 @@ type Datapoint struct { func (m *Datapoint) Reset() { *m = Datapoint{} } func (m *Datapoint) String() string { return proto.CompactTextString(m) } func (*Datapoint) ProtoMessage() {} -func (*Datapoint) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{11} } +func (*Datapoint) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{13} } func (m *Datapoint) GetTimestamp() int64 { if m != nil { @@ -723,7 +773,7 @@ type Tag struct { func (m *Tag) Reset() { *m = Tag{} } func (m *Tag) String() string { return proto.CompactTextString(m) } func (*Tag) ProtoMessage() {} -func (*Tag) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{12} } +func (*Tag) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{14} } func (m *Tag) GetName() []byte { if m != nil { @@ -747,7 +797,7 @@ type M3CompressedSeries struct { func (m *M3CompressedSeries) Reset() { *m = M3CompressedSeries{} } func (m *M3CompressedSeries) String() string { return proto.CompactTextString(m) } func (*M3CompressedSeries) ProtoMessage() {} -func (*M3CompressedSeries) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{13} } +func (*M3CompressedSeries) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{15} } func (m *M3CompressedSeries) GetCompressedTags() []byte { if m != nil { @@ -770,7 +820,7 @@ type M3CompressedValuesReplica struct { func (m *M3CompressedValuesReplica) Reset() { *m = M3CompressedValuesReplica{} } func (m *M3CompressedValuesReplica) String() string { return proto.CompactTextString(m) } func (*M3CompressedValuesReplica) ProtoMessage() {} -func (*M3CompressedValuesReplica) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{14} } +func (*M3CompressedValuesReplica) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{16} } func (m *M3CompressedValuesReplica) GetSegments() []*M3Segments { if m != nil { @@ -787,7 +837,7 @@ type M3Segments struct { func (m *M3Segments) Reset() { *m = M3Segments{} } func (m *M3Segments) String() string { return proto.CompactTextString(m) } func (*M3Segments) ProtoMessage() {} -func (*M3Segments) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{15} } +func (*M3Segments) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{17} } func (m *M3Segments) GetMerged() *M3Segment { if m != nil { @@ -813,7 +863,7 @@ type M3Segment struct { func (m *M3Segment) Reset() { *m = M3Segment{} } func (m *M3Segment) String() string { return proto.CompactTextString(m) } func (*M3Segment) ProtoMessage() {} -func (*M3Segment) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{16} } +func (*M3Segment) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{18} } func (m *M3Segment) GetHead() []byte { if m != nil { @@ -855,7 +905,7 @@ type SearchRequest struct { func (m *SearchRequest) Reset() { *m = SearchRequest{} } func (m *SearchRequest) String() string { return proto.CompactTextString(m) } func (*SearchRequest) ProtoMessage() {} -func (*SearchRequest) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{17} } +func (*SearchRequest) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{19} } type isSearchRequest_Matchers interface { isSearchRequest_Matchers() @@ -967,7 +1017,7 @@ type M3TagProperty struct { func (m *M3TagProperty) Reset() { *m = M3TagProperty{} } func (m *M3TagProperty) String() string { return proto.CompactTextString(m) } func (*M3TagProperty) ProtoMessage() {} -func (*M3TagProperty) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{18} } +func (*M3TagProperty) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{20} } func (m *M3TagProperty) GetId() []byte { if m != nil { @@ -990,7 +1040,7 @@ type M3TagProperties struct { func (m *M3TagProperties) Reset() { *m = M3TagProperties{} } func (m *M3TagProperties) String() string { return proto.CompactTextString(m) } func (*M3TagProperties) ProtoMessage() {} -func (*M3TagProperties) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{19} } +func (*M3TagProperties) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{21} } func (m *M3TagProperties) GetProperties() []*M3TagProperty { if m != nil { @@ -1007,7 +1057,7 @@ type TagProperty struct { func (m *TagProperty) Reset() { *m = TagProperty{} } func (m *TagProperty) String() string { return proto.CompactTextString(m) } func (*TagProperty) ProtoMessage() {} -func (*TagProperty) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{20} } +func (*TagProperty) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{22} } func (m *TagProperty) GetKey() []byte { if m != nil { @@ -1030,7 +1080,7 @@ type TagProperties struct { func (m *TagProperties) Reset() { *m = TagProperties{} } func (m *TagProperties) String() string { return proto.CompactTextString(m) } func (*TagProperties) ProtoMessage() {} -func (*TagProperties) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{21} } +func (*TagProperties) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{23} } func (m *TagProperties) GetProperties() []*TagProperty { if m != nil { @@ -1050,7 +1100,7 @@ type SearchResponse struct { func (m *SearchResponse) Reset() { *m = SearchResponse{} } func (m *SearchResponse) String() string { return proto.CompactTextString(m) } func (*SearchResponse) ProtoMessage() {} -func (*SearchResponse) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{22} } +func (*SearchResponse) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{24} } type isSearchResponse_Value interface { isSearchResponse_Value() @@ -1181,7 +1231,7 @@ type CompleteTagsRequestOptions struct { func (m *CompleteTagsRequestOptions) Reset() { *m = CompleteTagsRequestOptions{} } func (m *CompleteTagsRequestOptions) String() string { return proto.CompactTextString(m) } func (*CompleteTagsRequestOptions) ProtoMessage() {} -func (*CompleteTagsRequestOptions) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{23} } +func (*CompleteTagsRequestOptions) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{25} } func (m *CompleteTagsRequestOptions) GetType() CompleteTagsType { if m != nil { @@ -1228,7 +1278,7 @@ type CompleteTagsRequest struct { func (m *CompleteTagsRequest) Reset() { *m = CompleteTagsRequest{} } func (m *CompleteTagsRequest) String() string { return proto.CompactTextString(m) } func (*CompleteTagsRequest) ProtoMessage() {} -func (*CompleteTagsRequest) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{24} } +func (*CompleteTagsRequest) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{26} } type isCompleteTagsRequest_Matchers interface { isCompleteTagsRequest_Matchers() @@ -1325,7 +1375,7 @@ type TagNames struct { func (m *TagNames) Reset() { *m = TagNames{} } func (m *TagNames) String() string { return proto.CompactTextString(m) } func (*TagNames) ProtoMessage() {} -func (*TagNames) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{25} } +func (*TagNames) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{27} } func (m *TagNames) GetNames() [][]byte { if m != nil { @@ -1342,7 +1392,7 @@ type TagValue struct { func (m *TagValue) Reset() { *m = TagValue{} } func (m *TagValue) String() string { return proto.CompactTextString(m) } func (*TagValue) ProtoMessage() {} -func (*TagValue) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{26} } +func (*TagValue) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{28} } func (m *TagValue) GetKey() []byte { if m != nil { @@ -1365,7 +1415,7 @@ type TagValues struct { func (m *TagValues) Reset() { *m = TagValues{} } func (m *TagValues) String() string { return proto.CompactTextString(m) } func (*TagValues) ProtoMessage() {} -func (*TagValues) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{27} } +func (*TagValues) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{29} } func (m *TagValues) GetValues() []*TagValue { if m != nil { @@ -1385,7 +1435,7 @@ type CompleteTagsResponse struct { func (m *CompleteTagsResponse) Reset() { *m = CompleteTagsResponse{} } func (m *CompleteTagsResponse) String() string { return proto.CompactTextString(m) } func (*CompleteTagsResponse) ProtoMessage() {} -func (*CompleteTagsResponse) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{28} } +func (*CompleteTagsResponse) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{30} } type isCompleteTagsResponse_Value interface { isCompleteTagsResponse_Value() @@ -1514,7 +1564,7 @@ type ResultMetadata struct { func (m *ResultMetadata) Reset() { *m = ResultMetadata{} } func (m *ResultMetadata) String() string { return proto.CompactTextString(m) } func (*ResultMetadata) ProtoMessage() {} -func (*ResultMetadata) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{29} } +func (*ResultMetadata) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{31} } func (m *ResultMetadata) GetExhaustive() bool { if m != nil { @@ -1545,7 +1595,7 @@ type Warning struct { func (m *Warning) Reset() { *m = Warning{} } func (m *Warning) String() string { return proto.CompactTextString(m) } func (*Warning) ProtoMessage() {} -func (*Warning) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{30} } +func (*Warning) Descriptor() ([]byte, []int) { return fileDescriptorQuery, []int{32} } func (m *Warning) GetName() []byte { if m != nil { @@ -1568,7 +1618,9 @@ func init() { proto.RegisterType((*TagMatchers)(nil), "rpc.TagMatchers") proto.RegisterType((*TagMatcher)(nil), "rpc.TagMatcher") proto.RegisterType((*FetchOptions)(nil), "rpc.FetchOptions") - proto.RegisterType((*RestrictFetchOptions)(nil), "rpc.RestrictFetchOptions") + proto.RegisterType((*RestrictQueryOptions)(nil), "rpc.RestrictQueryOptions") + proto.RegisterType((*RestrictFetchType)(nil), "rpc.RestrictFetchType") + proto.RegisterType((*RestrictFetchTags)(nil), "rpc.RestrictFetchTags") proto.RegisterType((*FetchResponse)(nil), "rpc.FetchResponse") proto.RegisterType((*Series)(nil), "rpc.Series") proto.RegisterType((*SeriesMetadata)(nil), "rpc.SeriesMetadata") @@ -2086,7 +2138,45 @@ func (m *FetchOptions) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *RestrictFetchOptions) Marshal() (dAtA []byte, err error) { +func (m *RestrictQueryOptions) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RestrictQueryOptions) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.RestrictFetchType != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintQuery(dAtA, i, uint64(m.RestrictFetchType.Size())) + n5, err := m.RestrictFetchType.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + } + if m.RestrictFetchTags != nil { + dAtA[i] = 0x22 + i++ + i = encodeVarintQuery(dAtA, i, uint64(m.RestrictFetchTags.Size())) + n6, err := m.RestrictFetchTags.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n6 + } + return i, nil +} + +func (m *RestrictFetchType) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -2096,7 +2186,7 @@ func (m *RestrictFetchOptions) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *RestrictFetchOptions) MarshalTo(dAtA []byte) (int, error) { +func (m *RestrictFetchType) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -2110,11 +2200,47 @@ func (m *RestrictFetchOptions) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintQuery(dAtA, i, uint64(m.MetricsStoragePolicy.Size())) - n5, err := m.MetricsStoragePolicy.MarshalTo(dAtA[i:]) + n7, err := m.MetricsStoragePolicy.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n5 + i += n7 + } + return i, nil +} + +func (m *RestrictFetchTags) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RestrictFetchTags) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Restrict != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintQuery(dAtA, i, uint64(m.Restrict.Size())) + n8, err := m.Restrict.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n8 + } + if len(m.Strip) > 0 { + for _, b := range m.Strip { + dAtA[i] = 0x12 + i++ + i = encodeVarintQuery(dAtA, i, uint64(len(b))) + i += copy(dAtA[i:], b) + } } return i, nil } @@ -2150,11 +2276,11 @@ func (m *FetchResponse) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintQuery(dAtA, i, uint64(m.Meta.Size())) - n6, err := m.Meta.MarshalTo(dAtA[i:]) + n9, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n6 + i += n9 } return i, nil } @@ -2178,18 +2304,18 @@ func (m *Series) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintQuery(dAtA, i, uint64(m.Meta.Size())) - n7, err := m.Meta.MarshalTo(dAtA[i:]) + n10, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n7 + i += n10 } if m.Value != nil { - nn8, err := m.Value.MarshalTo(dAtA[i:]) + nn11, err := m.Value.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn8 + i += nn11 } return i, nil } @@ -2200,11 +2326,11 @@ func (m *Series_Decompressed) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintQuery(dAtA, i, uint64(m.Decompressed.Size())) - n9, err := m.Decompressed.MarshalTo(dAtA[i:]) + n12, err := m.Decompressed.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n9 + i += n12 } return i, nil } @@ -2214,11 +2340,11 @@ func (m *Series_Compressed) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x1a i++ i = encodeVarintQuery(dAtA, i, uint64(m.Compressed.Size())) - n10, err := m.Compressed.MarshalTo(dAtA[i:]) + n13, err := m.Compressed.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n10 + i += n13 } return i, nil } @@ -2442,11 +2568,11 @@ func (m *M3Segments) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintQuery(dAtA, i, uint64(m.Merged.Size())) - n11, err := m.Merged.MarshalTo(dAtA[i:]) + n14, err := m.Merged.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n11 + i += n14 } if len(m.Unmerged) > 0 { for _, msg := range m.Unmerged { @@ -2519,11 +2645,11 @@ func (m *SearchRequest) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if m.Matchers != nil { - nn12, err := m.Matchers.MarshalTo(dAtA[i:]) + nn15, err := m.Matchers.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn12 + i += nn15 } if m.Start != 0 { dAtA[i] = 0x10 @@ -2539,11 +2665,11 @@ func (m *SearchRequest) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintQuery(dAtA, i, uint64(m.Options.Size())) - n13, err := m.Options.MarshalTo(dAtA[i:]) + n16, err := m.Options.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n13 + i += n16 } return i, nil } @@ -2554,11 +2680,11 @@ func (m *SearchRequest_TagMatchers) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintQuery(dAtA, i, uint64(m.TagMatchers.Size())) - n14, err := m.TagMatchers.MarshalTo(dAtA[i:]) + n17, err := m.TagMatchers.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n14 + i += n17 } return i, nil } @@ -2698,21 +2824,21 @@ func (m *SearchResponse) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if m.Value != nil { - nn15, err := m.Value.MarshalTo(dAtA[i:]) + nn18, err := m.Value.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn15 + i += nn18 } if m.Meta != nil { dAtA[i] = 0x1a i++ i = encodeVarintQuery(dAtA, i, uint64(m.Meta.Size())) - n16, err := m.Meta.MarshalTo(dAtA[i:]) + n19, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n16 + i += n19 } return i, nil } @@ -2723,11 +2849,11 @@ func (m *SearchResponse_Decompressed) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintQuery(dAtA, i, uint64(m.Decompressed.Size())) - n17, err := m.Decompressed.MarshalTo(dAtA[i:]) + n20, err := m.Decompressed.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n17 + i += n20 } return i, nil } @@ -2737,11 +2863,11 @@ func (m *SearchResponse_Compressed) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintQuery(dAtA, i, uint64(m.Compressed.Size())) - n18, err := m.Compressed.MarshalTo(dAtA[i:]) + n21, err := m.Compressed.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n18 + i += n21 } return i, nil } @@ -2787,11 +2913,11 @@ func (m *CompleteTagsRequestOptions) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x2a i++ i = encodeVarintQuery(dAtA, i, uint64(m.Options.Size())) - n19, err := m.Options.MarshalTo(dAtA[i:]) + n22, err := m.Options.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n19 + i += n22 } return i, nil } @@ -2812,21 +2938,21 @@ func (m *CompleteTagsRequest) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if m.Matchers != nil { - nn20, err := m.Matchers.MarshalTo(dAtA[i:]) + nn23, err := m.Matchers.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn20 + i += nn23 } if m.Options != nil { dAtA[i] = 0x12 i++ i = encodeVarintQuery(dAtA, i, uint64(m.Options.Size())) - n21, err := m.Options.MarshalTo(dAtA[i:]) + n24, err := m.Options.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n21 + i += n24 } return i, nil } @@ -2837,11 +2963,11 @@ func (m *CompleteTagsRequest_TagMatchers) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintQuery(dAtA, i, uint64(m.TagMatchers.Size())) - n22, err := m.TagMatchers.MarshalTo(dAtA[i:]) + n25, err := m.TagMatchers.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n22 + i += n25 } return i, nil } @@ -2949,21 +3075,21 @@ func (m *CompleteTagsResponse) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if m.Value != nil { - nn23, err := m.Value.MarshalTo(dAtA[i:]) + nn26, err := m.Value.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += nn23 + i += nn26 } if m.Meta != nil { dAtA[i] = 0x1a i++ i = encodeVarintQuery(dAtA, i, uint64(m.Meta.Size())) - n24, err := m.Meta.MarshalTo(dAtA[i:]) + n27, err := m.Meta.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n24 + i += n27 } return i, nil } @@ -2974,11 +3100,11 @@ func (m *CompleteTagsResponse_Default) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintQuery(dAtA, i, uint64(m.Default.Size())) - n25, err := m.Default.MarshalTo(dAtA[i:]) + n28, err := m.Default.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n25 + i += n28 } return i, nil } @@ -2988,11 +3114,11 @@ func (m *CompleteTagsResponse_NamesOnly) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintQuery(dAtA, i, uint64(m.NamesOnly.Size())) - n26, err := m.NamesOnly.MarshalTo(dAtA[i:]) + n29, err := m.NamesOnly.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n26 + i += n29 } return i, nil } @@ -3034,22 +3160,22 @@ func (m *ResultMetadata) MarshalTo(dAtA []byte) (int, error) { } } if len(m.Resolutions) > 0 { - dAtA28 := make([]byte, len(m.Resolutions)*10) - var j27 int + dAtA31 := make([]byte, len(m.Resolutions)*10) + var j30 int for _, num1 := range m.Resolutions { num := uint64(num1) for num >= 1<<7 { - dAtA28[j27] = uint8(uint64(num)&0x7f | 0x80) + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j27++ + j30++ } - dAtA28[j27] = uint8(num) - j27++ + dAtA31[j30] = uint8(num) + j30++ } dAtA[i] = 0x1a i++ - i = encodeVarintQuery(dAtA, i, uint64(j27)) - i += copy(dAtA[i:], dAtA28[:j27]) + i = encodeVarintQuery(dAtA, i, uint64(j30)) + i += copy(dAtA[i:], dAtA31[:j30]) } return i, nil } @@ -3197,7 +3323,21 @@ func (m *FetchOptions) Size() (n int) { return n } -func (m *RestrictFetchOptions) Size() (n int) { +func (m *RestrictQueryOptions) Size() (n int) { + var l int + _ = l + if m.RestrictFetchType != nil { + l = m.RestrictFetchType.Size() + n += 1 + l + sovQuery(uint64(l)) + } + if m.RestrictFetchTags != nil { + l = m.RestrictFetchTags.Size() + n += 1 + l + sovQuery(uint64(l)) + } + return n +} + +func (m *RestrictFetchType) Size() (n int) { var l int _ = l if m.MetricsType != 0 { @@ -3210,6 +3350,22 @@ func (m *RestrictFetchOptions) Size() (n int) { return n } +func (m *RestrictFetchTags) Size() (n int) { + var l int + _ = l + if m.Restrict != nil { + l = m.Restrict.Size() + n += 1 + l + sovQuery(uint64(l)) + } + if len(m.Strip) > 0 { + for _, b := range m.Strip { + l = len(b) + n += 1 + l + sovQuery(uint64(l)) + } + } + return n +} + func (m *FetchResponse) Size() (n int) { var l int _ = l @@ -4247,7 +4403,7 @@ func (m *FetchOptions) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Restrict == nil { - m.Restrict = &RestrictFetchOptions{} + m.Restrict = &RestrictQueryOptions{} } if err := m.Restrict.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -4370,7 +4526,7 @@ func (m *FetchOptions) Unmarshal(dAtA []byte) error { } return nil } -func (m *RestrictFetchOptions) Unmarshal(dAtA []byte) error { +func (m *RestrictQueryOptions) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -4393,10 +4549,126 @@ func (m *RestrictFetchOptions) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: RestrictFetchOptions: wiretype end group for non-group") + return fmt.Errorf("proto: RestrictQueryOptions: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: RestrictFetchOptions: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: RestrictQueryOptions: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RestrictFetchType", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RestrictFetchType == nil { + m.RestrictFetchType = &RestrictFetchType{} + } + if err := m.RestrictFetchType.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RestrictFetchTags", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.RestrictFetchTags == nil { + m.RestrictFetchTags = &RestrictFetchTags{} + } + if err := m.RestrictFetchTags.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RestrictFetchType) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RestrictFetchType: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RestrictFetchType: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -4472,6 +4744,118 @@ func (m *RestrictFetchOptions) Unmarshal(dAtA []byte) error { } return nil } +func (m *RestrictFetchTags) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RestrictFetchTags: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RestrictFetchTags: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Restrict", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Restrict == nil { + m.Restrict = &TagMatchers{} + } + if err := m.Restrict.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Strip", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Strip = append(m.Strip, make([]byte, postIndex-iNdEx)) + copy(m.Strip[len(m.Strip)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipQuery(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthQuery + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *FetchResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -7385,104 +7769,108 @@ func init() { } var fileDescriptorQuery = []byte{ - // 1579 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0xdb, 0x52, 0x1b, 0x47, - 0x1a, 0xd6, 0x68, 0xd0, 0xe9, 0xd7, 0x01, 0xb9, 0x61, 0xd7, 0x82, 0xf5, 0xb2, 0xd4, 0xec, 0xae, - 0x97, 0xc5, 0x5e, 0x64, 0x83, 0xbd, 0x8e, 0x53, 0x95, 0x83, 0x00, 0x19, 0x28, 0x83, 0x84, 0x5b, - 0x43, 0xec, 0xa4, 0x92, 0x22, 0xad, 0x51, 0x5b, 0x4c, 0xa1, 0x39, 0x78, 0xa6, 0xe5, 0x18, 0x57, - 0x1e, 0x22, 0x95, 0x9b, 0x3c, 0x40, 0x2a, 0x79, 0x02, 0x3f, 0x42, 0x2e, 0x72, 0x99, 0x47, 0x48, - 0x39, 0xb9, 0xc8, 0x63, 0xa4, 0xba, 0xa7, 0xe7, 0x24, 0x89, 0xb2, 0xcb, 0x77, 0xd3, 0xff, 0xb1, - 0xff, 0xbf, 0xbf, 0xfe, 0xfe, 0x1e, 0xf8, 0x70, 0x68, 0xb2, 0xb3, 0x71, 0x7f, 0xc3, 0x70, 0xac, - 0xa6, 0xb5, 0x35, 0xe8, 0x37, 0xad, 0xad, 0xa6, 0xef, 0x19, 0xcd, 0x67, 0x63, 0xea, 0x5d, 0x34, - 0x87, 0xd4, 0xa6, 0x1e, 0x61, 0x74, 0xd0, 0x74, 0x3d, 0x87, 0x39, 0x4d, 0xcf, 0x35, 0xdc, 0x7e, - 0xa0, 0xdb, 0x10, 0x12, 0xa4, 0x7a, 0xae, 0xb1, 0xbc, 0x7b, 0x49, 0x10, 0x8b, 0x32, 0xcf, 0x34, - 0xfc, 0xa9, 0x30, 0xae, 0x33, 0x32, 0x8d, 0x0b, 0xb7, 0x2f, 0x3f, 0x82, 0x50, 0xda, 0x3c, 0x54, - 0xf7, 0x29, 0x19, 0xb1, 0x33, 0x4c, 0x9f, 0x8d, 0xa9, 0xcf, 0xb4, 0xa7, 0x50, 0x0b, 0x05, 0xbe, - 0xeb, 0xd8, 0x3e, 0x45, 0xd7, 0xa1, 0x36, 0x76, 0x99, 0x69, 0xd1, 0xdd, 0xb1, 0x47, 0x98, 0xe9, - 0xd8, 0x0d, 0x65, 0x55, 0x59, 0x2b, 0xe1, 0x09, 0x29, 0xba, 0x09, 0x57, 0x02, 0x49, 0x87, 0xd8, - 0x8e, 0x4f, 0x0d, 0xc7, 0x1e, 0xf8, 0x8d, 0xec, 0xaa, 0xb2, 0xa6, 0xe2, 0x69, 0x85, 0xf6, 0x83, - 0x02, 0x95, 0x07, 0x94, 0x19, 0x61, 0x62, 0xb4, 0x08, 0x39, 0x9f, 0x11, 0x8f, 0x89, 0xe8, 0x2a, - 0x0e, 0x16, 0xa8, 0x0e, 0x2a, 0xb5, 0x07, 0x32, 0x0c, 0xff, 0x44, 0x77, 0xa0, 0xcc, 0xc8, 0xf0, - 0x88, 0x30, 0xe3, 0x8c, 0x7a, 0x7e, 0x43, 0x5d, 0x55, 0xd6, 0xca, 0x9b, 0xf5, 0x0d, 0xcf, 0x35, - 0x36, 0xf4, 0x58, 0xbe, 0x9f, 0xc1, 0x49, 0x33, 0x74, 0x03, 0x0a, 0x8e, 0xcb, 0xb7, 0xe9, 0x37, - 0xe6, 0x84, 0xc7, 0x15, 0xe1, 0x21, 0x76, 0xd0, 0x0d, 0x14, 0x38, 0xb4, 0xd8, 0x06, 0x28, 0x5a, - 0xd2, 0x51, 0xfb, 0x18, 0xca, 0x89, 0xb0, 0xe8, 0x76, 0x3a, 0xbb, 0xb2, 0xaa, 0xae, 0x95, 0x37, - 0xe7, 0x27, 0xb2, 0xa7, 0x52, 0x6b, 0x9f, 0x03, 0xc4, 0x2a, 0x84, 0x60, 0xce, 0x26, 0x16, 0x15, - 0x55, 0x56, 0xb0, 0xf8, 0xe6, 0xa5, 0x3f, 0x27, 0xa3, 0x31, 0x15, 0x65, 0x56, 0x70, 0xb0, 0x40, - 0xff, 0x82, 0x39, 0x76, 0xe1, 0x52, 0x51, 0x61, 0x4d, 0x56, 0x28, 0xa3, 0xe8, 0x17, 0x2e, 0xc5, - 0x42, 0xab, 0xfd, 0x9e, 0x95, 0x7d, 0x94, 0x55, 0xf0, 0x60, 0x23, 0xd3, 0x32, 0xa3, 0x3e, 0x8a, - 0x05, 0xba, 0x0b, 0x45, 0x8f, 0xfa, 0x1c, 0x19, 0x4c, 0x64, 0x29, 0x6f, 0x2e, 0x89, 0x80, 0x58, - 0x0a, 0x53, 0x8d, 0x88, 0x4c, 0xd1, 0x3a, 0xd4, 0x47, 0x8e, 0x73, 0xde, 0x27, 0xc6, 0x79, 0x74, - 0xfa, 0xaa, 0x88, 0x3b, 0x25, 0x47, 0x77, 0xa1, 0x32, 0xb6, 0xc9, 0x70, 0xe8, 0xd1, 0x21, 0x87, - 0x9d, 0xe8, 0x73, 0x2d, 0xec, 0x33, 0xb1, 0x9d, 0x31, 0x0b, 0xe2, 0xe3, 0x94, 0x19, 0xba, 0x0d, - 0x90, 0x70, 0xca, 0x5d, 0xe6, 0x94, 0x30, 0x42, 0x3b, 0xb0, 0x10, 0xaf, 0xb8, 0xde, 0x32, 0x5f, - 0xd2, 0x41, 0x23, 0x7f, 0x99, 0xef, 0x2c, 0x6b, 0x0e, 0x57, 0xd3, 0x36, 0x46, 0xe3, 0x01, 0xc5, - 0xd4, 0x77, 0x46, 0x63, 0x51, 0x5b, 0x61, 0x55, 0x59, 0x2b, 0xe2, 0x69, 0x85, 0xf6, 0x9d, 0x02, - 0x8b, 0xb3, 0x7a, 0x85, 0x36, 0xa1, 0x2c, 0x6f, 0x1c, 0x3f, 0x14, 0xd1, 0xf4, 0xe8, 0xb0, 0x62, - 0x39, 0x4e, 0x1a, 0xa1, 0x87, 0xb0, 0x28, 0x97, 0x3d, 0xe6, 0x78, 0x64, 0x48, 0x8f, 0xc5, 0x95, - 0x94, 0x07, 0x73, 0x75, 0x23, 0xbc, 0xaa, 0x1b, 0x29, 0x35, 0x9e, 0xe9, 0xa4, 0x7d, 0x01, 0x55, - 0x79, 0x8f, 0xe4, 0x7d, 0xfd, 0x27, 0xe4, 0x7d, 0xea, 0x99, 0x34, 0x44, 0x67, 0x59, 0x6c, 0xa6, - 0x27, 0x44, 0x58, 0xaa, 0xd0, 0x7f, 0x60, 0xce, 0xa2, 0x8c, 0xc8, 0x94, 0x0b, 0x21, 0x16, 0xc6, - 0x23, 0x76, 0x44, 0x19, 0x19, 0x10, 0x46, 0xb0, 0x30, 0xd0, 0x5e, 0x29, 0x90, 0xef, 0xa5, 0x7d, - 0x94, 0x84, 0x4f, 0xa0, 0x4a, 0xfb, 0xa0, 0x0f, 0xa0, 0x32, 0xa0, 0x86, 0x63, 0xb9, 0x1e, 0xf5, - 0x7d, 0x3a, 0x88, 0xea, 0xe2, 0x0e, 0xbb, 0x09, 0x45, 0xe0, 0xbc, 0x9f, 0xc1, 0x29, 0x73, 0x74, - 0x1f, 0x20, 0xe1, 0xac, 0x26, 0x9c, 0x8f, 0xb6, 0x76, 0xa6, 0x9d, 0x13, 0xc6, 0xdb, 0x05, 0x79, - 0x93, 0xb4, 0x27, 0x50, 0x4b, 0x6f, 0x0d, 0xd5, 0x20, 0x6b, 0x0e, 0xe4, 0xb5, 0xcb, 0x9a, 0x03, - 0x74, 0x0d, 0x4a, 0x82, 0x62, 0x74, 0xd3, 0xa2, 0x92, 0x5f, 0x62, 0x01, 0x6a, 0x40, 0x81, 0xda, - 0x03, 0xa1, 0x0b, 0xf0, 0x1e, 0x2e, 0xb5, 0x3e, 0xa0, 0xe9, 0x1a, 0xd0, 0x06, 0x00, 0xcf, 0xe2, - 0x3a, 0xa6, 0xcd, 0xc2, 0xc6, 0xd7, 0x82, 0x82, 0x43, 0x31, 0x4e, 0x58, 0xa0, 0x6b, 0x30, 0xc7, - 0xc8, 0x90, 0xf3, 0x23, 0xb7, 0x2c, 0x86, 0x04, 0x82, 0x85, 0x54, 0xfb, 0x08, 0x4a, 0x91, 0x1b, - 0xdf, 0x28, 0x27, 0x4f, 0x9f, 0x11, 0xcb, 0x95, 0x97, 0x3a, 0x16, 0xa4, 0xb9, 0x43, 0x91, 0xdc, - 0xa1, 0x35, 0x41, 0xd5, 0xc9, 0xf0, 0xed, 0xc9, 0x46, 0x7b, 0x01, 0x68, 0xba, 0xb9, 0x9c, 0xfa, - 0xe3, 0x4a, 0x75, 0xbe, 0xdf, 0x20, 0xd2, 0x84, 0x14, 0xbd, 0xcf, 0xd9, 0xc5, 0x1d, 0x99, 0x06, - 0x09, 0x2b, 0x5a, 0x99, 0x3a, 0xaf, 0x4f, 0x78, 0x1e, 0x1f, 0x07, 0x66, 0x38, 0xb2, 0xd7, 0xf6, - 0x61, 0xe9, 0x52, 0x33, 0x74, 0x03, 0x8a, 0x3e, 0x1d, 0x5a, 0x34, 0x6e, 0xea, 0xbc, 0x0c, 0xdc, - 0x93, 0x62, 0x1c, 0x19, 0x68, 0x5f, 0x02, 0xc4, 0x72, 0x74, 0x1d, 0xf2, 0x16, 0xf5, 0x86, 0x74, - 0x20, 0xf1, 0x5a, 0x4b, 0x3b, 0x62, 0xa9, 0x45, 0xeb, 0x50, 0x1c, 0xdb, 0xd2, 0x32, 0x9b, 0x38, - 0xb7, 0xd8, 0x32, 0xd2, 0x6b, 0x0e, 0x94, 0x22, 0x31, 0x6f, 0xee, 0x19, 0x25, 0x21, 0xa4, 0xc4, - 0x37, 0x97, 0x31, 0x62, 0x8e, 0x64, 0x6f, 0xc5, 0x77, 0x1a, 0x68, 0xea, 0x24, 0xd0, 0xae, 0x41, - 0xa9, 0x3f, 0x72, 0x8c, 0xf3, 0x9e, 0xf9, 0x92, 0x0a, 0xca, 0x54, 0x71, 0x2c, 0xd0, 0x7e, 0x54, - 0xa0, 0xda, 0xa3, 0xc4, 0x8b, 0xc7, 0xe4, 0x9d, 0xc9, 0x01, 0xf4, 0x56, 0xe3, 0x2f, 0x1a, 0xae, - 0xd9, 0x19, 0xc3, 0x55, 0x8d, 0x87, 0xeb, 0x3b, 0x8f, 0xc9, 0x3d, 0xa8, 0x1e, 0x6d, 0xe9, 0x64, - 0x78, 0xec, 0x39, 0x2e, 0xf5, 0xd8, 0xc5, 0xd4, 0x75, 0x9b, 0x86, 0x52, 0x76, 0x16, 0x94, 0xb4, - 0x36, 0xcc, 0x27, 0x03, 0x71, 0x14, 0x6e, 0x02, 0xb8, 0xd1, 0x4a, 0xc2, 0x00, 0xc9, 0x33, 0x4a, - 0xa4, 0xc4, 0x09, 0x2b, 0xed, 0x9e, 0x18, 0xdb, 0xd1, 0x6e, 0xea, 0xa0, 0x9e, 0xd3, 0x0b, 0xb9, - 0x1d, 0xfe, 0x89, 0xfe, 0x0a, 0x79, 0x81, 0xfc, 0x70, 0x1f, 0x72, 0xa5, 0xb5, 0xa0, 0x9a, 0xce, - 0x7e, 0x6b, 0x46, 0xf6, 0xa8, 0xdf, 0x33, 0x73, 0xbf, 0x52, 0x38, 0xf9, 0x04, 0x87, 0x26, 0x39, - 0xf9, 0xbd, 0x09, 0x46, 0x0c, 0x8e, 0x0d, 0x4d, 0x84, 0x99, 0x45, 0x86, 0xff, 0x4f, 0x91, 0x61, - 0xc0, 0xa4, 0x8b, 0x53, 0xc5, 0x4f, 0x31, 0x61, 0x44, 0xd6, 0xea, 0x1b, 0x08, 0x3e, 0xa6, 0xcc, - 0x9f, 0x14, 0x58, 0xe6, 0xf7, 0x70, 0x44, 0x19, 0xe5, 0x47, 0x21, 0x11, 0x17, 0x0e, 0xba, 0xff, - 0xca, 0xe7, 0x48, 0x30, 0xe1, 0xfe, 0x22, 0x02, 0x26, 0xcd, 0xe3, 0x37, 0x09, 0x3f, 0xeb, 0xa7, - 0xe6, 0x88, 0x51, 0xaf, 0x43, 0x2c, 0xaa, 0x87, 0x34, 0x57, 0xc1, 0x13, 0xd2, 0x18, 0x95, 0xea, - 0x0c, 0x54, 0xce, 0xcd, 0x44, 0x65, 0xee, 0x4d, 0xa8, 0xd4, 0xbe, 0x55, 0x60, 0x61, 0x46, 0x19, - 0xef, 0x78, 0x71, 0xee, 0xc7, 0xa9, 0x83, 0xde, 0xff, 0x63, 0xaa, 0xf0, 0x74, 0x9f, 0x66, 0x5f, - 0x8f, 0x55, 0x28, 0xea, 0x64, 0xc8, 0x0b, 0x17, 0x55, 0x73, 0x22, 0x0e, 0xb0, 0x54, 0xc1, 0xc1, - 0x42, 0xbb, 0x23, 0x2c, 0x04, 0xfb, 0xbd, 0x01, 0xad, 0x6a, 0x02, 0xad, 0x9b, 0x50, 0x0a, 0xbd, - 0x7c, 0xf4, 0xef, 0xc8, 0x28, 0x40, 0x69, 0x35, 0x2c, 0x4e, 0xe8, 0x23, 0x9f, 0xef, 0x15, 0x58, - 0x4c, 0xef, 0x5f, 0x82, 0x74, 0x1d, 0x0a, 0x03, 0xfa, 0x94, 0x8c, 0x47, 0x2c, 0x45, 0x99, 0x51, - 0x82, 0xfd, 0x0c, 0x0e, 0x0d, 0xd0, 0xff, 0xa0, 0x24, 0xf6, 0xdd, 0xb5, 0x47, 0xe1, 0xbb, 0x25, - 0x4a, 0x27, 0xca, 0xdc, 0xcf, 0xe0, 0xd8, 0xe2, 0x1d, 0xd0, 0xf8, 0x35, 0xd4, 0xd2, 0x06, 0x68, - 0x05, 0x80, 0xbe, 0x38, 0x23, 0x63, 0x9f, 0x99, 0xcf, 0x03, 0x18, 0x16, 0x71, 0x42, 0x82, 0xd6, - 0xa0, 0xf8, 0x15, 0xf1, 0x6c, 0xd3, 0x8e, 0xc6, 0x6a, 0x45, 0xe4, 0x79, 0x1c, 0x08, 0x71, 0xa4, - 0x45, 0xab, 0x50, 0xf6, 0xa2, 0xa7, 0x1d, 0xff, 0x85, 0x50, 0xd7, 0x54, 0x9c, 0x14, 0x69, 0xf7, - 0xa0, 0x20, 0xdd, 0x66, 0xce, 0xd0, 0x06, 0x14, 0x2c, 0xea, 0xfb, 0x64, 0x18, 0x4e, 0xd1, 0x70, - 0xb9, 0x4e, 0xa1, 0x9c, 0x78, 0xa3, 0xa3, 0x12, 0xe4, 0xda, 0x8f, 0x4e, 0x5a, 0x87, 0xf5, 0x0c, - 0xaa, 0x40, 0xb1, 0xd3, 0xd5, 0x83, 0x95, 0x82, 0x00, 0xf2, 0xb8, 0xbd, 0xd7, 0x7e, 0x72, 0x5c, - 0xcf, 0xa2, 0x2a, 0x94, 0x3a, 0x5d, 0x5d, 0x2e, 0x55, 0xae, 0x6a, 0x3f, 0x39, 0xe8, 0xe9, 0xbd, - 0xfa, 0x9c, 0x54, 0xc9, 0x65, 0x0e, 0x15, 0x40, 0x6d, 0x1d, 0x1e, 0xd6, 0xf3, 0xeb, 0x06, 0x94, - 0x13, 0xaf, 0x4b, 0xd4, 0x80, 0xc5, 0x93, 0xce, 0xc3, 0x4e, 0xf7, 0x71, 0xe7, 0xf4, 0xa8, 0xad, - 0xe3, 0x83, 0x9d, 0xde, 0xa9, 0xfe, 0xe9, 0x71, 0xbb, 0x9e, 0x41, 0x7f, 0x87, 0xa5, 0x93, 0x4e, - 0x6b, 0x6f, 0x0f, 0xb7, 0xf7, 0x5a, 0x7a, 0x7b, 0x37, 0xad, 0x56, 0xd0, 0xdf, 0xe0, 0xea, 0x65, - 0xca, 0xec, 0xfa, 0x01, 0x54, 0x92, 0xcf, 0x68, 0x84, 0xa0, 0xb6, 0xdb, 0x7e, 0xd0, 0x3a, 0x39, - 0xd4, 0x4f, 0xbb, 0xc7, 0xfa, 0x41, 0xb7, 0x53, 0xcf, 0xa0, 0x2b, 0x50, 0x7d, 0xd0, 0xc5, 0x3b, - 0xed, 0xd3, 0x76, 0xa7, 0xb5, 0x7d, 0xd8, 0xde, 0xad, 0x2b, 0xdc, 0x2c, 0x10, 0xed, 0x1e, 0xf4, - 0x02, 0x59, 0x76, 0xfd, 0x26, 0xd4, 0x27, 0xb9, 0x02, 0x95, 0xa1, 0x20, 0xc3, 0xd5, 0x33, 0x7c, - 0xa1, 0xb7, 0xf6, 0x3a, 0xad, 0xa3, 0x76, 0x5d, 0xd9, 0xfc, 0x43, 0x81, 0xdc, 0x23, 0xfe, 0xbf, - 0x8b, 0x6e, 0x43, 0x3e, 0xf8, 0x1b, 0x45, 0x01, 0x57, 0xa6, 0xfe, 0x55, 0x97, 0x17, 0x52, 0x32, - 0x89, 0xe2, 0x5b, 0x90, 0x13, 0xc4, 0x80, 0x12, 0x24, 0x11, 0x3a, 0xa0, 0xa4, 0x28, 0xb0, 0xbf, - 0xa5, 0xa0, 0x2d, 0xfe, 0xc2, 0xe5, 0x74, 0x2d, 0x93, 0xa4, 0x06, 0xee, 0xf2, 0x42, 0x4a, 0x16, - 0x39, 0xb5, 0xa1, 0x92, 0xac, 0x08, 0x35, 0x2e, 0xe3, 0x85, 0xe5, 0xa5, 0x19, 0x9a, 0x30, 0xcc, - 0xf6, 0xd5, 0x9f, 0x5f, 0xaf, 0x28, 0xbf, 0xbc, 0x5e, 0x51, 0x7e, 0x7d, 0xbd, 0xa2, 0x7c, 0xf3, - 0xdb, 0x4a, 0xe6, 0xb3, 0x9c, 0xf8, 0xdf, 0xef, 0xe7, 0xc5, 0xff, 0xf9, 0xd6, 0x9f, 0x01, 0x00, - 0x00, 0xff, 0xff, 0xf3, 0x1f, 0xfe, 0x09, 0x2c, 0x10, 0x00, 0x00, + // 1640 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x58, 0xdb, 0x72, 0xdb, 0x4e, + 0x19, 0xb7, 0xac, 0xf8, 0xf4, 0xf9, 0x10, 0x67, 0x13, 0xfe, 0x75, 0x42, 0x09, 0x1e, 0x01, 0x7f, + 0x42, 0x5a, 0xe2, 0xd6, 0x69, 0x29, 0x65, 0x86, 0x83, 0x13, 0xbb, 0x49, 0xa6, 0x89, 0x9d, 0xae, + 0x15, 0x5a, 0x18, 0x98, 0xb0, 0x96, 0xb7, 0x8e, 0x26, 0xd6, 0xa1, 0x92, 0x5c, 0x9a, 0x0e, 0x0f, + 0xc1, 0x30, 0x3c, 0x01, 0x0c, 0x3c, 0x41, 0x1f, 0x81, 0x0b, 0x2e, 0x79, 0x04, 0xa6, 0x70, 0xc1, + 0x63, 0x30, 0xbb, 0x5a, 0x9d, 0x2c, 0x65, 0xda, 0xe9, 0x9d, 0xbf, 0xf3, 0x7e, 0xdf, 0xfe, 0xf6, + 0xb7, 0x2b, 0xc3, 0xcf, 0x66, 0xba, 0x77, 0xb5, 0x98, 0xec, 0x69, 0x96, 0xd1, 0x31, 0xf6, 0xa7, + 0x93, 0x8e, 0xb1, 0xdf, 0x71, 0x1d, 0xad, 0xf3, 0x66, 0x41, 0x9d, 0x9b, 0xce, 0x8c, 0x9a, 0xd4, + 0x21, 0x1e, 0x9d, 0x76, 0x6c, 0xc7, 0xf2, 0xac, 0x8e, 0x63, 0x6b, 0xf6, 0xc4, 0xb7, 0xed, 0x71, + 0x0d, 0x92, 0x1d, 0x5b, 0xdb, 0xea, 0xdf, 0x92, 0xc4, 0xa0, 0x9e, 0xa3, 0x6b, 0x6e, 0x2a, 0x8d, + 0x6d, 0xcd, 0x75, 0xed, 0xc6, 0x9e, 0x88, 0x1f, 0x7e, 0x2a, 0x65, 0x15, 0xea, 0xc7, 0x94, 0xcc, + 0xbd, 0x2b, 0x4c, 0xdf, 0x2c, 0xa8, 0xeb, 0x29, 0xaf, 0xa1, 0x11, 0x28, 0x5c, 0xdb, 0x32, 0x5d, + 0x8a, 0xbe, 0x86, 0xc6, 0xc2, 0xf6, 0x74, 0x83, 0xf6, 0x17, 0x0e, 0xf1, 0x74, 0xcb, 0x6c, 0x49, + 0x6d, 0x69, 0xa7, 0x82, 0x97, 0xb4, 0xe8, 0x3e, 0xac, 0xf9, 0x9a, 0x21, 0x31, 0x2d, 0x97, 0x6a, + 0x96, 0x39, 0x75, 0x5b, 0xf9, 0xb6, 0xb4, 0x23, 0xe3, 0xb4, 0x41, 0xf9, 0x9b, 0x04, 0xb5, 0x67, + 0xd4, 0xd3, 0x82, 0xc2, 0x68, 0x03, 0x0a, 0xae, 0x47, 0x1c, 0x8f, 0x67, 0x97, 0xb1, 0x2f, 0xa0, + 0x26, 0xc8, 0xd4, 0x9c, 0x8a, 0x34, 0xec, 0x27, 0x7a, 0x04, 0x55, 0x8f, 0xcc, 0xce, 0x88, 0xa7, + 0x5d, 0x51, 0xc7, 0x6d, 0xc9, 0x6d, 0x69, 0xa7, 0xda, 0x6d, 0xee, 0x39, 0xb6, 0xb6, 0xa7, 0x46, + 0xfa, 0xe3, 0x1c, 0x8e, 0xbb, 0xa1, 0x7b, 0x50, 0xb2, 0x6c, 0xb6, 0x4c, 0xb7, 0xb5, 0xc2, 0x23, + 0xd6, 0x78, 0x04, 0x5f, 0xc1, 0xc8, 0x37, 0xe0, 0xc0, 0xe3, 0x00, 0xa0, 0x6c, 0x88, 0x40, 0xe5, + 0x17, 0x50, 0x8d, 0xa5, 0x45, 0x0f, 0x93, 0xd5, 0xa5, 0xb6, 0xbc, 0x53, 0xed, 0xae, 0x2e, 0x55, + 0x4f, 0x94, 0x56, 0x7e, 0x03, 0x10, 0x99, 0x10, 0x82, 0x15, 0x93, 0x18, 0x94, 0x77, 0x59, 0xc3, + 0xfc, 0x37, 0x6b, 0xfd, 0x2d, 0x99, 0x2f, 0x28, 0x6f, 0xb3, 0x86, 0x7d, 0x01, 0x7d, 0x17, 0x56, + 0xbc, 0x1b, 0x9b, 0xf2, 0x0e, 0x1b, 0xa2, 0x43, 0x91, 0x45, 0xbd, 0xb1, 0x29, 0xe6, 0x56, 0xe5, + 0xbf, 0x79, 0x31, 0x47, 0xd1, 0x05, 0x4b, 0x36, 0xd7, 0x0d, 0x3d, 0x9c, 0x23, 0x17, 0xd0, 0x63, + 0x28, 0x3b, 0xd4, 0x65, 0xc8, 0xf0, 0x78, 0x95, 0x6a, 0x77, 0x93, 0x27, 0xc4, 0x42, 0xf9, 0x82, + 0xc1, 0x2b, 0x18, 0x44, 0xe8, 0x8a, 0x76, 0xa1, 0x39, 0xb7, 0xac, 0xeb, 0x09, 0xd1, 0xae, 0xc3, + 0xdd, 0x97, 0x79, 0xde, 0x94, 0x1e, 0x3d, 0x86, 0xda, 0xc2, 0x24, 0xb3, 0x99, 0x43, 0x67, 0x0c, + 0x76, 0x7c, 0xce, 0x8d, 0x60, 0xce, 0xc4, 0xb4, 0x16, 0x9e, 0x9f, 0x1f, 0x27, 0xdc, 0xd0, 0x43, + 0x80, 0x58, 0x50, 0xe1, 0xb6, 0xa0, 0x98, 0x13, 0x3a, 0x84, 0xf5, 0x48, 0x62, 0x76, 0x43, 0x7f, + 0x4f, 0xa7, 0xad, 0xe2, 0x6d, 0xb1, 0x59, 0xde, 0x0c, 0xae, 0xba, 0xa9, 0xcd, 0x17, 0x53, 0x8a, + 0xa9, 0x6b, 0xcd, 0x17, 0xbc, 0xb7, 0x52, 0x5b, 0xda, 0x29, 0xe3, 0xb4, 0x41, 0xf9, 0x8b, 0x04, + 0x1b, 0x59, 0xb3, 0x42, 0x7d, 0x58, 0x0b, 0xf4, 0x7c, 0x1b, 0xd4, 0x60, 0xcb, 0xaa, 0xdd, 0xaf, + 0x12, 0x13, 0x0e, 0xad, 0x38, 0x1d, 0x90, 0xce, 0x42, 0x66, 0x01, 0x50, 0xb3, 0xb2, 0x90, 0x99, + 0x8b, 0xd3, 0x01, 0xca, 0x9f, 0xa5, 0x8c, 0xc5, 0xa0, 0x2e, 0x54, 0x05, 0x27, 0xf0, 0xb5, 0x49, + 0x71, 0x38, 0x45, 0x7a, 0x1c, 0x77, 0x42, 0xcf, 0x61, 0x43, 0x88, 0x63, 0xcf, 0x72, 0xc8, 0x8c, + 0x9e, 0x73, 0xd2, 0x10, 0xd0, 0xb9, 0xb3, 0x17, 0x90, 0xc9, 0x5e, 0xc2, 0x8c, 0x33, 0x83, 0x94, + 0x97, 0x19, 0xcd, 0xa1, 0xfb, 0x31, 0x40, 0x4a, 0xd9, 0x67, 0x38, 0x86, 0x43, 0x4e, 0x0e, 0x8e, + 0x6e, 0xb7, 0xf2, 0x6d, 0x99, 0x9d, 0x10, 0x2e, 0x28, 0xbf, 0x85, 0xba, 0xa0, 0x10, 0x41, 0x55, + 0xdf, 0x81, 0xa2, 0x4b, 0x1d, 0x9d, 0x06, 0x07, 0xb3, 0xca, 0x53, 0x8e, 0xb9, 0x0a, 0x0b, 0x13, + 0xfa, 0x3e, 0xac, 0x18, 0xd4, 0x23, 0xa2, 0x97, 0xf5, 0x60, 0xbc, 0x8b, 0xb9, 0x77, 0x46, 0x3d, + 0x32, 0x25, 0x1e, 0xc1, 0xdc, 0x41, 0xf9, 0x20, 0x41, 0x71, 0x9c, 0x8c, 0x91, 0x62, 0x31, 0xbe, + 0x29, 0x19, 0x83, 0x7e, 0x0a, 0xb5, 0x29, 0xd5, 0x2c, 0xc3, 0x76, 0xa8, 0xeb, 0xd2, 0x69, 0x38, + 0x30, 0x16, 0xd0, 0x8f, 0x19, 0xfc, 0xe0, 0xe3, 0x1c, 0x4e, 0xb8, 0xa3, 0xa7, 0x00, 0xb1, 0x60, + 0x39, 0x16, 0x7c, 0xb6, 0x7f, 0x98, 0x0e, 0x8e, 0x39, 0x1f, 0x94, 0x04, 0x89, 0x28, 0xaf, 0xa0, + 0x91, 0x5c, 0x1a, 0x6a, 0x40, 0x5e, 0x9f, 0x0a, 0xc6, 0xc9, 0xeb, 0x53, 0x74, 0x17, 0x2a, 0x9c, + 0x5d, 0x55, 0xdd, 0xa0, 0x82, 0x5a, 0x23, 0x05, 0x6a, 0x41, 0x89, 0x9a, 0x53, 0x6e, 0xf3, 0x8f, + 0x7a, 0x20, 0x2a, 0x13, 0x40, 0xe9, 0x1e, 0xd0, 0x1e, 0x00, 0xab, 0x62, 0x5b, 0xba, 0xe9, 0x05, + 0x83, 0x6f, 0xf8, 0x0d, 0x07, 0x6a, 0x1c, 0xf3, 0x40, 0x77, 0x61, 0xc5, 0x63, 0xf0, 0xce, 0x73, + 0xcf, 0x72, 0xb0, 0xeb, 0x98, 0x6b, 0x95, 0x9f, 0x43, 0x25, 0x0c, 0x63, 0x0b, 0x65, 0xf7, 0x86, + 0xeb, 0x11, 0xc3, 0x16, 0x7c, 0x16, 0x29, 0x92, 0xb4, 0x29, 0x09, 0xda, 0x54, 0x3a, 0x20, 0xab, + 0x64, 0xf6, 0xf9, 0x3c, 0xab, 0xbc, 0x03, 0x94, 0x1e, 0x2e, 0xbb, 0xf5, 0xa2, 0x4e, 0xf9, 0x71, + 0xf4, 0x33, 0x2d, 0x69, 0xd1, 0x4f, 0x18, 0x8e, 0xed, 0xb9, 0xae, 0x91, 0xa0, 0xa3, 0xed, 0xd4, + 0x7e, 0xfd, 0x92, 0xd5, 0x71, 0xb1, 0xef, 0x86, 0x43, 0x7f, 0xe5, 0x18, 0x36, 0x6f, 0x75, 0x43, + 0xf7, 0xa0, 0xec, 0xd2, 0x99, 0x41, 0xa3, 0xa1, 0xae, 0x8a, 0xc4, 0x63, 0xa1, 0xc6, 0xa1, 0x83, + 0xf2, 0x3b, 0x80, 0x48, 0x8f, 0xbe, 0x86, 0xa2, 0x41, 0x9d, 0x19, 0x9d, 0x0a, 0xbc, 0x36, 0x92, + 0x81, 0x58, 0x58, 0xd1, 0x2e, 0x94, 0x17, 0xa6, 0xf0, 0xcc, 0xc7, 0xf6, 0x2d, 0xf2, 0x0c, 0xed, + 0x8a, 0x05, 0x95, 0x50, 0xcd, 0x86, 0x7b, 0x45, 0x49, 0x00, 0x29, 0xfe, 0x9b, 0xe9, 0x3c, 0xa2, + 0xcf, 0xc5, 0x6c, 0xf9, 0xef, 0x24, 0xd0, 0xe4, 0x65, 0xa0, 0xdd, 0x85, 0xca, 0x64, 0x6e, 0x69, + 0xd7, 0x63, 0xfd, 0x3d, 0xe5, 0x64, 0x27, 0xe3, 0x48, 0xa1, 0xfc, 0x5d, 0x82, 0xfa, 0x98, 0x12, + 0x27, 0x7a, 0x21, 0x3c, 0x5a, 0xbe, 0x7b, 0x3f, 0xeb, 0xe6, 0x0f, 0xdf, 0x15, 0xf9, 0x8c, 0x77, + 0x85, 0x1c, 0xbd, 0x2b, 0xbe, 0xf8, 0x85, 0x70, 0x04, 0xf5, 0xb3, 0x7d, 0x95, 0xcc, 0xce, 0x1d, + 0xcb, 0xa6, 0x8e, 0x77, 0x93, 0x3a, 0x6e, 0x69, 0x28, 0xe5, 0xb3, 0xa0, 0xa4, 0x0c, 0x60, 0x35, + 0x9e, 0x88, 0xa1, 0xb0, 0x0b, 0x60, 0x87, 0x92, 0x80, 0x01, 0x12, 0x7b, 0x14, 0x2b, 0x89, 0x63, + 0x5e, 0xca, 0x13, 0xfe, 0x62, 0x09, 0x57, 0xd3, 0x04, 0xf9, 0x9a, 0xde, 0x88, 0xe5, 0xb0, 0x9f, + 0xe8, 0x2b, 0x28, 0x72, 0xe4, 0x07, 0xeb, 0x10, 0x92, 0xd2, 0x83, 0x7a, 0xb2, 0xfa, 0x83, 0x8c, + 0xea, 0xe1, 0xbc, 0x33, 0x6b, 0x7f, 0x90, 0x18, 0xf9, 0xf8, 0x9b, 0x26, 0x38, 0xf9, 0xc7, 0x4b, + 0x8c, 0xe8, 0x6f, 0x1b, 0x5a, 0x4a, 0x93, 0x45, 0x86, 0x3f, 0x4a, 0x90, 0xa1, 0xcf, 0xa4, 0x1b, + 0xa9, 0xe6, 0x53, 0x4c, 0x18, 0x92, 0xb5, 0xfc, 0x09, 0x82, 0x8f, 0x28, 0xf3, 0x1f, 0x12, 0x6c, + 0xb1, 0x73, 0x38, 0xa7, 0x1e, 0xe5, 0x97, 0xab, 0x8f, 0xb8, 0xe0, 0x8e, 0xff, 0x81, 0x78, 0x89, + 0xf9, 0x57, 0xe7, 0x37, 0x78, 0xc2, 0xb8, 0x7b, 0xf4, 0x1c, 0x63, 0x7b, 0xfd, 0x5a, 0x9f, 0x7b, + 0xd4, 0x19, 0x12, 0x83, 0xaa, 0x01, 0xcd, 0xd5, 0xf0, 0x92, 0x36, 0x42, 0xa5, 0x9c, 0x81, 0xca, + 0x95, 0x4c, 0x54, 0x16, 0x3e, 0x85, 0x4a, 0xe5, 0x4f, 0x12, 0xac, 0x67, 0xb4, 0xf1, 0x85, 0x07, + 0xe7, 0x69, 0x54, 0xda, 0x9f, 0xfd, 0xb7, 0x53, 0x8d, 0x27, 0xe7, 0x94, 0x7d, 0x3c, 0xda, 0x50, + 0x56, 0xc9, 0x8c, 0x35, 0xce, 0xbb, 0x66, 0x44, 0xec, 0x63, 0xa9, 0x86, 0x7d, 0x41, 0x79, 0xc4, + 0x3d, 0x38, 0xfb, 0x7d, 0x02, 0xad, 0x72, 0x0c, 0xad, 0x5d, 0xa8, 0x04, 0x51, 0x2e, 0xfa, 0x5e, + 0xe8, 0xe4, 0xa3, 0xb4, 0x1e, 0x34, 0xc7, 0xed, 0x61, 0xcc, 0x5f, 0x25, 0xd8, 0x48, 0xae, 0x5f, + 0x80, 0x74, 0x17, 0x4a, 0x53, 0xfa, 0x9a, 0x2c, 0xe6, 0x5e, 0x82, 0x32, 0xc3, 0x02, 0xc7, 0x39, + 0x1c, 0x38, 0xa0, 0x1f, 0x42, 0x85, 0xaf, 0x7b, 0x64, 0xce, 0x83, 0x07, 0x51, 0x58, 0x8e, 0xb7, + 0x79, 0x9c, 0xc3, 0x91, 0xc7, 0x17, 0xa0, 0xf1, 0x0f, 0xd0, 0x48, 0x3a, 0xa0, 0x6d, 0x00, 0xfa, + 0xee, 0x8a, 0x2c, 0x5c, 0x4f, 0x7f, 0xeb, 0xc3, 0xb0, 0x8c, 0x63, 0x1a, 0xb4, 0x03, 0xe5, 0xdf, + 0x13, 0xc7, 0xd4, 0xcd, 0xf0, 0x5a, 0xad, 0xf1, 0x3a, 0x2f, 0x7d, 0x25, 0x0e, 0xad, 0xa8, 0x0d, + 0x55, 0x27, 0x7c, 0xd5, 0xb2, 0xaf, 0x27, 0x79, 0x47, 0xc6, 0x71, 0x95, 0xf2, 0x04, 0x4a, 0x22, + 0x2c, 0xf3, 0x0e, 0x6d, 0x41, 0xc9, 0xa0, 0xae, 0x4b, 0x66, 0xc1, 0x2d, 0x1a, 0x88, 0xbb, 0x14, + 0xaa, 0xb1, 0xcf, 0x13, 0x54, 0x81, 0xc2, 0xe0, 0xc5, 0x45, 0xef, 0xb4, 0x99, 0x43, 0x35, 0x28, + 0x0f, 0x47, 0xaa, 0x2f, 0x49, 0x08, 0xa0, 0x88, 0x07, 0x47, 0x83, 0x57, 0xe7, 0xcd, 0x3c, 0xaa, + 0x43, 0x65, 0x38, 0x52, 0x85, 0x28, 0x33, 0xd3, 0xe0, 0xd5, 0xc9, 0x58, 0x1d, 0x37, 0x57, 0x84, + 0x49, 0x88, 0x05, 0x54, 0x02, 0xb9, 0x77, 0x7a, 0xda, 0x2c, 0xee, 0x6a, 0x50, 0x8d, 0x3d, 0x5b, + 0x51, 0x0b, 0x36, 0x2e, 0x86, 0xcf, 0x87, 0xa3, 0x97, 0xc3, 0xcb, 0xb3, 0x81, 0x8a, 0x4f, 0x0e, + 0xc7, 0x97, 0xea, 0xaf, 0xce, 0x07, 0xcd, 0x1c, 0xfa, 0x16, 0x6c, 0x5e, 0x0c, 0x7b, 0x47, 0x47, + 0x78, 0x70, 0xd4, 0x53, 0x07, 0xfd, 0xa4, 0x59, 0x42, 0xdf, 0x84, 0x3b, 0xb7, 0x19, 0xf3, 0xbb, + 0x27, 0x50, 0x8b, 0x7f, 0x41, 0x20, 0x04, 0x8d, 0xfe, 0xe0, 0x59, 0xef, 0xe2, 0x54, 0xbd, 0x1c, + 0x9d, 0xab, 0x27, 0xa3, 0x61, 0x33, 0x87, 0xd6, 0xa0, 0xfe, 0x6c, 0x84, 0x0f, 0x07, 0x97, 0x83, + 0x61, 0xef, 0xe0, 0x74, 0xd0, 0x6f, 0x4a, 0xcc, 0xcd, 0x57, 0xf5, 0x4f, 0xc6, 0xbe, 0x2e, 0xbf, + 0x7b, 0x1f, 0x9a, 0xcb, 0x5c, 0x81, 0xaa, 0x50, 0x12, 0xe9, 0x9a, 0x39, 0x26, 0xa8, 0xbd, 0xa3, + 0x61, 0xef, 0x6c, 0xd0, 0x94, 0xba, 0xff, 0x93, 0xa0, 0xc0, 0xbf, 0x2f, 0xd0, 0x43, 0x28, 0xfa, + 0x1f, 0xe2, 0xc8, 0xe7, 0xca, 0xc4, 0x67, 0xfa, 0xd6, 0x7a, 0x42, 0x27, 0x50, 0xfc, 0x00, 0x0a, + 0x9c, 0x18, 0x50, 0x8c, 0x24, 0x82, 0x00, 0x14, 0x57, 0xf9, 0xfe, 0x0f, 0x24, 0xb4, 0xcf, 0x5e, + 0xb8, 0x8c, 0xae, 0x45, 0x91, 0xc4, 0x85, 0xbb, 0xb5, 0x9e, 0xd0, 0x85, 0x41, 0x03, 0xa8, 0xc5, + 0x3b, 0x42, 0xad, 0xdb, 0x78, 0x61, 0x6b, 0x33, 0xc3, 0x12, 0xa4, 0x39, 0xb8, 0xf3, 0xcf, 0x8f, + 0xdb, 0xd2, 0xbf, 0x3e, 0x6e, 0x4b, 0xff, 0xfe, 0xb8, 0x2d, 0xfd, 0xf1, 0x3f, 0xdb, 0xb9, 0x5f, + 0x17, 0xf8, 0x5f, 0x1d, 0x93, 0x22, 0xff, 0x6b, 0x62, 0xff, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, + 0x66, 0x87, 0x8b, 0xa9, 0x27, 0x11, 0x00, 0x00, } diff --git a/src/query/generated/proto/rpcpb/query.proto b/src/query/generated/proto/rpcpb/query.proto index acc3fa2f6f..a50a243762 100644 --- a/src/query/generated/proto/rpcpb/query.proto +++ b/src/query/generated/proto/rpcpb/query.proto @@ -56,7 +56,7 @@ enum MatcherType { message FetchOptions { int64 limit = 1; - RestrictFetchOptions restrict = 2; + RestrictQueryOptions restrict = 2; int64 lookbackDuration = 3; FanoutOption unaggregated = 4; FanoutOption aggregated = 5; @@ -64,11 +64,21 @@ message FetchOptions { bool includeResolution = 7; } -message RestrictFetchOptions { +message RestrictQueryOptions { + RestrictFetchType RestrictFetchType = 3; + RestrictFetchTags RestrictFetchTags = 4; +} + +message RestrictFetchType { MetricsType metricsType = 1; policypb.StoragePolicy metricsStoragePolicy = 2; } +message RestrictFetchTags { + TagMatchers restrict = 1; + repeated bytes strip = 2; +} + enum MetricsType { UNKNOWN_METRICS_TYPE = 0; UNAGGREGATED_METRICS_TYPE = 1; diff --git a/src/query/models/matcher.go b/src/query/models/matcher.go index 0991ff8b1a..0a10edfb8a 100644 --- a/src/query/models/matcher.go +++ b/src/query/models/matcher.go @@ -57,6 +57,15 @@ func NewMatcher(t MatchType, n, v []byte) (Matcher, error) { Value: v, } + if len(n) == 0 && t != MatchAll { + return Matcher{}, errors.New("name must be set unless using MatchAll") + } + + if len(v) == 0 && !(t == MatchAll || t == MatchField || t == MatchNotField) { + return Matcher{}, errors.New("field must be set unless using one " + + "of MatchField, MatchNotField, or MatchAll") + } + if t == MatchRegexp || t == MatchNotRegexp { re, err := regexp.Compile("^(?:" + string(v) + ")$") if err != nil { diff --git a/src/query/models/matcher_test.go b/src/query/models/matcher_test.go index af60022e57..7bb1de73e5 100644 --- a/src/query/models/matcher_test.go +++ b/src/query/models/matcher_test.go @@ -27,20 +27,12 @@ import ( "github.com/stretchr/testify/require" ) -func newMatcher(t *testing.T, mType MatchType, value string) Matcher { - m, err := NewMatcher(mType, []byte{}, []byte(value)) +func TestMatcherString(t *testing.T) { + m, err := NewMatcher(MatchEqual, []byte("foo"), []byte("bar")) require.NoError(t, err) require.NotNil(t, m) - - return m -} - -func TestMatcher_String(t *testing.T) { - m := newMatcher(t, MatchEqual, "foo") - m.Name = []byte(`key`) - - assert.Equal(t, `key="foo"`, m.String()) - assert.Equal(t, `key="foo"`, (&m).String()) + assert.Equal(t, `foo="bar"`, m.String()) + assert.Equal(t, `foo="bar"`, (&m).String()) } func TestMatchType(t *testing.T) { diff --git a/src/query/storage/fanout/storage.go b/src/query/storage/fanout/storage.go index a2062739ea..81430369f8 100644 --- a/src/query/storage/fanout/storage.go +++ b/src/query/storage/fanout/storage.go @@ -21,6 +21,7 @@ package fanout import ( + "bytes" "context" "fmt" "sync" @@ -332,9 +333,44 @@ func (s *fanoutStorage) CompleteTags( built := accumulatedTags.Build() built.Metadata = metadata + built = applyOptions(built, options) return &built, nil } +func applyOptions( + result storage.CompleteTagsResult, + opts *storage.FetchOptions, +) storage.CompleteTagsResult { + if opts.RestrictQueryOptions == nil { + return result + } + + filter := opts.RestrictQueryOptions.GetRestrictByTag().GetFilterByNames() + if len(filter) > 0 { + // Filter out unwanted tags inplace. + filteredList := result.CompletedTags[:0] + for _, s := range result.CompletedTags { + skip := false + for _, name := range filter { + if bytes.Equal(s.Name, name) { + skip = true + break + } + } + + if skip { + continue + } + + filteredList = append(filteredList, s) + } + + result.CompletedTags = filteredList + } + + return result +} + func (s *fanoutStorage) Write(ctx context.Context, query *storage.WriteQuery) error { // TODO: Consider removing this lookup on every write by maintaining diff --git a/src/query/storage/index.go b/src/query/storage/index.go index 0cff5702dc..58a2790748 100644 --- a/src/query/storage/index.go +++ b/src/query/storage/index.go @@ -123,9 +123,11 @@ func FetchOptionsToAggregateOptions( // FetchQueryToM3Query converts an m3coordinator fetch query to an M3 query. func FetchQueryToM3Query( fetchQuery *FetchQuery, + options *FetchOptions, ) (index.Query, error) { + fetchQuery = fetchQuery.WithAppliedOptions(options) matchers := fetchQuery.TagMatchers - // If no matchers provided, explicitly set this to an AllQuery + // If no matchers provided, explicitly set this to an AllQuery. if len(matchers) == 0 { return index.Query{ Query: idx.NewAllQuery(), diff --git a/src/query/storage/index_test.go b/src/query/storage/index_test.go index c1138063a5..9d7cedbf5d 100644 --- a/src/query/storage/index_test.go +++ b/src/query/storage/index_test.go @@ -201,7 +201,7 @@ func TestFetchQueryToM3Query(t *testing.T) { Interval: 15 * time.Second, } - m3Query, err := FetchQueryToM3Query(fetchQuery) + m3Query, err := FetchQueryToM3Query(fetchQuery, nil) require.NoError(t, err) assert.Equal(t, test.expected, m3Query.String()) }) diff --git a/src/query/storage/m3/cluster_resolver.go b/src/query/storage/m3/cluster_resolver.go index 7955cb08b5..4ddecca550 100644 --- a/src/query/storage/m3/cluster_resolver.go +++ b/src/query/storage/m3/cluster_resolver.go @@ -76,12 +76,12 @@ func resolveClusterNamespacesForQuery( now, start, end time.Time, clusters Clusters, opts *storage.FanoutOptions, - restrict *storage.RestrictFetchOptions, + restrict *storage.RestrictQueryOptions, ) (queryFanoutType, ClusterNamespaces, error) { - if restrict != nil { + if typeRestrict := restrict.GetRestrictByType(); typeRestrict != nil { // If a specific restriction is set, then attempt to satisfy. - return resolveClusterNamespacesForQueryWithRestrictFetchOptions(now, - start, clusters, restrict) + return resolveClusterNamespacesForQueryWithRestrictQueryOptions(now, + start, clusters, *typeRestrict) } // First check if the unaggregated cluster can fully satisfy the query range. @@ -295,18 +295,19 @@ func aggregatedNamespaces( return slices } -// resolveClusterNamespacesForQueryWithRestrictFetchOptions returns the cluster +// resolveClusterNamespacesForQueryWithRestrictQueryOptions returns the cluster // namespace referred to by the restrict fetch options or an error if it // cannot be found. -func resolveClusterNamespacesForQueryWithRestrictFetchOptions( +func resolveClusterNamespacesForQueryWithRestrictQueryOptions( now, start time.Time, clusters Clusters, - restrict *storage.RestrictFetchOptions, + restrict storage.RestrictByType, ) (queryFanoutType, ClusterNamespaces, error) { coversRangeFilter := newCoversRangeFilter(coversRangeFilterOptions{ now: now, queryStart: start, }) + result := func( namespace ClusterNamespace, err error, @@ -314,10 +315,12 @@ func resolveClusterNamespacesForQueryWithRestrictFetchOptions( if err != nil { return 0, nil, err } + if coversRangeFilter(namespace) { return namespaceCoversAllQueryRange, ClusterNamespaces{namespace}, nil } + return namespaceCoversPartialQueryRange, ClusterNamespaces{namespace}, nil } @@ -335,6 +338,7 @@ func resolveClusterNamespacesForQueryWithRestrictFetchOptions( fmt.Errorf("could not find namespace for storage policy: %v", restrict.StoragePolicy.String())) } + return result(ns, nil) default: return result(nil, diff --git a/src/query/storage/m3/cluster_resolver_test.go b/src/query/storage/m3/cluster_resolver_test.go index b93b95facf..c0fa180971 100644 --- a/src/query/storage/m3/cluster_resolver_test.go +++ b/src/query/storage/m3/cluster_resolver_test.go @@ -163,7 +163,7 @@ var testCases = []struct { name string queryLength time.Duration opts *storage.FanoutOptions - restrict *storage.RestrictFetchOptions + restrict *storage.RestrictQueryOptions expectedType queryFanoutType expectedClusterNames []string expectedErr error @@ -311,8 +311,10 @@ var testCases = []struct { { name: "restrict to unaggregated", queryLength: time.Hour * 1000, - restrict: &storage.RestrictFetchOptions{ - MetricsType: storage.UnaggregatedMetricsType, + restrict: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.UnaggregatedMetricsType, + }, }, expectedType: namespaceCoversPartialQueryRange, expectedClusterNames: []string{"UNAGG"}, @@ -320,10 +322,12 @@ var testCases = []struct { { name: "restrict to aggregate filtered", queryLength: time.Hour * 1000, - restrict: &storage.RestrictFetchOptions{ - MetricsType: storage.AggregatedMetricsType, - StoragePolicy: policy.MustParseStoragePolicy( - genResolution.String() + ":" + genRetentionFiltered.String()), + restrict: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.MustParseStoragePolicy( + genResolution.String() + ":" + genRetentionFiltered.String()), + }, }, expectedType: namespaceCoversPartialQueryRange, expectedClusterNames: []string{"AGG_FILTERED"}, @@ -331,10 +335,12 @@ var testCases = []struct { { name: "restrict to aggregate unfiltered", queryLength: time.Hour * 1000, - restrict: &storage.RestrictFetchOptions{ - MetricsType: storage.AggregatedMetricsType, - StoragePolicy: policy.MustParseStoragePolicy( - genResolution.String() + ":" + genRetentionUnfiltered.String()), + restrict: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.MustParseStoragePolicy( + genResolution.String() + ":" + genRetentionUnfiltered.String()), + }, }, expectedType: namespaceCoversPartialQueryRange, expectedClusterNames: []string{"AGG_NO_FILTER"}, @@ -342,17 +348,21 @@ var testCases = []struct { { name: "restrict with unknown metrics type", queryLength: time.Hour * 1000, - restrict: &storage.RestrictFetchOptions{ - MetricsType: storage.UnknownMetricsType, + restrict: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.UnknownMetricsType, + }, }, expectedErrContains: "unrecognized metrics type:", }, { name: "restrict with unknown storage policy", queryLength: time.Hour * 1000, - restrict: &storage.RestrictFetchOptions{ - MetricsType: storage.AggregatedMetricsType, - StoragePolicy: policy.MustParseStoragePolicy("1s:100d"), + restrict: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.MustParseStoragePolicy("1s:100d"), + }, }, expectedErrContains: "could not find namespace for storage policy:", }, diff --git a/src/query/storage/m3/storage.go b/src/query/storage/m3/storage.go index fb488bd9c4..165c126cb3 100644 --- a/src/query/storage/m3/storage.go +++ b/src/query/storage/m3/storage.go @@ -286,7 +286,7 @@ func (s *m3storage) fetchCompressed( default: } - m3query, err := storage.FetchQueryToM3Query(query) + m3query, err := storage.FetchQueryToM3Query(query, options) if err != nil { return nil, err } @@ -301,7 +301,7 @@ func (s *m3storage) fetchCompressed( query.End, s.clusters, options.FanoutOptions, - options.RestrictFetchOptions, + options.RestrictQueryOptions, ) if err != nil { return nil, err @@ -425,7 +425,7 @@ func (s *m3storage) CompleteTags( TagMatchers: query.TagMatchers, } - m3query, err := storage.FetchQueryToM3Query(fetchQuery) + m3query, err := storage.FetchQueryToM3Query(fetchQuery, options) if err != nil { return nil, err } @@ -552,7 +552,7 @@ func (s *m3storage) SearchCompressed( default: } - m3query, err := storage.FetchQueryToM3Query(query) + m3query, err := storage.FetchQueryToM3Query(query, options) if err != nil { return tagResult, noop, err } diff --git a/src/query/storage/restrict_query_options.go b/src/query/storage/restrict_query_options.go new file mode 100644 index 0000000000..19aa847fa7 --- /dev/null +++ b/src/query/storage/restrict_query_options.go @@ -0,0 +1,163 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package storage + +import ( + "bytes" + "fmt" + + "github.com/m3db/m3/src/metrics/policy" + "github.com/m3db/m3/src/query/models" +) + +// Validate will validate the restrict fetch options. +func (o *RestrictQueryOptions) Validate() error { + if o.RestrictByType != nil { + return o.RestrictByType.Validate() + } + + return nil +} + +// GetRestrictByType provides the type restrictions if present; nil otherwise. +func (o *RestrictQueryOptions) GetRestrictByType() *RestrictByType { + if o == nil { + return nil + } + + return o.RestrictByType +} + +// GetRestrictByTag provides the tag restrictions if present; nil otherwise. +func (o *RestrictQueryOptions) GetRestrictByTag() *RestrictByTag { + if o == nil { + return nil + } + + return o.RestrictByTag +} + +// GetMatchers provides the tag matchers by which results are restricted if +// present; nil otherwise. +func (o *RestrictByTag) GetMatchers() models.Matchers { + if o == nil { + return nil + } + + return o.Restrict +} + +// Validate will validate the restrict type restrictions. +func (o *RestrictByType) Validate() error { + switch o.MetricsType { + case UnaggregatedMetricsType: + if o.StoragePolicy != policy.EmptyStoragePolicy { + return fmt.Errorf( + "expected no storage policy for unaggregated metrics type, "+ + "instead got: %v", o.StoragePolicy.String()) + } + case AggregatedMetricsType: + if v := o.StoragePolicy.Resolution().Window; v <= 0 { + return fmt.Errorf( + "expected positive resolution window, instead got: %v", v) + } + if v := o.StoragePolicy.Resolution().Precision; v <= 0 { + return fmt.Errorf( + "expected positive resolution precision, instead got: %v", v) + } + if v := o.StoragePolicy.Retention().Duration(); v <= 0 { + return fmt.Errorf( + "expected positive retention, instead got: %v", v) + } + default: + return fmt.Errorf( + "unknown metrics type: %v", o.MetricsType) + } + return nil +} + +// GetFilterByNames returns the tag names to filter out of the response. +func (o *RestrictByTag) GetFilterByNames() [][]byte { + if o == nil { + return nil + } + + if o.Strip != nil { + return o.Strip + } + + o.Strip = make([][]byte, 0, len(o.Restrict)) + for _, r := range o.Restrict { + o.Strip = append(o.Strip, r.Name) + } + + return o.Strip +} + +// WithAppliedOptions returns a copy of the fetch query applied options +// that restricts the fetch with respect to labels that must be applied. +func (q *FetchQuery) WithAppliedOptions( + opts *FetchOptions, +) *FetchQuery { + result := *q + if opts == nil { + return &result + } + + restrictOpts := opts.RestrictQueryOptions + if restrictOpts == nil { + return &result + } + + restrict := restrictOpts.GetRestrictByTag().GetMatchers() + if len(restrict) == 0 { + return &result + } + + // Since must apply matchers will always be small (usually 1) + // it's better to not allocate intermediate datastructure and just + // perform n^2 matching. + existing := result.TagMatchers + for _, existingMatcher := range result.TagMatchers { + willBeOverridden := false + for _, matcher := range restrict { + if bytes.Equal(existingMatcher.Name, matcher.Name) { + willBeOverridden = true + break + } + } + + if willBeOverridden { + // We'll override this when we append the restrict matchers. + continue + } + + existing = append(existing, existingMatcher) + } + + // Now append the must apply matchers. + result.TagMatchers = append(existing, restrict...) + return &result +} + +func (q *FetchQuery) String() string { + return q.Raw +} diff --git a/src/query/storage/restrict_query_options_test.go b/src/query/storage/restrict_query_options_test.go new file mode 100644 index 0000000000..f0825001a5 --- /dev/null +++ b/src/query/storage/restrict_query_options_test.go @@ -0,0 +1,53 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package storage + +import ( + "testing" + + "github.com/m3db/m3/src/query/models" + + "github.com/stretchr/testify/require" +) + +func TestGetRestrict(t *testing.T) { + var opts *RestrictQueryOptions + require.Nil(t, opts.GetRestrictByTag()) + require.Nil(t, opts.GetRestrictByType()) + + opts = &RestrictQueryOptions{} + require.Nil(t, opts.GetRestrictByTag()) + require.Nil(t, opts.GetRestrictByType()) + + opts.RestrictByTag = &RestrictByTag{} + require.NotNil(t, opts.GetRestrictByTag()) + + matcher, err := models.NewMatcher(models.MatchEqual, []byte("f"), []byte("b")) + require.NoError(t, err) + matchers := models.Matchers{matcher} + + opts.RestrictByTag.Restrict = matchers + require.Equal(t, matchers, opts.GetRestrictByTag().GetMatchers()) + + byType := &RestrictByType{} + opts.RestrictByType = byType + require.Equal(t, byType, opts.GetRestrictByType()) +} diff --git a/src/query/storage/types.go b/src/query/storage/types.go index 139bc59600..9aeb599a5b 100644 --- a/src/query/storage/types.go +++ b/src/query/storage/types.go @@ -22,14 +22,12 @@ package storage import ( "context" - "errors" "fmt" "time" "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/query/block" "github.com/m3db/m3/src/query/cost" - "github.com/m3db/m3/src/query/generated/proto/rpcpb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/ts" xtime "github.com/m3db/m3/src/x/time" @@ -37,10 +35,6 @@ import ( "github.com/uber-go/tally" ) -var ( - errNoRestrictFetchOptionsProtoMsg = errors.New("no restrict fetch options proto message") -) - // Type describes the type of storage. type Type int @@ -107,10 +101,6 @@ type FetchQuery struct { Interval time.Duration `json:"interval"` } -func (q *FetchQuery) String() string { - return q.Raw -} - // FetchOptions represents the options for fetch query. type FetchOptions struct { // Remote is set when this fetch is originated by a remote grpc call. @@ -121,9 +111,9 @@ type FetchOptions struct { BlockType models.FetchedBlockType // FanoutOptions are the options for the fetch namespace fanout. FanoutOptions *FanoutOptions - // RestrictFetchOptions restricts the fetch to a specific set of + // RestrictQueryOptions restricts the fetch to a specific set of // conditions. - RestrictFetchOptions *RestrictFetchOptions + RestrictQueryOptions *RestrictQueryOptions // Step is the configured step size. Step time.Duration // LookbackDuration if set overrides the default lookback duration. @@ -199,18 +189,27 @@ func (o *FetchOptions) QueryFetchOptions( if r.Limit <= 0 { r.Limit = queryCtx.Options.LimitMaxTimeseries } - if r.RestrictFetchOptions == nil && queryCtx.Options.RestrictFetchType != nil { + + // Use inbuilt options for type restriction if none found. + if r.RestrictQueryOptions.GetRestrictByType() == nil && + queryCtx.Options.RestrictFetchType != nil { v := queryCtx.Options.RestrictFetchType - restrict := RestrictFetchOptions{ + restrict := &RestrictByType{ MetricsType: MetricsType(v.MetricsType), StoragePolicy: v.StoragePolicy, } + if err := restrict.Validate(); err != nil { return nil, err } - r.RestrictFetchOptions = &restrict + if r.RestrictQueryOptions == nil { + r.RestrictQueryOptions = &RestrictQueryOptions{} + } + + r.RestrictQueryOptions.RestrictByType = restrict } + return r, nil } @@ -220,8 +219,8 @@ func (o *FetchOptions) Clone() *FetchOptions { return &result } -// RestrictFetchOptions restricts the fetch to a specific set of conditions. -type RestrictFetchOptions struct { +// RestrictByType are specific restrictions to stick to a single data type. +type RestrictByType struct { // MetricsType restricts the type of metrics being returned. MetricsType MetricsType // StoragePolicy is required if metrics type is not unaggregated @@ -229,95 +228,27 @@ type RestrictFetchOptions struct { StoragePolicy policy.StoragePolicy } -// NewRestrictFetchOptionsFromProto returns a restrict fetch options from -// protobuf message. -// TODO: (arnikola) extract these out of types.go -func NewRestrictFetchOptionsFromProto( - p *rpcpb.RestrictFetchOptions, -) (RestrictFetchOptions, error) { - var result RestrictFetchOptions - - if p == nil { - return result, errNoRestrictFetchOptionsProtoMsg - } - - switch p.MetricsType { - case rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE: - result.MetricsType = UnaggregatedMetricsType - case rpcpb.MetricsType_AGGREGATED_METRICS_TYPE: - result.MetricsType = AggregatedMetricsType - } - - if p.MetricsStoragePolicy != nil { - storagePolicy, err := policy.NewStoragePolicyFromProto( - p.MetricsStoragePolicy) - if err != nil { - return result, err - } - - result.StoragePolicy = storagePolicy - } - - // Validate the resulting options. - if err := result.Validate(); err != nil { - return result, err - } - - return result, nil -} - -// Validate will validate the restrict fetch options. -func (o RestrictFetchOptions) Validate() error { - switch o.MetricsType { - case UnaggregatedMetricsType: - if o.StoragePolicy != policy.EmptyStoragePolicy { - return fmt.Errorf( - "expected no storage policy for unaggregated metrics type, "+ - "instead got: %v", o.StoragePolicy.String()) - } - case AggregatedMetricsType: - if v := o.StoragePolicy.Resolution().Window; v <= 0 { - return fmt.Errorf( - "expected positive resolution window, instead got: %v", v) - } - if v := o.StoragePolicy.Resolution().Precision; v <= 0 { - return fmt.Errorf( - "expected positive resolution precision, instead got: %v", v) - } - if v := o.StoragePolicy.Retention().Duration(); v <= 0 { - return fmt.Errorf( - "expected positive retention, instead got: %v", v) - } - default: - return fmt.Errorf( - "unknown metrics type: %v", o.MetricsType) - } - return nil +// RestrictByTag are specific restrictions to enforce behavior for given +// tags. +type RestrictByTag struct { + // Restrict is a set of override matchers to apply to a fetch + // regardless of the existing fetch matchers, they should replace any + // existing matchers part of a fetch if they collide. + Restrict models.Matchers + // Strip is a set of tag names to strip from the response. + // + // NB: If this is unset, but Restrict is set, all tag names appearing in any + // of the Restrict matchers are removed. + Strip [][]byte } -// Proto returns the protobuf message that corresponds to RestrictFetchOptions. -func (o RestrictFetchOptions) Proto() (*rpcpb.RestrictFetchOptions, error) { - if err := o.Validate(); err != nil { - return nil, err - } - - result := &rpcpb.RestrictFetchOptions{} - - switch o.MetricsType { - case UnaggregatedMetricsType: - result.MetricsType = rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE - case AggregatedMetricsType: - result.MetricsType = rpcpb.MetricsType_AGGREGATED_METRICS_TYPE - - storagePolicyProto, err := o.StoragePolicy.Proto() - if err != nil { - return nil, err - } - - result.MetricsStoragePolicy = storagePolicyProto - } - - return result, nil +// RestrictQueryOptions restricts the query to a specific set of conditions. +type RestrictQueryOptions struct { + // RestrictByType are specific restrictions to stick to a single data type. + RestrictByType *RestrictByType + // RestrictByTag are specific restrictions to enforce behavior for given + // tags. + RestrictByTag *RestrictByTag } // Querier handles queries against a storage. diff --git a/src/query/storage/types_test.go b/src/query/storage/types_test.go deleted file mode 100644 index 9774fd34e3..0000000000 --- a/src/query/storage/types_test.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package storage - -import ( - "fmt" - "math" - "strings" - "testing" - "time" - - "github.com/m3db/m3/src/metrics/generated/proto/policypb" - "github.com/m3db/m3/src/metrics/policy" - "github.com/m3db/m3/src/query/generated/proto/rpcpb" - xtime "github.com/m3db/m3/src/x/time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNewRestrictFetchOptionsFromProto(t *testing.T) { - tests := []struct { - value *rpcpb.RestrictFetchOptions - expected RestrictFetchOptions - errContains string - }{ - { - value: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE, - }, - expected: RestrictFetchOptions{ - MetricsType: UnaggregatedMetricsType, - }, - }, - { - value: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, - MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ - WindowSize: int64(time.Minute), - Precision: int64(time.Second), - }, - Retention: &policypb.Retention{ - Period: int64(24 * time.Hour), - }, - }, - }, - expected: RestrictFetchOptions{ - MetricsType: AggregatedMetricsType, - StoragePolicy: policy.NewStoragePolicy(time.Minute, - xtime.Second, 24*time.Hour), - }, - }, - { - value: nil, - errContains: errNoRestrictFetchOptionsProtoMsg.Error(), - }, - { - value: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_UNKNOWN_METRICS_TYPE, - }, - errContains: "unknown metrics type:", - }, - { - value: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE, - MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ - WindowSize: int64(time.Minute), - Precision: int64(time.Second), - }, - Retention: &policypb.Retention{ - Period: int64(24 * time.Hour), - }, - }, - }, - errContains: "expected no storage policy for unaggregated metrics", - }, - { - value: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, - MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ - WindowSize: -1, - }, - }, - }, - errContains: "unable to convert from duration to time unit", - }, - { - value: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, - MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ - WindowSize: int64(time.Minute), - Precision: int64(-1), - }, - }, - }, - errContains: "unable to convert from duration to time unit", - }, - { - value: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, - MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ - WindowSize: int64(time.Minute), - Precision: int64(time.Second), - }, - Retention: &policypb.Retention{ - Period: int64(-1), - }, - }, - }, - errContains: "expected positive retention", - }, - } - for _, test := range tests { - t.Run(fmt.Sprintf("%s", test.value), func(t *testing.T) { - result, err := NewRestrictFetchOptionsFromProto(test.value) - if test.errContains == "" { - require.NoError(t, err) - assert.Equal(t, test.expected, result) - return - } - - require.Error(t, err) - assert.True(t, - strings.Contains(err.Error(), test.errContains), - fmt.Sprintf("err=%v, want_contains=%v", err.Error(), test.errContains)) - }) - } -} - -func TestRestrictFetchOptionsProto(t *testing.T) { - tests := []struct { - value RestrictFetchOptions - expected *rpcpb.RestrictFetchOptions - errContains string - }{ - { - value: RestrictFetchOptions{ - MetricsType: UnaggregatedMetricsType, - }, - expected: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE, - }, - }, - { - value: RestrictFetchOptions{ - MetricsType: AggregatedMetricsType, - StoragePolicy: policy.NewStoragePolicy(time.Minute, - xtime.Second, 24*time.Hour), - }, - expected: &rpcpb.RestrictFetchOptions{ - MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, - MetricsStoragePolicy: &policypb.StoragePolicy{ - Resolution: &policypb.Resolution{ - WindowSize: int64(time.Minute), - Precision: int64(time.Second), - }, - Retention: &policypb.Retention{ - Period: int64(24 * time.Hour), - }, - }, - }, - }, - { - value: RestrictFetchOptions{ - MetricsType: MetricsType(uint(math.MaxUint16)), - }, - errContains: "unknown metrics type:", - }, - { - value: RestrictFetchOptions{ - MetricsType: UnaggregatedMetricsType, - StoragePolicy: policy.NewStoragePolicy(time.Minute, - xtime.Second, 24*time.Hour), - }, - errContains: "expected no storage policy for unaggregated metrics", - }, - } - for _, test := range tests { - t.Run(fmt.Sprintf("%s", test.value), func(t *testing.T) { - result, err := test.value.Proto() - if test.errContains == "" { - require.NoError(t, err) - assert.Equal(t, test.expected, result) - return - } - - require.Error(t, err) - assert.True(t, strings.Contains(err.Error(), test.errContains)) - }) - } -} diff --git a/src/query/tsdb/remote/codecs.go b/src/query/tsdb/remote/codecs.go index 773850a18d..706866a597 100644 --- a/src/query/tsdb/remote/codecs.go +++ b/src/query/tsdb/remote/codecs.go @@ -27,8 +27,10 @@ import ( "strings" "time" + "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/query/api/v1/handler" "github.com/m3db/m3/src/query/block" + "github.com/m3db/m3/src/query/generated/proto/rpcpb" rpc "github.com/m3db/m3/src/query/generated/proto/rpcpb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage" @@ -119,6 +121,10 @@ func encodeFetchRequest( } func encodeTagMatchers(modelMatchers models.Matchers) (*rpc.TagMatchers, error) { + if modelMatchers == nil { + return nil, nil + } + matchers := make([]*rpc.TagMatcher, len(modelMatchers)) for i, matcher := range modelMatchers { t, err := encodeMatcherTypeToProto(matcher.Type) @@ -148,7 +154,7 @@ func encodeFanoutOption(opt storage.FanoutOption) (rpc.FanoutOption, error) { return rpc.FanoutOption_FORCE_ENABLED, nil } - return 0, fmt.Errorf("unknown fanout option for proto encoding: %v\n", opt) + return 0, fmt.Errorf("unknown fanout option for proto encoding: %v", opt) } func encodeFetchOptions(options *storage.FetchOptions) (*rpc.FetchOptions, error) { @@ -180,8 +186,8 @@ func encodeFetchOptions(options *storage.FetchOptions) (*rpc.FetchOptions, error } result.AggregatedOptimized = aggOpt - if v := options.RestrictFetchOptions; v != nil { - restrict, err := v.Proto() + if v := options.RestrictQueryOptions; v != nil { + restrict, err := encodeRestrictQueryOptions(v) if err != nil { return nil, err } @@ -196,6 +202,72 @@ func encodeFetchOptions(options *storage.FetchOptions) (*rpc.FetchOptions, error return result, nil } +func encodeRestrictQueryOptionsByType( + o *storage.RestrictByType, +) (*rpcpb.RestrictFetchType, error) { + if o == nil { + return nil, nil + } + + if err := o.Validate(); err != nil { + return nil, err + } + + result := &rpcpb.RestrictFetchType{} + switch o.MetricsType { + case storage.UnaggregatedMetricsType: + result.MetricsType = rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE + case storage.AggregatedMetricsType: + result.MetricsType = rpcpb.MetricsType_AGGREGATED_METRICS_TYPE + + storagePolicyProto, err := o.StoragePolicy.Proto() + if err != nil { + return nil, err + } + + result.MetricsStoragePolicy = storagePolicyProto + } + + return result, nil +} + +func encodeRestrictQueryOptionsByTag( + o *storage.RestrictByTag, +) (*rpcpb.RestrictFetchTags, error) { + if o == nil { + return nil, nil + } + + matchers, err := encodeTagMatchers(o.GetMatchers()) + if err != nil { + return nil, err + } + + return &rpcpb.RestrictFetchTags{ + Restrict: matchers, + Strip: o.Strip, + }, nil +} + +func encodeRestrictQueryOptions( + o *storage.RestrictQueryOptions, +) (*rpcpb.RestrictQueryOptions, error) { + byType, err := encodeRestrictQueryOptionsByType(o.GetRestrictByType()) + if err != nil { + return nil, err + } + + byTags, err := encodeRestrictQueryOptionsByTag(o.GetRestrictByTag()) + if err != nil { + return nil, err + } + + return &rpcpb.RestrictQueryOptions{ + RestrictFetchType: byType, + RestrictFetchTags: byTags, + }, nil +} + func encodeMatcherTypeToProto(t models.MatchType) (rpc.MatcherType, error) { switch t { case models.MatchEqual: @@ -302,7 +374,77 @@ func decodeFanoutOption(opt rpc.FanoutOption) (storage.FanoutOption, error) { return storage.FanoutForceEnable, nil } - return 0, fmt.Errorf("unknown fanout option for proto encoding: %v\n", opt) + return 0, fmt.Errorf("unknown fanout option for proto encoding: %v", opt) +} + +func decodeRestrictQueryOptionsByType( + p *rpc.RestrictFetchType, +) (*storage.RestrictByType, error) { + if p == nil { + return nil, nil + } + + result := &storage.RestrictByType{} + switch p.GetMetricsType() { + case rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE: + result.MetricsType = storage.UnaggregatedMetricsType + case rpcpb.MetricsType_AGGREGATED_METRICS_TYPE: + result.MetricsType = storage.AggregatedMetricsType + } + + if p.GetMetricsStoragePolicy() != nil { + storagePolicy, err := policy.NewStoragePolicyFromProto( + p.MetricsStoragePolicy) + if err != nil { + return result, err + } + + result.StoragePolicy = storagePolicy + } + + if err := result.Validate(); err != nil { + return nil, err + + } + + return result, nil +} + +func decodeRestrictQueryOptionsByTag( + p *rpc.RestrictFetchTags, +) (*storage.RestrictByTag, error) { + if p == nil { + return nil, nil + } + + matchers, err := decodeTagMatchers(p.GetRestrict()) + if err != nil { + return nil, err + } + + return &storage.RestrictByTag{ + Restrict: matchers, + Strip: p.Strip, + }, nil +} + +func decodeRestrictQueryOptions( + p *rpc.RestrictQueryOptions, +) (*storage.RestrictQueryOptions, error) { + byType, err := decodeRestrictQueryOptionsByType(p.GetRestrictFetchType()) + if err != nil { + return nil, err + } + + byTag, err := decodeRestrictQueryOptionsByTag(p.GetRestrictFetchTags()) + if err != nil { + return nil, err + } + + return &storage.RestrictQueryOptions{ + RestrictByType: byType, + RestrictByTag: byTag, + }, nil } func decodeFetchOptions(rpcFetchOptions *rpc.FetchOptions) (*storage.FetchOptions, error) { @@ -336,11 +478,12 @@ func decodeFetchOptions(rpcFetchOptions *rpc.FetchOptions) (*storage.FetchOption } if v := rpcFetchOptions.Restrict; v != nil { - restrict, err := storage.NewRestrictFetchOptionsFromProto(v) + restrict, err := decodeRestrictQueryOptions(v) if err != nil { return nil, err } - result.RestrictFetchOptions = &restrict + + result.RestrictQueryOptions = restrict } if v := rpcFetchOptions.LookbackDuration; v > 0 { diff --git a/src/query/tsdb/remote/codecs_test.go b/src/query/tsdb/remote/codecs_test.go index 1996f18e5c..e0b5a0d8c2 100644 --- a/src/query/tsdb/remote/codecs_test.go +++ b/src/query/tsdb/remote/codecs_test.go @@ -22,18 +22,24 @@ package remote import ( "context" + "fmt" + "math" "net/http" + "strings" "testing" "time" + "github.com/m3db/m3/src/metrics/generated/proto/policypb" "github.com/m3db/m3/src/metrics/policy" "github.com/m3db/m3/src/query/api/v1/handler" + "github.com/m3db/m3/src/query/generated/proto/rpcpb" rpc "github.com/m3db/m3/src/query/generated/proto/rpcpb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/storage" "github.com/m3db/m3/src/query/test" "github.com/m3db/m3/src/query/util/logging" "github.com/m3db/m3/src/x/instrument" + xtime "github.com/m3db/m3/src/x/time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -126,9 +132,11 @@ func TestEncodeFetchMessage(t *testing.T) { rQ, start, end := createStorageFetchQuery(t) fetchOpts := storage.NewFetchOptions() fetchOpts.Limit = 42 - fetchOpts.RestrictFetchOptions = &storage.RestrictFetchOptions{ - MetricsType: storage.AggregatedMetricsType, - StoragePolicy: policy.MustParseStoragePolicy("1m:14d"), + fetchOpts.RestrictQueryOptions = &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.MustParseStoragePolicy("1m:14d"), + }, } lookback := time.Minute fetchOpts.LookbackDuration = &lookback @@ -149,11 +157,15 @@ func TestEncodeFetchMessage(t *testing.T) { require.NotNil(t, grpcQ.Options) assert.Equal(t, int64(42), grpcQ.Options.Limit) require.NotNil(t, grpcQ.Options.Restrict) - assert.Equal(t, rpc.MetricsType_AGGREGATED_METRICS_TYPE, grpcQ.Options.Restrict.MetricsType) - require.NotNil(t, grpcQ.Options.Restrict.MetricsStoragePolicy) - expectedStoragePolicyProto, err := fetchOpts.RestrictFetchOptions.StoragePolicy.Proto() + require.NotNil(t, grpcQ.Options.Restrict.RestrictFetchType) + assert.Equal(t, rpc.MetricsType_AGGREGATED_METRICS_TYPE, + grpcQ.Options.Restrict.RestrictFetchType.MetricsType) + require.NotNil(t, grpcQ.Options.Restrict.RestrictFetchType.MetricsStoragePolicy) + expectedStoragePolicyProto, err := fetchOpts.RestrictQueryOptions. + RestrictByType.StoragePolicy.Proto() require.NoError(t, err) - assert.Equal(t, expectedStoragePolicyProto, grpcQ.Options.Restrict.MetricsStoragePolicy) + assert.Equal(t, expectedStoragePolicyProto, grpcQ.Options.Restrict. + RestrictFetchType.MetricsStoragePolicy) assert.Equal(t, lookback, time.Duration(grpcQ.Options.LookbackDuration)) } @@ -161,9 +173,11 @@ func TestEncodeDecodeFetchQuery(t *testing.T) { rQ, _, _ := createStorageFetchQuery(t) fetchOpts := storage.NewFetchOptions() fetchOpts.Limit = 42 - fetchOpts.RestrictFetchOptions = &storage.RestrictFetchOptions{ - MetricsType: storage.AggregatedMetricsType, - StoragePolicy: policy.MustParseStoragePolicy("1m:14d"), + fetchOpts.RestrictQueryOptions = &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.MustParseStoragePolicy("1m:14d"), + }, } lookback := time.Minute fetchOpts.LookbackDuration = &lookback @@ -177,10 +191,12 @@ func TestEncodeDecodeFetchQuery(t *testing.T) { require.NoError(t, err) require.NotNil(t, revertedOpts) require.Equal(t, fetchOpts.Limit, revertedOpts.Limit) - require.Equal(t, fetchOpts.RestrictFetchOptions.MetricsType, - revertedOpts.RestrictFetchOptions.MetricsType) - require.Equal(t, fetchOpts.RestrictFetchOptions.StoragePolicy.String(), - revertedOpts.RestrictFetchOptions.StoragePolicy.String()) + require.Equal(t, fetchOpts.RestrictQueryOptions. + RestrictByType.MetricsType, + revertedOpts.RestrictQueryOptions.RestrictByType.MetricsType) + require.Equal(t, fetchOpts.RestrictQueryOptions. + RestrictByType.StoragePolicy.String(), + revertedOpts.RestrictQueryOptions.RestrictByType.StoragePolicy.String()) require.NotNil(t, revertedOpts.LookbackDuration) require.Equal(t, lookback, *revertedOpts.LookbackDuration) @@ -220,3 +236,269 @@ func TestRetrieveMetadata(t *testing.T) { require.Equal(t, requestID, logging.ReadContextID(encodedCtx)) } + +func TestNewRestrictQueryOptionsFromProto(t *testing.T) { + tests := []struct { + value *rpcpb.RestrictQueryOptions + expected *storage.RestrictQueryOptions + errContains string + }{ + { + value: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE, + }, + }, + expected: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.UnaggregatedMetricsType, + }, + }, + }, + { + value: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, + MetricsStoragePolicy: &policypb.StoragePolicy{ + Resolution: &policypb.Resolution{ + WindowSize: int64(time.Minute), + Precision: int64(time.Second), + }, + Retention: &policypb.Retention{ + Period: int64(24 * time.Hour), + }, + }, + }, + RestrictFetchTags: &rpc.RestrictFetchTags{ + Restrict: &rpc.TagMatchers{ + TagMatchers: []*rpc.TagMatcher{ + newRPCMatcher(rpc.MatcherType_NOTREGEXP, "foo", "bar"), + newRPCMatcher(rpc.MatcherType_EQUAL, "baz", "qux"), + }, + }, + Strip: [][]byte{ + []byte("foobar"), + }, + }, + }, + expected: &storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.NewStoragePolicy(time.Minute, + xtime.Second, 24*time.Hour), + }, + RestrictByTag: &storage.RestrictByTag{ + Restrict: []models.Matcher{ + mustNewMatcher(models.MatchNotRegexp, "foo", "bar"), + mustNewMatcher(models.MatchEqual, "baz", "qux"), + }, + Strip: [][]byte{ + []byte("foobar"), + }, + }, + }, + }, + { + value: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_UNKNOWN_METRICS_TYPE, + }, + }, + errContains: "unknown metrics type:", + }, + { + value: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE, + MetricsStoragePolicy: &policypb.StoragePolicy{ + Resolution: &policypb.Resolution{ + WindowSize: int64(time.Minute), + Precision: int64(time.Second), + }, + Retention: &policypb.Retention{ + Period: int64(24 * time.Hour), + }, + }, + }, + }, + errContains: "expected no storage policy for unaggregated metrics", + }, + { + value: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, + MetricsStoragePolicy: &policypb.StoragePolicy{ + Resolution: &policypb.Resolution{ + WindowSize: -1, + }, + }, + }, + }, + errContains: "unable to convert from duration to time unit", + }, + { + value: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, + MetricsStoragePolicy: &policypb.StoragePolicy{ + Resolution: &policypb.Resolution{ + WindowSize: int64(time.Minute), + Precision: int64(-1), + }, + }, + }, + }, + errContains: "unable to convert from duration to time unit", + }, + { + value: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, + MetricsStoragePolicy: &policypb.StoragePolicy{ + Resolution: &policypb.Resolution{ + WindowSize: int64(time.Minute), + Precision: int64(time.Second), + }, + Retention: &policypb.Retention{ + Period: int64(-1), + }, + }, + }, + }, + errContains: "expected positive retention", + }, + } + for _, test := range tests { + t.Run(fmt.Sprintf("%+v", test), func(t *testing.T) { + result, err := decodeRestrictQueryOptions(test.value) + if test.errContains == "" { + require.NoError(t, err) + assert.Equal(t, test.expected, result) + return + } + + require.Error(t, err) + assert.True(t, + strings.Contains(err.Error(), test.errContains), + fmt.Sprintf("err=%v, want_contains=%v", err.Error(), test.errContains)) + }) + } +} + +func mustNewMatcher(t models.MatchType, n, v string) models.Matcher { + matcher, err := models.NewMatcher(t, []byte(n), []byte(v)) + if err != nil { + panic(err) + } + + return matcher +} + +func newRPCMatcher(t rpc.MatcherType, n, v string) *rpc.TagMatcher { + return &rpc.TagMatcher{Name: []byte(n), Value: []byte(v), Type: t} +} + +func TestRestrictQueryOptionsProto(t *testing.T) { + tests := []struct { + value storage.RestrictQueryOptions + expected *rpcpb.RestrictQueryOptions + errContains string + }{ + { + value: storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.UnaggregatedMetricsType, + }, + RestrictByTag: &storage.RestrictByTag{ + Restrict: []models.Matcher{ + mustNewMatcher(models.MatchNotRegexp, "foo", "bar"), + }, + Strip: [][]byte{[]byte("foobar")}, + }, + }, + expected: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_UNAGGREGATED_METRICS_TYPE, + }, + RestrictFetchTags: &rpcpb.RestrictFetchTags{ + Restrict: &rpc.TagMatchers{ + TagMatchers: []*rpc.TagMatcher{ + newRPCMatcher(rpc.MatcherType_NOTREGEXP, "foo", "bar"), + }, + }, + Strip: [][]byte{[]byte("foobar")}, + }, + }, + }, + { + value: storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.AggregatedMetricsType, + StoragePolicy: policy.NewStoragePolicy(time.Minute, + xtime.Second, 24*time.Hour), + }, + RestrictByTag: &storage.RestrictByTag{ + Restrict: models.Matchers{ + mustNewMatcher(models.MatchNotRegexp, "foo", "bar"), + mustNewMatcher(models.MatchEqual, "baz", "qux"), + }, + Strip: [][]byte{[]byte("foobar")}, + }, + }, + expected: &rpcpb.RestrictQueryOptions{ + RestrictFetchType: &rpcpb.RestrictFetchType{ + MetricsType: rpcpb.MetricsType_AGGREGATED_METRICS_TYPE, + MetricsStoragePolicy: &policypb.StoragePolicy{ + Resolution: &policypb.Resolution{ + WindowSize: int64(time.Minute), + Precision: int64(time.Second), + }, + Retention: &policypb.Retention{ + Period: int64(24 * time.Hour), + }, + }, + }, + RestrictFetchTags: &rpcpb.RestrictFetchTags{ + Restrict: &rpc.TagMatchers{ + TagMatchers: []*rpc.TagMatcher{ + newRPCMatcher(rpc.MatcherType_NOTREGEXP, "foo", "bar"), + newRPCMatcher(rpc.MatcherType_EQUAL, "baz", "qux"), + }, + }, + Strip: [][]byte{[]byte("foobar")}, + }, + }, + }, + { + value: storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.MetricsType(uint(math.MaxUint16)), + }, + }, + errContains: "unknown metrics type:", + }, + { + value: storage.RestrictQueryOptions{ + RestrictByType: &storage.RestrictByType{ + MetricsType: storage.UnaggregatedMetricsType, + StoragePolicy: policy.NewStoragePolicy(time.Minute, + xtime.Second, 24*time.Hour), + }, + }, + errContains: "expected no storage policy for unaggregated metrics", + }, + } + for _, test := range tests { + t.Run(fmt.Sprintf("%+v", test.value), func(t *testing.T) { + result, err := encodeRestrictQueryOptions(&test.value) + if test.errContains == "" { + require.NoError(t, err) + require.Equal(t, test.expected, result) + return + } + + require.Error(t, err) + assert.True(t, strings.Contains(err.Error(), test.errContains)) + }) + } +}