From ee5f56509c4bdce8484c4cdf56002304fa46a41b Mon Sep 17 00:00:00 2001 From: Artem Nikolayevsky Date: Wed, 17 Apr 2019 12:29:59 -0400 Subject: [PATCH 1/4] [query] add list tags endpoint --- src/query/api/v1/handler/prometheus/common.go | 29 ++++ .../v1/handler/prometheus/native/list_tags.go | 90 ++++++++++ .../prometheus/native/list_tags_test.go | 154 ++++++++++++++++++ src/query/api/v1/httpd/handler.go | 7 + src/query/errors/handler.go | 2 + src/query/models/matcher.go | 2 + src/query/models/types.go | 1 + src/query/storage/index.go | 3 + src/query/storage/index_test.go | 9 + src/query/tsdb/remote/codecs.go | 2 + 10 files changed, 299 insertions(+) create mode 100644 src/query/api/v1/handler/prometheus/native/list_tags.go create mode 100644 src/query/api/v1/handler/prometheus/native/list_tags_test.go diff --git a/src/query/api/v1/handler/prometheus/common.go b/src/query/api/v1/handler/prometheus/common.go index 696bf02cba..5ec181e575 100644 --- a/src/query/api/v1/handler/prometheus/common.go +++ b/src/query/api/v1/handler/prometheus/common.go @@ -304,6 +304,35 @@ func renderDefaultTagCompletionResultsJSON( return jw.Close() } +// RenderListTagResultsJSON renders list tag results to json format. +func RenderListTagResultsJSON( + w io.Writer, + result *storage.CompleteTagsResult, +) error { + if !result.CompleteNameOnly { + return errors.ErrWithNames + } + + jw := json.NewWriter(w) + jw.BeginObject() + + jw.BeginObjectField("status") + jw.WriteString("success") + + jw.BeginObjectField("data") + jw.BeginArray() + + for _, t := range result.CompletedTags { + jw.WriteString(string(t.Name)) + } + + jw.EndArray() + + jw.EndObject() + + return jw.Close() +} + // RenderTagCompletionResultsJSON renders tag completion results to json format. func RenderTagCompletionResultsJSON( w io.Writer, diff --git a/src/query/api/v1/handler/prometheus/native/list_tags.go b/src/query/api/v1/handler/prometheus/native/list_tags.go new file mode 100644 index 0000000000..78a8c2ffa4 --- /dev/null +++ b/src/query/api/v1/handler/prometheus/native/list_tags.go @@ -0,0 +1,90 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package native + +import ( + "context" + "net/http" + "time" + + "github.com/m3db/m3/src/query/api/v1/handler" + "github.com/m3db/m3/src/query/api/v1/handler/prometheus" + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/query/util/logging" + "github.com/m3db/m3/src/x/net/http" + + "go.uber.org/zap" +) + +const ( + // ListTagsURL is the url for listing tags. + ListTagsURL = handler.RoutePrefixV1 + "/labels" +) + +var ( + // ListTagsHTTPMethods are the HTTP methods used with this resource. + ListTagsHTTPMethods = []string{http.MethodGet, http.MethodPost} +) + +// ListTagsHandler represents a handler for list tags endpoint. +type ListTagsHandler struct { + storage storage.Storage +} + +// NewListTagsHandler returns a new instance of handler. +func NewListTagsHandler( + storage storage.Storage, +) http.Handler { + return &ListTagsHandler{ + storage: storage, + } +} + +func (h *ListTagsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := context.WithValue(r.Context(), handler.HeaderKey, r.Header) + logger := logging.WithContext(ctx) + w.Header().Set("Content-Type", "application/json") + + query := &storage.CompleteTagsQuery{ + CompleteNameOnly: true, + TagMatchers: models.Matchers{{Type: models.MatchAll}}, + + // NB: necessarily spans entire possible query range. + Start: time.Time{}, + End: time.Now(), + } + + opts := storage.NewFetchOptions() + result, err := h.storage.CompleteTags(ctx, query, opts) + if err != nil { + logger.Error("unable to complete tags", zap.Error(err)) + xhttp.Error(w, err, http.StatusBadRequest) + return + } + + // TODO: Support multiple result types + if err = prometheus.RenderListTagResultsJSON(w, result); err != nil { + logger.Error("unable to render results", zap.Error(err)) + xhttp.Error(w, err, http.StatusBadRequest) + return + } +} diff --git a/src/query/api/v1/handler/prometheus/native/list_tags_test.go b/src/query/api/v1/handler/prometheus/native/list_tags_test.go new file mode 100644 index 0000000000..a3d0d0745a --- /dev/null +++ b/src/query/api/v1/handler/prometheus/native/list_tags_test.go @@ -0,0 +1,154 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package native + +import ( + "encoding/json" + "math" + "net/http" + "net/url" + "testing" + "time" + + "github.com/m3db/m3/src/query/models" + "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/query/util/logging" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +type listTagsMatcher struct{} + +func (m *listTagsMatcher) String() string { return "list tags query" } +func (m *listTagsMatcher) Matches(x interface{}) bool { + q, ok := x.(*storage.CompleteTagsQuery) + if !ok { + return false + } + + if !q.Start.Equal(time.Time{}) { + return false + } + + // NB: end time for the query should be roughly `Now` + diff := q.End.Sub(time.Now()) + absDiff := time.Duration(math.Abs(float64(diff))) + if absDiff > time.Second { + return false + } + + if !q.CompleteNameOnly { + return false + } + + if len(q.FilterNameTags) != 0 { + return false + } + + if len(q.TagMatchers) != 1 { + return false + } + + return models.MatchAll == q.TagMatchers[0].Type +} + +var _ gomock.Matcher = &listTagsMatcher{} + +func b(s string) []byte { return []byte(s) } +func bs(ss ...string) [][]byte { + bb := make([][]byte, len(ss)) + for i, s := range ss { + bb[i] = b(s) + } + + return bb +} + +func setupStorage(ctrl *gomock.Controller) storage.Storage { + store := storage.NewMockStorage(ctrl) + result := &storage.CompleteTagsResult{ + CompleteNameOnly: true, + CompletedTags: []storage.CompletedTag{ + {Name: b("bar")}, + {Name: b("baz")}, + {Name: b("foo")}, + }, + } + + store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). + Return(result, nil) + + return store +} + +type writer struct { + results []string +} + +var _ http.ResponseWriter = &writer{} + +func (w *writer) WriteHeader(_ int) {} +func (w *writer) Header() http.Header { return make(http.Header) } +func (w *writer) Write(b []byte) (int, error) { + if w.results == nil { + w.results = make([]string, 0, 10) + } + + w.results = append(w.results, string(b)) + return len(b), nil +} + +type result struct { + Status string `json:"status"` + Data []string `json:"data"` +} + +func TestFind(t *testing.T) { + logging.InitWithCores(nil) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // setup storage and handler + store := setupStorage(ctrl) + handler := NewListTagsHandler(store) + + // execute the query + w := &writer{} + req := &http.Request{ + URL: &url.URL{ + RawQuery: "", + }, + } + + handler.ServeHTTP(w, req) + require.Equal(t, 1, len(w.results)) + var r result + json.Unmarshal([]byte(w.results[0]), &r) + + ex := result{ + Status: "success", + Data: []string{"bar", "baz", "foo"}, + } + + require.Equal(t, ex, r) +} diff --git a/src/query/api/v1/httpd/handler.go b/src/query/api/v1/httpd/handler.go index 7e62b67f98..ff07a8d73a 100644 --- a/src/query/api/v1/httpd/handler.go +++ b/src/query/api/v1/httpd/handler.go @@ -217,6 +217,13 @@ func (h *Handler) RegisterRoutes() error { wrapped(remote.NewTagValuesHandler(h.storage)).ServeHTTP, ).Methods(remote.TagValuesHTTPMethod) + // List tag endpoints + for _, method := range native.ListTagsHTTPMethods { + h.router.HandleFunc(native.ListTagsURL, + wrapped(native.NewCompleteTagsHandler(h.storage)).ServeHTTP, + ).Methods(method) + } + // Series match endpoints h.router.HandleFunc(remote.PromSeriesMatchURL, wrapped(remote.NewPromSeriesMatchHandler(h.storage, h.tagOptions)).ServeHTTP, diff --git a/src/query/errors/handler.go b/src/query/errors/handler.go index b942eb4d41..e066472a3c 100644 --- a/src/query/errors/handler.go +++ b/src/query/errors/handler.go @@ -40,6 +40,8 @@ var ( ErrInvalidMatchers = errors.New("invalid matchers") // ErrNamesOnly is returned when label values results are name only ErrNamesOnly = errors.New("can not render label values; result has label names only") + // ErrWithNames is returned when label values results are name only + ErrWithNames = errors.New("can not render label list; result has label names and values") // ErrMultipleResults is returned when there are multiple label values results ErrMultipleResults = errors.New("can not render label values; multiple results detected") ) diff --git a/src/query/models/matcher.go b/src/query/models/matcher.go index de53ec586d..350f0ee948 100644 --- a/src/query/models/matcher.go +++ b/src/query/models/matcher.go @@ -38,6 +38,8 @@ func (m MatchType) String() string { return "=~" case MatchNotRegexp: return "!~" + case MatchAll: + return "*" default: return "unknown match type" } diff --git a/src/query/models/types.go b/src/query/models/types.go index dfff733a5f..f211d6d80a 100644 --- a/src/query/models/types.go +++ b/src/query/models/types.go @@ -120,6 +120,7 @@ const ( MatchNotEqual MatchRegexp MatchNotRegexp + MatchAll ) // Matcher models the matching of a label. diff --git a/src/query/storage/index.go b/src/query/storage/index.go index de293185fa..769fc77e4f 100644 --- a/src/query/storage/index.go +++ b/src/query/storage/index.go @@ -192,6 +192,9 @@ func matcherToQuery(matcher models.Matcher) (idx.Query, error) { } return query, nil + case models.MatchAll: + return idx.NewAllQuery(), nil + default: return idx.Query{}, fmt.Errorf("unsupported query type: %v", matcher) } diff --git a/src/query/storage/index_test.go b/src/query/storage/index_test.go index 95a4bb9e93..4b40a7a054 100644 --- a/src/query/storage/index_test.go +++ b/src/query/storage/index_test.go @@ -158,6 +158,15 @@ func TestFetchQueryToM3Query(t *testing.T) { expected: "all()", matchers: models.Matchers{}, }, + { + name: "all matchers", + expected: "all()", + matchers: models.Matchers{ + { + Type: models.MatchAll, + }, + }, + }, } for _, test := range tests { diff --git a/src/query/tsdb/remote/codecs.go b/src/query/tsdb/remote/codecs.go index 380260f8b2..d2d18bb38a 100644 --- a/src/query/tsdb/remote/codecs.go +++ b/src/query/tsdb/remote/codecs.go @@ -192,6 +192,8 @@ func encodeMatcherTypeToProto(t models.MatchType) (rpc.MatcherType, error) { return rpc.MatcherType_REGEXP, nil case models.MatchNotRegexp: return rpc.MatcherType_NOTREGEXP, nil + case models.MatchAll: + return rpc.MatcherType_EXISTS, nil default: return rpc.MatcherType_EQUAL, fmt.Errorf("Unknown matcher type for proto encoding") } From e3e001d9c778239170f28ce33c26308be65eb897 Mon Sep 17 00:00:00 2001 From: Artem Nikolayevsky Date: Wed, 17 Apr 2019 16:02:52 -0400 Subject: [PATCH 2/4] Use correct handler --- src/query/api/v1/httpd/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/api/v1/httpd/handler.go b/src/query/api/v1/httpd/handler.go index ff07a8d73a..53552a5244 100644 --- a/src/query/api/v1/httpd/handler.go +++ b/src/query/api/v1/httpd/handler.go @@ -220,7 +220,7 @@ func (h *Handler) RegisterRoutes() error { // List tag endpoints for _, method := range native.ListTagsHTTPMethods { h.router.HandleFunc(native.ListTagsURL, - wrapped(native.NewCompleteTagsHandler(h.storage)).ServeHTTP, + wrapped(native.NewListTagsHandler(h.storage)).ServeHTTP, ).Methods(method) } From fd7fe63a53a55b495f8a603a13762197cbbde92f Mon Sep 17 00:00:00 2001 From: Artem Nikolayevsky Date: Mon, 29 Apr 2019 13:46:41 -0400 Subject: [PATCH 3/4] PR responses --- .../v1/handler/prometheus/native/list_tags.go | 3 +- .../prometheus/native/list_tags_test.go | 78 ++++++++++++------- 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/query/api/v1/handler/prometheus/native/list_tags.go b/src/query/api/v1/handler/prometheus/native/list_tags.go index 78a8c2ffa4..c014c73b92 100644 --- a/src/query/api/v1/handler/prometheus/native/list_tags.go +++ b/src/query/api/v1/handler/prometheus/native/list_tags.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Uber Technologies, Inc. +// Copyright (c) 2019 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -81,7 +81,6 @@ func (h *ListTagsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // TODO: Support multiple result types if err = prometheus.RenderListTagResultsJSON(w, result); err != nil { logger.Error("unable to render results", zap.Error(err)) xhttp.Error(w, err, http.StatusBadRequest) diff --git a/src/query/api/v1/handler/prometheus/native/list_tags_test.go b/src/query/api/v1/handler/prometheus/native/list_tags_test.go index a3d0d0745a..051676aa74 100644 --- a/src/query/api/v1/handler/prometheus/native/list_tags_test.go +++ b/src/query/api/v1/handler/prometheus/native/list_tags_test.go @@ -22,6 +22,7 @@ package native import ( "encoding/json" + "errors" "math" "net/http" "net/url" @@ -74,31 +75,6 @@ func (m *listTagsMatcher) Matches(x interface{}) bool { var _ gomock.Matcher = &listTagsMatcher{} func b(s string) []byte { return []byte(s) } -func bs(ss ...string) [][]byte { - bb := make([][]byte, len(ss)) - for i, s := range ss { - bb[i] = b(s) - } - - return bb -} - -func setupStorage(ctrl *gomock.Controller) storage.Storage { - store := storage.NewMockStorage(ctrl) - result := &storage.CompleteTagsResult{ - CompleteNameOnly: true, - CompletedTags: []storage.CompletedTag{ - {Name: b("bar")}, - {Name: b("baz")}, - {Name: b("foo")}, - }, - } - - store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). - Return(result, nil) - - return store -} type writer struct { results []string @@ -122,14 +98,26 @@ type result struct { Data []string `json:"data"` } -func TestFind(t *testing.T) { +func TestListTags(t *testing.T) { logging.InitWithCores(nil) ctrl := gomock.NewController(t) defer ctrl.Finish() // setup storage and handler - store := setupStorage(ctrl) + store := storage.NewMockStorage(ctrl) + storeResult := &storage.CompleteTagsResult{ + CompleteNameOnly: true, + CompletedTags: []storage.CompletedTag{ + {Name: b("bar")}, + {Name: b("baz")}, + {Name: b("foo")}, + }, + } + + store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). + Return(storeResult, nil) + handler := NewListTagsHandler(store) // execute the query @@ -152,3 +140,39 @@ func TestFind(t *testing.T) { require.Equal(t, ex, r) } + +type errResult struct { + Error string `json:"error"` +} + +func TestListErrorTags(t *testing.T) { + logging.InitWithCores(nil) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // setup storage and handler + store := storage.NewMockStorage(ctrl) + store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). + Return(nil, errors.New("err")) + handler := NewListTagsHandler(store) + + // execute the query + w := &writer{} + req := &http.Request{ + URL: &url.URL{ + RawQuery: "", + }, + } + + handler.ServeHTTP(w, req) + require.Equal(t, 1, len(w.results)) + var r errResult + json.Unmarshal([]byte(w.results[0]), &r) + + ex := errResult{ + Error: "err", + } + + require.Equal(t, ex, r) +} From efe5ed963750a0816b81b71657ae117e7ab8dc64 Mon Sep 17 00:00:00 2001 From: Artem Nikolayevsky Date: Mon, 29 Apr 2019 16:14:17 -0400 Subject: [PATCH 4/4] Refactor test to use httptest.NewRequest() --- .../prometheus/native/list_tags_test.go | 97 ++++++------------- 1 file changed, 32 insertions(+), 65 deletions(-) diff --git a/src/query/api/v1/handler/prometheus/native/list_tags_test.go b/src/query/api/v1/handler/prometheus/native/list_tags_test.go index 051676aa74..f28f02bf1f 100644 --- a/src/query/api/v1/handler/prometheus/native/list_tags_test.go +++ b/src/query/api/v1/handler/prometheus/native/list_tags_test.go @@ -21,11 +21,11 @@ package native import ( - "encoding/json" "errors" + "fmt" + "io/ioutil" "math" - "net/http" - "net/url" + "net/http/httptest" "testing" "time" @@ -76,28 +76,6 @@ var _ gomock.Matcher = &listTagsMatcher{} func b(s string) []byte { return []byte(s) } -type writer struct { - results []string -} - -var _ http.ResponseWriter = &writer{} - -func (w *writer) WriteHeader(_ int) {} -func (w *writer) Header() http.Header { return make(http.Header) } -func (w *writer) Write(b []byte) (int, error) { - if w.results == nil { - w.results = make([]string, 0, 10) - } - - w.results = append(w.results, string(b)) - return len(b), nil -} - -type result struct { - Status string `json:"status"` - Data []string `json:"data"` -} - func TestListTags(t *testing.T) { logging.InitWithCores(nil) @@ -115,34 +93,24 @@ func TestListTags(t *testing.T) { }, } - store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). - Return(storeResult, nil) - handler := NewListTagsHandler(store) + for _, method := range []string{"GET", "POST"} { + store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). + Return(storeResult, nil) - // execute the query - w := &writer{} - req := &http.Request{ - URL: &url.URL{ - RawQuery: "", - }, - } + req := httptest.NewRequest(method, "/labels", nil) + w := httptest.NewRecorder() - handler.ServeHTTP(w, req) - require.Equal(t, 1, len(w.results)) - var r result - json.Unmarshal([]byte(w.results[0]), &r) + handler.ServeHTTP(w, req) + body := w.Result().Body + defer body.Close() - ex := result{ - Status: "success", - Data: []string{"bar", "baz", "foo"}, - } + r, err := ioutil.ReadAll(body) + require.NoError(t, err) - require.Equal(t, ex, r) -} - -type errResult struct { - Error string `json:"error"` + ex := `{"status":"success","data":["bar","baz","foo"]}` + require.Equal(t, ex, string(r)) + } } func TestListErrorTags(t *testing.T) { @@ -153,26 +121,25 @@ func TestListErrorTags(t *testing.T) { // setup storage and handler store := storage.NewMockStorage(ctrl) - store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). - Return(nil, errors.New("err")) handler := NewListTagsHandler(store) - // execute the query - w := &writer{} - req := &http.Request{ - URL: &url.URL{ - RawQuery: "", - }, - } + for _, method := range []string{"GET", "POST"} { + store.EXPECT().CompleteTags(gomock.Any(), &listTagsMatcher{}, gomock.Any()). + Return(nil, errors.New("err")) - handler.ServeHTTP(w, req) - require.Equal(t, 1, len(w.results)) - var r errResult - json.Unmarshal([]byte(w.results[0]), &r) + req := httptest.NewRequest(method, "/labels", nil) + w := httptest.NewRecorder() - ex := errResult{ - Error: "err", - } + handler.ServeHTTP(w, req) + body := w.Result().Body + defer body.Close() - require.Equal(t, ex, r) + r, err := ioutil.ReadAll(body) + require.NoError(t, err) + + ex := `{"error":"err"}` + // NB: error handler adds a newline to the output. + ex = fmt.Sprintf("%s\n", ex) + require.Equal(t, ex, string(r)) + } }