From 3fdbae6dfd606aeefc944426b42a431b0cf33cb6 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Wed, 6 Nov 2019 16:56:48 -0500 Subject: [PATCH 01/19] Added configuration and CLI support Signed-off-by: Joe Elliott --- pkg/cassandra/config/config.go | 2 ++ plugin/storage/cassandra/options.go | 32 +++++++++++++++++++++--- plugin/storage/cassandra/options_test.go | 9 +++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pkg/cassandra/config/config.go b/pkg/cassandra/config/config.go index 8aba2e11680..733406b90f1 100644 --- a/pkg/cassandra/config/config.go +++ b/pkg/cassandra/config/config.go @@ -45,6 +45,8 @@ type Configuration struct { DisableAutoDiscovery bool `yaml:"disable_auto_discovery"` EnableDependenciesV2 bool `yaml:"enable_dependencies_v2"` TLS TLS + TagIndexWhitelist []string `yaml:"tag-index-whitelist"` + TagIndexBlacklist []string `yaml:"tag-index-blacklist"` } // Authenticator holds the authentication properties needed to connect to a Cassandra cluster diff --git a/plugin/storage/cassandra/options.go b/plugin/storage/cassandra/options.go index fcf5383c568..395a246e9c3 100644 --- a/plugin/storage/cassandra/options.go +++ b/plugin/storage/cassandra/options.go @@ -50,6 +50,8 @@ const ( suffixServerName = ".tls.server-name" suffixVerifyHost = ".tls.verify-host" suffixEnableDependenciesV2 = ".enable-dependencies-v2" + suffixTagIndexBlacklist = ".tag-index-blacklist" + suffixTagIndexWhitelist = ".tag-index-whitelist" // common storage settings suffixSpanStoreWriteCacheTTL = ".span-store-write-cache-ttl" @@ -69,10 +71,12 @@ type Options struct { // preparing the actual config.Configuration. type namespaceConfig struct { config.Configuration - servers string - namespace string - primary bool - Enabled bool + servers string + tagIndexBlacklist string + tagIndexWhitelist string + namespace string + primary bool + Enabled bool } // NewOptions creates a new Options struct. @@ -213,6 +217,14 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { nsConfig.namespace+suffixEnableDependenciesV2, nsConfig.EnableDependenciesV2, "(deprecated) Jaeger will automatically detect the version of the dependencies table") + flagSet.String( + nsConfig.namespace+suffixTagIndexBlacklist, + nsConfig.tagIndexWhitelist, + "The comma-separated list of tags to blacklist") + flagSet.String( + nsConfig.namespace+suffixTagIndexWhitelist, + nsConfig.tagIndexBlacklist, + "The comma-separated list of tags to whitelist") } // InitFromViper initializes Options with properties from viper @@ -250,11 +262,15 @@ func (cfg *namespaceConfig) initFromViper(v *viper.Viper) { cfg.TLS.EnableHostVerification = v.GetBool(cfg.namespace + suffixVerifyHost) cfg.EnableDependenciesV2 = v.GetBool(cfg.namespace + suffixEnableDependenciesV2) cfg.DisableCompression = v.GetBool(cfg.namespace + suffixDisableCompression) + cfg.tagIndexBlacklist = stripWhiteSpace(v.GetString(cfg.namespace + suffixTagIndexBlacklist)) + cfg.tagIndexWhitelist = stripWhiteSpace(v.GetString(cfg.namespace + suffixTagIndexWhitelist)) } // GetPrimary returns primary configuration. func (opt *Options) GetPrimary() *config.Configuration { opt.primary.Servers = strings.Split(opt.primary.servers, ",") + opt.primary.TagIndexBlacklist = strings.Split(opt.primary.tagIndexBlacklist, ",") + opt.primary.TagIndexWhitelist = strings.Split(opt.primary.tagIndexWhitelist, ",") return &opt.primary.Configuration } @@ -273,6 +289,14 @@ func (opt *Options) Get(namespace string) *config.Configuration { nsCfg.servers = opt.primary.servers } nsCfg.Servers = strings.Split(nsCfg.servers, ",") + if nsCfg.tagIndexBlacklist == "" { + nsCfg.tagIndexBlacklist = opt.primary.tagIndexBlacklist + } + nsCfg.TagIndexBlacklist = strings.Split(nsCfg.tagIndexBlacklist, ",") + if nsCfg.tagIndexWhitelist == "" { + nsCfg.tagIndexWhitelist = opt.primary.tagIndexWhitelist + } + nsCfg.TagIndexWhitelist = strings.Split(nsCfg.tagIndexWhitelist, ",") return &nsCfg.Configuration } diff --git a/plugin/storage/cassandra/options_test.go b/plugin/storage/cassandra/options_test.go index aca6b020146..c71a140ac67 100644 --- a/plugin/storage/cassandra/options_test.go +++ b/plugin/storage/cassandra/options_test.go @@ -43,6 +43,8 @@ func TestOptions(t *testing.T) { assert.Equal(t, primary.Servers, aux.Servers) assert.Equal(t, primary.ConnectionsPerHost, aux.ConnectionsPerHost) assert.Equal(t, primary.ReconnectInterval, aux.ReconnectInterval) + assert.Equal(t, primary.TagIndexBlacklist, aux.TagIndexBlacklist) + assert.Equal(t, primary.TagIndexWhitelist, aux.TagIndexWhitelist) } func TestOptionsWithFlags(t *testing.T) { @@ -60,11 +62,14 @@ func TestOptionsWithFlags(t *testing.T) { "--cas.consistency=ONE", "--cas.proto-version=3", "--cas.socket-keep-alive=42s", + "--cas.tag-index-blacklist=blerg, blarg,blorg ", + "--cas.tag-index-whitelist=blerg, blarg,blorg ", // enable aux with a couple overrides "--cas-aux.enabled=true", "--cas-aux.keyspace=jaeger-archive", "--cas-aux.servers=3.3.3.3, 4.4.4.4", "--cas-aux.enable-dependencies-v2=true", + "--cas-aux.tag-index-whitelist=foo,bar", }) opts.InitFromViper(v) @@ -74,6 +79,8 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, []string{"1.1.1.1", "2.2.2.2"}, primary.Servers) assert.Equal(t, "ONE", primary.Consistency) assert.Equal(t, false, primary.EnableDependenciesV2) + assert.Equal(t, []string{"blerg", "blarg", "blorg"}, primary.TagIndexBlacklist) + assert.Equal(t, []string{"blerg", "blarg", "blorg"}, primary.TagIndexWhitelist) aux := opts.Get("cas-aux") require.NotNil(t, aux) @@ -88,4 +95,6 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, 3, aux.ProtoVersion) assert.Equal(t, 42*time.Second, aux.SocketKeepAlive) assert.Equal(t, true, aux.EnableDependenciesV2) + assert.Equal(t, []string{"blerg", "blarg", "blorg"}, aux.TagIndexBlacklist) + assert.Equal(t, []string{"foo", "bar"}, aux.TagIndexWhitelist) } From 83107158f177dccbfa69cb33c0126bb4f98c374e Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 7 Nov 2019 08:33:23 -0500 Subject: [PATCH 02/19] Added filters Signed-off-by: Joe Elliott --- .../spanstore/dbmodel/tag_filter_blacklist.go | 61 +++++++++++++++++++ .../dbmodel/tag_filter_blacklist_test.go | 61 +++++++++++++++++++ .../spanstore/dbmodel/tag_filter_whitelist.go | 61 +++++++++++++++++++ .../dbmodel/tag_filter_whitelist_test.go | 61 +++++++++++++++++++ 4 files changed, 244 insertions(+) create mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go create mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go create mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go create mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go new file mode 100644 index 00000000000..afdecb106ab --- /dev/null +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go @@ -0,0 +1,61 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbmodel + +import ( + "github.com/jaegertracing/jaeger/model" +) + +// BlacklistTagFilter filters out all tags in its tags slice +type BlacklistTagFilter struct { + tags map[string]struct{} +} + +// NewBlacklistTagFilter creates a BlacklistTagFilter with the provided tags +func NewBlacklistTagFilter(tags []string) BlacklistTagFilter { + mapTags := make(map[string]struct{}) + for _, t := range tags { + mapTags[t] = struct{}{} + } + return BlacklistTagFilter{ + tags: mapTags, + } +} + +// FilterProcessTags implements TagFilter +func (tf BlacklistTagFilter) FilterProcessTags(span *model.Span, processTags model.KeyValues) model.KeyValues { + return tf.filter(processTags) +} + +// FilterTags implements TagFilter +func (tf BlacklistTagFilter) FilterTags(span *model.Span, tags model.KeyValues) model.KeyValues { + return tf.filter(tags) +} + +// FilterLogFields implements TagFilter +func (tf BlacklistTagFilter) FilterLogFields(span *model.Span, logFields model.KeyValues) model.KeyValues { + return tf.filter(logFields) +} + +func (tf BlacklistTagFilter) filter(processTags model.KeyValues) model.KeyValues { + var tags model.KeyValues + for _, t := range processTags { + if _, ok := tf.tags[t.Key]; !ok { + tags = append(tags, t) + } + } + return tags +} diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go new file mode 100644 index 00000000000..59a905921f6 --- /dev/null +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbmodel + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/jaegertracing/jaeger/model" +) + +func TestBlacklistFilter(t *testing.T) { + // expected + tt := [][][]string{ + [][]string{ + []string{"a", "b", "c"}, // input + []string{"a"}, // filter + []string{"b", "c"}, // expected + }, + [][]string{ + []string{"a", "b", "c"}, + []string{"A"}, + []string{"a", "b", "c"}, + }, + } + + for _, test := range tt { + input := test[0] + filter := test[1] + expected := test[2] + + var inputKVs model.KeyValues + for _, i := range input { + inputKVs = append(inputKVs, model.String(i, "")) + } + var expectedKVs model.KeyValues + for _, e := range expected { + expectedKVs = append(expectedKVs, model.String(e, "")) + } + + tf := NewBlacklistTagFilter(filter) + actualKVs := tf.filter(inputKVs) + actualKVs.Sort() + expectedKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + } +} diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go new file mode 100644 index 00000000000..c00dfcbceab --- /dev/null +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go @@ -0,0 +1,61 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbmodel + +import ( + "github.com/jaegertracing/jaeger/model" +) + +// WhitelistTagFilter filters out all tags in its tags slice +type WhitelistTagFilter struct { + tags map[string]struct{} +} + +// NewWhitelistTagFilter creates a WhitelistTagFilter with the provided tags +func NewWhitelistTagFilter(tags []string) WhitelistTagFilter { + mapTags := make(map[string]struct{}) + for _, t := range tags { + mapTags[t] = struct{}{} + } + return WhitelistTagFilter{ + tags: mapTags, + } +} + +// FilterProcessTags implements TagFilter +func (tf WhitelistTagFilter) FilterProcessTags(span *model.Span, processTags model.KeyValues) model.KeyValues { + return tf.filter(processTags) +} + +// FilterTags implements TagFilter +func (tf WhitelistTagFilter) FilterTags(span *model.Span, tags model.KeyValues) model.KeyValues { + return tf.filter(tags) +} + +// FilterLogFields implements TagFilter +func (tf WhitelistTagFilter) FilterLogFields(span *model.Span, logFields model.KeyValues) model.KeyValues { + return tf.filter(logFields) +} + +func (tf WhitelistTagFilter) filter(processTags model.KeyValues) model.KeyValues { + var tags model.KeyValues + for _, t := range processTags { + if _, ok := tf.tags[t.Key]; ok { + tags = append(tags, t) + } + } + return tags +} diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go new file mode 100644 index 00000000000..33e4119e626 --- /dev/null +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbmodel + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/jaegertracing/jaeger/model" +) + +func TestWhitelistFilter(t *testing.T) { + // expected + tt := [][][]string{ + [][]string{ + []string{"a", "b", "c"}, // input + []string{"a"}, // filter + []string{"a"}, // expected + }, + [][]string{ + []string{"a", "b", "c"}, + []string{"A"}, + []string{}, + }, + } + + for _, test := range tt { + input := test[0] + filter := test[1] + expected := test[2] + + var inputKVs model.KeyValues + for _, i := range input { + inputKVs = append(inputKVs, model.String(i, "")) + } + var expectedKVs model.KeyValues + for _, e := range expected { + expectedKVs = append(expectedKVs, model.String(e, "")) + } + + tf := NewWhitelistTagFilter(filter) + actualKVs := tf.filter(inputKVs) + actualKVs.Sort() + expectedKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + } +} From e67acd9347efce9d5cacc3971802c50c3f8ef863 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 7 Nov 2019 09:20:51 -0500 Subject: [PATCH 03/19] Create White and Blacklist filters based on options Signed-off-by: Joe Elliott --- plugin/storage/cassandra/factory.go | 28 +++++++++++++++-- plugin/storage/cassandra/factory_test.go | 39 ++++++++++++++++++++++++ plugin/storage/cassandra/options.go | 16 +++++++--- 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/plugin/storage/cassandra/factory.go b/plugin/storage/cassandra/factory.go index 09d3a10bf73..b468b0f937c 100644 --- a/plugin/storage/cassandra/factory.go +++ b/plugin/storage/cassandra/factory.go @@ -16,6 +16,7 @@ package cassandra import ( + "errors" "flag" "github.com/spf13/viper" @@ -26,6 +27,7 @@ import ( "github.com/jaegertracing/jaeger/pkg/cassandra/config" cDepStore "github.com/jaegertracing/jaeger/plugin/storage/cassandra/dependencystore" cSpanStore "github.com/jaegertracing/jaeger/plugin/storage/cassandra/spanstore" + "github.com/jaegertracing/jaeger/plugin/storage/cassandra/spanstore/dbmodel" "github.com/jaegertracing/jaeger/storage" "github.com/jaegertracing/jaeger/storage/dependencystore" "github.com/jaegertracing/jaeger/storage/spanstore" @@ -102,7 +104,11 @@ func (f *Factory) CreateSpanReader() (spanstore.Reader, error) { // CreateSpanWriter implements storage.Factory func (f *Factory) CreateSpanWriter() (spanstore.Writer, error) { - return cSpanStore.NewSpanWriter(f.primarySession, f.Options.SpanStoreWriteCacheTTL, f.primaryMetricsFactory, f.logger), nil + options, err := writerOptions(f.Options.GetPrimary()) + if err != nil { + return nil, err + } + return cSpanStore.NewSpanWriter(f.primarySession, f.Options.SpanStoreWriteCacheTTL, f.primaryMetricsFactory, f.logger, options...), nil } // CreateDependencyReader implements storage.Factory @@ -124,5 +130,23 @@ func (f *Factory) CreateArchiveSpanWriter() (spanstore.Writer, error) { if f.archiveSession == nil { return nil, storage.ErrArchiveStorageNotConfigured } - return cSpanStore.NewSpanWriter(f.archiveSession, f.Options.SpanStoreWriteCacheTTL, f.archiveMetricsFactory, f.logger), nil + options, err := writerOptions(f.Options.Get(archiveStorageConfig)) + if err != nil { + return nil, err + } + return cSpanStore.NewSpanWriter(f.archiveSession, f.Options.SpanStoreWriteCacheTTL, f.archiveMetricsFactory, f.logger, options...), nil +} + +func writerOptions(config *config.Configuration) ([]cSpanStore.Option, error) { + if len(config.TagIndexBlacklist) > 0 && len(config.TagIndexWhitelist) > 0 { + return nil, errors.New("only one of TagIndexBlacklist and TagIndexWhitelist can be specified") + } + + var options []cSpanStore.Option + if len(config.TagIndexBlacklist) > 0 { + options = append(options, cSpanStore.TagFilter(dbmodel.NewBlacklistTagFilter(config.TagIndexBlacklist))) + } else if len(config.TagIndexWhitelist) > 0 { + options = append(options, cSpanStore.TagFilter(dbmodel.NewWhitelistTagFilter(config.TagIndexWhitelist))) + } + return options, nil } diff --git a/plugin/storage/cassandra/factory_test.go b/plugin/storage/cassandra/factory_test.go index d301cc31d3a..aee08c85f29 100644 --- a/plugin/storage/cassandra/factory_test.go +++ b/plugin/storage/cassandra/factory_test.go @@ -100,3 +100,42 @@ func TestCassandraFactory(t *testing.T) { _, err = f.CreateArchiveSpanWriter() assert.NoError(t, err) } + +func TestExclusiveWhiteListBlackList(t *testing.T) { + logger, logBuf := testutils.NewLogger() + f := NewFactory() + v, command := config.Viperize(f.AddFlags) + command.ParseFlags([]string{"--cassandra-archive.enabled=true", + "--cassandra.enable-dependencies-v2=true", + "--cassandra.tag-index-whitelist=a,b,c", + "--cassandra.tag-index-blacklist=a,b,c"}) + f.InitFromViper(v) + + // after InitFromViper, f.primaryConfig points to a real session builder that will fail in unit tests, + // so we override it with a mock. + f.primaryConfig = newMockSessionBuilder(nil, errors.New("made-up error")) + assert.EqualError(t, f.Initialize(metrics.NullFactory, zap.NewNop()), "made-up error") + + var ( + session = &mocks.Session{} + query = &mocks.Query{} + ) + session.On("Query", mock.AnythingOfType("string"), mock.Anything).Return(query) + query.On("Exec").Return(nil) + f.primaryConfig = newMockSessionBuilder(session, nil) + f.archiveConfig = newMockSessionBuilder(nil, errors.New("made-up error")) + assert.EqualError(t, f.Initialize(metrics.NullFactory, zap.NewNop()), "made-up error") + + f.archiveConfig = nil + assert.NoError(t, f.Initialize(metrics.NullFactory, logger)) + assert.Contains(t, logBuf.String(), "Cassandra archive storage configuration is empty, skipping") + + _, err := f.CreateSpanWriter() + assert.EqualError(t, err, "only one of TagIndexBlacklist and TagIndexWhitelist can be specified") + + f.archiveConfig = &mockSessionBuilder{} + assert.NoError(t, f.Initialize(metrics.NullFactory, zap.NewNop())) + + _, err = f.CreateArchiveSpanWriter() + assert.EqualError(t, err, "only one of TagIndexBlacklist and TagIndexWhitelist can be specified") +} diff --git a/plugin/storage/cassandra/options.go b/plugin/storage/cassandra/options.go index 395a246e9c3..53901453a20 100644 --- a/plugin/storage/cassandra/options.go +++ b/plugin/storage/cassandra/options.go @@ -269,8 +269,12 @@ func (cfg *namespaceConfig) initFromViper(v *viper.Viper) { // GetPrimary returns primary configuration. func (opt *Options) GetPrimary() *config.Configuration { opt.primary.Servers = strings.Split(opt.primary.servers, ",") - opt.primary.TagIndexBlacklist = strings.Split(opt.primary.tagIndexBlacklist, ",") - opt.primary.TagIndexWhitelist = strings.Split(opt.primary.tagIndexWhitelist, ",") + if len(opt.primary.tagIndexBlacklist) > 0 { + opt.primary.TagIndexBlacklist = strings.Split(opt.primary.tagIndexBlacklist, ",") + } + if len(opt.primary.tagIndexWhitelist) > 0 { + opt.primary.TagIndexWhitelist = strings.Split(opt.primary.tagIndexWhitelist, ",") + } return &opt.primary.Configuration } @@ -292,11 +296,15 @@ func (opt *Options) Get(namespace string) *config.Configuration { if nsCfg.tagIndexBlacklist == "" { nsCfg.tagIndexBlacklist = opt.primary.tagIndexBlacklist } - nsCfg.TagIndexBlacklist = strings.Split(nsCfg.tagIndexBlacklist, ",") + if len(nsCfg.tagIndexBlacklist) > 0 { + nsCfg.TagIndexBlacklist = strings.Split(nsCfg.tagIndexBlacklist, ",") + } if nsCfg.tagIndexWhitelist == "" { nsCfg.tagIndexWhitelist = opt.primary.tagIndexWhitelist } - nsCfg.TagIndexWhitelist = strings.Split(nsCfg.tagIndexWhitelist, ",") + if len(nsCfg.tagIndexWhitelist) > 0 { + nsCfg.TagIndexWhitelist = strings.Split(nsCfg.tagIndexWhitelist, ",") + } return &nsCfg.Configuration } From d75054c27d627a0bd295307888dd0b94c2235c4e Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 7 Nov 2019 10:07:08 -0500 Subject: [PATCH 04/19] make fmt Signed-off-by: Joe Elliott --- .../dbmodel/tag_filter_blacklist_test.go | 16 ++++++++-------- .../dbmodel/tag_filter_whitelist_test.go | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go index 59a905921f6..e70b5aa98bb 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go @@ -26,15 +26,15 @@ import ( func TestBlacklistFilter(t *testing.T) { // expected tt := [][][]string{ - [][]string{ - []string{"a", "b", "c"}, // input - []string{"a"}, // filter - []string{"b", "c"}, // expected + { + {"a", "b", "c"}, // input + {"a"}, // filter + {"b", "c"}, // expected }, - [][]string{ - []string{"a", "b", "c"}, - []string{"A"}, - []string{"a", "b", "c"}, + { + {"a", "b", "c"}, + {"A"}, + {"a", "b", "c"}, }, } diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go index 33e4119e626..cab4eccae51 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go @@ -26,15 +26,15 @@ import ( func TestWhitelistFilter(t *testing.T) { // expected tt := [][][]string{ - [][]string{ - []string{"a", "b", "c"}, // input - []string{"a"}, // filter - []string{"a"}, // expected + { + {"a", "b", "c"}, // input + {"a"}, // filter + {"a"}, // expected }, - [][]string{ - []string{"a", "b", "c"}, - []string{"A"}, - []string{}, + { + {"a", "b", "c"}, + {"A"}, + {}, }, } From 1c49ef760d81d60e98f7896438bbc259037ca55c Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 7 Nov 2019 10:52:14 -0500 Subject: [PATCH 05/19] Expanded filter tests Signed-off-by: Joe Elliott --- .../spanstore/dbmodel/tag_filter_blacklist_test.go | 14 +++++++++++++- .../spanstore/dbmodel/tag_filter_whitelist_test.go | 14 +++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go index e70b5aa98bb..c7cebaa8ea3 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go @@ -51,11 +51,23 @@ func TestBlacklistFilter(t *testing.T) { for _, e := range expected { expectedKVs = append(expectedKVs, model.String(e, "")) } + expectedKVs.Sort() tf := NewBlacklistTagFilter(filter) actualKVs := tf.filter(inputKVs) actualKVs.Sort() - expectedKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterLogFields(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterProcessTags(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterTags(nil, inputKVs) + actualKVs.Sort() assert.Equal(t, actualKVs, expectedKVs) } } diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go index cab4eccae51..7fe714e542d 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go @@ -51,11 +51,23 @@ func TestWhitelistFilter(t *testing.T) { for _, e := range expected { expectedKVs = append(expectedKVs, model.String(e, "")) } + expectedKVs.Sort() tf := NewWhitelistTagFilter(filter) actualKVs := tf.filter(inputKVs) actualKVs.Sort() - expectedKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterLogFields(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterProcessTags(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterTags(nil, inputKVs) + actualKVs.Sort() assert.Equal(t, actualKVs, expectedKVs) } } From c691d2debde4e8cfd2f26fbb360a6f68fea9ddee Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 7 Nov 2019 11:15:20 -0500 Subject: [PATCH 06/19] Expanded factory tests Signed-off-by: Joe Elliott --- plugin/storage/cassandra/factory_test.go | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/plugin/storage/cassandra/factory_test.go b/plugin/storage/cassandra/factory_test.go index aee08c85f29..10fe0e8ca8c 100644 --- a/plugin/storage/cassandra/factory_test.go +++ b/plugin/storage/cassandra/factory_test.go @@ -139,3 +139,29 @@ func TestExclusiveWhiteListBlackList(t *testing.T) { _, err = f.CreateArchiveSpanWriter() assert.EqualError(t, err, "only one of TagIndexBlacklist and TagIndexWhitelist can be specified") } + +func TestWriterOptions(t *testing.T) { + opts := NewOptions("cassandra") + v, command := config.Viperize(opts.AddFlags) + command.ParseFlags([]string{"--cassandra.tag-index-whitelist=a,b,c"}) + opts.InitFromViper(v) + + options, _ := writerOptions(opts.GetPrimary()) + assert.Len(t, options, 1) + + opts = NewOptions("cassandra") + v, command = config.Viperize(opts.AddFlags) + command.ParseFlags([]string{"--cassandra.tag-index-blacklist=a,b,c"}) + opts.InitFromViper(v) + + options, _ = writerOptions(opts.GetPrimary()) + assert.Len(t, options, 1) + + opts = NewOptions("cassandra") + v, command = config.Viperize(opts.AddFlags) + command.ParseFlags([]string{""}) + opts.InitFromViper(v) + + options, _ = writerOptions(opts.GetPrimary()) + assert.Len(t, options, 0) +} From 9726f1f1ed6c9ac91599263f5de2516e78cef495 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Thu, 7 Nov 2019 21:03:41 -0500 Subject: [PATCH 07/19] Moved tag filters out of cassandra connection config Signed-off-by: Joe Elliott --- pkg/cassandra/config/config.go | 2 - plugin/storage/cassandra/factory.go | 19 ++++--- plugin/storage/cassandra/factory_test.go | 6 +- plugin/storage/cassandra/options.go | 72 ++++++++++++------------ plugin/storage/cassandra/options_test.go | 20 ++++--- 5 files changed, 64 insertions(+), 55 deletions(-) diff --git a/pkg/cassandra/config/config.go b/pkg/cassandra/config/config.go index 733406b90f1..8aba2e11680 100644 --- a/pkg/cassandra/config/config.go +++ b/pkg/cassandra/config/config.go @@ -45,8 +45,6 @@ type Configuration struct { DisableAutoDiscovery bool `yaml:"disable_auto_discovery"` EnableDependenciesV2 bool `yaml:"enable_dependencies_v2"` TLS TLS - TagIndexWhitelist []string `yaml:"tag-index-whitelist"` - TagIndexBlacklist []string `yaml:"tag-index-blacklist"` } // Authenticator holds the authentication properties needed to connect to a Cassandra cluster diff --git a/plugin/storage/cassandra/factory.go b/plugin/storage/cassandra/factory.go index b468b0f937c..6a471e3a89d 100644 --- a/plugin/storage/cassandra/factory.go +++ b/plugin/storage/cassandra/factory.go @@ -104,7 +104,7 @@ func (f *Factory) CreateSpanReader() (spanstore.Reader, error) { // CreateSpanWriter implements storage.Factory func (f *Factory) CreateSpanWriter() (spanstore.Writer, error) { - options, err := writerOptions(f.Options.GetPrimary()) + options, err := writerOptions(f.Options) if err != nil { return nil, err } @@ -130,23 +130,26 @@ func (f *Factory) CreateArchiveSpanWriter() (spanstore.Writer, error) { if f.archiveSession == nil { return nil, storage.ErrArchiveStorageNotConfigured } - options, err := writerOptions(f.Options.Get(archiveStorageConfig)) + options, err := writerOptions(f.Options) if err != nil { return nil, err } return cSpanStore.NewSpanWriter(f.archiveSession, f.Options.SpanStoreWriteCacheTTL, f.archiveMetricsFactory, f.logger, options...), nil } -func writerOptions(config *config.Configuration) ([]cSpanStore.Option, error) { - if len(config.TagIndexBlacklist) > 0 && len(config.TagIndexWhitelist) > 0 { +func writerOptions(opts *Options) ([]cSpanStore.Option, error) { + tagIndexBlacklist := opts.GetTagIndexBlacklist() + tagIndexWhitelist := opts.GetTagIndexWhitelist() + + if len(tagIndexBlacklist) > 0 && len(tagIndexWhitelist) > 0 { return nil, errors.New("only one of TagIndexBlacklist and TagIndexWhitelist can be specified") } var options []cSpanStore.Option - if len(config.TagIndexBlacklist) > 0 { - options = append(options, cSpanStore.TagFilter(dbmodel.NewBlacklistTagFilter(config.TagIndexBlacklist))) - } else if len(config.TagIndexWhitelist) > 0 { - options = append(options, cSpanStore.TagFilter(dbmodel.NewWhitelistTagFilter(config.TagIndexWhitelist))) + if len(tagIndexBlacklist) > 0 { + options = append(options, cSpanStore.TagFilter(dbmodel.NewBlacklistTagFilter(tagIndexBlacklist))) + } else if len(tagIndexWhitelist) > 0 { + options = append(options, cSpanStore.TagFilter(dbmodel.NewWhitelistTagFilter(tagIndexWhitelist))) } return options, nil } diff --git a/plugin/storage/cassandra/factory_test.go b/plugin/storage/cassandra/factory_test.go index 10fe0e8ca8c..99c814ee11c 100644 --- a/plugin/storage/cassandra/factory_test.go +++ b/plugin/storage/cassandra/factory_test.go @@ -146,7 +146,7 @@ func TestWriterOptions(t *testing.T) { command.ParseFlags([]string{"--cassandra.tag-index-whitelist=a,b,c"}) opts.InitFromViper(v) - options, _ := writerOptions(opts.GetPrimary()) + options, _ := writerOptions(opts) assert.Len(t, options, 1) opts = NewOptions("cassandra") @@ -154,7 +154,7 @@ func TestWriterOptions(t *testing.T) { command.ParseFlags([]string{"--cassandra.tag-index-blacklist=a,b,c"}) opts.InitFromViper(v) - options, _ = writerOptions(opts.GetPrimary()) + options, _ = writerOptions(opts) assert.Len(t, options, 1) opts = NewOptions("cassandra") @@ -162,6 +162,6 @@ func TestWriterOptions(t *testing.T) { command.ParseFlags([]string{""}) opts.InitFromViper(v) - options, _ = writerOptions(opts.GetPrimary()) + options, _ = writerOptions(opts) assert.Len(t, options, 0) } diff --git a/plugin/storage/cassandra/options.go b/plugin/storage/cassandra/options.go index 53901453a20..6614d3e54f1 100644 --- a/plugin/storage/cassandra/options.go +++ b/plugin/storage/cassandra/options.go @@ -50,11 +50,11 @@ const ( suffixServerName = ".tls.server-name" suffixVerifyHost = ".tls.verify-host" suffixEnableDependenciesV2 = ".enable-dependencies-v2" - suffixTagIndexBlacklist = ".tag-index-blacklist" - suffixTagIndexWhitelist = ".tag-index-whitelist" // common storage settings suffixSpanStoreWriteCacheTTL = ".span-store-write-cache-ttl" + suffixTagIndexBlacklist = ".tag-index-blacklist" + suffixTagIndexWhitelist = ".tag-index-whitelist" ) // Options contains various type of Cassandra configs and provides the ability @@ -64,6 +64,8 @@ type Options struct { primary *namespaceConfig others map[string]*namespaceConfig SpanStoreWriteCacheTTL time.Duration + tagIndexBlacklist string + tagIndexWhitelist string } // the Servers field in config.Configuration is a list, which we cannot represent with flags. @@ -71,12 +73,10 @@ type Options struct { // preparing the actual config.Configuration. type namespaceConfig struct { config.Configuration - servers string - tagIndexBlacklist string - tagIndexWhitelist string - namespace string - primary bool - Enabled bool + servers string + namespace string + primary bool + Enabled bool } // NewOptions creates a new Options struct. @@ -120,6 +120,15 @@ func (opt *Options) AddFlags(flagSet *flag.FlagSet) { flagSet.Duration(opt.primary.namespace+suffixSpanStoreWriteCacheTTL, opt.SpanStoreWriteCacheTTL, "The duration to wait before rewriting an existing service or operation name") + flagSet.String( + opt.primary.namespace+suffixTagIndexBlacklist, + opt.tagIndexBlacklist, + "The comma-separated list of tags to blacklist") + flagSet.String( + opt.primary.namespace+suffixTagIndexWhitelist, + opt.tagIndexWhitelist, + "The comma-separated list of tags to whitelist") + } func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { @@ -217,14 +226,6 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { nsConfig.namespace+suffixEnableDependenciesV2, nsConfig.EnableDependenciesV2, "(deprecated) Jaeger will automatically detect the version of the dependencies table") - flagSet.String( - nsConfig.namespace+suffixTagIndexBlacklist, - nsConfig.tagIndexWhitelist, - "The comma-separated list of tags to blacklist") - flagSet.String( - nsConfig.namespace+suffixTagIndexWhitelist, - nsConfig.tagIndexBlacklist, - "The comma-separated list of tags to whitelist") } // InitFromViper initializes Options with properties from viper @@ -234,6 +235,9 @@ func (opt *Options) InitFromViper(v *viper.Viper) { cfg.initFromViper(v) } opt.SpanStoreWriteCacheTTL = v.GetDuration(opt.primary.namespace + suffixSpanStoreWriteCacheTTL) + opt.tagIndexBlacklist = stripWhiteSpace(v.GetString(opt.primary.namespace + suffixTagIndexBlacklist)) + opt.tagIndexWhitelist = stripWhiteSpace(v.GetString(opt.primary.namespace + suffixTagIndexWhitelist)) + } func (cfg *namespaceConfig) initFromViper(v *viper.Viper) { @@ -262,19 +266,11 @@ func (cfg *namespaceConfig) initFromViper(v *viper.Viper) { cfg.TLS.EnableHostVerification = v.GetBool(cfg.namespace + suffixVerifyHost) cfg.EnableDependenciesV2 = v.GetBool(cfg.namespace + suffixEnableDependenciesV2) cfg.DisableCompression = v.GetBool(cfg.namespace + suffixDisableCompression) - cfg.tagIndexBlacklist = stripWhiteSpace(v.GetString(cfg.namespace + suffixTagIndexBlacklist)) - cfg.tagIndexWhitelist = stripWhiteSpace(v.GetString(cfg.namespace + suffixTagIndexWhitelist)) } // GetPrimary returns primary configuration. func (opt *Options) GetPrimary() *config.Configuration { opt.primary.Servers = strings.Split(opt.primary.servers, ",") - if len(opt.primary.tagIndexBlacklist) > 0 { - opt.primary.TagIndexBlacklist = strings.Split(opt.primary.tagIndexBlacklist, ",") - } - if len(opt.primary.tagIndexWhitelist) > 0 { - opt.primary.TagIndexWhitelist = strings.Split(opt.primary.tagIndexWhitelist, ",") - } return &opt.primary.Configuration } @@ -293,19 +289,25 @@ func (opt *Options) Get(namespace string) *config.Configuration { nsCfg.servers = opt.primary.servers } nsCfg.Servers = strings.Split(nsCfg.servers, ",") - if nsCfg.tagIndexBlacklist == "" { - nsCfg.tagIndexBlacklist = opt.primary.tagIndexBlacklist - } - if len(nsCfg.tagIndexBlacklist) > 0 { - nsCfg.TagIndexBlacklist = strings.Split(nsCfg.tagIndexBlacklist, ",") - } - if nsCfg.tagIndexWhitelist == "" { - nsCfg.tagIndexWhitelist = opt.primary.tagIndexWhitelist + return &nsCfg.Configuration +} + +// GetTagIndexBlacklist returns the list of blacklisted tags +func (opt *Options) GetTagIndexBlacklist() []string { + if len(opt.tagIndexBlacklist) > 0 { + return strings.Split(opt.tagIndexBlacklist, ",") } - if len(nsCfg.tagIndexWhitelist) > 0 { - nsCfg.TagIndexWhitelist = strings.Split(nsCfg.tagIndexWhitelist, ",") + + return []string{} +} + +// GetTagIndexWhitelist returns the list of whitelisted tags +func (opt *Options) GetTagIndexWhitelist() []string { + if len(opt.tagIndexWhitelist) > 0 { + return strings.Split(opt.tagIndexWhitelist, ",") } - return &nsCfg.Configuration + + return []string{} } // stripWhiteSpace removes all whitespace characters from a string diff --git a/plugin/storage/cassandra/options_test.go b/plugin/storage/cassandra/options_test.go index c71a140ac67..d981c66e64d 100644 --- a/plugin/storage/cassandra/options_test.go +++ b/plugin/storage/cassandra/options_test.go @@ -43,8 +43,6 @@ func TestOptions(t *testing.T) { assert.Equal(t, primary.Servers, aux.Servers) assert.Equal(t, primary.ConnectionsPerHost, aux.ConnectionsPerHost) assert.Equal(t, primary.ReconnectInterval, aux.ReconnectInterval) - assert.Equal(t, primary.TagIndexBlacklist, aux.TagIndexBlacklist) - assert.Equal(t, primary.TagIndexWhitelist, aux.TagIndexWhitelist) } func TestOptionsWithFlags(t *testing.T) { @@ -63,7 +61,7 @@ func TestOptionsWithFlags(t *testing.T) { "--cas.proto-version=3", "--cas.socket-keep-alive=42s", "--cas.tag-index-blacklist=blerg, blarg,blorg ", - "--cas.tag-index-whitelist=blerg, blarg,blorg ", + "--cas.tag-index-whitelist=flerg, flarg,florg ", // enable aux with a couple overrides "--cas-aux.enabled=true", "--cas-aux.keyspace=jaeger-archive", @@ -79,8 +77,8 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, []string{"1.1.1.1", "2.2.2.2"}, primary.Servers) assert.Equal(t, "ONE", primary.Consistency) assert.Equal(t, false, primary.EnableDependenciesV2) - assert.Equal(t, []string{"blerg", "blarg", "blorg"}, primary.TagIndexBlacklist) - assert.Equal(t, []string{"blerg", "blarg", "blorg"}, primary.TagIndexWhitelist) + assert.Equal(t, []string{"blerg", "blarg", "blorg"}, opts.GetTagIndexBlacklist()) + assert.Equal(t, []string{"flerg", "flarg", "florg"}, opts.GetTagIndexWhitelist()) aux := opts.Get("cas-aux") require.NotNil(t, aux) @@ -95,6 +93,14 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, 3, aux.ProtoVersion) assert.Equal(t, 42*time.Second, aux.SocketKeepAlive) assert.Equal(t, true, aux.EnableDependenciesV2) - assert.Equal(t, []string{"blerg", "blarg", "blorg"}, aux.TagIndexBlacklist) - assert.Equal(t, []string{"foo", "bar"}, aux.TagIndexWhitelist) +} + +func TestEmptyBlackWhiteLists(t *testing.T) { + opts := NewOptions("cas") + v, command := config.Viperize(opts.AddFlags) + command.ParseFlags([]string{}) + opts.InitFromViper(v) + + assert.Equal(t, []string{}, opts.GetTagIndexBlacklist()) + assert.Equal(t, []string{}, opts.GetTagIndexWhitelist()) } From 9ffdb9b30d5c49f4cc20aadf8f972318a6e5d80a Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 08:27:19 -0500 Subject: [PATCH 08/19] Addressed review items Signed-off-by: Joe Elliott --- jaeger-ui | 2 +- plugin/storage/cassandra/factory.go | 4 ++-- plugin/storage/cassandra/factory_test.go | 2 +- plugin/storage/cassandra/options.go | 16 ++++++++-------- plugin/storage/cassandra/options_test.go | 9 ++++----- .../spanstore/dbmodel/tag_filter_blacklist.go | 7 +++---- .../dbmodel/tag_filter_blacklist_test.go | 1 - .../spanstore/dbmodel/tag_filter_whitelist.go | 15 +++++++-------- .../dbmodel/tag_filter_whitelist_test.go | 1 - 9 files changed, 26 insertions(+), 31 deletions(-) diff --git a/jaeger-ui b/jaeger-ui index df4c89751f9..6f78c8d7071 160000 --- a/jaeger-ui +++ b/jaeger-ui @@ -1 +1 @@ -Subproject commit df4c89751f9a3ad27bf483ecdac0a37a1eaf314f +Subproject commit 6f78c8d70714887ef4ba93c22219a3f783c12b63 diff --git a/plugin/storage/cassandra/factory.go b/plugin/storage/cassandra/factory.go index 6a471e3a89d..6045e25e21a 100644 --- a/plugin/storage/cassandra/factory.go +++ b/plugin/storage/cassandra/factory.go @@ -138,8 +138,8 @@ func (f *Factory) CreateArchiveSpanWriter() (spanstore.Writer, error) { } func writerOptions(opts *Options) ([]cSpanStore.Option, error) { - tagIndexBlacklist := opts.GetTagIndexBlacklist() - tagIndexWhitelist := opts.GetTagIndexWhitelist() + tagIndexBlacklist := opts.TagIndexBlacklist() + tagIndexWhitelist := opts.TagIndexWhitelist() if len(tagIndexBlacklist) > 0 && len(tagIndexWhitelist) > 0 { return nil, errors.New("only one of TagIndexBlacklist and TagIndexWhitelist can be specified") diff --git a/plugin/storage/cassandra/factory_test.go b/plugin/storage/cassandra/factory_test.go index 99c814ee11c..a031334fa66 100644 --- a/plugin/storage/cassandra/factory_test.go +++ b/plugin/storage/cassandra/factory_test.go @@ -101,7 +101,7 @@ func TestCassandraFactory(t *testing.T) { assert.NoError(t, err) } -func TestExclusiveWhiteListBlackList(t *testing.T) { +func TestExclusiveWhitelistBlacklist(t *testing.T) { logger, logBuf := testutils.NewLogger() f := NewFactory() v, command := config.Viperize(f.AddFlags) diff --git a/plugin/storage/cassandra/options.go b/plugin/storage/cassandra/options.go index 6614d3e54f1..49898948f1f 100644 --- a/plugin/storage/cassandra/options.go +++ b/plugin/storage/cassandra/options.go @@ -123,11 +123,11 @@ func (opt *Options) AddFlags(flagSet *flag.FlagSet) { flagSet.String( opt.primary.namespace+suffixTagIndexBlacklist, opt.tagIndexBlacklist, - "The comma-separated list of tags to blacklist") + "The comma-separated list of span tags to blacklist from being indexed. All other tags will be indexed. Mutually exclusive with the whitelist option.") flagSet.String( opt.primary.namespace+suffixTagIndexWhitelist, opt.tagIndexWhitelist, - "The comma-separated list of tags to whitelist") + "The comma-separated list of span tags to whitelist for being indexed. All other tags will not be indexed. Mutually exclusive with the blacklist option.") } @@ -292,22 +292,22 @@ func (opt *Options) Get(namespace string) *config.Configuration { return &nsCfg.Configuration } -// GetTagIndexBlacklist returns the list of blacklisted tags -func (opt *Options) GetTagIndexBlacklist() []string { +// TagIndexBlacklist returns the list of blacklisted tags +func (opt *Options) TagIndexBlacklist() []string { if len(opt.tagIndexBlacklist) > 0 { return strings.Split(opt.tagIndexBlacklist, ",") } - return []string{} + return nil } -// GetTagIndexWhitelist returns the list of whitelisted tags -func (opt *Options) GetTagIndexWhitelist() []string { +// TagIndexWhitelist returns the list of whitelisted tags +func (opt *Options) TagIndexWhitelist() []string { if len(opt.tagIndexWhitelist) > 0 { return strings.Split(opt.tagIndexWhitelist, ",") } - return []string{} + return nil } // stripWhiteSpace removes all whitespace characters from a string diff --git a/plugin/storage/cassandra/options_test.go b/plugin/storage/cassandra/options_test.go index d981c66e64d..ffcdac1e87c 100644 --- a/plugin/storage/cassandra/options_test.go +++ b/plugin/storage/cassandra/options_test.go @@ -67,7 +67,6 @@ func TestOptionsWithFlags(t *testing.T) { "--cas-aux.keyspace=jaeger-archive", "--cas-aux.servers=3.3.3.3, 4.4.4.4", "--cas-aux.enable-dependencies-v2=true", - "--cas-aux.tag-index-whitelist=foo,bar", }) opts.InitFromViper(v) @@ -77,8 +76,8 @@ func TestOptionsWithFlags(t *testing.T) { assert.Equal(t, []string{"1.1.1.1", "2.2.2.2"}, primary.Servers) assert.Equal(t, "ONE", primary.Consistency) assert.Equal(t, false, primary.EnableDependenciesV2) - assert.Equal(t, []string{"blerg", "blarg", "blorg"}, opts.GetTagIndexBlacklist()) - assert.Equal(t, []string{"flerg", "flarg", "florg"}, opts.GetTagIndexWhitelist()) + assert.Equal(t, []string{"blerg", "blarg", "blorg"}, opts.TagIndexBlacklist()) + assert.Equal(t, []string{"flerg", "flarg", "florg"}, opts.TagIndexWhitelist()) aux := opts.Get("cas-aux") require.NotNil(t, aux) @@ -101,6 +100,6 @@ func TestEmptyBlackWhiteLists(t *testing.T) { command.ParseFlags([]string{}) opts.InitFromViper(v) - assert.Equal(t, []string{}, opts.GetTagIndexBlacklist()) - assert.Equal(t, []string{}, opts.GetTagIndexWhitelist()) + assert.Len(t, opts.TagIndexBlacklist(), 0) + assert.Len(t, opts.TagIndexWhitelist(), 0) } diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go index afdecb106ab..106ae0561ac 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go @@ -1,5 +1,4 @@ // Copyright (c) 2019 The Jaeger Authors. -// Copyright (c) 2017 Uber Technologies, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -51,11 +50,11 @@ func (tf BlacklistTagFilter) FilterLogFields(span *model.Span, logFields model.K } func (tf BlacklistTagFilter) filter(processTags model.KeyValues) model.KeyValues { - var tags model.KeyValues + var kvs model.KeyValues for _, t := range processTags { if _, ok := tf.tags[t.Key]; !ok { - tags = append(tags, t) + kvs = append(kvs, t) } } - return tags + return kvs } diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go index c7cebaa8ea3..69848927212 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go @@ -1,5 +1,4 @@ // Copyright (c) 2019 The Jaeger Authors. -// Copyright (c) 2017 Uber Technologies, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go index c00dfcbceab..941532acb98 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go @@ -1,5 +1,4 @@ // Copyright (c) 2019 The Jaeger Authors. -// Copyright (c) 2017 Uber Technologies, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -36,8 +35,8 @@ func NewWhitelistTagFilter(tags []string) WhitelistTagFilter { } // FilterProcessTags implements TagFilter -func (tf WhitelistTagFilter) FilterProcessTags(span *model.Span, processTags model.KeyValues) model.KeyValues { - return tf.filter(processTags) +func (tf WhitelistTagFilter) FilterProcessTags(span *model.Span, tags model.KeyValues) model.KeyValues { + return tf.filter(tags) } // FilterTags implements TagFilter @@ -50,12 +49,12 @@ func (tf WhitelistTagFilter) FilterLogFields(span *model.Span, logFields model.K return tf.filter(logFields) } -func (tf WhitelistTagFilter) filter(processTags model.KeyValues) model.KeyValues { - var tags model.KeyValues - for _, t := range processTags { +func (tf WhitelistTagFilter) filter(tags model.KeyValues) model.KeyValues { + var kvs model.KeyValues + for _, t := range tags { if _, ok := tf.tags[t.Key]; ok { - tags = append(tags, t) + kvs = append(kvs, t) } } - return tags + return kvs } diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go index 7fe714e542d..61c187ee543 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go @@ -1,5 +1,4 @@ // Copyright (c) 2019 The Jaeger Authors. -// Copyright (c) 2017 Uber Technologies, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From e86345b23ea789c39d8ec74bdcbcb22f837fb33b Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 08:47:27 -0500 Subject: [PATCH 09/19] Combined white and blacklist filters Signed-off-by: Joe Elliott --- .../dbmodel/tag_filter_blacklist_test.go | 72 ------------------- ...blacklist.go => tag_filter_exact_match.go} | 28 ++++---- .../spanstore/dbmodel/tag_filter_whitelist.go | 60 ---------------- .../dbmodel/tag_filter_whitelist_test.go | 72 ------------------- 4 files changed, 16 insertions(+), 216 deletions(-) delete mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go rename plugin/storage/cassandra/spanstore/dbmodel/{tag_filter_blacklist.go => tag_filter_exact_match.go} (52%) delete mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go delete mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go deleted file mode 100644 index 69848927212..00000000000 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2019 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dbmodel - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/jaegertracing/jaeger/model" -) - -func TestBlacklistFilter(t *testing.T) { - // expected - tt := [][][]string{ - { - {"a", "b", "c"}, // input - {"a"}, // filter - {"b", "c"}, // expected - }, - { - {"a", "b", "c"}, - {"A"}, - {"a", "b", "c"}, - }, - } - - for _, test := range tt { - input := test[0] - filter := test[1] - expected := test[2] - - var inputKVs model.KeyValues - for _, i := range input { - inputKVs = append(inputKVs, model.String(i, "")) - } - var expectedKVs model.KeyValues - for _, e := range expected { - expectedKVs = append(expectedKVs, model.String(e, "")) - } - expectedKVs.Sort() - - tf := NewBlacklistTagFilter(filter) - actualKVs := tf.filter(inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - - actualKVs = tf.FilterLogFields(nil, inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - - actualKVs = tf.FilterProcessTags(nil, inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - - actualKVs = tf.FilterTags(nil, inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - } -} diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go similarity index 52% rename from plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go rename to plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go index 106ae0561ac..7dca920e121 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_blacklist.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go @@ -18,41 +18,45 @@ import ( "github.com/jaegertracing/jaeger/model" ) -// BlacklistTagFilter filters out all tags in its tags slice -type BlacklistTagFilter struct { - tags map[string]struct{} +// ExactMatchTagFilter filters out all tags in its tags slice +type ExactMatchTagFilter struct { + tags map[string]struct{} + dropMatches bool } -// NewBlacklistTagFilter creates a BlacklistTagFilter with the provided tags -func NewBlacklistTagFilter(tags []string) BlacklistTagFilter { +// NewExactMatchTagFilter creates a ExactMatchTagFilter with the provided tags. Passing +// dropMatches true will exhibit blacklist behavior. Passing dropMatches false +// will exhibit whitelist behavior. +func NewExactMatchTagFilter(tags []string, dropMatches bool) ExactMatchTagFilter { mapTags := make(map[string]struct{}) for _, t := range tags { mapTags[t] = struct{}{} } - return BlacklistTagFilter{ - tags: mapTags, + return ExactMatchTagFilter{ + tags: mapTags, + dropMatches: dropMatches, } } // FilterProcessTags implements TagFilter -func (tf BlacklistTagFilter) FilterProcessTags(span *model.Span, processTags model.KeyValues) model.KeyValues { +func (tf ExactMatchTagFilter) FilterProcessTags(span *model.Span, processTags model.KeyValues) model.KeyValues { return tf.filter(processTags) } // FilterTags implements TagFilter -func (tf BlacklistTagFilter) FilterTags(span *model.Span, tags model.KeyValues) model.KeyValues { +func (tf ExactMatchTagFilter) FilterTags(span *model.Span, tags model.KeyValues) model.KeyValues { return tf.filter(tags) } // FilterLogFields implements TagFilter -func (tf BlacklistTagFilter) FilterLogFields(span *model.Span, logFields model.KeyValues) model.KeyValues { +func (tf ExactMatchTagFilter) FilterLogFields(span *model.Span, logFields model.KeyValues) model.KeyValues { return tf.filter(logFields) } -func (tf BlacklistTagFilter) filter(processTags model.KeyValues) model.KeyValues { +func (tf ExactMatchTagFilter) filter(processTags model.KeyValues) model.KeyValues { var kvs model.KeyValues for _, t := range processTags { - if _, ok := tf.tags[t.Key]; !ok { + if _, ok := tf.tags[t.Key]; ok == !tf.dropMatches { kvs = append(kvs, t) } } diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go deleted file mode 100644 index 941532acb98..00000000000 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2019 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dbmodel - -import ( - "github.com/jaegertracing/jaeger/model" -) - -// WhitelistTagFilter filters out all tags in its tags slice -type WhitelistTagFilter struct { - tags map[string]struct{} -} - -// NewWhitelistTagFilter creates a WhitelistTagFilter with the provided tags -func NewWhitelistTagFilter(tags []string) WhitelistTagFilter { - mapTags := make(map[string]struct{}) - for _, t := range tags { - mapTags[t] = struct{}{} - } - return WhitelistTagFilter{ - tags: mapTags, - } -} - -// FilterProcessTags implements TagFilter -func (tf WhitelistTagFilter) FilterProcessTags(span *model.Span, tags model.KeyValues) model.KeyValues { - return tf.filter(tags) -} - -// FilterTags implements TagFilter -func (tf WhitelistTagFilter) FilterTags(span *model.Span, tags model.KeyValues) model.KeyValues { - return tf.filter(tags) -} - -// FilterLogFields implements TagFilter -func (tf WhitelistTagFilter) FilterLogFields(span *model.Span, logFields model.KeyValues) model.KeyValues { - return tf.filter(logFields) -} - -func (tf WhitelistTagFilter) filter(tags model.KeyValues) model.KeyValues { - var kvs model.KeyValues - for _, t := range tags { - if _, ok := tf.tags[t.Key]; ok { - kvs = append(kvs, t) - } - } - return kvs -} diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go deleted file mode 100644 index 61c187ee543..00000000000 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_whitelist_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2019 The Jaeger Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dbmodel - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/jaegertracing/jaeger/model" -) - -func TestWhitelistFilter(t *testing.T) { - // expected - tt := [][][]string{ - { - {"a", "b", "c"}, // input - {"a"}, // filter - {"a"}, // expected - }, - { - {"a", "b", "c"}, - {"A"}, - {}, - }, - } - - for _, test := range tt { - input := test[0] - filter := test[1] - expected := test[2] - - var inputKVs model.KeyValues - for _, i := range input { - inputKVs = append(inputKVs, model.String(i, "")) - } - var expectedKVs model.KeyValues - for _, e := range expected { - expectedKVs = append(expectedKVs, model.String(e, "")) - } - expectedKVs.Sort() - - tf := NewWhitelistTagFilter(filter) - actualKVs := tf.filter(inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - - actualKVs = tf.FilterLogFields(nil, inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - - actualKVs = tf.FilterProcessTags(nil, inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - - actualKVs = tf.FilterTags(nil, inputKVs) - actualKVs.Sort() - assert.Equal(t, actualKVs, expectedKVs) - } -} From eca658afb14cba98f5cbb28fe21542f9d046d29d Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 08:56:09 -0500 Subject: [PATCH 10/19] Added exact match tests Signed-off-by: Joe Elliott --- .../dbmodel/tag_filter_exact_match_test.go | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go new file mode 100644 index 00000000000..61e8351d212 --- /dev/null +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go @@ -0,0 +1,121 @@ +// Copyright (c) 2019 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dbmodel + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/jaegertracing/jaeger/model" +) + +func TestBlacklistFilter(t *testing.T) { + // expected + tt := [][][]string{ + { + {"a", "b", "c"}, // input + {"a"}, // filter + {"b", "c"}, // expected + }, + { + {"a", "b", "c"}, + {"A"}, + {"a", "b", "c"}, + }, + } + + for _, test := range tt { + input := test[0] + filter := test[1] + expected := test[2] + + var inputKVs model.KeyValues + for _, i := range input { + inputKVs = append(inputKVs, model.String(i, "")) + } + var expectedKVs model.KeyValues + for _, e := range expected { + expectedKVs = append(expectedKVs, model.String(e, "")) + } + expectedKVs.Sort() + + tf := NewExactMatchTagFilter(filter, true) + actualKVs := tf.filter(inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterLogFields(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterProcessTags(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterTags(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + } +} + +func TestWhitelistFilter(t *testing.T) { + // expected + tt := [][][]string{ + { + {"a", "b", "c"}, // input + {"a"}, // filter + {"a"}, // expected + }, + { + {"a", "b", "c"}, + {"A"}, + {}, + }, + } + + for _, test := range tt { + input := test[0] + filter := test[1] + expected := test[2] + + var inputKVs model.KeyValues + for _, i := range input { + inputKVs = append(inputKVs, model.String(i, "")) + } + var expectedKVs model.KeyValues + for _, e := range expected { + expectedKVs = append(expectedKVs, model.String(e, "")) + } + expectedKVs.Sort() + + tf := NewExactMatchTagFilter(filter, false) + actualKVs := tf.filter(inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterLogFields(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterProcessTags(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + + actualKVs = tf.FilterTags(nil, inputKVs) + actualKVs.Sort() + assert.Equal(t, actualKVs, expectedKVs) + } +} From 63834ab20d32d457a9206399ca1a263d611c17c5 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 09:05:31 -0500 Subject: [PATCH 11/19] processTags => tags Signed-off-by: Joe Elliott --- .../cassandra/spanstore/dbmodel/tag_filter_exact_match.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go index 7dca920e121..da2a6a2fea3 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go @@ -53,9 +53,9 @@ func (tf ExactMatchTagFilter) FilterLogFields(span *model.Span, logFields model. return tf.filter(logFields) } -func (tf ExactMatchTagFilter) filter(processTags model.KeyValues) model.KeyValues { +func (tf ExactMatchTagFilter) filter(tags model.KeyValues) model.KeyValues { var kvs model.KeyValues - for _, t := range processTags { + for _, t := range tags { if _, ok := tf.tags[t.Key]; ok == !tf.dropMatches { kvs = append(kvs, t) } From 08f3487dc5d6e20059ea6b948388532c4305bf2a Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 09:07:17 -0500 Subject: [PATCH 12/19] Create exact match tag filters Signed-off-by: Joe Elliott --- plugin/storage/cassandra/factory.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/storage/cassandra/factory.go b/plugin/storage/cassandra/factory.go index 6045e25e21a..0f279ca95b8 100644 --- a/plugin/storage/cassandra/factory.go +++ b/plugin/storage/cassandra/factory.go @@ -147,9 +147,9 @@ func writerOptions(opts *Options) ([]cSpanStore.Option, error) { var options []cSpanStore.Option if len(tagIndexBlacklist) > 0 { - options = append(options, cSpanStore.TagFilter(dbmodel.NewBlacklistTagFilter(tagIndexBlacklist))) + options = append(options, cSpanStore.TagFilter(dbmodel.NewExactMatchTagFilter(tagIndexBlacklist, true))) } else if len(tagIndexWhitelist) > 0 { - options = append(options, cSpanStore.TagFilter(dbmodel.NewWhitelistTagFilter(tagIndexWhitelist))) + options = append(options, cSpanStore.TagFilter(dbmodel.NewExactMatchTagFilter(tagIndexWhitelist, false))) } return options, nil } From 6b4866a1d6a379aead712b12ec67298727ed0c22 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 09:09:36 -0500 Subject: [PATCH 13/19] Better var name Signed-off-by: Joe Elliott --- .../cassandra/spanstore/dbmodel/tag_filter_exact_match.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go index da2a6a2fea3..1e78a843997 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go @@ -54,11 +54,11 @@ func (tf ExactMatchTagFilter) FilterLogFields(span *model.Span, logFields model. } func (tf ExactMatchTagFilter) filter(tags model.KeyValues) model.KeyValues { - var kvs model.KeyValues + var filteredTags model.KeyValues for _, t := range tags { if _, ok := tf.tags[t.Key]; ok == !tf.dropMatches { - kvs = append(kvs, t) + filteredTags = append(filteredTags, t) } } - return kvs + return filteredTags } From ed29bb6c12bbb474c1f1077673334d8ee2822b48 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 13:12:09 -0500 Subject: [PATCH 14/19] Reverted submodule Signed-off-by: Joe Elliott --- jaeger-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jaeger-ui b/jaeger-ui index 6f78c8d7071..7580270c449 160000 --- a/jaeger-ui +++ b/jaeger-ui @@ -1 +1 @@ -Subproject commit 6f78c8d70714887ef4ba93c22219a3f783c12b63 +Subproject commit 7580270c449cc5092738de948ce7b928bbc37449 From 0d3ba343bf2a89c55a5f1bc61ab9897997e8a26e Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 13:17:17 -0500 Subject: [PATCH 15/19] Created convenience factory methods for white/blacklist Signed-off-by: Joe Elliott --- plugin/storage/cassandra/factory.go | 4 ++-- .../spanstore/dbmodel/tag_filter_exact_match.go | 10 ++++++++++ .../spanstore/dbmodel/tag_filter_exact_match_test.go | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/plugin/storage/cassandra/factory.go b/plugin/storage/cassandra/factory.go index 0f279ca95b8..0c9d5c0071b 100644 --- a/plugin/storage/cassandra/factory.go +++ b/plugin/storage/cassandra/factory.go @@ -147,9 +147,9 @@ func writerOptions(opts *Options) ([]cSpanStore.Option, error) { var options []cSpanStore.Option if len(tagIndexBlacklist) > 0 { - options = append(options, cSpanStore.TagFilter(dbmodel.NewExactMatchTagFilter(tagIndexBlacklist, true))) + options = append(options, cSpanStore.TagFilter(dbmodel.NewBlacklistFilter(tagIndexBlacklist))) } else if len(tagIndexWhitelist) > 0 { - options = append(options, cSpanStore.TagFilter(dbmodel.NewExactMatchTagFilter(tagIndexWhitelist, false))) + options = append(options, cSpanStore.TagFilter(dbmodel.NewWhitelistFilter(tagIndexWhitelist))) } return options, nil } diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go index 1e78a843997..b747ff38704 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go @@ -38,6 +38,16 @@ func NewExactMatchTagFilter(tags []string, dropMatches bool) ExactMatchTagFilter } } +// NewBlacklistFilter is a convenience method for creating a blacklist ExactMatchTagFilter +func NewBlacklistFilter(tags []string) ExactMatchTagFilter { + return NewExactMatchTagFilter(tags, true) +} + +// NewWhitelistFilter is a convenience method for creating a whitelist ExactMatchTagFilter +func NewWhitelistFilter(tags []string) ExactMatchTagFilter { + return NewExactMatchTagFilter(tags, false) +} + // FilterProcessTags implements TagFilter func (tf ExactMatchTagFilter) FilterProcessTags(span *model.Span, processTags model.KeyValues) model.KeyValues { return tf.filter(processTags) diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go index 61e8351d212..89a35d714ba 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go @@ -52,7 +52,7 @@ func TestBlacklistFilter(t *testing.T) { } expectedKVs.Sort() - tf := NewExactMatchTagFilter(filter, true) + tf := NewBlacklistFilter(filter) actualKVs := tf.filter(inputKVs) actualKVs.Sort() assert.Equal(t, actualKVs, expectedKVs) @@ -101,7 +101,7 @@ func TestWhitelistFilter(t *testing.T) { } expectedKVs.Sort() - tf := NewExactMatchTagFilter(filter, false) + tf := NewWhitelistFilter(filter) actualKVs := tf.filter(inputKVs) actualKVs.Sort() assert.Equal(t, actualKVs, expectedKVs) From 8f042cc3465e757cd07f29ebe642c11c5d5f408f Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 13:19:52 -0500 Subject: [PATCH 16/19] Hid original factory method Signed-off-by: Joe Elliott --- .../cassandra/spanstore/dbmodel/tag_filter_exact_match.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go index b747ff38704..51917a9bff4 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match.go @@ -24,10 +24,10 @@ type ExactMatchTagFilter struct { dropMatches bool } -// NewExactMatchTagFilter creates a ExactMatchTagFilter with the provided tags. Passing +// newExactMatchTagFilter creates a ExactMatchTagFilter with the provided tags. Passing // dropMatches true will exhibit blacklist behavior. Passing dropMatches false // will exhibit whitelist behavior. -func NewExactMatchTagFilter(tags []string, dropMatches bool) ExactMatchTagFilter { +func newExactMatchTagFilter(tags []string, dropMatches bool) ExactMatchTagFilter { mapTags := make(map[string]struct{}) for _, t := range tags { mapTags[t] = struct{}{} @@ -40,12 +40,12 @@ func NewExactMatchTagFilter(tags []string, dropMatches bool) ExactMatchTagFilter // NewBlacklistFilter is a convenience method for creating a blacklist ExactMatchTagFilter func NewBlacklistFilter(tags []string) ExactMatchTagFilter { - return NewExactMatchTagFilter(tags, true) + return newExactMatchTagFilter(tags, true) } // NewWhitelistFilter is a convenience method for creating a whitelist ExactMatchTagFilter func NewWhitelistFilter(tags []string) ExactMatchTagFilter { - return NewExactMatchTagFilter(tags, false) + return newExactMatchTagFilter(tags, false) } // FilterProcessTags implements TagFilter From b80d70c14260048493a86e2b9b53c138a07c1428 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 13:25:13 -0500 Subject: [PATCH 17/19] Clarified tests Signed-off-by: Joe Elliott --- .../dbmodel/tag_filter_exact_match_test.go | 58 +++++++++---------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go index 89a35d714ba..c90d38f56c1 100644 --- a/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go +++ b/plugin/storage/cassandra/spanstore/dbmodel/tag_filter_exact_match_test.go @@ -23,36 +23,35 @@ import ( ) func TestBlacklistFilter(t *testing.T) { - // expected - tt := [][][]string{ + tt := []struct { + input []string + filter []string + expected []string + }{ { - {"a", "b", "c"}, // input - {"a"}, // filter - {"b", "c"}, // expected + input: []string{"a", "b", "c"}, + filter: []string{"a"}, + expected: []string{"b", "c"}, }, { - {"a", "b", "c"}, - {"A"}, - {"a", "b", "c"}, + input: []string{"a", "b", "c"}, + filter: []string{"A"}, + expected: []string{"a", "b", "c"}, }, } for _, test := range tt { - input := test[0] - filter := test[1] - expected := test[2] - var inputKVs model.KeyValues - for _, i := range input { + for _, i := range test.input { inputKVs = append(inputKVs, model.String(i, "")) } var expectedKVs model.KeyValues - for _, e := range expected { + for _, e := range test.expected { expectedKVs = append(expectedKVs, model.String(e, "")) } expectedKVs.Sort() - tf := NewBlacklistFilter(filter) + tf := NewBlacklistFilter(test.filter) actualKVs := tf.filter(inputKVs) actualKVs.Sort() assert.Equal(t, actualKVs, expectedKVs) @@ -72,36 +71,35 @@ func TestBlacklistFilter(t *testing.T) { } func TestWhitelistFilter(t *testing.T) { - // expected - tt := [][][]string{ + tt := []struct { + input []string + filter []string + expected []string + }{ { - {"a", "b", "c"}, // input - {"a"}, // filter - {"a"}, // expected + input: []string{"a", "b", "c"}, + filter: []string{"a"}, + expected: []string{"a"}, }, { - {"a", "b", "c"}, - {"A"}, - {}, + input: []string{"a", "b", "c"}, + filter: []string{"A"}, + expected: []string{}, }, } for _, test := range tt { - input := test[0] - filter := test[1] - expected := test[2] - var inputKVs model.KeyValues - for _, i := range input { + for _, i := range test.input { inputKVs = append(inputKVs, model.String(i, "")) } var expectedKVs model.KeyValues - for _, e := range expected { + for _, e := range test.expected { expectedKVs = append(expectedKVs, model.String(e, "")) } expectedKVs.Sort() - tf := NewWhitelistFilter(filter) + tf := NewWhitelistFilter(test.filter) actualKVs := tf.filter(inputKVs) actualKVs.Sort() assert.Equal(t, actualKVs, expectedKVs) From 2f6fed6173b1c8deecadcb94daa88014d6c3b659 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 14:03:22 -0500 Subject: [PATCH 18/19] Fixed submodule? Signed-off-by: Joe Elliott --- jaeger-ui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jaeger-ui b/jaeger-ui index 7580270c449..df4c89751f9 160000 --- a/jaeger-ui +++ b/jaeger-ui @@ -1 +1 @@ -Subproject commit 7580270c449cc5092738de948ce7b928bbc37449 +Subproject commit df4c89751f9a3ad27bf483ecdac0a37a1eaf314f From a0bfe43ca183296782fd988262a1a52ce2c0fca7 Mon Sep 17 00:00:00 2001 From: Joe Elliott Date: Fri, 8 Nov 2019 14:34:46 -0500 Subject: [PATCH 19/19] Removed blank line Signed-off-by: Joe Elliott --- plugin/storage/cassandra/options.go | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/storage/cassandra/options.go b/plugin/storage/cassandra/options.go index 49898948f1f..9207c774b40 100644 --- a/plugin/storage/cassandra/options.go +++ b/plugin/storage/cassandra/options.go @@ -237,7 +237,6 @@ func (opt *Options) InitFromViper(v *viper.Viper) { opt.SpanStoreWriteCacheTTL = v.GetDuration(opt.primary.namespace + suffixSpanStoreWriteCacheTTL) opt.tagIndexBlacklist = stripWhiteSpace(v.GetString(opt.primary.namespace + suffixTagIndexBlacklist)) opt.tagIndexWhitelist = stripWhiteSpace(v.GetString(opt.primary.namespace + suffixTagIndexWhitelist)) - } func (cfg *namespaceConfig) initFromViper(v *viper.Viper) {