From e4cacd0a2f4b362b9a0a03efef4311c45aa5899a Mon Sep 17 00:00:00 2001 From: Santosh Kumar Bhiravabhatla <36130219+bhiravabhatla@users.noreply.github.com> Date: Sat, 27 Feb 2021 04:21:42 +0530 Subject: [PATCH] [es] Move all mapping related code to mappings package (#2822) * 2813 - Move all mapping related code to mappings package & implement @yurishkuro's feedback on refactoring Signed-off-by: santosh * 2813 - Fix name of the index templates Signed-off-by: santosh * 2813 - Rename ESPrefix to IndexPrefix for clarity & Implement @yurishkuro's feedback Signed-off-by: santosh * 2813 - Fix fmt issue Signed-off-by: santosh * 2813 - Fix issue with adding hyphen to index prefix Signed-off-by: santosh * 2813 - Used golang embed package instead of esc to embed index templates & implement feedback on tests Signed-off-by: santosh --- Makefile | 8 +- cmd/esmapping-generator/app/flags.go | 30 +- cmd/esmapping-generator/app/flags_test.go | 10 +- .../app/renderer/render.go | 27 +- .../app/renderer/render_test.go | 10 +- plugin/storage/es/dependencystore/storage.go | 3 +- plugin/storage/es/esRollover.py | 4 +- plugin/storage/es/factory.go | 76 +--- plugin/storage/es/factory_test.go | 251 ------------- .../fixtures/jaeger-dependencies-7.json | 10 + .../fixtures/jaeger-dependencies.json | 10 + .../mappings/fixtures/jaeger-service-7.json | 48 +++ .../es/mappings/fixtures/jaeger-service.json | 49 +++ .../es/mappings/fixtures/jaeger-span-7.json | 164 +++++++++ .../es/mappings/fixtures/jaeger-span.json | 165 +++++++++ plugin/storage/es/mappings/gen_assets.go | 341 ------------------ .../es/mappings/jaeger-dependencies-7.json | 4 +- .../es/mappings/jaeger-dependencies.json | 4 +- .../storage/es/mappings/jaeger-service-7.json | 12 +- .../storage/es/mappings/jaeger-service.json | 4 +- plugin/storage/es/mappings/jaeger-span-7.json | 14 +- plugin/storage/es/mappings/jaeger-span.json | 4 +- plugin/storage/es/mappings/mapping.go | 86 +++++ plugin/storage/es/mappings/mapping_test.go | 314 ++++++++++++++++ plugin/storage/es/spanstore/writer_test.go | 2 +- .../storage/integration/elasticsearch_test.go | 14 +- 26 files changed, 937 insertions(+), 727 deletions(-) create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-dependencies-7.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-dependencies.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-service-7.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-service.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-span-7.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-span.json delete mode 100644 plugin/storage/es/mappings/gen_assets.go create mode 100644 plugin/storage/es/mappings/mapping.go create mode 100644 plugin/storage/es/mappings/mapping_test.go diff --git a/Makefile b/Makefile index d4190ff749f..34376633c82 100644 --- a/Makefile +++ b/Makefile @@ -193,10 +193,6 @@ go-lint: @$(GOLINT) $(ALL_PKGS) | grep -v _nolint.go >> $(LINT_LOG) || true; @[ ! -s "$(LINT_LOG)" ] || (echo "Lint Failures" | cat - $(LINT_LOG) && false) -.PHONY: elasticsearch-mappings -elasticsearch-mappings: - esc -pkg mappings -o plugin/storage/es/mappings/gen_assets.go -ignore assets -prefix plugin/storage/es/mappings plugin/storage/es/mappings - .PHONY: build-examples build-examples: $(GOBUILD) -o ./examples/hotrod/hotrod-$(GOOS)-$(GOARCH) ./examples/hotrod/main.go @@ -244,7 +240,7 @@ build-all-in-one-debug build-agent-debug build-query-debug build-collector-debug build-all-in-one-debug build-agent-debug build-query-debug build-collector-debug build-ingester-debug: SUFFIX = -debug .PHONY: build-all-in-one build-all-in-one-debug -build-all-in-one build-all-in-one-debug: build-ui elasticsearch-mappings +build-all-in-one build-all-in-one-debug: build-ui $(GOBUILD) $(DISABLE_OPTIMIZATIONS) -tags ui -o ./cmd/all-in-one/all-in-one$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/all-in-one/main.go .PHONY: build-agent build-agent-debug @@ -256,7 +252,7 @@ build-query build-query-debug: build-ui $(GOBUILD) $(DISABLE_OPTIMIZATIONS) -tags ui -o ./cmd/query/query$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/query/main.go .PHONY: build-collector build-collector-debug -build-collector build-collector-debug: elasticsearch-mappings +build-collector build-collector-debug: $(GOBUILD) $(DISABLE_OPTIMIZATIONS) -o ./cmd/collector/collector$(SUFFIX)-$(GOOS)-$(GOARCH) $(BUILD_INFO) ./cmd/collector/main.go .PHONY: build-ingester build-ingester-debug diff --git a/cmd/esmapping-generator/app/flags.go b/cmd/esmapping-generator/app/flags.go index b5d4c18d43b..6d2590fd5bf 100644 --- a/cmd/esmapping-generator/app/flags.go +++ b/cmd/esmapping-generator/app/flags.go @@ -20,21 +20,21 @@ import ( // Options represent configurable parameters for jaeger-esmapping-generator type Options struct { - Mapping string - EsVersion int64 - Shards int64 - Replicas int64 - EsPrefix string - UseILM string // using string as util is being used in python and using bool leads to type issues. + Mapping string + EsVersion uint + Shards int64 + Replicas int64 + IndexPrefix string + UseILM string // using string as util is being used in python and using bool leads to type issues. } const ( - mappingFlag = "mapping" - esVersionFlag = "es-version" - shardsFlag = "shards" - replicasFlag = "replicas" - esPrefixFlag = "es-prefix" - useILMFlag = "use-ilm" + mappingFlag = "mapping" + esVersionFlag = "es-version" + shardsFlag = "shards" + replicasFlag = "replicas" + indexPrefixFlag = "index-prefix" + useILMFlag = "use-ilm" ) // AddFlags adds flags for esmapping-generator main program @@ -44,7 +44,7 @@ func (o *Options) AddFlags(command *cobra.Command) { mappingFlag, "", "The index mapping the template will be applied to. Pass either jaeger-span or jaeger-service") - command.Flags().Int64Var( + command.Flags().UintVar( &o.EsVersion, esVersionFlag, 7, @@ -60,8 +60,8 @@ func (o *Options) AddFlags(command *cobra.Command) { 1, "The number of replicas per index in Elasticsearch") command.Flags().StringVar( - &o.EsPrefix, - esPrefixFlag, + &o.IndexPrefix, + indexPrefixFlag, "", "Specifies index prefix") command.Flags().StringVar( diff --git a/cmd/esmapping-generator/app/flags_test.go b/cmd/esmapping-generator/app/flags_test.go index 1e217479494..ca66a0ff649 100644 --- a/cmd/esmapping-generator/app/flags_test.go +++ b/cmd/esmapping-generator/app/flags_test.go @@ -28,10 +28,10 @@ func TestOptionsWithDefaultFlags(t *testing.T) { o.AddFlags(&c) assert.Equal(t, "", o.Mapping) - assert.Equal(t, int64(7), o.EsVersion) + assert.Equal(t, uint(7), o.EsVersion) assert.Equal(t, int64(5), o.Shards) assert.Equal(t, int64(1), o.Replicas) - assert.Equal(t, "", o.EsPrefix) + assert.Equal(t, "", o.IndexPrefix) assert.Equal(t, "false", o.UseILM) } @@ -45,14 +45,14 @@ func TestOptionsWithFlags(t *testing.T) { "--es-version=6", "--shards=5", "--replicas=1", - "--es-prefix=test", + "--index-prefix=test", "--use-ilm=true", }) require.NoError(t, err) assert.Equal(t, "jaeger-span", o.Mapping) - assert.Equal(t, int64(6), o.EsVersion) + assert.Equal(t, uint(6), o.EsVersion) assert.Equal(t, int64(5), o.Shards) assert.Equal(t, int64(1), o.Replicas) - assert.Equal(t, "test", o.EsPrefix) + assert.Equal(t, "test", o.IndexPrefix) assert.Equal(t, "true", o.UseILM) } diff --git a/cmd/esmapping-generator/app/renderer/render.go b/cmd/esmapping-generator/app/renderer/render.go index 7895637917a..2e96703f37b 100644 --- a/cmd/esmapping-generator/app/renderer/render.go +++ b/cmd/esmapping-generator/app/renderer/render.go @@ -18,8 +18,8 @@ import ( "strconv" "github.com/jaegertracing/jaeger/cmd/esmapping-generator/app" - estemplate "github.com/jaegertracing/jaeger/pkg/es" - "github.com/jaegertracing/jaeger/plugin/storage/es" + "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/plugin/storage/es/mappings" ) var supportedMappings = map[string]struct{}{ @@ -28,15 +28,22 @@ var supportedMappings = map[string]struct{}{ } // GetMappingAsString returns rendered index templates as string -func GetMappingAsString(builder estemplate.TemplateBuilder, opt *app.Options) (string, error) { - if opt.EsVersion == 7 { - enableILM, err := strconv.ParseBool(opt.UseILM) - if err != nil { - return "", err - } - return es.FixMapping(builder, es.LoadMapping("/"+opt.Mapping+"-7.json"), opt.Shards, opt.Replicas, opt.EsPrefix, enableILM) +func GetMappingAsString(builder es.TemplateBuilder, opt *app.Options) (string, error) { + + enableILM, err := strconv.ParseBool(opt.UseILM) + if err != nil { + return "", err + } + + mappingBuilder := mappings.MappingBuilder{ + TemplateBuilder: builder, + Shards: opt.Shards, + Replicas: opt.Replicas, + EsVersion: opt.EsVersion, + IndexPrefix: opt.IndexPrefix, + UseILM: enableILM, } - return es.FixMapping(builder, es.LoadMapping("/"+opt.Mapping+".json"), opt.Shards, opt.Replicas, "", false) + return mappingBuilder.GetMapping(opt.Mapping) } // IsValidOption checks if passed option is a valid index template. diff --git a/cmd/esmapping-generator/app/renderer/render_test.go b/cmd/esmapping-generator/app/renderer/render_test.go index f1b710f6252..e6130bd80e7 100644 --- a/cmd/esmapping-generator/app/renderer/render_test.go +++ b/cmd/esmapping-generator/app/renderer/render_test.go @@ -50,23 +50,23 @@ func Test_getMappingAsString(t *testing.T) { wantErr error }{ { - name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"}, + name: "ES version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true"}, want: "ES version 7", }, { - name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"}, + name: "ES version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false"}, want: "ES version 6", }, { - name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "false"}, + name: "Parse Error version 6", args: app.Options{Mapping: "jaeger-span", EsVersion: 6, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "false"}, wantErr: errors.New("parse error"), }, { - name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "true"}, + name: "Parse Error version 7", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "true"}, wantErr: errors.New("parse error"), }, { - name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, EsPrefix: "test", UseILM: "foo"}, + name: "Parse bool error", args: app.Options{Mapping: "jaeger-span", EsVersion: 7, Shards: 5, Replicas: 1, IndexPrefix: "test", UseILM: "foo"}, wantErr: errors.New("strconv.ParseBool: parsing \"foo\": invalid syntax"), }, } diff --git a/plugin/storage/es/dependencystore/storage.go b/plugin/storage/es/dependencystore/storage.go index 64ea820c6fe..9a1f818de20 100644 --- a/plugin/storage/es/dependencystore/storage.go +++ b/plugin/storage/es/dependencystore/storage.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" "github.com/olivere/elastic" @@ -47,7 +48,7 @@ type DependencyStore struct { // NewDependencyStore returns a DependencyStore func NewDependencyStore(client es.Client, logger *zap.Logger, indexPrefix, indexDateLayout string, maxDocCount int) *DependencyStore { var prefix string - if indexPrefix != "" { + if indexPrefix != "" && !strings.HasSuffix(indexPrefix, "-") { prefix = indexPrefix + "-" } return &DependencyStore{ diff --git a/plugin/storage/es/esRollover.py b/plugin/storage/es/esRollover.py index bae5a893313..5697c2b7c8c 100755 --- a/plugin/storage/es/esRollover.py +++ b/plugin/storage/es/esRollover.py @@ -194,10 +194,10 @@ def str2bool(v): return v.lower() in ('true', '1') -def fix_mapping(template_name, esVersion, shards, replicas, esprefix, use_ilm): +def fix_mapping(template_name, esVersion, shards, replicas, indexPrefix, use_ilm): output = subprocess.Popen(['esmapping-generator', '--mapping', template_name, '--es-version', str(esVersion), '--shards', str(shards), '--replicas', - str(replicas), '--es-prefix', esprefix, '--use-ilm', str(use_ilm)], + str(replicas), '--index-prefix', indexPrefix, '--use-ilm', str(use_ilm)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) mapping, stderr = output.communicate() diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go index 205c9a216d7..4f4bf496d7c 100644 --- a/plugin/storage/es/factory.go +++ b/plugin/storage/es/factory.go @@ -16,7 +16,6 @@ package es import ( - "bytes" "flag" "fmt" "io" @@ -172,7 +171,16 @@ func createSpanWriter( return nil, err } - spanMapping, serviceMapping, err := GetSpanServiceMappings(es.TextTemplateBuilder{}, cfg.GetNumShards(), cfg.GetNumReplicas(), client.GetVersion(), cfg.GetIndexPrefix(), cfg.GetUseILM()) + mappingBuilder := mappings.MappingBuilder{ + TemplateBuilder: es.TextTemplateBuilder{}, + Shards: cfg.GetNumShards(), + Replicas: cfg.GetNumReplicas(), + EsVersion: cfg.GetVersion(), + IndexPrefix: cfg.GetIndexPrefix(), + UseILM: cfg.GetUseILM(), + } + + spanMapping, serviceMapping, err := mappingBuilder.GetSpanServiceMappings() if err != nil { return nil, err } @@ -197,70 +205,6 @@ func createSpanWriter( return writer, nil } -// GetSpanServiceMappings returns span and service mappings -func GetSpanServiceMappings(tb es.TemplateBuilder, shards, replicas int64, esVersion uint, esPrefix string, useILM bool) (string, string, error) { - if esVersion == 7 { - spanMapping, err := FixMapping(tb, LoadMapping("/jaeger-span-7.json"), shards, replicas, esPrefix, useILM) - if err != nil { - return "", "", err - } - serviceMapping, err := FixMapping(tb, LoadMapping("/jaeger-service-7.json"), shards, replicas, esPrefix, useILM) - if err != nil { - return "", "", err - } - return spanMapping, serviceMapping, nil - } - spanMapping, err := FixMapping(tb, LoadMapping("/jaeger-span.json"), shards, replicas, "", false) - if err != nil { - return "", "", err - } - serviceMapping, err := FixMapping(tb, LoadMapping("/jaeger-service.json"), shards, replicas, "", false) - if err != nil { - return "", "", err - } - return spanMapping, serviceMapping, nil -} - -// GetDependenciesMappings returns dependencies mappings -func GetDependenciesMappings(tb es.TemplateBuilder, shards, replicas int64, esVersion uint) (string, error) { - if esVersion == 7 { - return FixMapping(tb, LoadMapping("/jaeger-dependencies-7.json"), shards, replicas, "", false) - } - return FixMapping(tb, LoadMapping("/jaeger-dependencies.json"), shards, replicas, "", false) -} - -// LoadMapping returns index mappings from go assets as strings -func LoadMapping(name string) string { - s, _ := mappings.FSString(false, name) - return s -} - -// FixMapping parses the index mappings with given values and returns parsed template as string -func FixMapping(tb es.TemplateBuilder, mapping string, shards, replicas int64, esPrefix string, useILM bool) (string, error) { - - tmpl, err := tb.Parse(mapping) - if err != nil { - return "", err - } - writer := new(bytes.Buffer) - - if esPrefix != "" { - esPrefix += "-" - } - values := struct { - NumberOfShards int64 - NumberOfReplicas int64 - ESPrefix string - UseILM bool - }{shards, replicas, esPrefix, useILM} - - if err := tmpl.Execute(writer, values); err != nil { - return "", err - } - - return writer.String(), nil -} - var _ io.Closer = (*Factory)(nil) // Close closes the resources held by the factory diff --git a/plugin/storage/es/factory_test.go b/plugin/storage/es/factory_test.go index 885017e8966..1e31df71bb4 100644 --- a/plugin/storage/es/factory_test.go +++ b/plugin/storage/es/factory_test.go @@ -16,13 +16,9 @@ package es import ( - "bytes" "context" "errors" - "io/ioutil" - "os" "testing" - "text/template" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -178,49 +174,6 @@ func TestTagKeysAsFields(t *testing.T) { } } -func TestFactory_LoadMapping(t *testing.T) { - tests := []struct { - name string - esPrefix string - useILM bool - }{ - {name: "/jaeger-span.json"}, - {name: "/jaeger-service.json"}, - {name: "/jaeger-span.json"}, - {name: "/jaeger-service.json"}, - {name: "/jaeger-span-7.json", esPrefix: "test", useILM: true}, - {name: "/jaeger-service-7.json", esPrefix: "test", useILM: true}, - {name: "/jaeger-dependencies.json"}, - {name: "/jaeger-dependencies-7.json"}, - } - for _, test := range tests { - mapping := LoadMapping(test.name) - writer := new(bytes.Buffer) - f, err := os.Open("mappings/" + test.name) - require.NoError(t, err) - b, err := ioutil.ReadAll(f) - require.NoError(t, err) - assert.Equal(t, string(b), mapping) - tempMapping, err := template.New("mapping").Parse(mapping) - require.NoError(t, err) - esPrefixTemplateVal := test.esPrefix - if esPrefixTemplateVal != "" { - esPrefixTemplateVal += "-" - } - values := struct { - NumberOfShards int64 - NumberOfReplicas int64 - ESPrefix string - UseILM bool - }{10, 0, esPrefixTemplateVal, test.useILM} - err = tempMapping.Execute(writer, values) - require.NoError(t, err) - actualMapping, err := FixMapping(es.TextTemplateBuilder{}, mapping, 10, 0, test.esPrefix, test.useILM) - require.NoError(t, err) - assert.Equal(t, writer.String(), actualMapping) - } -} - func TestCreateTemplateError(t *testing.T) { f := NewFactory() f.primaryConfig = &mockClientBuilder{createTemplateError: errors.New("template-error"), Configuration: escfg.Configuration{Enabled: true, CreateIndexTemplates: true}} @@ -276,207 +229,3 @@ func TestNewOptions(t *testing.T) { assert.Equal(t, archiveCfg, o.others[archiveNamespace].Configuration) assert.Equal(t, archiveNamespace, o.others[archiveNamespace].namespace) } - -func TestFixMapping(t *testing.T) { - tests := []struct { - name string - templateBuilderMockFunc func() *mocks.TemplateBuilder - err string - }{ - { - name: "templateRenderSuccess", - templateBuilderMockFunc: func() *mocks.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(nil) - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "", - }, - { - name: "templateRenderFailure", - templateBuilderMockFunc: func() *mocks.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template exec error")) - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "template exec error", - }, - { - name: "templateLoadError", - templateBuilderMockFunc: func() *mocks.TemplateBuilder { - tb := mocks.TemplateBuilder{} - tb.On("Parse", mock.Anything).Return(nil, errors.New("template load error")) - return &tb - }, - err: "template load error", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := FixMapping(test.templateBuilderMockFunc(), "test", 3, 5, "test", true) - if test.err != "" { - assert.EqualError(t, err, test.err) - } else { - assert.NoError(t, err) - } - - }) - } -} - -func TestGetSpanServiceMappings(t *testing.T) { - type args struct { - shards int64 - replicas int64 - esVersion uint - esPrefix string - useILM bool - } - tests := []struct { - name string - args args - mockNewTextTemplateBuilder func() es.TemplateBuilder - err string - }{ - { - name: "ES Version 7", - args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - esPrefix: "test", - useILM: true, - }, - mockNewTextTemplateBuilder: func() es.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(nil) - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "", - }, - { - name: "ES Version 7 Service Error", - args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - esPrefix: "test", - useILM: true, - }, - mockNewTextTemplateBuilder: func() es.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "template load error", - }, - - { - name: "ES Version < 7", - args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - esPrefix: "test", - useILM: true, - }, - mockNewTextTemplateBuilder: func() es.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(nil) - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "", - }, - { - name: "ES Version < 7 Service Error", - args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - esPrefix: "test", - useILM: true, - }, - mockNewTextTemplateBuilder: func() es.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "template load error", - }, - { - name: "ES Version < 7 Span Error", - args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - esPrefix: "test", - useILM: true, - }, - mockNewTextTemplateBuilder: func() es.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "template load error", - }, - { - name: "ES Version 7 Span Error", - args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - esPrefix: "test", - useILM: true, - }, - mockNewTextTemplateBuilder: func() es.TemplateBuilder { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() - tb.On("Parse", mock.Anything).Return(&ta, nil) - return &tb - }, - err: "template load error", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, _, err := GetSpanServiceMappings(test.mockNewTextTemplateBuilder(), test.args.shards, test.args.replicas, - test.args.esVersion, test.args.esPrefix, - test.args.useILM) - if test.err != "" { - assert.EqualError(t, err, test.err) - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestGetDependenciesMappings(t *testing.T) { - tb := mocks.TemplateBuilder{} - ta := mocks.TemplateApplier{} - ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) - tb.On("Parse", mock.Anything).Return(&ta, nil) - - _, err := GetDependenciesMappings(&tb, 5, 5, 7) - assert.EqualError(t, err, "template load error") - _, err = GetDependenciesMappings(&tb, 5, 5, 6) - assert.EqualError(t, err, "template load error") -} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-dependencies-7.json b/plugin/storage/es/mappings/fixtures/jaeger-dependencies-7.json new file mode 100644 index 00000000000..6c4f46fe96d --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-dependencies-7.json @@ -0,0 +1,10 @@ +{ + "index_patterns": "*jaeger-dependencies-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + }, + "mappings":{} +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-dependencies.json b/plugin/storage/es/mappings/fixtures/jaeger-dependencies.json new file mode 100644 index 00000000000..42659ac9d03 --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-dependencies.json @@ -0,0 +1,10 @@ +{ + "template": "*jaeger-dependencies-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + }, + "mappings":{} +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-service-7.json b/plugin/storage/es/mappings/fixtures/jaeger-service-7.json new file mode 100644 index 00000000000..6f1ad6697b5 --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-service-7.json @@ -0,0 +1,48 @@ +{ + "index_patterns": "*test-jaeger-service-*", + "aliases": { + "test-jaeger-service-read" : {} + }, + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + ,"lifecycle": { + "name": "jaeger-ilm-policy", + "rollover_alias": "test-jaeger-service-write" + } + }, + "mappings":{ + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ], + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + } + } + } +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-service.json b/plugin/storage/es/mappings/fixtures/jaeger-service.json new file mode 100644 index 00000000000..9ba447775ed --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-service.json @@ -0,0 +1,49 @@ +{ + "template": "*jaeger-service-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true, + "index.mapper.dynamic":false + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":false + }, + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ] + }, + "service":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span-7.json b/plugin/storage/es/mappings/fixtures/jaeger-span-7.json new file mode 100644 index 00000000000..590cf6172af --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-span-7.json @@ -0,0 +1,164 @@ +{ + "index_patterns": "*test-jaeger-span-*", + "aliases": { + "test-jaeger-span-read": {} + }, + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + ,"lifecycle": { + "name": "jaeger-ilm-policy", + "rollover_alias": "test-jaeger-span-write" + } + }, + "mappings":{ + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ], + "properties":{ + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "parentSpanID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + }, + "startTime":{ + "type":"long" + }, + "startTimeMillis":{ + "type":"date", + "format":"epoch_millis" + }, + "duration":{ + "type":"long" + }, + "flags":{ + "type":"integer" + }, + "logs":{ + "type":"nested", + "dynamic":false, + "properties":{ + "timestamp":{ + "type":"long" + }, + "fields":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "process":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "references":{ + "type":"nested", + "dynamic":false, + "properties":{ + "refType":{ + "type":"keyword", + "ignore_above":256 + }, + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + } +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span.json b/plugin/storage/es/mappings/fixtures/jaeger-span.json new file mode 100644 index 00000000000..e7a89878009 --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-span.json @@ -0,0 +1,165 @@ +{ + "template": "*jaeger-span-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true, + "index.mapper.dynamic":false + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":false + }, + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ] + }, + "span":{ + "properties":{ + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "parentSpanID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + }, + "startTime":{ + "type":"long" + }, + "startTimeMillis":{ + "type":"date", + "format":"epoch_millis" + }, + "duration":{ + "type":"long" + }, + "flags":{ + "type":"integer" + }, + "logs":{ + "type":"nested", + "dynamic":false, + "properties":{ + "timestamp":{ + "type":"long" + }, + "fields":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "process":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "references":{ + "type":"nested", + "dynamic":false, + "properties":{ + "refType":{ + "type":"keyword", + "ignore_above":256 + }, + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + } + } +} diff --git a/plugin/storage/es/mappings/gen_assets.go b/plugin/storage/es/mappings/gen_assets.go deleted file mode 100644 index c3f97e3406b..00000000000 --- a/plugin/storage/es/mappings/gen_assets.go +++ /dev/null @@ -1,341 +0,0 @@ -// Code generated by "esc -pkg mappings -o plugin/storage/es/mappings/gen_assets.go -ignore assets -prefix plugin/storage/es/mappings plugin/storage/es/mappings"; DO NOT EDIT. - -package mappings - -import ( - "bytes" - "compress/gzip" - "encoding/base64" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "path" - "sync" - "time" -) - -type _escLocalFS struct{} - -var _escLocal _escLocalFS - -type _escStaticFS struct{} - -var _escStatic _escStaticFS - -type _escDirectory struct { - fs http.FileSystem - name string -} - -type _escFile struct { - compressed string - size int64 - modtime int64 - local string - isDir bool - - once sync.Once - data []byte - name string -} - -func (_escLocalFS) Open(name string) (http.File, error) { - f, present := _escData[path.Clean(name)] - if !present { - return nil, os.ErrNotExist - } - return os.Open(f.local) -} - -func (_escStaticFS) prepare(name string) (*_escFile, error) { - f, present := _escData[path.Clean(name)] - if !present { - return nil, os.ErrNotExist - } - var err error - f.once.Do(func() { - f.name = path.Base(name) - if f.size == 0 { - return - } - var gr *gzip.Reader - b64 := base64.NewDecoder(base64.StdEncoding, bytes.NewBufferString(f.compressed)) - gr, err = gzip.NewReader(b64) - if err != nil { - return - } - f.data, err = ioutil.ReadAll(gr) - }) - if err != nil { - return nil, err - } - return f, nil -} - -func (fs _escStaticFS) Open(name string) (http.File, error) { - f, err := fs.prepare(name) - if err != nil { - return nil, err - } - return f.File() -} - -func (dir _escDirectory) Open(name string) (http.File, error) { - return dir.fs.Open(dir.name + name) -} - -func (f *_escFile) File() (http.File, error) { - type httpFile struct { - *bytes.Reader - *_escFile - } - return &httpFile{ - Reader: bytes.NewReader(f.data), - _escFile: f, - }, nil -} - -func (f *_escFile) Close() error { - return nil -} - -func (f *_escFile) Readdir(count int) ([]os.FileInfo, error) { - if !f.isDir { - return nil, fmt.Errorf(" escFile.Readdir: '%s' is not directory", f.name) - } - - fis, ok := _escDirs[f.local] - if !ok { - return nil, fmt.Errorf(" escFile.Readdir: '%s' is directory, but we have no info about content of this dir, local=%s", f.name, f.local) - } - limit := count - if count <= 0 || limit > len(fis) { - limit = len(fis) - } - - if len(fis) == 0 && count > 0 { - return nil, io.EOF - } - - return fis[0:limit], nil -} - -func (f *_escFile) Stat() (os.FileInfo, error) { - return f, nil -} - -func (f *_escFile) Name() string { - return f.name -} - -func (f *_escFile) Size() int64 { - return f.size -} - -func (f *_escFile) Mode() os.FileMode { - return 0 -} - -func (f *_escFile) ModTime() time.Time { - return time.Unix(f.modtime, 0) -} - -func (f *_escFile) IsDir() bool { - return f.isDir -} - -func (f *_escFile) Sys() interface{} { - return f -} - -// FS returns a http.Filesystem for the embedded assets. If useLocal is true, -// the filesystem's contents are instead used. -func FS(useLocal bool) http.FileSystem { - if useLocal { - return _escLocal - } - return _escStatic -} - -// Dir returns a http.Filesystem for the embedded assets on a given prefix dir. -// If useLocal is true, the filesystem's contents are instead used. -func Dir(useLocal bool, name string) http.FileSystem { - if useLocal { - return _escDirectory{fs: _escLocal, name: name} - } - return _escDirectory{fs: _escStatic, name: name} -} - -// FSByte returns the named file from the embedded assets. If useLocal is -// true, the filesystem's contents are instead used. -func FSByte(useLocal bool, name string) ([]byte, error) { - if useLocal { - f, err := _escLocal.Open(name) - if err != nil { - return nil, err - } - b, err := ioutil.ReadAll(f) - _ = f.Close() - return b, err - } - f, err := _escStatic.prepare(name) - if err != nil { - return nil, err - } - return f.data, nil -} - -// FSMustByte is the same as FSByte, but panics if name is not present. -func FSMustByte(useLocal bool, name string) []byte { - b, err := FSByte(useLocal, name) - if err != nil { - panic(err) - } - return b -} - -// FSString is the string version of FSByte. -func FSString(useLocal bool, name string) (string, error) { - b, err := FSByte(useLocal, name) - return string(b), err -} - -// FSMustString is the string version of FSMustByte. -func FSMustString(useLocal bool, name string) string { - return string(FSMustByte(useLocal, name)) -} - -var _escData = map[string]*_escFile{ - - "/.nocover": { - name: ".nocover", - local: "plugin/storage/es/mappings/.nocover", - size: 43, - modtime: 1551330067, - compressed: ` -H4sIAAAAAAAC/youSSzJzFYoSEzOTkxPVcjILy4pVkgsLcnXTU/NSy1KLElNUUjLzEkt1uMCBAAA//8y -IKK1KwAAAA== -`, - }, - - "/jaeger-dependencies-7.json": { - name: "jaeger-dependencies-7.json", - local: "plugin/storage/es/mappings/jaeger-dependencies-7.json", - size: 279, - modtime: 1612587018, - compressed: ` -H4sIAAAAAAAC/2yPQWrDMBBF9z6FmGVIRDfd+BAttAcwivTtTJGnqmYMBaO7F5l00dDt478Hfx+cI5aE -76kEM1RRGh2dPgIW1EtCgSRIZOjlROe+VpixLEpjl391L9t6RZ0+50lvoaae2XfnXw78Or8f0LV2/l+q -KJljeNTe7vhRXEMpLIsXqCFNMyMn9ZlXNhqfn/5sK742qKmPId7gIeGaQaPVDYNzR5fuvX6qDW34CQAA -//8AfUo/FwEAAA== -`, - }, - - "/jaeger-dependencies.json": { - name: "jaeger-dependencies.json", - local: "plugin/storage/es/mappings/jaeger-dependencies.json", - size: 273, - modtime: 1612587018, - compressed: ` -H4sIAAAAAAAC/2yPQWrDMBBF9z7FMMuQiG660SFaaA9gFOvbmSJPVc0YCsZ3LzbZNGT7+O/BXzsidsy1 -JAdH4tNXwoR2yajQDB0Edjnxed8Z3EUn47hrRCya8Rt0ma9o/ffY2y21bBxpXSm8Hfh9/Dwgbdv5udRQ -iwzpUfu440dxTrWKTkFhjtyPgpItFJnFOb6+/Ns2/CwwtzCk4YYATdcCjt4WdERHl++9/dTWbd1fAAAA -//8zWep3EQEAAA== -`, - }, - - "/jaeger-service-7.json": { - name: "jaeger-service-7.json", - local: "plugin/storage/es/mappings/jaeger-service-7.json", - size: 1148, - modtime: 1612587018, - compressed: ` -H4sIAAAAAAAC/8ySQY/TMBCF7/kVozmuWgshLYfcOSDBglhxQsiaOpPU4NjGdrsbRf7vKMbbZgvdEwdO -kWbmvXzzPHMDgNp2/Cg9pcTBRmwBb+YZxNv7T4F7/Qg5fyceOGwjh6NWvL3BTQMwz1vQPYgvkd+9/wA5 -L15kNEVeTBZrAHzZKTB1CC3Mizg/ubLtql3klLQdIrbVrrAKexh3HKTrZdxT6MrvZhB3pfyxvy9FyMXw -L6LA3mhFVXbWfa71S+VI3ms7CMsxcSd7zaaLwuhRJ2xvXz2bDfzzwDFFoUjtWbClnWFsUzhw2e6PyAA2 -aHTPalLLYM2tOFoalwrW0LQZt94ZrSbcnIeCM8YdOcgS/TL+cuQPQSfGos+XeZe1sa57Dr2bLI1aycSj -N5SW5/1aAVa00ZOViYYoR/JP2tqrls+rAJgmz9jiD54eXOhWa/1OdLAusKSdOzK2r2/frNp5PYue0l6O -lNQeW0w0iBs8tXNzoVgx++AUx/hfYFcWcQ2/fL/VY/PBeQ5JczzDYX3iu3I2qy2v4l5FPWHi8htK2tl/ -4Xq6utzk5lcAAAD//16uKiV8BAAA -`, - }, - - "/jaeger-service.json": { - name: "jaeger-service.json", - local: "plugin/storage/es/mappings/jaeger-service.json", - size: 1056, - modtime: 1612587018, - compressed: ` -H4sIAAAAAAAC/8xTy47bIBTd+yvQXUYJqiqlCz4ildplVaEbc23TAqaA00aR/73CQ8aPibKaxXjhx+E8 -OBhuFWOQyHqDiUAw2P1CaikcIoWLrumwg32mREpJuzaCyArGQDtF/7gb7JmC7BsZOwwqgmC3G+OnCf7a -fJ9ANo77x6JA3ugat7JvBd4KLXqvXcsdxURKNpqMitxoqxOI46cVN9CfgWKKvMa6I04Oz4ZApDDQG08K -XF0dWl2DaNBEqhibkqEkzrWlogYHk+QdyRgaM38yBi9hajbLV6nCGJQseV/2COLHq3i2yavu0cmEbZQW -/TJiGi2T2+L5h149gYDfdP3bBwX77bhuXR9I4rm/EIjPxy8rwrjmg8fUSYup7kBAwpbvYEEYqwe6VQsf -+ppi/GBFyqz4s0Ll7We1cINyMhYbwIfeU0ia4mobFOIJLa2rPav1pNKiDuRATLp37+deLZ/5PlZj9T8A -AP//IuFOPCAEAAA= -`, - }, - - "/jaeger-span-7.json": { - name: "jaeger-span-7.json", - local: "plugin/storage/es/mappings/jaeger-span-7.json", - size: 3680, - modtime: 1612587018, - compressed: ` -H4sIAAAAAAAC/+xXwW7bMAy95ysEHovEGAZ0B5+3Q4G1G9buNAwGY9OOWlnSJCZtYPjfBytO6sR2OmDu -MAw7BZb4nh4pPsauZkKA1Bk9JRaZyWkPsYCLqhLRh9vPjnL5JOr6Hqkgt/AW9eIC5jMhqmohZC6ir56u -Pl6Lum6IUEn01DA0vELAGRpHmDWBDbDeM5LOWipPzFIXHuKWKoiM9LpckktMnvgVuiwcVYnoJix/ym/D -oqgD4QDIkVUyxVPYl3b5FFiitVIXkSbPlCW5JJX5SMlSMsSXb45iHf1Yk2cfpZiuKCKNS0UQs1tTCKuq -Xr2EmIOSOaXbtAltiyYEaCybZ2jrJVW5sEbJdAvzfYgzSpkNuSTUvAk+U+tHJ5kgQOu9mEOpQ8rQpvpc -72yrsZRpwlRahdzc6rf28L3O5pos6oSx8EmJdo9t91rK41UhgLeWIIYH2j4alx1SandloY2jBJdmQxC/ -vXzX2a67sWCRV0mJnK4gBsYiuoDDdj07QXQ0W2dS8v6vkN1qicbkh9/vbaNZZyw5luSfxQE7TOnqfVfu -uNRRmQeJYNGR5luLekJSPy1dUwZkafRNMMpUIhkd38lhRmV0AeOQa6mU9EPADJm6OnLjSmSIgaxJV0m5 -A/aYs/Uuw1/XkissBhVIzc0o6COUGQbsxl1XdDsMIM5ReZofeanXkjsqWZJnLO2Yj7pJ9E2yG7Vj2J7A -syLPCg2bD7Ttr75k+Rdt38sqIDao1vTHTmMs7gLt65w3G3uqxyfxfuYddd5oH3lyG5nSqdEnnciMo9Pe -LO8pZTgH/d+l/2SXOsrJkU7p1Uako3wo6+nauv9mMCl//0/9d+nPXMeJR4f9eRT+avc24MPJajrouykH -3dQdN/LKGj4sZvXsZwAAAP//iNEyH2AOAAA= -`, - }, - - "/jaeger-span.json": { - name: "jaeger-span.json", - local: "plugin/storage/es/mappings/jaeger-span.json", - size: 3826, - modtime: 1612587018, - compressed: ` -H4sIAAAAAAAC/+xWTW/bPAy++1cIPBap8eIFuoPPu+ywDlh7GwaDsWlHrb4mMd2Cwv998EeX2JWdDYiH -AVsOSUzpIfmI5GM9J0IAk3YKmSATcPWAVJO/Dg7N9RVs2vVAzNLUAbJ2uxAgTUnfUrPXW/K5rfKwQ18G -yMTzs0hvO/OH6q4ziqbZxEGenJIFTmEfB/MUqNE5aerUUGAq80qSKkOqpJYM2c1/o72evuwpcEgLLHaU -ksGtIsjY7+mVT/JpeTCoZQFZhSpQIkQXGYaIR9p5SRXuFecvltaGSh0fhYA+WHl01n4GKkLAECt/OfMA -2acf4KOb9tQdmpyxDrlGdxqiWx2Sm9rbah4cQQaPdPhqfQmb6bqsjfWU49Y+EWT/37wZbWjG+8Eh73KN -XOwgA8Y6vYKTDU0SwY1YOG8LCuEPIzJklS4RGv59Tk68dUU5qb7z1pFnSWHUA+yxoHdvx5yW+CxwOeEB -Dj0ZvnNoVnAe1nHbHhCytOYWNV0+aUbP93LOs7KmhmXge6mUDHF42YriKKvKeo0MGZCzxS7XPTgaodz3 -vH81s0phPZOPNNyKcxyn7Bysl8wxkbHmbSYjG2nq3qHUFBi1mx/XMbHYJPbSPe8hku6ZlM+k3S0/0iFm -P68zP6E1EZYd6gnVnn57VMb6vnO+Ztxk6bk594p4EeBJvy50XiD/JAt6LSIrvCoYF15HdvtABcM5B//6 -+6/ub08VeTIFrS7Jnqr4aVx6KGJ3mhXixK4hlwhztmSvpn5u4ieg1SscnewLn/rMJF9eWNfp1cUbfP/b -fjdJk3wPAAD//5ROSoHyDgAA -`, - }, - - "/": { - name: "/", - local: `plugin/storage/es/mappings`, - isDir: true, - }, -} - -var _escDirs = map[string][]os.FileInfo{ - - "plugin/storage/es/mappings": { - _escData["/.nocover"], - _escData["/jaeger-dependencies-7.json"], - _escData["/jaeger-dependencies.json"], - _escData["/jaeger-service-7.json"], - _escData["/jaeger-service.json"], - _escData["/jaeger-span-7.json"], - _escData["/jaeger-span.json"], - }, -} diff --git a/plugin/storage/es/mappings/jaeger-dependencies-7.json b/plugin/storage/es/mappings/jaeger-dependencies-7.json index af6be27c337..770f94a680d 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-7.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-7.json @@ -1,8 +1,8 @@ { "index_patterns": "*jaeger-dependencies-*", "settings":{ - "index.number_of_shards": {{ .NumberOfShards }}, - "index.number_of_replicas": {{ .NumberOfReplicas }}, + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true }, diff --git a/plugin/storage/es/mappings/jaeger-dependencies.json b/plugin/storage/es/mappings/jaeger-dependencies.json index dfa5a49e959..cdfa62d9547 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies.json +++ b/plugin/storage/es/mappings/jaeger-dependencies.json @@ -1,8 +1,8 @@ { "template": "*jaeger-dependencies-*", "settings":{ - "index.number_of_shards": {{ .NumberOfShards }}, - "index.number_of_replicas": {{ .NumberOfReplicas }}, + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true }, diff --git a/plugin/storage/es/mappings/jaeger-service-7.json b/plugin/storage/es/mappings/jaeger-service-7.json index f37d3f85115..2dd1eb63333 100644 --- a/plugin/storage/es/mappings/jaeger-service-7.json +++ b/plugin/storage/es/mappings/jaeger-service-7.json @@ -1,19 +1,19 @@ { - "index_patterns": "*{{ .ESPrefix }}jaeger-service-*", + "index_patterns": "*{{ .IndexPrefix }}jaeger-service-*", {{- if .UseILM }} "aliases": { - "{{ .ESPrefix }}jaeger-service-read" : {} + "{{ .IndexPrefix }}jaeger-service-read" : {} }, {{- end }} "settings":{ - "index.number_of_shards": {{ .NumberOfShards }}, - "index.number_of_replicas": {{ .NumberOfReplicas }}, + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true - {{ if .UseILM }} + {{- if .UseILM }} ,"lifecycle": { "name": "jaeger-ilm-policy", - "rollover_alias": "{{ .ESPrefix }}jaeger-service-write" + "rollover_alias": "{{ .IndexPrefix }}jaeger-service-write" } {{- end }} }, diff --git a/plugin/storage/es/mappings/jaeger-service.json b/plugin/storage/es/mappings/jaeger-service.json index 750aa8236ea..1adb8e0bf38 100644 --- a/plugin/storage/es/mappings/jaeger-service.json +++ b/plugin/storage/es/mappings/jaeger-service.json @@ -1,8 +1,8 @@ { "template": "*jaeger-service-*", "settings":{ - "index.number_of_shards": {{ .NumberOfShards }}, - "index.number_of_replicas": {{ .NumberOfReplicas }}, + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json index 9a95e463f86..79a495b1c16 100644 --- a/plugin/storage/es/mappings/jaeger-span-7.json +++ b/plugin/storage/es/mappings/jaeger-span-7.json @@ -1,21 +1,21 @@ { - "index_patterns": "*{{ .ESPrefix }}jaeger-span-*", + "index_patterns": "*{{ .IndexPrefix }}jaeger-span-*", {{- if .UseILM }} "aliases": { - "{{ .ESPrefix }}jaeger-span-read": {} + "{{ .IndexPrefix }}jaeger-span-read": {} }, {{- end }} "settings":{ - "index.number_of_shards": {{ .NumberOfShards }}, - "index.number_of_replicas": {{ .NumberOfReplicas }}, + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true - {{ if .UseILM }} + {{- if .UseILM }} ,"lifecycle": { "name": "jaeger-ilm-policy", - "rollover_alias": "{{ .ESPrefix }}jaeger-span-write" + "rollover_alias": "{{ .IndexPrefix }}jaeger-span-write" } - {{ end }} + {{- end }} }, "mappings":{ "dynamic_templates":[ diff --git a/plugin/storage/es/mappings/jaeger-span.json b/plugin/storage/es/mappings/jaeger-span.json index b3c5da90a12..3c73f923c4c 100644 --- a/plugin/storage/es/mappings/jaeger-span.json +++ b/plugin/storage/es/mappings/jaeger-span.json @@ -1,8 +1,8 @@ { "template": "*jaeger-span-*", "settings":{ - "index.number_of_shards": {{ .NumberOfShards }}, - "index.number_of_replicas": {{ .NumberOfReplicas }}, + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go new file mode 100644 index 00000000000..3a059111034 --- /dev/null +++ b/plugin/storage/es/mappings/mapping.go @@ -0,0 +1,86 @@ +// Copyright (c) 2021 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mappings + +import ( + "bytes" + "embed" + "strings" + + "github.com/jaegertracing/jaeger/pkg/es" +) + +//go:embed *.json +// MAPPINGS contains embeded index templates. +var MAPPINGS embed.FS + +// MappingBuilder holds parameters required to render an elasticsearch index template +type MappingBuilder struct { + TemplateBuilder es.TemplateBuilder + Shards int64 + Replicas int64 + EsVersion uint + IndexPrefix string + UseILM bool +} + +// GetMapping returns the rendered mapping based on elasticsearch version +func (mb *MappingBuilder) GetMapping(mapping string) (string, error) { + if mb.EsVersion == 7 { + return mb.fixMapping(mapping + "-7.json") + } + return mb.fixMapping(mapping + ".json") +} + +// GetSpanServiceMappings returns span and service mappings +func (mb *MappingBuilder) GetSpanServiceMappings() (string, string, error) { + spanMapping, err := mb.GetMapping("jaeger-span") + if err != nil { + return "", "", err + } + serviceMapping, err := mb.GetMapping("jaeger-service") + if err != nil { + return "", "", err + } + return spanMapping, serviceMapping, nil +} + +// GetDependenciesMappings returns dependencies mappings +func (mb *MappingBuilder) GetDependenciesMappings() (string, error) { + return mb.GetMapping("jaeger-dependencies") +} + +func loadMapping(name string) string { + s, _ := MAPPINGS.ReadFile(name) + return string(s) +} + +func (mb *MappingBuilder) fixMapping(mapping string) (string, error) { + + tmpl, err := mb.TemplateBuilder.Parse(loadMapping(mapping)) + if err != nil { + return "", err + } + writer := new(bytes.Buffer) + + if mb.IndexPrefix != "" && !strings.HasSuffix(mb.IndexPrefix, "-") { + mb.IndexPrefix += "-" + } + if err := tmpl.Execute(writer, mb); err != nil { + return "", err + } + + return writer.String(), nil +} diff --git a/plugin/storage/es/mappings/mapping_test.go b/plugin/storage/es/mappings/mapping_test.go new file mode 100644 index 00000000000..683f68f4d5b --- /dev/null +++ b/plugin/storage/es/mappings/mapping_test.go @@ -0,0 +1,314 @@ +// Copyright (c) 2021 The Jaeger Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mappings + +import ( + "embed" + "errors" + "io/ioutil" + "os" + "testing" + "text/template" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/pkg/es/mocks" +) + +//go:embed fixtures/*.json +var FIXTURES embed.FS + +func TestMappingBuilder_GetMapping(t *testing.T) { + tests := []struct { + mapping string + esVersion uint + }{ + {mapping: "jaeger-span", esVersion: 7}, + {mapping: "jaeger-span", esVersion: 6}, + {mapping: "jaeger-service", esVersion: 7}, + {mapping: "jaeger-service", esVersion: 6}, + {mapping: "jaeger-dependencies", esVersion: 7}, + {mapping: "jaeger-dependencies", esVersion: 6}, + } + for _, tt := range tests { + t.Run(tt.mapping, func(t *testing.T) { + mb := &MappingBuilder{ + TemplateBuilder: es.TextTemplateBuilder{}, + Shards: 3, + Replicas: 3, + EsVersion: tt.esVersion, + IndexPrefix: "test-", + UseILM: true, + } + got, err := mb.GetMapping(tt.mapping) + require.NoError(t, err) + var wantbytes []byte + if tt.esVersion == 7 { + wantbytes, err = FIXTURES.ReadFile("fixtures/" + tt.mapping + "-7.json") + require.NoError(t, err) + } else { + wantbytes, err = FIXTURES.ReadFile("fixtures/" + tt.mapping + ".json") + require.NoError(t, err) + } + want := string(wantbytes) + assert.Equal(t, got, want) + }) + } +} + +func TestMappingBuilder_loadMapping(t *testing.T) { + tests := []struct { + name string + }{ + {name: "jaeger-span.json"}, + {name: "jaeger-service.json"}, + {name: "jaeger-span-7.json"}, + {name: "jaeger-service-7.json"}, + {name: "jaeger-dependencies.json"}, + {name: "jaeger-dependencies-7.json"}, + } + for _, test := range tests { + mapping := loadMapping(test.name) + f, err := os.Open("./" + test.name) + require.NoError(t, err) + b, err := ioutil.ReadAll(f) + require.NoError(t, err) + assert.Equal(t, string(b), mapping) + _, err = template.New("mapping").Parse(mapping) + require.NoError(t, err) + } +} + +func TestMappingBuilder_fixMapping(t *testing.T) { + tests := []struct { + name string + templateBuilderMockFunc func() *mocks.TemplateBuilder + err string + }{ + { + name: "templateRenderSuccess", + templateBuilderMockFunc: func() *mocks.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "", + }, + { + name: "templateRenderFailure", + templateBuilderMockFunc: func() *mocks.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template exec error")) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template exec error", + }, + { + name: "templateLoadError", + templateBuilderMockFunc: func() *mocks.TemplateBuilder { + tb := mocks.TemplateBuilder{} + tb.On("Parse", mock.Anything).Return(nil, errors.New("template load error")) + return &tb + }, + err: "template load error", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mappingBuilder := MappingBuilder{ + TemplateBuilder: test.templateBuilderMockFunc(), + Shards: 3, + Replicas: 5, + EsVersion: 7, + IndexPrefix: "test", + UseILM: true, + } + _, err := mappingBuilder.fixMapping("test") + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } + + }) + } +} + +func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { + type args struct { + shards int64 + replicas int64 + esVersion uint + indexPrefix string + useILM bool + } + tests := []struct { + name string + args args + mockNewTextTemplateBuilder func() es.TemplateBuilder + err string + }{ + { + name: "ES Version 7", + args: args{ + shards: 3, + replicas: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "", + }, + { + name: "ES Version 7 Service Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + + { + name: "ES Version < 7", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "", + }, + { + name: "ES Version < 7 Service Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + { + name: "ES Version < 7 Span Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + { + name: "ES Version 7 Span Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mappingBuilder := MappingBuilder{ + TemplateBuilder: test.mockNewTextTemplateBuilder(), + Shards: test.args.shards, + Replicas: test.args.replicas, + EsVersion: test.args.esVersion, + IndexPrefix: test.args.indexPrefix, + UseILM: test.args.useILM, + } + _, _, err := mappingBuilder.GetSpanServiceMappings() + if test.err != "" { + assert.EqualError(t, err, test.err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestMappingBuilder_GetDependenciesMappings(t *testing.T) { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) + tb.On("Parse", mock.Anything).Return(&ta, nil) + + mappingBuilder := MappingBuilder{ + TemplateBuilder: &tb, + } + _, err := mappingBuilder.GetDependenciesMappings() + assert.EqualError(t, err, "template load error") +} diff --git a/plugin/storage/es/spanstore/writer_test.go b/plugin/storage/es/spanstore/writer_test.go index 6c40fbd8371..b2f208015b0 100644 --- a/plugin/storage/es/spanstore/writer_test.go +++ b/plugin/storage/es/spanstore/writer_test.go @@ -255,7 +255,7 @@ func TestCreateTemplates(t *testing.T) { for _, test := range tests { withSpanWriter(func(w *spanWriterTest) { prefix := "" - if test.indexPrefix != "" { + if test.indexPrefix != "" && !strings.HasSuffix(test.indexPrefix, "-") { prefix = test.indexPrefix + "-" } w.client.On("CreateTemplate", prefix+"jaeger-span").Return(test.spanTemplateService()) diff --git a/plugin/storage/integration/elasticsearch_test.go b/plugin/storage/integration/elasticsearch_test.go index 7eb372f98ce..f684f9d6bbe 100644 --- a/plugin/storage/integration/elasticsearch_test.go +++ b/plugin/storage/integration/elasticsearch_test.go @@ -34,8 +34,8 @@ import ( estemplate "github.com/jaegertracing/jaeger/pkg/es" eswrapper "github.com/jaegertracing/jaeger/pkg/es/wrapper" "github.com/jaegertracing/jaeger/pkg/testutils" - "github.com/jaegertracing/jaeger/plugin/storage/es" "github.com/jaegertracing/jaeger/plugin/storage/es/dependencystore" + "github.com/jaegertracing/jaeger/plugin/storage/es/mappings" "github.com/jaegertracing/jaeger/plugin/storage/es/spanstore" ) @@ -108,7 +108,15 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro return err } client := eswrapper.WrapESClient(s.client, bp, esVersion) - spanMapping, serviceMapping, err := es.GetSpanServiceMappings(estemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion(), indexPrefix, false) + mappingBuilder := mappings.MappingBuilder{ + TemplateBuilder: estemplate.TextTemplateBuilder{}, + Shards: 5, + Replicas: 1, + EsVersion: client.GetVersion(), + IndexPrefix: indexPrefix, + UseILM: false, + } + spanMapping, serviceMapping, err := mappingBuilder.GetSpanServiceMappings() if err != nil { return err } @@ -138,7 +146,7 @@ func (s *ESStorageIntegration) initSpanstore(allTagsAsFields, archive bool) erro MaxDocCount: defaultMaxDocCount, }) dependencyStore := dependencystore.NewDependencyStore(client, s.logger, indexPrefix, indexDateLayout, defaultMaxDocCount) - depMapping, err := es.GetDependenciesMappings(estemplate.TextTemplateBuilder{}, 5, 1, client.GetVersion()) + depMapping, err := mappingBuilder.GetDependenciesMappings() if err != nil { return err }