From db54d19ed93402df635b95dd146be84f14107802 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Wed, 3 Jul 2024 13:58:14 +0200 Subject: [PATCH] Bring back ES6 compatibility and e2e test Signed-off-by: Pavol Loffay --- .github/workflows/ci-elasticsearch.yml | 3 + .../elasticsearch/v6/docker-compose.yml | 14 ++ .../fixtures/jaeger-dependencies.json | 10 ++ .../es/mappings/fixtures/jaeger-sampling.json | 10 ++ .../es/mappings/fixtures/jaeger-service.json | 49 ++++++ .../es/mappings/fixtures/jaeger-span.json | 165 ++++++++++++++++++ .../es/mappings/jaeger-dependencies.json | 10 ++ .../storage/es/mappings/jaeger-sampling.json | 10 ++ .../storage/es/mappings/jaeger-service.json | 49 ++++++ plugin/storage/es/mappings/jaeger-span.json | 165 ++++++++++++++++++ plugin/storage/es/mappings/mapping.go | 4 +- plugin/storage/es/mappings/mapping_test.go | 70 +++++++- 12 files changed, 557 insertions(+), 2 deletions(-) create mode 100644 docker-compose/elasticsearch/v6/docker-compose.yml create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-dependencies.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-sampling.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-service.json create mode 100644 plugin/storage/es/mappings/fixtures/jaeger-span.json create mode 100644 plugin/storage/es/mappings/jaeger-dependencies.json create mode 100644 plugin/storage/es/mappings/jaeger-sampling.json create mode 100644 plugin/storage/es/mappings/jaeger-service.json create mode 100644 plugin/storage/es/mappings/jaeger-span.json diff --git a/.github/workflows/ci-elasticsearch.yml b/.github/workflows/ci-elasticsearch.yml index 108bea334db..111f37dc7cc 100644 --- a/.github/workflows/ci-elasticsearch.yml +++ b/.github/workflows/ci-elasticsearch.yml @@ -22,6 +22,9 @@ jobs: fail-fast: false matrix: version: + - major: 6.x + distribution: elasticsearch + jaeger: v1 - major: 7.x distribution: elasticsearch jaeger: v1 diff --git a/docker-compose/elasticsearch/v6/docker-compose.yml b/docker-compose/elasticsearch/v6/docker-compose.yml new file mode 100644 index 00000000000..ed8797d9b53 --- /dev/null +++ b/docker-compose/elasticsearch/v6/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.8' + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.23 + environment: + - discovery.type=single-node + - http.host=0.0.0.0 + - transport.host=127.0.0.1 + - xpack.security.enabled=false # Disable security features + - xpack.security.http.ssl.enabled=false # Disable HTTPS + - xpack.monitoring.enabled=false # Disable monitoring features + ports: + - "9200:9200" \ No newline at end of file diff --git a/plugin/storage/es/mappings/fixtures/jaeger-dependencies.json b/plugin/storage/es/mappings/fixtures/jaeger-dependencies.json new file mode 100644 index 00000000000..42659ac9d03 --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-dependencies.json @@ -0,0 +1,10 @@ +{ + "template": "*jaeger-dependencies-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + }, + "mappings":{} +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-sampling.json b/plugin/storage/es/mappings/fixtures/jaeger-sampling.json new file mode 100644 index 00000000000..87304011017 --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-sampling.json @@ -0,0 +1,10 @@ +{ + "template": "*jaeger-sampling-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + }, + "mappings":{} +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-service.json b/plugin/storage/es/mappings/fixtures/jaeger-service.json new file mode 100644 index 00000000000..9ba447775ed --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-service.json @@ -0,0 +1,49 @@ +{ + "template": "*jaeger-service-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true, + "index.mapper.dynamic":false + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":false + }, + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ] + }, + "service":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } +} diff --git a/plugin/storage/es/mappings/fixtures/jaeger-span.json b/plugin/storage/es/mappings/fixtures/jaeger-span.json new file mode 100644 index 00000000000..e7a89878009 --- /dev/null +++ b/plugin/storage/es/mappings/fixtures/jaeger-span.json @@ -0,0 +1,165 @@ +{ + "template": "*jaeger-span-*", + "settings":{ + "index.number_of_shards": 3, + "index.number_of_replicas": 3, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true, + "index.mapper.dynamic":false + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":false + }, + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ] + }, + "span":{ + "properties":{ + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "parentSpanID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + }, + "startTime":{ + "type":"long" + }, + "startTimeMillis":{ + "type":"date", + "format":"epoch_millis" + }, + "duration":{ + "type":"long" + }, + "flags":{ + "type":"integer" + }, + "logs":{ + "type":"nested", + "dynamic":false, + "properties":{ + "timestamp":{ + "type":"long" + }, + "fields":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "process":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "references":{ + "type":"nested", + "dynamic":false, + "properties":{ + "refType":{ + "type":"keyword", + "ignore_above":256 + }, + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + } + } +} diff --git a/plugin/storage/es/mappings/jaeger-dependencies.json b/plugin/storage/es/mappings/jaeger-dependencies.json new file mode 100644 index 00000000000..cdfa62d9547 --- /dev/null +++ b/plugin/storage/es/mappings/jaeger-dependencies.json @@ -0,0 +1,10 @@ +{ + "template": "*jaeger-dependencies-*", + "settings":{ + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true + }, + "mappings":{} +} diff --git a/plugin/storage/es/mappings/jaeger-sampling.json b/plugin/storage/es/mappings/jaeger-sampling.json new file mode 100644 index 00000000000..458d490a357 --- /dev/null +++ b/plugin/storage/es/mappings/jaeger-sampling.json @@ -0,0 +1,10 @@ +{ + "template": "*jaeger-sampling-*", + "settings":{ + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":false + }, + "mappings":{} +} diff --git a/plugin/storage/es/mappings/jaeger-service.json b/plugin/storage/es/mappings/jaeger-service.json new file mode 100644 index 00000000000..1adb8e0bf38 --- /dev/null +++ b/plugin/storage/es/mappings/jaeger-service.json @@ -0,0 +1,49 @@ +{ + "template": "*jaeger-service-*", + "settings":{ + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true, + "index.mapper.dynamic":false + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":false + }, + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ] + }, + "service":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } +} diff --git a/plugin/storage/es/mappings/jaeger-span.json b/plugin/storage/es/mappings/jaeger-span.json new file mode 100644 index 00000000000..3c73f923c4c --- /dev/null +++ b/plugin/storage/es/mappings/jaeger-span.json @@ -0,0 +1,165 @@ +{ + "template": "*jaeger-span-*", + "settings":{ + "index.number_of_shards": {{ .Shards }}, + "index.number_of_replicas": {{ .Replicas }}, + "index.mapping.nested_fields.limit":50, + "index.requests.cache.enable":true, + "index.mapper.dynamic":false + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":false + }, + "dynamic_templates":[ + { + "span_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"tag.*" + } + }, + { + "process_tags_map":{ + "mapping":{ + "type":"keyword", + "ignore_above":256 + }, + "path_match":"process.tag.*" + } + } + ] + }, + "span":{ + "properties":{ + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "parentSpanID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + }, + "operationName":{ + "type":"keyword", + "ignore_above":256 + }, + "startTime":{ + "type":"long" + }, + "startTimeMillis":{ + "type":"date", + "format":"epoch_millis" + }, + "duration":{ + "type":"long" + }, + "flags":{ + "type":"integer" + }, + "logs":{ + "type":"nested", + "dynamic":false, + "properties":{ + "timestamp":{ + "type":"long" + }, + "fields":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "process":{ + "properties":{ + "serviceName":{ + "type":"keyword", + "ignore_above":256 + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + }, + "references":{ + "type":"nested", + "dynamic":false, + "properties":{ + "refType":{ + "type":"keyword", + "ignore_above":256 + }, + "traceID":{ + "type":"keyword", + "ignore_above":256 + }, + "spanID":{ + "type":"keyword", + "ignore_above":256 + } + } + }, + "tag":{ + "type":"object" + }, + "tags":{ + "type":"nested", + "dynamic":false, + "properties":{ + "key":{ + "type":"keyword", + "ignore_above":256 + }, + "value":{ + "type":"keyword", + "ignore_above":256 + }, + "tagType":{ + "type":"keyword", + "ignore_above":256 + } + } + } + } + } + } +} diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go index deee2a1cba4..877087fea67 100644 --- a/plugin/storage/es/mappings/mapping.go +++ b/plugin/storage/es/mappings/mapping.go @@ -46,8 +46,10 @@ type MappingBuilder struct { func (mb *MappingBuilder) GetMapping(mapping string) (string, error) { if mb.EsVersion == 8 { return mb.fixMapping(mapping + "-8.json") + } else if mb.EsVersion == 7 { + return mb.fixMapping(mapping + "-7.json") } - return mb.fixMapping(mapping + "-7.json") + return mb.fixMapping(mapping + ".json") } // GetSpanServiceMappings returns span and service mappings diff --git a/plugin/storage/es/mappings/mapping_test.go b/plugin/storage/es/mappings/mapping_test.go index efa56c335bf..167c6893d1b 100644 --- a/plugin/storage/es/mappings/mapping_test.go +++ b/plugin/storage/es/mappings/mapping_test.go @@ -42,10 +42,13 @@ func TestMappingBuilder_GetMapping(t *testing.T) { }{ {mapping: "jaeger-span", esVersion: 8}, {mapping: "jaeger-span", esVersion: 7}, + {mapping: "jaeger-span", esVersion: 6}, {mapping: "jaeger-service", esVersion: 8}, {mapping: "jaeger-service", esVersion: 7}, + {mapping: "jaeger-service", esVersion: 6}, {mapping: "jaeger-dependencies", esVersion: 8}, {mapping: "jaeger-dependencies", esVersion: 7}, + {mapping: "jaeger-dependencies", esVersion: 6}, } for _, tt := range tests { t.Run(tt.mapping, func(t *testing.T) { @@ -64,7 +67,10 @@ func TestMappingBuilder_GetMapping(t *testing.T) { got, err := mb.GetMapping(tt.mapping) require.NoError(t, err) var wantbytes []byte - fileSuffix := fmt.Sprintf("-%d", tt.esVersion) + fileSuffix := "" + if tt.esVersion >= 7 { + fileSuffix = fmt.Sprintf("-%d", tt.esVersion) + } wantbytes, err = FIXTURES.ReadFile("fixtures/" + tt.mapping + fileSuffix + ".json") require.NoError(t, err) want := string(wantbytes) @@ -77,10 +83,13 @@ func TestMappingBuilder_loadMapping(t *testing.T) { tests := []struct { name string }{ + {name: "jaeger-span.json"}, {name: "jaeger-span-7.json"}, {name: "jaeger-span-8.json"}, + {name: "jaeger-service.json"}, {name: "jaeger-service-7.json"}, {name: "jaeger-service-8.json"}, + {name: "jaeger-dependencies.json"}, {name: "jaeger-dependencies-7.json"}, {name: "jaeger-dependencies-8.json"}, } @@ -210,6 +219,65 @@ func TestMappingBuilder_GetSpanServiceMappings(t *testing.T) { }, err: "template load error", }, + + { + name: "ES Version < 7", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "", + }, + { + name: "ES Version < 7 Service Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(nil).Once() + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")).Once() + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, + { + name: "ES Version < 7 Span Error", + args: args{ + shards: 3, + replicas: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", + }, + mockNewTextTemplateBuilder: func() es.TemplateBuilder { + tb := mocks.TemplateBuilder{} + ta := mocks.TemplateApplier{} + ta.On("Execute", mock.Anything, mock.Anything).Return(errors.New("template load error")) + tb.On("Parse", mock.Anything).Return(&ta, nil) + return &tb + }, + err: "template load error", + }, { name: "ES Version 7 Span Error", args: args{