From 6e519913bb0d89a83e195a4500270aff3f525182 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 15 Dec 2022 17:44:54 +0100 Subject: [PATCH 01/18] Add dynamic mappings and properties - pending to create if they do not exist --- internal/builder/_static/ecs_mappings.json | 886 +++++++++++++++++++++ internal/builder/dynamic_mappings.go | 193 +++++ internal/builder/packages.go | 7 + 3 files changed, 1086 insertions(+) create mode 100644 internal/builder/_static/ecs_mappings.json create mode 100644 internal/builder/dynamic_mappings.go diff --git a/internal/builder/_static/ecs_mappings.json b/internal/builder/_static/ecs_mappings.json new file mode 100644 index 000000000..1de642bd1 --- /dev/null +++ b/internal/builder/_static/ecs_mappings.json @@ -0,0 +1,886 @@ +{ + "mappings": { + "properties": { + "@timestamp": { + "type": "date", + "ignore_malformed": false + } + }, + "dynamic_templates": [ + { + "data_stream_to_constant": { + "path_match": "data_stream.*", + "mapping": { + "type": "constant_keyword" + } + } + }, + { + "resolved_ip_to_ip": { + "match": "resolved_ip", + "mapping": { + "type": "ip" + } + } + }, + { + "forwarded_ip_to_ip": { + "match_mapping_type": "string", + "match": "forwarded_ip", + "mapping": { + "type": "ip" + } + } + }, + { + "ip_to_ip": { + "match_mapping_type": "string", + "match": "ip", + "mapping": { + "type": "ip" + } + } + }, + { + "port_to_long": { + "match": "port", + "mapping": { + "type": "long" + } + } + }, + { + "thread_id_to_long": { + "path_match": "*.thread.id", + "mapping": { + "type": "long" + } + } + }, + { + "status_code_to_long": { + "match": "status_code", + "mapping": { + "type": "long" + } + } + }, + { + "line_to_long": { + "path_match": "*.file.line", + "mapping": { + "type": "long" + } + } + }, + { + "priority_to_long": { + "path_match": "log.syslog.priority", + "mapping": { + "type": "long" + } + } + }, + { + "code_to_long": { + "path_match": "*.facility.code", + "mapping": { + "type": "long" + } + } + }, + { + "code_to_long": { + "path_match": "*.severity.code", + "mapping": { + "type": "long" + } + } + }, + { + "bytes_to_long": { + "match": "bytes", + "path_unmatch": "*.data.bytes", + "mapping": { + "type": "long" + } + } + }, + { + "packets_to_long": { + "match": "packets", + "mapping": { + "type": "long" + } + } + }, + { + "public_key_exponent_to_long": { + "match": "public_key_exponent", + "mapping": { + "type": "long" + } + } + }, + { + "severity_to_long": { + "path_match": "event.severity", + "mapping": { + "type": "long" + } + } + }, + { + "duration_to_long": { + "path_match": "event.duration", + "mapping": { + "type": "long" + } + } + }, + { + "pid_to_long": { + "match": "pid", + "mapping": { + "type": "long" + } + } + }, + { + "uptime_to_long": { + "match": "uptime", + "mapping": { + "type": "long" + } + } + }, + { + "sequence_to_long": { + "match": "sequence", + "mapping": { + "type": "long" + } + } + }, + { + "entropy_to_long": { + "match": "*entropy", + "mapping": { + "type": "long" + } + } + }, + { + "size_to_long": { + "match": "*size", + "mapping": { + "type": "long" + } + } + }, + { + "entrypoint_to_long": { + "match": "entrypoint", + "mapping": { + "type": "long" + } + } + }, + { + "ttl_to_long": { + "match": "ttl", + "mapping": { + "type": "long" + } + } + }, + { + "major_to_long": { + "match": "major", + "mapping": { + "type": "long" + } + } + }, + { + "minor_to_long": { + "match": "minor", + "mapping": { + "type": "long" + } + } + }, + { + "as_number_to_long": { + "path_match": "*.as.number", + "mapping": { + "type": "long" + } + } + }, + { + "pgid_to_long": { + "match": "pgid", + "mapping": { + "type": "long" + } + } + }, + { + "exit_code_to_long": { + "match": "exit_code", + "mapping": { + "type": "long" + } + } + }, + { + "chi_to_long": { + "match": "chi2", + "mapping": { + "type": "long" + } + } + }, + { + "args_count_to_long": { + "match": "args_count", + "mapping": { + "type": "long" + } + } + }, + { + "virtual_address_to_long": { + "match": "virtual_address", + "mapping": { + "type": "long" + } + } + }, + { + "io_text_to_wildcard": { + "path_match": "*.io.text", + "mapping": { + "type": "wildcard" + } + } + }, + { + "strings_to_wildcard": { + "path_match": "registry.data.strings", + "mapping": { + "type": "wildcard" + } + } + }, + { + "path_to_wildcard": { + "path_match": "*url.path", + "mapping": { + "type": "wildcard" + } + } + }, + { + "message_id_to_wildcard": { + "match": "message_id", + "mapping": { + "type": "wildcard" + } + } + }, + { + "command_line_to_multifield": { + "match": "command_line", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "error_stack_trace_to_multifield": { + "match": "stack_trace", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "http_content_to_multifield": { + "path_match": "*.body.content", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "url_full_to_multifield": { + "path_match": "*.url.full", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "url_original_to_multifield": { + "path_match": "*.url.original", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "user_agent_original_to_multifield": { + "path_match": "user_agent.original", + "mapping": { + "type": "wildcard", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "error_message_to_match_only": { + "path_match": "error.message", + "mapping": { + "type": "match_only_text" + } + } + }, + { + "message_match_only_text": { + "path_match": "message", + "mapping": { + "type": "match_only_text" + } + } + }, + { + "agent_name_to_keyword": { + "path_match": "agent.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "service_name_to_keyword": { + "path_match": "*.service.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "sections_name_to_keyword": { + "path_match": "*.sections.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "resource_name_to_keyword": { + "path_match": "*.resource.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "observer_name_to_keyword": { + "path_match": "observer.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "question_name_to_keyword": { + "path_match": "*.question.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "group_name_to_keyword": { + "path_match": "*.group.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "geo_name_to_keyword": { + "path_match": "*.geo.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "host_name_to_keyword": { + "path_match": "host.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "severity_name_to_keyword": { + "path_match": "*.severity.name", + "mapping": { + "type": "keyword" + } + } + }, + { + "title_to_multifield": { + "match": "title", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "executable_to_multifield": { + "match": "executable", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "file_path_to_multifield": { + "path_match": "*.file.path", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "file_target_path_to_multifield": { + "path_match": "*.file.target_path", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "name_to_multifield": { + "match": "name", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "full_name_to_multifield": { + "match": "full_name", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "os_full_to_multifield": { + "path_match": "*.os.full", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "working_directory_to_multifield": { + "match": "working_directory", + "mapping": { + "type": "keyword", + "fields": { + "text": { + "type": "match_only_text" + } + } + } + } + }, + { + "timestamp_to_date": { + "match": "timestamp", + "mapping": { + "type": "date" + } + } + }, + { + "delivery_timestamp_to_date": { + "match": "delivery_timestamp", + "mapping": { + "type": "date" + } + } + }, + { + "not_after_to_date": { + "match": "not_after", + "mapping": { + "type": "date" + } + } + }, + { + "not_before_to_date": { + "match": "not_before", + "mapping": { + "type": "date" + } + } + }, + { + "accessed_to_date": { + "match": "accessed", + "mapping": { + "type": "date" + } + } + }, + { + "origination_timestamp_to_date": { + "match": "origination_timestamp", + "mapping": { + "type": "date" + } + } + }, + { + "created_to_date": { + "match": "created", + "mapping": { + "type": "date" + } + } + }, + { + "installed_to_date": { + "match": "installed", + "mapping": { + "type": "date" + } + } + }, + { + "creation_date_to_date": { + "match": "creation_date", + "mapping": { + "type": "date" + } + } + }, + { + "ctime_to_date": { + "match": "ctime", + "mapping": { + "type": "date" + } + } + }, + { + "mtime_to_date": { + "match": "mtime", + "mapping": { + "type": "date" + } + } + }, + { + "ingested_to_date": { + "match": "ingested", + "mapping": { + "type": "date" + } + } + }, + { + "start_to_date": { + "match": "start", + "mapping": { + "type": "date" + } + } + }, + { + "end_to_date": { + "match": "end", + "mapping": { + "type": "date" + } + } + }, + { + "score_base_to_float": { + "path_match": "*.score.base", + "mapping": { + "type": "float" + } + } + }, + { + "score_temporal_to_float": { + "path_match": "*.score.temporal", + "mapping": { + "type": "float" + } + } + }, + { + "score_to_float": { + "match": "*_score", + "mapping": { + "type": "float" + } + } + }, + { + "score_norm_to_float": { + "match": "*_score_norm", + "mapping": { + "type": "float" + } + } + }, + { + "usage_to_float": { + "match": "usage", + "mapping": { + "type": "scaled_float", + "scaling_factor": 1000 + } + } + }, + { + "location_to_geo_point": { + "match": "location", + "mapping": { + "type": "geo_point" + } + } + }, + { + "same_as_process_to_boolean": { + "match": "same_as_process", + "mapping": { + "type": "boolean" + } + } + }, + { + "established_to_boolean": { + "match": "established", + "mapping": { + "type": "boolean" + } + } + }, + { + "resumed_to_boolean": { + "match": "resumed", + "mapping": { + "type": "boolean" + } + } + }, + { + "max_bytes_per_process_exceeded_to_boolean": { + "match": "max_bytes_per_process_exceeded", + "mapping": { + "type": "boolean" + } + } + }, + { + "interactive_to_boolean": { + "match": "interactive", + "mapping": { + "type": "boolean" + } + } + }, + { + "exists_to_boolean": { + "match": "exists", + "mapping": { + "type": "boolean" + } + } + }, + { + "trusted_to_boolean": { + "match": "trusted", + "mapping": { + "type": "boolean" + } + } + }, + { + "valid_to_boolean": { + "match": "valid", + "mapping": { + "type": "boolean" + } + } + }, + { + "go_stripped_to_boolean": { + "match": "go_stripped", + "mapping": { + "type": "boolean" + } + } + }, + { + "coldstart_to_boolean": { + "match": "coldstart", + "mapping": { + "type": "boolean" + } + } + }, + { + "exports_to_flattened": { + "match": "exports", + "mapping": { + "type": "flattened" + } + } + }, + { + "structured_data_to_flattened": { + "match": "structured_data", + "mapping": { + "type": "flattened" + } + } + }, + { + "imports_to_flattened": { + "match": "*imports", + "mapping": { + "type": "flattened" + } + } + }, + { + "attachments_to_nested": { + "match": "attachments", + "mapping": { + "type": "nested" + } + } + }, + { + "segments_to_nested": { + "match": "segments", + "mapping": { + "type": "nested" + } + } + }, + { + "elf_sections_to_nested": { + "path_match": "*.elf.sections", + "mapping": { + "type": "nested" + } + } + }, + { + "pe_sections_to_nested": { + "path_match": "*.pe.sections", + "mapping": { + "type": "nested" + } + } + }, + { + "macho_sections_to_nested": { + "path_match": "*.macho.sections", + "mapping": { + "type": "nested" + } + } + }, + { + "trigger_to_nested": { + "match": "trigger", + "mapping": { + "type": "nested" + } + } + } + ] + } +} diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go new file mode 100644 index 000000000..b24e043dd --- /dev/null +++ b/internal/builder/dynamic_mappings.go @@ -0,0 +1,193 @@ +package builder + +import ( + _ "embed" + "os" + "path/filepath" + "strconv" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" + + "github.com/elastic/elastic-package/internal/formatter" + "github.com/elastic/elastic-package/internal/logger" + "github.com/elastic/elastic-package/internal/packages" +) + +//go:embed _static/ecs_mappings.json +var staticEcsMappings string + +func addDynamicMappings(destinationDir string) error { + // Get raw datastream setFieldmanifest + dataStreamManifests, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", packages.DataStreamManifestFile)) + if err != nil { + return err + } + + for _, datastream := range dataStreamManifests { + logger.Infof("Adding mappings to datastream %s", datastream) + addDynamicMappingElements(datastream) + } + + packageManifest := filepath.Join(destinationDir, packages.PackageManifestFile) + + m, err := packages.ReadPackageManifest(packageManifest) + if err != nil { + return err + } + if m.Type == "input" { + logger.Infof("Adding mappings to package manifest %s", packageManifest) + addDynamicMappingElements(packageManifest) + } + + return nil +} + +func addDynamicMappingElements(path string) ([]byte, error) { + type ecsTemplates struct { + Mappings struct { + Properties map[string]interface{} `yaml:"properties"` + DynamicTemplates []map[string]interface{} `yaml:"dynamic_templates"` + } `yaml:"mappings"` + } + + var ecsMappings ecsTemplates + err := yaml.Unmarshal([]byte(staticEcsMappings), &ecsMappings) + if err != nil { + return nil, err + } + + contents, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var doc yaml.Node + err = yaml.Unmarshal(contents, &doc) + if err != nil { + return nil, err + } + + logger.Infof("Number of dynamic templates to be added: %d", len(ecsMappings.Mappings.DynamicTemplates)) + var templates yaml.Node + err = templates.Encode(ecsMappings.Mappings.DynamicTemplates) + if err != nil { + return nil, err + } + + err = appendListElements(&doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) + if err != nil { + logger.Errorf("Error appending elems %s", err) + return nil, err + } + + var properties yaml.Node + err = properties.Encode(ecsMappings.Mappings.Properties) + if err != nil { + logger.Errorf("Error encoding properties %s", err) + return nil, err + } + logger.Infof("Number of properties to be added: %d", len(ecsMappings.Mappings.Properties)) + + err = appendMapElements(&doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) + if err != nil { + logger.Errorf("Error appending properties %s", err) + return nil, err + } + + contents, err = formatResult(&doc) + if err != nil { + logger.Errorf("Error formatting %s", err) + return nil, err + } + + logger.Infof("New Contents manifest:\n%s", contents) + return nil, nil + +} + +func appendListElements(root *yaml.Node, path []string, values *yaml.Node) error { + if len(path) == 0 { + root.Content = append(root.Content, values.Content...) + return nil + } + + key := path[0] + rest := path[1:] + + switch root.Kind { + case yaml.DocumentNode: + return appendListElements(root.Content[0], path, values) + case yaml.MappingNode: + for i := 0; i < len(root.Content); i += 2 { + child := root.Content[i] + if child.Value == key { + return appendListElements(root.Content[i+1], rest, values) + } + } + case yaml.SequenceNode: + index, err := strconv.Atoi(key) + if err != nil { + return err + } + return appendListElements(root.Content[index], rest, values) + } + return nil +} + +func appendMapElements(root *yaml.Node, path []string, values *yaml.Node) error { + if len(path) == 0 { + contents, _ := yaml.Marshal(root.Content) + logger.Infof("appendMapElements> Node kind: %s", root.Kind) + logger.Infof("appendMapElements> Values kind: %s", values.Kind) + if len(values.Content) > 0 { + logger.Infof("appendMapElements> First Values kind: %s", values.Content[0].Kind) + logger.Infof("appendMapElements> Second Values kind: %s", values.Content[1].Kind) + } + logger.Infof("appendMapElements> First root child kind: %s", root.Content[0].Kind) + logger.Infof("appendMapElements> Second root child kind: %s", root.Content[1].Kind) + logger.Infof("appendMapElements> Node to update:\n%s", string(contents)) + contents, _ = yaml.Marshal(values.Content) + logger.Infof("appendMapElements> Values to add :\n%s", string(contents)) + + root.Content = append(root.Content, values.Content...) + return nil + } + + key := path[0] + rest := path[1:] + + switch root.Kind { + case yaml.DocumentNode: + return appendMapElements(root.Content[0], path, values) + case yaml.MappingNode: + for i := 0; i < len(root.Content); i += 2 { + child := root.Content[i] + if child.Value == key { + return appendMapElements(root.Content[i+1], rest, values) + } + } + // TODO not found + // create + case yaml.SequenceNode: + index, err := strconv.Atoi(key) + if err != nil { + return err + } + return appendMapElements(root.Content[index], rest, values) + } + return nil +} + +func formatResult(result interface{}) ([]byte, error) { + d, err := yaml.Marshal(result) + if err != nil { + logger.Errorf("formatResult error > %s", err) + return nil, errors.New("failed to encode") + } + d, _, err = formatter.YAMLFormatter(d) + if err != nil { + return nil, errors.New("failed to format") + } + return d, nil +} diff --git a/internal/builder/packages.go b/internal/builder/packages.go index d4000873d..8db61df4c 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -180,6 +180,12 @@ func BuildPackage(options BuildOptions) (string, error) { return "", errors.Wrap(err, "resolving external fields failed") } + logger.Debug("Add dynamic mappings") + err = addDynamicMappings(destinationDir) + if err != nil { + return "", errors.Wrap(err, "adding dynamic mappings") + } + if options.CreateZip { return buildZippedPackage(options, destinationDir) } @@ -189,6 +195,7 @@ func BuildPackage(options BuildOptions) (string, error) { return destinationDir, nil } + logger.Infof("Validating package %s", destinationDir) err = validator.ValidateFromPath(destinationDir) if err != nil { return "", errors.Wrap(err, "invalid content found in built package") From 89e943c9993c6b79ef7c332eb34e459589d42762 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 15 Dec 2022 17:52:47 +0100 Subject: [PATCH 02/18] Write files in destinationDir --- internal/builder/dynamic_mappings.go | 78 ++++++++++------------------ 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index b24e043dd..08f94e771 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package builder import ( @@ -26,7 +30,14 @@ func addDynamicMappings(destinationDir string) error { for _, datastream := range dataStreamManifests { logger.Infof("Adding mappings to datastream %s", datastream) - addDynamicMappingElements(datastream) + contents, err := addDynamicMappingElements(datastream) + if err != nil { + return err + } + err = os.WriteFile(datastream, contents, 0664) + if err != nil { + return err + } } packageManifest := filepath.Join(destinationDir, packages.PackageManifestFile) @@ -37,7 +48,14 @@ func addDynamicMappings(destinationDir string) error { } if m.Type == "input" { logger.Infof("Adding mappings to package manifest %s", packageManifest) - addDynamicMappingElements(packageManifest) + contents, err := addDynamicMappingElements(packageManifest) + if err != nil { + return err + } + os.WriteFile(packageManifest, contents, 0664) + if err != nil { + return err + } } return nil @@ -75,7 +93,7 @@ func addDynamicMappingElements(path string) ([]byte, error) { return nil, err } - err = appendListElements(&doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) + err = appendElements(&doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) if err != nil { logger.Errorf("Error appending elems %s", err) return nil, err @@ -89,7 +107,7 @@ func addDynamicMappingElements(path string) ([]byte, error) { } logger.Infof("Number of properties to be added: %d", len(ecsMappings.Mappings.Properties)) - err = appendMapElements(&doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) + err = appendElements(&doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) if err != nil { logger.Errorf("Error appending properties %s", err) return nil, err @@ -102,54 +120,12 @@ func addDynamicMappingElements(path string) ([]byte, error) { } logger.Infof("New Contents manifest:\n%s", contents) - return nil, nil - -} - -func appendListElements(root *yaml.Node, path []string, values *yaml.Node) error { - if len(path) == 0 { - root.Content = append(root.Content, values.Content...) - return nil - } + return contents, nil - key := path[0] - rest := path[1:] - - switch root.Kind { - case yaml.DocumentNode: - return appendListElements(root.Content[0], path, values) - case yaml.MappingNode: - for i := 0; i < len(root.Content); i += 2 { - child := root.Content[i] - if child.Value == key { - return appendListElements(root.Content[i+1], rest, values) - } - } - case yaml.SequenceNode: - index, err := strconv.Atoi(key) - if err != nil { - return err - } - return appendListElements(root.Content[index], rest, values) - } - return nil } -func appendMapElements(root *yaml.Node, path []string, values *yaml.Node) error { +func appendElements(root *yaml.Node, path []string, values *yaml.Node) error { if len(path) == 0 { - contents, _ := yaml.Marshal(root.Content) - logger.Infof("appendMapElements> Node kind: %s", root.Kind) - logger.Infof("appendMapElements> Values kind: %s", values.Kind) - if len(values.Content) > 0 { - logger.Infof("appendMapElements> First Values kind: %s", values.Content[0].Kind) - logger.Infof("appendMapElements> Second Values kind: %s", values.Content[1].Kind) - } - logger.Infof("appendMapElements> First root child kind: %s", root.Content[0].Kind) - logger.Infof("appendMapElements> Second root child kind: %s", root.Content[1].Kind) - logger.Infof("appendMapElements> Node to update:\n%s", string(contents)) - contents, _ = yaml.Marshal(values.Content) - logger.Infof("appendMapElements> Values to add :\n%s", string(contents)) - root.Content = append(root.Content, values.Content...) return nil } @@ -159,12 +135,12 @@ func appendMapElements(root *yaml.Node, path []string, values *yaml.Node) error switch root.Kind { case yaml.DocumentNode: - return appendMapElements(root.Content[0], path, values) + return appendElements(root.Content[0], path, values) case yaml.MappingNode: for i := 0; i < len(root.Content); i += 2 { child := root.Content[i] if child.Value == key { - return appendMapElements(root.Content[i+1], rest, values) + return appendElements(root.Content[i+1], rest, values) } } // TODO not found @@ -174,7 +150,7 @@ func appendMapElements(root *yaml.Node, path []string, values *yaml.Node) error if err != nil { return err } - return appendMapElements(root.Content[index], rest, values) + return appendElements(root.Content[index], rest, values) } return nil } From 9b7624bf31a18c6606ba1b4dccf83ba5815dfa1d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 15 Dec 2022 18:57:20 +0100 Subject: [PATCH 03/18] Create nodes if they do not exist --- internal/builder/dynamic_mappings.go | 43 +++++++++++++++++----------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 08f94e771..95c9e5add 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -22,14 +22,12 @@ import ( var staticEcsMappings string func addDynamicMappings(destinationDir string) error { - // Get raw datastream setFieldmanifest dataStreamManifests, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", packages.DataStreamManifestFile)) if err != nil { return err } for _, datastream := range dataStreamManifests { - logger.Infof("Adding mappings to datastream %s", datastream) contents, err := addDynamicMappingElements(datastream) if err != nil { return err @@ -47,7 +45,6 @@ func addDynamicMappings(destinationDir string) error { return err } if m.Type == "input" { - logger.Infof("Adding mappings to package manifest %s", packageManifest) contents, err := addDynamicMappingElements(packageManifest) if err != nil { return err @@ -86,26 +83,22 @@ func addDynamicMappingElements(path string) ([]byte, error) { return nil, err } - logger.Infof("Number of dynamic templates to be added: %d", len(ecsMappings.Mappings.DynamicTemplates)) - var templates yaml.Node + var templates, properties yaml.Node err = templates.Encode(ecsMappings.Mappings.DynamicTemplates) if err != nil { return nil, err } - - err = appendElements(&doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) + err = properties.Encode(ecsMappings.Mappings.Properties) if err != nil { - logger.Errorf("Error appending elems %s", err) + logger.Errorf("Error encoding properties %s", err) return nil, err } - var properties yaml.Node - err = properties.Encode(ecsMappings.Mappings.Properties) + err = appendElements(&doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) if err != nil { - logger.Errorf("Error encoding properties %s", err) + logger.Errorf("Error appending elems %s", err) return nil, err } - logger.Infof("Number of properties to be added: %d", len(ecsMappings.Mappings.Properties)) err = appendElements(&doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) if err != nil { @@ -119,9 +112,8 @@ func addDynamicMappingElements(path string) ([]byte, error) { return nil, err } - logger.Infof("New Contents manifest:\n%s", contents) + logger.Debugf("New Contents manifest:\n%s", contents) return contents, nil - } func appendElements(root *yaml.Node, path []string, values *yaml.Node) error { @@ -143,22 +135,39 @@ func appendElements(root *yaml.Node, path []string, values *yaml.Node) error { return appendElements(root.Content[i+1], rest, values) } } - // TODO not found - // create + newContentNodes := newYamlNode(key) + + root.Content = append(root.Content, newContentNodes...) + return appendElements(newContentNodes[1], rest, values) case yaml.SequenceNode: index, err := strconv.Atoi(key) if err != nil { return err } + if len(root.Content) >= index { + return errors.Errorf("index out of range in nodes from key %s", key) + } + return appendElements(root.Content[index], rest, values) } return nil } +func newYamlNode(key string) []*yaml.Node { + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + var childNode *yaml.Node + switch key { + case "dynamic_templates": + childNode = &yaml.Node{Kind: yaml.SequenceNode, Value: key} + default: + childNode = &yaml.Node{Kind: yaml.MappingNode, Value: key} + } + return []*yaml.Node{keyNode, childNode} +} + func formatResult(result interface{}) ([]byte, error) { d, err := yaml.Marshal(result) if err != nil { - logger.Errorf("formatResult error > %s", err) return nil, errors.New("failed to encode") } d, _, err = formatter.YAMLFormatter(d) From f51aef78870241c74769345eb3eb6bd5611b103c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 16 Dec 2022 16:32:49 +0100 Subject: [PATCH 04/18] Test new version package-spec --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 093e18e19..ad2effce1 100644 --- a/go.mod +++ b/go.mod @@ -160,3 +160,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/elastic/package-spec/v2 => github.com/mrodm/package-spec/v2 v2.0.0-20221216145924-0dcd8377445f diff --git a/go.sum b/go.sum index 2a98307b4..37badabaa 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,6 @@ github.com/elastic/go-windows v1.0.0 h1:qLURgZFkkrYyTTkvYpsZIgf83AUsdIHfvlJaqaZ7 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc= github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= -github.com/elastic/package-spec/v2 v2.2.0 h1:hNTtw1PmM9BcehF5g7E4NNNIXNTxravcbrwATucYOAU= -github.com/elastic/package-spec/v2 v2.2.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -384,6 +382,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mrodm/package-spec/v2 v2.0.0-20221216145924-0dcd8377445f h1:U2ic94qu45zFRDv+iRd8JzNbi25dppuXyyjC0rTgeIs= +github.com/mrodm/package-spec/v2 v2.0.0-20221216145924-0dcd8377445f/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= From c663afe567a3c307bb01b79b118eecb79f6fb3c9 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 16 Dec 2022 16:35:41 +0100 Subject: [PATCH 05/18] Check wether or not import setting is true in build manifest --- internal/builder/dynamic_mappings.go | 16 +++++++++++++++- internal/builder/packages.go | 2 +- .../packages/buildmanifest/build_manifest.go | 11 +++++++++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 95c9e5add..c502e012b 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -16,12 +16,26 @@ import ( "github.com/elastic/elastic-package/internal/formatter" "github.com/elastic/elastic-package/internal/logger" "github.com/elastic/elastic-package/internal/packages" + "github.com/elastic/elastic-package/internal/packages/buildmanifest" ) //go:embed _static/ecs_mappings.json var staticEcsMappings string -func addDynamicMappings(destinationDir string) error { +func addDynamicMappings(packageRoot, destinationDir string) error { + bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot) + if err != nil { + return errors.Wrap(err, "can't read build manifest") + } + if !ok { + logger.Debugf("Build manifest hasn't been defined for the package") + return nil + } + if !bm.ImportCommonDynamicMappings() { + logger.Debugf("Package doesn't have to import common dynamic mappings") + return nil + } + dataStreamManifests, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", packages.DataStreamManifestFile)) if err != nil { return err diff --git a/internal/builder/packages.go b/internal/builder/packages.go index 8db61df4c..bb38b4e28 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -181,7 +181,7 @@ func BuildPackage(options BuildOptions) (string, error) { } logger.Debug("Add dynamic mappings") - err = addDynamicMappings(destinationDir) + err = addDynamicMappings(options.PackageRoot, destinationDir) if err != nil { return "", errors.Wrap(err, "adding dynamic mappings") } diff --git a/internal/packages/buildmanifest/build_manifest.go b/internal/packages/buildmanifest/build_manifest.go index 956bada43..d32a1c2ef 100644 --- a/internal/packages/buildmanifest/build_manifest.go +++ b/internal/packages/buildmanifest/build_manifest.go @@ -8,9 +8,10 @@ import ( "os" "path/filepath" + "github.com/pkg/errors" + "github.com/elastic/go-ucfg" "github.com/elastic/go-ucfg/yaml" - "github.com/pkg/errors" ) // BuildManifest defines the manifest defining the building procedure. @@ -25,7 +26,8 @@ type Dependencies struct { // ECSDependency defines a dependency on ECS fields. type ECSDependency struct { - Reference string `config:"reference"` + Reference string `config:"reference"` + ImportCommonDynamicMappings bool `config:"import_common_dynamic_mappings"` } // HasDependencies function checks if there are any dependencies defined. @@ -33,6 +35,11 @@ func (bm *BuildManifest) HasDependencies() bool { return bm.Dependencies.ECS.Reference != "" } +// HasDependencies function checks if there are any dependencies defined. +func (bm *BuildManifest) ImportCommonDynamicMappings() bool { + return bm.Dependencies.ECS.ImportCommonDynamicMappings +} + // ReadBuildManifest function reads the package build manifest. func ReadBuildManifest(packageRoot string) (*BuildManifest, bool, error) { path := buildManifestPath(packageRoot) From 3c78387774917c6b426606a605709f0284de398d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 19 Dec 2022 15:49:51 +0100 Subject: [PATCH 06/18] Added list of mappings added into _meta field --- internal/builder/dynamic_mappings.go | 102 +++++++++++++++++++++------ 1 file changed, 81 insertions(+), 21 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index c502e012b..be71ca299 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -22,6 +22,13 @@ import ( //go:embed _static/ecs_mappings.json var staticEcsMappings string +type ecsTemplates struct { + Mappings struct { + Properties map[string]interface{} `yaml:"properties"` + DynamicTemplates []map[string]interface{} `yaml:"dynamic_templates"` + } `yaml:"mappings"` +} + func addDynamicMappings(packageRoot, destinationDir string) error { bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot) if err != nil { @@ -73,13 +80,6 @@ func addDynamicMappings(packageRoot, destinationDir string) error { } func addDynamicMappingElements(path string) ([]byte, error) { - type ecsTemplates struct { - Mappings struct { - Properties map[string]interface{} `yaml:"properties"` - DynamicTemplates []map[string]interface{} `yaml:"dynamic_templates"` - } `yaml:"mappings"` - } - var ecsMappings ecsTemplates err := yaml.Unmarshal([]byte(staticEcsMappings), &ecsMappings) if err != nil { @@ -97,37 +97,97 @@ func addDynamicMappingElements(path string) ([]byte, error) { return nil, err } - var templates, properties yaml.Node - err = templates.Encode(ecsMappings.Mappings.DynamicTemplates) + err = addEcsMappings(&doc, ecsMappings) if err != nil { return nil, err } - err = properties.Encode(ecsMappings.Mappings.Properties) + + err = addEcsMappingsListMeta(&doc, ecsMappings) if err != nil { - logger.Errorf("Error encoding properties %s", err) return nil, err } - err = appendElements(&doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) + contents, err = formatResult(&doc) if err != nil { - logger.Errorf("Error appending elems %s", err) + logger.Errorf("Error formatting %s", err) return nil, err } - err = appendElements(&doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) + logger.Debugf("New Contents manifest:\n%s", contents) + return contents, nil +} + +func addEcsMappings(doc *yaml.Node, mappings ecsTemplates) error { + var templates, properties yaml.Node + err := templates.Encode(mappings.Mappings.DynamicTemplates) + if err != nil { + return err + } + err = properties.Encode(mappings.Mappings.Properties) + if err != nil { + logger.Errorf("Error encoding properties %s", err) + return err + } + + err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) + if err != nil { + logger.Errorf("Error appending elems %s", err) + return err + } + + err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) if err != nil { logger.Errorf("Error appending properties %s", err) - return nil, err + return err } - contents, err = formatResult(&doc) + return nil +} + +func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { + type ecsMappingsAdded struct { + Properties []string `yaml:"properties"` + DynamicTemplates []string `yaml:"dynamic_templates"` + } + + var mappingsAdded ecsMappingsAdded + for property := range mappings.Mappings.Properties { + mappingsAdded.Properties = append(mappingsAdded.Properties, property) + } + + for _, dynamicTemplate := range mappings.Mappings.DynamicTemplates { + if len(dynamicTemplate) > 1 { + return errors.New("more than one dynamic template present") + } + + for title := range dynamicTemplate { + mappingsAdded.DynamicTemplates = append(mappingsAdded.DynamicTemplates, title) + } + } + + var properties yaml.Node + err := properties.Encode(mappingsAdded.Properties) if err != nil { - logger.Errorf("Error formatting %s", err) - return nil, err + return err } - logger.Debugf("New Contents manifest:\n%s", contents) - return contents, nil + err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "_meta", "ecs_properties_added"}, &properties) + if err != nil { + return err + } + + var dynamicTemplates yaml.Node + err = dynamicTemplates.Encode(mappingsAdded.DynamicTemplates) + if err != nil { + return err + } + + err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "_meta", "ecs_dynamic_templates_added"}, &dynamicTemplates) + if err != nil { + return err + } + + return nil } func appendElements(root *yaml.Node, path []string, values *yaml.Node) error { @@ -171,7 +231,7 @@ func newYamlNode(key string) []*yaml.Node { keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} var childNode *yaml.Node switch key { - case "dynamic_templates": + case "dynamic_templates", "ecs_properties_added", "ecs_dynamic_templates_added": childNode = &yaml.Node{Kind: yaml.SequenceNode, Value: key} default: childNode = &yaml.Node{Kind: yaml.MappingNode, Value: key} From 001a312857b78893b913f30ae4297d2f7940f074 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 19 Dec 2022 20:56:38 +0100 Subject: [PATCH 07/18] Add tests for methods adding ecs mappings --- internal/builder/dynamic_mappings.go | 6 +- internal/builder/dynamic_mappings_test.go | 100 ++++++++++++++++++ internal/builder/testdata/ecs.template.yml | 9 ++ internal/builder/testdata/empty.yml | 1 + internal/builder/testdata/existing.yml | 19 ++++ .../testdata/expected.empty.mappings.yml | 12 +++ .../builder/testdata/expected.empty.meta.yml | 9 ++ .../testdata/expected.existing.mappings.yml | 24 +++++ .../testdata/expected.existing.meta.yml | 20 ++++ 9 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 internal/builder/dynamic_mappings_test.go create mode 100644 internal/builder/testdata/ecs.template.yml create mode 100644 internal/builder/testdata/empty.yml create mode 100644 internal/builder/testdata/existing.yml create mode 100644 internal/builder/testdata/expected.empty.mappings.yml create mode 100644 internal/builder/testdata/expected.empty.meta.yml create mode 100644 internal/builder/testdata/expected.existing.mappings.yml create mode 100644 internal/builder/testdata/expected.existing.meta.yml diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index be71ca299..5f438caaa 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -192,7 +192,11 @@ func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { func appendElements(root *yaml.Node, path []string, values *yaml.Node) error { if len(path) == 0 { - root.Content = append(root.Content, values.Content...) + contents := values.Content + if values.Kind == yaml.DocumentNode { + contents = values.Content[0].Content + } + root.Content = append(root.Content, contents...) return nil } diff --git a/internal/builder/dynamic_mappings_test.go b/internal/builder/dynamic_mappings_test.go new file mode 100644 index 000000000..21a8a9bb7 --- /dev/null +++ b/internal/builder/dynamic_mappings_test.go @@ -0,0 +1,100 @@ +package builder + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestAddMappingsListMeta(t *testing.T) { + cases := []struct { + title string + baseDocPath string + ecsToAddPath string + expectedPath string + }{ + { + title: "Add new mappings", + baseDocPath: "testdata/empty.yml", + ecsToAddPath: "testdata/ecs.template.yml", + expectedPath: "testdata/expected.empty.meta.yml", + }, + { + title: "Append mappings", + baseDocPath: "testdata/existing.yml", + ecsToAddPath: "testdata/ecs.template.yml", + expectedPath: "testdata/expected.existing.meta.yml", + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + contentsBase, _ := os.ReadFile(c.baseDocPath) + contentsToAdd, _ := os.ReadFile(c.ecsToAddPath) + contentsExpected, _ := os.ReadFile(c.expectedPath) + + var base yaml.Node + err := yaml.Unmarshal(contentsBase, &base) + require.NoError(t, err) + + var template ecsTemplates + err = yaml.Unmarshal(contentsToAdd, &template) + require.NoError(t, err) + + err = addEcsMappingsListMeta(&base, template) + require.NoError(t, err) + + newYaml, _ := formatResult(&base) + + assert.Equal(t, string(contentsExpected), string(newYaml)) + }) + } +} + +func TestAddEcsMappings(t *testing.T) { + cases := []struct { + title string + baseDocPath string + ecsToAddPath string + expectedPath string + }{ + { + title: "Add new mappings from scratch", + baseDocPath: "testdata/empty.yml", + ecsToAddPath: "testdata/ecs.template.yml", + expectedPath: "testdata/expected.empty.mappings.yml", + }, + { + title: "Append mappings", + baseDocPath: "testdata/existing.yml", + ecsToAddPath: "testdata/ecs.template.yml", + expectedPath: "testdata/expected.existing.mappings.yml", + }, + } + + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + contentsBase, _ := os.ReadFile(c.baseDocPath) + contentsToAdd, _ := os.ReadFile(c.ecsToAddPath) + contentsExpected, _ := os.ReadFile(c.expectedPath) + + var base yaml.Node + err := yaml.Unmarshal(contentsBase, &base) + require.NoError(t, err) + + var template ecsTemplates + err = yaml.Unmarshal(contentsToAdd, &template) + require.NoError(t, err) + + err = addEcsMappings(&base, template) + require.NoError(t, err) + + newYaml, _ := formatResult(&base) + + assert.Equal(t, string(contentsExpected), string(newYaml)) + }) + } +} diff --git a/internal/builder/testdata/ecs.template.yml b/internal/builder/testdata/ecs.template.yml new file mode 100644 index 000000000..b8a0363fe --- /dev/null +++ b/internal/builder/testdata/ecs.template.yml @@ -0,0 +1,9 @@ +mappings: + properties: + datetime: + type: long + dynamic_templates: + - example_template: + patch_match: 'example.*' + mapping: + type: short diff --git a/internal/builder/testdata/empty.yml b/internal/builder/testdata/empty.yml new file mode 100644 index 000000000..670dee7f9 --- /dev/null +++ b/internal/builder/testdata/empty.yml @@ -0,0 +1 @@ +title: empty diff --git a/internal/builder/testdata/existing.yml b/internal/builder/testdata/existing.yml new file mode 100644 index 000000000..b98ae93ee --- /dev/null +++ b/internal/builder/testdata/existing.yml @@ -0,0 +1,19 @@ +title: existing + +elasticsearch: + index_template: + mappings: + _meta: + managed: true + ecs_properties_added: + - example + ecs_dynamic_templates_added: + - to_nested + properties: + example: + type: short + dynamic_templates: + - to_nested: + path: trigger + mapping: + type: nested diff --git a/internal/builder/testdata/expected.empty.mappings.yml b/internal/builder/testdata/expected.empty.mappings.yml new file mode 100644 index 000000000..034265900 --- /dev/null +++ b/internal/builder/testdata/expected.empty.mappings.yml @@ -0,0 +1,12 @@ +title: empty +elasticsearch: + index_template: + mappings: + dynamic_templates: + - example_template: + mapping: + type: short + patch_match: example.* + properties: + datetime: + type: long diff --git a/internal/builder/testdata/expected.empty.meta.yml b/internal/builder/testdata/expected.empty.meta.yml new file mode 100644 index 000000000..93121d810 --- /dev/null +++ b/internal/builder/testdata/expected.empty.meta.yml @@ -0,0 +1,9 @@ +title: empty +elasticsearch: + index_template: + mappings: + _meta: + ecs_properties_added: + - datetime + ecs_dynamic_templates_added: + - example_template diff --git a/internal/builder/testdata/expected.existing.mappings.yml b/internal/builder/testdata/expected.existing.mappings.yml new file mode 100644 index 000000000..ecf716f9c --- /dev/null +++ b/internal/builder/testdata/expected.existing.mappings.yml @@ -0,0 +1,24 @@ +title: existing +elasticsearch: + index_template: + mappings: + _meta: + managed: true + ecs_properties_added: + - example + ecs_dynamic_templates_added: + - to_nested + properties: + example: + type: short + datetime: + type: long + dynamic_templates: + - to_nested: + path: trigger + mapping: + type: nested + - example_template: + mapping: + type: short + patch_match: example.* diff --git a/internal/builder/testdata/expected.existing.meta.yml b/internal/builder/testdata/expected.existing.meta.yml new file mode 100644 index 000000000..664f38a81 --- /dev/null +++ b/internal/builder/testdata/expected.existing.meta.yml @@ -0,0 +1,20 @@ +title: existing +elasticsearch: + index_template: + mappings: + _meta: + managed: true + ecs_properties_added: + - example + - datetime + ecs_dynamic_templates_added: + - to_nested + - example_template + properties: + example: + type: short + dynamic_templates: + - to_nested: + path: trigger + mapping: + type: nested From ddea43837ad644294bbd2965cf2a9d0e34e15dfb Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Dec 2022 10:45:54 +0100 Subject: [PATCH 08/18] Add license header --- internal/builder/dynamic_mappings_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/builder/dynamic_mappings_test.go b/internal/builder/dynamic_mappings_test.go index 21a8a9bb7..a0fc40e34 100644 --- a/internal/builder/dynamic_mappings_test.go +++ b/internal/builder/dynamic_mappings_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package builder import ( From e85ada31e2f49dba6a5f9982618c469388a85b68 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Dec 2022 16:35:16 +0100 Subject: [PATCH 09/18] Update field used to import mappings --- go.mod | 2 +- go.sum | 4 ++-- internal/builder/dynamic_mappings.go | 7 +++---- internal/packages/buildmanifest/build_manifest.go | 8 ++++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index ad2effce1..5356c6b6e 100644 --- a/go.mod +++ b/go.mod @@ -161,4 +161,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/elastic/package-spec/v2 => github.com/mrodm/package-spec/v2 v2.0.0-20221216145924-0dcd8377445f +replace github.com/elastic/package-spec/v2 => github.com/mrodm/package-spec/v2 v2.0.0-20221220145202-70d4563fbf33 diff --git a/go.sum b/go.sum index 37badabaa..73f01ef1f 100644 --- a/go.sum +++ b/go.sum @@ -382,8 +382,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/mrodm/package-spec/v2 v2.0.0-20221216145924-0dcd8377445f h1:U2ic94qu45zFRDv+iRd8JzNbi25dppuXyyjC0rTgeIs= -github.com/mrodm/package-spec/v2 v2.0.0-20221216145924-0dcd8377445f/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= +github.com/mrodm/package-spec/v2 v2.0.0-20221220145202-70d4563fbf33 h1:jvgOEjqmj/Ns0Oj5qoP9eliNCo7trpVjNycHxoyqYb4= +github.com/mrodm/package-spec/v2 v2.0.0-20221220145202-70d4563fbf33/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 5f438caaa..496ef219c 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -35,13 +35,13 @@ func addDynamicMappings(packageRoot, destinationDir string) error { return errors.Wrap(err, "can't read build manifest") } if !ok { - logger.Debugf("Build manifest hasn't been defined for the package") + logger.Debug("Build manifest hasn't been defined for the package") return nil } - if !bm.ImportCommonDynamicMappings() { - logger.Debugf("Package doesn't have to import common dynamic mappings") + if !bm.ImportMappings() { return nil } + logger.Debug("Import ECS mappings into the built package") dataStreamManifests, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", packages.DataStreamManifestFile)) if err != nil { @@ -113,7 +113,6 @@ func addDynamicMappingElements(path string) ([]byte, error) { return nil, err } - logger.Debugf("New Contents manifest:\n%s", contents) return contents, nil } diff --git a/internal/packages/buildmanifest/build_manifest.go b/internal/packages/buildmanifest/build_manifest.go index d32a1c2ef..45f6345d9 100644 --- a/internal/packages/buildmanifest/build_manifest.go +++ b/internal/packages/buildmanifest/build_manifest.go @@ -26,8 +26,8 @@ type Dependencies struct { // ECSDependency defines a dependency on ECS fields. type ECSDependency struct { - Reference string `config:"reference"` - ImportCommonDynamicMappings bool `config:"import_common_dynamic_mappings"` + Reference string `config:"reference"` + ImportMappings bool `config:"import_mappings"` } // HasDependencies function checks if there are any dependencies defined. @@ -36,8 +36,8 @@ func (bm *BuildManifest) HasDependencies() bool { } // HasDependencies function checks if there are any dependencies defined. -func (bm *BuildManifest) ImportCommonDynamicMappings() bool { - return bm.Dependencies.ECS.ImportCommonDynamicMappings +func (bm *BuildManifest) ImportMappings() bool { + return bm.Dependencies.ECS.ImportMappings } // ReadBuildManifest function reads the package build manifest. From 44763fc20158e87a3238e76b9e68a24e8702f392 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Dec 2022 18:30:14 +0100 Subject: [PATCH 10/18] Rename dynamic template names to include a common prefix Review errors raised and their messages --- internal/builder/dynamic_mappings.go | 56 +++++++++++++------ .../testdata/expected.empty.mappings.yml | 2 +- .../builder/testdata/expected.empty.meta.yml | 2 +- .../testdata/expected.existing.mappings.yml | 2 +- .../testdata/expected.existing.meta.yml | 2 +- 5 files changed, 44 insertions(+), 20 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 496ef219c..c224da0a8 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -6,6 +6,7 @@ package builder import ( _ "embed" + "fmt" "os" "path/filepath" "strconv" @@ -22,6 +23,8 @@ import ( //go:embed _static/ecs_mappings.json var staticEcsMappings string +const prefixMapping = "_embedded_ecs" + type ecsTemplates struct { Mappings struct { Properties map[string]interface{} `yaml:"properties"` @@ -80,15 +83,14 @@ func addDynamicMappings(packageRoot, destinationDir string) error { } func addDynamicMappingElements(path string) ([]byte, error) { - var ecsMappings ecsTemplates - err := yaml.Unmarshal([]byte(staticEcsMappings), &ecsMappings) + ecsMappings, err := loadEcsMappings() if err != nil { - return nil, err + return nil, errors.New("can't load ecs mappings template") } contents, err := os.ReadFile(path) if err != nil { - return nil, err + return nil, errors.New("can't read manifest") } var doc yaml.Node @@ -109,40 +111,62 @@ func addDynamicMappingElements(path string) ([]byte, error) { contents, err = formatResult(&doc) if err != nil { - logger.Errorf("Error formatting %s", err) return nil, err } return contents, nil } +func loadEcsMappings() (ecsTemplates, error) { + var ecsMappings ecsTemplates + err := yaml.Unmarshal([]byte(staticEcsMappings), &ecsMappings) + if err != nil { + return ecsMappings, err + } + return ecsMappings, nil +} + func addEcsMappings(doc *yaml.Node, mappings ecsTemplates) error { var templates, properties yaml.Node err := templates.Encode(mappings.Mappings.DynamicTemplates) if err != nil { - return err + return errors.Wrap(err, "failed to encode dynamic templates") } err = properties.Encode(mappings.Mappings.Properties) if err != nil { - logger.Errorf("Error encoding properties %s", err) - return err + return errors.Wrap(err, "failed to encode properties") } + renameMappingsNames(&templates) + err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "dynamic_templates"}, &templates) if err != nil { - logger.Errorf("Error appending elems %s", err) - return err + return errors.Wrap(err, "failed to append dynamic templates") } err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) if err != nil { - logger.Errorf("Error appending properties %s", err) - return err + return errors.Wrap(err, "failed to append properties") } return nil } +func renameMappingsNames(doc *yaml.Node) { + switch doc.Kind { + case yaml.MappingNode: + for i := 0; i < len(doc.Content); i += 2 { + doc.Content[i].Value = fmt.Sprintf("%s.%s", prefixMapping, doc.Content[i].Value) + } + case yaml.SequenceNode: + for i := 0; i < len(doc.Content); i++ { + renameMappingsNames(doc.Content[i]) + } + case yaml.DocumentNode: + renameMappingsNames(doc.Content[0]) + } +} + func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { type ecsMappingsAdded struct { Properties []string `yaml:"properties"` @@ -156,18 +180,18 @@ func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { for _, dynamicTemplate := range mappings.Mappings.DynamicTemplates { if len(dynamicTemplate) > 1 { - return errors.New("more than one dynamic template present") + return errors.New("more than one dynamic template present in the same definition") } for title := range dynamicTemplate { - mappingsAdded.DynamicTemplates = append(mappingsAdded.DynamicTemplates, title) + mappingsAdded.DynamicTemplates = append(mappingsAdded.DynamicTemplates, fmt.Sprintf("%s.%s", prefixMapping, title)) } } var properties yaml.Node err := properties.Encode(mappingsAdded.Properties) if err != nil { - return err + return errors.Wrap(err, "failed to encode properties") } err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "_meta", "ecs_properties_added"}, &properties) @@ -178,7 +202,7 @@ func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { var dynamicTemplates yaml.Node err = dynamicTemplates.Encode(mappingsAdded.DynamicTemplates) if err != nil { - return err + return errors.Wrap(err, "failed to encode dynamic properties") } err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "_meta", "ecs_dynamic_templates_added"}, &dynamicTemplates) diff --git a/internal/builder/testdata/expected.empty.mappings.yml b/internal/builder/testdata/expected.empty.mappings.yml index 034265900..10b18a222 100644 --- a/internal/builder/testdata/expected.empty.mappings.yml +++ b/internal/builder/testdata/expected.empty.mappings.yml @@ -3,7 +3,7 @@ elasticsearch: index_template: mappings: dynamic_templates: - - example_template: + - _embedded_ecs.example_template: mapping: type: short patch_match: example.* diff --git a/internal/builder/testdata/expected.empty.meta.yml b/internal/builder/testdata/expected.empty.meta.yml index 93121d810..1b0eae209 100644 --- a/internal/builder/testdata/expected.empty.meta.yml +++ b/internal/builder/testdata/expected.empty.meta.yml @@ -6,4 +6,4 @@ elasticsearch: ecs_properties_added: - datetime ecs_dynamic_templates_added: - - example_template + - _embedded_ecs.example_template diff --git a/internal/builder/testdata/expected.existing.mappings.yml b/internal/builder/testdata/expected.existing.mappings.yml index ecf716f9c..bc1b9dacc 100644 --- a/internal/builder/testdata/expected.existing.mappings.yml +++ b/internal/builder/testdata/expected.existing.mappings.yml @@ -18,7 +18,7 @@ elasticsearch: path: trigger mapping: type: nested - - example_template: + - _embedded_ecs.example_template: mapping: type: short patch_match: example.* diff --git a/internal/builder/testdata/expected.existing.meta.yml b/internal/builder/testdata/expected.existing.meta.yml index 664f38a81..3ef576971 100644 --- a/internal/builder/testdata/expected.existing.meta.yml +++ b/internal/builder/testdata/expected.existing.meta.yml @@ -9,7 +9,7 @@ elasticsearch: - datetime ecs_dynamic_templates_added: - to_nested - - example_template + - _embedded_ecs.example_template properties: example: type: short From 311ee7037b06a7c7c3b81424e69c120fd6e70b1c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Dec 2022 18:44:15 +0100 Subject: [PATCH 11/18] Remove properties --- internal/builder/_static/ecs_mappings.json | 15 +++++---- internal/builder/dynamic_mappings.go | 32 ++----------------- .../testdata/expected.empty.mappings.yml | 3 -- .../builder/testdata/expected.empty.meta.yml | 2 -- .../testdata/expected.existing.mappings.yml | 2 -- .../testdata/expected.existing.meta.yml | 1 - 6 files changed, 12 insertions(+), 43 deletions(-) diff --git a/internal/builder/_static/ecs_mappings.json b/internal/builder/_static/ecs_mappings.json index 1de642bd1..f2f6f7660 100644 --- a/internal/builder/_static/ecs_mappings.json +++ b/internal/builder/_static/ecs_mappings.json @@ -1,12 +1,15 @@ { "mappings": { - "properties": { - "@timestamp": { - "type": "date", - "ignore_malformed": false - } - }, "dynamic_templates": [ + { + "ecs_timestamp": { + "path_match": "@timestamp", + "mapping": { + "type": "date", + "ignore_malformed": false + } + } + }, { "data_stream_to_constant": { "path_match": "data_stream.*", diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index c224da0a8..bc7f09555 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -27,7 +27,6 @@ const prefixMapping = "_embedded_ecs" type ecsTemplates struct { Mappings struct { - Properties map[string]interface{} `yaml:"properties"` DynamicTemplates []map[string]interface{} `yaml:"dynamic_templates"` } `yaml:"mappings"` } @@ -127,15 +126,11 @@ func loadEcsMappings() (ecsTemplates, error) { } func addEcsMappings(doc *yaml.Node, mappings ecsTemplates) error { - var templates, properties yaml.Node + var templates yaml.Node err := templates.Encode(mappings.Mappings.DynamicTemplates) if err != nil { return errors.Wrap(err, "failed to encode dynamic templates") } - err = properties.Encode(mappings.Mappings.Properties) - if err != nil { - return errors.Wrap(err, "failed to encode properties") - } renameMappingsNames(&templates) @@ -144,11 +139,6 @@ func addEcsMappings(doc *yaml.Node, mappings ecsTemplates) error { return errors.Wrap(err, "failed to append dynamic templates") } - err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "properties"}, &properties) - if err != nil { - return errors.Wrap(err, "failed to append properties") - } - return nil } @@ -169,14 +159,9 @@ func renameMappingsNames(doc *yaml.Node) { func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { type ecsMappingsAdded struct { - Properties []string `yaml:"properties"` DynamicTemplates []string `yaml:"dynamic_templates"` } - var mappingsAdded ecsMappingsAdded - for property := range mappings.Mappings.Properties { - mappingsAdded.Properties = append(mappingsAdded.Properties, property) - } for _, dynamicTemplate := range mappings.Mappings.DynamicTemplates { if len(dynamicTemplate) > 1 { @@ -188,21 +173,10 @@ func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { } } - var properties yaml.Node - err := properties.Encode(mappingsAdded.Properties) - if err != nil { - return errors.Wrap(err, "failed to encode properties") - } - - err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "_meta", "ecs_properties_added"}, &properties) - if err != nil { - return err - } - var dynamicTemplates yaml.Node - err = dynamicTemplates.Encode(mappingsAdded.DynamicTemplates) + err := dynamicTemplates.Encode(mappingsAdded.DynamicTemplates) if err != nil { - return errors.Wrap(err, "failed to encode dynamic properties") + return errors.Wrap(err, "failed to encode dynamic templates") } err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "_meta", "ecs_dynamic_templates_added"}, &dynamicTemplates) diff --git a/internal/builder/testdata/expected.empty.mappings.yml b/internal/builder/testdata/expected.empty.mappings.yml index 10b18a222..2de15663e 100644 --- a/internal/builder/testdata/expected.empty.mappings.yml +++ b/internal/builder/testdata/expected.empty.mappings.yml @@ -7,6 +7,3 @@ elasticsearch: mapping: type: short patch_match: example.* - properties: - datetime: - type: long diff --git a/internal/builder/testdata/expected.empty.meta.yml b/internal/builder/testdata/expected.empty.meta.yml index 1b0eae209..92a32c5e0 100644 --- a/internal/builder/testdata/expected.empty.meta.yml +++ b/internal/builder/testdata/expected.empty.meta.yml @@ -3,7 +3,5 @@ elasticsearch: index_template: mappings: _meta: - ecs_properties_added: - - datetime ecs_dynamic_templates_added: - _embedded_ecs.example_template diff --git a/internal/builder/testdata/expected.existing.mappings.yml b/internal/builder/testdata/expected.existing.mappings.yml index bc1b9dacc..dc79aced6 100644 --- a/internal/builder/testdata/expected.existing.mappings.yml +++ b/internal/builder/testdata/expected.existing.mappings.yml @@ -11,8 +11,6 @@ elasticsearch: properties: example: type: short - datetime: - type: long dynamic_templates: - to_nested: path: trigger diff --git a/internal/builder/testdata/expected.existing.meta.yml b/internal/builder/testdata/expected.existing.meta.yml index 3ef576971..010072680 100644 --- a/internal/builder/testdata/expected.existing.meta.yml +++ b/internal/builder/testdata/expected.existing.meta.yml @@ -6,7 +6,6 @@ elasticsearch: managed: true ecs_properties_added: - example - - datetime ecs_dynamic_templates_added: - to_nested - _embedded_ecs.example_template From f6fcff5bdada361ef22b8ecce876c121bfe2f121 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 20 Dec 2022 18:49:51 +0100 Subject: [PATCH 12/18] Remove the addition of _meta fields --- internal/builder/dynamic_mappings.go | 37 +-------------- internal/builder/dynamic_mappings_test.go | 45 ------------------- internal/builder/testdata/existing.yml | 6 --- .../builder/testdata/expected.empty.meta.yml | 7 --- .../testdata/expected.existing.mappings.yml | 6 --- .../testdata/expected.existing.meta.yml | 19 -------- 6 files changed, 1 insertion(+), 119 deletions(-) delete mode 100644 internal/builder/testdata/expected.empty.meta.yml delete mode 100644 internal/builder/testdata/expected.existing.meta.yml diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index bc7f09555..60af8b3c9 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -103,11 +103,6 @@ func addDynamicMappingElements(path string) ([]byte, error) { return nil, err } - err = addEcsMappingsListMeta(&doc, ecsMappings) - if err != nil { - return nil, err - } - contents, err = formatResult(&doc) if err != nil { return nil, err @@ -157,36 +152,6 @@ func renameMappingsNames(doc *yaml.Node) { } } -func addEcsMappingsListMeta(doc *yaml.Node, mappings ecsTemplates) error { - type ecsMappingsAdded struct { - DynamicTemplates []string `yaml:"dynamic_templates"` - } - var mappingsAdded ecsMappingsAdded - - for _, dynamicTemplate := range mappings.Mappings.DynamicTemplates { - if len(dynamicTemplate) > 1 { - return errors.New("more than one dynamic template present in the same definition") - } - - for title := range dynamicTemplate { - mappingsAdded.DynamicTemplates = append(mappingsAdded.DynamicTemplates, fmt.Sprintf("%s.%s", prefixMapping, title)) - } - } - - var dynamicTemplates yaml.Node - err := dynamicTemplates.Encode(mappingsAdded.DynamicTemplates) - if err != nil { - return errors.Wrap(err, "failed to encode dynamic templates") - } - - err = appendElements(doc, []string{"elasticsearch", "index_template", "mappings", "_meta", "ecs_dynamic_templates_added"}, &dynamicTemplates) - if err != nil { - return err - } - - return nil -} - func appendElements(root *yaml.Node, path []string, values *yaml.Node) error { if len(path) == 0 { contents := values.Content @@ -232,7 +197,7 @@ func newYamlNode(key string) []*yaml.Node { keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} var childNode *yaml.Node switch key { - case "dynamic_templates", "ecs_properties_added", "ecs_dynamic_templates_added": + case "dynamic_templates": childNode = &yaml.Node{Kind: yaml.SequenceNode, Value: key} default: childNode = &yaml.Node{Kind: yaml.MappingNode, Value: key} diff --git a/internal/builder/dynamic_mappings_test.go b/internal/builder/dynamic_mappings_test.go index a0fc40e34..3eca73aac 100644 --- a/internal/builder/dynamic_mappings_test.go +++ b/internal/builder/dynamic_mappings_test.go @@ -13,51 +13,6 @@ import ( "gopkg.in/yaml.v3" ) -func TestAddMappingsListMeta(t *testing.T) { - cases := []struct { - title string - baseDocPath string - ecsToAddPath string - expectedPath string - }{ - { - title: "Add new mappings", - baseDocPath: "testdata/empty.yml", - ecsToAddPath: "testdata/ecs.template.yml", - expectedPath: "testdata/expected.empty.meta.yml", - }, - { - title: "Append mappings", - baseDocPath: "testdata/existing.yml", - ecsToAddPath: "testdata/ecs.template.yml", - expectedPath: "testdata/expected.existing.meta.yml", - }, - } - - for _, c := range cases { - t.Run(c.title, func(t *testing.T) { - contentsBase, _ := os.ReadFile(c.baseDocPath) - contentsToAdd, _ := os.ReadFile(c.ecsToAddPath) - contentsExpected, _ := os.ReadFile(c.expectedPath) - - var base yaml.Node - err := yaml.Unmarshal(contentsBase, &base) - require.NoError(t, err) - - var template ecsTemplates - err = yaml.Unmarshal(contentsToAdd, &template) - require.NoError(t, err) - - err = addEcsMappingsListMeta(&base, template) - require.NoError(t, err) - - newYaml, _ := formatResult(&base) - - assert.Equal(t, string(contentsExpected), string(newYaml)) - }) - } -} - func TestAddEcsMappings(t *testing.T) { cases := []struct { title string diff --git a/internal/builder/testdata/existing.yml b/internal/builder/testdata/existing.yml index b98ae93ee..a17128725 100644 --- a/internal/builder/testdata/existing.yml +++ b/internal/builder/testdata/existing.yml @@ -3,12 +3,6 @@ title: existing elasticsearch: index_template: mappings: - _meta: - managed: true - ecs_properties_added: - - example - ecs_dynamic_templates_added: - - to_nested properties: example: type: short diff --git a/internal/builder/testdata/expected.empty.meta.yml b/internal/builder/testdata/expected.empty.meta.yml deleted file mode 100644 index 92a32c5e0..000000000 --- a/internal/builder/testdata/expected.empty.meta.yml +++ /dev/null @@ -1,7 +0,0 @@ -title: empty -elasticsearch: - index_template: - mappings: - _meta: - ecs_dynamic_templates_added: - - _embedded_ecs.example_template diff --git a/internal/builder/testdata/expected.existing.mappings.yml b/internal/builder/testdata/expected.existing.mappings.yml index dc79aced6..d00ddfeaf 100644 --- a/internal/builder/testdata/expected.existing.mappings.yml +++ b/internal/builder/testdata/expected.existing.mappings.yml @@ -2,12 +2,6 @@ title: existing elasticsearch: index_template: mappings: - _meta: - managed: true - ecs_properties_added: - - example - ecs_dynamic_templates_added: - - to_nested properties: example: type: short diff --git a/internal/builder/testdata/expected.existing.meta.yml b/internal/builder/testdata/expected.existing.meta.yml deleted file mode 100644 index 010072680..000000000 --- a/internal/builder/testdata/expected.existing.meta.yml +++ /dev/null @@ -1,19 +0,0 @@ -title: existing -elasticsearch: - index_template: - mappings: - _meta: - managed: true - ecs_properties_added: - - example - ecs_dynamic_templates_added: - - to_nested - - _embedded_ecs.example_template - properties: - example: - type: short - dynamic_templates: - - to_nested: - path: trigger - mapping: - type: nested From 17d6ac331ae4bd9c651f77e40e9fd5236f017d95 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Dec 2022 09:50:50 +0100 Subject: [PATCH 13/18] Add debug message before validation --- internal/builder/packages.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/builder/packages.go b/internal/builder/packages.go index bb38b4e28..abaa896d7 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -195,7 +195,7 @@ func BuildPackage(options BuildOptions) (string, error) { return destinationDir, nil } - logger.Infof("Validating package %s", destinationDir) + logger.Debugf("Validating built package (path: %s)", destinationDir) err = validator.ValidateFromPath(destinationDir) if err != nil { return "", errors.Wrap(err, "invalid content found in built package") @@ -218,6 +218,7 @@ func buildZippedPackage(options BuildOptions, destinationDir string) (string, er if options.SkipValidation { logger.Debug("Skip validation of the built .zip package") } else { + logger.Debugf("Validating built .zip package (path: %s)", zippedPackagePath) err = validator.ValidateFromZip(zippedPackagePath) if err != nil { return "", errors.Wrapf(err, "invalid content found in built zip package") From 31fa9a25402073ba6dc31da02c761e60ceb37859 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Dec 2022 10:07:29 +0100 Subject: [PATCH 14/18] Update debug messages --- internal/builder/dynamic_mappings.go | 1 + internal/builder/packages.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 60af8b3c9..0856e70fb 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -41,6 +41,7 @@ func addDynamicMappings(packageRoot, destinationDir string) error { return nil } if !bm.ImportMappings() { + logger.Debug("Package doesn't have to import ECS mappings") return nil } logger.Debug("Import ECS mappings into the built package") diff --git a/internal/builder/packages.go b/internal/builder/packages.go index abaa896d7..f7f40db17 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -180,7 +180,7 @@ func BuildPackage(options BuildOptions) (string, error) { return "", errors.Wrap(err, "resolving external fields failed") } - logger.Debug("Add dynamic mappings") + logger.Debug("Add dynamic mappings if needed") err = addDynamicMappings(options.PackageRoot, destinationDir) if err != nil { return "", errors.Wrap(err, "adding dynamic mappings") From aaa83e2fa7f8913b74333b0e3cbb9825e086bdfc Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Dec 2022 11:34:51 +0100 Subject: [PATCH 15/18] Add method to check if mappings should be imported --- internal/builder/dynamic_mappings.go | 79 +++++++++++++++++++--------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 0856e70fb..ecf8b1347 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strconv" + "github.com/Masterminds/semver" "github.com/pkg/errors" "gopkg.in/yaml.v3" @@ -25,6 +26,8 @@ var staticEcsMappings string const prefixMapping = "_embedded_ecs" +var semver2_3_0 = semver.MustParse("2.3.0") + type ecsTemplates struct { Mappings struct { DynamicTemplates []map[string]interface{} `yaml:"dynamic_templates"` @@ -32,43 +35,41 @@ type ecsTemplates struct { } func addDynamicMappings(packageRoot, destinationDir string) error { - bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot) + packageManifest := filepath.Join(destinationDir, packages.PackageManifestFile) + + m, err := packages.ReadPackageManifest(packageManifest) if err != nil { - return errors.Wrap(err, "can't read build manifest") - } - if !ok { - logger.Debug("Build manifest hasn't been defined for the package") - return nil - } - if !bm.ImportMappings() { - logger.Debug("Package doesn't have to import ECS mappings") - return nil + return err } - logger.Debug("Import ECS mappings into the built package") - dataStreamManifests, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", packages.DataStreamManifestFile)) + shouldImport, err := shouldImportEcsMappings(m.SpecVersion, packageRoot) if err != nil { return err } + if !shouldImport { + return nil + } - for _, datastream := range dataStreamManifests { - contents, err := addDynamicMappingElements(datastream) - if err != nil { - return err - } - err = os.WriteFile(datastream, contents, 0664) + logger.Debug("Import ECS mappings into the built package") + + switch m.Type { + case "integration": + dataStreamManifests, err := filepath.Glob(filepath.Join(destinationDir, "data_stream", "*", packages.DataStreamManifestFile)) if err != nil { return err } - } - - packageManifest := filepath.Join(destinationDir, packages.PackageManifestFile) - m, err := packages.ReadPackageManifest(packageManifest) - if err != nil { - return err - } - if m.Type == "input" { + for _, datastream := range dataStreamManifests { + contents, err := addDynamicMappingElements(datastream) + if err != nil { + return err + } + err = os.WriteFile(datastream, contents, 0664) + if err != nil { + return err + } + } + case "input": contents, err := addDynamicMappingElements(packageManifest) if err != nil { return err @@ -82,6 +83,32 @@ func addDynamicMappings(packageRoot, destinationDir string) error { return nil } +func shouldImportEcsMappings(specVersion, packageRoot string) (bool, error) { + v, err := semver.NewVersion(specVersion) + if err != nil { + return false, errors.Wrap(err, "invalid spec version") + } + + if v.LessThan(semver2_3_0) { + logger.Debugf("Required spec version >= %s to import ECS mappings", semver2_3_0.String()) + return false, nil + } + + bm, ok, err := buildmanifest.ReadBuildManifest(packageRoot) + if err != nil { + return false, errors.Wrap(err, "can't read build manifest") + } + if !ok { + logger.Debug("Build manifest hasn't been defined for the package") + return false, nil + } + if !bm.ImportMappings() { + logger.Debug("Package doesn't have to import ECS mappings") + return false, nil + } + return true, nil +} + func addDynamicMappingElements(path string) ([]byte, error) { ecsMappings, err := loadEcsMappings() if err != nil { From f21c828dc47f94701004b699f8717b3c0282c10d Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Wed, 21 Dec 2022 11:46:36 +0100 Subject: [PATCH 16/18] Use semver v3 --- internal/builder/dynamic_mappings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index ecf8b1347..9024d6fd4 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -11,7 +11,7 @@ import ( "path/filepath" "strconv" - "github.com/Masterminds/semver" + "github.com/Masterminds/semver/v3" "github.com/pkg/errors" "gopkg.in/yaml.v3" From a46de7bbc9ba6549aab70f35860d875657ca3fef Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 22 Dec 2022 14:36:14 +0100 Subject: [PATCH 17/18] Remove package-spec replace from go.mod --- go.mod | 2 -- go.sum | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ad1a606c4..427be0cbc 100644 --- a/go.mod +++ b/go.mod @@ -159,5 +159,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) - -replace github.com/elastic/package-spec/v2 => github.com/mrodm/package-spec/v2 v2.0.0-20221220145202-70d4563fbf33 diff --git a/go.sum b/go.sum index 822e6ac88..2c6067669 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gojsonschema v1.2.1 h1:cUMbgsz0wyEB4x7xf3zUEvUVDl6WCz2RKcQPul8OsQc= github.com/elastic/gojsonschema v1.2.1/go.mod h1:biw5eBS2Z4T02wjATMRSfecfjCmwaDPvuaqf844gLrg= +github.com/elastic/package-spec/v2 v2.2.0 h1:hNTtw1PmM9BcehF5g7E4NNNIXNTxravcbrwATucYOAU= +github.com/elastic/package-spec/v2 v2.2.0/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -292,8 +294,6 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/mrodm/package-spec/v2 v2.0.0-20221220145202-70d4563fbf33 h1:jvgOEjqmj/Ns0Oj5qoP9eliNCo7trpVjNycHxoyqYb4= -github.com/mrodm/package-spec/v2 v2.0.0-20221220145202-70d4563fbf33/go.mod h1:bGpJHojIN9FYyNbqNySJ7D3mDVqOe1sL2l4m7BCSjWo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= From 8290c657c2a7384b6111c11bbd3eec056a0efcb8 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 22 Dec 2022 19:23:56 +0100 Subject: [PATCH 18/18] Replace dot by a dash in the imported template names In kibana versions < 8.6.0 the dot in the names caused that packages were not installed. It was interpreted that the string after the dot was an inner element of the map, causing validation errors. --- internal/builder/dynamic_mappings.go | 2 +- internal/builder/testdata/expected.empty.mappings.yml | 2 +- internal/builder/testdata/expected.existing.mappings.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/builder/dynamic_mappings.go b/internal/builder/dynamic_mappings.go index 9024d6fd4..356bf9ed9 100644 --- a/internal/builder/dynamic_mappings.go +++ b/internal/builder/dynamic_mappings.go @@ -169,7 +169,7 @@ func renameMappingsNames(doc *yaml.Node) { switch doc.Kind { case yaml.MappingNode: for i := 0; i < len(doc.Content); i += 2 { - doc.Content[i].Value = fmt.Sprintf("%s.%s", prefixMapping, doc.Content[i].Value) + doc.Content[i].Value = fmt.Sprintf("%s-%s", prefixMapping, doc.Content[i].Value) } case yaml.SequenceNode: for i := 0; i < len(doc.Content); i++ { diff --git a/internal/builder/testdata/expected.empty.mappings.yml b/internal/builder/testdata/expected.empty.mappings.yml index 2de15663e..706abec3b 100644 --- a/internal/builder/testdata/expected.empty.mappings.yml +++ b/internal/builder/testdata/expected.empty.mappings.yml @@ -3,7 +3,7 @@ elasticsearch: index_template: mappings: dynamic_templates: - - _embedded_ecs.example_template: + - _embedded_ecs-example_template: mapping: type: short patch_match: example.* diff --git a/internal/builder/testdata/expected.existing.mappings.yml b/internal/builder/testdata/expected.existing.mappings.yml index d00ddfeaf..28f168641 100644 --- a/internal/builder/testdata/expected.existing.mappings.yml +++ b/internal/builder/testdata/expected.existing.mappings.yml @@ -10,7 +10,7 @@ elasticsearch: path: trigger mapping: type: nested - - _embedded_ecs.example_template: + - _embedded_ecs-example_template: mapping: type: short patch_match: example.*