From 3e9cceeb24d4b5c8eb53d4900e69fc404f9fdab3 Mon Sep 17 00:00:00 2001 From: ruflin Date: Mon, 29 Jun 2020 08:50:53 +0200 Subject: [PATCH 1/3] Introduce `elasticsearch.ingest_pipeline.name` as config option In https://github.com/elastic/package-registry/pull/552 `elasticsearch` as config block is introduced. As discussed there, it also makes sense to use this for the Ingest Pipeline config to have all Elasticsearch related parts in one place. For now no package uses this change and Kibana does not support it. As both ways are supported, this is not a breaking change yet. As a follow up, the package generation must be updated to adjust for this option and Kibana must be adapted to only support the new option. --- CHANGELOG.md | 1 + util/dataset.go | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 696410011..a4175c4ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Apply rule: first package found served. [#546](https://github.com/elastic/package-registry/pull/546) * Implement package watcher. [#553](https://github.com/elastic/package-registry/pull/553) * Add conditions as future replacement of requirements. [#519](https://github.com/elastic/package-registry/pull/519) +* Introduce `elasticsearch.ingest_pipeline.name` as config option. [#](https://github.com/elastic/package-registry/pull/) ### Deprecated diff --git a/util/dataset.go b/util/dataset.go index c212031f2..d110cdb6e 100644 --- a/util/dataset.go +++ b/util/dataset.go @@ -83,6 +83,10 @@ type Elasticsearch struct { IndexTemplateMappings map[string]interface{} `config:"index_template.mappings" json:"index_template.mappings,omitempty" yaml:"index_template.mappings,omitempty"` } +type Elasticsearch struct { + IngestPipelineName string `config:"ingest_pipeline.name,omitempty" json:"ingest_pipeline.name,omitempty" yaml:"ingest_pipeline.name,omitempty"` +} + type fieldEntry struct { name string aType string @@ -157,7 +161,18 @@ func (d *Dataset) Validate() error { return fmt.Errorf("type is not valid: %s", d.Type) } - if d.IngestPipeline == "" { + if d.Elasticsearch != nil && d.Elasticsearch.IngestPipelineName == "" { + // Check that no ingest pipeline exists in the directory except default + for _, path := range paths { + if filepath.Base(path) == "default.json" || filepath.Base(path) == "default.yml" { + d.Elasticsearch.IngestPipelineName = "default" + // TODO: remove because of legacy + d.IngestPipeline = "default" + break + } + } + // TODO: Remove, only here for legacy + } else if d.IngestPipeline == "" { // Check that no ingest pipeline exists in the directory except default for _, path := range paths { if filepath.Base(path) == "default.json" || filepath.Base(path) == "default.yml" { From b888df7a3f39b5fcb32c8015ba10723baff954d1 Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 1 Jul 2020 15:48:15 +0200 Subject: [PATCH 2/3] commit review --- util/dataset.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/util/dataset.go b/util/dataset.go index d110cdb6e..1ed636791 100644 --- a/util/dataset.go +++ b/util/dataset.go @@ -22,6 +22,10 @@ import ( const ( DirIngestPipeline = "ingest-pipeline" + + DefaultPipelineName = "default" + DefaultPipelineNameJSON = "default.json" + DefaultPipelineNameYAML = "default.yaml" ) var validTypes = map[string]string{ @@ -34,8 +38,10 @@ type Dataset struct { Type string `config:"type" json:"type" validate:"required"` Name string `config:"name" json:"name,omitempty" yaml:"name,omitempty"` - Title string `config:"title" json:"title" validate:"required"` - Release string `config:"release" json:"release"` + Title string `config:"title" json:"title" validate:"required"` + Release string `config:"release" json:"release"` + + // Deprecated: Replaced by elasticsearch.ingest_pipeline.name IngestPipeline string `config:"ingest_pipeline,omitempty" config:"ingest_pipeline" json:"ingest_pipeline,omitempty" yaml:"ingest_pipeline,omitempty"` Streams []Stream `config:"streams" json:"streams,omitempty" yaml:"streams,omitempty" ` Package string `json:"package,omitempty" yaml:"package,omitempty"` @@ -164,10 +170,10 @@ func (d *Dataset) Validate() error { if d.Elasticsearch != nil && d.Elasticsearch.IngestPipelineName == "" { // Check that no ingest pipeline exists in the directory except default for _, path := range paths { - if filepath.Base(path) == "default.json" || filepath.Base(path) == "default.yml" { - d.Elasticsearch.IngestPipelineName = "default" + if filepath.Base(path) == DefaultPipelineNameJSON || filepath.Base(path) == DefaultPipelineNameYAML { + d.Elasticsearch.IngestPipelineName = DefaultPipelineName // TODO: remove because of legacy - d.IngestPipeline = "default" + d.IngestPipeline = DefaultPipelineName break } } @@ -175,8 +181,8 @@ func (d *Dataset) Validate() error { } else if d.IngestPipeline == "" { // Check that no ingest pipeline exists in the directory except default for _, path := range paths { - if filepath.Base(path) == "default.json" || filepath.Base(path) == "default.yml" { - d.IngestPipeline = "default" + if filepath.Base(path) == DefaultPipelineNameJSON || filepath.Base(path) == DefaultPipelineNameYAML { + d.IngestPipeline = DefaultPipelineName break } } From 783ad9253d163792eaabbc8e1dc8bfc8f8f18157 Mon Sep 17 00:00:00 2001 From: ruflin Date: Wed, 1 Jul 2020 15:50:12 +0200 Subject: [PATCH 3/3] merge object --- util/dataset.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/util/dataset.go b/util/dataset.go index 1ed636791..f00f0d859 100644 --- a/util/dataset.go +++ b/util/dataset.go @@ -87,10 +87,7 @@ type Variable struct { type Elasticsearch struct { IndexTemplateSettings map[string]interface{} `config:"index_template.settings" json:"index_template.settings,omitempty" yaml:"index_template.settings,omitempty"` IndexTemplateMappings map[string]interface{} `config:"index_template.mappings" json:"index_template.mappings,omitempty" yaml:"index_template.mappings,omitempty"` -} - -type Elasticsearch struct { - IngestPipelineName string `config:"ingest_pipeline.name,omitempty" json:"ingest_pipeline.name,omitempty" yaml:"ingest_pipeline.name,omitempty"` + IngestPipelineName string `config:"ingest_pipeline.name,omitempty" json:"ingest_pipeline.name,omitempty" yaml:"ingest_pipeline.name,omitempty"` } type fieldEntry struct {