From a406836ec5b9eda79835853a11786ef062ed6ba5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <salvatore.campagna@elastic.co> Date: Thu, 1 Jun 2023 15:12:35 +0200 Subject: [PATCH 1/6] fix: do not copy index.default_pipeline and index.final_pipeline --- x-pack/plugin/rollup/qa/rest/build.gradle | 2 +- .../downsample/TransportDownsampleAction.java | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/rollup/qa/rest/build.gradle b/x-pack/plugin/rollup/qa/rest/build.gradle index e91e56d2d1ba4..1cffa7fcafc6a 100644 --- a/x-pack/plugin/rollup/qa/rest/build.gradle +++ b/x-pack/plugin/rollup/qa/rest/build.gradle @@ -17,7 +17,7 @@ dependencies { restResources { restApi { - include '_common', 'bulk', 'cluster', 'indices', 'search' + include '_common', 'bulk', 'cluster', 'indices', 'search', 'ingest.put_pipeline', 'ingest.delete_pipeline' } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 3f71701d3ae17..1adfbc8854661 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -74,8 +74,11 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_METRIC_PARAM; @@ -97,6 +100,14 @@ public class TransportDownsampleAction extends AcknowledgedTransportMasterNodeAc private final IndexScopedSettings indexScopedSettings; private final ThreadContext threadContext; + private static final Set<String> FORBIDDEN_SETTINGS = new HashSet<>( + Arrays.asList( + IndexSettings.DEFAULT_PIPELINE.getKey(), + IndexSettings.FINAL_PIPELINE.getKey(), + IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey() + ) + ); + /** * This is the cluster state task executor for cluster state update actions. */ @@ -584,12 +595,16 @@ private IndexMetadata.Builder copyIndexMetadata(IndexMetadata sourceIndexMetadat // the same rules with resize apply continue; } + if (FORBIDDEN_SETTINGS.contains(key)) { + continue; + } // Do not override settings that have already been set in the rollup index. // Also, we don't want to copy the `index.block.write` setting that we know // it is set in the source index settings. - if (IndexMetadata.SETTING_BLOCKS_WRITE.equals(key) || targetSettings.keys().contains(key)) { + if (targetSettings.keys().contains(key)) { continue; } + targetSettings.copy(key, sourceIndexMetadata.getSettings()); } From b97456927f4cc1bea45f6ae65d115d06ba910f22 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 1 Jun 2023 15:15:09 +0200 Subject: [PATCH 2/6] Update docs/changelog/96494.yaml --- docs/changelog/96494.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/96494.yaml diff --git a/docs/changelog/96494.yaml b/docs/changelog/96494.yaml new file mode 100644 index 0000000000000..9f2b58c5b4044 --- /dev/null +++ b/docs/changelog/96494.yaml @@ -0,0 +1,6 @@ +pr: 96494 +summary: Do not copy `index.default_pipeline` and `index.final_pipeline` +area: Rollup +type: bug +issues: + - 96478 From 624ab10994324a37832c5441a1b89e385da78b00 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <salvatore.campagna@elastic.co> Date: Thu, 1 Jun 2023 15:25:54 +0200 Subject: [PATCH 3/6] fix: use Set.of --- .../xpack/downsample/TransportDownsampleAction.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 1adfbc8854661..2c934c6433765 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -74,8 +74,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -100,12 +98,10 @@ public class TransportDownsampleAction extends AcknowledgedTransportMasterNodeAc private final IndexScopedSettings indexScopedSettings; private final ThreadContext threadContext; - private static final Set<String> FORBIDDEN_SETTINGS = new HashSet<>( - Arrays.asList( - IndexSettings.DEFAULT_PIPELINE.getKey(), - IndexSettings.FINAL_PIPELINE.getKey(), - IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey() - ) + private static final Set<String> FORBIDDEN_SETTINGS = Set.of( + IndexSettings.DEFAULT_PIPELINE.getKey(), + IndexSettings.FINAL_PIPELINE.getKey(), + IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey() ); /** From 2874499821df6e177f2971b648a5c8d4a96a8f32 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <salvatore.campagna@elastic.co> Date: Thu, 1 Jun 2023 15:45:32 +0200 Subject: [PATCH 4/6] fix: align comment to code --- .../xpack/downsample/TransportDownsampleAction.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 2c934c6433765..d175f41b561ef 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -591,12 +591,11 @@ private IndexMetadata.Builder copyIndexMetadata(IndexMetadata sourceIndexMetadat // the same rules with resize apply continue; } + // Do not copy index settings which are valid for the source index but not for the target index if (FORBIDDEN_SETTINGS.contains(key)) { continue; } // Do not override settings that have already been set in the rollup index. - // Also, we don't want to copy the `index.block.write` setting that we know - // it is set in the source index settings. if (targetSettings.keys().contains(key)) { continue; } From 8a3cfeecf4094f40d23f420f2c8172efa4aa99ac Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <salvatore.campagna@elastic.co> Date: Thu, 1 Jun 2023 15:46:19 +0200 Subject: [PATCH 5/6] test: check index.default_pipeline and index.final_pipeline are not copied --- .../rest-api-spec/test/rollup/60_settings.yml | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml diff --git a/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml new file mode 100644 index 0000000000000..8d3f0d278a8e5 --- /dev/null +++ b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml @@ -0,0 +1,90 @@ +--- +"Downsample index with pipeline": + - skip: + version: " - 8.4.99" + reason: "rollup renamed to downsample in 8.5.0" + + - do: + ingest.put_pipeline: + id: "pipeline" + body: > + { + "processors": [ + { + "set" : { + "field" : "field", + "value": 42 + } + } + ] + } + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index: + default_pipeline: "pipeline" + final_pipeline: "pipeline" + mode: time_series + routing_path: [metricset, k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + value: + type: double + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "value": 10}' + + - do: + indices.put_settings: + index: test + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test + target_index: test-rollup + body: > + { + "fixed_interval": "1h" + } + + - do: + indices.get_settings: + index: test + + - match: { test.settings.index.default_pipeline: "pipeline" } + - match: { test.settings.index.final_pipeline: "pipeline" } + + - do: + indices.get_settings: + index: test-rollup + + - match: { test-rollup.settings.index.default_pipeline: null } + - match: { test-rollup.settings.index.final_pipeline: null } + + +--- +teardown: + - do: + ingest.delete_pipeline: + id: "pipeline" + ignore: 404 From 66102fa723112236aedba18ca33485ebd0ea6e7a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <salvatore.campagna@elastic.co> Date: Thu, 1 Jun 2023 15:57:45 +0200 Subject: [PATCH 6/6] fic: check the pipeline works on source index --- .../rest-api-spec/test/rollup/60_settings.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml index 8d3f0d278a8e5..7b1b6d6a2134b 100644 --- a/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml +++ b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml @@ -52,6 +52,16 @@ - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "value": 10}' + - do: + search: + index: test + body: + query: + match_all: {} + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.field: 42 } + - do: indices.put_settings: index: test