From bbfef04c3e61830660eb51e138ff885a704199df Mon Sep 17 00:00:00 2001 From: Salvatore Campagna <93581129+salvatore-campagna@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:47:41 +0200 Subject: [PATCH] Do not copy index.default_pipeline and index.final_pipeline (#96494) (#96500) We need to skip copying the two settings, index.default_pipeline and index.final_pipeline, from the source index to the target index in order to avoid pipeline processing to be applied to the target (downsampled) index. Some pipeline scripts might fail due to the differences in mappings and settings between the downsampling source and target index. --- docs/changelog/96494.yaml | 6 ++ x-pack/plugin/rollup/qa/rest/build.gradle | 2 +- .../rest-api-spec/test/rollup/60_settings.yml | 100 ++++++++++++++++++ .../downsample/TransportDownsampleAction.java | 16 ++- 4 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/96494.yaml create mode 100644 x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml diff --git a/docs/changelog/96494.yaml b/docs/changelog/96494.yaml new file mode 100644 index 0000000000000..9f2b58c5b4044 --- /dev/null +++ b/docs/changelog/96494.yaml @@ -0,0 +1,6 @@ +pr: 96494 +summary: Do not copy `index.default_pipeline` and `index.final_pipeline` +area: Rollup +type: bug +issues: + - 96478 diff --git a/x-pack/plugin/rollup/qa/rest/build.gradle b/x-pack/plugin/rollup/qa/rest/build.gradle index e91e56d2d1ba4..1cffa7fcafc6a 100644 --- a/x-pack/plugin/rollup/qa/rest/build.gradle +++ b/x-pack/plugin/rollup/qa/rest/build.gradle @@ -17,7 +17,7 @@ dependencies { restResources { restApi { - include '_common', 'bulk', 'cluster', 'indices', 'search' + include '_common', 'bulk', 'cluster', 'indices', 'search', 'ingest.put_pipeline', 'ingest.delete_pipeline' } } diff --git a/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml new file mode 100644 index 0000000000000..7b1b6d6a2134b --- /dev/null +++ b/x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml @@ -0,0 +1,100 @@ +--- +"Downsample index with pipeline": + - skip: + version: " - 8.4.99" + reason: "rollup renamed to downsample in 8.5.0" + + - do: + ingest.put_pipeline: + id: "pipeline" + body: > + { + "processors": [ + { + "set" : { + "field" : "field", + "value": 42 + } + } + ] + } + + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index: + default_pipeline: "pipeline" + final_pipeline: "pipeline" + mode: time_series + routing_path: [metricset, k8s.pod.uid] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + value: + type: double + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "value": 10}' + + - do: + search: + index: test + body: + query: + match_all: {} + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.field: 42 } + + - do: + indices.put_settings: + index: test + body: + index.blocks.write: true + + - do: + indices.downsample: + index: test + target_index: test-rollup + body: > + { + "fixed_interval": "1h" + } + + - do: + indices.get_settings: + index: test + + - match: { test.settings.index.default_pipeline: "pipeline" } + - match: { test.settings.index.final_pipeline: "pipeline" } + + - do: + indices.get_settings: + index: test-rollup + + - match: { test-rollup.settings.index.default_pipeline: null } + - match: { test-rollup.settings.index.final_pipeline: null } + + +--- +teardown: + - do: + ingest.delete_pipeline: + id: "pipeline" + ignore: 404 diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 3f71701d3ae17..d175f41b561ef 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -76,6 +76,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_METRIC_PARAM; @@ -97,6 +98,12 @@ public class TransportDownsampleAction extends AcknowledgedTransportMasterNodeAc private final IndexScopedSettings indexScopedSettings; private final ThreadContext threadContext; + private static final Set FORBIDDEN_SETTINGS = Set.of( + IndexSettings.DEFAULT_PIPELINE.getKey(), + IndexSettings.FINAL_PIPELINE.getKey(), + IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey() + ); + /** * This is the cluster state task executor for cluster state update actions. */ @@ -584,12 +591,15 @@ private IndexMetadata.Builder copyIndexMetadata(IndexMetadata sourceIndexMetadat // the same rules with resize apply continue; } + // Do not copy index settings which are valid for the source index but not for the target index + if (FORBIDDEN_SETTINGS.contains(key)) { + continue; + } // Do not override settings that have already been set in the rollup index. - // Also, we don't want to copy the `index.block.write` setting that we know - // it is set in the source index settings. - if (IndexMetadata.SETTING_BLOCKS_WRITE.equals(key) || targetSettings.keys().contains(key)) { + if (targetSettings.keys().contains(key)) { continue; } + targetSettings.copy(key, sourceIndexMetadata.getSettings()); }