From 78734b936e0e0562f415c0fc72eb9397bb2eb01b Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 10 Oct 2023 11:41:08 +0300 Subject: [PATCH 01/16] Don't print synthetic source in mapping for bwc tests --- .../index/mapper/SourceFieldMapper.java | 29 ++++++++++--------- .../index/mapper/SourceFieldMapperTests.java | 8 +++++ .../query/SearchExecutionContextTests.java | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index c5d5dbec1ef15..3b88b508582a1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -105,16 +105,7 @@ public static class Builder extends MetadataFieldMapper.Builder { * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic * source mode. */ - private final Parameter mode = new Parameter<>( - "mode", - true, - () -> null, - (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), - m -> toType(m).enabled.explicit() ? null : toType(m).mode, - (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), - v -> v.toString().toLowerCase(Locale.ROOT) - ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) - .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured + private final Parameter mode; private final Parameter> includes = Parameter.stringArrayParam( "includes", false, @@ -128,9 +119,21 @@ public static class Builder extends MetadataFieldMapper.Builder { private final IndexMode indexMode; - public Builder(IndexMode indexMode) { + public Builder(IndexMode indexMode, IndexVersion indexVersion) { super(Defaults.NAME); this.indexMode = indexMode; + this.mode = new Parameter<>( + "mode", + true, + () -> getIndexMode() == IndexMode.TIME_SERIES && indexVersion.between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) + ? Mode.SYNTHETIC + : null, + (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), + m -> toType(m).enabled.explicit() ? null : toType(m).mode, + (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), + v -> v.toString().toLowerCase(Locale.ROOT) + ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) + .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured } public Builder setSynthetic() { @@ -188,7 +191,7 @@ private IndexMode getIndexMode() { c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT : DEFAULT, - c -> new Builder(c.getIndexSettings().getMode()) + c -> new Builder(c.getIndexSettings().getMode(), c.getIndexSettings().getIndexVersionCreated()) ); static final class SourceFieldType extends MappedFieldType { @@ -313,7 +316,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode).init(this); + return new Builder(indexMode, IndexVersion.current()).init(this); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index f683cb60c87c3..433ebc467483d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -12,6 +12,8 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -238,4 +240,10 @@ public void testSyntheticSourceInTimeSeries() throws IOException { assertTrue(mapper.sourceMapper().isSynthetic()); assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString()); } + + public void testSyntheticSourceInTimeSeriesBwc() throws IOException { + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, IndexVersion.V_8_8_0).build(); + assertTrue(sourceMapper.isSynthetic()); + assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", sourceMapper.toString()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 6d671a258c26a..9df1dc24c2793 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -381,7 +381,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, IndexVersion.current()).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); From 89641ee7b0720e167cbd8895f6f568d88e66197d Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 10 Oct 2023 11:54:28 +0300 Subject: [PATCH 02/16] Move comment. --- .../org/elasticsearch/index/mapper/SourceFieldMapper.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 3b88b508582a1..aeab22a6f5f35 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -101,10 +101,6 @@ public static class Builder extends MetadataFieldMapper.Builder { (previous, current, conflicts) -> (previous.value() == current.value()) || (previous.value() && current.value() == false) ); - /* - * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic - * source mode. - */ private final Parameter mode; private final Parameter> includes = Parameter.stringArrayParam( "includes", @@ -125,6 +121,7 @@ public Builder(IndexMode indexMode, IndexVersion indexVersion) { this.mode = new Parameter<>( "mode", true, + // The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic source mode. () -> getIndexMode() == IndexMode.TIME_SERIES && indexVersion.between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) ? Mode.SYNTHETIC : null, From 44e815635e2565c0b042cfe558a7451226c89488 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 10 Oct 2023 15:06:09 +0300 Subject: [PATCH 03/16] Don't print synthetic source in mapping for bwc tests #2 --- .../index/mapper/SourceFieldMapper.java | 38 +++++++++++++++---- .../index/mapper/SourceFieldMapperTests.java | 5 +-- .../query/SearchExecutionContextTests.java | 2 +- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index aeab22a6f5f35..a98c78409d951 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -114,17 +114,17 @@ public static class Builder extends MetadataFieldMapper.Builder { ); private final IndexMode indexMode; + private final boolean setDefaultSytheticMode; - public Builder(IndexMode indexMode, IndexVersion indexVersion) { + public Builder(IndexMode indexMode, boolean setDefaultSytheticMode) { super(Defaults.NAME); this.indexMode = indexMode; + this.setDefaultSytheticMode = setDefaultSytheticMode; this.mode = new Parameter<>( "mode", true, // The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic source mode. - () -> getIndexMode() == IndexMode.TIME_SERIES && indexVersion.between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) - ? Mode.SYNTHETIC - : null, + () -> getIndexMode() == IndexMode.TIME_SERIES && setDefaultSytheticMode ? Mode.SYNTHETIC : null, (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), m -> toType(m).enabled.explicit() ? null : toType(m).mode, (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), @@ -171,7 +171,8 @@ public SourceFieldMapper build() { enabled.get(), includes.getValue().toArray(String[]::new), excludes.getValue().toArray(String[]::new), - indexMode + indexMode, + setDefaultSytheticMode ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -188,7 +189,10 @@ private IndexMode getIndexMode() { c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT : DEFAULT, - c -> new Builder(c.getIndexSettings().getMode(), c.getIndexSettings().getIndexVersionCreated()) + c -> new Builder( + c.getIndexSettings().getMode(), + c.getIndexSettings().getIndexVersionCreated().between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) + ) ); static final class SourceFieldType extends MappedFieldType { @@ -230,8 +234,25 @@ public Query termQuery(Object value, SearchExecutionContext context) { private final SourceFilter sourceFilter; private final IndexMode indexMode; + private final boolean setDefaultSytheticMode; + + private SourceFieldMapper( + Mode mode, + Explicit enabled, + String[] includes, + String[] excludes, + IndexMode indexMode) { + this(mode, enabled, includes, excludes, indexMode, false); + } - private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { + private SourceFieldMapper( + Mode mode, + Explicit enabled, + String[] includes, + String[] excludes, + IndexMode indexMode, + boolean setDefaultSytheticMode + ) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; this.mode = mode; @@ -244,6 +265,7 @@ private SourceFieldMapper(Mode mode, Explicit enabled, String[] include } this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; + this.setDefaultSytheticMode = setDefaultSytheticMode; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -313,7 +335,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, IndexVersion.current()).init(this); + return new Builder(indexMode, setDefaultSytheticMode).init(this); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 433ebc467483d..2655e252aebfa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -242,8 +241,8 @@ public void testSyntheticSourceInTimeSeries() throws IOException { } public void testSyntheticSourceInTimeSeriesBwc() throws IOException { - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, IndexVersion.V_8_8_0).build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, true).build(); assertTrue(sourceMapper.isSynthetic()); - assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", sourceMapper.toString()); + assertEquals("{}", sourceMapper.toString()); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 9df1dc24c2793..253045fd15098 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -381,7 +381,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, IndexVersion.current()).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, false).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); From 034262c5d22229aa6e8a0b7e754fd806a521cfc4 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 10 Oct 2023 15:06:55 +0300 Subject: [PATCH 04/16] Don't print synthetic source in mapping for bwc tests #2 --- .../org/elasticsearch/index/mapper/SourceFieldMapper.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index a98c78409d951..9b25248f0a2cc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -236,12 +236,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { private final IndexMode indexMode; private final boolean setDefaultSytheticMode; - private SourceFieldMapper( - Mode mode, - Explicit enabled, - String[] includes, - String[] excludes, - IndexMode indexMode) { + private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { this(mode, enabled, includes, excludes, indexMode, false); } From 123c296a6fb8318b45228c9a3924cf37e6083f12 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 11 Oct 2023 10:37:35 +0300 Subject: [PATCH 05/16] Revert "Don't print synthetic source in mapping for bwc tests #2" This reverts commit 034262c5d22229aa6e8a0b7e754fd806a521cfc4. --- .../org/elasticsearch/index/mapper/SourceFieldMapper.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 9b25248f0a2cc..a98c78409d951 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -236,7 +236,12 @@ public Query termQuery(Object value, SearchExecutionContext context) { private final IndexMode indexMode; private final boolean setDefaultSytheticMode; - private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { + private SourceFieldMapper( + Mode mode, + Explicit enabled, + String[] includes, + String[] excludes, + IndexMode indexMode) { this(mode, enabled, includes, excludes, indexMode, false); } From c9d380b5acd15ec13314f9c9ad56b9ca4356a34c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 11 Oct 2023 10:37:41 +0300 Subject: [PATCH 06/16] Revert "Don't print synthetic source in mapping for bwc tests #2" This reverts commit 44e815635e2565c0b042cfe558a7451226c89488. --- .../index/mapper/SourceFieldMapper.java | 38 ++++--------------- .../index/mapper/SourceFieldMapperTests.java | 5 ++- .../query/SearchExecutionContextTests.java | 2 +- 3 files changed, 12 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index a98c78409d951..aeab22a6f5f35 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -114,17 +114,17 @@ public static class Builder extends MetadataFieldMapper.Builder { ); private final IndexMode indexMode; - private final boolean setDefaultSytheticMode; - public Builder(IndexMode indexMode, boolean setDefaultSytheticMode) { + public Builder(IndexMode indexMode, IndexVersion indexVersion) { super(Defaults.NAME); this.indexMode = indexMode; - this.setDefaultSytheticMode = setDefaultSytheticMode; this.mode = new Parameter<>( "mode", true, // The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic source mode. - () -> getIndexMode() == IndexMode.TIME_SERIES && setDefaultSytheticMode ? Mode.SYNTHETIC : null, + () -> getIndexMode() == IndexMode.TIME_SERIES && indexVersion.between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) + ? Mode.SYNTHETIC + : null, (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), m -> toType(m).enabled.explicit() ? null : toType(m).mode, (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), @@ -171,8 +171,7 @@ public SourceFieldMapper build() { enabled.get(), includes.getValue().toArray(String[]::new), excludes.getValue().toArray(String[]::new), - indexMode, - setDefaultSytheticMode + indexMode ); if (indexMode != null) { indexMode.validateSourceFieldMapper(sourceFieldMapper); @@ -189,10 +188,7 @@ private IndexMode getIndexMode() { c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT : DEFAULT, - c -> new Builder( - c.getIndexSettings().getMode(), - c.getIndexSettings().getIndexVersionCreated().between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) - ) + c -> new Builder(c.getIndexSettings().getMode(), c.getIndexSettings().getIndexVersionCreated()) ); static final class SourceFieldType extends MappedFieldType { @@ -234,25 +230,8 @@ public Query termQuery(Object value, SearchExecutionContext context) { private final SourceFilter sourceFilter; private final IndexMode indexMode; - private final boolean setDefaultSytheticMode; - - private SourceFieldMapper( - Mode mode, - Explicit enabled, - String[] includes, - String[] excludes, - IndexMode indexMode) { - this(mode, enabled, includes, excludes, indexMode, false); - } - private SourceFieldMapper( - Mode mode, - Explicit enabled, - String[] includes, - String[] excludes, - IndexMode indexMode, - boolean setDefaultSytheticMode - ) { + private SourceFieldMapper(Mode mode, Explicit enabled, String[] includes, String[] excludes, IndexMode indexMode) { super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED))); assert enabled.explicit() == false || mode == null; this.mode = mode; @@ -265,7 +244,6 @@ private SourceFieldMapper( } this.complete = stored() && sourceFilter == null; this.indexMode = indexMode; - this.setDefaultSytheticMode = setDefaultSytheticMode; } private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) { @@ -335,7 +313,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, setDefaultSytheticMode).init(this); + return new Builder(indexMode, IndexVersion.current()).init(this); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 2655e252aebfa..433ebc467483d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -241,8 +242,8 @@ public void testSyntheticSourceInTimeSeries() throws IOException { } public void testSyntheticSourceInTimeSeriesBwc() throws IOException { - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, true).build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, IndexVersion.V_8_8_0).build(); assertTrue(sourceMapper.isSynthetic()); - assertEquals("{}", sourceMapper.toString()); + assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", sourceMapper.toString()); } } diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 253045fd15098..9df1dc24c2793 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -381,7 +381,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, false).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, IndexVersion.current()).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); From 9b4c2e5e461bcf49298332ad755ac556e2c00851 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 11 Oct 2023 10:37:49 +0300 Subject: [PATCH 07/16] Revert "Don't print synthetic source in mapping for bwc tests (#100572)" This reverts commit 9322ab9b9163f70c9bf832f1b0a1985121393cfe. --- .../index/mapper/SourceFieldMapper.java | 34 +++++++++---------- .../index/mapper/SourceFieldMapperTests.java | 8 ----- .../query/SearchExecutionContextTests.java | 2 +- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index aeab22a6f5f35..c5d5dbec1ef15 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -101,7 +101,20 @@ public static class Builder extends MetadataFieldMapper.Builder { (previous, current, conflicts) -> (previous.value() == current.value()) || (previous.value() && current.value() == false) ); - private final Parameter mode; + /* + * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic + * source mode. + */ + private final Parameter mode = new Parameter<>( + "mode", + true, + () -> null, + (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), + m -> toType(m).enabled.explicit() ? null : toType(m).mode, + (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), + v -> v.toString().toLowerCase(Locale.ROOT) + ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) + .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured private final Parameter> includes = Parameter.stringArrayParam( "includes", false, @@ -115,22 +128,9 @@ public static class Builder extends MetadataFieldMapper.Builder { private final IndexMode indexMode; - public Builder(IndexMode indexMode, IndexVersion indexVersion) { + public Builder(IndexMode indexMode) { super(Defaults.NAME); this.indexMode = indexMode; - this.mode = new Parameter<>( - "mode", - true, - // The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic source mode. - () -> getIndexMode() == IndexMode.TIME_SERIES && indexVersion.between(IndexVersion.V_8_7_0, IndexVersion.V_8_10_0) - ? Mode.SYNTHETIC - : null, - (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)), - m -> toType(m).enabled.explicit() ? null : toType(m).mode, - (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)), - v -> v.toString().toLowerCase(Locale.ROOT) - ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED) - .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured } public Builder setSynthetic() { @@ -188,7 +188,7 @@ private IndexMode getIndexMode() { c -> c.getIndexSettings().getMode() == IndexMode.TIME_SERIES ? c.getIndexSettings().getIndexVersionCreated().onOrAfter(IndexVersion.V_8_7_0) ? TSDB_DEFAULT : TSDB_LEGACY_DEFAULT : DEFAULT, - c -> new Builder(c.getIndexSettings().getMode(), c.getIndexSettings().getIndexVersionCreated()) + c -> new Builder(c.getIndexSettings().getMode()) ); static final class SourceFieldType extends MappedFieldType { @@ -313,7 +313,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(indexMode, IndexVersion.current()).init(this); + return new Builder(indexMode).init(this); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index 433ebc467483d..f683cb60c87c3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -12,8 +12,6 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentParser; @@ -240,10 +238,4 @@ public void testSyntheticSourceInTimeSeries() throws IOException { assertTrue(mapper.sourceMapper().isSynthetic()); assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString()); } - - public void testSyntheticSourceInTimeSeriesBwc() throws IOException { - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(IndexMode.TIME_SERIES, IndexVersion.V_8_8_0).build(); - assertTrue(sourceMapper.isSynthetic()); - assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", sourceMapper.toString()); - } } diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 9df1dc24c2793..6d671a258c26a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -381,7 +381,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { public void testSyntheticSourceSearchLookup() throws IOException { // Build a mapping using synthetic source - SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, IndexVersion.current()).setSynthetic().build(); + SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add( new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); From 5be3557b47f7e57f6a08fe770e81d2652180d5fa Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 11 Oct 2023 10:44:40 +0300 Subject: [PATCH 08/16] Exclude synthetic source test from mixedClusterTests --- qa/mixed-cluster/build.gradle | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 08d64e2b9353b..68be7c5dd681d 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -41,6 +41,13 @@ excludeList.add('aggregations/filter/Standard queries get cached') excludeList.add('aggregations/filter/Terms lookup gets cached') excludeList.add('aggregations/filters_bucket/cache hits') +// The test checks that tsdb mappings report source as synthetic. +// This was broken between 8.7 (synthetic source became the defaul for +// tsdb) and 8.10 (fix ported to this version). The test doesn't +// add much in mixed cluster setting and may be contributing to flaky +// assertion errors like #100562. +excludeList.add('tsdb/20_mapping/Synthetic source') + BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { From 795c76719648081e99e9dcc12fd9fb6362f2e62d Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 11 Oct 2023 13:21:10 +0300 Subject: [PATCH 09/16] Update comment. --- qa/mixed-cluster/build.gradle | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 68be7c5dd681d..13256179b0a2b 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -42,10 +42,9 @@ excludeList.add('aggregations/filter/Terms lookup gets cached') excludeList.add('aggregations/filters_bucket/cache hits') // The test checks that tsdb mappings report source as synthetic. -// This was broken between 8.7 (synthetic source became the defaul for -// tsdb) and 8.10 (fix ported to this version). The test doesn't -// add much in mixed cluster setting and may be contributing to flaky -// assertion errors like #100562. +// It is supposed to be skipped (not needed) for versions before +// 8.10 but mixed cluster tests may not respect that - see the +// comment above. excludeList.add('tsdb/20_mapping/Synthetic source') BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> From a99951a8c9124e4e7f936e6c7e5e12ad9c37754a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 13 Oct 2023 11:25:30 +0300 Subject: [PATCH 10/16] Mute all tsdb tests in mixedClusterTests This is an interim step to stop sporadic test failures, while we try to fix version skip for mixed cluster tests. --- qa/mixed-cluster/build.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 13256179b0a2b..96c4f5f831b18 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -45,7 +45,9 @@ excludeList.add('aggregations/filters_bucket/cache hits') // It is supposed to be skipped (not needed) for versions before // 8.10 but mixed cluster tests may not respect that - see the // comment above. -excludeList.add('tsdb/20_mapping/Synthetic source') +// Tentatively disable all tsdb tests to avoid flakiness until +// version skip works properly for mixed cluster tests. +excludeList.add('tsdb/*/* source') BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> From 4f0e44f769a465a1f721502186d3ffcd0e874a81 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 13 Oct 2023 11:29:26 +0300 Subject: [PATCH 11/16] Remove old exclusion --- qa/mixed-cluster/build.gradle | 6 ------ 1 file changed, 6 deletions(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 8ae559b40116c..96c4f5f831b18 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -49,12 +49,6 @@ excludeList.add('aggregations/filters_bucket/cache hits') // version skip works properly for mixed cluster tests. excludeList.add('tsdb/*/* source') -// The test checks that tsdb mappings report source as synthetic. -// It is supposed to be skipped (not needed) for versions before -// 8.10 but mixed cluster tests may not respect that - see the -// comment above. -excludeList.add('tsdb/20_mapping/Synthetic source') - BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { From b41cac2f24b0bf51ff88f350fe8785f02bf4ecf6 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 13 Oct 2023 11:35:36 +0300 Subject: [PATCH 12/16] Add aggregations too --- qa/mixed-cluster/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 96c4f5f831b18..4483f3a363e2d 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -47,7 +47,8 @@ excludeList.add('aggregations/filters_bucket/cache hits') // comment above. // Tentatively disable all tsdb tests to avoid flakiness until // version skip works properly for mixed cluster tests. -excludeList.add('tsdb/*/* source') +excludeList.add('tsdb/*/*') +excludeList.add('aggregations/time_series/*') BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> From 1a75591ee10273b01d0305cc98461a3b14939358 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 13 Oct 2023 12:21:59 +0300 Subject: [PATCH 13/16] Mute tests for versions between 8.7-8.10 --- .../rest-api-spec/test/aggregations/time_series.yml | 5 +++++ qa/mixed-cluster/build.gradle | 1 - .../05_dimension_and_metric_in_non_tsdb_index.yml | 7 +++++++ .../rest-api-spec/test/tsdb/100_composite.yml | 5 +++-- .../resources/rest-api-spec/test/tsdb/10_settings.yml | 7 +++++++ .../rest-api-spec/test/tsdb/110_field_caps.yml | 5 +++-- .../rest-api-spec/test/tsdb/130_position_fields.yml | 5 +++-- .../rest-api-spec/test/tsdb/140_routing_path.yml | 8 ++++++++ .../rest-api-spec/test/tsdb/15_timestamp_mapping.yml | 5 +++++ .../resources/rest-api-spec/test/tsdb/20_mapping.yml | 11 +++++++---- .../rest-api-spec/test/tsdb/25_id_generation.yml | 5 +++-- .../resources/rest-api-spec/test/tsdb/30_snapshot.yml | 4 ++++ .../resources/rest-api-spec/test/tsdb/40_search.yml | 5 +++-- .../resources/rest-api-spec/test/tsdb/50_alias.yml | 5 +++-- .../rest-api-spec/test/tsdb/60_add_dimensions.yml | 6 ++++++ .../rest-api-spec/test/tsdb/70_dimension_types.yml | 7 +++++++ .../rest-api-spec/test/tsdb/80_index_resize.yml | 5 +++-- .../test/tsdb/90_unsupported_operations.yml | 5 +++-- 18 files changed, 80 insertions(+), 21 deletions(-) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml index 26c9b32a2f7a1..33ca0dc4779b6 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/time_series.yml @@ -1,4 +1,9 @@ +--- setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + - do: indices.create: index: tsdb diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 4483f3a363e2d..64eebef34eb2f 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -47,7 +47,6 @@ excludeList.add('aggregations/filters_bucket/cache hits') // comment above. // Tentatively disable all tsdb tests to avoid flakiness until // version skip works properly for mixed cluster tests. -excludeList.add('tsdb/*/*') excludeList.add('aggregations/time_series/*') BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml index a8c587f5c89cc..598919d75047d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/05_dimension_and_metric_in_non_tsdb_index.yml @@ -1,3 +1,10 @@ +--- +setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + +--- add time series mappings: - skip: version: " - 7.15.99" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml index 600ee9868907a..19d3defba1a4a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/100_composite.yml @@ -1,7 +1,8 @@ +--- setup: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.1.99,8.7.00 - 8.9.99" + reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml index d19ae05f2e402..922e0f4969669 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/10_settings.yml @@ -1,3 +1,10 @@ +--- +setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + +--- enable: - skip: version: " - 8.0.99" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/110_field_caps.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/110_field_caps.yml index 7e85066192454..03d8d952136e3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/110_field_caps.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/110_field_caps.yml @@ -1,7 +1,8 @@ +--- setup: - skip: - version: " - 8.4.99" - reason: metric params only on time series indexes introduced in 8.5.0 + version: " - 8.4.99,8.7.00 - 8.9.99" + reason: "metric params only on time series indexes introduced in 8.5.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml index 89e8ebb3e24ea..fa7a5ae1449a9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml @@ -1,8 +1,9 @@ --- setup: - skip: - version: " - 8.7.99" - reason: position metric introduced in 8.8.0 + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + - do: indices.create: index: locations diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 9ba89f803bf15..deaba75e5476a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -1,3 +1,11 @@ +--- +setup: + - skip: + version: " - 8.9.99" + reason: "counter field support added in 8.10" + features: close_to + +--- missing routing path field: - skip: features: close_to diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 21b5399b0eaad..1b61785ba3134 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -1,3 +1,8 @@ +--- +setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" --- date: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 21f30b6828f36..e4963832ddce3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -1,3 +1,10 @@ +--- +setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + +--- ecs style: - skip: version: " - 8.0.99" @@ -417,10 +424,6 @@ nested fields: --- "Synthetic source": - - skip: - version: " - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on" - - do: indices.create: index: tsdb-synthetic diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index cbf11c0b1d250..5b9ece81155e5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -1,7 +1,8 @@ +--- setup: - skip: - version: " - 8.1.99" - reason: id generation changed in 8.2 + version: " - 8.1.99,8.7.00 - 8.9.99" + reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml index 1e6dd9243fbe3..2ae4afa604170 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/30_snapshot.yml @@ -1,5 +1,9 @@ --- setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + - do: snapshot.create_repository: repository: test_repo diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index 48f650a05d6f9..23f08170d4f69 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -1,7 +1,8 @@ +--- setup: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.1.99,8.7.00 - 8.9.99" + reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml index 0035f9be3cfda..128ba3fe3e464 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/50_alias.yml @@ -1,7 +1,8 @@ +--- setup: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.1.99,8.7.00 - 8.9.99" + reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml index 2c151b9b7139f..969df4381bc0c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/60_add_dimensions.yml @@ -1,3 +1,9 @@ +--- +setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + --- add dimensions with put_mapping: - skip: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml index 2fb3209f6b845..f6c5ce2404a2a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/70_dimension_types.yml @@ -1,3 +1,10 @@ +--- +setup: + - skip: + version: "8.7.00 - 8.9.99" + reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + +--- keyword dimension: - skip: features: close_to diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml index f78f16780cb4f..9720ead43fd98 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/80_index_resize.yml @@ -1,7 +1,8 @@ +--- setup: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.1.99,8.7.00 - 8.9.99" + reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" features: "arbitrary_key" # Force allocating all shards to a single node so that we can shrink later. diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml index 7cb6af779623f..049b9670b6b46 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml @@ -1,7 +1,8 @@ +--- setup: - skip: - version: " - 8.1.99" - reason: tsdb indexing changed in 8.2.0 + version: " - 8.1.99,8.7.00 - 8.9.99" + reason: "tsdb indexing changed in 8.2.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: From 468c1c755deadf8671bdf33258584e27e24f88b8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 13 Oct 2023 12:23:10 +0300 Subject: [PATCH 14/16] Remove mute --- qa/mixed-cluster/build.gradle | 8 -------- 1 file changed, 8 deletions(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 64eebef34eb2f..08d64e2b9353b 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -41,14 +41,6 @@ excludeList.add('aggregations/filter/Standard queries get cached') excludeList.add('aggregations/filter/Terms lookup gets cached') excludeList.add('aggregations/filters_bucket/cache hits') -// The test checks that tsdb mappings report source as synthetic. -// It is supposed to be skipped (not needed) for versions before -// 8.10 but mixed cluster tests may not respect that - see the -// comment above. -// Tentatively disable all tsdb tests to avoid flakiness until -// version skip works properly for mixed cluster tests. -excludeList.add('aggregations/time_series/*') - BuildParams.bwcVersions.withWireCompatible { bwcVersion, baseName -> if (bwcVersion != VersionProperties.getElasticsearchVersion()) { From cef2533bb08949413b1cc702606aea61ca482b37 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 13 Oct 2023 14:11:38 +0300 Subject: [PATCH 15/16] Restore version skipping for position fields --- .../resources/rest-api-spec/test/tsdb/130_position_fields.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml index fa7a5ae1449a9..cb897c918305e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/130_position_fields.yml @@ -1,8 +1,8 @@ --- setup: - skip: - version: "8.7.00 - 8.9.99" - reason: "Synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" + version: " - 8.9.99" + reason: "position metric introduced in 8.8.0, synthetic source shows up in the mapping in 8.10 and on, may trigger assert failures in mixed cluster tests" - do: indices.create: From 29e794f16b3e27239b3e5e8283347c0adca5c8a8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 13 Oct 2023 15:08:25 +0300 Subject: [PATCH 16/16] Restore version skip for synthetic source --- .../resources/rest-api-spec/test/tsdb/20_mapping.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index e4963832ddce3..3d297e2181970 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -424,6 +424,10 @@ nested fields: --- "Synthetic source": + - skip: + version: " - 8.9.99" + reason: Synthetic source shows up in the mapping in 8.10 + - do: indices.create: index: tsdb-synthetic