From 27c84c1e50b39be5bd1b205606cf5dbf342b36e0 Mon Sep 17 00:00:00 2001 From: liketic Date: Fri, 30 Mar 2018 13:26:02 +0800 Subject: [PATCH 1/4] Use date format in mapping before fallback to default if not force format (#29282) --- .../index/mapper/RangeFieldMapper.java | 3 ++ .../index/mapper/RangeFieldTypeTests.java | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 1536db6510fc7..5959a817f256f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -292,6 +292,9 @@ public Query termQuery(Object value, QueryShardContext context) { public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, ShapeRelation relation, DateTimeZone timeZone, DateMathParser parser, QueryShardContext context) { failIfNotIndexed(); + if (parser == null) { + parser = dateMathParser(); + } return rangeType.rangeQuery(name(), hasDocValues(), lowerTerm, upperTerm, includeLower, includeUpper, relation, timeZone, parser, context); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 0c20153675af6..067f98c72a89d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -34,6 +34,8 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.joda.DateMathParser; +import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; @@ -98,6 +100,33 @@ public void testRangeQuery() throws Exception { ft.rangeQuery(from, to, includeLower, includeUpper, relation, null, null, context)); } + public void testRangeQueryUsingMappingFormat() throws Exception { + Settings indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); + QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), + writableRegistry(), null, null, () -> nowInMillis, null); + + Version version = randomFrom(Version.V_5_0_0_alpha1, Version.V_6_0_0_beta1, Version.CURRENT); + RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(RangeType.DATE, version); + ft.setName(FIELDNAME); + ft.setIndexOptions(IndexOptions.DOCS); + + FormatDateTimeFormatter formatter = Joda.forPattern( "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"); + ft.setDateTimeFormatter(formatter); + ShapeRelation relation = RandomPicks.randomFrom(random(), ShapeRelation.values()); + boolean includeLower = random().nextBoolean(); + boolean includeUpper = random().nextBoolean(); + Object from = nextFrom(); + Object to = nextTo(from); + + boolean hasDocValues = version.onOrAfter(Version.V_6_0_0_beta1); + assertEquals( + RangeType.DATE.rangeQuery(FIELDNAME, hasDocValues, from, to, includeLower, + includeUpper, relation, null, ft.dateMathParser(), context), + ft.rangeQuery(from, to, includeLower, includeUpper, relation, null, ft.dateMathParser(), context)); + } + private Query getExpectedRangeQuery(ShapeRelation relation, Object from, Object to, boolean includeLower, boolean includeUpper) { switch (type) { case DATE: From 9e0c04e397df0eb715e032c34d9eeb92f11190b4 Mon Sep 17 00:00:00 2001 From: liketic Date: Fri, 6 Apr 2018 21:04:05 +0800 Subject: [PATCH 2/4] Fix test --- .../index/mapper/RangeFieldTypeTests.java | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 067f98c72a89d..ee192849d3d24 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.RangeFieldMapper.RangeFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper.RangeType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.IndexSettingsModule; @@ -63,21 +64,21 @@ public void setupProperties() { addModifier(new Modifier("format", true) { @Override public void modify(MappedFieldType ft) { - ((RangeFieldMapper.RangeFieldType) ft).setDateTimeFormatter(Joda.forPattern("basic_week_date", Locale.ROOT)); + ((RangeFieldType) ft).setDateTimeFormatter(Joda.forPattern("basic_week_date", Locale.ROOT)); } }); addModifier(new Modifier("locale", true) { @Override public void modify(MappedFieldType ft) { - ((RangeFieldMapper.RangeFieldType) ft).setDateTimeFormatter(Joda.forPattern("date_optional_time", Locale.CANADA)); + ((RangeFieldType) ft).setDateTimeFormatter(Joda.forPattern("date_optional_time", Locale.CANADA)); } }); } } @Override - protected RangeFieldMapper.RangeFieldType createDefaultFieldType() { - return new RangeFieldMapper.RangeFieldType(type, Version.CURRENT); + protected RangeFieldType createDefaultFieldType() { + return new RangeFieldType(type, Version.CURRENT); } public void testRangeQuery() throws Exception { @@ -86,13 +87,13 @@ public void testRangeQuery() throws Exception { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null); - RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type, Version.CURRENT); + RangeFieldType ft = new RangeFieldType(type, Version.CURRENT); ft.setName(FIELDNAME); ft.setIndexOptions(IndexOptions.DOCS); - ShapeRelation relation = RandomPicks.randomFrom(random(), ShapeRelation.values()); - boolean includeLower = random().nextBoolean(); - boolean includeUpper = random().nextBoolean(); + ShapeRelation relation = randomFrom(ShapeRelation.values()); + boolean includeLower = randomBoolean(); + boolean includeUpper = randomBoolean(); Object from = nextFrom(); Object to = nextTo(from); @@ -100,31 +101,34 @@ public void testRangeQuery() throws Exception { ft.rangeQuery(from, to, includeLower, includeUpper, relation, null, null, context)); } - public void testRangeQueryUsingMappingFormat() throws Exception { + public void testRangeQueryUseDateFormatInMapping() throws Exception { Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", indexSettings); QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null); - Version version = randomFrom(Version.V_5_0_0_alpha1, Version.V_6_0_0_beta1, Version.CURRENT); - RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(RangeType.DATE, version); - ft.setName(FIELDNAME); - ft.setIndexOptions(IndexOptions.DOCS); + RangeFieldType fieldType = new RangeFieldType(RangeType.DATE, Version.CURRENT); + fieldType.setName(FIELDNAME); + fieldType.setIndexOptions(IndexOptions.DOCS); + ShapeRelation relation = randomFrom(ShapeRelation.values()); + DateTime from = DateTime.now(); + DateTime to = DateTime.now().plusDays(DISTANCE); - FormatDateTimeFormatter formatter = Joda.forPattern( "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"); - ft.setDateTimeFormatter(formatter); - ShapeRelation relation = RandomPicks.randomFrom(random(), ShapeRelation.values()); - boolean includeLower = random().nextBoolean(); - boolean includeUpper = random().nextBoolean(); - Object from = nextFrom(); - Object to = nextTo(from); + // Test default date format + DateMathParser defaultParser = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER); + assertEquals( + RangeType.DATE.rangeQuery(FIELDNAME, true, from, to, true, + true, relation, null, defaultParser, context), + fieldType.rangeQuery(from, to, true, true, relation, null, null, context)); - boolean hasDocValues = version.onOrAfter(Version.V_6_0_0_beta1); + // Test using date format in mapping + FormatDateTimeFormatter formatter = Joda.forPattern( "strict_date_optional_time||epoch_millis"); + fieldType.setDateTimeFormatter(formatter); assertEquals( - RangeType.DATE.rangeQuery(FIELDNAME, hasDocValues, from, to, includeLower, - includeUpper, relation, null, ft.dateMathParser(), context), - ft.rangeQuery(from, to, includeLower, includeUpper, relation, null, ft.dateMathParser(), context)); + RangeType.DATE.rangeQuery(FIELDNAME, true, from, to, true, + true, relation, null, fieldType.dateMathParser(), context), + fieldType.rangeQuery(from, to, true, true, relation, null, null, context)); } private Query getExpectedRangeQuery(ShapeRelation relation, Object from, Object to, boolean includeLower, boolean includeUpper) { @@ -306,14 +310,14 @@ public void testParseIp() { assertEquals(InetAddresses.forString("::1"), RangeFieldMapper.RangeType.IP.parse(new BytesRef("::1"), randomBoolean())); } - public void testTermQuery() throws Exception, IllegalArgumentException { + public void testTermQuery() throws Exception { // See https://github.com/elastic/elasticsearch/issues/25950 Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null); - RangeFieldMapper.RangeFieldType ft = new RangeFieldMapper.RangeFieldType(type, Version.CURRENT); + RangeFieldType ft = new RangeFieldType(type, Version.CURRENT); ft.setName(FIELDNAME); ft.setIndexOptions(IndexOptions.DOCS); From 3141e8834c6cb6b210cf92b73cc9e66336fc9ff2 Mon Sep 17 00:00:00 2001 From: liketic Date: Sun, 6 May 2018 12:43:50 +0800 Subject: [PATCH 3/4] Fix test --- .../index/mapper/RangeFieldTypeTests.java | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index ee192849d3d24..803ec60153d5e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.mapper; -import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.document.DoubleRange; import org.apache.lucene.document.FloatRange; import org.apache.lucene.document.InetAddressPoint; @@ -31,10 +30,10 @@ import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.joda.DateMathParser; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.network.InetAddresses; @@ -58,7 +57,7 @@ public class RangeFieldTypeTests extends FieldTypeTestCase { @Before public void setupProperties() { - type = RandomPicks.randomFrom(random(), RangeType.values()); + type = randomFrom(RangeType.values()); nowInMillis = randomNonNegativeLong(); if (type == RangeType.DATE) { addModifier(new Modifier("format", true) { @@ -82,11 +81,7 @@ protected RangeFieldType createDefaultFieldType() { } public void testRangeQuery() throws Exception { - Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); - QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), - writableRegistry(), null, null, () -> nowInMillis, null); + QueryShardContext context = createContext(); RangeFieldType ft = new RangeFieldType(type, Version.CURRENT); ft.setName(FIELDNAME); ft.setIndexOptions(IndexOptions.DOCS); @@ -101,34 +96,39 @@ public void testRangeQuery() throws Exception { ft.rangeQuery(from, to, includeLower, includeUpper, relation, null, null, context)); } - public void testRangeQueryUseDateFormatInMapping() throws Exception { + private QueryShardContext createContext() { Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", indexSettings); - QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); + return new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null); + } + public void testDateRangeQueryUsingMappingFormat() { + QueryShardContext context = createContext(); RangeFieldType fieldType = new RangeFieldType(RangeType.DATE, Version.CURRENT); fieldType.setName(FIELDNAME); fieldType.setIndexOptions(IndexOptions.DOCS); + fieldType.setHasDocValues(false); ShapeRelation relation = randomFrom(ShapeRelation.values()); - DateTime from = DateTime.now(); - DateTime to = DateTime.now().plusDays(DISTANCE); - // Test default date format - DateMathParser defaultParser = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER); - assertEquals( - RangeType.DATE.rangeQuery(FIELDNAME, true, from, to, true, - true, relation, null, defaultParser, context), - fieldType.rangeQuery(from, to, true, true, relation, null, null, context)); + // dates will break the default format + final String from = "2016-15-06T15:29:50+08:00"; + final String to = "2016-16-06T15:29:50+08:00"; + + ElasticsearchParseException ex = expectThrows(ElasticsearchParseException.class, + () -> fieldType.rangeQuery(from, to, true, true, relation, null, null, context)); + assertEquals("failed to parse date field [2016-15-06T15:29:50+08:00] with format [strict_date_optional_time||epoch_millis]", + ex.getMessage()); + + // setting mapping format which is compatible with those dates + final FormatDateTimeFormatter formatter = Joda.forPattern("yyyy-dd-MM'T'HH:mm:ssZZ"); + assertEquals(1465975790000L, formatter.parser().parseMillis(from)); + assertEquals(1466062190000L, formatter.parser().parseMillis(to)); - // Test using date format in mapping - FormatDateTimeFormatter formatter = Joda.forPattern( "strict_date_optional_time||epoch_millis"); fieldType.setDateTimeFormatter(formatter); - assertEquals( - RangeType.DATE.rangeQuery(FIELDNAME, true, from, to, true, - true, relation, null, fieldType.dateMathParser(), context), - fieldType.rangeQuery(from, to, true, true, relation, null, null, context)); + final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, null, context); + assertEquals("field:", query.toString()); } private Query getExpectedRangeQuery(ShapeRelation relation, Object from, Object to, boolean includeLower, boolean includeUpper) { @@ -312,11 +312,7 @@ public void testParseIp() { public void testTermQuery() throws Exception { // See https://github.com/elastic/elasticsearch/issues/25950 - Settings indexSettings = Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); - IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); - QueryShardContext context = new QueryShardContext(0, idxSettings, null, null, null, null, null, xContentRegistry(), - writableRegistry(), null, null, () -> nowInMillis, null); + QueryShardContext context = createContext(); RangeFieldType ft = new RangeFieldType(type, Version.CURRENT); ft.setName(FIELDNAME); ft.setIndexOptions(IndexOptions.DOCS); From 42c0ff8f1bd7233715eb8232f1397fde2f9a542d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 8 May 2018 14:39:39 +0200 Subject: [PATCH 4/4] Update changelog --- docs/CHANGELOG.asciidoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index 621ca5a6414d2..49558a409aa12 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -104,6 +104,8 @@ ones that the user is authorized to access in case field level security is enabl [float] === Bug Fixes +Use date format in `date_range` mapping before fallback to default ({pull}29310[#29310]) + Fixed prerelease version of elasticsearch in the `deb` package to sort before GA versions ({pull}29000[#29000]) @@ -169,6 +171,8 @@ Added put index template API to the high level rest client ({pull}30400[#30400]) [float] === Bug Fixes +Use date format in `date_range` mapping before fallback to default ({pull}29310[#29310]) + Do not ignore request analysis/similarity settings on index resize operations when the source index already contains such settings ({pull}30216[#30216]) Fix NPE when CumulativeSum agg encounters null value/empty bucket ({pull}29641[#29641])