From 479c0c78b797e14c28499fea030a39e361ea5b88 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 8 Feb 2019 15:41:00 -0500 Subject: [PATCH 01/12] Add fuzziness example (#37194) (#38648) The example in the Fuzziness section was actually relevant to the section above it, so I moved it there. I replaced it with an example of how to use the `fuzziness` parameter --- docs/reference/query-dsl/match-query.asciidoc | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc index 5c397d603bef3..b939364f12027 100644 --- a/docs/reference/query-dsl/match-query.asciidoc +++ b/docs/reference/query-dsl/match-query.asciidoc @@ -32,6 +32,25 @@ optional `should` clauses to match can be set using the <> parameter. +Here is an example when providing additional parameters (note the slight +change in structure, `message` is the field name): + +[source,js] +-------------------------------------------------- +GET /_search +{ + "query": { + "match" : { + "message" : { + "query" : "this is a test", + "operator" : "and" + } + } + } +} +-------------------------------------------------- +// CONSOLE + The `analyzer` can be set to control which analyzer will perform the analysis process on the text. It defaults to the field explicit mapping definition, or the default search analyzer. @@ -56,9 +75,6 @@ rewritten. Fuzzy transpositions (`ab` -> `ba`) are allowed by default but can be disabled by setting `fuzzy_transpositions` to `false`. -Here is an example when providing additional parameters (note the slight -change in structure, `message` is the field name): - [source,js] -------------------------------------------------- GET /_search @@ -66,8 +82,8 @@ GET /_search "query": { "match" : { "message" : { - "query" : "this is a test", - "operator" : "and" + "query" : "this is a testt", + "fuzziness": "AUTO" } } } From d253fb330cb96a6d3ccda68486fdb0a5de73a8d7 Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Fri, 8 Feb 2019 22:55:51 +0200 Subject: [PATCH 02/12] SQL: Replace joda with java time (#38437) Replace remaining usages of joda classes with java time. Fixes: #37703 --- .../xpack/sql/jdbc/TypeConverterTests.java | 17 +++--- .../search/extractor/FieldHitExtractor.java | 5 -- .../xpack/sql/parser/ExpressionBuilder.java | 33 ++++------ .../xpack/sql/type/DataTypeConversion.java | 5 +- .../xpack/sql/util/DateUtils.java | 61 ++++++------------- .../scalar/datetime/DateTimeTestUtils.java | 9 +-- .../sql/parser/EscapedFunctionsTests.java | 8 ++- .../sql/type/DataTypeConversionTests.java | 14 +++-- 8 files changed, 56 insertions(+), 96 deletions(-) diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java index 2e33f4e130741..588e3a6392e10 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java @@ -10,17 +10,18 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; -import org.joda.time.DateTime; -import org.joda.time.ReadableDateTime; import java.sql.Timestamp; +import java.time.Clock; +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZonedDateTime; import static org.hamcrest.Matchers.instanceOf; public class TypeConverterTests extends ESTestCase { - public void testFloatAsNative() throws Exception { assertThat(convertAsNative(42.0f, EsType.FLOAT), instanceOf(Float.class)); assertThat(convertAsNative(42.0, EsType.FLOAT), instanceOf(Float.class)); @@ -40,9 +41,9 @@ public void testDoubleAsNative() throws Exception { } public void testTimestampAsNative() throws Exception { - DateTime now = DateTime.now(); + ZonedDateTime now = ZonedDateTime.now(Clock.tick(Clock.system(ZoneId.of("Z")), Duration.ofMillis(1))); assertThat(convertAsNative(now, EsType.DATETIME), instanceOf(Timestamp.class)); - assertEquals(now.getMillis(), ((Timestamp) convertAsNative(now, EsType.DATETIME)).getTime()); + assertEquals(now.toInstant().toEpochMilli(), ((Timestamp) convertAsNative(now, EsType.DATETIME)).getTime()); } private Object convertAsNative(Object value, EsType type) throws Exception { @@ -50,11 +51,7 @@ private Object convertAsNative(Object value, EsType type) throws Exception { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); builder.field("value"); - if (value instanceof ReadableDateTime) { - builder.value(((ReadableDateTime) value).getMillis()); - } else { - builder.value(value); - } + builder.value(value); builder.endObject(); builder.close(); Object copy = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2().get("value"); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java index 589481247ac39..23747787b820b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java @@ -14,7 +14,6 @@ import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.DateUtils; -import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayDeque; @@ -132,10 +131,6 @@ private Object unwrapMultiValue(Object values) { if (values instanceof String) { return DateUtils.asDateTime(Long.parseLong(values.toString())); } - // returned by nested types... - if (values instanceof DateTime) { - return DateUtils.asDateTime((DateTime) values); - } } if (values instanceof Long || values instanceof Double || values instanceof String || values instanceof Boolean) { return values; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index fe8f5ac9925b1..05069ef42a14e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -111,15 +111,12 @@ import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypeConversion; import org.elasticsearch.xpack.sql.type.DataTypes; -import org.elasticsearch.xpack.sql.util.DateUtils; import org.elasticsearch.xpack.sql.util.StringUtils; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.DateTimeFormatterBuilder; -import org.joda.time.format.ISODateTimeFormat; import java.time.Duration; +import java.time.LocalTime; import java.time.Period; +import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAmount; import java.util.EnumSet; import java.util.List; @@ -127,9 +124,12 @@ import java.util.Map; import java.util.StringJoiner; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.sql.type.DataTypeConversion.conversionFor; +import static org.elasticsearch.xpack.sql.util.DateUtils.asDateOnly; +import static org.elasticsearch.xpack.sql.util.DateUtils.ofEscapedLiteral; abstract class ExpressionBuilder extends IdentifierBuilder { @@ -791,13 +791,11 @@ public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) { String string = string(ctx.string()); Source source = source(ctx); // parse yyyy-MM-dd - DateTime dt = null; try { - dt = ISODateTimeFormat.date().parseDateTime(string); - } catch(IllegalArgumentException ex) { + return new Literal(source, asDateOnly(string), DataType.DATE); + } catch(DateTimeParseException ex) { throw new ParsingException(source, "Invalid date received; {}", ex.getMessage()); } - return new Literal(source, DateUtils.asDateOnly(dt), DataType.DATE); } @Override @@ -806,10 +804,10 @@ public Literal visitTimeEscapedLiteral(TimeEscapedLiteralContext ctx) { Source source = source(ctx); // parse HH:mm:ss - DateTime dt = null; + LocalTime lt = null; try { - dt = ISODateTimeFormat.hourMinuteSecond().parseDateTime(string); - } catch (IllegalArgumentException ex) { + lt = LocalTime.parse(string, ISO_LOCAL_TIME); + } catch (DateTimeParseException ex) { throw new ParsingException(source, "Invalid time received; {}", ex.getMessage()); } @@ -822,18 +820,11 @@ public Literal visitTimestampEscapedLiteral(TimestampEscapedLiteralContext ctx) Source source = source(ctx); // parse yyyy-mm-dd hh:mm:ss(.f...) - DateTime dt = null; try { - DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .append(ISODateTimeFormat.date()) - .appendLiteral(" ") - .append(ISODateTimeFormat.hourMinuteSecondFraction()) - .toFormatter(); - dt = formatter.parseDateTime(string); - } catch (IllegalArgumentException ex) { + return new Literal(source, ofEscapedLiteral(string), DataType.DATETIME); + } catch (DateTimeParseException ex) { throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage()); } - return new Literal(source, DateUtils.asDateTime(dt), DataType.DATETIME); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java index bc89b0f1e1587..9dbb2a3abb6f6 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.sql.util.DateUtils; import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; import java.util.Locale; import java.util.function.DoubleFunction; import java.util.function.Function; @@ -546,8 +547,8 @@ private static Function fromString(Function conv return converter.apply(value.toString()); } catch (NumberFormatException e) { throw new SqlIllegalArgumentException(e, "cannot cast [{}] to [{}]", value, to); - } catch (IllegalArgumentException e) { - throw new SqlIllegalArgumentException(e, "cannot cast [{}] to [{}]:{}", value, to, e.getMessage()); + } catch (DateTimeParseException | IllegalArgumentException e) { + throw new SqlIllegalArgumentException(e, "cannot cast [{}] to [{}]: {}", value, to, e.getMessage()); } }; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java index deb7b9e9703c2..38db3cbe131cf 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java @@ -6,29 +6,35 @@ package org.elasticsearch.xpack.sql.util; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.xpack.sql.proto.StringUtils; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.ISODateTimeFormat; import java.time.Instant; -import java.time.LocalDateTime; +import java.time.LocalDate; import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; public final class DateUtils { - private static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000; - - // TODO: do we have a java.time based parser we can use instead? - private static final DateTimeFormatter UTC_DATE_FORMATTER = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC(); - public static final ZoneId UTC = ZoneId.of("Z"); public static final String DATE_PARSE_FORMAT = "epoch_millis"; + private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER = new DateTimeFormatterBuilder() + .append(ISO_LOCAL_DATE) + .appendLiteral(" ") + .append(ISO_LOCAL_TIME) + .toFormatter().withZone(UTC); + + private static final DateFormatter UTC_DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(UTC); + + private static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000L; + private DateUtils() {} /** @@ -56,22 +62,7 @@ public static ZonedDateTime asDateTime(long millis, ZoneId id) { * Parses the given string into a Date (SQL DATE type) using UTC as a default timezone. */ public static ZonedDateTime asDateOnly(String dateFormat) { - return asDateOnly(UTC_DATE_FORMATTER.parseDateTime(dateFormat)); - } - - public static ZonedDateTime asDateOnly(DateTime dateTime) { - LocalDateTime ldt = LocalDateTime.of( - dateTime.getYear(), - dateTime.getMonthOfYear(), - dateTime.getDayOfMonth(), - 0, - 0, - 0, - 0); - - return ZonedDateTime.ofStrict(ldt, - ZoneOffset.ofTotalSeconds(dateTime.getZone().getOffset(dateTime) / 1000), - org.elasticsearch.common.time.DateUtils.dateTimeZoneToZoneId(dateTime.getZone())); + return LocalDate.parse(dateFormat, ISO_LOCAL_DATE).atStartOfDay(UTC); } public static ZonedDateTime asDateOnly(ZonedDateTime zdt) { @@ -82,25 +73,13 @@ public static ZonedDateTime asDateOnly(ZonedDateTime zdt) { * Parses the given string into a DateTime using UTC as a default timezone. */ public static ZonedDateTime asDateTime(String dateFormat) { - return asDateTime(UTC_DATE_FORMATTER.parseDateTime(dateFormat)); + return DateFormatters.from(UTC_DATE_TIME_FORMATTER.parse(dateFormat)).withZoneSameInstant(UTC); } - public static ZonedDateTime asDateTime(DateTime dateTime) { - LocalDateTime ldt = LocalDateTime.of( - dateTime.getYear(), - dateTime.getMonthOfYear(), - dateTime.getDayOfMonth(), - dateTime.getHourOfDay(), - dateTime.getMinuteOfHour(), - dateTime.getSecondOfMinute(), - dateTime.getMillisOfSecond() * 1_000_000); - - return ZonedDateTime.ofStrict(ldt, - ZoneOffset.ofTotalSeconds(dateTime.getZone().getOffset(dateTime) / 1000), - org.elasticsearch.common.time.DateUtils.dateTimeZoneToZoneId(dateTime.getZone())); + public static ZonedDateTime ofEscapedLiteral(String dateFormat) { + return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER.withZone(UTC)); } - public static String toString(ZonedDateTime dateTime) { return StringUtils.toString(dateTime); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java index 2ae6e571ac9d2..4323cce234c54 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java @@ -7,22 +7,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.util.DateUtils; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import java.time.ZonedDateTime; -import static org.junit.Assert.assertEquals; - public class DateTimeTestUtils { private DateTimeTestUtils() {} public static ZonedDateTime dateTime(int year, int month, int day, int hour, int minute) { - DateTime dateTime = new DateTime(year, month, day, hour, minute, DateTimeZone.UTC); - ZonedDateTime zdt = ZonedDateTime.of(year, month, day, hour, minute, 0, 0, DateUtils.UTC); - assertEquals(dateTime.getMillis() / 1000, zdt.toEpochSecond()); - return zdt; + return ZonedDateTime.of(year, month, day, hour, minute, 0, 0, DateUtils.UTC); } public static ZonedDateTime dateTime(long millisSinceEpoch) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java index 8cbb0b528e9a6..bc3ea049c242e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java @@ -175,7 +175,8 @@ public void testDateLiteral() { public void testDateLiteralValidation() { ParsingException ex = expectThrows(ParsingException.class, () -> dateLiteral("2012-13-01")); - assertEquals("line 1:2: Invalid date received; Cannot parse \"2012-13-01\": Value 13 for monthOfYear must be in the range [1,12]", + assertEquals("line 1:2: Invalid date received; Text '2012-13-01' could not be parsed: " + + "Invalid value for MonthOfYear (valid values 1 - 12): 13", ex.getMessage()); } @@ -186,7 +187,8 @@ public void testTimeLiteralUnsupported() { public void testTimeLiteralValidation() { ParsingException ex = expectThrows(ParsingException.class, () -> timeLiteral("10:10:65")); - assertEquals("line 1:2: Invalid time received; Cannot parse \"10:10:65\": Value 65 for secondOfMinute must be in the range [0,59]", + assertEquals("line 1:2: Invalid time received; Text '10:10:65' could not be parsed: " + + "Invalid value for SecondOfMinute (valid values 0 - 59): 65", ex.getMessage()); } @@ -198,7 +200,7 @@ public void testTimestampLiteral() { public void testTimestampLiteralValidation() { ParsingException ex = expectThrows(ParsingException.class, () -> timestampLiteral("2012-01-01T10:01:02.3456")); assertEquals( - "line 1:2: Invalid timestamp received; Invalid format: \"2012-01-01T10:01:02.3456\" is malformed at \"T10:01:02.3456\"", + "line 1:2: Invalid timestamp received; Text '2012-01-01T10:01:02.3456' could not be parsed at index 10", ex.getMessage()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java index 686c97b91a06b..a72f9ee7f1244 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java @@ -150,10 +150,10 @@ public void testConversionToDate() { Conversion conversion = conversionFor(KEYWORD, to); assertNull(conversion.convert(null)); - assertEquals(date(0L), conversion.convert("1970-01-01T00:10:01Z")); - assertEquals(date(1483228800000L), conversion.convert("2017-01-01T00:11:00Z")); - assertEquals(date(-1672531200000L), conversion.convert("1917-01-01T00:11:00Z")); - assertEquals(date(18000000L), conversion.convert("1970-01-01T03:10:20-05:00")); + assertEquals(date(0L), conversion.convert("1970-01-01")); + assertEquals(date(1483228800000L), conversion.convert("2017-01-01")); + assertEquals(date(-1672531200000L), conversion.convert("1917-01-01")); + assertEquals(date(18000000L), conversion.convert("1970-01-01")); // double check back and forth conversion @@ -162,7 +162,7 @@ public void testConversionToDate() { Conversion back = conversionFor(KEYWORD, DATE); assertEquals(DateUtils.asDateOnly(zdt), back.convert(forward.convert(zdt))); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + assertEquals("cannot cast [0xff] to [date]: Text '0xff' could not be parsed at index 0", e.getMessage()); } } @@ -199,6 +199,7 @@ public void testConversionToDateTime() { Conversion conversion = conversionFor(KEYWORD, to); assertNull(conversion.convert(null)); + assertEquals(dateTime(0L), conversion.convert("1970-01-01")); assertEquals(dateTime(1000L), conversion.convert("1970-01-01T00:00:01Z")); assertEquals(dateTime(1483228800000L), conversion.convert("2017-01-01T00:00:00Z")); assertEquals(dateTime(1483228800000L), conversion.convert("2017-01-01T00:00:00Z")); @@ -211,7 +212,8 @@ public void testConversionToDateTime() { Conversion back = conversionFor(KEYWORD, DATETIME); assertEquals(dt, back.convert(forward.convert(dt))); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [datetime]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + assertEquals("cannot cast [0xff] to [datetime]: failed to parse date field [0xff] with format [date_optional_time]", + e.getMessage()); } } From a1c63a3e141fcc5c3dcc35d725572b6e5e7eed22 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 8 Feb 2019 13:09:53 -0800 Subject: [PATCH 03/12] make DateMathIndexExpressionsIntegrationIT more resilient (#38473) the internal cluster calls System.nanoTime() and System.currentTimeMillis() during evaluations of requests that need date-math index resolution. These are not mockable in these tests. As is, executing requests as-is in these test cases can potentially result in invalid responses when day-boundaries are hit mid test run. This change makes the test framework ignore failures due to day changes. Closes #31067. --- ...DateMathIndexExpressionsIntegrationIT.java | 97 +++++++++++++------ 1 file changed, 65 insertions(+), 32 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java index fe4ab2e363a88..ffe25b4a372e3 100644 --- a/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java @@ -19,6 +19,9 @@ package org.elasticsearch.indices; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -29,10 +32,14 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESIntegTestCase; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; +import org.junit.Before; + +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; @@ -41,15 +48,42 @@ import static org.hamcrest.Matchers.notNullValue; public class DateMathIndexExpressionsIntegrationIT extends ESIntegTestCase { + private ZonedDateTime now; + + @Before + public void setNow() { + now = ZonedDateTime.now(ZoneOffset.UTC); + } + + /** + * the internal cluster calls System.nanoTime() and System.currentTimeMillis() during evaluations of requests + * that need date-math index resolution. These are not mockable in these tests. As is, executing requests as-is + * in these test cases can potentially result in invalid responses when day-boundaries are hit mid test run. Instead + * of failing when index resolution with `now` is one day off, this method wraps calls with the assumption that + * the day did not change during the test run. + */ + public R dateSensitiveGet(ActionRequestBuilder builder) { + Runnable dayChangeAssumption = () -> assumeTrue("day changed between requests", + ZonedDateTime.now(ZoneOffset.UTC).getDayOfYear() == now.getDayOfYear()); + R response; + try { + response = builder.get(); + } catch (IndexNotFoundException e) { + // index resolver throws this if it does not find the exact index due to day changes + dayChangeAssumption.run(); + throw e; + } + dayChangeAssumption.run(); + return response; + } public void testIndexNameDateMathExpressions() { - DateTime now = new DateTime(DateTimeZone.UTC); - String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now); - String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1)); - String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2)); + String index1 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now); + String index2 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)); + String index3 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)); createIndex(index1, index2, index3); - GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(index1, index2, index3).get(); + GetSettingsResponse getSettingsResponse = dateSensitiveGet(client().admin().indices().prepareGetSettings(index1, index2, index3)); assertEquals(index1, getSettingsResponse.getSetting(index1, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(index2, getSettingsResponse.getSetting(index2, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(index3, getSettingsResponse.getSetting(index3, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); @@ -63,26 +97,26 @@ public void testIndexNameDateMathExpressions() { client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); - SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); + SearchResponse searchResponse = dateSensitiveGet(client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3)); assertHitCount(searchResponse, 3); assertSearchHits(searchResponse, "1", "2", "3"); - GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get(); + GetResponse getResponse = dateSensitiveGet(client().prepareGet(dateMathExp1, "type", "1")); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("1")); - getResponse = client().prepareGet(dateMathExp2, "type", "2").get(); + getResponse = dateSensitiveGet(client().prepareGet(dateMathExp2, "type", "2")); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("2")); - getResponse = client().prepareGet(dateMathExp3, "type", "3").get(); + getResponse = dateSensitiveGet(client().prepareGet(dateMathExp3, "type", "3")); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("3")); - MultiGetResponse mgetResponse = client().prepareMultiGet() + MultiGetResponse mgetResponse = dateSensitiveGet(client().prepareMultiGet() .add(dateMathExp1, "type", "1") .add(dateMathExp2, "type", "2") - .add(dateMathExp3, "type", "3").get(); + .add(dateMathExp3, "type", "3")); assertThat(mgetResponse.getResponses()[0].getResponse().isExists(), is(true)); assertThat(mgetResponse.getResponses()[0].getResponse().getId(), equalTo("1")); assertThat(mgetResponse.getResponses()[1].getResponse().isExists(), is(true)); @@ -90,29 +124,29 @@ public void testIndexNameDateMathExpressions() { assertThat(mgetResponse.getResponses()[2].getResponse().isExists(), is(true)); assertThat(mgetResponse.getResponses()[2].getResponse().getId(), equalTo("3")); - IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get(); + IndicesStatsResponse indicesStatsResponse = + dateSensitiveGet(client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3)); assertThat(indicesStatsResponse.getIndex(index1), notNullValue()); assertThat(indicesStatsResponse.getIndex(index2), notNullValue()); assertThat(indicesStatsResponse.getIndex(index3), notNullValue()); - DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get(); + DeleteResponse deleteResponse = dateSensitiveGet(client().prepareDelete(dateMathExp1, "type", "1")); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("1")); - deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get(); + deleteResponse = dateSensitiveGet(client().prepareDelete(dateMathExp2, "type", "2")); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("2")); - deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get(); + deleteResponse = dateSensitiveGet(client().prepareDelete(dateMathExp3, "type", "3")); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("3")); } - public void testAutoCreateIndexWithDateMathExpression() throws Exception { - DateTime now = new DateTime(DateTimeZone.UTC); - String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now); - String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1)); - String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2)); + public void testAutoCreateIndexWithDateMathExpression() { + String index1 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now); + String index2 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)); + String index3 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)); String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; @@ -122,29 +156,28 @@ public void testAutoCreateIndexWithDateMathExpression() throws Exception { client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); - SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); + SearchResponse searchResponse = dateSensitiveGet(client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3)); assertHitCount(searchResponse, 3); assertSearchHits(searchResponse, "1", "2", "3"); - IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get(); + IndicesStatsResponse indicesStatsResponse = + dateSensitiveGet(client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3)); assertThat(indicesStatsResponse.getIndex(index1), notNullValue()); assertThat(indicesStatsResponse.getIndex(index2), notNullValue()); assertThat(indicesStatsResponse.getIndex(index3), notNullValue()); } - public void testCreateIndexWithDateMathExpression() throws Exception { - DateTime now = new DateTime(DateTimeZone.UTC); - String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now); - String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1)); - String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2)); + public void testCreateIndexWithDateMathExpression() { + String index1 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now); + String index2 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)); + String index3 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)); String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; createIndex(dateMathExp1, dateMathExp2, dateMathExp3); - - GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(index1, index2, index3).get(); + GetSettingsResponse getSettingsResponse = dateSensitiveGet(client().admin().indices().prepareGetSettings(index1, index2, index3)); assertEquals(dateMathExp1, getSettingsResponse.getSetting(index1, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(dateMathExp2, getSettingsResponse.getSetting(index2, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(dateMathExp3, getSettingsResponse.getSetting(index3, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); From 9b75a709a2c3734187cc49762b9ef5d1b18f58dd Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Fri, 8 Feb 2019 23:02:53 +0100 Subject: [PATCH 04/12] Concurrent file chunk fetching for CCR restore (#38495) Adds the ability to fetch chunks from different files in parallel, configurable using the new `ccr.indices.recovery.max_concurrent_file_chunks` setting, which defaults to 5 in this PR. The implementation uses the parallel file writer functionality that is also used by peer recoveries. --- .../index/engine/CombinedDeletionPolicy.java | 7 + .../index/engine/InternalEngine.java | 5 + .../indices/recovery/MultiFileWriter.java | 213 ++++++++++++++++ .../indices/recovery/RecoveryTarget.java | 172 +------------ .../blobstore/FileRestoreContext.java | 28 +- .../PeerRecoveryTargetServiceTests.java | 2 +- .../indices/recovery/RecoveryStatusTests.java | 39 +-- .../index/engine/EngineTestCase.java | 6 + .../java/org/elasticsearch/xpack/ccr/Ccr.java | 4 +- .../elasticsearch/xpack/ccr/CcrSettings.java | 20 +- .../GetCcrRestoreFileChunkAction.java | 2 - .../xpack/ccr/repository/CcrRepository.java | 239 ++++++++++++------ .../repository/CcrRestoreSourceService.java | 12 +- .../xpack/ccr/IndexFollowingIT.java | 6 +- .../CcrRestoreSourceServiceTests.java | 14 +- 15 files changed, 472 insertions(+), 297 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java diff --git a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java index d10690379eddd..addb16d58d031 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java @@ -191,6 +191,13 @@ private static int indexOfKeptCommits(List commits, long return 0; } + /** + * Checks whether the deletion policy is holding on to snapshotted commits + */ + synchronized boolean hasSnapshottedCommits() { + return snapshottedCommits.isEmpty() == false; + } + /** * Checks if the deletion policy can release some index commits with the latest global checkpoint. */ diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index acedd8356ea9e..b2143dcc0407f 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -476,6 +476,11 @@ Translog getTranslog() { return translog; } + // Package private for testing purposes only + boolean hasSnapshottedCommits() { + return combinedDeletionPolicy.hasSnapshottedCommits(); + } + @Override public boolean isTranslogSyncNeeded() { return getTranslog().syncNeeded(); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java b/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java new file mode 100644 index 0000000000000..87a6d18671a6f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java @@ -0,0 +1,213 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.indices.recovery; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.StoreFileMetaData; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentMap; + +public class MultiFileWriter implements Releasable { + + public MultiFileWriter(Store store, RecoveryState.Index indexState, String tempFilePrefix, Logger logger, Runnable ensureOpen) { + this.store = store; + this.indexState = indexState; + this.tempFilePrefix = tempFilePrefix; + this.logger = logger; + this.ensureOpen = ensureOpen; + } + + private final Runnable ensureOpen; + private final Logger logger; + private final Store store; + private final RecoveryState.Index indexState; + private final String tempFilePrefix; + + private final ConcurrentMap openIndexOutputs = ConcurrentCollections.newConcurrentMap(); + private final ConcurrentMap fileChunkWriters = ConcurrentCollections.newConcurrentMap(); + + + final Map tempFileNames = ConcurrentCollections.newConcurrentMap(); + + public void writeFileChunk(StoreFileMetaData fileMetaData, long position, BytesReference content, boolean lastChunk) + throws IOException { + final FileChunkWriter writer = fileChunkWriters.computeIfAbsent(fileMetaData.name(), name -> new FileChunkWriter()); + writer.writeChunk(new FileChunk(fileMetaData, content, position, lastChunk)); + } + + /** Get a temporary name for the provided file name. */ + String getTempNameForFile(String origFile) { + return tempFilePrefix + origFile; + } + + public IndexOutput getOpenIndexOutput(String key) { + ensureOpen.run(); + return openIndexOutputs.get(key); + } + + /** remove and {@link IndexOutput} for a given file. It is the caller's responsibility to close it */ + public IndexOutput removeOpenIndexOutputs(String name) { + ensureOpen.run(); + return openIndexOutputs.remove(name); + } + + /** + * Creates an {@link IndexOutput} for the given file name. Note that the + * IndexOutput actually point at a temporary file. + *

+ * Note: You can use {@link #getOpenIndexOutput(String)} with the same filename to retrieve the same IndexOutput + * at a later stage + */ + public IndexOutput openAndPutIndexOutput(String fileName, StoreFileMetaData metaData, Store store) throws IOException { + ensureOpen.run(); + String tempFileName = getTempNameForFile(fileName); + if (tempFileNames.containsKey(tempFileName)) { + throw new IllegalStateException("output for file [" + fileName + "] has already been created"); + } + // add first, before it's created + tempFileNames.put(tempFileName, fileName); + IndexOutput indexOutput = store.createVerifyingOutput(tempFileName, metaData, IOContext.DEFAULT); + openIndexOutputs.put(fileName, indexOutput); + return indexOutput; + } + + private void innerWriteFileChunk(StoreFileMetaData fileMetaData, long position, + BytesReference content, boolean lastChunk) throws IOException { + final String name = fileMetaData.name(); + IndexOutput indexOutput; + if (position == 0) { + indexOutput = openAndPutIndexOutput(name, fileMetaData, store); + } else { + indexOutput = getOpenIndexOutput(name); + } + assert indexOutput.getFilePointer() == position : "file-pointer " + indexOutput.getFilePointer() + " != " + position; + BytesRefIterator iterator = content.iterator(); + BytesRef scratch; + while((scratch = iterator.next()) != null) { // we iterate over all pages - this is a 0-copy for all core impls + indexOutput.writeBytes(scratch.bytes, scratch.offset, scratch.length); + } + indexState.addRecoveredBytesToFile(name, content.length()); + if (indexOutput.getFilePointer() >= fileMetaData.length() || lastChunk) { + try { + Store.verify(indexOutput); + } finally { + // we are done + indexOutput.close(); + } + final String temporaryFileName = getTempNameForFile(name); + assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName) : + "expected: [" + temporaryFileName + "] in " + Arrays.toString(store.directory().listAll()); + store.directory().sync(Collections.singleton(temporaryFileName)); + IndexOutput remove = removeOpenIndexOutputs(name); + assert remove == null || remove == indexOutput; // remove maybe null if we got finished + } + } + + @Override + public void close() { + fileChunkWriters.clear(); + // clean open index outputs + Iterator> iterator = openIndexOutputs.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + logger.trace("closing IndexOutput file [{}]", entry.getValue()); + try { + entry.getValue().close(); + } catch (Exception e) { + logger.debug(() -> new ParameterizedMessage("error while closing recovery output [{}]", entry.getValue()), e); + } + iterator.remove(); + } + if (Strings.hasText(tempFilePrefix)) { + // trash temporary files + for (String file : tempFileNames.keySet()) { + logger.trace("cleaning temporary file [{}]", file); + store.deleteQuiet(file); + } + } + } + + /** renames all temporary files to their true name, potentially overriding existing files */ + public void renameAllTempFiles() throws IOException { + ensureOpen.run(); + store.renameTempFilesSafe(tempFileNames); + } + + static final class FileChunk { + final StoreFileMetaData md; + final BytesReference content; + final long position; + final boolean lastChunk; + FileChunk(StoreFileMetaData md, BytesReference content, long position, boolean lastChunk) { + this.md = md; + this.content = content; + this.position = position; + this.lastChunk = lastChunk; + } + } + + private final class FileChunkWriter { + // chunks can be delivered out of order, we need to buffer chunks if there's a gap between them. + final PriorityQueue pendingChunks = new PriorityQueue<>(Comparator.comparing(fc -> fc.position)); + long lastPosition = 0; + + void writeChunk(FileChunk newChunk) throws IOException { + synchronized (this) { + pendingChunks.add(newChunk); + } + while (true) { + final FileChunk chunk; + synchronized (this) { + chunk = pendingChunks.peek(); + if (chunk == null || chunk.position != lastPosition) { + return; + } + pendingChunks.remove(); + } + innerWriteFileChunk(chunk.md, chunk.position, chunk.content, chunk.lastChunk); + synchronized (this) { + assert lastPosition == chunk.position : "last_position " + lastPosition + " != chunk_position " + chunk.position; + lastPosition += chunk.content.length(); + if (chunk.lastChunk) { + assert pendingChunks.isEmpty() == true : "still have pending chunks [" + pendingChunks + "]"; + fileChunkWriters.remove(chunk.md.name()); + assert fileChunkWriters.containsValue(this) == false : "chunk writer [" + newChunk.md + "] was not removed"; + } + } + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index e63b9ba8fd5ea..76f2200a47d82 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -20,14 +20,9 @@ package org.elasticsearch.indices.recovery; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; @@ -39,7 +34,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.concurrent.AbstractRefCounted; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.seqno.ReplicationTracker; @@ -55,15 +49,7 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.PriorityQueue; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -85,15 +71,13 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget private final long recoveryId; private final IndexShard indexShard; private final DiscoveryNode sourceNode; - private final String tempFilePrefix; + private final MultiFileWriter multiFileWriter; private final Store store; private final PeerRecoveryTargetService.RecoveryListener listener; private final LongConsumer ensureClusterStateVersionCallback; private final AtomicBoolean finished = new AtomicBoolean(); - private final ConcurrentMap openIndexOutputs = ConcurrentCollections.newConcurrentMap(); - private final ConcurrentMap fileChunkWriters = ConcurrentCollections.newConcurrentMap(); private final CancellableThreads cancellableThreads; // last time this status was accessed @@ -102,8 +86,6 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget // latch that can be used to blockingly wait for RecoveryTarget to be closed private final CountDownLatch closedLatch = new CountDownLatch(1); - private final Map tempFileNames = ConcurrentCollections.newConcurrentMap(); - /** * Creates a new recovery target object that represents a recovery to the provided shard. * @@ -126,7 +108,9 @@ public RecoveryTarget(final IndexShard indexShard, this.indexShard = indexShard; this.sourceNode = sourceNode; this.shardId = indexShard.shardId(); - this.tempFilePrefix = RECOVERY_PREFIX + UUIDs.randomBase64UUID() + "."; + final String tempFilePrefix = RECOVERY_PREFIX + UUIDs.randomBase64UUID() + "."; + this.multiFileWriter = new MultiFileWriter(indexShard.store(), indexShard.recoveryState().getIndex(), tempFilePrefix, logger, + this::ensureRefCount); this.store = indexShard.store(); this.ensureClusterStateVersionCallback = ensureClusterStateVersionCallback; // make sure the store is not released until we are done. @@ -187,12 +171,6 @@ public RecoveryState.Stage stage() { return state().getStage(); } - /** renames all temporary files to their true name, potentially overriding existing files */ - public void renameAllTempFiles() throws IOException { - ensureRefCount(); - store.renameTempFilesSafe(tempFileNames); - } - /** * Closes the current recovery target and waits up to a certain timeout for resources to be freed. * Returns true if resetting the recovery was successful, false if the recovery target is already cancelled / failed or marked as done. @@ -274,7 +252,7 @@ public void notifyListener(RecoveryFailedException e, boolean sendShardFailure) /** mark the current recovery as done */ public void markAsDone() { if (finished.compareAndSet(false, true)) { - assert tempFileNames.isEmpty() : "not all temporary files are renamed"; + assert multiFileWriter.tempFileNames.isEmpty() : "not all temporary files are renamed"; try { // this might still throw an exception ie. if the shard is CLOSED due to some other event. // it's safer to decrement the reference in a try finally here. @@ -287,65 +265,12 @@ public void markAsDone() { } } - /** Get a temporary name for the provided file name. */ - public String getTempNameForFile(String origFile) { - return tempFilePrefix + origFile; - } - - public IndexOutput getOpenIndexOutput(String key) { - ensureRefCount(); - return openIndexOutputs.get(key); - } - - /** remove and {@link org.apache.lucene.store.IndexOutput} for a given file. It is the caller's responsibility to close it */ - public IndexOutput removeOpenIndexOutputs(String name) { - ensureRefCount(); - return openIndexOutputs.remove(name); - } - - /** - * Creates an {@link org.apache.lucene.store.IndexOutput} for the given file name. Note that the - * IndexOutput actually point at a temporary file. - *

- * Note: You can use {@link #getOpenIndexOutput(String)} with the same filename to retrieve the same IndexOutput - * at a later stage - */ - public IndexOutput openAndPutIndexOutput(String fileName, StoreFileMetaData metaData, Store store) throws IOException { - ensureRefCount(); - String tempFileName = getTempNameForFile(fileName); - if (tempFileNames.containsKey(tempFileName)) { - throw new IllegalStateException("output for file [" + fileName + "] has already been created"); - } - // add first, before it's created - tempFileNames.put(tempFileName, fileName); - IndexOutput indexOutput = store.createVerifyingOutput(tempFileName, metaData, IOContext.DEFAULT); - openIndexOutputs.put(fileName, indexOutput); - return indexOutput; - } - @Override protected void closeInternal() { try { - // clean open index outputs - Iterator> iterator = openIndexOutputs.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - logger.trace("closing IndexOutput file [{}]", entry.getValue()); - try { - entry.getValue().close(); - } catch (Exception e) { - logger.debug(() -> new ParameterizedMessage("error while closing recovery output [{}]", entry.getValue()), e); - } - iterator.remove(); - } - // trash temporary files - for (String file : tempFileNames.keySet()) { - logger.trace("cleaning temporary file [{}]", file); - store.deleteQuiet(file); - } + multiFileWriter.close(); } finally { // free store. increment happens in constructor - fileChunkWriters.clear(); store.decRef(); indexShard.recoveryStats().decCurrentAsTarget(); closedLatch.countDown(); @@ -470,7 +395,7 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa // first, we go and move files that were created with the recovery id suffix to // the actual names, its ok if we have a corrupted index here, since we have replicas // to recover from in case of a full cluster shutdown just when this code executes... - renameAllTempFiles(); + multiFileWriter.renameAllTempFiles(); final Store store = store(); store.incRef(); try { @@ -511,96 +436,21 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa } } - private void innerWriteFileChunk(StoreFileMetaData fileMetaData, long position, - BytesReference content, boolean lastChunk) throws IOException { - final Store store = store(); - final String name = fileMetaData.name(); - final RecoveryState.Index indexState = state().getIndex(); - IndexOutput indexOutput; - if (position == 0) { - indexOutput = openAndPutIndexOutput(name, fileMetaData, store); - } else { - indexOutput = getOpenIndexOutput(name); - } - assert indexOutput.getFilePointer() == position : "file-pointer " + indexOutput.getFilePointer() + " != " + position; - BytesRefIterator iterator = content.iterator(); - BytesRef scratch; - while((scratch = iterator.next()) != null) { // we iterate over all pages - this is a 0-copy for all core impls - indexOutput.writeBytes(scratch.bytes, scratch.offset, scratch.length); - } - indexState.addRecoveredBytesToFile(name, content.length()); - if (indexOutput.getFilePointer() >= fileMetaData.length() || lastChunk) { - try { - Store.verify(indexOutput); - } finally { - // we are done - indexOutput.close(); - } - final String temporaryFileName = getTempNameForFile(name); - assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName) : - "expected: [" + temporaryFileName + "] in " + Arrays.toString(store.directory().listAll()); - store.directory().sync(Collections.singleton(temporaryFileName)); - IndexOutput remove = removeOpenIndexOutputs(name); - assert remove == null || remove == indexOutput; // remove maybe null if we got finished - } - } - @Override public void writeFileChunk(StoreFileMetaData fileMetaData, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener listener) { try { state().getTranslog().totalOperations(totalTranslogOps); - final FileChunkWriter writer = fileChunkWriters.computeIfAbsent(fileMetaData.name(), name -> new FileChunkWriter()); - writer.writeChunk(new FileChunk(fileMetaData, content, position, lastChunk)); + multiFileWriter.writeFileChunk(fileMetaData, position, content, lastChunk); listener.onResponse(null); } catch (Exception e) { listener.onFailure(e); } } - private static final class FileChunk { - final StoreFileMetaData md; - final BytesReference content; - final long position; - final boolean lastChunk; - FileChunk(StoreFileMetaData md, BytesReference content, long position, boolean lastChunk) { - this.md = md; - this.content = content; - this.position = position; - this.lastChunk = lastChunk; - } - } - - private final class FileChunkWriter { - // chunks can be delivered out of order, we need to buffer chunks if there's a gap between them. - final PriorityQueue pendingChunks = new PriorityQueue<>(Comparator.comparing(fc -> fc.position)); - long lastPosition = 0; - - void writeChunk(FileChunk newChunk) throws IOException { - synchronized (this) { - pendingChunks.add(newChunk); - } - while (true) { - final FileChunk chunk; - synchronized (this) { - chunk = pendingChunks.peek(); - if (chunk == null || chunk.position != lastPosition) { - return; - } - pendingChunks.remove(); - } - innerWriteFileChunk(chunk.md, chunk.position, chunk.content, chunk.lastChunk); - synchronized (this) { - assert lastPosition == chunk.position : "last_position " + lastPosition + " != chunk_position " + chunk.position; - lastPosition += chunk.content.length(); - if (chunk.lastChunk) { - assert pendingChunks.isEmpty() == true : "still have pending chunks [" + pendingChunks + "]"; - fileChunkWriters.remove(chunk.md.name()); - assert fileChunkWriters.containsValue(this) == false : "chunk writer [" + newChunk.md + "] was not removed"; - } - } - } - } + /** Get a temporary name for the provided file name. */ + public String getTempNameForFile(String origFile) { + return multiFileWriter.getTempNameForFile(origFile); } Path translogLocation() { diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java index 2f837812ae2e2..c8185fea89523 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java @@ -62,14 +62,14 @@ */ public abstract class FileRestoreContext { - private static final Logger logger = LogManager.getLogger(FileRestoreContext.class); + protected static final Logger logger = LogManager.getLogger(FileRestoreContext.class); - private final String repositoryName; - private final IndexShard indexShard; - private final RecoveryState recoveryState; - private final SnapshotId snapshotId; - private final ShardId shardId; - private final int bufferSize; + protected final String repositoryName; + protected final IndexShard indexShard; + protected final RecoveryState recoveryState; + protected final SnapshotId snapshotId; + protected final ShardId shardId; + protected final int bufferSize; /** * Constructs new restore context @@ -183,7 +183,6 @@ public void restore(SnapshotFiles snapshotFiles) throws IOException { // list of all existing store files final List deleteIfExistFiles = Arrays.asList(store.directory().listAll()); - // restore the files from the snapshot to the Lucene store for (final BlobStoreIndexShardSnapshot.FileInfo fileToRecover : filesToRecover) { // if a file with a same physical name already exist in the store we need to delete it // before restoring it from the snapshot. We could be lenient and try to reuse the existing @@ -196,10 +195,9 @@ public void restore(SnapshotFiles snapshotFiles) throws IOException { logger.trace("[{}] [{}] deleting pre-existing file [{}]", shardId, snapshotId, physicalName); store.directory().deleteFile(physicalName); } - - logger.trace("[{}] [{}] restoring file [{}]", shardId, snapshotId, fileToRecover.name()); - restoreFile(fileToRecover, store); } + + restoreFiles(filesToRecover, store); } catch (IOException ex) { throw new IndexShardRestoreFailedException(shardId, "Failed to recover index", ex); } @@ -234,6 +232,14 @@ public void restore(SnapshotFiles snapshotFiles) throws IOException { } } + protected void restoreFiles(List filesToRecover, Store store) throws IOException { + // restore the files from the snapshot to the Lucene store + for (final BlobStoreIndexShardSnapshot.FileInfo fileToRecover : filesToRecover) { + logger.trace("[{}] [{}] restoring file [{}]", shardId, snapshotId, fileToRecover.name()); + restoreFile(fileToRecover, store); + } + } + protected abstract InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo); @SuppressWarnings("unchecked") diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java index a2ec88cf7b58c..41ea9a8bea74b 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java @@ -189,7 +189,7 @@ public void testWriteFileChunksConcurrently() throws Exception { for (Thread sender : senders) { sender.join(); } - recoveryTarget.renameAllTempFiles(); + recoveryTarget.cleanFiles(0, sourceSnapshot); recoveryTarget.decRef(); Store.MetadataSnapshot targetSnapshot = targetShard.snapshotStoreMetadata(); Store.RecoveryDiff diff = sourceSnapshot.recoveryDiff(targetSnapshot); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index ed1ee7708522d..b974d42d826bb 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -20,8 +20,6 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.IndexOutput; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; @@ -32,9 +30,6 @@ import java.util.Set; import java.util.regex.Pattern; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; - public class RecoveryStatusTests extends ESSingleNodeTestCase { private static final org.apache.lucene.util.Version MIN_SUPPORTED_LUCENE_VERSION = org.elasticsearch.Version.CURRENT .minimumIndexCompatibilityVersion().luceneVersion; @@ -42,35 +37,27 @@ public void testRenameTempFiles() throws IOException { IndexService service = createIndex("foo"); IndexShard indexShard = service.getShardOrNull(0); - DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); - RecoveryTarget status = new RecoveryTarget(indexShard, node, new PeerRecoveryTargetService.RecoveryListener() { - @Override - public void onRecoveryDone(RecoveryState state) { - } - - @Override - public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) { - } - }, version -> {}); - try (IndexOutput indexOutput = status.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength() - , "9z51nw", MIN_SUPPORTED_LUCENE_VERSION), status.store())) { + MultiFileWriter multiFileWriter = new MultiFileWriter(indexShard.store(), + indexShard.recoveryState().getIndex(), "recovery.test.", logger, () -> {}); + try (IndexOutput indexOutput = multiFileWriter.openAndPutIndexOutput("foo.bar", + new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", MIN_SUPPORTED_LUCENE_VERSION), indexShard.store())) { indexOutput.writeInt(1); - IndexOutput openIndexOutput = status.getOpenIndexOutput("foo.bar"); + IndexOutput openIndexOutput = multiFileWriter.getOpenIndexOutput("foo.bar"); assertSame(openIndexOutput, indexOutput); openIndexOutput.writeInt(1); CodecUtil.writeFooter(indexOutput); } try { - status.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", - MIN_SUPPORTED_LUCENE_VERSION), status.store()); + multiFileWriter.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", + MIN_SUPPORTED_LUCENE_VERSION), indexShard.store()); fail("file foo.bar is already opened and registered"); } catch (IllegalStateException ex) { assertEquals("output for file [foo.bar] has already been created", ex.getMessage()); // all well = it's already registered } - status.removeOpenIndexOutputs("foo.bar"); - Set strings = Sets.newHashSet(status.store().directory().listAll()); + multiFileWriter.removeOpenIndexOutputs("foo.bar"); + Set strings = Sets.newHashSet(indexShard.store().directory().listAll()); String expectedFile = null; for (String file : strings) { if (Pattern.compile("recovery[.][\\w-]+[.]foo[.]bar").matcher(file).matches()) { @@ -80,12 +67,10 @@ public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, bo } assertNotNull(expectedFile); indexShard.close("foo", false);// we have to close it here otherwise rename fails since the write.lock is held by the engine - status.renameAllTempFiles(); - strings = Sets.newHashSet(status.store().directory().listAll()); + multiFileWriter.renameAllTempFiles(); + strings = Sets.newHashSet(indexShard.store().directory().listAll()); assertTrue(strings.toString(), strings.contains("foo.bar")); assertFalse(strings.toString(), strings.contains(expectedFile)); - // we must fail the recovery because marking it as done will try to move the shard to POST_RECOVERY, - // which will fail because it's started - status.fail(new RecoveryFailedException(status.state(), "end of test. OK.", null), false); + multiFileWriter.close(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index e09455b55bd52..855f1b2e2fd72 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1088,6 +1088,12 @@ public static Translog getTranslog(Engine engine) { return internalEngine.getTranslog(); } + public static boolean hasSnapshottedCommits(Engine engine) { + assert engine instanceof InternalEngine : "only InternalEngines have snapshotted commits, got: " + engine.getClass(); + InternalEngine internalEngine = (InternalEngine) engine; + return internalEngine.hasSnapshottedCommits(); + } + public static final class PrimaryTermSupplier implements LongSupplier { private final AtomicLong term; diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index a7fa69e7abd39..d4c73c1c6e503 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -127,6 +127,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E private final CcrLicenseChecker ccrLicenseChecker; private final SetOnce restoreSourceService = new SetOnce<>(); private final SetOnce ccrSettings = new SetOnce<>(); + private final SetOnce threadPool = new SetOnce<>(); private Client client; private final boolean transportClientMode; @@ -171,6 +172,7 @@ public Collection createComponents( CcrSettings ccrSettings = new CcrSettings(settings, clusterService.getClusterSettings()); this.ccrSettings.set(ccrSettings); + this.threadPool.set(threadPool); CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService(threadPool, ccrSettings); this.restoreSourceService.set(restoreSourceService); return Arrays.asList( @@ -307,7 +309,7 @@ public List> getExecutorBuilders(Settings settings) { @Override public Map getInternalRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) { Repository.Factory repositoryFactory = - (metadata) -> new CcrRepository(metadata, client, ccrLicenseChecker, settings, ccrSettings.get()); + (metadata) -> new CcrRepository(metadata, client, ccrLicenseChecker, settings, ccrSettings.get(), threadPool.get()); return Collections.singletonMap(CcrRepository.TYPE, repositoryFactory); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java index 0e147f66d6ebc..9abcfb86e2b7c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java @@ -57,6 +57,12 @@ public final class CcrSettings { new ByteSizeValue(1, ByteSizeUnit.KB), new ByteSizeValue(1, ByteSizeUnit.GB), Setting.Property.Dynamic, Setting.Property.NodeScope); + /** + * Controls the maximum number of file chunk requests that are sent concurrently per recovery to the leader. + */ + public static final Setting INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING = + Setting.intSetting("ccr.indices.recovery.max_concurrent_file_chunks", 5, 1, 10, Property.Dynamic, Property.NodeScope); + /** * The leader must open resources for a ccr recovery. If there is no activity for this interval of time, * the leader will close the restore session. @@ -77,7 +83,7 @@ public final class CcrSettings { * * @return the settings */ - static List> getSettings() { + public static List> getSettings() { return Arrays.asList( XPackSettings.CCR_ENABLED_SETTING, CCR_FOLLOWING_INDEX_SETTING, @@ -86,6 +92,7 @@ static List> getSettings() { INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, CCR_AUTO_FOLLOW_WAIT_FOR_METADATA_TIMEOUT, RECOVERY_CHUNK_SIZE, + INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING, CCR_WAIT_FOR_METADATA_TIMEOUT); } @@ -93,14 +100,17 @@ static List> getSettings() { private volatile TimeValue recoveryActivityTimeout; private volatile TimeValue recoveryActionTimeout; private volatile ByteSizeValue chunkSize; + private volatile int maxConcurrentFileChunks; public CcrSettings(Settings settings, ClusterSettings clusterSettings) { this.recoveryActivityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings); this.recoveryActionTimeout = INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.get(settings); this.ccrRateLimiter = new CombinedRateLimiter(RECOVERY_MAX_BYTES_PER_SECOND.get(settings)); this.chunkSize = RECOVERY_MAX_BYTES_PER_SECOND.get(settings); + this.maxConcurrentFileChunks = INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING.get(settings); clusterSettings.addSettingsUpdateConsumer(RECOVERY_MAX_BYTES_PER_SECOND, this::setMaxBytesPerSec); clusterSettings.addSettingsUpdateConsumer(RECOVERY_CHUNK_SIZE, this::setChunkSize); + clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING, this::setMaxConcurrentFileChunks); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setRecoveryActivityTimeout); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, this::setRecoveryActionTimeout); } @@ -109,6 +119,10 @@ private void setChunkSize(ByteSizeValue chunkSize) { this.chunkSize = chunkSize; } + private void setMaxConcurrentFileChunks(int maxConcurrentFileChunks) { + this.maxConcurrentFileChunks = maxConcurrentFileChunks; + } + private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) { ccrRateLimiter.setMBPerSec(maxBytesPerSec); } @@ -125,6 +139,10 @@ public ByteSizeValue getChunkSize() { return chunkSize; } + public int getMaxConcurrentFileChunks() { + return maxConcurrentFileChunks; + } + public CombinedRateLimiter getRateLimiter() { return ccrRateLimiter; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java index cf8d2e5c55f48..37dfc84f46a01 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java @@ -51,7 +51,6 @@ public static class TransportGetCcrRestoreFileChunkAction extends HandledTransportAction { private final CcrRestoreSourceService restoreSourceService; - private final ThreadPool threadPool; private final BigArrays bigArrays; @Inject @@ -59,7 +58,6 @@ public TransportGetCcrRestoreFileChunkAction(BigArrays bigArrays, TransportServi CcrRestoreSourceService restoreSourceService) { super(NAME, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new, ThreadPool.Names.GENERIC); TransportActionProxy.registerProxyAction(transportService, NAME, GetCcrRestoreFileChunkResponse::new); - this.threadPool = transportService.getThreadPool(); this.restoreSourceService = restoreSourceService; this.bigArrays = bigArrays; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 833cd474450ff..356dcd1439b85 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -9,7 +9,10 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.apache.lucene.index.IndexCommit; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; @@ -22,26 +25,27 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.CombinedRateLimiter; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.EngineException; +import org.elasticsearch.index.seqno.LocalCheckpointTracker; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardRecoveryException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; -import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.snapshots.blobstore.SnapshotFiles; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.StoreFileMetaData; +import org.elasticsearch.indices.recovery.MultiFileWriter; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; @@ -51,6 +55,7 @@ import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotShardFailure; import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -65,15 +70,20 @@ import java.io.Closeable; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongConsumer; import java.util.function.Supplier; +import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; + /** * This repository relies on a remote cluster for Ccr restores. It is read-only so it can only be used to @@ -92,17 +102,19 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit private final String remoteClusterAlias; private final Client client; private final CcrLicenseChecker ccrLicenseChecker; + private final ThreadPool threadPool; private final CounterMetric throttledTime = new CounterMetric(); public CcrRepository(RepositoryMetaData metadata, Client client, CcrLicenseChecker ccrLicenseChecker, Settings settings, - CcrSettings ccrSettings) { + CcrSettings ccrSettings, ThreadPool threadPool) { this.metadata = metadata; this.ccrSettings = ccrSettings; assert metadata.name().startsWith(NAME_PREFIX) : "CcrRepository metadata.name() must start with: " + NAME_PREFIX; this.remoteClusterAlias = Strings.split(metadata.name(), NAME_PREFIX)[1]; this.ccrLicenseChecker = ccrLicenseChecker; this.client = client; + this.threadPool = threadPool; } @Override @@ -325,7 +337,7 @@ private RestoreSession openSession(String repositoryName, Client remoteClient, S PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse response = remoteClient.execute(PutCcrRestoreSessionAction.INSTANCE, new PutCcrRestoreSessionRequest(sessionUUID, leaderShardId)).actionGet(ccrSettings.getRecoveryActionTimeout()); return new RestoreSession(repositoryName, remoteClient, sessionUUID, response.getNode(), indexShard, recoveryState, - response.getStoreFileMetaData(), response.getMappingVersion(), ccrSettings, throttledTime::inc); + response.getStoreFileMetaData(), response.getMappingVersion(), threadPool, ccrSettings, throttledTime::inc); } private static class RestoreSession extends FileRestoreContext implements Closeable { @@ -337,107 +349,186 @@ private static class RestoreSession extends FileRestoreContext implements Closea private final long mappingVersion; private final CcrSettings ccrSettings; private final LongConsumer throttleListener; + private final ThreadPool threadPool; RestoreSession(String repositoryName, Client remoteClient, String sessionUUID, DiscoveryNode node, IndexShard indexShard, RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, long mappingVersion, - CcrSettings ccrSettings, LongConsumer throttleListener) { + ThreadPool threadPool, CcrSettings ccrSettings, LongConsumer throttleListener) { super(repositoryName, indexShard, SNAPSHOT_ID, recoveryState, Math.toIntExact(ccrSettings.getChunkSize().getBytes())); this.remoteClient = remoteClient; this.sessionUUID = sessionUUID; this.node = node; this.sourceMetaData = sourceMetaData; this.mappingVersion = mappingVersion; + this.threadPool = threadPool; this.ccrSettings = ccrSettings; this.throttleListener = throttleListener; } void restoreFiles() throws IOException { - ArrayList fileInfos = new ArrayList<>(); + ArrayList fileInfos = new ArrayList<>(); for (StoreFileMetaData fileMetaData : sourceMetaData) { ByteSizeValue fileSize = new ByteSizeValue(fileMetaData.length()); - fileInfos.add(new BlobStoreIndexShardSnapshot.FileInfo(fileMetaData.name(), fileMetaData, fileSize)); + fileInfos.add(new FileInfo(fileMetaData.name(), fileMetaData, fileSize)); } SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos); restore(snapshotFiles); } - @Override - protected InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo) { - return new RestoreFileInputStream(remoteClient, sessionUUID, node, fileInfo.metadata(), ccrSettings, throttleListener); - } + private static class FileSession { + FileSession(long lastTrackedSeqNo, long lastOffset) { + this.lastTrackedSeqNo = lastTrackedSeqNo; + this.lastOffset = lastOffset; + } - @Override - public void close() { - ClearCcrRestoreSessionRequest clearRequest = new ClearCcrRestoreSessionRequest(sessionUUID, node); - ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse response = - remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); + final long lastTrackedSeqNo; + final long lastOffset; } - } - - private static class RestoreFileInputStream extends InputStream { - private final Client remoteClient; - private final String sessionUUID; - private final DiscoveryNode node; - private final StoreFileMetaData fileToRecover; - private final CombinedRateLimiter rateLimiter; - private final CcrSettings ccrSettings; - private final LongConsumer throttleListener; - - private long pos = 0; + @Override + protected void restoreFiles(List filesToRecover, Store store) throws IOException { + logger.trace("[{}] starting CCR restore of {} files", shardId, filesToRecover); + + try (MultiFileWriter multiFileWriter = new MultiFileWriter(store, recoveryState.getIndex(), "", logger, () -> {})) { + final LocalCheckpointTracker requestSeqIdTracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED); + final AtomicReference> error = new AtomicReference<>(); + + final ArrayDeque remainingFiles = new ArrayDeque<>(filesToRecover); + final Map inFlightRequests = new HashMap<>(); + final Object mutex = new Object(); + + while (true) { + if (error.get() != null) { + break; + } + final FileInfo fileToRecover; + final FileSession fileSession; + synchronized (mutex) { + if (inFlightRequests.isEmpty() && remainingFiles.isEmpty()) { + break; + } + final long maxConcurrentFileChunks = ccrSettings.getMaxConcurrentFileChunks(); + if (remainingFiles.isEmpty() == false && inFlightRequests.size() < maxConcurrentFileChunks) { + for (int i = 0; i < maxConcurrentFileChunks; i++) { + if (remainingFiles.isEmpty()) { + break; + } + inFlightRequests.put(remainingFiles.pop(), new FileSession(NO_OPS_PERFORMED, 0)); + } + } + final Map.Entry minEntry = + inFlightRequests.entrySet().stream().min(Comparator.comparingLong(e -> e.getValue().lastTrackedSeqNo)).get(); + fileSession = minEntry.getValue(); + fileToRecover = minEntry.getKey(); + } + try { + requestSeqIdTracker.waitForOpsToComplete(fileSession.lastTrackedSeqNo); + synchronized (mutex) { + // if file has been removed in the mean-while, it means that restore of this file completed, so start working + // on the next one + if (inFlightRequests.containsKey(fileToRecover) == false) { + continue; + } + } + final long requestSeqId = requestSeqIdTracker.generateSeqNo(); + try { + synchronized (mutex) { + inFlightRequests.put(fileToRecover, new FileSession(requestSeqId, fileSession.lastOffset)); + } + final int bytesRequested = Math.toIntExact(Math.min(ccrSettings.getChunkSize().getBytes(), + fileToRecover.length() - fileSession.lastOffset)); + final GetCcrRestoreFileChunkRequest request = + new GetCcrRestoreFileChunkRequest(node, sessionUUID, fileToRecover.name(), bytesRequested); + logger.trace("[{}] [{}] fetching chunk for file [{}]", shardId, snapshotId, fileToRecover.name()); + + remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request, + ActionListener.wrap( + r -> threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + } + + @Override + protected void doRun() throws Exception { + final int actualChunkSize = r.getChunk().length(); + final long nanosPaused = ccrSettings.getRateLimiter().maybePause(actualChunkSize); + throttleListener.accept(nanosPaused); + final long newOffset = r.getOffset() + actualChunkSize; + assert newOffset <= fileToRecover.length(); + final boolean lastChunk = newOffset >= fileToRecover.length(); + multiFileWriter.writeFileChunk(fileToRecover.metadata(), r.getOffset(), r.getChunk(), + lastChunk); + if (lastChunk) { + synchronized (mutex) { + final FileSession session = inFlightRequests.remove(fileToRecover); + assert session != null : "session disappeared for " + fileToRecover.name(); + } + } else { + synchronized (mutex) { + final FileSession replaced = inFlightRequests.replace(fileToRecover, + new FileSession(requestSeqId, newOffset)); + assert replaced != null : "session disappeared for " + fileToRecover.name(); + } + } + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + } + }), + e -> { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + } + )); + } catch (Exception e) { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + throw e; + } + } catch (Exception e) { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + break; + } + + } + try { + requestSeqIdTracker.waitForOpsToComplete(requestSeqIdTracker.getMaxSeqNo()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ElasticsearchException(e); + } + if (error.get() != null) { + handleError(store, error.get().v2()); + } + } - private RestoreFileInputStream(Client remoteClient, String sessionUUID, DiscoveryNode node, StoreFileMetaData fileToRecover, - CcrSettings ccrSettings, LongConsumer throttleListener) { - this.remoteClient = remoteClient; - this.sessionUUID = sessionUUID; - this.node = node; - this.fileToRecover = fileToRecover; - this.ccrSettings = ccrSettings; - this.rateLimiter = ccrSettings.getRateLimiter(); - this.throttleListener = throttleListener; + logger.trace("[{}] completed CCR restore", shardId); } + private void handleError(Store store, Exception e) throws IOException { + final IOException corruptIndexException; + if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(e)) != null) { + try { + store.markStoreCorrupted(corruptIndexException); + } catch (IOException ioe) { + logger.warn("store cannot be marked as corrupted", e); + } + throw corruptIndexException; + } else { + ExceptionsHelper.reThrowIfNotNull(e); + } + } @Override - public int read() throws IOException { + protected InputStream fileInputStream(FileInfo fileInfo) { throw new UnsupportedOperationException(); } @Override - public int read(byte[] bytes, int off, int len) throws IOException { - long remainingBytes = fileToRecover.length() - pos; - if (remainingBytes <= 0) { - return 0; - } - - int bytesRequested = (int) Math.min(remainingBytes, len); - - long nanosPaused = rateLimiter.maybePause(bytesRequested); - throttleListener.accept(nanosPaused); - - String fileName = fileToRecover.name(); - GetCcrRestoreFileChunkRequest request = new GetCcrRestoreFileChunkRequest(node, sessionUUID, fileName, bytesRequested); - GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse response = - remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request).actionGet(ccrSettings.getRecoveryActionTimeout()); - BytesReference fileChunk = response.getChunk(); - - int bytesReceived = fileChunk.length(); - if (bytesReceived > bytesRequested) { - throw new IOException("More bytes [" + bytesReceived + "] received than requested [" + bytesRequested + "]"); - } - - long leaderOffset = response.getOffset(); - assert pos == leaderOffset : "Position [" + pos + "] should be equal to the leader file offset [" + leaderOffset + "]."; - - try (StreamInput streamInput = fileChunk.streamInput()) { - int bytesRead = streamInput.read(bytes, 0, bytesReceived); - assert bytesRead == bytesReceived : "Did not read the correct number of bytes"; - } - - pos += bytesReceived; - - return bytesReceived; + public void close() { + ClearCcrRestoreSessionRequest clearRequest = new ClearCcrRestoreSessionRequest(sessionUUID, node); + ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse response = + remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } - } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java index f093143112d3d..65fb14fdb95ec 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java @@ -42,8 +42,6 @@ import java.util.HashSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Consumer; import java.util.function.LongConsumer; public class CcrRestoreSourceService extends AbstractLifecycleComponent implements IndexEventListener { @@ -52,7 +50,6 @@ public class CcrRestoreSourceService extends AbstractLifecycleComponent implemen private final Map onGoingRestores = ConcurrentCollections.newConcurrentMap(); private final Map> sessionsForShard = new HashMap<>(); - private final CopyOnWriteArrayList> closeSessionListeners = new CopyOnWriteArrayList<>(); private final ThreadPool threadPool; private final CcrSettings ccrSettings; private final CounterMetric throttleTime = new CounterMetric(); @@ -93,12 +90,6 @@ protected synchronized void doClose() throws IOException { onGoingRestores.clear(); } - // TODO: The listeners are for testing. Once end-to-end file restore is implemented and can be tested, - // these should be removed. - public void addCloseSessionListener(Consumer listener) { - closeSessionListeners.add(listener); - } - public synchronized Store.MetadataSnapshot openSession(String sessionUUID, IndexShard indexShard) throws IOException { boolean success = false; RestoreSession restore = null; @@ -165,9 +156,7 @@ private void internalCloseSession(String sessionUUID, boolean throwIfSessionMiss } } } - closeSessionListeners.forEach(c -> c.accept(sessionUUID)); restore.decRef(); - } private Scheduler.Cancellable scheduleTimeout(String sessionUUID) { @@ -255,6 +244,7 @@ protected void closeInternal() { assert keyedLock.hasLockedKeys() == false : "Should not hold any file locks when closing"; timeoutTask.cancel(); IOUtils.closeWhileHandlingException(cachedInputs.values()); + IOUtils.closeWhileHandlingException(commitRef); } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 80bded6a5d1d3..d5c3304e966e6 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -114,7 +114,7 @@ public void testFollowIndex() throws Exception { final int firstBatchNumDocs; // Sometimes we want to index a lot of documents to ensure that the recovery works with larger files if (rarely()) { - firstBatchNumDocs = randomIntBetween(1800, 2000); + firstBatchNumDocs = randomIntBetween(1800, 10000); } else { firstBatchNumDocs = randomIntBetween(10, 64); } @@ -127,6 +127,7 @@ public void testFollowIndex() throws Exception { waitForDocs(firstBatchNumDocs, indexer); indexer.assertNoFailures(); + logger.info("Executing put follow"); boolean waitOnAll = randomBoolean(); final PutFollowAction.Request followRequest; @@ -176,6 +177,8 @@ public void testFollowIndex() throws Exception { logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); indexer.continueIndexing(secondBatchNumDocs); + waitForDocs(firstBatchNumDocs + secondBatchNumDocs, indexer); + final Map secondBatchNumDocsPerShard = new HashMap<>(); final ShardStats[] secondBatchShardStats = leaderClient().admin().indices().prepareStats("index1").get().getIndex("index1").getShards(); @@ -194,6 +197,7 @@ public void testFollowIndex() throws Exception { assertTrue("Doc with id [" + docId + "] is missing", getResponse.isExists()); }); } + pauseFollow("index2"); assertMaxSeqNoOfUpdatesIsTransferred(resolveLeaderIndex("index1"), resolveFollowerIndex("index2"), numberOfPrimaryShards); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java index 1c3c0da3d3c8a..2b5011d45139f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java @@ -12,10 +12,9 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; @@ -25,7 +24,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; @@ -39,10 +38,8 @@ public void setUp() throws Exception { super.setUp(); Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node").build(); taskQueue = new DeterministicTaskQueue(settings, random()); - Set> registeredSettings = Sets.newHashSet(CcrSettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, - CcrSettings.RECOVERY_MAX_BYTES_PER_SECOND, CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, - CcrSettings.RECOVERY_CHUNK_SIZE); - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, registeredSettings); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, CcrSettings.getSettings() + .stream().filter(s -> s.hasNodeScope()).collect(Collectors.toSet())); restoreSourceService = new CcrRestoreSourceService(taskQueue.getThreadPool(), new CcrSettings(Settings.EMPTY, clusterSettings)); } @@ -202,7 +199,10 @@ public void testGetSessionDoesNotLeakFileIfClosed() throws IOException { sessionReader.readFileBytes(files.get(1).name(), new BytesArray(new byte[10])); } + assertTrue(EngineTestCase.hasSnapshottedCommits(IndexShardTestCase.getEngine(indexShard))); restoreSourceService.closeSession(sessionUUID); + assertFalse(EngineTestCase.hasSnapshottedCommits(IndexShardTestCase.getEngine(indexShard))); + closeShards(indexShard); // Exception will be thrown if file is not closed. } From 431c4fd55e4f317312bb616c20518deb67e17180 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Fri, 8 Feb 2019 16:52:33 -0600 Subject: [PATCH 05/12] fix dissect doc "ip" --> "clientip" (#38545) Forward port of #38512. --- docs/reference/ingest/processors/dissect.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/ingest/processors/dissect.asciidoc b/docs/reference/ingest/processors/dissect.asciidoc index 0c04e7ed07396..aa54cb4411f3e 100644 --- a/docs/reference/ingest/processors/dissect.asciidoc +++ b/docs/reference/ingest/processors/dissect.asciidoc @@ -170,7 +170,7 @@ Named skip key modifier example | *Pattern* | `%{clientip} %{?ident} %{?auth} [%{@timestamp}]` | *Input* | 1.2.3.4 - - [30/Apr/1998:22:00:52 +0000] | *Result* a| -* ip = 1.2.3.4 +* clientip = 1.2.3.4 * @timestamp = 30/Apr/1998:22:00:52 +0000 |====== From 4775919cacde9bb271e1b6671c6e91642c7fa21e Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 8 Feb 2019 15:48:42 -0800 Subject: [PATCH 06/12] [DOCS] Adds placeholders for v8 highlights, breaking changes, release notes (#38641) --- docs/reference/migration/index.asciidoc | 4 +- docs/reference/migration/migrate_7_0.asciidoc | 64 -- .../migrate_7_0/aggregations.asciidoc | 34 - .../migration/migrate_7_0/analysis.asciidoc | 41 -- .../migration/migrate_7_0/api.asciidoc | 156 ----- .../migration/migrate_7_0/cluster.asciidoc | 27 - .../migration/migrate_7_0/discovery.asciidoc | 48 -- .../migration/migrate_7_0/indices.asciidoc | 110 ---- .../migration/migrate_7_0/java.asciidoc | 47 -- .../migration/migrate_7_0/logging.asciidoc | 42 -- .../migrate_7_0/low_level_restclient.asciidoc | 17 - .../migration/migrate_7_0/mappings.asciidoc | 74 --- .../migration/migrate_7_0/node.asciidoc | 15 - .../migration/migrate_7_0/packaging.asciidoc | 18 - .../migration/migrate_7_0/plugins.asciidoc | 77 --- .../migration/migrate_7_0/restclient.asciidoc | 32 - .../migration/migrate_7_0/scripting.asciidoc | 37 -- .../migration/migrate_7_0/search.asciidoc | 248 -------- .../migration/migrate_7_0/settings.asciidoc | 206 ------ .../migrate_7_0/snapshotstats.asciidoc | 15 - docs/reference/migration/migrate_8_0.asciidoc | 12 + docs/reference/release-notes.asciidoc | 6 +- .../release-notes/7.0.0-alpha1.asciidoc | 32 - .../release-notes/7.0.0-alpha2.asciidoc | 585 ------------------ .../release-notes/8.0.0-alpha1.asciidoc | 8 + .../release-notes/highlights-7.0.0.asciidoc | 9 - .../release-notes/highlights-8.0.0.asciidoc | 9 + .../release-notes/highlights.asciidoc | 4 +- 28 files changed, 36 insertions(+), 1941 deletions(-) delete mode 100644 docs/reference/migration/migrate_7_0.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/aggregations.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/analysis.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/api.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/cluster.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/discovery.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/indices.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/java.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/logging.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/low_level_restclient.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/mappings.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/node.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/packaging.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/plugins.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/restclient.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/scripting.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/search.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/settings.asciidoc delete mode 100644 docs/reference/migration/migrate_7_0/snapshotstats.asciidoc create mode 100644 docs/reference/migration/migrate_8_0.asciidoc delete mode 100644 docs/reference/release-notes/7.0.0-alpha1.asciidoc delete mode 100644 docs/reference/release-notes/7.0.0-alpha2.asciidoc create mode 100644 docs/reference/release-notes/8.0.0-alpha1.asciidoc delete mode 100644 docs/reference/release-notes/highlights-7.0.0.asciidoc create mode 100644 docs/reference/release-notes/highlights-8.0.0.asciidoc diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index aa74068419df0..e5ba82c406747 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -21,5 +21,7 @@ For more information, see <>. See also <> and <>. +* <> + -- -include::migrate_7_0.asciidoc[] +include::migrate_8_0.asciidoc[] diff --git a/docs/reference/migration/migrate_7_0.asciidoc b/docs/reference/migration/migrate_7_0.asciidoc deleted file mode 100644 index 25c2e3eef440d..0000000000000 --- a/docs/reference/migration/migrate_7_0.asciidoc +++ /dev/null @@ -1,64 +0,0 @@ -[[breaking-changes-7.0]] -== Breaking changes in 7.0 -++++ -7.0 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to Elasticsearch 7.0. - -See also <> and <>. - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> - -[float] -=== Indices created before 7.0 - -Elasticsearch 7.0 can read indices created in version 6.0 or above. An -Elasticsearch 7.0 node will not start in the presence of indices created in a -version of Elasticsearch before 6.0. - -[IMPORTANT] -.Reindex indices from Elasticsearch 5.x or before -========================================= - -Indices created in Elasticsearch 5.x or before will need to be reindexed with -Elasticsearch 6.x in order to be readable by Elasticsearch 7.x. - -========================================= - -include::migrate_7_0/aggregations.asciidoc[] -include::migrate_7_0/analysis.asciidoc[] -include::migrate_7_0/cluster.asciidoc[] -include::migrate_7_0/discovery.asciidoc[] -include::migrate_7_0/indices.asciidoc[] -include::migrate_7_0/mappings.asciidoc[] -include::migrate_7_0/search.asciidoc[] -include::migrate_7_0/packaging.asciidoc[] -include::migrate_7_0/plugins.asciidoc[] -include::migrate_7_0/api.asciidoc[] -include::migrate_7_0/java.asciidoc[] -include::migrate_7_0/settings.asciidoc[] -include::migrate_7_0/scripting.asciidoc[] -include::migrate_7_0/snapshotstats.asciidoc[] -include::migrate_7_0/restclient.asciidoc[] -include::migrate_7_0/low_level_restclient.asciidoc[] -include::migrate_7_0/logging.asciidoc[] -include::migrate_7_0/node.asciidoc[] diff --git a/docs/reference/migration/migrate_7_0/aggregations.asciidoc b/docs/reference/migration/migrate_7_0/aggregations.asciidoc deleted file mode 100644 index 2b8c2ed9cb783..0000000000000 --- a/docs/reference/migration/migrate_7_0/aggregations.asciidoc +++ /dev/null @@ -1,34 +0,0 @@ -[float] -[[breaking_70_aggregations_changes]] -=== Aggregations changes - -[float] -==== Deprecated `global_ordinals_hash` and `global_ordinals_low_cardinality` execution hints for terms aggregations have been removed - -These `execution_hint` are removed and should be replaced by `global_ordinals`. - -[float] -==== `search.max_buckets` in the cluster setting - -The dynamic cluster setting named `search.max_buckets` now defaults -to 10,000 (instead of unlimited in the previous version). -Requests that try to return more than the limit will fail with an exception. - -[float] -==== `missing` option of the `composite` aggregation has been removed - -The `missing` option of the `composite` aggregation, deprecated in 6.x, -has been removed. `missing_bucket` should be used instead. - -[float] -==== Replaced `params._agg` with `state` context variable in scripted metric aggregations - -The object used to share aggregation state between the scripts in a Scripted Metric -Aggregation is now a variable called `state` available in the script context, rather than -being provided via the `params` object as `params._agg`. - -[float] -==== Make metric aggregation script parameters `reduce_script` and `combine_script` mandatory - -The metric aggregation has been changed to require these two script parameters to ensure users are -explicitly defining how their data is processed. diff --git a/docs/reference/migration/migrate_7_0/analysis.asciidoc b/docs/reference/migration/migrate_7_0/analysis.asciidoc deleted file mode 100644 index 36ad41be09aa1..0000000000000 --- a/docs/reference/migration/migrate_7_0/analysis.asciidoc +++ /dev/null @@ -1,41 +0,0 @@ -[float] -[[breaking_70_analysis_changes]] -=== Analysis changes - -[float] -==== Limiting the number of tokens produced by _analyze - -To safeguard against out of memory errors, the number of tokens that can be produced -using the `_analyze` endpoint has been limited to 10000. This default limit can be changed -for a particular index with the index setting `index.analyze.max_token_count`. - -[float] -==== Limiting the length of an analyzed text during highlighting - -Highlighting a text that was indexed without offsets or term vectors, -requires analysis of this text in memory real time during the search request. -For large texts this analysis may take substantial amount of time and memory. -To protect against this, the maximum number of characters that will be analyzed has been -limited to 1000000. This default limit can be changed -for a particular index with the index setting `index.highlight.max_analyzed_offset`. - -[float] -==== `delimited_payload_filter` renaming - -The `delimited_payload_filter` was deprecated and renamed to `delimited_payload` in 6.2. -Using it in indices created before 7.0 will issue deprecation warnings. Using the old -name in new indices created in 7.0 will throw an error. Use the new name `delimited_payload` -instead. - -[float] -==== `standard` filter has been removed - -The `standard` token filter has been removed because it doesn't change anything in the stream. - -[float] -==== Deprecated standard_html_strip analyzer - -The `standard_html_strip` analyzer has been deprecated, and should be replaced -with a combination of the `standard` tokenizer and `html_strip` char_filter. -Indexes created using this analyzer will still be readable in elasticsearch 7.0, -but it will not be possible to create new indexes using it. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/api.asciidoc b/docs/reference/migration/migrate_7_0/api.asciidoc deleted file mode 100644 index 6c1d03760f904..0000000000000 --- a/docs/reference/migration/migrate_7_0/api.asciidoc +++ /dev/null @@ -1,156 +0,0 @@ -[float] -[[breaking_70_api_changes]] -=== API changes - -[float] -==== Internal Versioning is no longer supported for optimistic concurrency control - -Elasticsearch maintains a numeric version field for each document it stores. That field -is incremented by one with every change to the document. Until 7.0.0 the API allowed using -that field for optimistic concurrency control, i.e., making a write operation conditional -on the current document version. Sadly, that approach is flawed because the value of the -version doesn't always uniquely represent a change to the document. If a primary fails -while handling a write operation, it may expose a version that will then be reused by the -new primary. - -Due to that issue, internal versioning can no longer be used and is replaced by a new -method based on sequence numbers. See <> for more details. - -Note that the `external` versioning type is still fully supported. - -[float] -==== Camel case and underscore parameters deprecated in 6.x have been removed -A number of duplicate parameters deprecated in 6.x have been removed from -Bulk request, Multi Get request, Term Vectors request, and More Like This Query -requests. - -The following camel case parameters have been removed: - -* `opType` -* `versionType`, `_versionType` - -The following parameters starting with underscore have been removed: - -* `_parent` -* `_retry_on_conflict` -* `_routing` -* `_version` -* `_version_type` - -Instead of these removed parameters, use their non camel case equivalents without -starting underscore, e.g. use `version_type` instead of `_version_type` or `versionType`. - -[float] -==== Thread pool info - -In previous versions of Elasticsearch, the thread pool info returned in the -<> returned `min` and `max` values reflecting -the configured minimum and maximum number of threads that could be in each -thread pool. The trouble with this representation is that it does not align with -the configuration parameters used to configure thread pools. For -<>, the minimum number of threads is -configured by a parameter called `core` and the maximum number of threads is -configured by a parameter called `max`. For <>, there is only one configuration parameter along these lines and that -parameter is called `size`, reflecting the fixed number of threads in the -pool. This discrepancy between the API and the configuration parameters has been -rectified. Now, the API will report `core` and `max` for scaling thread pools, -and `size` for fixed thread pools. - -Similarly, in the cat thread pool API the existing `size` output has been -renamed to `pool_size` which reflects the number of threads currently in the -pool; the shortcut for this value has been changed from `s` to `psz`. The `min` -output has been renamed to `core` with a shortcut of `cr`, the shortcut for -`max` has been changed to `mx`, and the `size` output with a shortcut of `sz` -has been reused to report the configured number of threads in the pool. This -aligns the output of the API with the configuration values for thread -pools. Note that `core` and `max` will be populated for scaling thread pools, -and `size` will be populated for fixed thread pools. - -[float] -==== The parameter `fields` deprecated in 6.x has been removed from Bulk request -and Update request. The Update API returns `400 - Bad request` if request contains -unknown parameters (instead of ignored in the previous version). - -[float] -[[remove-suggest-metric]] -==== Remove support for `suggest` metric/index metric in indices stats and nodes stats APIs - -Previously, `suggest` stats were folded into `search` stats. Support for the -`suggest` metric on the indices stats and nodes stats APIs remained for -backwards compatibility. Backwards support for the `suggest` metric was -deprecated in 6.3.0 and now removed in 7.0.0. - -[[remove-field-caps-body]] - -In the past, `fields` could be provided either as a parameter, or as part of the request -body. Specifying `fields` in the request body as opposed to a parameter was deprecated -in 6.4.0, and is now unsupported in 7.0.0. - -[float] -==== `copy_settings` is deprecated on shrink and split APIs - -Versions of Elasticsearch prior to 6.4.0 did not copy index settings on shrink -and split operations. Starting with Elasticsearch 7.0.0, the default behavior -will be for such settings to be copied on such operations. To enable users in -6.4.0 to transition in 6.4.0 to the default behavior in 7.0.0, the -`copy_settings` parameter was added on the REST layer. As this behavior will be -the only behavior in 8.0.0, this parameter is deprecated in 7.0.0 for removal in -8.0.0. - -[float] -==== The deprecated stored script contexts have now been removed -When putting stored scripts, support for storing them with the deprecated `template` context or without a context is -now removed. Scripts must be stored using the `script` context as mentioned in the documentation. - -[float] -==== Removed Get Aliases API limitations when {security-features} are enabled - -The behavior and response codes of the get aliases API no longer vary -depending on whether {security-features} are enabled. Previously a -404 - NOT FOUND (IndexNotFoundException) could be returned in case the -current user was not authorized for any alias. An empty response with -status 200 - OK is now returned instead at all times. - -[float] -==== Put User API response no longer has `user` object - -The Put User API response was changed in 6.5.0 to add the `created` field -outside of the user object where it previously had been. In 7.0.0 the user -object has been removed in favor of the top level `created` field. - -[float] -==== Source filtering url parameters `_source_include` and `_source_exclude` have been removed - -The deprecated in 6.x url parameters are now removed. Use `_source_includes` and `_source_excludes` instead. - -[float] -==== Multi Search Request metadata validation - -MultiSearchRequests issued through `_msearch` now validate all keys in the metadata section. Previously unknown keys were ignored -while now an exception is thrown. - -[float] -==== Deprecated graph endpoints removed - -The deprecated graph endpoints (those with `/_graph/_explore`) have been -removed. - - -[float] -==== Deprecated `_termvector` endpoint removed - -The `_termvector` endpoint was deprecated in 2.0 and has now been removed. -The endpoint `_termvectors` (plural) should be used instead. - -[float] -==== When {security-features} are enabled, index monitoring APIs over restricted indices are not authorized implicitly anymore - -Restricted indices (currently only `.security-6` and `.security`) are special internal -indices that require setting the `allow_restricted_indices` flag on every index -permission that covers them. If this flag is `false` (default) the permission -will not cover these and actions against them will not be authorized. -However, the monitoring APIs were the only exception to this rule. This exception -has been forfeited and index monitoring privileges have to be granted explicitly, -using the `allow_restricted_indices` flag on the permission (as any other index -privilege). diff --git a/docs/reference/migration/migrate_7_0/cluster.asciidoc b/docs/reference/migration/migrate_7_0/cluster.asciidoc deleted file mode 100644 index bfe7d5df2d094..0000000000000 --- a/docs/reference/migration/migrate_7_0/cluster.asciidoc +++ /dev/null @@ -1,27 +0,0 @@ -[float] -[[breaking_70_cluster_changes]] -=== Cluster changes - -[float] -==== `:` is no longer allowed in cluster name - -Due to cross-cluster search using `:` to separate a cluster and index name, -cluster names may no longer contain `:`. - -[float] -==== New default for `wait_for_active_shards` parameter of the open index command - -The default value for the `wait_for_active_shards` parameter of the open index API -is changed from 0 to 1, which means that the command will now by default wait for all -primary shards of the opened index to be allocated. - -[float] -==== Shard preferences `_primary`, `_primary_first`, `_replica`, and `_replica_first` are removed -These shard preferences are removed in favour of the `_prefer_nodes` and `_only_nodes` preferences. - -[float] -==== Cluster-wide shard soft limit -Clusters now have soft limits on the total number of open shards in the cluster -based on the number of nodes and the `cluster.max_shards_per_node` cluster -setting, to prevent accidental operations that would destabilize the cluster. -More information can be found in the <>. diff --git a/docs/reference/migration/migrate_7_0/discovery.asciidoc b/docs/reference/migration/migrate_7_0/discovery.asciidoc deleted file mode 100644 index 56449625246cd..0000000000000 --- a/docs/reference/migration/migrate_7_0/discovery.asciidoc +++ /dev/null @@ -1,48 +0,0 @@ -[float] -[[breaking_70_discovery_changes]] -=== Discovery changes - -[float] -==== Cluster bootstrapping is required if discovery is configured - -The first time a cluster is started, `cluster.initial_master_nodes` must be set -to perform cluster bootstrapping. It should contain the names of the -master-eligible nodes in the initial cluster and be defined on every -master-eligible node in the cluster. See <> for an example, and the -<> describes this setting in more detail. - -The `discovery.zen.minimum_master_nodes` setting is permitted, but ignored, on -7.x nodes. - -[float] -==== Removing master-eligible nodes sometimes requires voting exclusions - -If you wish to remove half or more of the master-eligible nodes from a cluster, -you must first exclude the affected nodes from the voting configuration using -the <>. -If you remove fewer than half of the master-eligible nodes at the same time, -voting exclusions are not required. If you remove only master-ineligible nodes -such as data-only nodes or coordinating-only nodes, voting exclusions are not -required. Likewise, if you add nodes to the cluster, voting exclusions are not -required. - -[float] -==== Discovery configuration is required in production - -Production deployments of Elasticsearch now require at least one of the -following settings to be specified in the `elasticsearch.yml` configuration -file: - -- `discovery.seed_hosts` -- `discovery.seed_providers` -- `cluster.initial_master_nodes` - -[float] -==== New name for `no_master_block` setting - -The `discovery.zen.no_master_block` setting is now known as -`cluster.no_master_block`. Any value set for `discovery.zen.no_master_block` is -now ignored. You should remove this setting and, if needed, set -`cluster.no_master_block` appropriately after the upgrade. diff --git a/docs/reference/migration/migrate_7_0/indices.asciidoc b/docs/reference/migration/migrate_7_0/indices.asciidoc deleted file mode 100644 index f832ae7feb6d3..0000000000000 --- a/docs/reference/migration/migrate_7_0/indices.asciidoc +++ /dev/null @@ -1,110 +0,0 @@ -[float] -[[breaking_70_indices_changes]] -=== Indices changes - -[float] -==== Index creation no longer defaults to five shards -Previous versions of Elasticsearch defaulted to creating five shards per index. -Starting with 7.0.0, the default is now one shard per index. - -[float] -==== `:` is no longer allowed in index name - -Due to cross-cluster search using `:` to separate a cluster and index name, -index names may no longer contain `:`. - -[float] -==== `index.unassigned.node_left.delayed_timeout` may no longer be negative - -Negative values were interpreted as zero in earlier versions but are no -longer accepted. - -[float] -==== `_flush` and `_force_merge` will no longer refresh - -In previous versions issuing a `_flush` or `_force_merge` (with `flush=true`) -had the undocumented side-effect of refreshing the index which made new documents -visible to searches and non-realtime GET operations. From now on these operations -don't have this side-effect anymore. To make documents visible an explicit `_refresh` -call is needed unless the index is refreshed by the internal scheduler. - -[float] -==== Limit to the difference between max_size and min_size in NGramTokenFilter and NGramTokenizer - -To safeguard against creating too many index terms, the difference between `max_ngram` and -`min_ngram` in `NGramTokenFilter` and `NGramTokenizer` has been limited to 1. This default -limit can be changed with the index setting `index.max_ngram_diff`. Note that if the limit is -exceeded a error is thrown only for new indices. For existing pre-7.0 indices, a deprecation -warning is logged. - -[float] -==== Limit to the difference between max_size and min_size in ShingleTokenFilter - -To safeguard against creating too many tokens, the difference between `max_shingle_size` and -`min_shingle_size` in `ShingleTokenFilter` has been limited to 3. This default -limit can be changed with the index setting `index.max_shingle_diff`. Note that if the limit is -exceeded a error is thrown only for new indices. For existing pre-7.0 indices, a deprecation -warning is logged. - -[float] -==== Document distribution changes - -Indices created with version `7.0.0` onwards will have an automatic `index.number_of_routing_shards` -value set. This might change how documents are distributed across shards depending on how many -shards the index has. In order to maintain the exact same distribution as a pre `7.0.0` index, the -`index.number_of_routing_shards` must be set to the `index.number_of_shards` at index creation time. -Note: if the number of routing shards equals the number of shards `_split` operations are not supported. - -[float] -==== Skipped background refresh on search idle shards - -Shards belonging to an index that does not have an explicit -`index.refresh_interval` configured will no longer refresh in the background -once the shard becomes "search idle", ie the shard hasn't seen any search -traffic for `index.search.idle.after` seconds (defaults to `30s`). Searches -that access a search idle shard will be "parked" until the next refresh -happens. Indexing requests with `wait_for_refresh` will also trigger -a background refresh. - -[float] -==== Remove deprecated url parameters for Clear Indices Cache API - -The following previously deprecated url parameter have been removed: - -* `filter` - use `query` instead -* `filter_cache` - use `query` instead -* `request_cache` - use `request` instead -* `field_data` - use `fielddata` instead - -[float] -==== `network.breaker.inflight_requests.overhead` increased to 2 - -Previously the in flight requests circuit breaker considered only the raw byte representation. -By bumping the value of `network.breaker.inflight_requests.overhead` from 1 to 2, this circuit -breaker considers now also the memory overhead of representing the request as a structured object. - -[float] -==== Parent circuit breaker changes - -The parent circuit breaker defines a new setting `indices.breaker.total.use_real_memory` which is -`true` by default. This means that the parent circuit breaker will trip based on currently used -heap memory instead of only considering the reserved memory by child circuit breakers. When this -setting is `true`, the default parent breaker limit also changes from 70% to 95% of the JVM heap size. -The previous behavior can be restored by setting `indices.breaker.total.use_real_memory` to `false`. - -[float] -==== Field data circuit breaker changes -As doc values have been enabled by default in earlier versions of Elasticsearch, -there is less need for fielddata. Therefore, the default value of the setting -`indices.breaker.fielddata.limit` has been lowered from 60% to 40% of the JVM -heap size. - -[float] -==== `fix` value for `index.shard.check_on_startup` is removed - -Deprecated option value `fix` for setting `index.shard.check_on_startup` is not supported. - -[float] -==== `elasticsearch-translog` is removed - -Use the `elasticsearch-shard` tool to remove corrupted translog data. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/java.asciidoc b/docs/reference/migration/migrate_7_0/java.asciidoc deleted file mode 100644 index f34b1c6ca9906..0000000000000 --- a/docs/reference/migration/migrate_7_0/java.asciidoc +++ /dev/null @@ -1,47 +0,0 @@ -[float] -[[breaking_70_java_changes]] -=== Java API changes - -[float] -==== `isShardsAcked` deprecated in `6.2` has been removed - -`isShardsAcked` has been replaced by `isShardsAcknowledged` in -`CreateIndexResponse`, `RolloverResponse` and -`CreateIndexClusterStateUpdateResponse`. - -[float] -==== `prepareExecute` removed from the client api - -The `prepareExecute` method which created a request builder has been -removed from the client api. Instead, construct a builder for the -appropriate request directly. - -[float] -==== Some Aggregation classes have moved packages - -* All classes present in `org.elasticsearch.search.aggregations.metrics.*` packages -were moved to a single `org.elasticsearch.search.aggregations.metrics` package. - -* All classes present in `org.elasticsearch.search.aggregations.pipeline.*` packages -were moved to a single `org.elasticsearch.search.aggregations.pipeline` package. In -addition, `org.elasticsearch.search.aggregations.pipeline.PipelineAggregationBuilders` -was moved to `org.elasticsearch.search.aggregations.PipelineAggregationBuilders` - - -[float] -==== `Retry.withBackoff` methods with `Settings` removed - -The variants of `Retry.withBackoff` that included `Settings` have been removed -because `Settings` is no longer needed. - -[float] -==== Deprecated method `Client#termVector` removed - -The client method `termVector`, deprecated in 2.0, has been removed. The method -`termVectors` (plural) should be used instead. - -[float] -==== Deprecated constructor `AbstractLifecycleComponent(Settings settings)` removed - -The constructor `AbstractLifecycleComponent(Settings settings)`, deprecated in 6.7 -has been removed. The parameterless constructor should be used instead. diff --git a/docs/reference/migration/migrate_7_0/logging.asciidoc b/docs/reference/migration/migrate_7_0/logging.asciidoc deleted file mode 100644 index 1329def9a1878..0000000000000 --- a/docs/reference/migration/migrate_7_0/logging.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[float] -[[breaking_70_logging_changes]] -=== Logging changes - -[float] -==== New JSON format log files in `log` directory - -Elasticsearch now will produce additional log files in JSON format. They will be stored in `*.json` suffix files. -Following files should be expected now in log directory: -* ${cluster_name}_server.json -* ${cluster_name}_deprecation.json -* ${cluster_name}_index_search_slowlog.json -* ${cluster_name}_index_indexing_slowlog.json -* ${cluster_name}.log -* ${cluster_name}_deprecation.log -* ${cluster_name}_index_search_slowlog.log -* ${cluster_name}_index_indexing_slowlog.log -* ${cluster_name}_audit.json -* gc.log - -Note: You can configure which of these files are written by editing `log4j2.properties`. - -[float] -==== Log files ending with `*.log` deprecated -Log files with the `.log` file extension using the old pattern layout format -are now considered deprecated and the newly added JSON log file format with -the `.json` file extension should be used instead. -Note: GC logs which are written to the file `gc.log` will not be changed. - -[float] -==== Docker output in JSON format - -All Docker console logs are now in JSON format. You can distinguish logs streams with the `type` field. - -[float] -==== Audit plaintext log file removed, JSON file renamed - -Elasticsearch no longer produces the `${cluster_name}_access.log` plaintext -audit log file. The `${cluster_name}_audit.log` files also no longer exist; they -are replaced by `${cluster_name}_audit.json` files. When auditing is enabled, -auditing events are stored in these dedicated JSON log files on each node. - diff --git a/docs/reference/migration/migrate_7_0/low_level_restclient.asciidoc b/docs/reference/migration/migrate_7_0/low_level_restclient.asciidoc deleted file mode 100644 index 0820c7f01cc70..0000000000000 --- a/docs/reference/migration/migrate_7_0/low_level_restclient.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -[float] -[[breaking_70_low_level_restclient_changes]] -=== Low-level REST client changes - -[float] -==== Deprecated flavors of performRequest have been removed - -We deprecated the flavors of `performRequest` and `performRequestAsync` that -do not take `Request` objects in 6.4.0 in favor of the flavors that take -`Request` objects because those methods can be extended without breaking -backwards compatibility. - -[float] -==== Removed setHosts - -We deprecated `setHosts` in 6.4.0 in favor of `setNodes` because it supports -host metadata used by the `NodeSelector`. diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc deleted file mode 100644 index 653dd2fb4ca46..0000000000000 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ /dev/null @@ -1,74 +0,0 @@ -[float] -[[breaking_70_mappings_changes]] -=== Mapping changes - -[float] -==== The `_all` meta field is removed - -The `_all` field deprecated in 6 have now been removed. - -[float] -==== The `_uid` meta field is removed - -This field used to index a composite key formed of the `_type` and the `_id`. -Now that indices cannot have multiple types, this has been removed in favour -of `_id`. - -[float] -==== The `_default_` mapping is no longer allowed - -The `_default_` mapping has been deprecated in 6.0 and is now no longer allowed -in 7.0. Trying to configure a `_default_` mapping on 7.x indices will result in -an error. - -[float] -==== `index_options` for numeric fields has been removed - -The `index_options` field for numeric fields has been deprecated in 6 and has now been removed. - -[float] -==== Limiting the number of `nested` json objects - -To safeguard against out of memory errors, the number of nested json objects within a single -document across all fields has been limited to 10000. This default limit can be changed with -the index setting `index.mapping.nested_objects.limit`. - -[float] -==== The `update_all_types` option has been removed - -This option is useless now that all indices have at most one type. - -[float] -==== The `classic` similarity has been removed - -The `classic` similarity relied on coordination factors for scoring to be good -in presence of stopwords in the query. This feature has been removed from -Lucene, which means that the `classic` similarity now produces scores of lower -quality. It is advised to switch to `BM25` instead, which is widely accepted -as a better alternative. - -[float] -==== Similarities fail when unsupported options are provided - -An error will now be thrown when unknown configuration options are provided -to similarities. Such unknown parameters were ignored before. - -[float] -==== Changed default `geo_shape` indexing strategy - -`geo_shape` types now default to using a vector indexing approach based on Lucene's new -`LatLonShape` field type. This indexes shapes as a triangular mesh instead of decomposing -them into individual grid cells. To index using legacy prefix trees the `tree` parameter -must be explicitly set to one of `quadtree` or `geohash`. Note that these strategies are -now deprecated and will be removed in a future version. - -IMPORTANT NOTE: If using timed index creation from templates, the `geo_shape` mapping -should also be changed in the template to explicitly define `tree` to one of `geohash` -or `quadtree`. This will ensure compatibility with previously created indexes. - -[float] -==== Deprecated `geo_shape` parameters - -The following type parameters are deprecated for the `geo_shape` field type: `tree`, -`precision`, `tree_levels`, `distance_error_pct`, `points_only`, and `strategy`. They -will be removed in a future version. diff --git a/docs/reference/migration/migrate_7_0/node.asciidoc b/docs/reference/migration/migrate_7_0/node.asciidoc deleted file mode 100644 index 3b8a9d84e765d..0000000000000 --- a/docs/reference/migration/migrate_7_0/node.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[float] -[[breaking_70_node_changes]] -=== Node start up - -[float] -==== Nodes with left-behind data or metadata refuse to start -Repurposing an existing node by changing node.master or node.data to false can leave lingering on-disk metadata and -data around, which will not be accessible by the node's new role. Beside storing non-accessible data, this can lead -to situations where dangling indices are imported even though the node might not be able to host any shards, leading -to a red cluster health. To avoid this, - -* nodes with on-disk shard data and node.data set to false will refuse to start -* nodes with on-disk index/shard data and both node.master and node.data set to false will refuse to start - -Beware that such role changes done prior to the 7.0 upgrade could prevent node start up in 7.0. diff --git a/docs/reference/migration/migrate_7_0/packaging.asciidoc b/docs/reference/migration/migrate_7_0/packaging.asciidoc deleted file mode 100644 index e2380613d8f7b..0000000000000 --- a/docs/reference/migration/migrate_7_0/packaging.asciidoc +++ /dev/null @@ -1,18 +0,0 @@ -[float] -[[breaking_70_packaging_changes]] -=== Packaging changes - -[float] -[[systemd-service-file-config]] -==== systemd service file is no longer configuration - -The systemd service file `/usr/lib/systemd/system/elasticsearch.service` -was previously marked as a configuration file in rpm and deb packages. -Overrides to the systemd elasticsearch service should be made -in `/etc/systemd/system/elasticsearch.service.d/override.conf`. - -[float] -==== tar package no longer includes windows specific files - -The tar package previously included files in the `bin` directory meant only -for windows. These files have been removed. Use the `zip` package instead. diff --git a/docs/reference/migration/migrate_7_0/plugins.asciidoc b/docs/reference/migration/migrate_7_0/plugins.asciidoc deleted file mode 100644 index 05de7e85b8e92..0000000000000 --- a/docs/reference/migration/migrate_7_0/plugins.asciidoc +++ /dev/null @@ -1,77 +0,0 @@ -[float] -[[breaking_70_plugins_changes]] -=== Plugins changes - -[float] -==== Azure Repository plugin - -* The legacy azure settings which where starting with `cloud.azure.storage.` prefix have been removed. -This includes `account`, `key`, `default` and `timeout`. -You need to use settings which are starting with `azure.client.` prefix instead. - -* Global timeout setting `cloud.azure.storage.timeout` has been removed. -You must set it per azure client instead. Like `azure.client.default.timeout: 10s` for example. - -See {plugins}/repository-azure-repository-settings.html#repository-azure-repository-settings[Azure Repository settings]. - -[float] -==== Google Cloud Storage Repository plugin - -* The repository settings `application_name`, `connect_timeout` and `read_timeout` have been removed and -must now be specified in the client settings instead. - -See {plugins}/repository-gcs-client.html#repository-gcs-client[Google Cloud Storage Client Settings]. - -[float] -==== S3 Repository Plugin - -* The plugin now uses the path style access pattern for all requests. -In previous versions it was automatically determining whether to use virtual hosted style or path style -access. - -[float] -==== Analysis Plugin changes - -* The misspelled helper method `requriesAnalysisSettings(AnalyzerProvider provider)` has been -renamed to `requiresAnalysisSettings` - -[float] -==== File-based discovery plugin - -* This plugin has been removed since its functionality is now part of -Elasticsearch and requires no plugin. The location of the hosts file has moved -from `$ES_PATH_CONF/file-discovery/unicast_hosts.txt` to -`$ES_PATH_CONF/unicast_hosts.txt`. See <> for further information. - -[float] -==== Security Extensions - -As a consequence of the <>, -the `getRealmSettings` method has been removed from the `SecurityExtension` class, -and the `settings` method on `RealmConfig` now returns the node's (global) settings. -Custom security extensions should register their settings by implementing the standard -`Plugin.getSettings` method, and can retrieve them from `RealmConfig.settings()` or -using one of the `RealmConfig.getSetting` methods. -Each realm setting should be defined as an `AffixSetting` as shown in the example below: -[source,java] --------------------------------------------------- -Setting.AffixSetting MY_SETTING = Setting.affixKeySetting( - "xpack.security.authc.realms." + MY_REALM_TYPE + ".", "my_setting", - key -> Setting.simpleString(key, properties) -); --------------------------------------------------- - -The `RealmSettings.simpleString` method can be used as a convenience for the above. - -[float] -==== Tribe node removed - -Tribe node functionality has been removed in favor of -<>. - -[float] -==== Discovery implementations are no longer pluggable - -* The method `DiscoveryPlugin#getDiscoveryTypes()` was removed, so that plugins - can no longer provide their own discovery implementations. diff --git a/docs/reference/migration/migrate_7_0/restclient.asciidoc b/docs/reference/migration/migrate_7_0/restclient.asciidoc deleted file mode 100644 index 39d19c345cd95..0000000000000 --- a/docs/reference/migration/migrate_7_0/restclient.asciidoc +++ /dev/null @@ -1,32 +0,0 @@ -[float] -[[breaking_70_restclient_changes]] -=== High-level REST client changes - -[float] -==== API methods accepting `Header` argument have been removed - -All API methods accepting headers as a `Header` varargs argument, deprecated -since 6.4, have been removed in favour of the newly introduced methods that -accept instead a `RequestOptions` argument. In case you are not specifying any -header, e.g. `client.index(indexRequest)` becomes -`client.index(indexRequest, RequestOptions.DEFAULT)`. -In case you are specifying headers -e.g. `client.index(indexRequest, new Header("name" "value"))` becomes -`client.index(indexRequest, RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build());` - -[float] -==== Cluster Health API default to `cluster` level - -The Cluster Health API used to default to `shards` level to ease migration -from transport client that doesn't support the `level` parameter and always -returns information including indices and shards details. The level default -value has been aligned with the Elasticsearch default level: `cluster`. - -=== Low-level REST client changes - -[float] -==== Support for `maxRetryTimeout` removed from RestClient - -`RestClient` and `RestClientBuilder` no longer support the `maxRetryTimeout` -setting. The setting was removed as its counting mechanism was not accurate -and caused issues while adding little value. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/scripting.asciidoc b/docs/reference/migration/migrate_7_0/scripting.asciidoc deleted file mode 100644 index 99afca91e0119..0000000000000 --- a/docs/reference/migration/migrate_7_0/scripting.asciidoc +++ /dev/null @@ -1,37 +0,0 @@ -[float] -[[breaking_70_scripting_changes]] -=== Scripting changes - -[float] -==== getDate() and getDates() removed - -Fields of type `long` and `date` had `getDate()` and `getDates()` methods -(for multi valued fields) to get an object with date specific helper methods -for the current doc value. In 5.3.0, `date` fields were changed to expose -this same date object directly when calling `doc["myfield"].value`, and -the getter methods for date objects were deprecated. These methods have -now been removed. Instead, use `.value` on `date` fields, or explicitly -parse `long` fields into a date object using -`Instance.ofEpochMillis(doc["myfield"].value)`. - -[float] -==== Accessing missing document values will throw an error -`doc['field'].value` will throw an exception if -the document is missing a value for the field `field`. - -To check if a document is missing a value, you can use -`doc['field'].size() == 0`. - - -[float] -==== Script errors will return as `400` error codes - -Malformed scripts, either in search templates, ingest pipelines or search -requests, return `400 - Bad request` while they would previously return -`500 - Internal Server Error`. This also applies for stored scripts. - -[float] -==== getValues() removed - -The `ScriptDocValues#getValues()` method is deprecated in 6.6 and will -be removed in 7.0. Use `doc["foo"]` in place of `doc["foo"].values`. diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc deleted file mode 100644 index afe96fd8851a9..0000000000000 --- a/docs/reference/migration/migrate_7_0/search.asciidoc +++ /dev/null @@ -1,248 +0,0 @@ -[float] -[[breaking_70_search_changes]] -=== Search and Query DSL changes - -[float] -==== Changes to queries -* The default value for `transpositions` parameter of `fuzzy` query - has been changed to `true`. - -* The `query_string` options `use_dismax`, `split_on_whitespace`, - `all_fields`, `locale`, `auto_generate_phrase_query` and - `lowercase_expanded_terms` deprecated in 6.x have been removed. - -* Purely negative queries (only MUST_NOT clauses) now return a score of `0` - rather than `1`. - -* The boundary specified using geohashes in the `geo_bounding_box` query - now include entire geohash cell, instead of just geohash center. - -* Attempts to generate multi-term phrase queries against non-text fields - with a custom analyzer will now throw an exception. - -* An `envelope` crossing the dateline in a `geo_shape `query is now processed - correctly when specified using REST API instead of having its left and - right corners flipped. - -* Attempts to set `boost` on inner span queries will now throw a parsing exception. - -[float] -==== Adaptive replica selection enabled by default - -Adaptive replica selection has been enabled by default. If you wish to return to -the older round robin of search requests, you can use the -`cluster.routing.use_adaptive_replica_selection` setting: - -[source,js] --------------------------------------------------- -PUT /_cluster/settings -{ - "transient": { - "cluster.routing.use_adaptive_replica_selection": false - } -} --------------------------------------------------- -// CONSOLE - -[float] -==== Search API returns `400` for invalid requests - -The Search API returns `400 - Bad request` while it would previously return -`500 - Internal Server Error` in the following cases of invalid request: - -* the result window is too large -* sort is used in combination with rescore -* the rescore window is too large -* the number of slices is too large -* keep alive for scroll is too large -* number of filters in the adjacency matrix aggregation is too large -* script compilation errors - -[float] -==== Scroll queries cannot use the `request_cache` anymore - -Setting `request_cache:true` on a query that creates a scroll (`scroll=1m`) -has been deprecated in 6 and will now return a `400 - Bad request`. -Scroll queries are not meant to be cached. - -[float] -==== Scroll queries cannot use `rescore` anymore - -Including a rescore clause on a query that creates a scroll (`scroll=1m`) has -been deprecated in 6.5 and will now return a `400 - Bad request`. Allowing -rescore on scroll queries would break the scroll sort. In the 6.x line, the -rescore clause was silently ignored (for scroll queries), and it was allowed in -the 5.x line. - -[float] -==== Term Suggesters supported distance algorithms - -The following string distance algorithms were given additional names in 6.2 and -their existing names were deprecated. The deprecated names have now been -removed. - -* `levenstein` - replaced by `levenshtein` -* `jarowinkler` - replaced by `jaro_winkler` - -[float] -==== `popular` mode for Suggesters - -The `popular` mode for Suggesters (`term` and `phrase`) now uses the doc frequency -(instead of the sum of the doc frequency) of the input terms to compute the frequency -threshold for candidate suggestions. - -[float] -==== Limiting the number of terms that can be used in a Terms Query request - -Executing a Terms Query with a lot of terms may degrade the cluster performance, -as each additional term demands extra processing and memory. -To safeguard against this, the maximum number of terms that can be used in a -Terms Query request has been limited to 65536. This default maximum can be changed -for a particular index with the index setting `index.max_terms_count`. - -[float] -==== Limiting the length of regex that can be used in a Regexp Query request - -Executing a Regexp Query with a long regex string may degrade search performance. -To safeguard against this, the maximum length of regex that can be used in a -Regexp Query request has been limited to 1000. This default maximum can be changed -for a particular index with the index setting `index.max_regex_length`. - -[float] -==== Limiting the number of auto-expanded fields - -Executing queries that use automatic expansion of fields (e.g. `query_string`, `simple_query_string` -or `multi_match`) can have performance issues for indices with a large numbers of fields. -To safeguard against this, a hard limit of 1024 fields has been introduced for queries -using the "all fields" mode ("default_field": "*") or other fieldname expansions (e.g. "foo*"). - -[float] -==== Invalid `_search` request body - -Search requests with extra content after the main object will no longer be accepted -by the `_search` endpoint. A parsing exception will be thrown instead. - -[float] -==== Doc-value fields default format - -The format of doc-value fields is changing to be the same as what could be -obtained in 6.x with the special `use_field_mapping` format. This is mostly a -change for date fields, which are now formatted based on the format that is -configured in the mappings by default. This behavior can be changed by -specifying a <> within the doc-value -field. - -[float] -==== Context Completion Suggester - -The ability to query and index context enabled suggestions without context, -deprecated in 6.x, has been removed. Context enabled suggestion queries -without contexts have to visit every suggestion, which degrades the search performance -considerably. - -For geo context the value of the `path` parameter is now validated against the mapping, -and the context is only accepted if `path` points to a field with `geo_point` type. - -[float] -==== Semantics changed for `max_concurrent_shard_requests` - -`max_concurrent_shard_requests` used to limit the total number of concurrent shard -requests a single high level search request can execute. In 7.0 this changed to be the -max number of concurrent shard requests per node. The default is now `5`. - -[float] -==== `max_score` set to `null` when scores are not tracked - -`max_score` used to be set to `0` whenever scores are not tracked. `null` is now used -instead which is a more appropriate value for a scenario where scores are not available. - -[float] -==== Negative boosts are not allowed - -Setting a negative `boost` for a query or a field, deprecated in 6x, is not allowed in this version. -To deboost a specific query or field you can use a `boost` comprise between 0 and 1. - -[float] -==== Negative scores are not allowed in Function Score Query - -Negative scores in the Function Score Query are deprecated in 6.x, and are -not allowed in this version. If a negative score is produced as a result -of computation (e.g. in `script_score` or `field_value_factor` functions), -an error will be thrown. - -[float] -==== The filter context has been removed - -The `filter` context has been removed from Elasticsearch's query builders, -the distinction between queries and filters is now decided in Lucene depending -on whether queries need to access score or not. As a result `bool` queries with -`should` clauses that don't need to access the score will no longer set their -`minimum_should_match` to 1. This behavior has been deprecated in the previous -major version. - -[float] -==== `hits.total` is now an object in the search response - -The total hits that match the search request is now returned as an object -with a `value` and a `relation`. `value` indicates the number of hits that -match and `relation` indicates whether the value is accurate (`eq`) or a lower bound -(`gte`): - -``` -{ - "hits": { - "total": { <1> - "value": 1000, - "relation": "eq" - }, - ... - } -} -``` - -The "total" object in the response indicates that the query matches exactly 1000 -documents ("eq"). The `value` is always accurate (`"relation": "eq"`) when -`track_total_hits` is set to true in the request. -You can also retrieve `hits.total` as a number in the rest response by adding -`rest_total_hits_as_int=true` in the request parameter of the search request. -This parameter has been added to ease the transition to the new format and -will be removed in the next major version (8.0). - -[float] -==== `hits.total` is omitted in the response if `track_total_hits` is disabled (false) - -If `track_total_hits` is set to `false` in the search request the search response -will set `hits.total` to null and the object will not be displayed in the rest -layer. You can add `rest_total_hits_as_int=true` in the search request parameters -to get the old format back (`"total": -1`). - -[float] -==== `track_total_hits` defaults to 10,000 - -By default search request will count the total hits accurately up to `10,000` -documents. If the total number of hits that match the query is greater than this - value, the response will indicate that the returned value is a lower bound: - -[source,js] --------------------------------------------------- -{ - "_shards": ... - "timed_out": false, - "took": 100, - "hits": { - "max_score": 1.0, - "total" : { - "value": 10000, <1> - "relation": "gte" <2> - }, - "hits": ... - } -} --------------------------------------------------- -// NOTCONSOLE - -<1> There are at least 10000 documents that match the query -<2> This is a lower bound (`"gte"`). - -You can force the count to always be accurate by setting `"track_total_hits` -to true explicitly in the search request. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc deleted file mode 100644 index 2e5631b378652..0000000000000 --- a/docs/reference/migration/migrate_7_0/settings.asciidoc +++ /dev/null @@ -1,206 +0,0 @@ -[float] -[[breaking_70_settings_changes]] -=== Settings changes - -[float] -==== The default for `node.name` is now the hostname - -`node.name` now defaults to the hostname at the time when Elasticsearch -is started. Previously the default node name was the first eight characters -of the node id. It can still be configured explicitly in `elasticsearch.yml`. - -[float] -==== Percolator - -* The deprecated `index.percolator.map_unmapped_fields_as_string` setting has been removed in favour of - the `index.percolator.map_unmapped_fields_as_text` setting. - -[float] -==== Index thread pool - -* Internally, single-document index/delete/update requests are executed as bulk - requests with a single-document payload. This means that these requests are - executed on the bulk thread pool. As such, the indexing thread pool is no - longer needed and has been removed. As such, the settings - `thread_pool.index.size` and `thread_pool.index.queue_size` have been removed. - -[float] -[[write-thread-pool-fallback]] -==== Write thread pool fallback - -* The bulk thread pool was replaced by the write thread pool in 6.3.0. However, - for backwards compatibility reasons the name `bulk` was still usable as fallback - settings `thread_pool.bulk.size` and `thread_pool.bulk.queue_size` for - `thread_pool.write.size` and `thread_pool.write.queue_size`, respectively, and - the system property `es.thread_pool.write.use_bulk_as_display_name` was - available to keep the display output in APIs as `bulk` instead of `write`. - These fallback settings and this system property have been removed. - -[float] -==== Disabling memory-mapping - -* The setting `node.store.allow_mmapfs` has been renamed to `node.store.allow_mmap`. - -[float] -[[remove-http-enabled]] -==== Http enabled setting removed - -* The setting `http.enabled` previously allowed disabling binding to HTTP, only allowing -use of the transport client. This setting has been removed, as the transport client -will be removed in the future, thus requiring HTTP to always be enabled. - -[float] -[[remove-http-pipelining-setting]] -==== Http pipelining setting removed - -* The setting `http.pipelining` previously allowed disabling HTTP pipelining support. -This setting has been removed, as disabling http pipelining support on the server -provided little value. The setting `http.pipelining.max_events` can still be used to -limit the number of pipelined requests in-flight. - -[float] -==== Cross-cluster search settings renamed - -The cross-cluster search remote cluster connection infrastructure is also used -in cross-cluster replication. This means that the setting names -`search.remote.*` used for configuring cross-cluster search belie the fact that -they also apply to other situations where a connection to a remote cluster as -used. Therefore, these settings have been renamed from `search.remote.*` to -`cluster.remote.*`. For backwards compatibility purposes, we will fallback to -`search.remote.*` if `cluster.remote.*` is not set. For any such settings stored -in the cluster state, or set on dynamic settings updates, we will automatically -upgrade the setting from `search.remote.*` to `cluster.remote.*`. The fallback -settings will be removed in 8.0.0. - -[float] -[[audit-logfile-local-node-info]] -==== Audit logfile local node info - -The following settings have been removed: - -- `xpack.security.audit.logfile.prefix.emit_node_host_address`, instead use - `xpack.security.audit.logfile.emit_node_host_address` -- `xpack.security.audit.logfile.prefix.emit_node_host_name`, instead use - `xpack.security.audit.logfile.emit_node_host_name` -- `xpack.security.audit.logfile.prefix.emit_node_name`, instead use - `xpack.security.audit.logfile.emit_node_name` - -The new settings have the same meaning as the removed ones, but the `prefix` -name component is no longer meaningful as logfile audit entries are structured -JSON documents and are not prefixed by anything. -Moreover, `xpack.security.audit.logfile.emit_node_name` has changed its default -from `true` to `false`. All other settings mentioned before, have kept their -default value of `false`. - -[float] -[[include-realm-type-in-setting]] -==== Security realms settings - -The settings for all security realms must now include the realm type as part -of the setting name, and the explicit `type` setting has been removed. - -A realm that was previous configured as: -[source,yaml] --------------------------------------------------- -xpack.security.authc.realms: - ldap1: - type: ldap - order: 1 - url: "ldaps://ldap.example.com/" --------------------------------------------------- - -Must be migrated to: -[source,yaml] --------------------------------------------------- -xpack.security.authc.realms: - ldap.ldap1: - order: 1 - url: "ldaps://ldap.example.com/" --------------------------------------------------- - -Any realm specific secure settings that have been stored in the elasticsearch -keystore (such as ldap bind passwords, or passwords for ssl keys) must be updated -in a similar way. - -[float] -[[tls-setting-fallback]] -==== TLS/SSL settings - -The default TLS/SSL settings, which were prefixed by `xpack.ssl`, have been removed. -The removal of these default settings also removes the ability for a component to -fallback to a default configuration when using TLS. Each component (realm, transport, http, -http client, etc) must now be configured with their own settings for TLS if it is being -used. - -[float] -[[tls-v1-removed]] -==== TLS v1.0 disabled - -TLS version 1.0 is now disabled by default as it suffers from -https://www.owasp.org/index.php/Transport_Layer_Protection_Cheat_Sheet#Rule_-_Only_Support_Strong_Protocols[known security issues]. -The default protocols are now TLSv1.3 (if supported), TLSv1.2 and TLSv1.1. -You can enable TLS v1.0 by configuring the relevant `ssl.supported_protocols` setting to include `"TLSv1"`, for example: -[source,yaml] --------------------------------------------------- -xpack.security.http.ssl.supported_protocols: [ "TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1" ] --------------------------------------------------- - -[float] -[[trial-explicit-security]] -==== Security on Trial Licenses - -On trial licenses, `xpack.security.enabled` defaults to `false`. - -In prior versions, a trial license would automatically enable security if either - -* `xpack.security.transport.enabled` was `true`; _or_ -* the trial license was generated on a version of X-Pack from 6.2 or earlier. - -This behaviour has been now removed, so security is only enabled if: - -* `xpack.security.enabled` is `true`; _or_ -* `xpack.security.enabled` is not set, and a gold or platinum license is installed. - -[float] -[[watcher-notifications-account-settings]] -==== Watcher notifications account settings - -The following settings have been removed in favor of the secure variants. -The <> have to be defined inside each cluster -node's keystore, i.e., they are not to be specified via the cluster settings API. - -- `xpack.notification.email.account..smtp.password`, instead use -`xpack.notification.email.account..smtp.secure_password` -- `xpack.notification.hipchat.account..auth_token`, instead use -`xpack.notification.hipchat.account..secure_auth_token` -- `xpack.notification.jira.account..url`, instead use -`xpack.notification.jira.account..secure_url` -- `xpack.notification.jira.account..user`, instead use -`xpack.notification.jira.account..secure_user` -- `xpack.notification.jira.account..password`, instead use -`xpack.notification.jira.account..secure_password` -- `xpack.notification.pagerduty.account..service_api_key`, instead use -`xpack.notification.pagerduty.account..secure_service_api_key` -- `xpack.notification.slack.account..url`, instead use -`xpack.notification.slack.account..secure_url` - -[float] -[[remove-audit-index-output]] -==== Audit index output type removed - -All the settings under the `xpack.security.audit.index` namespace have been -removed. In addition, the `xpack.security.audit.outputs` setting has been -removed as well. - -These settings enabled and configured the audit index output type. This output -type has been removed because it was unreliable in certain scenarios and this -could have lead to dropping audit events while the operations on the system -were allowed to continue as usual. The recommended replacement is the -use of the `logfile` audit output type and using other components from the -Elastic Stack to handle the indexing part. - -[float] -[[ingest-user-agent-ecs-always]] -==== Ingest User Agent processor always uses `ecs` output format -The deprecated `ecs` setting for the user agent ingest processor has been -removed. https://github.com/elastic/ecs[ECS] format is now the default. diff --git a/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc b/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc deleted file mode 100644 index 2098eb3574ca8..0000000000000 --- a/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[float] -[[breaking_70_snapshotstats_changes]] -=== Snapshot stats changes - -Snapshot stats details are provided in a new structured way: - -* `total` section for all the files that are referenced by the snapshot. -* `incremental` section for those files that actually needed to be copied over as part of the incremental snapshotting. -* In case of a snapshot that's still in progress, there's also a `processed` section for files that are in the process of being copied. - -[float] -==== Deprecated `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` snapshot stats properties have been removed - -* Properties `number_of_files` and `total_size_in_bytes` are removed and should be replaced by values of nested object `total`. -* Properties `processed_files` and `processed_size_in_bytes` are removed and should be replaced by values of nested object `processed`. \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc new file mode 100644 index 0000000000000..4477090dc16b6 --- /dev/null +++ b/docs/reference/migration/migrate_8_0.asciidoc @@ -0,0 +1,12 @@ +[[breaking-changes-8.0]] +== Breaking changes in 8.0 +++++ +8.0 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.0. + +See also <> and <>. + +coming[8.0.0] \ No newline at end of file diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 1c31e1cb3db47..b087f70ac83d5 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,10 +6,8 @@ This section summarizes the changes in each release. -* <> -* <> +* <> -- -include::release-notes/7.0.0-alpha2.asciidoc[] -include::release-notes/7.0.0-alpha1.asciidoc[] +include::release-notes/8.0.0-alpha1.asciidoc[] diff --git a/docs/reference/release-notes/7.0.0-alpha1.asciidoc b/docs/reference/release-notes/7.0.0-alpha1.asciidoc deleted file mode 100644 index 758e88d34024c..0000000000000 --- a/docs/reference/release-notes/7.0.0-alpha1.asciidoc +++ /dev/null @@ -1,32 +0,0 @@ -[[release-notes-7.0.0-alpha1]] -== {es} version 7.0.0-alpha1 - -The changes listed below have been released for the first time in Elasticsearch 7.0.0-alpha1. - -[[breaking-7.0.0-alpha1]] -[float] -=== Breaking changes - -Core:: -* Tribe node has been removed in favor of Cross-Cluster-Search - -Cross-Cluster-Search:: -* `http_addresses` has been removed from the <> API - because it is expensive to fetch and no longer needed by Kibana. - -Rest API:: -* The Clear Cache API only supports `POST` as HTTP method -* `CircuitBreakingException` was previously mapped to HTTP status code 503 and is now - mapped as HTTP status code 429. - -Aggregations:: -* The Percentiles and PercentileRanks aggregations now return `null` in the REST response, - instead of `NaN`. This makes it consistent with the rest of the aggregations. Note: - this only applies to the REST response, the java objects continue to return `NaN` (also - consistent with other aggregations) - -Suggesters:: -* Plugins that register suggesters can now define their own types of suggestions and must - explicitly indicate the type of suggestion that they produce. Existing plugins will - require changes to their plugin registration. See the `custom-suggester` example - plugin {pull}30284[#30284] diff --git a/docs/reference/release-notes/7.0.0-alpha2.asciidoc b/docs/reference/release-notes/7.0.0-alpha2.asciidoc deleted file mode 100644 index b07088d6cfed6..0000000000000 --- a/docs/reference/release-notes/7.0.0-alpha2.asciidoc +++ /dev/null @@ -1,585 +0,0 @@ -[[release-notes-7.0.0-alpha2]] -== {es} version 7.0.0-alpha2 - -[[breaking-7.0.0-alpha2]] -[float] -=== Breaking changes - -Authentication:: -* Enhance Invalidate Token API {pull}35388[#35388] (issues: {issue}34556[#34556], {issue}35115[#35115]) - -Circuit Breakers:: -* Lower fielddata circuit breaker's default limit {pull}27162[#27162] (issue: {issue}27130[#27130]) - -CCR:: -* Change get autofollow patterns API response format {pull}36203[#36203] (issue: {issue}36049[#36049]) - -Index APIs:: -* Always enforce cluster-wide shard limit {pull}34892[#34892] (issues: {issue}20705[#20705], {issue}34021[#34021]) - -Ranking:: -* Forbid negative scores in function_score query {pull}35709[#35709] (issue: {issue}33309[#33309]) - -Scripting:: -* Delete deprecated getValues from ScriptDocValues {pull}36183[#36183] (issue: {issue}22919[#22919]) - -Search:: -* Remove the deprecated _termvector endpoint. {pull}36131[#36131] (issues: {issue}36098[#36098], {issue}8484[#8484]) -* Remove deprecated Graph endpoints {pull}35956[#35956] -* Validate metadata on `_msearch` {pull}35938[#35938] (issue: {issue}35869[#35869]) -* Make hits.total an object in the search response {pull}35849[#35849] (issue: {issue}33028[#33028]) -* Remove the distinction between query and filter context in QueryBuilders {pull}35354[#35354] (issue: {issue}35293[#35293]) -* Throw a parsing exception when boost is set in span_or query (#28390) {pull}34112[#34112] (issue: {issue}28390[#28390]) - -ZenDiscovery:: -* Best-effort cluster formation if unconfigured {pull}36215[#36215] - -[[breaking-java-7.0.0-alpha2]] -[float] -=== Breaking Java changes - -ZenDiscovery:: -* Make node field in JoinRequest private {pull}36405[#36405] - -[[deprecation-7.0.0-alpha2]] -[float] -=== Deprecations - -Core:: -* Deprecate use of scientific notation in epoch time parsing {pull}36691[#36691] -* Add backcompat for joda time formats {pull}36531[#36531] - -Machine Learning:: -* Deprecate X-Pack centric ML endpoints {pull}36315[#36315] (issue: {issue}35958[#35958]) - -Mapping:: -* Deprecate types in index API {pull}36575[#36575] (issues: {issue}35190[#35190], {issue}35790[#35790]) -* Deprecate uses of _type as a field name in queries {pull}36503[#36503] (issue: {issue}35190[#35190]) -* Deprecate types in update_by_query and delete_by_query {pull}36365[#36365] (issue: {issue}35190[#35190]) -* For msearch templates, make sure to use the right name for deprecation logging. {pull}36344[#36344] -* Deprecate types in termvector and mtermvector requests. {pull}36182[#36182] -* Deprecate types in update requests. {pull}36181[#36181] -* Deprecate types in document delete requests. {pull}36087[#36087] -* Deprecate types in get, exists, and multi get. {pull}35930[#35930] -* Deprecate types in search and multi search templates. {pull}35669[#35669] -* Deprecate types in explain requests. {pull}35611[#35611] -* Deprecate types in validate query requests. {pull}35575[#35575] -* Deprecate types in count and msearch. {pull}35421[#35421] (issue: {issue}34041[#34041]) - -Migration:: -* Deprecate X-Pack centric Migration endpoints {pull}35976[#35976] (issue: {issue}35958[#35958]) - -Monitoring:: -* Deprecate /_xpack/monitoring/* in favor of /_monitoring/* {pull}36130[#36130] (issue: {issue}35958[#35958]) - -Rollup:: -* Re-deprecate xpack rollup endpoints {pull}36451[#36451] (issue: {issue}36044[#36044]) -* Deprecate X-Pack centric rollup endpoints {pull}35962[#35962] (issue: {issue}35958[#35958]) - -Scripting:: -* Adds deprecation logging to ScriptDocValues#getValues. {pull}34279[#34279] (issue: {issue}22919[#22919]) -* Conditionally use java time api in scripting {pull}31441[#31441] - -Search:: -* Remove X-Pack centric graph endpoints {pull}36010[#36010] (issue: {issue}35958[#35958]) - -Security:: -* Deprecate X-Pack centric license endpoints {pull}35959[#35959] (issue: {issue}35958[#35958]) -* Deprecate /_xpack/security/* in favor of /_security/* {pull}36293[#36293] (issue: {issue}35958[#35958]) - -SQL:: -* Deprecate X-Pack SQL translate endpoint {pull}36030[#36030] -* Deprecate X-Pack centric SQL endpoints {pull}35964[#35964] (issue: {issue}35958[#35958]) - -Watcher:: -* Deprecate X-Pack centric watcher endpoints {pull}36218[#36218] (issue: {issue}35958[#35958]) - - -[[feature-7.0.0-alpha2]] -[float] -=== New features - -Analysis:: -* Add support for inlined user dictionary in Nori {pull}36123[#36123] (issue: {issue}35842[#35842]) -* Add a prebuilt ICU Analyzer {pull}34958[#34958] (issue: {issue}34285[#34285]) - -Java High Level REST Client:: -* Add rollup search {pull}36334[#36334] (issue: {issue}29827[#29827]) - -Java Low Level REST Client:: -* Make warning behavior pluggable per request {pull}36345[#36345] -* Add PreferHasAttributeNodeSelector {pull}36005[#36005] - -Geo:: -* Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach {pull}36751[#36751] (issue: {issue}35320[#35320]) -* Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach {pull}35320[#35320] (issue: {issue}32039[#32039]) - -Machine Learning:: -* Add delayed datacheck to the datafeed job runner {pull}35387[#35387] (issue: {issue}35131[#35131]) - -Mapping:: -* Make typeless APIs usable with indices whose type name is different from `_doc` {pull}35790[#35790] (issue: {issue}35190[#35190]) - -SQL:: -* Introduce HISTOGRAM grouping function {pull}36510[#36510] (issue: {issue}36509[#36509]) -* DATABASE() and USER() system functions {pull}35946[#35946] (issue: {issue}35863[#35863]) -* Introduce INTERVAL support {pull}35521[#35521] (issue: {issue}29990[#29990]) - -Search:: -* Add intervals query {pull}36135[#36135] (issues: {issue}29636[#29636], {issue}32406[#32406]) -* Added soft limit to open scroll contexts #25244 {pull}36009[#36009] (issue: {issue}25244[#25244]) - -[[enhancement-7.0.0-alpha2]] -[float] -=== Enhancements - -Aggregations:: -* Added keyed response to pipeline percentile aggregations 22302 {pull}36392[#36392] (issue: {issue}22302[#22302]) -* Enforce max_buckets limit only in the final reduction phase {pull}36152[#36152] (issues: {issue}32125[#32125], {issue}35921[#35921]) -* Histogram aggs: add empty buckets only in the final reduce step {pull}35921[#35921] -* Handles exists query in composite aggs {pull}35758[#35758] -* Added parent validation for auto date histogram {pull}35670[#35670] - -Analysis:: -* Allow word_delimiter_graph_filter to not adjust internal offsets {pull}36699[#36699] (issues: {issue}33710[#33710], {issue}34741[#34741]) -* Ensure TokenFilters only produce single tokens when parsing synonyms {pull}34331[#34331] (issue: {issue}34298[#34298]) - -Audit:: -* Add "request.id" to file audit logs {pull}35536[#35536] - -Authentication:: -* Invalidate Token API enhancements - HLRC {pull}36362[#36362] (issue: {issue}35388[#35388]) -* Add DEBUG/TRACE logs for LDAP bind {pull}36028[#36028] -* Add Tests for findSamlRealm {pull}35905[#35905] -* Add realm information for Authenticate API {pull}35648[#35648] -* Formal support for "password_hash" in Put User {pull}35242[#35242] (issue: {issue}34729[#34729]) - -Authorization:: -* Improve exact index matching performance {pull}36017[#36017] -* `manage_token` privilege for `kibana_system` {pull}35751[#35751] -* Grant .tasks access to kibana_system role {pull}35573[#35573] - -Build:: -* Sounds like typo in exception message {pull}35458[#35458] -* Allow set section in setup section of REST tests {pull}34678[#34678] - -CCR:: -* Add time since last auto follow fetch to auto follow stats {pull}36542[#36542] (issues: {issue}33007[#33007], {issue}35895[#35895]) -* Clean followed leader index UUIDs in auto follow metadata {pull}36408[#36408] (issue: {issue}33007[#33007]) -* Change AutofollowCoordinator to use wait_for_metadata_version {pull}36264[#36264] (issues: {issue}33007[#33007], {issue}35895[#35895]) -* Refactor AutoFollowCoordinator to track leader indices per remote cluster {pull}36031[#36031] (issues: {issue}33007[#33007], {issue}35895[#35895]) - -Core:: -* Override the JVM DNS cache policy {pull}36570[#36570] -* Replace usages of AtomicBoolean based block of code by the RunOnce class {pull}35553[#35553] (issue: {issue}35489[#35489]) -* Added wait_for_metadata_version parameter to cluster state api. {pull}35535[#35535] -* Extract RunOnce into a dedicated class {pull}35489[#35489] -* Introduce elasticsearch-core jar {pull}28191[#28191] (issue: {issue}27933[#27933]) -* Rename core module to server {pull}28180[#28180] (issue: {issue}27933[#27933]) - -CRUD:: -* Rename seq# powered optimistic concurrency control parameters to ifSeqNo/ifPrimaryTerm {pull}36757[#36757] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Expose Sequence Number based Optimistic Concurrency Control in the rest layer {pull}36721[#36721] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Add doc's sequence number + primary term to GetResult and use it for updates {pull}36680[#36680] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Add seq no powered optimistic locking support to the index and delete transport actions {pull}36619[#36619] (issues: {issue}10708[#10708], {issue}36148[#36148]) - -Distributed:: -* [Close Index API] Mark shard copy as stale if needed during shard verification {pull}36755[#36755] -* [Close Index API] Refactor MetaDataIndexStateService {pull}36354[#36354] (issue: {issue}36249[#36249]) -* [Close Index API] Add TransportShardCloseAction for pre-closing verifications {pull}36249[#36249] -* TransportResyncReplicationAction should not honour blocks {pull}35795[#35795] (issues: {issue}35332[#35332], {issue}35597[#35597]) -* Expose all permits acquisition in IndexShard and TransportReplicationAction {pull}35540[#35540] (issue: {issue}33888[#33888]) -* [RCI] Check blocks while having index shard permit in TransportReplicationAction {pull}35332[#35332] (issue: {issue}33888[#33888]) - -Engine:: -* Add sequence numbers based optimistic concurrency control support to Engine {pull}36467[#36467] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Require soft-deletes when access changes snapshot {pull}36446[#36446] -* Use delCount of SegmentInfos to calculate numDocs {pull}36323[#36323] -* Always configure soft-deletes field of IndexWriterConfig {pull}36196[#36196] (issue: {issue}36141[#36141]) -* Enable soft-deletes by default on 7.0.0 or later {pull}36141[#36141] -* Always return false from `refreshNeeded` on ReadOnlyEngine {pull}35837[#35837] (issue: {issue}35785[#35785]) -* Add a `_freeze` / `_unfreeze` API {pull}35592[#35592] (issue: {issue}34352[#34352]) -* [RCI] Add IndexShardOperationPermits.asyncBlockOperations(ActionListener) {pull}34902[#34902] (issue: {issue}33888[#33888]) - -Features:: -* Simplify deprecation issue levels {pull}36326[#36326] - -Index APIs:: -* Add cluster-wide shard limit warnings {pull}34021[#34021] (issues: {issue}20705[#20705], {issue}32856[#32856]) - -Ingest:: -* Grok fix duplicate patterns JAVACLASS and JAVAFILE {pull}35886[#35886] -* Implement Drop Processor {pull}32278[#32278] (issue: {issue}23726[#23726]) - -Java High Level REST Client:: -* Add get users action {pull}36332[#36332] (issue: {issue}29827[#29827]) -* Add delete template API {pull}36320[#36320] (issue: {issue}27205[#27205]) -* Implement get-user-privileges API {pull}36292[#36292] -* Get Deprecation Info API {pull}36279[#36279] (issue: {issue}29827[#29827]) -* Add support for Follow Stats API {pull}36253[#36253] (issue: {issue}33824[#33824]) -* Add support for CCR Stats API {pull}36213[#36213] (issue: {issue}33824[#33824]) -* Put Role {pull}36209[#36209] (issue: {issue}29827[#29827]) -* Add index templates exist API {pull}36132[#36132] (issue: {issue}27205[#27205]) -* Add support for CCR Get Auto Follow Pattern apis {pull}36049[#36049] (issue: {issue}33824[#33824]) -* Add support for CCR Delete Auto Follow Pattern API {pull}35981[#35981] (issue: {issue}33824[#33824]) -* Remove fromXContent from IndexUpgradeInfoResponse {pull}35934[#35934] -* Add delete expired data API {pull}35906[#35906] (issue: {issue}29827[#29827]) -* Execute watch API {pull}35868[#35868] (issue: {issue}29827[#29827]) -* Add ability to put user with a password hash {pull}35844[#35844] (issue: {issue}35242[#35242]) -* Add ML find file structure API {pull}35833[#35833] (issue: {issue}29827[#29827]) -* Add support for get roles API {pull}35787[#35787] (issue: {issue}29827[#29827]) -* Added support for CCR Put Auto Follow Pattern API {pull}35780[#35780] (issue: {issue}33824[#33824]) -* XPack ML info action {pull}35777[#35777] (issue: {issue}29827[#29827]) -* ML Delete event from Calendar {pull}35760[#35760] (issue: {issue}29827[#29827]) -* Add ML revert model snapshot API {pull}35750[#35750] (issue: {issue}29827[#29827]) -* ML Get Calendar Events {pull}35747[#35747] (issue: {issue}29827[#29827]) -* Add high-level REST client API for `_freeze` and `_unfreeze` {pull}35723[#35723] (issue: {issue}34352[#34352]) -* Fix issue in equals impl for GlobalOperationPrivileges {pull}35721[#35721] -* ML Delete job from calendar {pull}35713[#35713] (issue: {issue}29827[#29827]) -* ML Add Event To Calendar API {pull}35704[#35704] (issue: {issue}29827[#29827]) -* Add ML update model snapshot API (#35537) {pull}35694[#35694] (issue: {issue}29827[#29827]) -* Add support for CCR Unfollow API {pull}35693[#35693] (issue: {issue}33824[#33824]) -* Clean up PutLicenseResponse {pull}35689[#35689] (issue: {issue}35547[#35547]) -* Clean up StartBasicResponse {pull}35688[#35688] (issue: {issue}35547[#35547]) -* Add support for put privileges API {pull}35679[#35679] -* ML Add Job to Calendar API {pull}35666[#35666] (issue: {issue}29827[#29827]) -* Add support for CCR Resume Follow API {pull}35638[#35638] (issue: {issue}33824[#33824]) -* Add support for get application privileges API {pull}35556[#35556] (issue: {issue}29827[#29827]) -* Clean up XPackInfoResponse class and related tests {pull}35547[#35547] -* Add parameters to stopRollupJob API {pull}35545[#35545] (issue: {issue}34811[#34811]) -* Add ML delete model snapshot API {pull}35537[#35537] (issue: {issue}29827[#29827]) -* Add get watch API {pull}35531[#35531] (issue: {issue}29827[#29827]) -* Add ML Update Filter API {pull}35522[#35522] (issue: {issue}29827[#29827]) -* Add ml get filters api {pull}35502[#35502] (issue: {issue}29827[#29827]) -* Add ML get model snapshots API {pull}35487[#35487] (issue: {issue}29827[#29827]) -* Add "_has_privileges" API to Security Client {pull}35479[#35479] (issue: {issue}29827[#29827]) -* Add Delete Privileges API to HLRC {pull}35454[#35454] (issue: {issue}29827[#29827]) -* Add support for CCR Put Follow API {pull}35409[#35409] -* Add ML delete filter action {pull}35382[#35382] (issue: {issue}29827[#29827]) -* Add delete user action {pull}35294[#35294] (issue: {issue}29827[#29827]) -* HLRC for _mtermvectors {pull}35266[#35266] (issues: {issue}27205[#27205], {issue}33447[#33447]) -* Reindex API with wait_for_completion false {pull}35202[#35202] (issue: {issue}27205[#27205]) -* Add watcher stats API {pull}35185[#35185] (issue: {issue}29827[#29827]) -* HLRC support for getTask {pull}35166[#35166] (issue: {issue}27205[#27205]) -* Add GetRollupIndexCaps API {pull}35102[#35102] (issue: {issue}29827[#29827]) -* HLRC: migration api - upgrade {pull}34898[#34898] (issue: {issue}29827[#29827]) -* Add stop rollup job support to HL REST Client {pull}34702[#34702] (issue: {issue}29827[#29827]) -* Bulk Api support for global parameters {pull}34528[#34528] (issue: {issue}26026[#26026]) -* Add delete rollup job support to HL REST Client {pull}34066[#34066] (issue: {issue}29827[#29827]) -* Add support for get license basic/trial status API {pull}33176[#33176] (issue: {issue}29827[#29827]) -* Add machine learning open job {pull}32860[#32860] (issue: {issue}29827[#29827]) -* Add ML HLRC wrapper and put_job API call {pull}32726[#32726] -* Add Get Snapshots High Level REST API {pull}31537[#31537] (issue: {issue}27205[#27205]) - -Java Low Level REST Client:: -* On retry timeout add root exception {pull}25576[#25576] - -Monitoring:: -* Make Exporters Async {pull}35765[#35765] (issue: {issue}35743[#35743]) - -Geo:: -* Adds a name of the field to geopoint parsing errors {pull}36529[#36529] (issue: {issue}15965[#15965]) -* Add support to ShapeBuilders for building Lucene geometry {pull}35707[#35707] (issue: {issue}35320[#35320]) -* Add ST_WktToSQL function {pull}35416[#35416] (issue: {issue}29872[#29872]) - -License:: -* Require acknowledgement to start_trial license {pull}30135[#30135] (issue: {issue}30134[#30134]) - -Machine Learning:: -* Create the ML annotations index {pull}36731[#36731] (issues: {issue}26034[#26034], {issue}33376[#33376]) -* Split in batches and migrate all jobs and datafeeds {pull}36716[#36716] (issue: {issue}32905[#32905]) -* Add cluster setting to enable/disable config migration {pull}36700[#36700] (issue: {issue}32905[#32905]) -* Add audits when deprecation warnings occur with datafeed start {pull}36233[#36233] -* Add lazy parsing for DatafeedConfig:Aggs,Query {pull}36117[#36117] -* Add support for lazy nodes (#29991) {pull}34538[#34538] (issue: {issue}29991[#29991]) - -Network:: -* Unify transport settings naming {pull}36623[#36623] -* Add sni name to SSLEngine in netty transport {pull}33144[#33144] (issue: {issue}32517[#32517]) -* Add cors support to NioHttpServerTransport {pull}30827[#30827] (issue: {issue}28898[#28898]) -* Reintroduce mandatory http pipelining support {pull}30820[#30820] -* Make http pipelining support mandatory {pull}30695[#30695] (issues: {issue}28898[#28898], {issue}29500[#29500]) -* Add nio http server transport {pull}29587[#29587] (issue: {issue}28898[#28898]) -* Selectors operate on channel contexts {pull}28468[#28468] (issue: {issue}27260[#27260]) -* Unify nio read / write channel contexts {pull}28160[#28160] (issue: {issue}27260[#27260]) -* Create nio-transport plugin for NioTransport {pull}27949[#27949] (issue: {issue}27260[#27260]) -* Add elasticsearch-nio jar for base nio classes {pull}27801[#27801] (issue: {issue}27802[#27802]) -* Add NioGroup for use in different transports {pull}27737[#27737] (issue: {issue}27260[#27260]) -* Add read timeouts to http module {pull}27713[#27713] -* Implement byte array reusage in `NioTransport` {pull}27696[#27696] (issue: {issue}27563[#27563]) -* Introduce resizable inbound byte buffer {pull}27551[#27551] (issue: {issue}27563[#27563]) -* Decouple nio constructs from the tcp transport {pull}27484[#27484] (issue: {issue}27260[#27260]) -* Remove manual tracking of registered channels {pull}27445[#27445] (issue: {issue}27260[#27260]) -* Remove tcp profile from low level nio channel {pull}27441[#27441] (issue: {issue}27260[#27260]) -* Decouple `ChannelFactory` from Tcp classes {pull}27286[#27286] (issue: {issue}27260[#27260]) - -Packaging:: -* Introduce Docker images build {pull}36246[#36246] -* Move creation of temporary directory to Java {pull}36002[#36002] (issue: {issue}31003[#31003]) - -Plugins:: -* Plugin install: don't print download progress in batch mode {pull}36361[#36361] - -Ranking:: -* Vector field {pull}33022[#33022] (issue: {issue}31615[#31615]) - -Recovery:: -* Exposed engine must include all operations below global checkpoint during rollback {pull}36159[#36159] (issue: {issue}32867[#32867]) - -Rollup:: -* Add non-X-Pack centric rollup endpoints {pull}36383[#36383] (issues: {issue}35958[#35958], {issue}35962[#35962]) -* Add more diagnostic stats to job {pull}35471[#35471] -* Add `wait_for_completion` option to StopRollupJob API {pull}34811[#34811] (issue: {issue}34574[#34574]) - -Scripting:: -* Update joda compat methods to use compat class {pull}36654[#36654] -* [Painless] Add boxed type to boxed type casts for method/return {pull}36571[#36571] -* [Painless] Add def to boxed type casts {pull}36506[#36506] - -Settings:: -* Add user-defined cluster metadata {pull}33325[#33325] (issue: {issue}33220[#33220]) - -Search:: -* Add copy constructor to SearchRequest {pull}36641[#36641] (issue: {issue}32125[#32125]) -* Add raw sort values to SearchSortValues transport serialization {pull}36617[#36617] (issue: {issue}32125[#32125]) -* Add sort and collapse info to SearchHits transport serialization {pull}36555[#36555] (issue: {issue}32125[#32125]) -* Add default methods to DocValueFormat {pull}36480[#36480] -* Respect indices options on _msearch {pull}35887[#35887] -* Allow efficient can_match phases on frozen indices {pull}35431[#35431] (issues: {issue}34352[#34352], {issue}34357[#34357]) -* Add a new query type - ScriptScoreQuery {pull}34533[#34533] (issues: {issue}23850[#23850], {issue}27588[#27588], {issue}30303[#30303]) - -Security:: -* Make credentials mandatory when launching xpack/migrate {pull}36197[#36197] (issues: {issue}29847[#29847], {issue}33972[#33972]) - -Snapshot/Restore:: -* Allow Parallel Restore Operations {pull}36397[#36397] -* Repo Creation out of ClusterStateTask {pull}36157[#36157] (issue: {issue}9488[#9488]) -* Add read-only repository verification {pull}35731[#35731] (issue: {issue}35703[#35703]) - -SQL:: -* Extend the ODBC metric by differentiating between 32 and 64bit platforms {pull}36753[#36753] (issue: {issue}36740[#36740]) -* Fix wrong appliance of StackOverflow limit for IN {pull}36724[#36724] (issue: {issue}36592[#36592]) -* Introduce NOW/CURRENT_TIMESTAMP function {pull}36562[#36562] (issue: {issue}36534[#36534]) -* Move requests' parameters to requests JSON body {pull}36149[#36149] (issue: {issue}35992[#35992]) -* Make INTERVAL millis optional {pull}36043[#36043] (issue: {issue}36032[#36032]) -* Implement data type verification for conditionals {pull}35916[#35916] (issue: {issue}35907[#35907]) -* Implement GREATEST and LEAST functions {pull}35879[#35879] (issue: {issue}35878[#35878]) -* Implement null safe equality operator `<=>` {pull}35873[#35873] (issue: {issue}35871[#35871]) -* SYS COLUMNS returns ODBC specific schema {pull}35870[#35870] (issue: {issue}35376[#35376]) -* Polish grammar for intervals {pull}35853[#35853] -* Add filtering to SYS TYPES {pull}35852[#35852] (issue: {issue}35342[#35342]) -* Implement NULLIF(expr1, expr2) function {pull}35826[#35826] (issue: {issue}35818[#35818]) -* Lock down JDBC driver {pull}35798[#35798] (issue: {issue}35437[#35437]) -* Implement NVL(expr1, expr2) {pull}35794[#35794] (issue: {issue}35782[#35782]) -* Implement ISNULL(expr1, expr2) {pull}35793[#35793] (issue: {issue}35781[#35781]) -* Implement IFNULL variant of COALESCE {pull}35762[#35762] (issue: {issue}35749[#35749]) -* XPack FeatureSet functionality {pull}35725[#35725] (issue: {issue}34821[#34821]) -* Perform lazy evaluation of mismatched mappings {pull}35676[#35676] (issues: {issue}35659[#35659], {issue}35675[#35675]) -* Improve validation of unsupported fields {pull}35675[#35675] (issue: {issue}35673[#35673]) -* Move internals from Joda to java.time {pull}35649[#35649] (issue: {issue}35633[#35633]) - -Stats:: -* Handle OS pretty name on old OS without OS release {pull}35453[#35453] (issue: {issue}35440[#35440]) - -Task Management:: -* Periodically try to reassign unassigned persistent tasks {pull}36069[#36069] (issue: {issue}35792[#35792]) -* Only require task permissions {pull}35667[#35667] (issue: {issue}35573[#35573]) -* Retry if task can't be written {pull}35054[#35054] (issue: {issue}33764[#33764]) - -ZenDiscovery:: -* Add discovery types to cluster stats {pull}36442[#36442] -* Introduce `zen2` discovery type {pull}36298[#36298] -* Zen2: Persist cluster states the old way on non-master-eligible nodes {pull}36247[#36247] (issue: {issue}3[#3]) -* [Zen2] Storage layer WriteStateException propagation {pull}36052[#36052] -* [Zen2] Implement Tombstone REST APIs {pull}36007[#36007] -* [Zen2] Update default for USE_ZEN2 to true {pull}35998[#35998] -* [Zen2] Add warning if cluster fails to form fast enough {pull}35993[#35993] -* [Zen2] Allow Setting a List of Bootstrap Nodes to Wait for {pull}35847[#35847] -* [Zen2] VotingTombstone class {pull}35832[#35832] -* [Zen2] PersistedState interface implementation {pull}35819[#35819] -* [Zen2] Support rolling upgrades from Zen1 {pull}35737[#35737] -* [Zen2] Add lag detector {pull}35685[#35685] -* [Zen2] Move ClusterState fields to be persisted to ClusterState.MetaData {pull}35625[#35625] -* [Zen2] Introduce ClusterBootstrapService {pull}35488[#35488] -* [Zen2] Introduce vote withdrawal {pull}35446[#35446] -* Zen2: Add basic Zen1 transport-level BWC {pull}35443[#35443] - -[[bug-7.0.0-alpha2]] -[float] -=== Bug fixes - -Aggregations:: -* fix MultiValuesSourceFieldConfig toXContent {pull}36525[#36525] (issue: {issue}36474[#36474]) -* Cache the score of the parent document in the nested agg {pull}36019[#36019] (issues: {issue}34555[#34555], {issue}35985[#35985]) -* Correct implemented interface of ParsedReverseNested {pull}35455[#35455] (issue: {issue}35449[#35449]) -* Handle IndexOrDocValuesQuery in composite aggregation {pull}35392[#35392] - -Audit:: -* Fix origin.type for connection_* events {pull}36410[#36410] -* Fix IndexAuditTrail rolling restart on rollover edge {pull}35988[#35988] (issue: {issue}33867[#33867]) - -Authentication:: -* Fix kerberos setting registration {pull}35986[#35986] (issues: {issue}30241[#30241], {issue}35942[#35942]) -* Add support for Kerberos V5 Oid {pull}35764[#35764] (issue: {issue}34763[#34763]) - -Build:: -* Use explicit deps on test tasks for check {pull}36325[#36325] -* Fix jdbc jar pom to not include deps {pull}36036[#36036] (issue: {issue}32014[#32014]) -* Fix official plugins list {pull}35661[#35661] (issue: {issue}35623[#35623]) - -CCR:: -* Fix follow stats API's follower index filtering feature {pull}36647[#36647] -* AutoFollowCoordinator should tolerate that auto follow patterns may be removed {pull}35945[#35945] (issue: {issue}35937[#35937]) -* Only auto follow indices when all primary shards have started {pull}35814[#35814] (issue: {issue}35480[#35480]) -* Avoid NPE in follower stats when no tasks metadata {pull}35802[#35802] -* Fix the names of CCR stats endpoints in usage API {pull}35438[#35438] - -Circuit Breakers:: -* Modify `BigArrays` to take name of circuit breaker {pull}36461[#36461] (issue: {issue}31435[#31435]) - -Core:: -* Fix CompositeBytesReference#slice to not throw AIOOBE with legal offsets. {pull}35955[#35955] (issue: {issue}35950[#35950]) -* Suppress CachedTimeThread in hot threads output {pull}35558[#35558] (issue: {issue}23175[#23175]) -* Upgrade to Joda 2.10.1 {pull}35410[#35410] (issue: {issue}33749[#33749]) - -Distributed:: -* Combine the execution of an exclusive replica operation with primary term update {pull}36116[#36116] (issue: {issue}35850[#35850]) -* ActiveShardCount should not fail when closing the index {pull}35936[#35936] - -Engine:: -* Set Lucene version upon index creation. {pull}36038[#36038] (issue: {issue}33826[#33826]) -* Wrap can_match reader with ElasticsearchDirectoryReader {pull}35857[#35857] -* Copy checkpoint atomically when rolling generation {pull}35407[#35407] - -Geo:: -* More robust handling of ignore_malformed in geoshape parsing {pull}35603[#35603] (issues: {issue}34047[#34047], {issue}34498[#34498]) -* Better handling of malformed geo_points {pull}35554[#35554] (issue: {issue}35419[#35419]) -* Enables coerce support in WKT polygon parser {pull}35414[#35414] (issue: {issue}35059[#35059]) - -Index APIs:: -* Fix duplicate phrase in shrink/split error message {pull}36734[#36734] (issue: {issue}36729[#36729]) -* Raise a 404 exception when document source is not found (#33384) {pull}34083[#34083] (issue: {issue}33384[#33384]) - -Ingest:: -* Fix on_failure with Drop processor {pull}36686[#36686] (issue: {issue}36151[#36151]) -* Support default pipelines + bulk upserts {pull}36618[#36618] (issue: {issue}36219[#36219]) -* Support default pipeline through an alias {pull}36231[#36231] (issue: {issue}35817[#35817]) - -License:: -* Do not serialize basic license exp in x-pack info {pull}30848[#30848] -* Update versions for start_trial after backport {pull}30218[#30218] (issue: {issue}30135[#30135]) - -Machine Learning:: -* Interrupt Grok in file structure finder timeout {pull}36588[#36588] -* Prevent stack overflow while copying ML jobs and datafeeds {pull}36370[#36370] (issue: {issue}36360[#36360]) -* Adjust file structure finder parser config {pull}35935[#35935] -* Fix find_file_structure NPE with should_trim_fields {pull}35465[#35465] (issue: {issue}35462[#35462]) -* Prevent notifications being created on deletion of a non existent job {pull}35337[#35337] (issues: {issue}34058[#34058], {issue}35336[#35336]) -* Clear Job#finished_time when it is opened (#32605) {pull}32755[#32755] -* Fix thread leak when waiting for job flush (#32196) {pull}32541[#32541] (issue: {issue}32196[#32196]) -* Fix CPoissonMeanConjugate sampling error. {ml-pull}335[#335] - -Network:: -* Do not resolve addresses in remote connection info {pull}36671[#36671] (issue: {issue}35658[#35658]) -* Always compress based on the settings {pull}36522[#36522] (issue: {issue}36399[#36399]) -* http.publish_host Should Contain CNAME {pull}32806[#32806] (issue: {issue}22029[#22029]) -* Adjust SSLDriver behavior for JDK11 changes {pull}32145[#32145] (issues: {issue}32122[#32122], {issue}32144[#32144]) -* Add TRACE, CONNECT, and PATCH http methods {pull}31035[#31035] (issue: {issue}31017[#31017]) -* Transport client: Don't validate node in handshake {pull}30737[#30737] (issue: {issue}30141[#30141]) -* Fix issue with finishing handshake in ssl driver {pull}30580[#30580] -* Remove potential nio selector leak {pull}27825[#27825] -* Fix issue where the incorrect buffers are written {pull}27695[#27695] (issue: {issue}27551[#27551]) -* Do not set SO_LINGER on server channels {pull}26997[#26997] -* Do not set SO_LINGER to 0 when not shutting down {pull}26871[#26871] (issue: {issue}26764[#26764]) -* Release pipelined http responses on close {pull}26226[#26226] - -Packaging:: -* Fix error message when package install fails due to missing Java {pull}36077[#36077] (issue: {issue}31845[#31845]) -* Add missing entries to conffiles {pull}35810[#35810] (issue: {issue}35691[#35691]) - -Plugins:: -* Ensure that azure stream has socket privileges {pull}28751[#28751] (issue: {issue}28662[#28662]) - -Recovery:: -* Register ResyncTask.Status as a NamedWriteable {pull}36610[#36610] - -Rollup:: -* Fix rollup search statistics {pull}36674[#36674] - -Scripting:: -* Properly support no-offset date formatting {pull}36316[#36316] (issue: {issue}36306[#36306]) -* [Painless] Generate Bridge Methods {pull}36097[#36097] -* Fix serialization bug in painless execute api request {pull}36075[#36075] (issue: {issue}36050[#36050]) -* Actually add joda time back to whitelist {pull}35965[#35965] (issue: {issue}35915[#35915]) -* Add back joda to whitelist {pull}35915[#35915] (issue: {issue}35913[#35913]) - -Settings:: -* Correctly Identify Noop Updates {pull}36560[#36560] (issue: {issue}36496[#36496]) - -SQL:: -* Fix translation of LIKE/RLIKE keywords {pull}36672[#36672] (issues: {issue}36039[#36039], {issue}36584[#36584]) -* Scripting support for casting functions CAST and CONVERT {pull}36640[#36640] (issue: {issue}36061[#36061]) -* Fix translation to painless for conditionals {pull}36636[#36636] (issue: {issue}36631[#36631]) -* Concat should be always not nullable {pull}36601[#36601] (issue: {issue}36169[#36169]) -* Fix MOD() for long and integer arguments {pull}36599[#36599] (issue: {issue}36364[#36364]) -* Fix issue with complex HAVING and GROUP BY ordinal {pull}36594[#36594] (issue: {issue}36059[#36059]) -* Be lenient for tests involving comparison to H2 but strict for csv spec tests {pull}36498[#36498] (issue: {issue}36483[#36483]) -* Non ISO 8601 versions of DAY_OF_WEEK and WEEK_OF_YEAR functions {pull}36358[#36358] (issue: {issue}36263[#36263]) -* Do not ignore all fields whose names start with underscore {pull}36214[#36214] (issue: {issue}36206[#36206]) -* Fix issue with wrong data type for scripted Grouping keys {pull}35969[#35969] (issue: {issue}35662[#35662]) -* Fix translation of math functions to painless {pull}35910[#35910] (issue: {issue}35654[#35654]) -* Fix jdbc jar to include deps {pull}35602[#35602] -* Fix query translation for scripted queries {pull}35408[#35408] (issue: {issue}35232[#35232]) -* Clear the cursor if nested inner hits are enough to fulfill the query required limits {pull}35398[#35398] (issue: {issue}35176[#35176]) -* Introduce IsNull node to simplify expressions {pull}35206[#35206] (issues: {issue}34876[#34876], {issue}35171[#35171]) -* The SSL default configuration shouldn't override the https protocol if used {pull}34635[#34635] (issue: {issue}33817[#33817]) -* Minor fix for javadoc {pull}32573[#32573] (issue: {issue}32553[#32553]) - -Search:: -* Inner hits fail to propagate doc-value format. {pull}36310[#36310] -* Fix custom AUTO issue with Fuzziness#toXContent {pull}35807[#35807] (issue: {issue}33462[#33462]) -* Fix analyzed prefix query in query_string {pull}35756[#35756] (issue: {issue}31702[#31702]) -* Fix problem with MatchNoDocsQuery in disjunction queries {pull}35726[#35726] (issue: {issue}34708[#34708]) -* Fix phrase_slop in query_string query {pull}35533[#35533] (issue: {issue}35125[#35125]) -* Add a More Like This query routing requirement check (#29678) {pull}33974[#33974] - -Security:: -* Remove license state listeners on closeables {pull}36308[#36308] (issues: {issue}33328[#33328], {issue}35627[#35627], {issue}35628[#35628]) - -Snapshot/Restore:: -* Upgrade GCS Dependencies to 1.55.0 {pull}36634[#36634] (issues: {issue}35229[#35229], {issue}35459[#35459]) -* Improve Resilience SnapshotShardService {pull}36113[#36113] (issue: {issue}32265[#32265]) -* Keep SnapshotsInProgress State in Sync with Routing Table {pull}35710[#35710] -* Ensure that gcs client creation is privileged {pull}25938[#25938] (issue: {issue}25932[#25932]) -* Make calls to CloudBlobContainer#exists privileged {pull}25937[#25937] (issue: {issue}25931[#25931]) - -Watcher:: -* Watcher accounts constructed lazily {pull}36656[#36656] -* Only trigger a watch if new or schedule/changed {pull}35908[#35908] -* Fix Watcher NotificationService's secure settings {pull}35610[#35610] (issue: {issue}35378[#35378]) -* Fix integration tests to ensure correct start/stop of Watcher {pull}35271[#35271] (issues: {issue}29877[#29877], {issue}30705[#30705], {issue}33291[#33291], {issue}34448[#34448], {issue}34462[#34462]) - -ZenDiscovery:: -* [Zen2] Respect the no_master_block setting {pull}36478[#36478] -* Cancel GetDiscoveredNodesAction when bootstrapped {pull}36423[#36423] (issues: {issue}36380[#36380], {issue}36381[#36381]) -* [Zen2] Only elect master-eligible nodes {pull}35996[#35996] -* [Zen2] Remove duplicate discovered peers {pull}35505[#35505] - - -[[regression-7.0.0-alpha2]] -[float] -=== Regressions - -Scripting:: -* Use Number as a return value for BucketAggregationScript {pull}35653[#35653] (issue: {issue}35351[#35351]) - - -[[upgrade-7.0.0-alpha2]] -[float] -=== Upgrades - -Ingest:: -* Update geolite2 database in ingest geoip plugin {pull}33840[#33840] - -Network:: -* Upgrade Netty 4.3.32.Final {pull}36102[#36102] (issue: {issue}35360[#35360]) diff --git a/docs/reference/release-notes/8.0.0-alpha1.asciidoc b/docs/reference/release-notes/8.0.0-alpha1.asciidoc new file mode 100644 index 0000000000000..4693221f629cf --- /dev/null +++ b/docs/reference/release-notes/8.0.0-alpha1.asciidoc @@ -0,0 +1,8 @@ +[[release-notes-8.0.0-alpha1]] +== {es} version 8.0.0-alpha1 + +The changes listed below have been released for the first time in {es} +8.0.0-alpha1. + +coming[8.0.0] + diff --git a/docs/reference/release-notes/highlights-7.0.0.asciidoc b/docs/reference/release-notes/highlights-7.0.0.asciidoc deleted file mode 100644 index d01d543c8257e..0000000000000 --- a/docs/reference/release-notes/highlights-7.0.0.asciidoc +++ /dev/null @@ -1,9 +0,0 @@ -[[release-highlights-7.0.0]] -== 7.0.0 release highlights -++++ -7.0.0 -++++ - -coming[7.0.0] - -See also <> and <>. diff --git a/docs/reference/release-notes/highlights-8.0.0.asciidoc b/docs/reference/release-notes/highlights-8.0.0.asciidoc new file mode 100644 index 0000000000000..419059edd0650 --- /dev/null +++ b/docs/reference/release-notes/highlights-8.0.0.asciidoc @@ -0,0 +1,9 @@ +[[release-highlights-8.0.0]] +== 8.0.0 release highlights +++++ +8.0.0 +++++ + +coming[8.0.0] + +See also <> and <>. diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 5b20b67061d03..6ff7c75a235db 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -9,8 +9,8 @@ This section summarizes the most important changes in each release. For the full list, see <> and <>. -* <> +* <> -- -include::highlights-7.0.0.asciidoc[] \ No newline at end of file +include::highlights-8.0.0.asciidoc[] \ No newline at end of file From 8f4693f0307e573c6b8f34408312bad2fd92d6e4 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 8 Feb 2019 19:47:33 -0500 Subject: [PATCH 07/12] Drop support for the low-level REST client on JDK 7 (#38540) This commit bumps the minimum compiler version on the low-level REST client to JDK 8. --- client/rest/build.gradle | 4 ++-- client/sniffer/build.gradle | 4 ++-- client/test/build.gradle | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 6b22b7b909909..7bbcb1df85694 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -22,8 +22,8 @@ apply plugin: 'elasticsearch.build' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' -targetCompatibility = JavaVersion.VERSION_1_7 -sourceCompatibility = JavaVersion.VERSION_1_7 +targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_1_8 group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-client' diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index 382a3f3c9d121..fffc1b711b25c 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -20,8 +20,8 @@ apply plugin: 'elasticsearch.build' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' -targetCompatibility = JavaVersion.VERSION_1_7 -sourceCompatibility = JavaVersion.VERSION_1_7 +targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_1_8 group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-client-sniffer' diff --git a/client/test/build.gradle b/client/test/build.gradle index f184cfbb73c3d..25cf23672dac6 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -18,8 +18,8 @@ */ apply plugin: 'elasticsearch.build' -targetCompatibility = JavaVersion.VERSION_1_7 -sourceCompatibility = JavaVersion.VERSION_1_7 +targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_1_8 group = "${group}.client.test" From 06674235b2e0761ac307d50a499122587dfd68fd Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 8 Feb 2019 16:57:36 -0800 Subject: [PATCH 08/12] [DOCS] Fixes broken migration links (#38655) --- docs/reference/migration/apis/deprecation.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/migration/apis/deprecation.asciidoc b/docs/reference/migration/apis/deprecation.asciidoc index 88de3f5d6e3fa..b52079726d868 100644 --- a/docs/reference/migration/apis/deprecation.asciidoc +++ b/docs/reference/migration/apis/deprecation.asciidoc @@ -54,7 +54,7 @@ Example response: { "level" : "critical", "message" : "Cluster name cannot contain ':'", - "url" : "{ref}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_cluster_name", + "url" : "{ref-70}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_cluster_name", "details" : "This cluster is named [mycompany:logging], which contains the illegal character ':'." } ], @@ -64,7 +64,7 @@ Example response: { "level" : "warning", "message" : "Index name cannot contain ':'", - "url" : "{ref}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_index_name", + "url" : "{ref-70}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_index_name", "details" : "This index is named [logs:apache], which contains the illegal character ':'." } ] From 115ad24953630129aa69dc0b56cd868b3b142aee Mon Sep 17 00:00:00 2001 From: Marios Trivyzas Date: Sat, 9 Feb 2019 09:17:31 +0200 Subject: [PATCH 09/12] SQL: Relax StackOverflow circuit breaker for constants (#38572) Constant numbers (of any form: integers, decimals, negatives, scientific) and strings shouldn't increase the depth counters as they don't contribute to the increment of the stack depth. Fixes: #38571 --- .../xpack/sql/parser/SqlParser.java | 7 +++-- .../xpack/sql/parser/SqlParserTests.java | 26 ++++++++++++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java index 0bc02c1ba6f27..9920293794ea0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.parser; import com.carrotsearch.hppc.ObjectShortHashMap; - import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonToken; @@ -37,8 +36,6 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.StatementContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.StatementDefaultContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.UnquoteIdentifierContext; -import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ValueExpressionContext; -import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ValueExpressionDefaultContext; import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; @@ -242,7 +239,6 @@ static class CircuitBreakerListener extends SqlBaseBaseListener { ENTER_EXIT_RULE_MAPPING.put(StatementDefaultContext.class.getSimpleName(), StatementContext.class.getSimpleName()); ENTER_EXIT_RULE_MAPPING.put(QueryPrimaryDefaultContext.class.getSimpleName(), QueryTermContext.class.getSimpleName()); ENTER_EXIT_RULE_MAPPING.put(BooleanDefaultContext.class.getSimpleName(), BooleanExpressionContext.class.getSimpleName()); - ENTER_EXIT_RULE_MAPPING.put(ValueExpressionDefaultContext.class.getSimpleName(), ValueExpressionContext.class.getSimpleName()); } private boolean insideIn = false; @@ -265,6 +261,9 @@ public void enterEveryRule(ParserRuleContext ctx) { if (ctx.getClass() != UnquoteIdentifierContext.class && ctx.getClass() != QuoteIdentifierContext.class && ctx.getClass() != BackQuotedIdentifierContext.class && + ctx.getClass() != SqlBaseParser.ConstantContext.class && + ctx.getClass() != SqlBaseParser.NumberContext.class && + ctx.getClass() != SqlBaseParser.ValueExpressionContext.class && (insideIn == false || ctx.getClass() != PrimaryExpressionContext.class)) { int currentDepth = depthCounts.putOrAdd(ctx.getClass().getSimpleName(), (short) 1, (short) 1); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java index dd44a8e464ae4..8b275468f482a 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java @@ -294,9 +294,18 @@ public void testLimitToPreventStackOverflowFromLargeComplexSubselectTree() { } public void testLimitStackOverflowForInAndLiteralsIsNotApplied() { - int noChildren = 100_000; + int noChildren = 10_000; LogicalPlan plan = parseStatement("SELECT * FROM t WHERE a IN(" + - Joiner.on(",").join(nCopies(noChildren, "a + b")) + ")"); + Joiner.on(",").join(nCopies(noChildren, "a + 10")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-(-a - 10)")) + "," + + Joiner.on(",").join(nCopies(noChildren, "20")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-20")) + "," + + Joiner.on(",").join(nCopies(noChildren, "20.1234")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-20.4321")) + "," + + Joiner.on(",").join(nCopies(noChildren, "1.1234E56")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-1.4321E-65")) + "," + + Joiner.on(",").join(nCopies(noChildren, "'foo'")) + "," + + Joiner.on(",").join(nCopies(noChildren, "'bar'")) + ")"); assertEquals(With.class, plan.getClass()); assertEquals(Project.class, ((With) plan).child().getClass()); @@ -305,8 +314,17 @@ public void testLimitStackOverflowForInAndLiteralsIsNotApplied() { assertEquals(In.class, filter.condition().getClass()); In in = (In) filter.condition(); assertEquals("?a", in.value().toString()); - assertEquals(noChildren, in.list().size()); - assertThat(in.list().get(0).toString(), startsWith("Add[?a,?b]")); + assertEquals(noChildren * 2 + 8, in.list().size()); + assertThat(in.list().get(0).toString(), startsWith("Add[?a,10]#")); + assertThat(in.list().get(noChildren).toString(), startsWith("Neg[Sub[Neg[?a]#")); + assertEquals("20", in.list().get(noChildren * 2).toString()); + assertEquals("-20", in.list().get(noChildren * 2 + 1).toString()); + assertEquals("20.1234", in.list().get(noChildren * 2 + 2).toString()); + assertEquals("-20.4321", in.list().get(noChildren * 2 + 3).toString()); + assertEquals("1.1234E56", in.list().get(noChildren * 2 + 4).toString()); + assertEquals("-1.4321E-65", in.list().get(noChildren * 2 + 5).toString()); + assertEquals("'foo'=foo", in.list().get(noChildren * 2 + 6).toString()); + assertEquals("'bar'=bar", in.list().get(noChildren * 2 + 7).toString()); } public void testDecrementOfDepthCounter() { From 4e9b1cfd4df38c652bba36b4b4b538ce7c714b6e Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Sat, 9 Feb 2019 09:28:32 +0200 Subject: [PATCH 10/12] SQL: Prevent grouping over grouping functions (#38649) Improve verifier to disallow grouping over grouping functions (e.g. HISTOGRAM over HISTOGRAM). Close #38308 --- .../xpack/sql/analysis/analyzer/Verifier.java | 22 ++++++++++++++++--- .../xpack/sql/expression/Expression.java | 7 ------ .../xpack/sql/expression/Expressions.java | 4 ++-- .../analyzer/VerifierErrorMessagesTests.java | 16 +++++++++++--- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index ac59b08dbb726..52d53538bb2f8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -593,20 +593,36 @@ private static void checkGroupingFunctionInGroupBy(LogicalPlan p, Set l // check if the query has a grouping function (Histogram) but no GROUP BY if (p instanceof Project) { Project proj = (Project) p; - proj.projections().forEach(e -> e.forEachDown(f -> + proj.projections().forEach(e -> e.forEachDown(f -> localFailures.add(fail(f, "[{}] needs to be part of the grouping", Expressions.name(f))), GroupingFunction.class)); } else if (p instanceof Aggregate) { - // if it does have a GROUP BY, check if the groupings contain the grouping functions (Histograms) + // if it does have a GROUP BY, check if the groupings contain the grouping functions (Histograms) Aggregate a = (Aggregate) p; a.aggregates().forEach(agg -> agg.forEachDown(e -> { - if (a.groupings().size() == 0 + if (a.groupings().size() == 0 || Expressions.anyMatch(a.groupings(), g -> g instanceof Function && e.functionEquals((Function) g)) == false) { localFailures.add(fail(e, "[{}] needs to be part of the grouping", Expressions.name(e))); } + else { + checkGroupingFunctionTarget(e, localFailures); + } + }, GroupingFunction.class)); + + a.groupings().forEach(g -> g.forEachDown(e -> { + checkGroupingFunctionTarget(e, localFailures); }, GroupingFunction.class)); } } + private static void checkGroupingFunctionTarget(GroupingFunction f, Set localFailures) { + f.field().forEachDown(e -> { + if (e instanceof GroupingFunction) { + localFailures.add(fail(f.field(), "Cannot embed grouping functions within each other, found [{}] in [{}]", + Expressions.name(f.field()), Expressions.name(f))); + } + }); + } + private static void checkFilterOnAggs(LogicalPlan p, Set localFailures) { if (p instanceof Filter) { Filter filter = (Filter) p; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java index 745cc36e34a57..616c337e64c9a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java @@ -14,9 +14,6 @@ import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.List; -import java.util.Locale; - -import static java.lang.String.format; /** * In a SQL statement, an Expression is whatever a user specifies inside an @@ -39,10 +36,6 @@ public TypeResolution(String message) { this(true, message); } - TypeResolution(String message, Object... args) { - this(true, format(Locale.ROOT, message, args)); - } - private TypeResolution(boolean unresolved, String message) { this.failed = unresolved; this.message = message; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java index 04d660642c8b2..648aff5254561 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java @@ -18,9 +18,9 @@ import java.util.StringJoiner; import java.util.function.Predicate; -import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN; public final class Expressions { @@ -186,7 +186,7 @@ public static TypeResolution typeMustBe(Expression e, String... acceptedTypes) { return predicate.test(e.dataType()) || DataTypes.isNull(e.dataType())? TypeResolution.TYPE_RESOLVED : - new TypeResolution(format(Locale.ROOT, "[%s]%s argument must be [%s], found value [%s] type [%s]", + new TypeResolution(format(null, "[{}]{} argument must be [{}], found value [{}] type [{}]", operationName, paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT), acceptedTypesForErrorMsg(acceptedTypes), diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 558d92351b069..eec483ca219b8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -566,10 +566,20 @@ public void testGroupByScalarOnTopOfGrouping() { } public void testAggsInHistogram() { - assertEquals("1:47: Cannot use an aggregate [MAX] for grouping", - error("SELECT MAX(date) FROM test GROUP BY HISTOGRAM(MAX(int), 1)")); + assertEquals("1:37: Cannot use an aggregate [MAX] for grouping", + error("SELECT MAX(date) FROM test GROUP BY MAX(int)")); } - + + public void testGroupingsInHistogram() { + assertEquals( + "1:47: Cannot embed grouping functions within each other, found [HISTOGRAM(int, 1)] in [HISTOGRAM(HISTOGRAM(int, 1), 1)]", + error("SELECT MAX(date) FROM test GROUP BY HISTOGRAM(HISTOGRAM(int, 1), 1)")); + } + + public void testCastInHistogram() { + accept("SELECT MAX(date) FROM test GROUP BY HISTOGRAM(CAST(int AS LONG), 1)"); + } + public void testHistogramNotInGrouping() { assertEquals("1:8: [HISTOGRAM(date, INTERVAL 1 MONTH)] needs to be part of the grouping", error("SELECT HISTOGRAM(date, INTERVAL 1 MONTH) AS h FROM test")); From 03afa437adc2aacaac7bb80307be380f64254bc7 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sat, 9 Feb 2019 09:22:43 -0500 Subject: [PATCH 11/12] Avoid polluting download stats on builds (#38660) Recently we changed where we source released artifacts for usage in backwards compatibility tests. We now source these from artifacts.elastic.co. To avoid polluting the download stats from builds, we want to add the X-Elastic-No-KPI header to requests from artifacts.elastic.co. To do this, we hack the Ivy feature of custom HTTP header credentials and specify our desired headers. --- .../groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index f240ebb52c8ba..4f88a9791eb9d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -39,6 +39,7 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.artifacts.ResolvedArtifact import org.gradle.api.artifacts.dsl.RepositoryHandler +import org.gradle.api.credentials.HttpHeaderCredentials import org.gradle.api.execution.TaskExecutionGraph import org.gradle.api.plugins.JavaBasePlugin import org.gradle.api.plugins.JavaPlugin @@ -50,6 +51,7 @@ import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.compile.GroovyCompile import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.javadoc.Javadoc +import org.gradle.authentication.http.HttpHeaderAuthentication import org.gradle.internal.jvm.Jvm import org.gradle.process.ExecResult import org.gradle.process.ExecSpec @@ -570,6 +572,14 @@ class BuildPlugin implements Plugin { patternLayout { artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]" } + // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats + credentials(HttpHeaderCredentials) { + name = "X-Elastic-No-KPI" + value = "1" + } + authentication { + header(HttpHeaderAuthentication) + } } repos.maven { name "elastic" From c0328d5dcd95876bc0d7bcffc0c1c37c52741b7e Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Sat, 9 Feb 2019 16:07:36 +0100 Subject: [PATCH 12/12] Fix timezone fallback in ingest processor (#38407) If no timezone was specified in the date processor, then the conversion would lead to wrong time, as UTC was assumed by default, leading to incorrectly parsed dates. This commit does not assume a default timezone and will thus not format the dates in a wrong way. --- .../ingest/common/DateFormat.java | 18 +++- .../ingest/common/DateProcessor.java | 2 +- .../ingest/common/DateFormatTests.java | 13 +++ .../ingest/common/DateProcessorTests.java | 13 +++ .../test/ingest/30_date_processor.yml | 91 +++++++++++++++++++ 5 files changed, 132 insertions(+), 5 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 05aa75944d2f9..7301ad8a9e919 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -87,10 +87,16 @@ Function getFunction(String format, ZoneId zoneId, Locale format = format.substring(1); } + boolean isUtc = ZoneOffset.UTC.equals(zoneId); + int year = LocalDate.now(ZoneOffset.UTC).getYear(); - DateFormatter formatter = DateFormatter.forPattern(format) - .withLocale(locale) - .withZone(zoneId); + DateFormatter dateFormatter = DateFormatter.forPattern(format) + .withLocale(locale); + // if UTC zone is set here, the the time zone specified in the format will be ignored, leading to wrong dates + if (isUtc == false) { + dateFormatter = dateFormatter.withZone(zoneId); + } + final DateFormatter formatter = dateFormatter; return text -> { TemporalAccessor accessor = formatter.parse(text); // if there is no year, we fall back to the current one and @@ -106,7 +112,11 @@ Function getFunction(String format, ZoneId zoneId, Locale accessor = newTime.withZoneSameLocal(zoneId); } - return DateFormatters.from(accessor); + if (isUtc) { + return DateFormatters.from(accessor).withZoneSameInstant(ZoneOffset.UTC); + } else { + return DateFormatters.from(accessor); + } }; } }; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index e7ad1356977e0..390279bdbb5d7 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -43,7 +43,7 @@ public final class DateProcessor extends AbstractProcessor { public static final String TYPE = "date"; static final String DEFAULT_TARGET_FIELD = "@timestamp"; - public static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); private final TemplateScript.Factory timezone; private final TemplateScript.Factory locale; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 136c9f7f69a0a..e44e62be8629a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.test.ESTestCase; @@ -43,6 +44,14 @@ public void testParseJava() { equalTo("11 24 01:29:01")); } + public void testParseJavaWithTimeZone() { + Function javaFunction = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", + ZoneOffset.UTC, Locale.ROOT); + ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100"); + String expectedDateTime = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").withZone(ZoneOffset.UTC).format(datetime); + assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z")); + } + public void testParseJavaDefaultYear() { String format = randomFrom("8dd/MM", "dd/MM"); ZoneId timezone = DateUtils.of("Europe/Amsterdam"); @@ -70,6 +79,10 @@ public void testParseUnixWithMsPrecision() { public void testParseISO8601() { assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(), equalTo(978336000000L)); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), + equalTo("2001-01-01T08:00Z")); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), + equalTo("2001-01-01T08:00Z")); } public void testParseISO8601Failure() { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index 2e32e3fd0ebd2..7582056e0b6b6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -29,6 +29,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -97,6 +98,18 @@ public void testJavaPatternMultipleFormats() { } } + public void testJavaPatternNoTimezone() { + DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), + null, null, + "date_as_string", Arrays.asList("yyyy dd MM HH:mm:ss XXX"), "date_as_date"); + + Map document = new HashMap<>(); + document.put("date_as_string", "2010 12 06 00:00:00 -02:00"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + dateProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T02:00:00.000Z")); + } + public void testInvalidJavaPattern() { try { DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml index b2e83c640388a..99e90064da013 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -39,3 +39,94 @@ teardown: id: 1 - match: { _source.date_source_field: "12/06/2010" } - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" } + +--- +"Test date processor with no timezone configured": + + - do: + ingest.put_pipeline: + id: "my_pipeline" + # sample formats from beats, featuring mongodb, icinga, apache + body: > + { + "description": "_description", + "processors": [ + { + "date" : { + "field" : "date_source_1", + "target_field" : "date_target_1", + "formats" : ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ" ] + } + }, + { + "date" : { + "field" : "date_source_2", + "target_field" : "date_target_2", + "formats" : ["yyyy-MM-dd HH:mm:ss Z" ] + } + }, + { + "date" : { + "field" : "date_source_3", + "target_field" : "date_target_3", + "formats" : [ "dd/MMM/yyyy:H:m:s Z" ] + } + }, + { + "date" : { + "field" : "date_source_4", + "target_field" : "date_target_4", + "formats" : [ "UNIX" ] + } + }, + { + "date" : { + "field" : "date_source_5", + "target_field" : "date_target_5", + "formats" : [ "UNIX_MS" ] + } + }, + { + "date" : { + "field" : "date_source_6", + "target_field" : "date_target_6", + "formats" : [ "TAI64N" ] + } + }, + { + "date" : { + "field" : "date_source_7", + "target_field" : "date_target_7", + "formats" : [ "ISO8601" ] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: { date_source_1: "2018-02-05T13:44:56.657+0100", date_source_2: "2017-04-04 13:43:09 +0200", date_source_3: "10/Aug/2018:09:45:56 +0200", date_source_4: "1", date_source_5: "1", date_source_6: "4000000050d506482dbdf024", date_source_7: "2018-02-05T13:44:56.657+0100" } + + - do: + get: + index: test + id: 1 + - match: { _source.date_source_1: "2018-02-05T13:44:56.657+0100" } + - match: { _source.date_target_1: "2018-02-05T12:44:56.657Z" } + - match: { _source.date_source_2: "2017-04-04 13:43:09 +0200" } + - match: { _source.date_target_2: "2017-04-04T11:43:09.000Z" } + - match: { _source.date_source_3: "10/Aug/2018:09:45:56 +0200" } + - match: { _source.date_target_3: "2018-08-10T07:45:56.000Z" } + - match: { _source.date_source_4: "1" } + - match: { _source.date_target_4: "1970-01-01T00:00:01.000Z" } + - match: { _source.date_source_5: "1" } + - match: { _source.date_target_5: "1970-01-01T00:00:00.001Z" } + - match: { _source.date_source_6: "4000000050d506482dbdf024" } + - match: { _source.date_target_6: "2012-12-22T01:00:46.767Z" } + - match: { _source.date_source_7: "2018-02-05T13:44:56.657+0100" } + - match: { _source.date_target_7: "2018-02-05T12:44:56.657Z" } +