diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index f240ebb52c8ba..4f88a9791eb9d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -39,6 +39,7 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.artifacts.ResolvedArtifact import org.gradle.api.artifacts.dsl.RepositoryHandler +import org.gradle.api.credentials.HttpHeaderCredentials import org.gradle.api.execution.TaskExecutionGraph import org.gradle.api.plugins.JavaBasePlugin import org.gradle.api.plugins.JavaPlugin @@ -50,6 +51,7 @@ import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.compile.GroovyCompile import org.gradle.api.tasks.compile.JavaCompile import org.gradle.api.tasks.javadoc.Javadoc +import org.gradle.authentication.http.HttpHeaderAuthentication import org.gradle.internal.jvm.Jvm import org.gradle.process.ExecResult import org.gradle.process.ExecSpec @@ -570,6 +572,14 @@ class BuildPlugin implements Plugin { patternLayout { artifact "elasticsearch/[module]-[revision](-[classifier]).[ext]" } + // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats + credentials(HttpHeaderCredentials) { + name = "X-Elastic-No-KPI" + value = "1" + } + authentication { + header(HttpHeaderAuthentication) + } } repos.maven { name "elastic" diff --git a/client/rest/build.gradle b/client/rest/build.gradle index 6b22b7b909909..7bbcb1df85694 100644 --- a/client/rest/build.gradle +++ b/client/rest/build.gradle @@ -22,8 +22,8 @@ apply plugin: 'elasticsearch.build' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' -targetCompatibility = JavaVersion.VERSION_1_7 -sourceCompatibility = JavaVersion.VERSION_1_7 +targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_1_8 group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-client' diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle index 382a3f3c9d121..fffc1b711b25c 100644 --- a/client/sniffer/build.gradle +++ b/client/sniffer/build.gradle @@ -20,8 +20,8 @@ apply plugin: 'elasticsearch.build' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' -targetCompatibility = JavaVersion.VERSION_1_7 -sourceCompatibility = JavaVersion.VERSION_1_7 +targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_1_8 group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-client-sniffer' diff --git a/client/test/build.gradle b/client/test/build.gradle index f184cfbb73c3d..25cf23672dac6 100644 --- a/client/test/build.gradle +++ b/client/test/build.gradle @@ -18,8 +18,8 @@ */ apply plugin: 'elasticsearch.build' -targetCompatibility = JavaVersion.VERSION_1_7 -sourceCompatibility = JavaVersion.VERSION_1_7 +targetCompatibility = JavaVersion.VERSION_1_8 +sourceCompatibility = JavaVersion.VERSION_1_8 group = "${group}.client.test" diff --git a/docs/reference/ingest/processors/dissect.asciidoc b/docs/reference/ingest/processors/dissect.asciidoc index 0c04e7ed07396..aa54cb4411f3e 100644 --- a/docs/reference/ingest/processors/dissect.asciidoc +++ b/docs/reference/ingest/processors/dissect.asciidoc @@ -170,7 +170,7 @@ Named skip key modifier example | *Pattern* | `%{clientip} %{?ident} %{?auth} [%{@timestamp}]` | *Input* | 1.2.3.4 - - [30/Apr/1998:22:00:52 +0000] | *Result* a| -* ip = 1.2.3.4 +* clientip = 1.2.3.4 * @timestamp = 30/Apr/1998:22:00:52 +0000 |====== diff --git a/docs/reference/migration/apis/deprecation.asciidoc b/docs/reference/migration/apis/deprecation.asciidoc index 88de3f5d6e3fa..b52079726d868 100644 --- a/docs/reference/migration/apis/deprecation.asciidoc +++ b/docs/reference/migration/apis/deprecation.asciidoc @@ -54,7 +54,7 @@ Example response: { "level" : "critical", "message" : "Cluster name cannot contain ':'", - "url" : "{ref}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_cluster_name", + "url" : "{ref-70}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_cluster_name", "details" : "This cluster is named [mycompany:logging], which contains the illegal character ':'." } ], @@ -64,7 +64,7 @@ Example response: { "level" : "warning", "message" : "Index name cannot contain ':'", - "url" : "{ref}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_index_name", + "url" : "{ref-70}/breaking-changes-7.0.html#_literal_literal_is_no_longer_allowed_in_index_name", "details" : "This index is named [logs:apache], which contains the illegal character ':'." } ] diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index aa74068419df0..e5ba82c406747 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -21,5 +21,7 @@ For more information, see <>. See also <> and <>. +* <> + -- -include::migrate_7_0.asciidoc[] +include::migrate_8_0.asciidoc[] diff --git a/docs/reference/migration/migrate_7_0.asciidoc b/docs/reference/migration/migrate_7_0.asciidoc deleted file mode 100644 index 25c2e3eef440d..0000000000000 --- a/docs/reference/migration/migrate_7_0.asciidoc +++ /dev/null @@ -1,64 +0,0 @@ -[[breaking-changes-7.0]] -== Breaking changes in 7.0 -++++ -7.0 -++++ - -This section discusses the changes that you need to be aware of when migrating -your application to Elasticsearch 7.0. - -See also <> and <>. - -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> - -[float] -=== Indices created before 7.0 - -Elasticsearch 7.0 can read indices created in version 6.0 or above. An -Elasticsearch 7.0 node will not start in the presence of indices created in a -version of Elasticsearch before 6.0. - -[IMPORTANT] -.Reindex indices from Elasticsearch 5.x or before -========================================= - -Indices created in Elasticsearch 5.x or before will need to be reindexed with -Elasticsearch 6.x in order to be readable by Elasticsearch 7.x. - -========================================= - -include::migrate_7_0/aggregations.asciidoc[] -include::migrate_7_0/analysis.asciidoc[] -include::migrate_7_0/cluster.asciidoc[] -include::migrate_7_0/discovery.asciidoc[] -include::migrate_7_0/indices.asciidoc[] -include::migrate_7_0/mappings.asciidoc[] -include::migrate_7_0/search.asciidoc[] -include::migrate_7_0/packaging.asciidoc[] -include::migrate_7_0/plugins.asciidoc[] -include::migrate_7_0/api.asciidoc[] -include::migrate_7_0/java.asciidoc[] -include::migrate_7_0/settings.asciidoc[] -include::migrate_7_0/scripting.asciidoc[] -include::migrate_7_0/snapshotstats.asciidoc[] -include::migrate_7_0/restclient.asciidoc[] -include::migrate_7_0/low_level_restclient.asciidoc[] -include::migrate_7_0/logging.asciidoc[] -include::migrate_7_0/node.asciidoc[] diff --git a/docs/reference/migration/migrate_7_0/aggregations.asciidoc b/docs/reference/migration/migrate_7_0/aggregations.asciidoc deleted file mode 100644 index 2b8c2ed9cb783..0000000000000 --- a/docs/reference/migration/migrate_7_0/aggregations.asciidoc +++ /dev/null @@ -1,34 +0,0 @@ -[float] -[[breaking_70_aggregations_changes]] -=== Aggregations changes - -[float] -==== Deprecated `global_ordinals_hash` and `global_ordinals_low_cardinality` execution hints for terms aggregations have been removed - -These `execution_hint` are removed and should be replaced by `global_ordinals`. - -[float] -==== `search.max_buckets` in the cluster setting - -The dynamic cluster setting named `search.max_buckets` now defaults -to 10,000 (instead of unlimited in the previous version). -Requests that try to return more than the limit will fail with an exception. - -[float] -==== `missing` option of the `composite` aggregation has been removed - -The `missing` option of the `composite` aggregation, deprecated in 6.x, -has been removed. `missing_bucket` should be used instead. - -[float] -==== Replaced `params._agg` with `state` context variable in scripted metric aggregations - -The object used to share aggregation state between the scripts in a Scripted Metric -Aggregation is now a variable called `state` available in the script context, rather than -being provided via the `params` object as `params._agg`. - -[float] -==== Make metric aggregation script parameters `reduce_script` and `combine_script` mandatory - -The metric aggregation has been changed to require these two script parameters to ensure users are -explicitly defining how their data is processed. diff --git a/docs/reference/migration/migrate_7_0/analysis.asciidoc b/docs/reference/migration/migrate_7_0/analysis.asciidoc deleted file mode 100644 index 36ad41be09aa1..0000000000000 --- a/docs/reference/migration/migrate_7_0/analysis.asciidoc +++ /dev/null @@ -1,41 +0,0 @@ -[float] -[[breaking_70_analysis_changes]] -=== Analysis changes - -[float] -==== Limiting the number of tokens produced by _analyze - -To safeguard against out of memory errors, the number of tokens that can be produced -using the `_analyze` endpoint has been limited to 10000. This default limit can be changed -for a particular index with the index setting `index.analyze.max_token_count`. - -[float] -==== Limiting the length of an analyzed text during highlighting - -Highlighting a text that was indexed without offsets or term vectors, -requires analysis of this text in memory real time during the search request. -For large texts this analysis may take substantial amount of time and memory. -To protect against this, the maximum number of characters that will be analyzed has been -limited to 1000000. This default limit can be changed -for a particular index with the index setting `index.highlight.max_analyzed_offset`. - -[float] -==== `delimited_payload_filter` renaming - -The `delimited_payload_filter` was deprecated and renamed to `delimited_payload` in 6.2. -Using it in indices created before 7.0 will issue deprecation warnings. Using the old -name in new indices created in 7.0 will throw an error. Use the new name `delimited_payload` -instead. - -[float] -==== `standard` filter has been removed - -The `standard` token filter has been removed because it doesn't change anything in the stream. - -[float] -==== Deprecated standard_html_strip analyzer - -The `standard_html_strip` analyzer has been deprecated, and should be replaced -with a combination of the `standard` tokenizer and `html_strip` char_filter. -Indexes created using this analyzer will still be readable in elasticsearch 7.0, -but it will not be possible to create new indexes using it. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/api.asciidoc b/docs/reference/migration/migrate_7_0/api.asciidoc deleted file mode 100644 index 6c1d03760f904..0000000000000 --- a/docs/reference/migration/migrate_7_0/api.asciidoc +++ /dev/null @@ -1,156 +0,0 @@ -[float] -[[breaking_70_api_changes]] -=== API changes - -[float] -==== Internal Versioning is no longer supported for optimistic concurrency control - -Elasticsearch maintains a numeric version field for each document it stores. That field -is incremented by one with every change to the document. Until 7.0.0 the API allowed using -that field for optimistic concurrency control, i.e., making a write operation conditional -on the current document version. Sadly, that approach is flawed because the value of the -version doesn't always uniquely represent a change to the document. If a primary fails -while handling a write operation, it may expose a version that will then be reused by the -new primary. - -Due to that issue, internal versioning can no longer be used and is replaced by a new -method based on sequence numbers. See <> for more details. - -Note that the `external` versioning type is still fully supported. - -[float] -==== Camel case and underscore parameters deprecated in 6.x have been removed -A number of duplicate parameters deprecated in 6.x have been removed from -Bulk request, Multi Get request, Term Vectors request, and More Like This Query -requests. - -The following camel case parameters have been removed: - -* `opType` -* `versionType`, `_versionType` - -The following parameters starting with underscore have been removed: - -* `_parent` -* `_retry_on_conflict` -* `_routing` -* `_version` -* `_version_type` - -Instead of these removed parameters, use their non camel case equivalents without -starting underscore, e.g. use `version_type` instead of `_version_type` or `versionType`. - -[float] -==== Thread pool info - -In previous versions of Elasticsearch, the thread pool info returned in the -<> returned `min` and `max` values reflecting -the configured minimum and maximum number of threads that could be in each -thread pool. The trouble with this representation is that it does not align with -the configuration parameters used to configure thread pools. For -<>, the minimum number of threads is -configured by a parameter called `core` and the maximum number of threads is -configured by a parameter called `max`. For <>, there is only one configuration parameter along these lines and that -parameter is called `size`, reflecting the fixed number of threads in the -pool. This discrepancy between the API and the configuration parameters has been -rectified. Now, the API will report `core` and `max` for scaling thread pools, -and `size` for fixed thread pools. - -Similarly, in the cat thread pool API the existing `size` output has been -renamed to `pool_size` which reflects the number of threads currently in the -pool; the shortcut for this value has been changed from `s` to `psz`. The `min` -output has been renamed to `core` with a shortcut of `cr`, the shortcut for -`max` has been changed to `mx`, and the `size` output with a shortcut of `sz` -has been reused to report the configured number of threads in the pool. This -aligns the output of the API with the configuration values for thread -pools. Note that `core` and `max` will be populated for scaling thread pools, -and `size` will be populated for fixed thread pools. - -[float] -==== The parameter `fields` deprecated in 6.x has been removed from Bulk request -and Update request. The Update API returns `400 - Bad request` if request contains -unknown parameters (instead of ignored in the previous version). - -[float] -[[remove-suggest-metric]] -==== Remove support for `suggest` metric/index metric in indices stats and nodes stats APIs - -Previously, `suggest` stats were folded into `search` stats. Support for the -`suggest` metric on the indices stats and nodes stats APIs remained for -backwards compatibility. Backwards support for the `suggest` metric was -deprecated in 6.3.0 and now removed in 7.0.0. - -[[remove-field-caps-body]] - -In the past, `fields` could be provided either as a parameter, or as part of the request -body. Specifying `fields` in the request body as opposed to a parameter was deprecated -in 6.4.0, and is now unsupported in 7.0.0. - -[float] -==== `copy_settings` is deprecated on shrink and split APIs - -Versions of Elasticsearch prior to 6.4.0 did not copy index settings on shrink -and split operations. Starting with Elasticsearch 7.0.0, the default behavior -will be for such settings to be copied on such operations. To enable users in -6.4.0 to transition in 6.4.0 to the default behavior in 7.0.0, the -`copy_settings` parameter was added on the REST layer. As this behavior will be -the only behavior in 8.0.0, this parameter is deprecated in 7.0.0 for removal in -8.0.0. - -[float] -==== The deprecated stored script contexts have now been removed -When putting stored scripts, support for storing them with the deprecated `template` context or without a context is -now removed. Scripts must be stored using the `script` context as mentioned in the documentation. - -[float] -==== Removed Get Aliases API limitations when {security-features} are enabled - -The behavior and response codes of the get aliases API no longer vary -depending on whether {security-features} are enabled. Previously a -404 - NOT FOUND (IndexNotFoundException) could be returned in case the -current user was not authorized for any alias. An empty response with -status 200 - OK is now returned instead at all times. - -[float] -==== Put User API response no longer has `user` object - -The Put User API response was changed in 6.5.0 to add the `created` field -outside of the user object where it previously had been. In 7.0.0 the user -object has been removed in favor of the top level `created` field. - -[float] -==== Source filtering url parameters `_source_include` and `_source_exclude` have been removed - -The deprecated in 6.x url parameters are now removed. Use `_source_includes` and `_source_excludes` instead. - -[float] -==== Multi Search Request metadata validation - -MultiSearchRequests issued through `_msearch` now validate all keys in the metadata section. Previously unknown keys were ignored -while now an exception is thrown. - -[float] -==== Deprecated graph endpoints removed - -The deprecated graph endpoints (those with `/_graph/_explore`) have been -removed. - - -[float] -==== Deprecated `_termvector` endpoint removed - -The `_termvector` endpoint was deprecated in 2.0 and has now been removed. -The endpoint `_termvectors` (plural) should be used instead. - -[float] -==== When {security-features} are enabled, index monitoring APIs over restricted indices are not authorized implicitly anymore - -Restricted indices (currently only `.security-6` and `.security`) are special internal -indices that require setting the `allow_restricted_indices` flag on every index -permission that covers them. If this flag is `false` (default) the permission -will not cover these and actions against them will not be authorized. -However, the monitoring APIs were the only exception to this rule. This exception -has been forfeited and index monitoring privileges have to be granted explicitly, -using the `allow_restricted_indices` flag on the permission (as any other index -privilege). diff --git a/docs/reference/migration/migrate_7_0/cluster.asciidoc b/docs/reference/migration/migrate_7_0/cluster.asciidoc deleted file mode 100644 index bfe7d5df2d094..0000000000000 --- a/docs/reference/migration/migrate_7_0/cluster.asciidoc +++ /dev/null @@ -1,27 +0,0 @@ -[float] -[[breaking_70_cluster_changes]] -=== Cluster changes - -[float] -==== `:` is no longer allowed in cluster name - -Due to cross-cluster search using `:` to separate a cluster and index name, -cluster names may no longer contain `:`. - -[float] -==== New default for `wait_for_active_shards` parameter of the open index command - -The default value for the `wait_for_active_shards` parameter of the open index API -is changed from 0 to 1, which means that the command will now by default wait for all -primary shards of the opened index to be allocated. - -[float] -==== Shard preferences `_primary`, `_primary_first`, `_replica`, and `_replica_first` are removed -These shard preferences are removed in favour of the `_prefer_nodes` and `_only_nodes` preferences. - -[float] -==== Cluster-wide shard soft limit -Clusters now have soft limits on the total number of open shards in the cluster -based on the number of nodes and the `cluster.max_shards_per_node` cluster -setting, to prevent accidental operations that would destabilize the cluster. -More information can be found in the <>. diff --git a/docs/reference/migration/migrate_7_0/discovery.asciidoc b/docs/reference/migration/migrate_7_0/discovery.asciidoc deleted file mode 100644 index 56449625246cd..0000000000000 --- a/docs/reference/migration/migrate_7_0/discovery.asciidoc +++ /dev/null @@ -1,48 +0,0 @@ -[float] -[[breaking_70_discovery_changes]] -=== Discovery changes - -[float] -==== Cluster bootstrapping is required if discovery is configured - -The first time a cluster is started, `cluster.initial_master_nodes` must be set -to perform cluster bootstrapping. It should contain the names of the -master-eligible nodes in the initial cluster and be defined on every -master-eligible node in the cluster. See <> for an example, and the -<> describes this setting in more detail. - -The `discovery.zen.minimum_master_nodes` setting is permitted, but ignored, on -7.x nodes. - -[float] -==== Removing master-eligible nodes sometimes requires voting exclusions - -If you wish to remove half or more of the master-eligible nodes from a cluster, -you must first exclude the affected nodes from the voting configuration using -the <>. -If you remove fewer than half of the master-eligible nodes at the same time, -voting exclusions are not required. If you remove only master-ineligible nodes -such as data-only nodes or coordinating-only nodes, voting exclusions are not -required. Likewise, if you add nodes to the cluster, voting exclusions are not -required. - -[float] -==== Discovery configuration is required in production - -Production deployments of Elasticsearch now require at least one of the -following settings to be specified in the `elasticsearch.yml` configuration -file: - -- `discovery.seed_hosts` -- `discovery.seed_providers` -- `cluster.initial_master_nodes` - -[float] -==== New name for `no_master_block` setting - -The `discovery.zen.no_master_block` setting is now known as -`cluster.no_master_block`. Any value set for `discovery.zen.no_master_block` is -now ignored. You should remove this setting and, if needed, set -`cluster.no_master_block` appropriately after the upgrade. diff --git a/docs/reference/migration/migrate_7_0/indices.asciidoc b/docs/reference/migration/migrate_7_0/indices.asciidoc deleted file mode 100644 index f832ae7feb6d3..0000000000000 --- a/docs/reference/migration/migrate_7_0/indices.asciidoc +++ /dev/null @@ -1,110 +0,0 @@ -[float] -[[breaking_70_indices_changes]] -=== Indices changes - -[float] -==== Index creation no longer defaults to five shards -Previous versions of Elasticsearch defaulted to creating five shards per index. -Starting with 7.0.0, the default is now one shard per index. - -[float] -==== `:` is no longer allowed in index name - -Due to cross-cluster search using `:` to separate a cluster and index name, -index names may no longer contain `:`. - -[float] -==== `index.unassigned.node_left.delayed_timeout` may no longer be negative - -Negative values were interpreted as zero in earlier versions but are no -longer accepted. - -[float] -==== `_flush` and `_force_merge` will no longer refresh - -In previous versions issuing a `_flush` or `_force_merge` (with `flush=true`) -had the undocumented side-effect of refreshing the index which made new documents -visible to searches and non-realtime GET operations. From now on these operations -don't have this side-effect anymore. To make documents visible an explicit `_refresh` -call is needed unless the index is refreshed by the internal scheduler. - -[float] -==== Limit to the difference between max_size and min_size in NGramTokenFilter and NGramTokenizer - -To safeguard against creating too many index terms, the difference between `max_ngram` and -`min_ngram` in `NGramTokenFilter` and `NGramTokenizer` has been limited to 1. This default -limit can be changed with the index setting `index.max_ngram_diff`. Note that if the limit is -exceeded a error is thrown only for new indices. For existing pre-7.0 indices, a deprecation -warning is logged. - -[float] -==== Limit to the difference between max_size and min_size in ShingleTokenFilter - -To safeguard against creating too many tokens, the difference between `max_shingle_size` and -`min_shingle_size` in `ShingleTokenFilter` has been limited to 3. This default -limit can be changed with the index setting `index.max_shingle_diff`. Note that if the limit is -exceeded a error is thrown only for new indices. For existing pre-7.0 indices, a deprecation -warning is logged. - -[float] -==== Document distribution changes - -Indices created with version `7.0.0` onwards will have an automatic `index.number_of_routing_shards` -value set. This might change how documents are distributed across shards depending on how many -shards the index has. In order to maintain the exact same distribution as a pre `7.0.0` index, the -`index.number_of_routing_shards` must be set to the `index.number_of_shards` at index creation time. -Note: if the number of routing shards equals the number of shards `_split` operations are not supported. - -[float] -==== Skipped background refresh on search idle shards - -Shards belonging to an index that does not have an explicit -`index.refresh_interval` configured will no longer refresh in the background -once the shard becomes "search idle", ie the shard hasn't seen any search -traffic for `index.search.idle.after` seconds (defaults to `30s`). Searches -that access a search idle shard will be "parked" until the next refresh -happens. Indexing requests with `wait_for_refresh` will also trigger -a background refresh. - -[float] -==== Remove deprecated url parameters for Clear Indices Cache API - -The following previously deprecated url parameter have been removed: - -* `filter` - use `query` instead -* `filter_cache` - use `query` instead -* `request_cache` - use `request` instead -* `field_data` - use `fielddata` instead - -[float] -==== `network.breaker.inflight_requests.overhead` increased to 2 - -Previously the in flight requests circuit breaker considered only the raw byte representation. -By bumping the value of `network.breaker.inflight_requests.overhead` from 1 to 2, this circuit -breaker considers now also the memory overhead of representing the request as a structured object. - -[float] -==== Parent circuit breaker changes - -The parent circuit breaker defines a new setting `indices.breaker.total.use_real_memory` which is -`true` by default. This means that the parent circuit breaker will trip based on currently used -heap memory instead of only considering the reserved memory by child circuit breakers. When this -setting is `true`, the default parent breaker limit also changes from 70% to 95% of the JVM heap size. -The previous behavior can be restored by setting `indices.breaker.total.use_real_memory` to `false`. - -[float] -==== Field data circuit breaker changes -As doc values have been enabled by default in earlier versions of Elasticsearch, -there is less need for fielddata. Therefore, the default value of the setting -`indices.breaker.fielddata.limit` has been lowered from 60% to 40% of the JVM -heap size. - -[float] -==== `fix` value for `index.shard.check_on_startup` is removed - -Deprecated option value `fix` for setting `index.shard.check_on_startup` is not supported. - -[float] -==== `elasticsearch-translog` is removed - -Use the `elasticsearch-shard` tool to remove corrupted translog data. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/java.asciidoc b/docs/reference/migration/migrate_7_0/java.asciidoc deleted file mode 100644 index f34b1c6ca9906..0000000000000 --- a/docs/reference/migration/migrate_7_0/java.asciidoc +++ /dev/null @@ -1,47 +0,0 @@ -[float] -[[breaking_70_java_changes]] -=== Java API changes - -[float] -==== `isShardsAcked` deprecated in `6.2` has been removed - -`isShardsAcked` has been replaced by `isShardsAcknowledged` in -`CreateIndexResponse`, `RolloverResponse` and -`CreateIndexClusterStateUpdateResponse`. - -[float] -==== `prepareExecute` removed from the client api - -The `prepareExecute` method which created a request builder has been -removed from the client api. Instead, construct a builder for the -appropriate request directly. - -[float] -==== Some Aggregation classes have moved packages - -* All classes present in `org.elasticsearch.search.aggregations.metrics.*` packages -were moved to a single `org.elasticsearch.search.aggregations.metrics` package. - -* All classes present in `org.elasticsearch.search.aggregations.pipeline.*` packages -were moved to a single `org.elasticsearch.search.aggregations.pipeline` package. In -addition, `org.elasticsearch.search.aggregations.pipeline.PipelineAggregationBuilders` -was moved to `org.elasticsearch.search.aggregations.PipelineAggregationBuilders` - - -[float] -==== `Retry.withBackoff` methods with `Settings` removed - -The variants of `Retry.withBackoff` that included `Settings` have been removed -because `Settings` is no longer needed. - -[float] -==== Deprecated method `Client#termVector` removed - -The client method `termVector`, deprecated in 2.0, has been removed. The method -`termVectors` (plural) should be used instead. - -[float] -==== Deprecated constructor `AbstractLifecycleComponent(Settings settings)` removed - -The constructor `AbstractLifecycleComponent(Settings settings)`, deprecated in 6.7 -has been removed. The parameterless constructor should be used instead. diff --git a/docs/reference/migration/migrate_7_0/logging.asciidoc b/docs/reference/migration/migrate_7_0/logging.asciidoc deleted file mode 100644 index 1329def9a1878..0000000000000 --- a/docs/reference/migration/migrate_7_0/logging.asciidoc +++ /dev/null @@ -1,42 +0,0 @@ -[float] -[[breaking_70_logging_changes]] -=== Logging changes - -[float] -==== New JSON format log files in `log` directory - -Elasticsearch now will produce additional log files in JSON format. They will be stored in `*.json` suffix files. -Following files should be expected now in log directory: -* ${cluster_name}_server.json -* ${cluster_name}_deprecation.json -* ${cluster_name}_index_search_slowlog.json -* ${cluster_name}_index_indexing_slowlog.json -* ${cluster_name}.log -* ${cluster_name}_deprecation.log -* ${cluster_name}_index_search_slowlog.log -* ${cluster_name}_index_indexing_slowlog.log -* ${cluster_name}_audit.json -* gc.log - -Note: You can configure which of these files are written by editing `log4j2.properties`. - -[float] -==== Log files ending with `*.log` deprecated -Log files with the `.log` file extension using the old pattern layout format -are now considered deprecated and the newly added JSON log file format with -the `.json` file extension should be used instead. -Note: GC logs which are written to the file `gc.log` will not be changed. - -[float] -==== Docker output in JSON format - -All Docker console logs are now in JSON format. You can distinguish logs streams with the `type` field. - -[float] -==== Audit plaintext log file removed, JSON file renamed - -Elasticsearch no longer produces the `${cluster_name}_access.log` plaintext -audit log file. The `${cluster_name}_audit.log` files also no longer exist; they -are replaced by `${cluster_name}_audit.json` files. When auditing is enabled, -auditing events are stored in these dedicated JSON log files on each node. - diff --git a/docs/reference/migration/migrate_7_0/low_level_restclient.asciidoc b/docs/reference/migration/migrate_7_0/low_level_restclient.asciidoc deleted file mode 100644 index 0820c7f01cc70..0000000000000 --- a/docs/reference/migration/migrate_7_0/low_level_restclient.asciidoc +++ /dev/null @@ -1,17 +0,0 @@ -[float] -[[breaking_70_low_level_restclient_changes]] -=== Low-level REST client changes - -[float] -==== Deprecated flavors of performRequest have been removed - -We deprecated the flavors of `performRequest` and `performRequestAsync` that -do not take `Request` objects in 6.4.0 in favor of the flavors that take -`Request` objects because those methods can be extended without breaking -backwards compatibility. - -[float] -==== Removed setHosts - -We deprecated `setHosts` in 6.4.0 in favor of `setNodes` because it supports -host metadata used by the `NodeSelector`. diff --git a/docs/reference/migration/migrate_7_0/mappings.asciidoc b/docs/reference/migration/migrate_7_0/mappings.asciidoc deleted file mode 100644 index 653dd2fb4ca46..0000000000000 --- a/docs/reference/migration/migrate_7_0/mappings.asciidoc +++ /dev/null @@ -1,74 +0,0 @@ -[float] -[[breaking_70_mappings_changes]] -=== Mapping changes - -[float] -==== The `_all` meta field is removed - -The `_all` field deprecated in 6 have now been removed. - -[float] -==== The `_uid` meta field is removed - -This field used to index a composite key formed of the `_type` and the `_id`. -Now that indices cannot have multiple types, this has been removed in favour -of `_id`. - -[float] -==== The `_default_` mapping is no longer allowed - -The `_default_` mapping has been deprecated in 6.0 and is now no longer allowed -in 7.0. Trying to configure a `_default_` mapping on 7.x indices will result in -an error. - -[float] -==== `index_options` for numeric fields has been removed - -The `index_options` field for numeric fields has been deprecated in 6 and has now been removed. - -[float] -==== Limiting the number of `nested` json objects - -To safeguard against out of memory errors, the number of nested json objects within a single -document across all fields has been limited to 10000. This default limit can be changed with -the index setting `index.mapping.nested_objects.limit`. - -[float] -==== The `update_all_types` option has been removed - -This option is useless now that all indices have at most one type. - -[float] -==== The `classic` similarity has been removed - -The `classic` similarity relied on coordination factors for scoring to be good -in presence of stopwords in the query. This feature has been removed from -Lucene, which means that the `classic` similarity now produces scores of lower -quality. It is advised to switch to `BM25` instead, which is widely accepted -as a better alternative. - -[float] -==== Similarities fail when unsupported options are provided - -An error will now be thrown when unknown configuration options are provided -to similarities. Such unknown parameters were ignored before. - -[float] -==== Changed default `geo_shape` indexing strategy - -`geo_shape` types now default to using a vector indexing approach based on Lucene's new -`LatLonShape` field type. This indexes shapes as a triangular mesh instead of decomposing -them into individual grid cells. To index using legacy prefix trees the `tree` parameter -must be explicitly set to one of `quadtree` or `geohash`. Note that these strategies are -now deprecated and will be removed in a future version. - -IMPORTANT NOTE: If using timed index creation from templates, the `geo_shape` mapping -should also be changed in the template to explicitly define `tree` to one of `geohash` -or `quadtree`. This will ensure compatibility with previously created indexes. - -[float] -==== Deprecated `geo_shape` parameters - -The following type parameters are deprecated for the `geo_shape` field type: `tree`, -`precision`, `tree_levels`, `distance_error_pct`, `points_only`, and `strategy`. They -will be removed in a future version. diff --git a/docs/reference/migration/migrate_7_0/node.asciidoc b/docs/reference/migration/migrate_7_0/node.asciidoc deleted file mode 100644 index 3b8a9d84e765d..0000000000000 --- a/docs/reference/migration/migrate_7_0/node.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[float] -[[breaking_70_node_changes]] -=== Node start up - -[float] -==== Nodes with left-behind data or metadata refuse to start -Repurposing an existing node by changing node.master or node.data to false can leave lingering on-disk metadata and -data around, which will not be accessible by the node's new role. Beside storing non-accessible data, this can lead -to situations where dangling indices are imported even though the node might not be able to host any shards, leading -to a red cluster health. To avoid this, - -* nodes with on-disk shard data and node.data set to false will refuse to start -* nodes with on-disk index/shard data and both node.master and node.data set to false will refuse to start - -Beware that such role changes done prior to the 7.0 upgrade could prevent node start up in 7.0. diff --git a/docs/reference/migration/migrate_7_0/packaging.asciidoc b/docs/reference/migration/migrate_7_0/packaging.asciidoc deleted file mode 100644 index e2380613d8f7b..0000000000000 --- a/docs/reference/migration/migrate_7_0/packaging.asciidoc +++ /dev/null @@ -1,18 +0,0 @@ -[float] -[[breaking_70_packaging_changes]] -=== Packaging changes - -[float] -[[systemd-service-file-config]] -==== systemd service file is no longer configuration - -The systemd service file `/usr/lib/systemd/system/elasticsearch.service` -was previously marked as a configuration file in rpm and deb packages. -Overrides to the systemd elasticsearch service should be made -in `/etc/systemd/system/elasticsearch.service.d/override.conf`. - -[float] -==== tar package no longer includes windows specific files - -The tar package previously included files in the `bin` directory meant only -for windows. These files have been removed. Use the `zip` package instead. diff --git a/docs/reference/migration/migrate_7_0/plugins.asciidoc b/docs/reference/migration/migrate_7_0/plugins.asciidoc deleted file mode 100644 index 05de7e85b8e92..0000000000000 --- a/docs/reference/migration/migrate_7_0/plugins.asciidoc +++ /dev/null @@ -1,77 +0,0 @@ -[float] -[[breaking_70_plugins_changes]] -=== Plugins changes - -[float] -==== Azure Repository plugin - -* The legacy azure settings which where starting with `cloud.azure.storage.` prefix have been removed. -This includes `account`, `key`, `default` and `timeout`. -You need to use settings which are starting with `azure.client.` prefix instead. - -* Global timeout setting `cloud.azure.storage.timeout` has been removed. -You must set it per azure client instead. Like `azure.client.default.timeout: 10s` for example. - -See {plugins}/repository-azure-repository-settings.html#repository-azure-repository-settings[Azure Repository settings]. - -[float] -==== Google Cloud Storage Repository plugin - -* The repository settings `application_name`, `connect_timeout` and `read_timeout` have been removed and -must now be specified in the client settings instead. - -See {plugins}/repository-gcs-client.html#repository-gcs-client[Google Cloud Storage Client Settings]. - -[float] -==== S3 Repository Plugin - -* The plugin now uses the path style access pattern for all requests. -In previous versions it was automatically determining whether to use virtual hosted style or path style -access. - -[float] -==== Analysis Plugin changes - -* The misspelled helper method `requriesAnalysisSettings(AnalyzerProvider provider)` has been -renamed to `requiresAnalysisSettings` - -[float] -==== File-based discovery plugin - -* This plugin has been removed since its functionality is now part of -Elasticsearch and requires no plugin. The location of the hosts file has moved -from `$ES_PATH_CONF/file-discovery/unicast_hosts.txt` to -`$ES_PATH_CONF/unicast_hosts.txt`. See <> for further information. - -[float] -==== Security Extensions - -As a consequence of the <>, -the `getRealmSettings` method has been removed from the `SecurityExtension` class, -and the `settings` method on `RealmConfig` now returns the node's (global) settings. -Custom security extensions should register their settings by implementing the standard -`Plugin.getSettings` method, and can retrieve them from `RealmConfig.settings()` or -using one of the `RealmConfig.getSetting` methods. -Each realm setting should be defined as an `AffixSetting` as shown in the example below: -[source,java] --------------------------------------------------- -Setting.AffixSetting MY_SETTING = Setting.affixKeySetting( - "xpack.security.authc.realms." + MY_REALM_TYPE + ".", "my_setting", - key -> Setting.simpleString(key, properties) -); --------------------------------------------------- - -The `RealmSettings.simpleString` method can be used as a convenience for the above. - -[float] -==== Tribe node removed - -Tribe node functionality has been removed in favor of -<>. - -[float] -==== Discovery implementations are no longer pluggable - -* The method `DiscoveryPlugin#getDiscoveryTypes()` was removed, so that plugins - can no longer provide their own discovery implementations. diff --git a/docs/reference/migration/migrate_7_0/restclient.asciidoc b/docs/reference/migration/migrate_7_0/restclient.asciidoc deleted file mode 100644 index 39d19c345cd95..0000000000000 --- a/docs/reference/migration/migrate_7_0/restclient.asciidoc +++ /dev/null @@ -1,32 +0,0 @@ -[float] -[[breaking_70_restclient_changes]] -=== High-level REST client changes - -[float] -==== API methods accepting `Header` argument have been removed - -All API methods accepting headers as a `Header` varargs argument, deprecated -since 6.4, have been removed in favour of the newly introduced methods that -accept instead a `RequestOptions` argument. In case you are not specifying any -header, e.g. `client.index(indexRequest)` becomes -`client.index(indexRequest, RequestOptions.DEFAULT)`. -In case you are specifying headers -e.g. `client.index(indexRequest, new Header("name" "value"))` becomes -`client.index(indexRequest, RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build());` - -[float] -==== Cluster Health API default to `cluster` level - -The Cluster Health API used to default to `shards` level to ease migration -from transport client that doesn't support the `level` parameter and always -returns information including indices and shards details. The level default -value has been aligned with the Elasticsearch default level: `cluster`. - -=== Low-level REST client changes - -[float] -==== Support for `maxRetryTimeout` removed from RestClient - -`RestClient` and `RestClientBuilder` no longer support the `maxRetryTimeout` -setting. The setting was removed as its counting mechanism was not accurate -and caused issues while adding little value. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/scripting.asciidoc b/docs/reference/migration/migrate_7_0/scripting.asciidoc deleted file mode 100644 index 99afca91e0119..0000000000000 --- a/docs/reference/migration/migrate_7_0/scripting.asciidoc +++ /dev/null @@ -1,37 +0,0 @@ -[float] -[[breaking_70_scripting_changes]] -=== Scripting changes - -[float] -==== getDate() and getDates() removed - -Fields of type `long` and `date` had `getDate()` and `getDates()` methods -(for multi valued fields) to get an object with date specific helper methods -for the current doc value. In 5.3.0, `date` fields were changed to expose -this same date object directly when calling `doc["myfield"].value`, and -the getter methods for date objects were deprecated. These methods have -now been removed. Instead, use `.value` on `date` fields, or explicitly -parse `long` fields into a date object using -`Instance.ofEpochMillis(doc["myfield"].value)`. - -[float] -==== Accessing missing document values will throw an error -`doc['field'].value` will throw an exception if -the document is missing a value for the field `field`. - -To check if a document is missing a value, you can use -`doc['field'].size() == 0`. - - -[float] -==== Script errors will return as `400` error codes - -Malformed scripts, either in search templates, ingest pipelines or search -requests, return `400 - Bad request` while they would previously return -`500 - Internal Server Error`. This also applies for stored scripts. - -[float] -==== getValues() removed - -The `ScriptDocValues#getValues()` method is deprecated in 6.6 and will -be removed in 7.0. Use `doc["foo"]` in place of `doc["foo"].values`. diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc deleted file mode 100644 index afe96fd8851a9..0000000000000 --- a/docs/reference/migration/migrate_7_0/search.asciidoc +++ /dev/null @@ -1,248 +0,0 @@ -[float] -[[breaking_70_search_changes]] -=== Search and Query DSL changes - -[float] -==== Changes to queries -* The default value for `transpositions` parameter of `fuzzy` query - has been changed to `true`. - -* The `query_string` options `use_dismax`, `split_on_whitespace`, - `all_fields`, `locale`, `auto_generate_phrase_query` and - `lowercase_expanded_terms` deprecated in 6.x have been removed. - -* Purely negative queries (only MUST_NOT clauses) now return a score of `0` - rather than `1`. - -* The boundary specified using geohashes in the `geo_bounding_box` query - now include entire geohash cell, instead of just geohash center. - -* Attempts to generate multi-term phrase queries against non-text fields - with a custom analyzer will now throw an exception. - -* An `envelope` crossing the dateline in a `geo_shape `query is now processed - correctly when specified using REST API instead of having its left and - right corners flipped. - -* Attempts to set `boost` on inner span queries will now throw a parsing exception. - -[float] -==== Adaptive replica selection enabled by default - -Adaptive replica selection has been enabled by default. If you wish to return to -the older round robin of search requests, you can use the -`cluster.routing.use_adaptive_replica_selection` setting: - -[source,js] --------------------------------------------------- -PUT /_cluster/settings -{ - "transient": { - "cluster.routing.use_adaptive_replica_selection": false - } -} --------------------------------------------------- -// CONSOLE - -[float] -==== Search API returns `400` for invalid requests - -The Search API returns `400 - Bad request` while it would previously return -`500 - Internal Server Error` in the following cases of invalid request: - -* the result window is too large -* sort is used in combination with rescore -* the rescore window is too large -* the number of slices is too large -* keep alive for scroll is too large -* number of filters in the adjacency matrix aggregation is too large -* script compilation errors - -[float] -==== Scroll queries cannot use the `request_cache` anymore - -Setting `request_cache:true` on a query that creates a scroll (`scroll=1m`) -has been deprecated in 6 and will now return a `400 - Bad request`. -Scroll queries are not meant to be cached. - -[float] -==== Scroll queries cannot use `rescore` anymore - -Including a rescore clause on a query that creates a scroll (`scroll=1m`) has -been deprecated in 6.5 and will now return a `400 - Bad request`. Allowing -rescore on scroll queries would break the scroll sort. In the 6.x line, the -rescore clause was silently ignored (for scroll queries), and it was allowed in -the 5.x line. - -[float] -==== Term Suggesters supported distance algorithms - -The following string distance algorithms were given additional names in 6.2 and -their existing names were deprecated. The deprecated names have now been -removed. - -* `levenstein` - replaced by `levenshtein` -* `jarowinkler` - replaced by `jaro_winkler` - -[float] -==== `popular` mode for Suggesters - -The `popular` mode for Suggesters (`term` and `phrase`) now uses the doc frequency -(instead of the sum of the doc frequency) of the input terms to compute the frequency -threshold for candidate suggestions. - -[float] -==== Limiting the number of terms that can be used in a Terms Query request - -Executing a Terms Query with a lot of terms may degrade the cluster performance, -as each additional term demands extra processing and memory. -To safeguard against this, the maximum number of terms that can be used in a -Terms Query request has been limited to 65536. This default maximum can be changed -for a particular index with the index setting `index.max_terms_count`. - -[float] -==== Limiting the length of regex that can be used in a Regexp Query request - -Executing a Regexp Query with a long regex string may degrade search performance. -To safeguard against this, the maximum length of regex that can be used in a -Regexp Query request has been limited to 1000. This default maximum can be changed -for a particular index with the index setting `index.max_regex_length`. - -[float] -==== Limiting the number of auto-expanded fields - -Executing queries that use automatic expansion of fields (e.g. `query_string`, `simple_query_string` -or `multi_match`) can have performance issues for indices with a large numbers of fields. -To safeguard against this, a hard limit of 1024 fields has been introduced for queries -using the "all fields" mode ("default_field": "*") or other fieldname expansions (e.g. "foo*"). - -[float] -==== Invalid `_search` request body - -Search requests with extra content after the main object will no longer be accepted -by the `_search` endpoint. A parsing exception will be thrown instead. - -[float] -==== Doc-value fields default format - -The format of doc-value fields is changing to be the same as what could be -obtained in 6.x with the special `use_field_mapping` format. This is mostly a -change for date fields, which are now formatted based on the format that is -configured in the mappings by default. This behavior can be changed by -specifying a <> within the doc-value -field. - -[float] -==== Context Completion Suggester - -The ability to query and index context enabled suggestions without context, -deprecated in 6.x, has been removed. Context enabled suggestion queries -without contexts have to visit every suggestion, which degrades the search performance -considerably. - -For geo context the value of the `path` parameter is now validated against the mapping, -and the context is only accepted if `path` points to a field with `geo_point` type. - -[float] -==== Semantics changed for `max_concurrent_shard_requests` - -`max_concurrent_shard_requests` used to limit the total number of concurrent shard -requests a single high level search request can execute. In 7.0 this changed to be the -max number of concurrent shard requests per node. The default is now `5`. - -[float] -==== `max_score` set to `null` when scores are not tracked - -`max_score` used to be set to `0` whenever scores are not tracked. `null` is now used -instead which is a more appropriate value for a scenario where scores are not available. - -[float] -==== Negative boosts are not allowed - -Setting a negative `boost` for a query or a field, deprecated in 6x, is not allowed in this version. -To deboost a specific query or field you can use a `boost` comprise between 0 and 1. - -[float] -==== Negative scores are not allowed in Function Score Query - -Negative scores in the Function Score Query are deprecated in 6.x, and are -not allowed in this version. If a negative score is produced as a result -of computation (e.g. in `script_score` or `field_value_factor` functions), -an error will be thrown. - -[float] -==== The filter context has been removed - -The `filter` context has been removed from Elasticsearch's query builders, -the distinction between queries and filters is now decided in Lucene depending -on whether queries need to access score or not. As a result `bool` queries with -`should` clauses that don't need to access the score will no longer set their -`minimum_should_match` to 1. This behavior has been deprecated in the previous -major version. - -[float] -==== `hits.total` is now an object in the search response - -The total hits that match the search request is now returned as an object -with a `value` and a `relation`. `value` indicates the number of hits that -match and `relation` indicates whether the value is accurate (`eq`) or a lower bound -(`gte`): - -``` -{ - "hits": { - "total": { <1> - "value": 1000, - "relation": "eq" - }, - ... - } -} -``` - -The "total" object in the response indicates that the query matches exactly 1000 -documents ("eq"). The `value` is always accurate (`"relation": "eq"`) when -`track_total_hits` is set to true in the request. -You can also retrieve `hits.total` as a number in the rest response by adding -`rest_total_hits_as_int=true` in the request parameter of the search request. -This parameter has been added to ease the transition to the new format and -will be removed in the next major version (8.0). - -[float] -==== `hits.total` is omitted in the response if `track_total_hits` is disabled (false) - -If `track_total_hits` is set to `false` in the search request the search response -will set `hits.total` to null and the object will not be displayed in the rest -layer. You can add `rest_total_hits_as_int=true` in the search request parameters -to get the old format back (`"total": -1`). - -[float] -==== `track_total_hits` defaults to 10,000 - -By default search request will count the total hits accurately up to `10,000` -documents. If the total number of hits that match the query is greater than this - value, the response will indicate that the returned value is a lower bound: - -[source,js] --------------------------------------------------- -{ - "_shards": ... - "timed_out": false, - "took": 100, - "hits": { - "max_score": 1.0, - "total" : { - "value": 10000, <1> - "relation": "gte" <2> - }, - "hits": ... - } -} --------------------------------------------------- -// NOTCONSOLE - -<1> There are at least 10000 documents that match the query -<2> This is a lower bound (`"gte"`). - -You can force the count to always be accurate by setting `"track_total_hits` -to true explicitly in the search request. \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc deleted file mode 100644 index 2e5631b378652..0000000000000 --- a/docs/reference/migration/migrate_7_0/settings.asciidoc +++ /dev/null @@ -1,206 +0,0 @@ -[float] -[[breaking_70_settings_changes]] -=== Settings changes - -[float] -==== The default for `node.name` is now the hostname - -`node.name` now defaults to the hostname at the time when Elasticsearch -is started. Previously the default node name was the first eight characters -of the node id. It can still be configured explicitly in `elasticsearch.yml`. - -[float] -==== Percolator - -* The deprecated `index.percolator.map_unmapped_fields_as_string` setting has been removed in favour of - the `index.percolator.map_unmapped_fields_as_text` setting. - -[float] -==== Index thread pool - -* Internally, single-document index/delete/update requests are executed as bulk - requests with a single-document payload. This means that these requests are - executed on the bulk thread pool. As such, the indexing thread pool is no - longer needed and has been removed. As such, the settings - `thread_pool.index.size` and `thread_pool.index.queue_size` have been removed. - -[float] -[[write-thread-pool-fallback]] -==== Write thread pool fallback - -* The bulk thread pool was replaced by the write thread pool in 6.3.0. However, - for backwards compatibility reasons the name `bulk` was still usable as fallback - settings `thread_pool.bulk.size` and `thread_pool.bulk.queue_size` for - `thread_pool.write.size` and `thread_pool.write.queue_size`, respectively, and - the system property `es.thread_pool.write.use_bulk_as_display_name` was - available to keep the display output in APIs as `bulk` instead of `write`. - These fallback settings and this system property have been removed. - -[float] -==== Disabling memory-mapping - -* The setting `node.store.allow_mmapfs` has been renamed to `node.store.allow_mmap`. - -[float] -[[remove-http-enabled]] -==== Http enabled setting removed - -* The setting `http.enabled` previously allowed disabling binding to HTTP, only allowing -use of the transport client. This setting has been removed, as the transport client -will be removed in the future, thus requiring HTTP to always be enabled. - -[float] -[[remove-http-pipelining-setting]] -==== Http pipelining setting removed - -* The setting `http.pipelining` previously allowed disabling HTTP pipelining support. -This setting has been removed, as disabling http pipelining support on the server -provided little value. The setting `http.pipelining.max_events` can still be used to -limit the number of pipelined requests in-flight. - -[float] -==== Cross-cluster search settings renamed - -The cross-cluster search remote cluster connection infrastructure is also used -in cross-cluster replication. This means that the setting names -`search.remote.*` used for configuring cross-cluster search belie the fact that -they also apply to other situations where a connection to a remote cluster as -used. Therefore, these settings have been renamed from `search.remote.*` to -`cluster.remote.*`. For backwards compatibility purposes, we will fallback to -`search.remote.*` if `cluster.remote.*` is not set. For any such settings stored -in the cluster state, or set on dynamic settings updates, we will automatically -upgrade the setting from `search.remote.*` to `cluster.remote.*`. The fallback -settings will be removed in 8.0.0. - -[float] -[[audit-logfile-local-node-info]] -==== Audit logfile local node info - -The following settings have been removed: - -- `xpack.security.audit.logfile.prefix.emit_node_host_address`, instead use - `xpack.security.audit.logfile.emit_node_host_address` -- `xpack.security.audit.logfile.prefix.emit_node_host_name`, instead use - `xpack.security.audit.logfile.emit_node_host_name` -- `xpack.security.audit.logfile.prefix.emit_node_name`, instead use - `xpack.security.audit.logfile.emit_node_name` - -The new settings have the same meaning as the removed ones, but the `prefix` -name component is no longer meaningful as logfile audit entries are structured -JSON documents and are not prefixed by anything. -Moreover, `xpack.security.audit.logfile.emit_node_name` has changed its default -from `true` to `false`. All other settings mentioned before, have kept their -default value of `false`. - -[float] -[[include-realm-type-in-setting]] -==== Security realms settings - -The settings for all security realms must now include the realm type as part -of the setting name, and the explicit `type` setting has been removed. - -A realm that was previous configured as: -[source,yaml] --------------------------------------------------- -xpack.security.authc.realms: - ldap1: - type: ldap - order: 1 - url: "ldaps://ldap.example.com/" --------------------------------------------------- - -Must be migrated to: -[source,yaml] --------------------------------------------------- -xpack.security.authc.realms: - ldap.ldap1: - order: 1 - url: "ldaps://ldap.example.com/" --------------------------------------------------- - -Any realm specific secure settings that have been stored in the elasticsearch -keystore (such as ldap bind passwords, or passwords for ssl keys) must be updated -in a similar way. - -[float] -[[tls-setting-fallback]] -==== TLS/SSL settings - -The default TLS/SSL settings, which were prefixed by `xpack.ssl`, have been removed. -The removal of these default settings also removes the ability for a component to -fallback to a default configuration when using TLS. Each component (realm, transport, http, -http client, etc) must now be configured with their own settings for TLS if it is being -used. - -[float] -[[tls-v1-removed]] -==== TLS v1.0 disabled - -TLS version 1.0 is now disabled by default as it suffers from -https://www.owasp.org/index.php/Transport_Layer_Protection_Cheat_Sheet#Rule_-_Only_Support_Strong_Protocols[known security issues]. -The default protocols are now TLSv1.3 (if supported), TLSv1.2 and TLSv1.1. -You can enable TLS v1.0 by configuring the relevant `ssl.supported_protocols` setting to include `"TLSv1"`, for example: -[source,yaml] --------------------------------------------------- -xpack.security.http.ssl.supported_protocols: [ "TLSv1.3", "TLSv1.2", "TLSv1.1", "TLSv1" ] --------------------------------------------------- - -[float] -[[trial-explicit-security]] -==== Security on Trial Licenses - -On trial licenses, `xpack.security.enabled` defaults to `false`. - -In prior versions, a trial license would automatically enable security if either - -* `xpack.security.transport.enabled` was `true`; _or_ -* the trial license was generated on a version of X-Pack from 6.2 or earlier. - -This behaviour has been now removed, so security is only enabled if: - -* `xpack.security.enabled` is `true`; _or_ -* `xpack.security.enabled` is not set, and a gold or platinum license is installed. - -[float] -[[watcher-notifications-account-settings]] -==== Watcher notifications account settings - -The following settings have been removed in favor of the secure variants. -The <> have to be defined inside each cluster -node's keystore, i.e., they are not to be specified via the cluster settings API. - -- `xpack.notification.email.account..smtp.password`, instead use -`xpack.notification.email.account..smtp.secure_password` -- `xpack.notification.hipchat.account..auth_token`, instead use -`xpack.notification.hipchat.account..secure_auth_token` -- `xpack.notification.jira.account..url`, instead use -`xpack.notification.jira.account..secure_url` -- `xpack.notification.jira.account..user`, instead use -`xpack.notification.jira.account..secure_user` -- `xpack.notification.jira.account..password`, instead use -`xpack.notification.jira.account..secure_password` -- `xpack.notification.pagerduty.account..service_api_key`, instead use -`xpack.notification.pagerduty.account..secure_service_api_key` -- `xpack.notification.slack.account..url`, instead use -`xpack.notification.slack.account..secure_url` - -[float] -[[remove-audit-index-output]] -==== Audit index output type removed - -All the settings under the `xpack.security.audit.index` namespace have been -removed. In addition, the `xpack.security.audit.outputs` setting has been -removed as well. - -These settings enabled and configured the audit index output type. This output -type has been removed because it was unreliable in certain scenarios and this -could have lead to dropping audit events while the operations on the system -were allowed to continue as usual. The recommended replacement is the -use of the `logfile` audit output type and using other components from the -Elastic Stack to handle the indexing part. - -[float] -[[ingest-user-agent-ecs-always]] -==== Ingest User Agent processor always uses `ecs` output format -The deprecated `ecs` setting for the user agent ingest processor has been -removed. https://github.com/elastic/ecs[ECS] format is now the default. diff --git a/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc b/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc deleted file mode 100644 index 2098eb3574ca8..0000000000000 --- a/docs/reference/migration/migrate_7_0/snapshotstats.asciidoc +++ /dev/null @@ -1,15 +0,0 @@ -[float] -[[breaking_70_snapshotstats_changes]] -=== Snapshot stats changes - -Snapshot stats details are provided in a new structured way: - -* `total` section for all the files that are referenced by the snapshot. -* `incremental` section for those files that actually needed to be copied over as part of the incremental snapshotting. -* In case of a snapshot that's still in progress, there's also a `processed` section for files that are in the process of being copied. - -[float] -==== Deprecated `number_of_files`, `processed_files`, `total_size_in_bytes` and `processed_size_in_bytes` snapshot stats properties have been removed - -* Properties `number_of_files` and `total_size_in_bytes` are removed and should be replaced by values of nested object `total`. -* Properties `processed_files` and `processed_size_in_bytes` are removed and should be replaced by values of nested object `processed`. \ No newline at end of file diff --git a/docs/reference/migration/migrate_8_0.asciidoc b/docs/reference/migration/migrate_8_0.asciidoc new file mode 100644 index 0000000000000..4477090dc16b6 --- /dev/null +++ b/docs/reference/migration/migrate_8_0.asciidoc @@ -0,0 +1,12 @@ +[[breaking-changes-8.0]] +== Breaking changes in 8.0 +++++ +8.0 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.0. + +See also <> and <>. + +coming[8.0.0] \ No newline at end of file diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc index 5c397d603bef3..b939364f12027 100644 --- a/docs/reference/query-dsl/match-query.asciidoc +++ b/docs/reference/query-dsl/match-query.asciidoc @@ -32,6 +32,25 @@ optional `should` clauses to match can be set using the <> parameter. +Here is an example when providing additional parameters (note the slight +change in structure, `message` is the field name): + +[source,js] +-------------------------------------------------- +GET /_search +{ + "query": { + "match" : { + "message" : { + "query" : "this is a test", + "operator" : "and" + } + } + } +} +-------------------------------------------------- +// CONSOLE + The `analyzer` can be set to control which analyzer will perform the analysis process on the text. It defaults to the field explicit mapping definition, or the default search analyzer. @@ -56,9 +75,6 @@ rewritten. Fuzzy transpositions (`ab` -> `ba`) are allowed by default but can be disabled by setting `fuzzy_transpositions` to `false`. -Here is an example when providing additional parameters (note the slight -change in structure, `message` is the field name): - [source,js] -------------------------------------------------- GET /_search @@ -66,8 +82,8 @@ GET /_search "query": { "match" : { "message" : { - "query" : "this is a test", - "operator" : "and" + "query" : "this is a testt", + "fuzziness": "AUTO" } } } diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 1c31e1cb3db47..b087f70ac83d5 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,10 +6,8 @@ This section summarizes the changes in each release. -* <> -* <> +* <> -- -include::release-notes/7.0.0-alpha2.asciidoc[] -include::release-notes/7.0.0-alpha1.asciidoc[] +include::release-notes/8.0.0-alpha1.asciidoc[] diff --git a/docs/reference/release-notes/7.0.0-alpha1.asciidoc b/docs/reference/release-notes/7.0.0-alpha1.asciidoc deleted file mode 100644 index 758e88d34024c..0000000000000 --- a/docs/reference/release-notes/7.0.0-alpha1.asciidoc +++ /dev/null @@ -1,32 +0,0 @@ -[[release-notes-7.0.0-alpha1]] -== {es} version 7.0.0-alpha1 - -The changes listed below have been released for the first time in Elasticsearch 7.0.0-alpha1. - -[[breaking-7.0.0-alpha1]] -[float] -=== Breaking changes - -Core:: -* Tribe node has been removed in favor of Cross-Cluster-Search - -Cross-Cluster-Search:: -* `http_addresses` has been removed from the <> API - because it is expensive to fetch and no longer needed by Kibana. - -Rest API:: -* The Clear Cache API only supports `POST` as HTTP method -* `CircuitBreakingException` was previously mapped to HTTP status code 503 and is now - mapped as HTTP status code 429. - -Aggregations:: -* The Percentiles and PercentileRanks aggregations now return `null` in the REST response, - instead of `NaN`. This makes it consistent with the rest of the aggregations. Note: - this only applies to the REST response, the java objects continue to return `NaN` (also - consistent with other aggregations) - -Suggesters:: -* Plugins that register suggesters can now define their own types of suggestions and must - explicitly indicate the type of suggestion that they produce. Existing plugins will - require changes to their plugin registration. See the `custom-suggester` example - plugin {pull}30284[#30284] diff --git a/docs/reference/release-notes/7.0.0-alpha2.asciidoc b/docs/reference/release-notes/7.0.0-alpha2.asciidoc deleted file mode 100644 index b07088d6cfed6..0000000000000 --- a/docs/reference/release-notes/7.0.0-alpha2.asciidoc +++ /dev/null @@ -1,585 +0,0 @@ -[[release-notes-7.0.0-alpha2]] -== {es} version 7.0.0-alpha2 - -[[breaking-7.0.0-alpha2]] -[float] -=== Breaking changes - -Authentication:: -* Enhance Invalidate Token API {pull}35388[#35388] (issues: {issue}34556[#34556], {issue}35115[#35115]) - -Circuit Breakers:: -* Lower fielddata circuit breaker's default limit {pull}27162[#27162] (issue: {issue}27130[#27130]) - -CCR:: -* Change get autofollow patterns API response format {pull}36203[#36203] (issue: {issue}36049[#36049]) - -Index APIs:: -* Always enforce cluster-wide shard limit {pull}34892[#34892] (issues: {issue}20705[#20705], {issue}34021[#34021]) - -Ranking:: -* Forbid negative scores in function_score query {pull}35709[#35709] (issue: {issue}33309[#33309]) - -Scripting:: -* Delete deprecated getValues from ScriptDocValues {pull}36183[#36183] (issue: {issue}22919[#22919]) - -Search:: -* Remove the deprecated _termvector endpoint. {pull}36131[#36131] (issues: {issue}36098[#36098], {issue}8484[#8484]) -* Remove deprecated Graph endpoints {pull}35956[#35956] -* Validate metadata on `_msearch` {pull}35938[#35938] (issue: {issue}35869[#35869]) -* Make hits.total an object in the search response {pull}35849[#35849] (issue: {issue}33028[#33028]) -* Remove the distinction between query and filter context in QueryBuilders {pull}35354[#35354] (issue: {issue}35293[#35293]) -* Throw a parsing exception when boost is set in span_or query (#28390) {pull}34112[#34112] (issue: {issue}28390[#28390]) - -ZenDiscovery:: -* Best-effort cluster formation if unconfigured {pull}36215[#36215] - -[[breaking-java-7.0.0-alpha2]] -[float] -=== Breaking Java changes - -ZenDiscovery:: -* Make node field in JoinRequest private {pull}36405[#36405] - -[[deprecation-7.0.0-alpha2]] -[float] -=== Deprecations - -Core:: -* Deprecate use of scientific notation in epoch time parsing {pull}36691[#36691] -* Add backcompat for joda time formats {pull}36531[#36531] - -Machine Learning:: -* Deprecate X-Pack centric ML endpoints {pull}36315[#36315] (issue: {issue}35958[#35958]) - -Mapping:: -* Deprecate types in index API {pull}36575[#36575] (issues: {issue}35190[#35190], {issue}35790[#35790]) -* Deprecate uses of _type as a field name in queries {pull}36503[#36503] (issue: {issue}35190[#35190]) -* Deprecate types in update_by_query and delete_by_query {pull}36365[#36365] (issue: {issue}35190[#35190]) -* For msearch templates, make sure to use the right name for deprecation logging. {pull}36344[#36344] -* Deprecate types in termvector and mtermvector requests. {pull}36182[#36182] -* Deprecate types in update requests. {pull}36181[#36181] -* Deprecate types in document delete requests. {pull}36087[#36087] -* Deprecate types in get, exists, and multi get. {pull}35930[#35930] -* Deprecate types in search and multi search templates. {pull}35669[#35669] -* Deprecate types in explain requests. {pull}35611[#35611] -* Deprecate types in validate query requests. {pull}35575[#35575] -* Deprecate types in count and msearch. {pull}35421[#35421] (issue: {issue}34041[#34041]) - -Migration:: -* Deprecate X-Pack centric Migration endpoints {pull}35976[#35976] (issue: {issue}35958[#35958]) - -Monitoring:: -* Deprecate /_xpack/monitoring/* in favor of /_monitoring/* {pull}36130[#36130] (issue: {issue}35958[#35958]) - -Rollup:: -* Re-deprecate xpack rollup endpoints {pull}36451[#36451] (issue: {issue}36044[#36044]) -* Deprecate X-Pack centric rollup endpoints {pull}35962[#35962] (issue: {issue}35958[#35958]) - -Scripting:: -* Adds deprecation logging to ScriptDocValues#getValues. {pull}34279[#34279] (issue: {issue}22919[#22919]) -* Conditionally use java time api in scripting {pull}31441[#31441] - -Search:: -* Remove X-Pack centric graph endpoints {pull}36010[#36010] (issue: {issue}35958[#35958]) - -Security:: -* Deprecate X-Pack centric license endpoints {pull}35959[#35959] (issue: {issue}35958[#35958]) -* Deprecate /_xpack/security/* in favor of /_security/* {pull}36293[#36293] (issue: {issue}35958[#35958]) - -SQL:: -* Deprecate X-Pack SQL translate endpoint {pull}36030[#36030] -* Deprecate X-Pack centric SQL endpoints {pull}35964[#35964] (issue: {issue}35958[#35958]) - -Watcher:: -* Deprecate X-Pack centric watcher endpoints {pull}36218[#36218] (issue: {issue}35958[#35958]) - - -[[feature-7.0.0-alpha2]] -[float] -=== New features - -Analysis:: -* Add support for inlined user dictionary in Nori {pull}36123[#36123] (issue: {issue}35842[#35842]) -* Add a prebuilt ICU Analyzer {pull}34958[#34958] (issue: {issue}34285[#34285]) - -Java High Level REST Client:: -* Add rollup search {pull}36334[#36334] (issue: {issue}29827[#29827]) - -Java Low Level REST Client:: -* Make warning behavior pluggable per request {pull}36345[#36345] -* Add PreferHasAttributeNodeSelector {pull}36005[#36005] - -Geo:: -* Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach {pull}36751[#36751] (issue: {issue}35320[#35320]) -* Integrate Lucene's LatLonShape (BKD Backed GeoShapes) as default `geo_shape` indexing approach {pull}35320[#35320] (issue: {issue}32039[#32039]) - -Machine Learning:: -* Add delayed datacheck to the datafeed job runner {pull}35387[#35387] (issue: {issue}35131[#35131]) - -Mapping:: -* Make typeless APIs usable with indices whose type name is different from `_doc` {pull}35790[#35790] (issue: {issue}35190[#35190]) - -SQL:: -* Introduce HISTOGRAM grouping function {pull}36510[#36510] (issue: {issue}36509[#36509]) -* DATABASE() and USER() system functions {pull}35946[#35946] (issue: {issue}35863[#35863]) -* Introduce INTERVAL support {pull}35521[#35521] (issue: {issue}29990[#29990]) - -Search:: -* Add intervals query {pull}36135[#36135] (issues: {issue}29636[#29636], {issue}32406[#32406]) -* Added soft limit to open scroll contexts #25244 {pull}36009[#36009] (issue: {issue}25244[#25244]) - -[[enhancement-7.0.0-alpha2]] -[float] -=== Enhancements - -Aggregations:: -* Added keyed response to pipeline percentile aggregations 22302 {pull}36392[#36392] (issue: {issue}22302[#22302]) -* Enforce max_buckets limit only in the final reduction phase {pull}36152[#36152] (issues: {issue}32125[#32125], {issue}35921[#35921]) -* Histogram aggs: add empty buckets only in the final reduce step {pull}35921[#35921] -* Handles exists query in composite aggs {pull}35758[#35758] -* Added parent validation for auto date histogram {pull}35670[#35670] - -Analysis:: -* Allow word_delimiter_graph_filter to not adjust internal offsets {pull}36699[#36699] (issues: {issue}33710[#33710], {issue}34741[#34741]) -* Ensure TokenFilters only produce single tokens when parsing synonyms {pull}34331[#34331] (issue: {issue}34298[#34298]) - -Audit:: -* Add "request.id" to file audit logs {pull}35536[#35536] - -Authentication:: -* Invalidate Token API enhancements - HLRC {pull}36362[#36362] (issue: {issue}35388[#35388]) -* Add DEBUG/TRACE logs for LDAP bind {pull}36028[#36028] -* Add Tests for findSamlRealm {pull}35905[#35905] -* Add realm information for Authenticate API {pull}35648[#35648] -* Formal support for "password_hash" in Put User {pull}35242[#35242] (issue: {issue}34729[#34729]) - -Authorization:: -* Improve exact index matching performance {pull}36017[#36017] -* `manage_token` privilege for `kibana_system` {pull}35751[#35751] -* Grant .tasks access to kibana_system role {pull}35573[#35573] - -Build:: -* Sounds like typo in exception message {pull}35458[#35458] -* Allow set section in setup section of REST tests {pull}34678[#34678] - -CCR:: -* Add time since last auto follow fetch to auto follow stats {pull}36542[#36542] (issues: {issue}33007[#33007], {issue}35895[#35895]) -* Clean followed leader index UUIDs in auto follow metadata {pull}36408[#36408] (issue: {issue}33007[#33007]) -* Change AutofollowCoordinator to use wait_for_metadata_version {pull}36264[#36264] (issues: {issue}33007[#33007], {issue}35895[#35895]) -* Refactor AutoFollowCoordinator to track leader indices per remote cluster {pull}36031[#36031] (issues: {issue}33007[#33007], {issue}35895[#35895]) - -Core:: -* Override the JVM DNS cache policy {pull}36570[#36570] -* Replace usages of AtomicBoolean based block of code by the RunOnce class {pull}35553[#35553] (issue: {issue}35489[#35489]) -* Added wait_for_metadata_version parameter to cluster state api. {pull}35535[#35535] -* Extract RunOnce into a dedicated class {pull}35489[#35489] -* Introduce elasticsearch-core jar {pull}28191[#28191] (issue: {issue}27933[#27933]) -* Rename core module to server {pull}28180[#28180] (issue: {issue}27933[#27933]) - -CRUD:: -* Rename seq# powered optimistic concurrency control parameters to ifSeqNo/ifPrimaryTerm {pull}36757[#36757] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Expose Sequence Number based Optimistic Concurrency Control in the rest layer {pull}36721[#36721] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Add doc's sequence number + primary term to GetResult and use it for updates {pull}36680[#36680] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Add seq no powered optimistic locking support to the index and delete transport actions {pull}36619[#36619] (issues: {issue}10708[#10708], {issue}36148[#36148]) - -Distributed:: -* [Close Index API] Mark shard copy as stale if needed during shard verification {pull}36755[#36755] -* [Close Index API] Refactor MetaDataIndexStateService {pull}36354[#36354] (issue: {issue}36249[#36249]) -* [Close Index API] Add TransportShardCloseAction for pre-closing verifications {pull}36249[#36249] -* TransportResyncReplicationAction should not honour blocks {pull}35795[#35795] (issues: {issue}35332[#35332], {issue}35597[#35597]) -* Expose all permits acquisition in IndexShard and TransportReplicationAction {pull}35540[#35540] (issue: {issue}33888[#33888]) -* [RCI] Check blocks while having index shard permit in TransportReplicationAction {pull}35332[#35332] (issue: {issue}33888[#33888]) - -Engine:: -* Add sequence numbers based optimistic concurrency control support to Engine {pull}36467[#36467] (issues: {issue}10708[#10708], {issue}36148[#36148]) -* Require soft-deletes when access changes snapshot {pull}36446[#36446] -* Use delCount of SegmentInfos to calculate numDocs {pull}36323[#36323] -* Always configure soft-deletes field of IndexWriterConfig {pull}36196[#36196] (issue: {issue}36141[#36141]) -* Enable soft-deletes by default on 7.0.0 or later {pull}36141[#36141] -* Always return false from `refreshNeeded` on ReadOnlyEngine {pull}35837[#35837] (issue: {issue}35785[#35785]) -* Add a `_freeze` / `_unfreeze` API {pull}35592[#35592] (issue: {issue}34352[#34352]) -* [RCI] Add IndexShardOperationPermits.asyncBlockOperations(ActionListener) {pull}34902[#34902] (issue: {issue}33888[#33888]) - -Features:: -* Simplify deprecation issue levels {pull}36326[#36326] - -Index APIs:: -* Add cluster-wide shard limit warnings {pull}34021[#34021] (issues: {issue}20705[#20705], {issue}32856[#32856]) - -Ingest:: -* Grok fix duplicate patterns JAVACLASS and JAVAFILE {pull}35886[#35886] -* Implement Drop Processor {pull}32278[#32278] (issue: {issue}23726[#23726]) - -Java High Level REST Client:: -* Add get users action {pull}36332[#36332] (issue: {issue}29827[#29827]) -* Add delete template API {pull}36320[#36320] (issue: {issue}27205[#27205]) -* Implement get-user-privileges API {pull}36292[#36292] -* Get Deprecation Info API {pull}36279[#36279] (issue: {issue}29827[#29827]) -* Add support for Follow Stats API {pull}36253[#36253] (issue: {issue}33824[#33824]) -* Add support for CCR Stats API {pull}36213[#36213] (issue: {issue}33824[#33824]) -* Put Role {pull}36209[#36209] (issue: {issue}29827[#29827]) -* Add index templates exist API {pull}36132[#36132] (issue: {issue}27205[#27205]) -* Add support for CCR Get Auto Follow Pattern apis {pull}36049[#36049] (issue: {issue}33824[#33824]) -* Add support for CCR Delete Auto Follow Pattern API {pull}35981[#35981] (issue: {issue}33824[#33824]) -* Remove fromXContent from IndexUpgradeInfoResponse {pull}35934[#35934] -* Add delete expired data API {pull}35906[#35906] (issue: {issue}29827[#29827]) -* Execute watch API {pull}35868[#35868] (issue: {issue}29827[#29827]) -* Add ability to put user with a password hash {pull}35844[#35844] (issue: {issue}35242[#35242]) -* Add ML find file structure API {pull}35833[#35833] (issue: {issue}29827[#29827]) -* Add support for get roles API {pull}35787[#35787] (issue: {issue}29827[#29827]) -* Added support for CCR Put Auto Follow Pattern API {pull}35780[#35780] (issue: {issue}33824[#33824]) -* XPack ML info action {pull}35777[#35777] (issue: {issue}29827[#29827]) -* ML Delete event from Calendar {pull}35760[#35760] (issue: {issue}29827[#29827]) -* Add ML revert model snapshot API {pull}35750[#35750] (issue: {issue}29827[#29827]) -* ML Get Calendar Events {pull}35747[#35747] (issue: {issue}29827[#29827]) -* Add high-level REST client API for `_freeze` and `_unfreeze` {pull}35723[#35723] (issue: {issue}34352[#34352]) -* Fix issue in equals impl for GlobalOperationPrivileges {pull}35721[#35721] -* ML Delete job from calendar {pull}35713[#35713] (issue: {issue}29827[#29827]) -* ML Add Event To Calendar API {pull}35704[#35704] (issue: {issue}29827[#29827]) -* Add ML update model snapshot API (#35537) {pull}35694[#35694] (issue: {issue}29827[#29827]) -* Add support for CCR Unfollow API {pull}35693[#35693] (issue: {issue}33824[#33824]) -* Clean up PutLicenseResponse {pull}35689[#35689] (issue: {issue}35547[#35547]) -* Clean up StartBasicResponse {pull}35688[#35688] (issue: {issue}35547[#35547]) -* Add support for put privileges API {pull}35679[#35679] -* ML Add Job to Calendar API {pull}35666[#35666] (issue: {issue}29827[#29827]) -* Add support for CCR Resume Follow API {pull}35638[#35638] (issue: {issue}33824[#33824]) -* Add support for get application privileges API {pull}35556[#35556] (issue: {issue}29827[#29827]) -* Clean up XPackInfoResponse class and related tests {pull}35547[#35547] -* Add parameters to stopRollupJob API {pull}35545[#35545] (issue: {issue}34811[#34811]) -* Add ML delete model snapshot API {pull}35537[#35537] (issue: {issue}29827[#29827]) -* Add get watch API {pull}35531[#35531] (issue: {issue}29827[#29827]) -* Add ML Update Filter API {pull}35522[#35522] (issue: {issue}29827[#29827]) -* Add ml get filters api {pull}35502[#35502] (issue: {issue}29827[#29827]) -* Add ML get model snapshots API {pull}35487[#35487] (issue: {issue}29827[#29827]) -* Add "_has_privileges" API to Security Client {pull}35479[#35479] (issue: {issue}29827[#29827]) -* Add Delete Privileges API to HLRC {pull}35454[#35454] (issue: {issue}29827[#29827]) -* Add support for CCR Put Follow API {pull}35409[#35409] -* Add ML delete filter action {pull}35382[#35382] (issue: {issue}29827[#29827]) -* Add delete user action {pull}35294[#35294] (issue: {issue}29827[#29827]) -* HLRC for _mtermvectors {pull}35266[#35266] (issues: {issue}27205[#27205], {issue}33447[#33447]) -* Reindex API with wait_for_completion false {pull}35202[#35202] (issue: {issue}27205[#27205]) -* Add watcher stats API {pull}35185[#35185] (issue: {issue}29827[#29827]) -* HLRC support for getTask {pull}35166[#35166] (issue: {issue}27205[#27205]) -* Add GetRollupIndexCaps API {pull}35102[#35102] (issue: {issue}29827[#29827]) -* HLRC: migration api - upgrade {pull}34898[#34898] (issue: {issue}29827[#29827]) -* Add stop rollup job support to HL REST Client {pull}34702[#34702] (issue: {issue}29827[#29827]) -* Bulk Api support for global parameters {pull}34528[#34528] (issue: {issue}26026[#26026]) -* Add delete rollup job support to HL REST Client {pull}34066[#34066] (issue: {issue}29827[#29827]) -* Add support for get license basic/trial status API {pull}33176[#33176] (issue: {issue}29827[#29827]) -* Add machine learning open job {pull}32860[#32860] (issue: {issue}29827[#29827]) -* Add ML HLRC wrapper and put_job API call {pull}32726[#32726] -* Add Get Snapshots High Level REST API {pull}31537[#31537] (issue: {issue}27205[#27205]) - -Java Low Level REST Client:: -* On retry timeout add root exception {pull}25576[#25576] - -Monitoring:: -* Make Exporters Async {pull}35765[#35765] (issue: {issue}35743[#35743]) - -Geo:: -* Adds a name of the field to geopoint parsing errors {pull}36529[#36529] (issue: {issue}15965[#15965]) -* Add support to ShapeBuilders for building Lucene geometry {pull}35707[#35707] (issue: {issue}35320[#35320]) -* Add ST_WktToSQL function {pull}35416[#35416] (issue: {issue}29872[#29872]) - -License:: -* Require acknowledgement to start_trial license {pull}30135[#30135] (issue: {issue}30134[#30134]) - -Machine Learning:: -* Create the ML annotations index {pull}36731[#36731] (issues: {issue}26034[#26034], {issue}33376[#33376]) -* Split in batches and migrate all jobs and datafeeds {pull}36716[#36716] (issue: {issue}32905[#32905]) -* Add cluster setting to enable/disable config migration {pull}36700[#36700] (issue: {issue}32905[#32905]) -* Add audits when deprecation warnings occur with datafeed start {pull}36233[#36233] -* Add lazy parsing for DatafeedConfig:Aggs,Query {pull}36117[#36117] -* Add support for lazy nodes (#29991) {pull}34538[#34538] (issue: {issue}29991[#29991]) - -Network:: -* Unify transport settings naming {pull}36623[#36623] -* Add sni name to SSLEngine in netty transport {pull}33144[#33144] (issue: {issue}32517[#32517]) -* Add cors support to NioHttpServerTransport {pull}30827[#30827] (issue: {issue}28898[#28898]) -* Reintroduce mandatory http pipelining support {pull}30820[#30820] -* Make http pipelining support mandatory {pull}30695[#30695] (issues: {issue}28898[#28898], {issue}29500[#29500]) -* Add nio http server transport {pull}29587[#29587] (issue: {issue}28898[#28898]) -* Selectors operate on channel contexts {pull}28468[#28468] (issue: {issue}27260[#27260]) -* Unify nio read / write channel contexts {pull}28160[#28160] (issue: {issue}27260[#27260]) -* Create nio-transport plugin for NioTransport {pull}27949[#27949] (issue: {issue}27260[#27260]) -* Add elasticsearch-nio jar for base nio classes {pull}27801[#27801] (issue: {issue}27802[#27802]) -* Add NioGroup for use in different transports {pull}27737[#27737] (issue: {issue}27260[#27260]) -* Add read timeouts to http module {pull}27713[#27713] -* Implement byte array reusage in `NioTransport` {pull}27696[#27696] (issue: {issue}27563[#27563]) -* Introduce resizable inbound byte buffer {pull}27551[#27551] (issue: {issue}27563[#27563]) -* Decouple nio constructs from the tcp transport {pull}27484[#27484] (issue: {issue}27260[#27260]) -* Remove manual tracking of registered channels {pull}27445[#27445] (issue: {issue}27260[#27260]) -* Remove tcp profile from low level nio channel {pull}27441[#27441] (issue: {issue}27260[#27260]) -* Decouple `ChannelFactory` from Tcp classes {pull}27286[#27286] (issue: {issue}27260[#27260]) - -Packaging:: -* Introduce Docker images build {pull}36246[#36246] -* Move creation of temporary directory to Java {pull}36002[#36002] (issue: {issue}31003[#31003]) - -Plugins:: -* Plugin install: don't print download progress in batch mode {pull}36361[#36361] - -Ranking:: -* Vector field {pull}33022[#33022] (issue: {issue}31615[#31615]) - -Recovery:: -* Exposed engine must include all operations below global checkpoint during rollback {pull}36159[#36159] (issue: {issue}32867[#32867]) - -Rollup:: -* Add non-X-Pack centric rollup endpoints {pull}36383[#36383] (issues: {issue}35958[#35958], {issue}35962[#35962]) -* Add more diagnostic stats to job {pull}35471[#35471] -* Add `wait_for_completion` option to StopRollupJob API {pull}34811[#34811] (issue: {issue}34574[#34574]) - -Scripting:: -* Update joda compat methods to use compat class {pull}36654[#36654] -* [Painless] Add boxed type to boxed type casts for method/return {pull}36571[#36571] -* [Painless] Add def to boxed type casts {pull}36506[#36506] - -Settings:: -* Add user-defined cluster metadata {pull}33325[#33325] (issue: {issue}33220[#33220]) - -Search:: -* Add copy constructor to SearchRequest {pull}36641[#36641] (issue: {issue}32125[#32125]) -* Add raw sort values to SearchSortValues transport serialization {pull}36617[#36617] (issue: {issue}32125[#32125]) -* Add sort and collapse info to SearchHits transport serialization {pull}36555[#36555] (issue: {issue}32125[#32125]) -* Add default methods to DocValueFormat {pull}36480[#36480] -* Respect indices options on _msearch {pull}35887[#35887] -* Allow efficient can_match phases on frozen indices {pull}35431[#35431] (issues: {issue}34352[#34352], {issue}34357[#34357]) -* Add a new query type - ScriptScoreQuery {pull}34533[#34533] (issues: {issue}23850[#23850], {issue}27588[#27588], {issue}30303[#30303]) - -Security:: -* Make credentials mandatory when launching xpack/migrate {pull}36197[#36197] (issues: {issue}29847[#29847], {issue}33972[#33972]) - -Snapshot/Restore:: -* Allow Parallel Restore Operations {pull}36397[#36397] -* Repo Creation out of ClusterStateTask {pull}36157[#36157] (issue: {issue}9488[#9488]) -* Add read-only repository verification {pull}35731[#35731] (issue: {issue}35703[#35703]) - -SQL:: -* Extend the ODBC metric by differentiating between 32 and 64bit platforms {pull}36753[#36753] (issue: {issue}36740[#36740]) -* Fix wrong appliance of StackOverflow limit for IN {pull}36724[#36724] (issue: {issue}36592[#36592]) -* Introduce NOW/CURRENT_TIMESTAMP function {pull}36562[#36562] (issue: {issue}36534[#36534]) -* Move requests' parameters to requests JSON body {pull}36149[#36149] (issue: {issue}35992[#35992]) -* Make INTERVAL millis optional {pull}36043[#36043] (issue: {issue}36032[#36032]) -* Implement data type verification for conditionals {pull}35916[#35916] (issue: {issue}35907[#35907]) -* Implement GREATEST and LEAST functions {pull}35879[#35879] (issue: {issue}35878[#35878]) -* Implement null safe equality operator `<=>` {pull}35873[#35873] (issue: {issue}35871[#35871]) -* SYS COLUMNS returns ODBC specific schema {pull}35870[#35870] (issue: {issue}35376[#35376]) -* Polish grammar for intervals {pull}35853[#35853] -* Add filtering to SYS TYPES {pull}35852[#35852] (issue: {issue}35342[#35342]) -* Implement NULLIF(expr1, expr2) function {pull}35826[#35826] (issue: {issue}35818[#35818]) -* Lock down JDBC driver {pull}35798[#35798] (issue: {issue}35437[#35437]) -* Implement NVL(expr1, expr2) {pull}35794[#35794] (issue: {issue}35782[#35782]) -* Implement ISNULL(expr1, expr2) {pull}35793[#35793] (issue: {issue}35781[#35781]) -* Implement IFNULL variant of COALESCE {pull}35762[#35762] (issue: {issue}35749[#35749]) -* XPack FeatureSet functionality {pull}35725[#35725] (issue: {issue}34821[#34821]) -* Perform lazy evaluation of mismatched mappings {pull}35676[#35676] (issues: {issue}35659[#35659], {issue}35675[#35675]) -* Improve validation of unsupported fields {pull}35675[#35675] (issue: {issue}35673[#35673]) -* Move internals from Joda to java.time {pull}35649[#35649] (issue: {issue}35633[#35633]) - -Stats:: -* Handle OS pretty name on old OS without OS release {pull}35453[#35453] (issue: {issue}35440[#35440]) - -Task Management:: -* Periodically try to reassign unassigned persistent tasks {pull}36069[#36069] (issue: {issue}35792[#35792]) -* Only require task permissions {pull}35667[#35667] (issue: {issue}35573[#35573]) -* Retry if task can't be written {pull}35054[#35054] (issue: {issue}33764[#33764]) - -ZenDiscovery:: -* Add discovery types to cluster stats {pull}36442[#36442] -* Introduce `zen2` discovery type {pull}36298[#36298] -* Zen2: Persist cluster states the old way on non-master-eligible nodes {pull}36247[#36247] (issue: {issue}3[#3]) -* [Zen2] Storage layer WriteStateException propagation {pull}36052[#36052] -* [Zen2] Implement Tombstone REST APIs {pull}36007[#36007] -* [Zen2] Update default for USE_ZEN2 to true {pull}35998[#35998] -* [Zen2] Add warning if cluster fails to form fast enough {pull}35993[#35993] -* [Zen2] Allow Setting a List of Bootstrap Nodes to Wait for {pull}35847[#35847] -* [Zen2] VotingTombstone class {pull}35832[#35832] -* [Zen2] PersistedState interface implementation {pull}35819[#35819] -* [Zen2] Support rolling upgrades from Zen1 {pull}35737[#35737] -* [Zen2] Add lag detector {pull}35685[#35685] -* [Zen2] Move ClusterState fields to be persisted to ClusterState.MetaData {pull}35625[#35625] -* [Zen2] Introduce ClusterBootstrapService {pull}35488[#35488] -* [Zen2] Introduce vote withdrawal {pull}35446[#35446] -* Zen2: Add basic Zen1 transport-level BWC {pull}35443[#35443] - -[[bug-7.0.0-alpha2]] -[float] -=== Bug fixes - -Aggregations:: -* fix MultiValuesSourceFieldConfig toXContent {pull}36525[#36525] (issue: {issue}36474[#36474]) -* Cache the score of the parent document in the nested agg {pull}36019[#36019] (issues: {issue}34555[#34555], {issue}35985[#35985]) -* Correct implemented interface of ParsedReverseNested {pull}35455[#35455] (issue: {issue}35449[#35449]) -* Handle IndexOrDocValuesQuery in composite aggregation {pull}35392[#35392] - -Audit:: -* Fix origin.type for connection_* events {pull}36410[#36410] -* Fix IndexAuditTrail rolling restart on rollover edge {pull}35988[#35988] (issue: {issue}33867[#33867]) - -Authentication:: -* Fix kerberos setting registration {pull}35986[#35986] (issues: {issue}30241[#30241], {issue}35942[#35942]) -* Add support for Kerberos V5 Oid {pull}35764[#35764] (issue: {issue}34763[#34763]) - -Build:: -* Use explicit deps on test tasks for check {pull}36325[#36325] -* Fix jdbc jar pom to not include deps {pull}36036[#36036] (issue: {issue}32014[#32014]) -* Fix official plugins list {pull}35661[#35661] (issue: {issue}35623[#35623]) - -CCR:: -* Fix follow stats API's follower index filtering feature {pull}36647[#36647] -* AutoFollowCoordinator should tolerate that auto follow patterns may be removed {pull}35945[#35945] (issue: {issue}35937[#35937]) -* Only auto follow indices when all primary shards have started {pull}35814[#35814] (issue: {issue}35480[#35480]) -* Avoid NPE in follower stats when no tasks metadata {pull}35802[#35802] -* Fix the names of CCR stats endpoints in usage API {pull}35438[#35438] - -Circuit Breakers:: -* Modify `BigArrays` to take name of circuit breaker {pull}36461[#36461] (issue: {issue}31435[#31435]) - -Core:: -* Fix CompositeBytesReference#slice to not throw AIOOBE with legal offsets. {pull}35955[#35955] (issue: {issue}35950[#35950]) -* Suppress CachedTimeThread in hot threads output {pull}35558[#35558] (issue: {issue}23175[#23175]) -* Upgrade to Joda 2.10.1 {pull}35410[#35410] (issue: {issue}33749[#33749]) - -Distributed:: -* Combine the execution of an exclusive replica operation with primary term update {pull}36116[#36116] (issue: {issue}35850[#35850]) -* ActiveShardCount should not fail when closing the index {pull}35936[#35936] - -Engine:: -* Set Lucene version upon index creation. {pull}36038[#36038] (issue: {issue}33826[#33826]) -* Wrap can_match reader with ElasticsearchDirectoryReader {pull}35857[#35857] -* Copy checkpoint atomically when rolling generation {pull}35407[#35407] - -Geo:: -* More robust handling of ignore_malformed in geoshape parsing {pull}35603[#35603] (issues: {issue}34047[#34047], {issue}34498[#34498]) -* Better handling of malformed geo_points {pull}35554[#35554] (issue: {issue}35419[#35419]) -* Enables coerce support in WKT polygon parser {pull}35414[#35414] (issue: {issue}35059[#35059]) - -Index APIs:: -* Fix duplicate phrase in shrink/split error message {pull}36734[#36734] (issue: {issue}36729[#36729]) -* Raise a 404 exception when document source is not found (#33384) {pull}34083[#34083] (issue: {issue}33384[#33384]) - -Ingest:: -* Fix on_failure with Drop processor {pull}36686[#36686] (issue: {issue}36151[#36151]) -* Support default pipelines + bulk upserts {pull}36618[#36618] (issue: {issue}36219[#36219]) -* Support default pipeline through an alias {pull}36231[#36231] (issue: {issue}35817[#35817]) - -License:: -* Do not serialize basic license exp in x-pack info {pull}30848[#30848] -* Update versions for start_trial after backport {pull}30218[#30218] (issue: {issue}30135[#30135]) - -Machine Learning:: -* Interrupt Grok in file structure finder timeout {pull}36588[#36588] -* Prevent stack overflow while copying ML jobs and datafeeds {pull}36370[#36370] (issue: {issue}36360[#36360]) -* Adjust file structure finder parser config {pull}35935[#35935] -* Fix find_file_structure NPE with should_trim_fields {pull}35465[#35465] (issue: {issue}35462[#35462]) -* Prevent notifications being created on deletion of a non existent job {pull}35337[#35337] (issues: {issue}34058[#34058], {issue}35336[#35336]) -* Clear Job#finished_time when it is opened (#32605) {pull}32755[#32755] -* Fix thread leak when waiting for job flush (#32196) {pull}32541[#32541] (issue: {issue}32196[#32196]) -* Fix CPoissonMeanConjugate sampling error. {ml-pull}335[#335] - -Network:: -* Do not resolve addresses in remote connection info {pull}36671[#36671] (issue: {issue}35658[#35658]) -* Always compress based on the settings {pull}36522[#36522] (issue: {issue}36399[#36399]) -* http.publish_host Should Contain CNAME {pull}32806[#32806] (issue: {issue}22029[#22029]) -* Adjust SSLDriver behavior for JDK11 changes {pull}32145[#32145] (issues: {issue}32122[#32122], {issue}32144[#32144]) -* Add TRACE, CONNECT, and PATCH http methods {pull}31035[#31035] (issue: {issue}31017[#31017]) -* Transport client: Don't validate node in handshake {pull}30737[#30737] (issue: {issue}30141[#30141]) -* Fix issue with finishing handshake in ssl driver {pull}30580[#30580] -* Remove potential nio selector leak {pull}27825[#27825] -* Fix issue where the incorrect buffers are written {pull}27695[#27695] (issue: {issue}27551[#27551]) -* Do not set SO_LINGER on server channels {pull}26997[#26997] -* Do not set SO_LINGER to 0 when not shutting down {pull}26871[#26871] (issue: {issue}26764[#26764]) -* Release pipelined http responses on close {pull}26226[#26226] - -Packaging:: -* Fix error message when package install fails due to missing Java {pull}36077[#36077] (issue: {issue}31845[#31845]) -* Add missing entries to conffiles {pull}35810[#35810] (issue: {issue}35691[#35691]) - -Plugins:: -* Ensure that azure stream has socket privileges {pull}28751[#28751] (issue: {issue}28662[#28662]) - -Recovery:: -* Register ResyncTask.Status as a NamedWriteable {pull}36610[#36610] - -Rollup:: -* Fix rollup search statistics {pull}36674[#36674] - -Scripting:: -* Properly support no-offset date formatting {pull}36316[#36316] (issue: {issue}36306[#36306]) -* [Painless] Generate Bridge Methods {pull}36097[#36097] -* Fix serialization bug in painless execute api request {pull}36075[#36075] (issue: {issue}36050[#36050]) -* Actually add joda time back to whitelist {pull}35965[#35965] (issue: {issue}35915[#35915]) -* Add back joda to whitelist {pull}35915[#35915] (issue: {issue}35913[#35913]) - -Settings:: -* Correctly Identify Noop Updates {pull}36560[#36560] (issue: {issue}36496[#36496]) - -SQL:: -* Fix translation of LIKE/RLIKE keywords {pull}36672[#36672] (issues: {issue}36039[#36039], {issue}36584[#36584]) -* Scripting support for casting functions CAST and CONVERT {pull}36640[#36640] (issue: {issue}36061[#36061]) -* Fix translation to painless for conditionals {pull}36636[#36636] (issue: {issue}36631[#36631]) -* Concat should be always not nullable {pull}36601[#36601] (issue: {issue}36169[#36169]) -* Fix MOD() for long and integer arguments {pull}36599[#36599] (issue: {issue}36364[#36364]) -* Fix issue with complex HAVING and GROUP BY ordinal {pull}36594[#36594] (issue: {issue}36059[#36059]) -* Be lenient for tests involving comparison to H2 but strict for csv spec tests {pull}36498[#36498] (issue: {issue}36483[#36483]) -* Non ISO 8601 versions of DAY_OF_WEEK and WEEK_OF_YEAR functions {pull}36358[#36358] (issue: {issue}36263[#36263]) -* Do not ignore all fields whose names start with underscore {pull}36214[#36214] (issue: {issue}36206[#36206]) -* Fix issue with wrong data type for scripted Grouping keys {pull}35969[#35969] (issue: {issue}35662[#35662]) -* Fix translation of math functions to painless {pull}35910[#35910] (issue: {issue}35654[#35654]) -* Fix jdbc jar to include deps {pull}35602[#35602] -* Fix query translation for scripted queries {pull}35408[#35408] (issue: {issue}35232[#35232]) -* Clear the cursor if nested inner hits are enough to fulfill the query required limits {pull}35398[#35398] (issue: {issue}35176[#35176]) -* Introduce IsNull node to simplify expressions {pull}35206[#35206] (issues: {issue}34876[#34876], {issue}35171[#35171]) -* The SSL default configuration shouldn't override the https protocol if used {pull}34635[#34635] (issue: {issue}33817[#33817]) -* Minor fix for javadoc {pull}32573[#32573] (issue: {issue}32553[#32553]) - -Search:: -* Inner hits fail to propagate doc-value format. {pull}36310[#36310] -* Fix custom AUTO issue with Fuzziness#toXContent {pull}35807[#35807] (issue: {issue}33462[#33462]) -* Fix analyzed prefix query in query_string {pull}35756[#35756] (issue: {issue}31702[#31702]) -* Fix problem with MatchNoDocsQuery in disjunction queries {pull}35726[#35726] (issue: {issue}34708[#34708]) -* Fix phrase_slop in query_string query {pull}35533[#35533] (issue: {issue}35125[#35125]) -* Add a More Like This query routing requirement check (#29678) {pull}33974[#33974] - -Security:: -* Remove license state listeners on closeables {pull}36308[#36308] (issues: {issue}33328[#33328], {issue}35627[#35627], {issue}35628[#35628]) - -Snapshot/Restore:: -* Upgrade GCS Dependencies to 1.55.0 {pull}36634[#36634] (issues: {issue}35229[#35229], {issue}35459[#35459]) -* Improve Resilience SnapshotShardService {pull}36113[#36113] (issue: {issue}32265[#32265]) -* Keep SnapshotsInProgress State in Sync with Routing Table {pull}35710[#35710] -* Ensure that gcs client creation is privileged {pull}25938[#25938] (issue: {issue}25932[#25932]) -* Make calls to CloudBlobContainer#exists privileged {pull}25937[#25937] (issue: {issue}25931[#25931]) - -Watcher:: -* Watcher accounts constructed lazily {pull}36656[#36656] -* Only trigger a watch if new or schedule/changed {pull}35908[#35908] -* Fix Watcher NotificationService's secure settings {pull}35610[#35610] (issue: {issue}35378[#35378]) -* Fix integration tests to ensure correct start/stop of Watcher {pull}35271[#35271] (issues: {issue}29877[#29877], {issue}30705[#30705], {issue}33291[#33291], {issue}34448[#34448], {issue}34462[#34462]) - -ZenDiscovery:: -* [Zen2] Respect the no_master_block setting {pull}36478[#36478] -* Cancel GetDiscoveredNodesAction when bootstrapped {pull}36423[#36423] (issues: {issue}36380[#36380], {issue}36381[#36381]) -* [Zen2] Only elect master-eligible nodes {pull}35996[#35996] -* [Zen2] Remove duplicate discovered peers {pull}35505[#35505] - - -[[regression-7.0.0-alpha2]] -[float] -=== Regressions - -Scripting:: -* Use Number as a return value for BucketAggregationScript {pull}35653[#35653] (issue: {issue}35351[#35351]) - - -[[upgrade-7.0.0-alpha2]] -[float] -=== Upgrades - -Ingest:: -* Update geolite2 database in ingest geoip plugin {pull}33840[#33840] - -Network:: -* Upgrade Netty 4.3.32.Final {pull}36102[#36102] (issue: {issue}35360[#35360]) diff --git a/docs/reference/release-notes/8.0.0-alpha1.asciidoc b/docs/reference/release-notes/8.0.0-alpha1.asciidoc new file mode 100644 index 0000000000000..4693221f629cf --- /dev/null +++ b/docs/reference/release-notes/8.0.0-alpha1.asciidoc @@ -0,0 +1,8 @@ +[[release-notes-8.0.0-alpha1]] +== {es} version 8.0.0-alpha1 + +The changes listed below have been released for the first time in {es} +8.0.0-alpha1. + +coming[8.0.0] + diff --git a/docs/reference/release-notes/highlights-7.0.0.asciidoc b/docs/reference/release-notes/highlights-7.0.0.asciidoc deleted file mode 100644 index d01d543c8257e..0000000000000 --- a/docs/reference/release-notes/highlights-7.0.0.asciidoc +++ /dev/null @@ -1,9 +0,0 @@ -[[release-highlights-7.0.0]] -== 7.0.0 release highlights -++++ -7.0.0 -++++ - -coming[7.0.0] - -See also <> and <>. diff --git a/docs/reference/release-notes/highlights-8.0.0.asciidoc b/docs/reference/release-notes/highlights-8.0.0.asciidoc new file mode 100644 index 0000000000000..419059edd0650 --- /dev/null +++ b/docs/reference/release-notes/highlights-8.0.0.asciidoc @@ -0,0 +1,9 @@ +[[release-highlights-8.0.0]] +== 8.0.0 release highlights +++++ +8.0.0 +++++ + +coming[8.0.0] + +See also <> and <>. diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 5b20b67061d03..6ff7c75a235db 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -9,8 +9,8 @@ This section summarizes the most important changes in each release. For the full list, see <> and <>. -* <> +* <> -- -include::highlights-7.0.0.asciidoc[] \ No newline at end of file +include::highlights-8.0.0.asciidoc[] \ No newline at end of file diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java index 05aa75944d2f9..7301ad8a9e919 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java @@ -87,10 +87,16 @@ Function getFunction(String format, ZoneId zoneId, Locale format = format.substring(1); } + boolean isUtc = ZoneOffset.UTC.equals(zoneId); + int year = LocalDate.now(ZoneOffset.UTC).getYear(); - DateFormatter formatter = DateFormatter.forPattern(format) - .withLocale(locale) - .withZone(zoneId); + DateFormatter dateFormatter = DateFormatter.forPattern(format) + .withLocale(locale); + // if UTC zone is set here, the the time zone specified in the format will be ignored, leading to wrong dates + if (isUtc == false) { + dateFormatter = dateFormatter.withZone(zoneId); + } + final DateFormatter formatter = dateFormatter; return text -> { TemporalAccessor accessor = formatter.parse(text); // if there is no year, we fall back to the current one and @@ -106,7 +112,11 @@ Function getFunction(String format, ZoneId zoneId, Locale accessor = newTime.withZoneSameLocal(zoneId); } - return DateFormatters.from(accessor); + if (isUtc) { + return DateFormatters.from(accessor).withZoneSameInstant(ZoneOffset.UTC); + } else { + return DateFormatters.from(accessor); + } }; } }; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index e7ad1356977e0..390279bdbb5d7 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -43,7 +43,7 @@ public final class DateProcessor extends AbstractProcessor { public static final String TYPE = "date"; static final String DEFAULT_TARGET_FIELD = "@timestamp"; - public static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); + private static final DateFormatter FORMATTER = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); private final TemplateScript.Factory timezone; private final TemplateScript.Factory locale; diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java index 136c9f7f69a0a..e44e62be8629a 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.test.ESTestCase; @@ -43,6 +44,14 @@ public void testParseJava() { equalTo("11 24 01:29:01")); } + public void testParseJavaWithTimeZone() { + Function javaFunction = DateFormat.Java.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZZ", + ZoneOffset.UTC, Locale.ROOT); + ZonedDateTime datetime = javaFunction.apply("2018-02-05T13:44:56.657+0100"); + String expectedDateTime = DateFormatter.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").withZone(ZoneOffset.UTC).format(datetime); + assertThat(expectedDateTime, is("2018-02-05T12:44:56.657Z")); + } + public void testParseJavaDefaultYear() { String format = randomFrom("8dd/MM", "dd/MM"); ZoneId timezone = DateUtils.of("Europe/Amsterdam"); @@ -70,6 +79,10 @@ public void testParseUnixWithMsPrecision() { public void testParseISO8601() { assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toInstant().toEpochMilli(), equalTo(978336000000L)); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), + equalTo("2001-01-01T08:00Z")); + assertThat(DateFormat.Iso8601.getFunction(null, ZoneOffset.UTC, null).apply("2001-01-01T00:00:00-0800").toString(), + equalTo("2001-01-01T08:00Z")); } public void testParseISO8601Failure() { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java index 2e32e3fd0ebd2..7582056e0b6b6 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java @@ -29,6 +29,7 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -97,6 +98,18 @@ public void testJavaPatternMultipleFormats() { } } + public void testJavaPatternNoTimezone() { + DateProcessor dateProcessor = new DateProcessor(randomAlphaOfLength(10), + null, null, + "date_as_string", Arrays.asList("yyyy dd MM HH:mm:ss XXX"), "date_as_date"); + + Map document = new HashMap<>(); + document.put("date_as_string", "2010 12 06 00:00:00 -02:00"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + dateProcessor.execute(ingestDocument); + assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo("2010-06-12T02:00:00.000Z")); + } + public void testInvalidJavaPattern() { try { DateProcessor processor = new DateProcessor(randomAlphaOfLength(10), diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml index b2e83c640388a..99e90064da013 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yml @@ -39,3 +39,94 @@ teardown: id: 1 - match: { _source.date_source_field: "12/06/2010" } - match: { _source.date_target_field: "2010-06-12T00:00:00.000+02:00" } + +--- +"Test date processor with no timezone configured": + + - do: + ingest.put_pipeline: + id: "my_pipeline" + # sample formats from beats, featuring mongodb, icinga, apache + body: > + { + "description": "_description", + "processors": [ + { + "date" : { + "field" : "date_source_1", + "target_field" : "date_target_1", + "formats" : ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ" ] + } + }, + { + "date" : { + "field" : "date_source_2", + "target_field" : "date_target_2", + "formats" : ["yyyy-MM-dd HH:mm:ss Z" ] + } + }, + { + "date" : { + "field" : "date_source_3", + "target_field" : "date_target_3", + "formats" : [ "dd/MMM/yyyy:H:m:s Z" ] + } + }, + { + "date" : { + "field" : "date_source_4", + "target_field" : "date_target_4", + "formats" : [ "UNIX" ] + } + }, + { + "date" : { + "field" : "date_source_5", + "target_field" : "date_target_5", + "formats" : [ "UNIX_MS" ] + } + }, + { + "date" : { + "field" : "date_source_6", + "target_field" : "date_target_6", + "formats" : [ "TAI64N" ] + } + }, + { + "date" : { + "field" : "date_source_7", + "target_field" : "date_target_7", + "formats" : [ "ISO8601" ] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "my_pipeline" + body: { date_source_1: "2018-02-05T13:44:56.657+0100", date_source_2: "2017-04-04 13:43:09 +0200", date_source_3: "10/Aug/2018:09:45:56 +0200", date_source_4: "1", date_source_5: "1", date_source_6: "4000000050d506482dbdf024", date_source_7: "2018-02-05T13:44:56.657+0100" } + + - do: + get: + index: test + id: 1 + - match: { _source.date_source_1: "2018-02-05T13:44:56.657+0100" } + - match: { _source.date_target_1: "2018-02-05T12:44:56.657Z" } + - match: { _source.date_source_2: "2017-04-04 13:43:09 +0200" } + - match: { _source.date_target_2: "2017-04-04T11:43:09.000Z" } + - match: { _source.date_source_3: "10/Aug/2018:09:45:56 +0200" } + - match: { _source.date_target_3: "2018-08-10T07:45:56.000Z" } + - match: { _source.date_source_4: "1" } + - match: { _source.date_target_4: "1970-01-01T00:00:01.000Z" } + - match: { _source.date_source_5: "1" } + - match: { _source.date_target_5: "1970-01-01T00:00:00.001Z" } + - match: { _source.date_source_6: "4000000050d506482dbdf024" } + - match: { _source.date_target_6: "2012-12-22T01:00:46.767Z" } + - match: { _source.date_source_7: "2018-02-05T13:44:56.657+0100" } + - match: { _source.date_target_7: "2018-02-05T12:44:56.657Z" } + diff --git a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java index d10690379eddd..addb16d58d031 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java @@ -191,6 +191,13 @@ private static int indexOfKeptCommits(List commits, long return 0; } + /** + * Checks whether the deletion policy is holding on to snapshotted commits + */ + synchronized boolean hasSnapshottedCommits() { + return snapshottedCommits.isEmpty() == false; + } + /** * Checks if the deletion policy can release some index commits with the latest global checkpoint. */ diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 3ec35bb07661c..10de784fb3554 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -476,6 +476,11 @@ Translog getTranslog() { return translog; } + // Package private for testing purposes only + boolean hasSnapshottedCommits() { + return combinedDeletionPolicy.hasSnapshottedCommits(); + } + @Override public boolean isTranslogSyncNeeded() { return getTranslog().syncNeeded(); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java b/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java new file mode 100644 index 0000000000000..87a6d18671a6f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/indices/recovery/MultiFileWriter.java @@ -0,0 +1,213 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.indices.recovery; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.StoreFileMetaData; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.concurrent.ConcurrentMap; + +public class MultiFileWriter implements Releasable { + + public MultiFileWriter(Store store, RecoveryState.Index indexState, String tempFilePrefix, Logger logger, Runnable ensureOpen) { + this.store = store; + this.indexState = indexState; + this.tempFilePrefix = tempFilePrefix; + this.logger = logger; + this.ensureOpen = ensureOpen; + } + + private final Runnable ensureOpen; + private final Logger logger; + private final Store store; + private final RecoveryState.Index indexState; + private final String tempFilePrefix; + + private final ConcurrentMap openIndexOutputs = ConcurrentCollections.newConcurrentMap(); + private final ConcurrentMap fileChunkWriters = ConcurrentCollections.newConcurrentMap(); + + + final Map tempFileNames = ConcurrentCollections.newConcurrentMap(); + + public void writeFileChunk(StoreFileMetaData fileMetaData, long position, BytesReference content, boolean lastChunk) + throws IOException { + final FileChunkWriter writer = fileChunkWriters.computeIfAbsent(fileMetaData.name(), name -> new FileChunkWriter()); + writer.writeChunk(new FileChunk(fileMetaData, content, position, lastChunk)); + } + + /** Get a temporary name for the provided file name. */ + String getTempNameForFile(String origFile) { + return tempFilePrefix + origFile; + } + + public IndexOutput getOpenIndexOutput(String key) { + ensureOpen.run(); + return openIndexOutputs.get(key); + } + + /** remove and {@link IndexOutput} for a given file. It is the caller's responsibility to close it */ + public IndexOutput removeOpenIndexOutputs(String name) { + ensureOpen.run(); + return openIndexOutputs.remove(name); + } + + /** + * Creates an {@link IndexOutput} for the given file name. Note that the + * IndexOutput actually point at a temporary file. + *

+ * Note: You can use {@link #getOpenIndexOutput(String)} with the same filename to retrieve the same IndexOutput + * at a later stage + */ + public IndexOutput openAndPutIndexOutput(String fileName, StoreFileMetaData metaData, Store store) throws IOException { + ensureOpen.run(); + String tempFileName = getTempNameForFile(fileName); + if (tempFileNames.containsKey(tempFileName)) { + throw new IllegalStateException("output for file [" + fileName + "] has already been created"); + } + // add first, before it's created + tempFileNames.put(tempFileName, fileName); + IndexOutput indexOutput = store.createVerifyingOutput(tempFileName, metaData, IOContext.DEFAULT); + openIndexOutputs.put(fileName, indexOutput); + return indexOutput; + } + + private void innerWriteFileChunk(StoreFileMetaData fileMetaData, long position, + BytesReference content, boolean lastChunk) throws IOException { + final String name = fileMetaData.name(); + IndexOutput indexOutput; + if (position == 0) { + indexOutput = openAndPutIndexOutput(name, fileMetaData, store); + } else { + indexOutput = getOpenIndexOutput(name); + } + assert indexOutput.getFilePointer() == position : "file-pointer " + indexOutput.getFilePointer() + " != " + position; + BytesRefIterator iterator = content.iterator(); + BytesRef scratch; + while((scratch = iterator.next()) != null) { // we iterate over all pages - this is a 0-copy for all core impls + indexOutput.writeBytes(scratch.bytes, scratch.offset, scratch.length); + } + indexState.addRecoveredBytesToFile(name, content.length()); + if (indexOutput.getFilePointer() >= fileMetaData.length() || lastChunk) { + try { + Store.verify(indexOutput); + } finally { + // we are done + indexOutput.close(); + } + final String temporaryFileName = getTempNameForFile(name); + assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName) : + "expected: [" + temporaryFileName + "] in " + Arrays.toString(store.directory().listAll()); + store.directory().sync(Collections.singleton(temporaryFileName)); + IndexOutput remove = removeOpenIndexOutputs(name); + assert remove == null || remove == indexOutput; // remove maybe null if we got finished + } + } + + @Override + public void close() { + fileChunkWriters.clear(); + // clean open index outputs + Iterator> iterator = openIndexOutputs.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + logger.trace("closing IndexOutput file [{}]", entry.getValue()); + try { + entry.getValue().close(); + } catch (Exception e) { + logger.debug(() -> new ParameterizedMessage("error while closing recovery output [{}]", entry.getValue()), e); + } + iterator.remove(); + } + if (Strings.hasText(tempFilePrefix)) { + // trash temporary files + for (String file : tempFileNames.keySet()) { + logger.trace("cleaning temporary file [{}]", file); + store.deleteQuiet(file); + } + } + } + + /** renames all temporary files to their true name, potentially overriding existing files */ + public void renameAllTempFiles() throws IOException { + ensureOpen.run(); + store.renameTempFilesSafe(tempFileNames); + } + + static final class FileChunk { + final StoreFileMetaData md; + final BytesReference content; + final long position; + final boolean lastChunk; + FileChunk(StoreFileMetaData md, BytesReference content, long position, boolean lastChunk) { + this.md = md; + this.content = content; + this.position = position; + this.lastChunk = lastChunk; + } + } + + private final class FileChunkWriter { + // chunks can be delivered out of order, we need to buffer chunks if there's a gap between them. + final PriorityQueue pendingChunks = new PriorityQueue<>(Comparator.comparing(fc -> fc.position)); + long lastPosition = 0; + + void writeChunk(FileChunk newChunk) throws IOException { + synchronized (this) { + pendingChunks.add(newChunk); + } + while (true) { + final FileChunk chunk; + synchronized (this) { + chunk = pendingChunks.peek(); + if (chunk == null || chunk.position != lastPosition) { + return; + } + pendingChunks.remove(); + } + innerWriteFileChunk(chunk.md, chunk.position, chunk.content, chunk.lastChunk); + synchronized (this) { + assert lastPosition == chunk.position : "last_position " + lastPosition + " != chunk_position " + chunk.position; + lastPosition += chunk.content.length(); + if (chunk.lastChunk) { + assert pendingChunks.isEmpty() == true : "still have pending chunks [" + pendingChunks + "]"; + fileChunkWriters.remove(chunk.md.name()); + assert fileChunkWriters.containsValue(this) == false : "chunk writer [" + newChunk.md + "] was not removed"; + } + } + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index e63b9ba8fd5ea..76f2200a47d82 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -20,14 +20,9 @@ package org.elasticsearch.indices.recovery; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; @@ -39,7 +34,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.concurrent.AbstractRefCounted; -import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MapperException; import org.elasticsearch.index.seqno.ReplicationTracker; @@ -55,15 +49,7 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.PriorityQueue; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -85,15 +71,13 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget private final long recoveryId; private final IndexShard indexShard; private final DiscoveryNode sourceNode; - private final String tempFilePrefix; + private final MultiFileWriter multiFileWriter; private final Store store; private final PeerRecoveryTargetService.RecoveryListener listener; private final LongConsumer ensureClusterStateVersionCallback; private final AtomicBoolean finished = new AtomicBoolean(); - private final ConcurrentMap openIndexOutputs = ConcurrentCollections.newConcurrentMap(); - private final ConcurrentMap fileChunkWriters = ConcurrentCollections.newConcurrentMap(); private final CancellableThreads cancellableThreads; // last time this status was accessed @@ -102,8 +86,6 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget // latch that can be used to blockingly wait for RecoveryTarget to be closed private final CountDownLatch closedLatch = new CountDownLatch(1); - private final Map tempFileNames = ConcurrentCollections.newConcurrentMap(); - /** * Creates a new recovery target object that represents a recovery to the provided shard. * @@ -126,7 +108,9 @@ public RecoveryTarget(final IndexShard indexShard, this.indexShard = indexShard; this.sourceNode = sourceNode; this.shardId = indexShard.shardId(); - this.tempFilePrefix = RECOVERY_PREFIX + UUIDs.randomBase64UUID() + "."; + final String tempFilePrefix = RECOVERY_PREFIX + UUIDs.randomBase64UUID() + "."; + this.multiFileWriter = new MultiFileWriter(indexShard.store(), indexShard.recoveryState().getIndex(), tempFilePrefix, logger, + this::ensureRefCount); this.store = indexShard.store(); this.ensureClusterStateVersionCallback = ensureClusterStateVersionCallback; // make sure the store is not released until we are done. @@ -187,12 +171,6 @@ public RecoveryState.Stage stage() { return state().getStage(); } - /** renames all temporary files to their true name, potentially overriding existing files */ - public void renameAllTempFiles() throws IOException { - ensureRefCount(); - store.renameTempFilesSafe(tempFileNames); - } - /** * Closes the current recovery target and waits up to a certain timeout for resources to be freed. * Returns true if resetting the recovery was successful, false if the recovery target is already cancelled / failed or marked as done. @@ -274,7 +252,7 @@ public void notifyListener(RecoveryFailedException e, boolean sendShardFailure) /** mark the current recovery as done */ public void markAsDone() { if (finished.compareAndSet(false, true)) { - assert tempFileNames.isEmpty() : "not all temporary files are renamed"; + assert multiFileWriter.tempFileNames.isEmpty() : "not all temporary files are renamed"; try { // this might still throw an exception ie. if the shard is CLOSED due to some other event. // it's safer to decrement the reference in a try finally here. @@ -287,65 +265,12 @@ public void markAsDone() { } } - /** Get a temporary name for the provided file name. */ - public String getTempNameForFile(String origFile) { - return tempFilePrefix + origFile; - } - - public IndexOutput getOpenIndexOutput(String key) { - ensureRefCount(); - return openIndexOutputs.get(key); - } - - /** remove and {@link org.apache.lucene.store.IndexOutput} for a given file. It is the caller's responsibility to close it */ - public IndexOutput removeOpenIndexOutputs(String name) { - ensureRefCount(); - return openIndexOutputs.remove(name); - } - - /** - * Creates an {@link org.apache.lucene.store.IndexOutput} for the given file name. Note that the - * IndexOutput actually point at a temporary file. - *

- * Note: You can use {@link #getOpenIndexOutput(String)} with the same filename to retrieve the same IndexOutput - * at a later stage - */ - public IndexOutput openAndPutIndexOutput(String fileName, StoreFileMetaData metaData, Store store) throws IOException { - ensureRefCount(); - String tempFileName = getTempNameForFile(fileName); - if (tempFileNames.containsKey(tempFileName)) { - throw new IllegalStateException("output for file [" + fileName + "] has already been created"); - } - // add first, before it's created - tempFileNames.put(tempFileName, fileName); - IndexOutput indexOutput = store.createVerifyingOutput(tempFileName, metaData, IOContext.DEFAULT); - openIndexOutputs.put(fileName, indexOutput); - return indexOutput; - } - @Override protected void closeInternal() { try { - // clean open index outputs - Iterator> iterator = openIndexOutputs.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - logger.trace("closing IndexOutput file [{}]", entry.getValue()); - try { - entry.getValue().close(); - } catch (Exception e) { - logger.debug(() -> new ParameterizedMessage("error while closing recovery output [{}]", entry.getValue()), e); - } - iterator.remove(); - } - // trash temporary files - for (String file : tempFileNames.keySet()) { - logger.trace("cleaning temporary file [{}]", file); - store.deleteQuiet(file); - } + multiFileWriter.close(); } finally { // free store. increment happens in constructor - fileChunkWriters.clear(); store.decRef(); indexShard.recoveryStats().decCurrentAsTarget(); closedLatch.countDown(); @@ -470,7 +395,7 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa // first, we go and move files that were created with the recovery id suffix to // the actual names, its ok if we have a corrupted index here, since we have replicas // to recover from in case of a full cluster shutdown just when this code executes... - renameAllTempFiles(); + multiFileWriter.renameAllTempFiles(); final Store store = store(); store.incRef(); try { @@ -511,96 +436,21 @@ public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaDa } } - private void innerWriteFileChunk(StoreFileMetaData fileMetaData, long position, - BytesReference content, boolean lastChunk) throws IOException { - final Store store = store(); - final String name = fileMetaData.name(); - final RecoveryState.Index indexState = state().getIndex(); - IndexOutput indexOutput; - if (position == 0) { - indexOutput = openAndPutIndexOutput(name, fileMetaData, store); - } else { - indexOutput = getOpenIndexOutput(name); - } - assert indexOutput.getFilePointer() == position : "file-pointer " + indexOutput.getFilePointer() + " != " + position; - BytesRefIterator iterator = content.iterator(); - BytesRef scratch; - while((scratch = iterator.next()) != null) { // we iterate over all pages - this is a 0-copy for all core impls - indexOutput.writeBytes(scratch.bytes, scratch.offset, scratch.length); - } - indexState.addRecoveredBytesToFile(name, content.length()); - if (indexOutput.getFilePointer() >= fileMetaData.length() || lastChunk) { - try { - Store.verify(indexOutput); - } finally { - // we are done - indexOutput.close(); - } - final String temporaryFileName = getTempNameForFile(name); - assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName) : - "expected: [" + temporaryFileName + "] in " + Arrays.toString(store.directory().listAll()); - store.directory().sync(Collections.singleton(temporaryFileName)); - IndexOutput remove = removeOpenIndexOutputs(name); - assert remove == null || remove == indexOutput; // remove maybe null if we got finished - } - } - @Override public void writeFileChunk(StoreFileMetaData fileMetaData, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener listener) { try { state().getTranslog().totalOperations(totalTranslogOps); - final FileChunkWriter writer = fileChunkWriters.computeIfAbsent(fileMetaData.name(), name -> new FileChunkWriter()); - writer.writeChunk(new FileChunk(fileMetaData, content, position, lastChunk)); + multiFileWriter.writeFileChunk(fileMetaData, position, content, lastChunk); listener.onResponse(null); } catch (Exception e) { listener.onFailure(e); } } - private static final class FileChunk { - final StoreFileMetaData md; - final BytesReference content; - final long position; - final boolean lastChunk; - FileChunk(StoreFileMetaData md, BytesReference content, long position, boolean lastChunk) { - this.md = md; - this.content = content; - this.position = position; - this.lastChunk = lastChunk; - } - } - - private final class FileChunkWriter { - // chunks can be delivered out of order, we need to buffer chunks if there's a gap between them. - final PriorityQueue pendingChunks = new PriorityQueue<>(Comparator.comparing(fc -> fc.position)); - long lastPosition = 0; - - void writeChunk(FileChunk newChunk) throws IOException { - synchronized (this) { - pendingChunks.add(newChunk); - } - while (true) { - final FileChunk chunk; - synchronized (this) { - chunk = pendingChunks.peek(); - if (chunk == null || chunk.position != lastPosition) { - return; - } - pendingChunks.remove(); - } - innerWriteFileChunk(chunk.md, chunk.position, chunk.content, chunk.lastChunk); - synchronized (this) { - assert lastPosition == chunk.position : "last_position " + lastPosition + " != chunk_position " + chunk.position; - lastPosition += chunk.content.length(); - if (chunk.lastChunk) { - assert pendingChunks.isEmpty() == true : "still have pending chunks [" + pendingChunks + "]"; - fileChunkWriters.remove(chunk.md.name()); - assert fileChunkWriters.containsValue(this) == false : "chunk writer [" + newChunk.md + "] was not removed"; - } - } - } - } + /** Get a temporary name for the provided file name. */ + public String getTempNameForFile(String origFile) { + return multiFileWriter.getTempNameForFile(origFile); } Path translogLocation() { diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java index 2f837812ae2e2..c8185fea89523 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/FileRestoreContext.java @@ -62,14 +62,14 @@ */ public abstract class FileRestoreContext { - private static final Logger logger = LogManager.getLogger(FileRestoreContext.class); + protected static final Logger logger = LogManager.getLogger(FileRestoreContext.class); - private final String repositoryName; - private final IndexShard indexShard; - private final RecoveryState recoveryState; - private final SnapshotId snapshotId; - private final ShardId shardId; - private final int bufferSize; + protected final String repositoryName; + protected final IndexShard indexShard; + protected final RecoveryState recoveryState; + protected final SnapshotId snapshotId; + protected final ShardId shardId; + protected final int bufferSize; /** * Constructs new restore context @@ -183,7 +183,6 @@ public void restore(SnapshotFiles snapshotFiles) throws IOException { // list of all existing store files final List deleteIfExistFiles = Arrays.asList(store.directory().listAll()); - // restore the files from the snapshot to the Lucene store for (final BlobStoreIndexShardSnapshot.FileInfo fileToRecover : filesToRecover) { // if a file with a same physical name already exist in the store we need to delete it // before restoring it from the snapshot. We could be lenient and try to reuse the existing @@ -196,10 +195,9 @@ public void restore(SnapshotFiles snapshotFiles) throws IOException { logger.trace("[{}] [{}] deleting pre-existing file [{}]", shardId, snapshotId, physicalName); store.directory().deleteFile(physicalName); } - - logger.trace("[{}] [{}] restoring file [{}]", shardId, snapshotId, fileToRecover.name()); - restoreFile(fileToRecover, store); } + + restoreFiles(filesToRecover, store); } catch (IOException ex) { throw new IndexShardRestoreFailedException(shardId, "Failed to recover index", ex); } @@ -234,6 +232,14 @@ public void restore(SnapshotFiles snapshotFiles) throws IOException { } } + protected void restoreFiles(List filesToRecover, Store store) throws IOException { + // restore the files from the snapshot to the Lucene store + for (final BlobStoreIndexShardSnapshot.FileInfo fileToRecover : filesToRecover) { + logger.trace("[{}] [{}] restoring file [{}]", shardId, snapshotId, fileToRecover.name()); + restoreFile(fileToRecover, store); + } + } + protected abstract InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo); @SuppressWarnings("unchecked") diff --git a/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java b/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java index fe4ab2e363a88..ffe25b4a372e3 100644 --- a/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java +++ b/server/src/test/java/org/elasticsearch/indices/DateMathIndexExpressionsIntegrationIT.java @@ -19,6 +19,9 @@ package org.elasticsearch.indices; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -29,10 +32,14 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.test.ESIntegTestCase; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; +import org.junit.Before; + +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Locale; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; @@ -41,15 +48,42 @@ import static org.hamcrest.Matchers.notNullValue; public class DateMathIndexExpressionsIntegrationIT extends ESIntegTestCase { + private ZonedDateTime now; + + @Before + public void setNow() { + now = ZonedDateTime.now(ZoneOffset.UTC); + } + + /** + * the internal cluster calls System.nanoTime() and System.currentTimeMillis() during evaluations of requests + * that need date-math index resolution. These are not mockable in these tests. As is, executing requests as-is + * in these test cases can potentially result in invalid responses when day-boundaries are hit mid test run. Instead + * of failing when index resolution with `now` is one day off, this method wraps calls with the assumption that + * the day did not change during the test run. + */ + public R dateSensitiveGet(ActionRequestBuilder builder) { + Runnable dayChangeAssumption = () -> assumeTrue("day changed between requests", + ZonedDateTime.now(ZoneOffset.UTC).getDayOfYear() == now.getDayOfYear()); + R response; + try { + response = builder.get(); + } catch (IndexNotFoundException e) { + // index resolver throws this if it does not find the exact index due to day changes + dayChangeAssumption.run(); + throw e; + } + dayChangeAssumption.run(); + return response; + } public void testIndexNameDateMathExpressions() { - DateTime now = new DateTime(DateTimeZone.UTC); - String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now); - String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1)); - String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2)); + String index1 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now); + String index2 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)); + String index3 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)); createIndex(index1, index2, index3); - GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(index1, index2, index3).get(); + GetSettingsResponse getSettingsResponse = dateSensitiveGet(client().admin().indices().prepareGetSettings(index1, index2, index3)); assertEquals(index1, getSettingsResponse.getSetting(index1, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(index2, getSettingsResponse.getSetting(index2, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(index3, getSettingsResponse.getSetting(index3, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); @@ -63,26 +97,26 @@ public void testIndexNameDateMathExpressions() { client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); - SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); + SearchResponse searchResponse = dateSensitiveGet(client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3)); assertHitCount(searchResponse, 3); assertSearchHits(searchResponse, "1", "2", "3"); - GetResponse getResponse = client().prepareGet(dateMathExp1, "type", "1").get(); + GetResponse getResponse = dateSensitiveGet(client().prepareGet(dateMathExp1, "type", "1")); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("1")); - getResponse = client().prepareGet(dateMathExp2, "type", "2").get(); + getResponse = dateSensitiveGet(client().prepareGet(dateMathExp2, "type", "2")); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("2")); - getResponse = client().prepareGet(dateMathExp3, "type", "3").get(); + getResponse = dateSensitiveGet(client().prepareGet(dateMathExp3, "type", "3")); assertThat(getResponse.isExists(), is(true)); assertThat(getResponse.getId(), equalTo("3")); - MultiGetResponse mgetResponse = client().prepareMultiGet() + MultiGetResponse mgetResponse = dateSensitiveGet(client().prepareMultiGet() .add(dateMathExp1, "type", "1") .add(dateMathExp2, "type", "2") - .add(dateMathExp3, "type", "3").get(); + .add(dateMathExp3, "type", "3")); assertThat(mgetResponse.getResponses()[0].getResponse().isExists(), is(true)); assertThat(mgetResponse.getResponses()[0].getResponse().getId(), equalTo("1")); assertThat(mgetResponse.getResponses()[1].getResponse().isExists(), is(true)); @@ -90,29 +124,29 @@ public void testIndexNameDateMathExpressions() { assertThat(mgetResponse.getResponses()[2].getResponse().isExists(), is(true)); assertThat(mgetResponse.getResponses()[2].getResponse().getId(), equalTo("3")); - IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get(); + IndicesStatsResponse indicesStatsResponse = + dateSensitiveGet(client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3)); assertThat(indicesStatsResponse.getIndex(index1), notNullValue()); assertThat(indicesStatsResponse.getIndex(index2), notNullValue()); assertThat(indicesStatsResponse.getIndex(index3), notNullValue()); - DeleteResponse deleteResponse = client().prepareDelete(dateMathExp1, "type", "1").get(); + DeleteResponse deleteResponse = dateSensitiveGet(client().prepareDelete(dateMathExp1, "type", "1")); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("1")); - deleteResponse = client().prepareDelete(dateMathExp2, "type", "2").get(); + deleteResponse = dateSensitiveGet(client().prepareDelete(dateMathExp2, "type", "2")); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("2")); - deleteResponse = client().prepareDelete(dateMathExp3, "type", "3").get(); + deleteResponse = dateSensitiveGet(client().prepareDelete(dateMathExp3, "type", "3")); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); assertThat(deleteResponse.getId(), equalTo("3")); } - public void testAutoCreateIndexWithDateMathExpression() throws Exception { - DateTime now = new DateTime(DateTimeZone.UTC); - String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now); - String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1)); - String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2)); + public void testAutoCreateIndexWithDateMathExpression() { + String index1 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now); + String index2 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)); + String index3 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)); String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; @@ -122,29 +156,28 @@ public void testAutoCreateIndexWithDateMathExpression() throws Exception { client().prepareIndex(dateMathExp3, "type", "3").setSource("{}", XContentType.JSON).get(); refresh(); - SearchResponse searchResponse = client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3).get(); + SearchResponse searchResponse = dateSensitiveGet(client().prepareSearch(dateMathExp1, dateMathExp2, dateMathExp3)); assertHitCount(searchResponse, 3); assertSearchHits(searchResponse, "1", "2", "3"); - IndicesStatsResponse indicesStatsResponse = client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3).get(); + IndicesStatsResponse indicesStatsResponse = + dateSensitiveGet(client().admin().indices().prepareStats(dateMathExp1, dateMathExp2, dateMathExp3)); assertThat(indicesStatsResponse.getIndex(index1), notNullValue()); assertThat(indicesStatsResponse.getIndex(index2), notNullValue()); assertThat(indicesStatsResponse.getIndex(index3), notNullValue()); } - public void testCreateIndexWithDateMathExpression() throws Exception { - DateTime now = new DateTime(DateTimeZone.UTC); - String index1 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now); - String index2 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(1)); - String index3 = ".marvel-" + DateTimeFormat.forPattern("YYYY.MM.dd").print(now.minusDays(2)); + public void testCreateIndexWithDateMathExpression() { + String index1 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now); + String index2 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(1)); + String index3 = ".marvel-" + DateTimeFormatter.ofPattern("yyyy.MM.dd", Locale.ROOT).format(now.minusDays(2)); String dateMathExp1 = "<.marvel-{now/d}>"; String dateMathExp2 = "<.marvel-{now/d-1d}>"; String dateMathExp3 = "<.marvel-{now/d-2d}>"; createIndex(dateMathExp1, dateMathExp2, dateMathExp3); - - GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(index1, index2, index3).get(); + GetSettingsResponse getSettingsResponse = dateSensitiveGet(client().admin().indices().prepareGetSettings(index1, index2, index3)); assertEquals(dateMathExp1, getSettingsResponse.getSetting(index1, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(dateMathExp2, getSettingsResponse.getSetting(index2, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); assertEquals(dateMathExp3, getSettingsResponse.getSetting(index3, IndexMetaData.SETTING_INDEX_PROVIDED_NAME)); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java index a2ec88cf7b58c..41ea9a8bea74b 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/PeerRecoveryTargetServiceTests.java @@ -189,7 +189,7 @@ public void testWriteFileChunksConcurrently() throws Exception { for (Thread sender : senders) { sender.join(); } - recoveryTarget.renameAllTempFiles(); + recoveryTarget.cleanFiles(0, sourceSnapshot); recoveryTarget.decRef(); Store.MetadataSnapshot targetSnapshot = targetShard.snapshotStoreMetadata(); Store.RecoveryDiff diff = sourceSnapshot.recoveryDiff(targetSnapshot); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java index ed1ee7708522d..b974d42d826bb 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoveryStatusTests.java @@ -20,8 +20,6 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.IndexOutput; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.shard.IndexShard; @@ -32,9 +30,6 @@ import java.util.Set; import java.util.regex.Pattern; -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; - public class RecoveryStatusTests extends ESSingleNodeTestCase { private static final org.apache.lucene.util.Version MIN_SUPPORTED_LUCENE_VERSION = org.elasticsearch.Version.CURRENT .minimumIndexCompatibilityVersion().luceneVersion; @@ -42,35 +37,27 @@ public void testRenameTempFiles() throws IOException { IndexService service = createIndex("foo"); IndexShard indexShard = service.getShardOrNull(0); - DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); - RecoveryTarget status = new RecoveryTarget(indexShard, node, new PeerRecoveryTargetService.RecoveryListener() { - @Override - public void onRecoveryDone(RecoveryState state) { - } - - @Override - public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) { - } - }, version -> {}); - try (IndexOutput indexOutput = status.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength() - , "9z51nw", MIN_SUPPORTED_LUCENE_VERSION), status.store())) { + MultiFileWriter multiFileWriter = new MultiFileWriter(indexShard.store(), + indexShard.recoveryState().getIndex(), "recovery.test.", logger, () -> {}); + try (IndexOutput indexOutput = multiFileWriter.openAndPutIndexOutput("foo.bar", + new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", MIN_SUPPORTED_LUCENE_VERSION), indexShard.store())) { indexOutput.writeInt(1); - IndexOutput openIndexOutput = status.getOpenIndexOutput("foo.bar"); + IndexOutput openIndexOutput = multiFileWriter.getOpenIndexOutput("foo.bar"); assertSame(openIndexOutput, indexOutput); openIndexOutput.writeInt(1); CodecUtil.writeFooter(indexOutput); } try { - status.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", - MIN_SUPPORTED_LUCENE_VERSION), status.store()); + multiFileWriter.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw", + MIN_SUPPORTED_LUCENE_VERSION), indexShard.store()); fail("file foo.bar is already opened and registered"); } catch (IllegalStateException ex) { assertEquals("output for file [foo.bar] has already been created", ex.getMessage()); // all well = it's already registered } - status.removeOpenIndexOutputs("foo.bar"); - Set strings = Sets.newHashSet(status.store().directory().listAll()); + multiFileWriter.removeOpenIndexOutputs("foo.bar"); + Set strings = Sets.newHashSet(indexShard.store().directory().listAll()); String expectedFile = null; for (String file : strings) { if (Pattern.compile("recovery[.][\\w-]+[.]foo[.]bar").matcher(file).matches()) { @@ -80,12 +67,10 @@ public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, bo } assertNotNull(expectedFile); indexShard.close("foo", false);// we have to close it here otherwise rename fails since the write.lock is held by the engine - status.renameAllTempFiles(); - strings = Sets.newHashSet(status.store().directory().listAll()); + multiFileWriter.renameAllTempFiles(); + strings = Sets.newHashSet(indexShard.store().directory().listAll()); assertTrue(strings.toString(), strings.contains("foo.bar")); assertFalse(strings.toString(), strings.contains(expectedFile)); - // we must fail the recovery because marking it as done will try to move the shard to POST_RECOVERY, - // which will fail because it's started - status.fail(new RecoveryFailedException(status.state(), "end of test. OK.", null), false); + multiFileWriter.close(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index e09455b55bd52..855f1b2e2fd72 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1088,6 +1088,12 @@ public static Translog getTranslog(Engine engine) { return internalEngine.getTranslog(); } + public static boolean hasSnapshottedCommits(Engine engine) { + assert engine instanceof InternalEngine : "only InternalEngines have snapshotted commits, got: " + engine.getClass(); + InternalEngine internalEngine = (InternalEngine) engine; + return internalEngine.hasSnapshottedCommits(); + } + public static final class PrimaryTermSupplier implements LongSupplier { private final AtomicLong term; diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index a7fa69e7abd39..d4c73c1c6e503 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -127,6 +127,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E private final CcrLicenseChecker ccrLicenseChecker; private final SetOnce restoreSourceService = new SetOnce<>(); private final SetOnce ccrSettings = new SetOnce<>(); + private final SetOnce threadPool = new SetOnce<>(); private Client client; private final boolean transportClientMode; @@ -171,6 +172,7 @@ public Collection createComponents( CcrSettings ccrSettings = new CcrSettings(settings, clusterService.getClusterSettings()); this.ccrSettings.set(ccrSettings); + this.threadPool.set(threadPool); CcrRestoreSourceService restoreSourceService = new CcrRestoreSourceService(threadPool, ccrSettings); this.restoreSourceService.set(restoreSourceService); return Arrays.asList( @@ -307,7 +309,7 @@ public List> getExecutorBuilders(Settings settings) { @Override public Map getInternalRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) { Repository.Factory repositoryFactory = - (metadata) -> new CcrRepository(metadata, client, ccrLicenseChecker, settings, ccrSettings.get()); + (metadata) -> new CcrRepository(metadata, client, ccrLicenseChecker, settings, ccrSettings.get(), threadPool.get()); return Collections.singletonMap(CcrRepository.TYPE, repositoryFactory); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java index 0e147f66d6ebc..9abcfb86e2b7c 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java @@ -57,6 +57,12 @@ public final class CcrSettings { new ByteSizeValue(1, ByteSizeUnit.KB), new ByteSizeValue(1, ByteSizeUnit.GB), Setting.Property.Dynamic, Setting.Property.NodeScope); + /** + * Controls the maximum number of file chunk requests that are sent concurrently per recovery to the leader. + */ + public static final Setting INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING = + Setting.intSetting("ccr.indices.recovery.max_concurrent_file_chunks", 5, 1, 10, Property.Dynamic, Property.NodeScope); + /** * The leader must open resources for a ccr recovery. If there is no activity for this interval of time, * the leader will close the restore session. @@ -77,7 +83,7 @@ public final class CcrSettings { * * @return the settings */ - static List> getSettings() { + public static List> getSettings() { return Arrays.asList( XPackSettings.CCR_ENABLED_SETTING, CCR_FOLLOWING_INDEX_SETTING, @@ -86,6 +92,7 @@ static List> getSettings() { INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, CCR_AUTO_FOLLOW_WAIT_FOR_METADATA_TIMEOUT, RECOVERY_CHUNK_SIZE, + INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING, CCR_WAIT_FOR_METADATA_TIMEOUT); } @@ -93,14 +100,17 @@ static List> getSettings() { private volatile TimeValue recoveryActivityTimeout; private volatile TimeValue recoveryActionTimeout; private volatile ByteSizeValue chunkSize; + private volatile int maxConcurrentFileChunks; public CcrSettings(Settings settings, ClusterSettings clusterSettings) { this.recoveryActivityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings); this.recoveryActionTimeout = INDICES_RECOVERY_ACTION_TIMEOUT_SETTING.get(settings); this.ccrRateLimiter = new CombinedRateLimiter(RECOVERY_MAX_BYTES_PER_SECOND.get(settings)); this.chunkSize = RECOVERY_MAX_BYTES_PER_SECOND.get(settings); + this.maxConcurrentFileChunks = INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING.get(settings); clusterSettings.addSettingsUpdateConsumer(RECOVERY_MAX_BYTES_PER_SECOND, this::setMaxBytesPerSec); clusterSettings.addSettingsUpdateConsumer(RECOVERY_CHUNK_SIZE, this::setChunkSize); + clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING, this::setMaxConcurrentFileChunks); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setRecoveryActivityTimeout); clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, this::setRecoveryActionTimeout); } @@ -109,6 +119,10 @@ private void setChunkSize(ByteSizeValue chunkSize) { this.chunkSize = chunkSize; } + private void setMaxConcurrentFileChunks(int maxConcurrentFileChunks) { + this.maxConcurrentFileChunks = maxConcurrentFileChunks; + } + private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) { ccrRateLimiter.setMBPerSec(maxBytesPerSec); } @@ -125,6 +139,10 @@ public ByteSizeValue getChunkSize() { return chunkSize; } + public int getMaxConcurrentFileChunks() { + return maxConcurrentFileChunks; + } + public CombinedRateLimiter getRateLimiter() { return ccrRateLimiter; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java index cf8d2e5c55f48..37dfc84f46a01 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java @@ -51,7 +51,6 @@ public static class TransportGetCcrRestoreFileChunkAction extends HandledTransportAction { private final CcrRestoreSourceService restoreSourceService; - private final ThreadPool threadPool; private final BigArrays bigArrays; @Inject @@ -59,7 +58,6 @@ public TransportGetCcrRestoreFileChunkAction(BigArrays bigArrays, TransportServi CcrRestoreSourceService restoreSourceService) { super(NAME, transportService, actionFilters, GetCcrRestoreFileChunkRequest::new, ThreadPool.Names.GENERIC); TransportActionProxy.registerProxyAction(transportService, NAME, GetCcrRestoreFileChunkResponse::new); - this.threadPool = transportService.getThreadPool(); this.restoreSourceService = restoreSourceService; this.bigArrays = bigArrays; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 4d14e4cdd7b31..1896a25c91b26 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -8,8 +8,13 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.IndexCommit; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.AddRetentionLeaseAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -23,26 +28,27 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.CombinedRateLimiter; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.index.Index; import org.elasticsearch.index.engine.EngineException; +import org.elasticsearch.index.seqno.LocalCheckpointTracker; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardRecoveryException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException; import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus; -import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo; import org.elasticsearch.index.snapshots.blobstore.SnapshotFiles; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.StoreFileMetaData; +import org.elasticsearch.indices.recovery.MultiFileWriter; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.Repository; @@ -52,6 +58,7 @@ import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotShardFailure; import org.elasticsearch.snapshots.SnapshotState; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -66,15 +73,20 @@ import java.io.Closeable; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongConsumer; import java.util.function.Supplier; +import static org.elasticsearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; + /** * This repository relies on a remote cluster for Ccr restores. It is read-only so it can only be used to @@ -82,6 +94,8 @@ */ public class CcrRepository extends AbstractLifecycleComponent implements Repository { + private static final Logger logger = LogManager.getLogger(CcrRepository.class); + public static final String LATEST = "_latest_"; public static final String TYPE = "_ccr_"; public static final String NAME_PREFIX = "_ccr_"; @@ -93,17 +107,19 @@ public class CcrRepository extends AbstractLifecycleComponent implements Reposit private final String remoteClusterAlias; private final Client client; private final CcrLicenseChecker ccrLicenseChecker; + private final ThreadPool threadPool; private final CounterMetric throttledTime = new CounterMetric(); public CcrRepository(RepositoryMetaData metadata, Client client, CcrLicenseChecker ccrLicenseChecker, Settings settings, - CcrSettings ccrSettings) { + CcrSettings ccrSettings, ThreadPool threadPool) { this.metadata = metadata; this.ccrSettings = ccrSettings; assert metadata.name().startsWith(NAME_PREFIX) : "CcrRepository metadata.name() must start with: " + NAME_PREFIX; this.remoteClusterAlias = Strings.split(metadata.name(), NAME_PREFIX)[1]; this.ccrLicenseChecker = ccrLicenseChecker; this.client = client; + this.threadPool = threadPool; } @Override @@ -287,8 +303,8 @@ public void restoreShard(IndexShard indexShard, SnapshotId snapshotId, Version v final Client remoteClient = client.getRemoteClusterClient(remoteClusterAlias); - final String retentionLeaseId = - indexShard.shardId().getIndex().getUUID() + "-following-" + leaderUUID + "-" + UUIDs.randomBase64UUID(); + final String retentionLeaseId = indexShard.shardId().getIndex().getUUID() + "-following-" + leaderUUID; + logger.trace("requesting leader primary to add retention lease [" + retentionLeaseId + "]"); remoteClient .execute(AddRetentionLeaseAction.INSTANCE, new AddRetentionLeaseAction.Request(leaderShardId, retentionLeaseId, 0, "ccr")) .actionGet(ccrSettings.getRecoveryActionTimeout()); @@ -333,7 +349,7 @@ private RestoreSession openSession(String repositoryName, Client remoteClient, S PutCcrRestoreSessionAction.PutCcrRestoreSessionResponse response = remoteClient.execute(PutCcrRestoreSessionAction.INSTANCE, new PutCcrRestoreSessionRequest(sessionUUID, leaderShardId)).actionGet(ccrSettings.getRecoveryActionTimeout()); return new RestoreSession(repositoryName, remoteClient, sessionUUID, response.getNode(), indexShard, recoveryState, - response.getStoreFileMetaData(), response.getMappingVersion(), ccrSettings, throttledTime::inc); + response.getStoreFileMetaData(), response.getMappingVersion(), threadPool, ccrSettings, throttledTime::inc); } private static class RestoreSession extends FileRestoreContext implements Closeable { @@ -345,107 +361,186 @@ private static class RestoreSession extends FileRestoreContext implements Closea private final long mappingVersion; private final CcrSettings ccrSettings; private final LongConsumer throttleListener; + private final ThreadPool threadPool; RestoreSession(String repositoryName, Client remoteClient, String sessionUUID, DiscoveryNode node, IndexShard indexShard, RecoveryState recoveryState, Store.MetadataSnapshot sourceMetaData, long mappingVersion, - CcrSettings ccrSettings, LongConsumer throttleListener) { + ThreadPool threadPool, CcrSettings ccrSettings, LongConsumer throttleListener) { super(repositoryName, indexShard, SNAPSHOT_ID, recoveryState, Math.toIntExact(ccrSettings.getChunkSize().getBytes())); this.remoteClient = remoteClient; this.sessionUUID = sessionUUID; this.node = node; this.sourceMetaData = sourceMetaData; this.mappingVersion = mappingVersion; + this.threadPool = threadPool; this.ccrSettings = ccrSettings; this.throttleListener = throttleListener; } void restoreFiles() throws IOException { - ArrayList fileInfos = new ArrayList<>(); + ArrayList fileInfos = new ArrayList<>(); for (StoreFileMetaData fileMetaData : sourceMetaData) { ByteSizeValue fileSize = new ByteSizeValue(fileMetaData.length()); - fileInfos.add(new BlobStoreIndexShardSnapshot.FileInfo(fileMetaData.name(), fileMetaData, fileSize)); + fileInfos.add(new FileInfo(fileMetaData.name(), fileMetaData, fileSize)); } SnapshotFiles snapshotFiles = new SnapshotFiles(LATEST, fileInfos); restore(snapshotFiles); } - @Override - protected InputStream fileInputStream(BlobStoreIndexShardSnapshot.FileInfo fileInfo) { - return new RestoreFileInputStream(remoteClient, sessionUUID, node, fileInfo.metadata(), ccrSettings, throttleListener); - } + private static class FileSession { + FileSession(long lastTrackedSeqNo, long lastOffset) { + this.lastTrackedSeqNo = lastTrackedSeqNo; + this.lastOffset = lastOffset; + } - @Override - public void close() { - ClearCcrRestoreSessionRequest clearRequest = new ClearCcrRestoreSessionRequest(sessionUUID, node); - ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse response = - remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); + final long lastTrackedSeqNo; + final long lastOffset; } - } - private static class RestoreFileInputStream extends InputStream { - - private final Client remoteClient; - private final String sessionUUID; - private final DiscoveryNode node; - private final StoreFileMetaData fileToRecover; - private final CombinedRateLimiter rateLimiter; - private final CcrSettings ccrSettings; - private final LongConsumer throttleListener; - - private long pos = 0; + @Override + protected void restoreFiles(List filesToRecover, Store store) throws IOException { + logger.trace("[{}] starting CCR restore of {} files", shardId, filesToRecover); + + try (MultiFileWriter multiFileWriter = new MultiFileWriter(store, recoveryState.getIndex(), "", logger, () -> {})) { + final LocalCheckpointTracker requestSeqIdTracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED); + final AtomicReference> error = new AtomicReference<>(); + + final ArrayDeque remainingFiles = new ArrayDeque<>(filesToRecover); + final Map inFlightRequests = new HashMap<>(); + final Object mutex = new Object(); + + while (true) { + if (error.get() != null) { + break; + } + final FileInfo fileToRecover; + final FileSession fileSession; + synchronized (mutex) { + if (inFlightRequests.isEmpty() && remainingFiles.isEmpty()) { + break; + } + final long maxConcurrentFileChunks = ccrSettings.getMaxConcurrentFileChunks(); + if (remainingFiles.isEmpty() == false && inFlightRequests.size() < maxConcurrentFileChunks) { + for (int i = 0; i < maxConcurrentFileChunks; i++) { + if (remainingFiles.isEmpty()) { + break; + } + inFlightRequests.put(remainingFiles.pop(), new FileSession(NO_OPS_PERFORMED, 0)); + } + } + final Map.Entry minEntry = + inFlightRequests.entrySet().stream().min(Comparator.comparingLong(e -> e.getValue().lastTrackedSeqNo)).get(); + fileSession = minEntry.getValue(); + fileToRecover = minEntry.getKey(); + } + try { + requestSeqIdTracker.waitForOpsToComplete(fileSession.lastTrackedSeqNo); + synchronized (mutex) { + // if file has been removed in the mean-while, it means that restore of this file completed, so start working + // on the next one + if (inFlightRequests.containsKey(fileToRecover) == false) { + continue; + } + } + final long requestSeqId = requestSeqIdTracker.generateSeqNo(); + try { + synchronized (mutex) { + inFlightRequests.put(fileToRecover, new FileSession(requestSeqId, fileSession.lastOffset)); + } + final int bytesRequested = Math.toIntExact(Math.min(ccrSettings.getChunkSize().getBytes(), + fileToRecover.length() - fileSession.lastOffset)); + final GetCcrRestoreFileChunkRequest request = + new GetCcrRestoreFileChunkRequest(node, sessionUUID, fileToRecover.name(), bytesRequested); + logger.trace("[{}] [{}] fetching chunk for file [{}]", shardId, snapshotId, fileToRecover.name()); + + remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request, + ActionListener.wrap( + r -> threadPool.generic().execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + } + + @Override + protected void doRun() throws Exception { + final int actualChunkSize = r.getChunk().length(); + final long nanosPaused = ccrSettings.getRateLimiter().maybePause(actualChunkSize); + throttleListener.accept(nanosPaused); + final long newOffset = r.getOffset() + actualChunkSize; + assert newOffset <= fileToRecover.length(); + final boolean lastChunk = newOffset >= fileToRecover.length(); + multiFileWriter.writeFileChunk(fileToRecover.metadata(), r.getOffset(), r.getChunk(), + lastChunk); + if (lastChunk) { + synchronized (mutex) { + final FileSession session = inFlightRequests.remove(fileToRecover); + assert session != null : "session disappeared for " + fileToRecover.name(); + } + } else { + synchronized (mutex) { + final FileSession replaced = inFlightRequests.replace(fileToRecover, + new FileSession(requestSeqId, newOffset)); + assert replaced != null : "session disappeared for " + fileToRecover.name(); + } + } + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + } + }), + e -> { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + } + )); + } catch (Exception e) { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + requestSeqIdTracker.markSeqNoAsCompleted(requestSeqId); + throw e; + } + } catch (Exception e) { + error.compareAndSet(null, Tuple.tuple(fileToRecover.metadata(), e)); + break; + } + + } + try { + requestSeqIdTracker.waitForOpsToComplete(requestSeqIdTracker.getMaxSeqNo()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ElasticsearchException(e); + } + if (error.get() != null) { + handleError(store, error.get().v2()); + } + } - private RestoreFileInputStream(Client remoteClient, String sessionUUID, DiscoveryNode node, StoreFileMetaData fileToRecover, - CcrSettings ccrSettings, LongConsumer throttleListener) { - this.remoteClient = remoteClient; - this.sessionUUID = sessionUUID; - this.node = node; - this.fileToRecover = fileToRecover; - this.ccrSettings = ccrSettings; - this.rateLimiter = ccrSettings.getRateLimiter(); - this.throttleListener = throttleListener; + logger.trace("[{}] completed CCR restore", shardId); } + private void handleError(Store store, Exception e) throws IOException { + final IOException corruptIndexException; + if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(e)) != null) { + try { + store.markStoreCorrupted(corruptIndexException); + } catch (IOException ioe) { + logger.warn("store cannot be marked as corrupted", e); + } + throw corruptIndexException; + } else { + ExceptionsHelper.reThrowIfNotNull(e); + } + } @Override - public int read() throws IOException { + protected InputStream fileInputStream(FileInfo fileInfo) { throw new UnsupportedOperationException(); } @Override - public int read(byte[] bytes, int off, int len) throws IOException { - long remainingBytes = fileToRecover.length() - pos; - if (remainingBytes <= 0) { - return 0; - } - - int bytesRequested = (int) Math.min(remainingBytes, len); - - long nanosPaused = rateLimiter.maybePause(bytesRequested); - throttleListener.accept(nanosPaused); - - String fileName = fileToRecover.name(); - GetCcrRestoreFileChunkRequest request = new GetCcrRestoreFileChunkRequest(node, sessionUUID, fileName, bytesRequested); - GetCcrRestoreFileChunkAction.GetCcrRestoreFileChunkResponse response = - remoteClient.execute(GetCcrRestoreFileChunkAction.INSTANCE, request).actionGet(ccrSettings.getRecoveryActionTimeout()); - BytesReference fileChunk = response.getChunk(); - - int bytesReceived = fileChunk.length(); - if (bytesReceived > bytesRequested) { - throw new IOException("More bytes [" + bytesReceived + "] received than requested [" + bytesRequested + "]"); - } - - long leaderOffset = response.getOffset(); - assert pos == leaderOffset : "Position [" + pos + "] should be equal to the leader file offset [" + leaderOffset + "]."; - - try (StreamInput streamInput = fileChunk.streamInput()) { - int bytesRead = streamInput.read(bytes, 0, bytesReceived); - assert bytesRead == bytesReceived : "Did not read the correct number of bytes"; - } - - pos += bytesReceived; - - return bytesReceived; + public void close() { + ClearCcrRestoreSessionRequest clearRequest = new ClearCcrRestoreSessionRequest(sessionUUID, node); + ClearCcrRestoreSessionAction.ClearCcrRestoreSessionResponse response = + remoteClient.execute(ClearCcrRestoreSessionAction.INSTANCE, clearRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } - } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java index f093143112d3d..65fb14fdb95ec 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceService.java @@ -42,8 +42,6 @@ import java.util.HashSet; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Consumer; import java.util.function.LongConsumer; public class CcrRestoreSourceService extends AbstractLifecycleComponent implements IndexEventListener { @@ -52,7 +50,6 @@ public class CcrRestoreSourceService extends AbstractLifecycleComponent implemen private final Map onGoingRestores = ConcurrentCollections.newConcurrentMap(); private final Map> sessionsForShard = new HashMap<>(); - private final CopyOnWriteArrayList> closeSessionListeners = new CopyOnWriteArrayList<>(); private final ThreadPool threadPool; private final CcrSettings ccrSettings; private final CounterMetric throttleTime = new CounterMetric(); @@ -93,12 +90,6 @@ protected synchronized void doClose() throws IOException { onGoingRestores.clear(); } - // TODO: The listeners are for testing. Once end-to-end file restore is implemented and can be tested, - // these should be removed. - public void addCloseSessionListener(Consumer listener) { - closeSessionListeners.add(listener); - } - public synchronized Store.MetadataSnapshot openSession(String sessionUUID, IndexShard indexShard) throws IOException { boolean success = false; RestoreSession restore = null; @@ -165,9 +156,7 @@ private void internalCloseSession(String sessionUUID, boolean throwIfSessionMiss } } } - closeSessionListeners.forEach(c -> c.accept(sessionUUID)); restore.decRef(); - } private Scheduler.Cancellable scheduleTimeout(String sessionUUID) { @@ -255,6 +244,7 @@ protected void closeInternal() { assert keyedLock.hasLockedKeys() == false : "Should not hold any file locks when closing"; timeoutTask.cancel(); IOUtils.closeWhileHandlingException(cachedInputs.values()); + IOUtils.closeWhileHandlingException(commitRef); } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index d41c7bf1405ef..d5c3304e966e6 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -104,7 +104,7 @@ public class IndexFollowingIT extends CcrIntegTestCase { public void testFollowIndex() throws Exception { - final int numberOfPrimaryShards = randomIntBetween(1, 1); + final int numberOfPrimaryShards = randomIntBetween(1, 3); int numberOfReplicas = between(0, 1); final String leaderIndexSettings = getIndexSettings(numberOfPrimaryShards, numberOfReplicas, singletonMap(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), "true")); @@ -114,7 +114,7 @@ public void testFollowIndex() throws Exception { final int firstBatchNumDocs; // Sometimes we want to index a lot of documents to ensure that the recovery works with larger files if (rarely()) { - firstBatchNumDocs = randomIntBetween(1800, 2000); + firstBatchNumDocs = randomIntBetween(1800, 10000); } else { firstBatchNumDocs = randomIntBetween(10, 64); } @@ -127,21 +127,22 @@ public void testFollowIndex() throws Exception { waitForDocs(firstBatchNumDocs, indexer); indexer.assertNoFailures(); + logger.info("Executing put follow"); boolean waitOnAll = randomBoolean(); final PutFollowAction.Request followRequest; if (waitOnAll) { - followRequest = putFollow("index1", "follow", ActiveShardCount.ALL); + followRequest = putFollow("index1", "index2", ActiveShardCount.ALL); } else { - followRequest = putFollow("index1", "follow", ActiveShardCount.ONE); + followRequest = putFollow("index1", "index2", ActiveShardCount.ONE); } PutFollowAction.Response response = followerClient().execute(PutFollowAction.INSTANCE, followRequest).get(); assertTrue(response.isFollowIndexCreated()); assertTrue(response.isFollowIndexShardsAcked()); assertTrue(response.isIndexFollowingStarted()); - ClusterHealthRequest healthRequest = Requests.clusterHealthRequest("follow").waitForNoRelocatingShards(true); - ClusterIndexHealth indexHealth = followerClient().admin().cluster().health(healthRequest).get().getIndices().get("follow"); + ClusterHealthRequest healthRequest = Requests.clusterHealthRequest("index2").waitForNoRelocatingShards(true); + ClusterIndexHealth indexHealth = followerClient().admin().cluster().health(healthRequest).get().getIndices().get("index2"); for (ClusterShardHealth shardHealth : indexHealth.getShards().values()) { if (waitOnAll) { assertTrue(shardHealth.isPrimaryActive()); @@ -165,17 +166,19 @@ public void testFollowIndex() throws Exception { for (String docId : indexer.getIds()) { assertBusy(() -> { - final GetResponse getResponse = followerClient().prepareGet("follow", "_doc", docId).get(); + final GetResponse getResponse = followerClient().prepareGet("index2", "_doc", docId).get(); assertTrue("Doc with id [" + docId + "] is missing", getResponse.isExists()); }); } - pauseFollow("follow"); - followerClient().execute(ResumeFollowAction.INSTANCE, resumeFollow("follow")).get(); + pauseFollow("index2"); + followerClient().execute(ResumeFollowAction.INSTANCE, resumeFollow("index2")).get(); final int secondBatchNumDocs = randomIntBetween(2, 64); logger.info("Indexing [{}] docs as second batch", secondBatchNumDocs); indexer.continueIndexing(secondBatchNumDocs); + waitForDocs(firstBatchNumDocs + secondBatchNumDocs, indexer); + final Map secondBatchNumDocsPerShard = new HashMap<>(); final ShardStats[] secondBatchShardStats = leaderClient().admin().indices().prepareStats("index1").get().getIndex("index1").getShards(); @@ -190,12 +193,13 @@ public void testFollowIndex() throws Exception { for (String docId : indexer.getIds()) { assertBusy(() -> { - final GetResponse getResponse = followerClient().prepareGet("follow", "_doc", docId).get(); + final GetResponse getResponse = followerClient().prepareGet("index2", "_doc", docId).get(); assertTrue("Doc with id [" + docId + "] is missing", getResponse.isExists()); }); } - pauseFollow("follow"); - assertMaxSeqNoOfUpdatesIsTransferred(resolveLeaderIndex("index1"), resolveFollowerIndex("follow"), numberOfPrimaryShards); + + pauseFollow("index2"); + assertMaxSeqNoOfUpdatesIsTransferred(resolveLeaderIndex("index1"), resolveFollowerIndex("index2"), numberOfPrimaryShards); } } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java index 1c3c0da3d3c8a..2b5011d45139f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java @@ -12,10 +12,9 @@ import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; @@ -25,7 +24,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.node.Node.NODE_NAME_SETTING; @@ -39,10 +38,8 @@ public void setUp() throws Exception { super.setUp(); Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node").build(); taskQueue = new DeterministicTaskQueue(settings, random()); - Set> registeredSettings = Sets.newHashSet(CcrSettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, - CcrSettings.RECOVERY_MAX_BYTES_PER_SECOND, CcrSettings.INDICES_RECOVERY_ACTION_TIMEOUT_SETTING, - CcrSettings.RECOVERY_CHUNK_SIZE); - ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, registeredSettings); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, CcrSettings.getSettings() + .stream().filter(s -> s.hasNodeScope()).collect(Collectors.toSet())); restoreSourceService = new CcrRestoreSourceService(taskQueue.getThreadPool(), new CcrSettings(Settings.EMPTY, clusterSettings)); } @@ -202,7 +199,10 @@ public void testGetSessionDoesNotLeakFileIfClosed() throws IOException { sessionReader.readFileBytes(files.get(1).name(), new BytesArray(new byte[10])); } + assertTrue(EngineTestCase.hasSnapshottedCommits(IndexShardTestCase.getEngine(indexShard))); restoreSourceService.closeSession(sessionUUID); + assertFalse(EngineTestCase.hasSnapshottedCommits(IndexShardTestCase.getEngine(indexShard))); + closeShards(indexShard); // Exception will be thrown if file is not closed. } diff --git a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java index 2e33f4e130741..588e3a6392e10 100644 --- a/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java +++ b/x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/TypeConverterTests.java @@ -10,17 +10,18 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; -import org.joda.time.DateTime; -import org.joda.time.ReadableDateTime; import java.sql.Timestamp; +import java.time.Clock; +import java.time.Duration; +import java.time.ZoneId; +import java.time.ZonedDateTime; import static org.hamcrest.Matchers.instanceOf; public class TypeConverterTests extends ESTestCase { - public void testFloatAsNative() throws Exception { assertThat(convertAsNative(42.0f, EsType.FLOAT), instanceOf(Float.class)); assertThat(convertAsNative(42.0, EsType.FLOAT), instanceOf(Float.class)); @@ -40,9 +41,9 @@ public void testDoubleAsNative() throws Exception { } public void testTimestampAsNative() throws Exception { - DateTime now = DateTime.now(); + ZonedDateTime now = ZonedDateTime.now(Clock.tick(Clock.system(ZoneId.of("Z")), Duration.ofMillis(1))); assertThat(convertAsNative(now, EsType.DATETIME), instanceOf(Timestamp.class)); - assertEquals(now.getMillis(), ((Timestamp) convertAsNative(now, EsType.DATETIME)).getTime()); + assertEquals(now.toInstant().toEpochMilli(), ((Timestamp) convertAsNative(now, EsType.DATETIME)).getTime()); } private Object convertAsNative(Object value, EsType type) throws Exception { @@ -50,11 +51,7 @@ private Object convertAsNative(Object value, EsType type) throws Exception { XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); builder.field("value"); - if (value instanceof ReadableDateTime) { - builder.value(((ReadableDateTime) value).getMillis()); - } else { - builder.value(value); - } + builder.value(value); builder.endObject(); builder.close(); Object copy = XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2().get("value"); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index ac59b08dbb726..52d53538bb2f8 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -593,20 +593,36 @@ private static void checkGroupingFunctionInGroupBy(LogicalPlan p, Set l // check if the query has a grouping function (Histogram) but no GROUP BY if (p instanceof Project) { Project proj = (Project) p; - proj.projections().forEach(e -> e.forEachDown(f -> + proj.projections().forEach(e -> e.forEachDown(f -> localFailures.add(fail(f, "[{}] needs to be part of the grouping", Expressions.name(f))), GroupingFunction.class)); } else if (p instanceof Aggregate) { - // if it does have a GROUP BY, check if the groupings contain the grouping functions (Histograms) + // if it does have a GROUP BY, check if the groupings contain the grouping functions (Histograms) Aggregate a = (Aggregate) p; a.aggregates().forEach(agg -> agg.forEachDown(e -> { - if (a.groupings().size() == 0 + if (a.groupings().size() == 0 || Expressions.anyMatch(a.groupings(), g -> g instanceof Function && e.functionEquals((Function) g)) == false) { localFailures.add(fail(e, "[{}] needs to be part of the grouping", Expressions.name(e))); } + else { + checkGroupingFunctionTarget(e, localFailures); + } + }, GroupingFunction.class)); + + a.groupings().forEach(g -> g.forEachDown(e -> { + checkGroupingFunctionTarget(e, localFailures); }, GroupingFunction.class)); } } + private static void checkGroupingFunctionTarget(GroupingFunction f, Set localFailures) { + f.field().forEachDown(e -> { + if (e instanceof GroupingFunction) { + localFailures.add(fail(f.field(), "Cannot embed grouping functions within each other, found [{}] in [{}]", + Expressions.name(f.field()), Expressions.name(f))); + } + }); + } + private static void checkFilterOnAggs(LogicalPlan p, Set localFailures) { if (p instanceof Filter) { Filter filter = (Filter) p; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java index 589481247ac39..23747787b820b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java @@ -14,7 +14,6 @@ import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.DateUtils; -import org.joda.time.DateTime; import java.io.IOException; import java.util.ArrayDeque; @@ -132,10 +131,6 @@ private Object unwrapMultiValue(Object values) { if (values instanceof String) { return DateUtils.asDateTime(Long.parseLong(values.toString())); } - // returned by nested types... - if (values instanceof DateTime) { - return DateUtils.asDateTime((DateTime) values); - } } if (values instanceof Long || values instanceof Double || values instanceof String || values instanceof Boolean) { return values; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java index 745cc36e34a57..616c337e64c9a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expression.java @@ -14,9 +14,6 @@ import org.elasticsearch.xpack.sql.util.StringUtils; import java.util.List; -import java.util.Locale; - -import static java.lang.String.format; /** * In a SQL statement, an Expression is whatever a user specifies inside an @@ -39,10 +36,6 @@ public TypeResolution(String message) { this(true, message); } - TypeResolution(String message, Object... args) { - this(true, format(Locale.ROOT, message, args)); - } - private TypeResolution(boolean unresolved, String message) { this.failed = unresolved; this.message = message; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java index 04d660642c8b2..648aff5254561 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java @@ -18,9 +18,9 @@ import java.util.StringJoiner; import java.util.function.Predicate; -import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN; public final class Expressions { @@ -186,7 +186,7 @@ public static TypeResolution typeMustBe(Expression e, String... acceptedTypes) { return predicate.test(e.dataType()) || DataTypes.isNull(e.dataType())? TypeResolution.TYPE_RESOLVED : - new TypeResolution(format(Locale.ROOT, "[%s]%s argument must be [%s], found value [%s] type [%s]", + new TypeResolution(format(null, "[{}]{} argument must be [{}], found value [{}] type [{}]", operationName, paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT), acceptedTypesForErrorMsg(acceptedTypes), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index fe8f5ac9925b1..05069ef42a14e 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -111,15 +111,12 @@ import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypeConversion; import org.elasticsearch.xpack.sql.type.DataTypes; -import org.elasticsearch.xpack.sql.util.DateUtils; import org.elasticsearch.xpack.sql.util.StringUtils; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.DateTimeFormatterBuilder; -import org.joda.time.format.ISODateTimeFormat; import java.time.Duration; +import java.time.LocalTime; import java.time.Period; +import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAmount; import java.util.EnumSet; import java.util.List; @@ -127,9 +124,12 @@ import java.util.Map; import java.util.StringJoiner; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.sql.type.DataTypeConversion.conversionFor; +import static org.elasticsearch.xpack.sql.util.DateUtils.asDateOnly; +import static org.elasticsearch.xpack.sql.util.DateUtils.ofEscapedLiteral; abstract class ExpressionBuilder extends IdentifierBuilder { @@ -791,13 +791,11 @@ public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) { String string = string(ctx.string()); Source source = source(ctx); // parse yyyy-MM-dd - DateTime dt = null; try { - dt = ISODateTimeFormat.date().parseDateTime(string); - } catch(IllegalArgumentException ex) { + return new Literal(source, asDateOnly(string), DataType.DATE); + } catch(DateTimeParseException ex) { throw new ParsingException(source, "Invalid date received; {}", ex.getMessage()); } - return new Literal(source, DateUtils.asDateOnly(dt), DataType.DATE); } @Override @@ -806,10 +804,10 @@ public Literal visitTimeEscapedLiteral(TimeEscapedLiteralContext ctx) { Source source = source(ctx); // parse HH:mm:ss - DateTime dt = null; + LocalTime lt = null; try { - dt = ISODateTimeFormat.hourMinuteSecond().parseDateTime(string); - } catch (IllegalArgumentException ex) { + lt = LocalTime.parse(string, ISO_LOCAL_TIME); + } catch (DateTimeParseException ex) { throw new ParsingException(source, "Invalid time received; {}", ex.getMessage()); } @@ -822,18 +820,11 @@ public Literal visitTimestampEscapedLiteral(TimestampEscapedLiteralContext ctx) Source source = source(ctx); // parse yyyy-mm-dd hh:mm:ss(.f...) - DateTime dt = null; try { - DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .append(ISODateTimeFormat.date()) - .appendLiteral(" ") - .append(ISODateTimeFormat.hourMinuteSecondFraction()) - .toFormatter(); - dt = formatter.parseDateTime(string); - } catch (IllegalArgumentException ex) { + return new Literal(source, ofEscapedLiteral(string), DataType.DATETIME); + } catch (DateTimeParseException ex) { throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage()); } - return new Literal(source, DateUtils.asDateTime(dt), DataType.DATETIME); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java index 0bc02c1ba6f27..9920293794ea0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.parser; import com.carrotsearch.hppc.ObjectShortHashMap; - import org.antlr.v4.runtime.BaseErrorListener; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonToken; @@ -37,8 +36,6 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.StatementContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.StatementDefaultContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.UnquoteIdentifierContext; -import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ValueExpressionContext; -import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ValueExpressionDefaultContext; import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; @@ -242,7 +239,6 @@ static class CircuitBreakerListener extends SqlBaseBaseListener { ENTER_EXIT_RULE_MAPPING.put(StatementDefaultContext.class.getSimpleName(), StatementContext.class.getSimpleName()); ENTER_EXIT_RULE_MAPPING.put(QueryPrimaryDefaultContext.class.getSimpleName(), QueryTermContext.class.getSimpleName()); ENTER_EXIT_RULE_MAPPING.put(BooleanDefaultContext.class.getSimpleName(), BooleanExpressionContext.class.getSimpleName()); - ENTER_EXIT_RULE_MAPPING.put(ValueExpressionDefaultContext.class.getSimpleName(), ValueExpressionContext.class.getSimpleName()); } private boolean insideIn = false; @@ -265,6 +261,9 @@ public void enterEveryRule(ParserRuleContext ctx) { if (ctx.getClass() != UnquoteIdentifierContext.class && ctx.getClass() != QuoteIdentifierContext.class && ctx.getClass() != BackQuotedIdentifierContext.class && + ctx.getClass() != SqlBaseParser.ConstantContext.class && + ctx.getClass() != SqlBaseParser.NumberContext.class && + ctx.getClass() != SqlBaseParser.ValueExpressionContext.class && (insideIn == false || ctx.getClass() != PrimaryExpressionContext.class)) { int currentDepth = depthCounts.putOrAdd(ctx.getClass().getSimpleName(), (short) 1, (short) 1); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java index bc89b0f1e1587..9dbb2a3abb6f6 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.sql.util.DateUtils; import java.time.ZonedDateTime; +import java.time.format.DateTimeParseException; import java.util.Locale; import java.util.function.DoubleFunction; import java.util.function.Function; @@ -546,8 +547,8 @@ private static Function fromString(Function conv return converter.apply(value.toString()); } catch (NumberFormatException e) { throw new SqlIllegalArgumentException(e, "cannot cast [{}] to [{}]", value, to); - } catch (IllegalArgumentException e) { - throw new SqlIllegalArgumentException(e, "cannot cast [{}] to [{}]:{}", value, to, e.getMessage()); + } catch (DateTimeParseException | IllegalArgumentException e) { + throw new SqlIllegalArgumentException(e, "cannot cast [{}] to [{}]: {}", value, to, e.getMessage()); } }; } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java index deb7b9e9703c2..38db3cbe131cf 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java @@ -6,29 +6,35 @@ package org.elasticsearch.xpack.sql.util; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.xpack.sql.proto.StringUtils; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.ISODateTimeFormat; import java.time.Instant; -import java.time.LocalDateTime; +import java.time.LocalDate; import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; public final class DateUtils { - private static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000; - - // TODO: do we have a java.time based parser we can use instead? - private static final DateTimeFormatter UTC_DATE_FORMATTER = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC(); - public static final ZoneId UTC = ZoneId.of("Z"); public static final String DATE_PARSE_FORMAT = "epoch_millis"; + private static final DateTimeFormatter DATE_TIME_ESCAPED_LITERAL_FORMATTER = new DateTimeFormatterBuilder() + .append(ISO_LOCAL_DATE) + .appendLiteral(" ") + .append(ISO_LOCAL_TIME) + .toFormatter().withZone(UTC); + + private static final DateFormatter UTC_DATE_TIME_FORMATTER = DateFormatter.forPattern("date_optional_time").withZone(UTC); + + private static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000L; + private DateUtils() {} /** @@ -56,22 +62,7 @@ public static ZonedDateTime asDateTime(long millis, ZoneId id) { * Parses the given string into a Date (SQL DATE type) using UTC as a default timezone. */ public static ZonedDateTime asDateOnly(String dateFormat) { - return asDateOnly(UTC_DATE_FORMATTER.parseDateTime(dateFormat)); - } - - public static ZonedDateTime asDateOnly(DateTime dateTime) { - LocalDateTime ldt = LocalDateTime.of( - dateTime.getYear(), - dateTime.getMonthOfYear(), - dateTime.getDayOfMonth(), - 0, - 0, - 0, - 0); - - return ZonedDateTime.ofStrict(ldt, - ZoneOffset.ofTotalSeconds(dateTime.getZone().getOffset(dateTime) / 1000), - org.elasticsearch.common.time.DateUtils.dateTimeZoneToZoneId(dateTime.getZone())); + return LocalDate.parse(dateFormat, ISO_LOCAL_DATE).atStartOfDay(UTC); } public static ZonedDateTime asDateOnly(ZonedDateTime zdt) { @@ -82,25 +73,13 @@ public static ZonedDateTime asDateOnly(ZonedDateTime zdt) { * Parses the given string into a DateTime using UTC as a default timezone. */ public static ZonedDateTime asDateTime(String dateFormat) { - return asDateTime(UTC_DATE_FORMATTER.parseDateTime(dateFormat)); + return DateFormatters.from(UTC_DATE_TIME_FORMATTER.parse(dateFormat)).withZoneSameInstant(UTC); } - public static ZonedDateTime asDateTime(DateTime dateTime) { - LocalDateTime ldt = LocalDateTime.of( - dateTime.getYear(), - dateTime.getMonthOfYear(), - dateTime.getDayOfMonth(), - dateTime.getHourOfDay(), - dateTime.getMinuteOfHour(), - dateTime.getSecondOfMinute(), - dateTime.getMillisOfSecond() * 1_000_000); - - return ZonedDateTime.ofStrict(ldt, - ZoneOffset.ofTotalSeconds(dateTime.getZone().getOffset(dateTime) / 1000), - org.elasticsearch.common.time.DateUtils.dateTimeZoneToZoneId(dateTime.getZone())); + public static ZonedDateTime ofEscapedLiteral(String dateFormat) { + return ZonedDateTime.parse(dateFormat, DATE_TIME_ESCAPED_LITERAL_FORMATTER.withZone(UTC)); } - public static String toString(ZonedDateTime dateTime) { return StringUtils.toString(dateTime); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 558d92351b069..eec483ca219b8 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -566,10 +566,20 @@ public void testGroupByScalarOnTopOfGrouping() { } public void testAggsInHistogram() { - assertEquals("1:47: Cannot use an aggregate [MAX] for grouping", - error("SELECT MAX(date) FROM test GROUP BY HISTOGRAM(MAX(int), 1)")); + assertEquals("1:37: Cannot use an aggregate [MAX] for grouping", + error("SELECT MAX(date) FROM test GROUP BY MAX(int)")); } - + + public void testGroupingsInHistogram() { + assertEquals( + "1:47: Cannot embed grouping functions within each other, found [HISTOGRAM(int, 1)] in [HISTOGRAM(HISTOGRAM(int, 1), 1)]", + error("SELECT MAX(date) FROM test GROUP BY HISTOGRAM(HISTOGRAM(int, 1), 1)")); + } + + public void testCastInHistogram() { + accept("SELECT MAX(date) FROM test GROUP BY HISTOGRAM(CAST(int AS LONG), 1)"); + } + public void testHistogramNotInGrouping() { assertEquals("1:8: [HISTOGRAM(date, INTERVAL 1 MONTH)] needs to be part of the grouping", error("SELECT HISTOGRAM(date, INTERVAL 1 MONTH) AS h FROM test")); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java index 2ae6e571ac9d2..4323cce234c54 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java @@ -7,22 +7,15 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.datetime; import org.elasticsearch.xpack.sql.util.DateUtils; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import java.time.ZonedDateTime; -import static org.junit.Assert.assertEquals; - public class DateTimeTestUtils { private DateTimeTestUtils() {} public static ZonedDateTime dateTime(int year, int month, int day, int hour, int minute) { - DateTime dateTime = new DateTime(year, month, day, hour, minute, DateTimeZone.UTC); - ZonedDateTime zdt = ZonedDateTime.of(year, month, day, hour, minute, 0, 0, DateUtils.UTC); - assertEquals(dateTime.getMillis() / 1000, zdt.toEpochSecond()); - return zdt; + return ZonedDateTime.of(year, month, day, hour, minute, 0, 0, DateUtils.UTC); } public static ZonedDateTime dateTime(long millisSinceEpoch) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java index 8cbb0b528e9a6..bc3ea049c242e 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java @@ -175,7 +175,8 @@ public void testDateLiteral() { public void testDateLiteralValidation() { ParsingException ex = expectThrows(ParsingException.class, () -> dateLiteral("2012-13-01")); - assertEquals("line 1:2: Invalid date received; Cannot parse \"2012-13-01\": Value 13 for monthOfYear must be in the range [1,12]", + assertEquals("line 1:2: Invalid date received; Text '2012-13-01' could not be parsed: " + + "Invalid value for MonthOfYear (valid values 1 - 12): 13", ex.getMessage()); } @@ -186,7 +187,8 @@ public void testTimeLiteralUnsupported() { public void testTimeLiteralValidation() { ParsingException ex = expectThrows(ParsingException.class, () -> timeLiteral("10:10:65")); - assertEquals("line 1:2: Invalid time received; Cannot parse \"10:10:65\": Value 65 for secondOfMinute must be in the range [0,59]", + assertEquals("line 1:2: Invalid time received; Text '10:10:65' could not be parsed: " + + "Invalid value for SecondOfMinute (valid values 0 - 59): 65", ex.getMessage()); } @@ -198,7 +200,7 @@ public void testTimestampLiteral() { public void testTimestampLiteralValidation() { ParsingException ex = expectThrows(ParsingException.class, () -> timestampLiteral("2012-01-01T10:01:02.3456")); assertEquals( - "line 1:2: Invalid timestamp received; Invalid format: \"2012-01-01T10:01:02.3456\" is malformed at \"T10:01:02.3456\"", + "line 1:2: Invalid timestamp received; Text '2012-01-01T10:01:02.3456' could not be parsed at index 10", ex.getMessage()); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java index dd44a8e464ae4..8b275468f482a 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/SqlParserTests.java @@ -294,9 +294,18 @@ public void testLimitToPreventStackOverflowFromLargeComplexSubselectTree() { } public void testLimitStackOverflowForInAndLiteralsIsNotApplied() { - int noChildren = 100_000; + int noChildren = 10_000; LogicalPlan plan = parseStatement("SELECT * FROM t WHERE a IN(" + - Joiner.on(",").join(nCopies(noChildren, "a + b")) + ")"); + Joiner.on(",").join(nCopies(noChildren, "a + 10")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-(-a - 10)")) + "," + + Joiner.on(",").join(nCopies(noChildren, "20")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-20")) + "," + + Joiner.on(",").join(nCopies(noChildren, "20.1234")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-20.4321")) + "," + + Joiner.on(",").join(nCopies(noChildren, "1.1234E56")) + "," + + Joiner.on(",").join(nCopies(noChildren, "-1.4321E-65")) + "," + + Joiner.on(",").join(nCopies(noChildren, "'foo'")) + "," + + Joiner.on(",").join(nCopies(noChildren, "'bar'")) + ")"); assertEquals(With.class, plan.getClass()); assertEquals(Project.class, ((With) plan).child().getClass()); @@ -305,8 +314,17 @@ public void testLimitStackOverflowForInAndLiteralsIsNotApplied() { assertEquals(In.class, filter.condition().getClass()); In in = (In) filter.condition(); assertEquals("?a", in.value().toString()); - assertEquals(noChildren, in.list().size()); - assertThat(in.list().get(0).toString(), startsWith("Add[?a,?b]")); + assertEquals(noChildren * 2 + 8, in.list().size()); + assertThat(in.list().get(0).toString(), startsWith("Add[?a,10]#")); + assertThat(in.list().get(noChildren).toString(), startsWith("Neg[Sub[Neg[?a]#")); + assertEquals("20", in.list().get(noChildren * 2).toString()); + assertEquals("-20", in.list().get(noChildren * 2 + 1).toString()); + assertEquals("20.1234", in.list().get(noChildren * 2 + 2).toString()); + assertEquals("-20.4321", in.list().get(noChildren * 2 + 3).toString()); + assertEquals("1.1234E56", in.list().get(noChildren * 2 + 4).toString()); + assertEquals("-1.4321E-65", in.list().get(noChildren * 2 + 5).toString()); + assertEquals("'foo'=foo", in.list().get(noChildren * 2 + 6).toString()); + assertEquals("'bar'=bar", in.list().get(noChildren * 2 + 7).toString()); } public void testDecrementOfDepthCounter() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java index 686c97b91a06b..a72f9ee7f1244 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java @@ -150,10 +150,10 @@ public void testConversionToDate() { Conversion conversion = conversionFor(KEYWORD, to); assertNull(conversion.convert(null)); - assertEquals(date(0L), conversion.convert("1970-01-01T00:10:01Z")); - assertEquals(date(1483228800000L), conversion.convert("2017-01-01T00:11:00Z")); - assertEquals(date(-1672531200000L), conversion.convert("1917-01-01T00:11:00Z")); - assertEquals(date(18000000L), conversion.convert("1970-01-01T03:10:20-05:00")); + assertEquals(date(0L), conversion.convert("1970-01-01")); + assertEquals(date(1483228800000L), conversion.convert("2017-01-01")); + assertEquals(date(-1672531200000L), conversion.convert("1917-01-01")); + assertEquals(date(18000000L), conversion.convert("1970-01-01")); // double check back and forth conversion @@ -162,7 +162,7 @@ public void testConversionToDate() { Conversion back = conversionFor(KEYWORD, DATE); assertEquals(DateUtils.asDateOnly(zdt), back.convert(forward.convert(zdt))); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + assertEquals("cannot cast [0xff] to [date]: Text '0xff' could not be parsed at index 0", e.getMessage()); } } @@ -199,6 +199,7 @@ public void testConversionToDateTime() { Conversion conversion = conversionFor(KEYWORD, to); assertNull(conversion.convert(null)); + assertEquals(dateTime(0L), conversion.convert("1970-01-01")); assertEquals(dateTime(1000L), conversion.convert("1970-01-01T00:00:01Z")); assertEquals(dateTime(1483228800000L), conversion.convert("2017-01-01T00:00:00Z")); assertEquals(dateTime(1483228800000L), conversion.convert("2017-01-01T00:00:00Z")); @@ -211,7 +212,8 @@ public void testConversionToDateTime() { Conversion back = conversionFor(KEYWORD, DATETIME); assertEquals(dt, back.convert(forward.convert(dt))); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [datetime]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + assertEquals("cannot cast [0xff] to [datetime]: failed to parse date field [0xff] with format [date_optional_time]", + e.getMessage()); } }