Merge branch 'master' into feature/data_stream_support_routing

* master: (128 commits)
  Mute DieWithDignityIT (elastic#77283)
  Fix randomization in MlNodeShutdownIT (elastic#77281)
  Add target_node_name for REPLACE shutdown type (elastic#77151)
  [DOCS] Adds information about version compatibility headers (elastic#77096)
  Fix template equals when mappings are wrapped (elastic#77008)
  Fix TextFieldMapper Retaining a Reference to its Builder (elastic#77251)
  Move die with dignity to be a test module (elastic#77136)
  Update task names for rest compatiblity (elastic#75267)
  [ML] adjusting bwc serialization for elastic#77256 (elastic#77257)
  Move `index.hidden` from Static to Dynamic settings (elastic#77218)
  Handle cgroups v2 in `OsProbe` (elastic#77128)
  Choose postings format from FieldMapper instead of MappedFieldType (elastic#77234)
  Add segment sorter for data streams (elastic#75195)
  Update skip after backport (elastic#77212)
  [ML] adding new defer_definition_decompression parameter to put trained model API (elastic#77189)
  [ML] Fix bug in inference stats persister for when feature reset is called
  Only check replicas in cancelling existing recoveries. (elastic#60564)
  Format `AbstractFilteringTestCase` (elastic#77217)
  [DOCS] Fixes line breaks. (elastic#77248)
  Convert 'routing' values in REST API tests to strings
  ...

# Conflicts:
#	server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

.ci/bwcVersions

@@ -40,6 +40,7 @@ BWC_VERSION:
   - "7.13.4"
   - "7.14.0"
   - "7.14.1"
+  - "7.14.2"
   - "7.15.0"
   - "7.16.0"
   - "8.0.0"

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix.yml

@@ -36,6 +36,7 @@
             - centos-8-packaging
             - debian-9-packaging
             - debian-10-packaging
+            - debian-11-packaging
             - opensuse-15-1-packaging
             - oraclelinux-7-packaging
             - oraclelinux-8-packaging

.git-blame-ignore-revs

@@ -14,3 +14,6 @@
 
 # Format more snapshot / restore relate projects
 559c4e6ef4f9173bbb59043bacd0ac36c7281040
+
+# Format aggregations and related code (server and x-pack)
+d71544976608bdb53fa4d29521fb328e1033ee2f

BUILDING.md

@@ -0,0 +1,120 @@
+Building Elasticsearch with Gradle
+=============================
+
+Elasticsearch is built using the [Gradle](https://gradle.org/) open source build tools.
+
+This document provides a general guidelines for using and working on the elasticsearch build logic.
+
+## Build logic organisation
+
+The Elasticsearch project contains 3 build-related projects that are included into the Elasticsearch build as a [composite build](https://docs.gradle.org/current/userguide/composite_builds.html).
+
+### `build-conventions`
+
+This project contains build conventions that are applied to all elasticsearch projects.
+
+### `build-tools`
+
+This project contains all build logic that we publish for third party elasticsearch plugin authors.
+We provide the following plugins:
+
+- `elasticsearch.esplugin` - A gradle plugin for building an elasticsearch plugin.
+- `elasticsearch.testclusters` - A gradle plugin for setting up es clusters for testing within a build.
+
+This project is published as part of the elasticsearch release and accessible by
+`org.elasticsearch.gradle:build-tools:<versionNumber>`.
+These build tools are also used by the `elasticsearch-hadoop` project maintained by elastic.
+
+### `build-tools-internal`
+
+This project contains all elasticsearch project specific build logic that is not meant to be shared
+with other internal or external projects.
+
+## Build guidelines
+
+This is an intentionally small set of guidelines to build users and authors
+to ensure we keep the build consistent. We also publish elasticsearch build logic
+as `build-tools` to be usuable by thirdparty elasticsearch plugin authors. This is
+also used by other elastic teams like `elasticsearch-hadoop`.
+Breaking changes should therefore be avoided and an appropriate deprecation cycle
+should be followed.
+
+### Stay up to date
+
+The elasticsearch build usually uses the latest Gradle GA release. We stay as close to the
+latest Gradle releases as possible. In certain cases an update is blocked by a breaking behaviour
+in Gradle. We're usually in contact with the gradle team here or working on a fix
+in our build logic to resolve this.
+
+**The Elasticsearch build will fail if any deprecated Gradle API is used.**
+
+### Make a change in the build
+
+There are a few guidelines to follow that should make your life easier to make changes to the elasticsearch build.
+Please add a member of the `es-delivery` team as a reviewer if you're making non-trivial changes to the build.
+
+#### Custom Plugin and Task implementations
+
+Build logic that is used across multiple subprojects should considered to be moved into a Gradle plugin with according Gradle task implmentation.
+Elasticsearch specific build logic is located in the `build-tools-internal` subproject including integration tests.
+
+- Gradle plugins and Tasks should be written in Java
+- We use a groovy and spock for setting up Gradle integration tests.
+  (see https://github.com/elastic/elasticsearch/blob/master/build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy)
+
+#### Declaring tasks
+
+The elasticsearch build makes use of the [task avoidance API](https://docs.gradle.org/current/userguide/task_configuration_avoidance.html) to keep the configuration time of the build low.
+
+When declaring tasks (in build scripts or custom plugins) this means that we want to _register_ a task like:
+
+    tasks.register('someTask') { ... }
+
+instead of eagerly _creating_ the task:
+
+    task someTask { ... }
+
+The major difference between these two syntaxes is, that the configuration block of an registered task will only be executed when the task is actually created due to the build requires that task to run. The configuration block of an eagerly created tasks will be executed immediately.
+
+By actually doing less in the gradle configuration time as only creating tasks that are requested as part of the build and by only running the configurations for those requested tasks, using the task avoidance api contributes a major part in keeping our build fast.
+
+#### Adding additional integration tests
+
+Additional integration tests for a certain elasticsearch modules that are specific to certain cluster configuration can be declared in a separate so called `qa` subproject of your module.
+
+The benefit of a dedicated project for these tests are:
+- `qa` projects are dedicated two specific usecases and easier to maintain
+- It keeps the specific test logic separated from the common test logic.
+- You can run those tests in parallel to other projects of the build.
+
+#### Using test fixtures
+
+Sometimes we want to share test fixtures to setup the code under test across multiple projects. There are basically two ways doing so.
+
+Ideally we would use the build-in [java-test-fixtures](https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures) gradle plugin.
+This plugin relies on having a separate sourceSet for the test fixtures code.
+
+In the elasticsearch codebase we have test fixtures and actual tests within the same sourceSet. Therefore we introduced the `elasticsearch.internal-test-artifact` plugin to provides another build artifact of your project based on the `test` sourceSet.
+
+
+This artifact can be resolved by the consumer project as shown in the example below:
+
+```
+dependencies {
+  //add the test fixtures of `:providing-project` to testImplementation configuration.
+  testImplementation(testArtifact(project(":fixture-providing-project')))
+}
+```
+
+This test artifact mechanism makes use of the concept of [component capabilities](https://docs.gradle.org/current/userguide/component_capabilities.html)
+similar to how the gradle build-in `java-test-fixtures` plugin works.
+
+`testArtifact` is a shortcut declared in the elasticsearch build. Alternatively you can declare the dependency via
+
+```
+dependencies {
+  testImplementation(project(":fixture-providing-project')) {
+    requireCapabilities("org.elasticsearch.gradle:fixture-providing-project-test-artifacts")
+  }
+}
+```

CONTRIBUTING.md

@@ -69,6 +69,8 @@ cycle.
 * Lines that are not part of your change should not be edited (e.g. don't format
   unchanged lines, don't reorder existing imports)
 * Add the appropriate [license headers](#license-headers) to any new files
+* For contributions involving the elasticsearch build you can find (details about the build setup in the
+* [BUILDING](BUILDING.md) file
 
 ### Submitting your changes
 

build-tools-internal/build.gradle

@@ -159,8 +159,8 @@ gradlePlugin {
       implementationClass = 'org.elasticsearch.gradle.internal.rest.compat.YamlRestCompatTestPlugin'
     }
     yamlRestTest {
-      id = 'elasticsearch.yaml-rest-test'
-      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.YamlRestTestPlugin'
+      id = 'elasticsearch.internal-yaml-rest-test'
+      implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
     }
   }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy

@@ -63,6 +63,6 @@ class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest {
         then:
         result.task(":checkoutBwcBranch").outcome == TaskOutcome.SUCCESS
         result.task(":consumer:register").outcome == TaskOutcome.SUCCESS
-        normalized(result.output).contains("/cloned/build/checkout")
+        result.output.contains("./build/checkout")
     }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveCheckPluginFuncTest.groovy

@@ -80,11 +80,16 @@ unknown license content line 2
             }
         """
 
+
+
         when:
-        def result = gradleRunner(":darwin-tar:check").buildAndFail()
+        def runner = gradleRunner(":darwin-tar:check")
+        println "{runner.getClass()} = ${runner.getClass()}"
+        def result = runner.buildAndFail()
+        println "result.getClass() = ${result.getClass()}"
         then:
         result.task(":darwin-tar:checkLicense").outcome == TaskOutcome.FAILED
-        normalized(result.output).contains("> expected line [2] in " +
+        result.output.contains("> expected line [2] in " +
                 "[./darwin-tar/build/tar-extracted/elasticsearch-${VersionProperties.getElasticsearch()}/LICENSE.txt] " +
                 "to be [elastic license coorp stuff line 2] but was [unknown license content line 2]")
     }
@@ -110,7 +115,7 @@ Copyright 2009-2018 Acme Coorp"""
         def result = gradleRunner(":darwin-tar:checkNotice").buildAndFail()
         then:
         result.task(":darwin-tar:checkNotice").outcome == TaskOutcome.FAILED
-        normalized(result.output).contains("> expected line [2] in " +
+        result.output.contains("> expected line [2] in " +
                 "[./darwin-tar/build/tar-extracted/elasticsearch-${VersionProperties.getElasticsearch()}/NOTICE.txt] " +
                 "to be [Copyright 2009-2021 Elasticsearch] but was [Copyright 2009-2018 Acme Coorp]")
     }
@@ -146,8 +151,7 @@ Copyright 2009-2021 Elasticsearch"""
         def result = gradleRunner(":darwin-tar:check").buildAndFail()
         then:
         result.task(":darwin-tar:checkMlCppNotice").outcome == TaskOutcome.FAILED
-        normalized(result.output)
-                .contains("> expected [./darwin-tar/build/tar-extracted/elasticsearch-" +
+        result.output.contains("> expected [./darwin-tar/build/tar-extracted/elasticsearch-" +
                         "${VersionProperties.getElasticsearch()}/modules/x-pack-ml/NOTICE.txt " +
                         "to contain [foo license] but it did not")
     }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy

@@ -17,6 +17,7 @@ import spock.lang.Unroll
 /*
  * Test is ignored on ARM since this test case tests the ability to build certain older BWC branches that we don't support on ARM
  */
+
 @IgnoreIf({ Architecture.current() == Architecture.AARCH64 })
 class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
@@ -138,9 +139,8 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
 
         and: "assemble task triggered"
         result.output.contains("[7.10.1] > Task :distribution:archives:darwin-tar:assemble")
-        normalized(result.output)
-                .contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-7.10/distribution/archives/darwin-tar/" +
-                        "build/distributions/elasticsearch-7.10.1-SNAPSHOT-darwin-x86_64.tar.gz")
+        result.output.contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-7.10/distribution/archives/darwin-tar/" +
+                "build/distributions/elasticsearch-7.10.1-SNAPSHOT-darwin-x86_64.tar.gz")
     }
 
     def "bwc expanded distribution folder can be resolved as bwc project artifact"() {
@@ -177,11 +177,9 @@ class InternalDistributionBwcSetupPluginFuncTest extends AbstractGitAwareGradleF
         result.task(":distribution:bwc:minor:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
         and: "assemble task triggered"
         result.output.contains("[7.12.0] > Task :distribution:archives:darwin-tar:extractedAssemble")
-        normalized(result.output)
-                .contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-7.x/" +
+        result.output.contains("expandedRootPath /distribution/bwc/minor/build/bwc/checkout-7.x/" +
                         "distribution/archives/darwin-tar/build/install")
-        normalized(result.output)
-                .contains("nested folder /distribution/bwc/minor/build/bwc/checkout-7.x/" +
+        result.output.contains("nested folder /distribution/bwc/minor/build/bwc/checkout-7.x/" +
                         "distribution/archives/darwin-tar/build/install/elasticsearch-7.12.0-SNAPSHOT")
     }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -182,7 +182,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         }
 
         then:
-        normalized(result.output).contains("Unpacking $expectedArchiveName using $transformType") == false
+        result.output.contains("Unpacking $expectedArchiveName using $transformType") == false
 
         where:
         platform  | expectedArchiveName       | transformType
@@ -214,8 +214,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
             return "/java/GA/" + versionPath + "/GPL/" + filename;
         } else if (vendor.equals(VENDOR_AZUL)) {
             final String module = isMac(platform) ? "macosx" : platform;
-            // we only test zulu 15 darwin aarch64 for now
-            return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu16.28.11-ca-jdk16.0.0-${module}_${arch}.tar.gz";
+            // we only test zulu 16 darwin aarch64 for now
+            return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu16.32.15-ca-jdk16.0.2-${module}_${arch}.tar.gz";
         }
     }
 

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy

@@ -33,7 +33,7 @@ class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradleFuncTest {
         assertOutputContains(result.output, "> Check failed. License header problems were found. Full details: ./build/reports/licenseHeaders/rat.xml")
         assertOutputContains(result.output, "./src/main/java/org/acme/UnknownLicensed.java")
         assertOutputContains(result.output, "./src/main/java/org/acme/UnapprovedLicensed.java")
-        normalized(result.output).contains("./src/main/java/org/acme/DualLicensed.java") == false
+        result.output.contains("./src/main/java/org/acme/DualLicensed.java") == false
     }
 
     def "can filter source files"() {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rerun/InternalTestRerunPluginFuncTest.groovy

@@ -57,7 +57,7 @@ class InternalTestRerunPluginFuncTest extends AbstractGradleFuncTest {
         def result = gradleRunner("test").buildAndFail()
         result.output.contains("total executions: 2") == false
         and: "no jvm system exit tracing provided"
-        normalized(result.output).contains("""Test jvm exited unexpectedly.
+        result.output.contains("""Test jvm exited unexpectedly.
 Test jvm system exit trace:""") == false
     }
 
@@ -119,7 +119,7 @@ Test jvm system exit trace:""") == false
         result.output.contains("AnotherTest6 total executions: 2")
         // triggered only in the second overall run
         and: 'Tracing is provided'
-        normalized(result.output).contains("""================
+        result.output.contains("""================
 Test jvm exited unexpectedly.
 Test jvm system exit trace (run: 1)
 Gradle Test Executor 1 > AnotherTest6 > someTest
@@ -200,11 +200,11 @@ Gradle Test Executor 1 > AnotherTest6 > someTest
         result.output.contains("JdkKillingTest total executions: 5")
         result.output.contains("Max retries(4) hit")
         and: 'Tracing is provided'
-        normalized(result.output).contains("Test jvm system exit trace (run: 1)")
-        normalized(result.output).contains("Test jvm system exit trace (run: 2)")
-        normalized(result.output).contains("Test jvm system exit trace (run: 3)")
-        normalized(result.output).contains("Test jvm system exit trace (run: 4)")
-        normalized(result.output).contains("Test jvm system exit trace (run: 5)")
+        result.output.contains("Test jvm system exit trace (run: 1)")
+        result.output.contains("Test jvm system exit trace (run: 2)")
+        result.output.contains("Test jvm system exit trace (run: 3)")
+        result.output.contains("Test jvm system exit trace (run: 4)")
+        result.output.contains("Test jvm system exit trace (run: 5)")
     }
 
     private String testMethodContent(boolean withSystemExit, boolean fail, int timesFailing = 1) {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestTestPluginFuncTest.groovy → build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/InternalYamlRestTestPluginFuncTest.groovy

@@ -11,13 +11,13 @@ package org.elasticsearch.gradle.internal.test.rest
 import org.elasticsearch.gradle.fixtures.AbstractRestResourcesFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
-class YamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
+class InternalYamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTest does nothing when there are no tests"() {
         given:
         buildFile << """
         plugins {
-          id 'elasticsearch.yaml-rest-test'
+          id 'elasticsearch.internal-yaml-rest-test'
         }
         """
 
@@ -34,7 +34,7 @@ class YamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         given:
         internalBuild()
         buildFile << """
-            apply plugin: 'elasticsearch.yaml-rest-test'
+            apply plugin: 'elasticsearch.internal-yaml-rest-test'
 
             dependencies {
                yamlRestTestImplementation "junit:junit:4.12"
@@ -67,8 +67,8 @@ class YamlRestTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         file("/build/classes/java/yamlRestTest/MockIT.class").exists()
 
         // check that our copied specs and tests are on the yamlRestTest classpath
-        normalized(result.output).contains("./build/restResources/yamlSpecs")
-        normalized(result.output).contains("./build/restResources/yamlTests")
+        result.output.contains("./build/restResources/yamlSpecs")
+        result.output.contains("./build/restResources/yamlTests")
 
         when:
         result = gradleRunner("yamlRestTest").build()