Merge branch 'upstream/master' into rollup-index-improvement

* upstream/master: (167 commits)
  Mute FrozenSearchableSnapshotsIntegTests#testCreateAndRestorePartialSearchableSnapshot
  Mute LdapSessionFactoryTests#testSslTrustIsReloaded
  Fix spotless violation from last commit
  Mute GeoGridTilerTestCase#testGeoGridSetValuesBoundingBoxes_UnboundedGeoShapeCellValues
  Small formatting clean up (elastic#84144)
  Always re-run Feature migrations which have encountered errors (elastic#83918)
  [DOCS] Clarify `orientation` usage for WKT and GeoJSON polygons (elastic#84025)
  Group field caps response by index mapping hash (elastic#83494)
  Shrink join queries in slow log (elastic#83914)
  TSDB: Reject the nested object fields that are configured time_series_dimension (elastic#83920)
  [DOCS] Remove note about partial response from Bulk API docs (elastic#84053)
  Allow regular data streams to be migrated to tsdb data streams. (elastic#83843)
  [DOCS] Fix `ignore_unavailable` parameter definition (elastic#84071)
  Make Metadata extend AbstractCollection (elastic#83791)
  Add API specs for OpenID Connect APIs
  Revert "Clean up for superuser role name references (elastic#83627)" (elastic#84096)
  Update Lucene analysis base url (elastic#84094)
  Avoid null threadContext in ResultDeduplicator (elastic#84093)
  Use static empty store files metadata (elastic#84034)
  Preserve context in snapshotDeletionListeners (elastic#84089)
  ...

# Conflicts:
#	x-pack/plugin/rollup/build.gradle

.ci/bwcVersions

@@ -51,5 +51,6 @@ BWC_VERSION:
   - "7.17.0"
   - "7.17.1"
   - "8.0.0"
+  - "8.0.1"
   - "8.1.0"
   - "8.2.0"

.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml → .ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc-snapshots.yml

@@ -1,7 +1,9 @@
 ---
-jjbb-template: generic-gradle-unix.yml
+jjbb-template: matrix-gradle-unix.yml
 vars:
-  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc-snapshots
   - job-display-name: "elastic / elasticsearch # %BRANCH% - intake bwc"
   - job-description: Elasticsearch %BRANCH% branch intake backwards compatibility checks.
-  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files bwcTestSnapshots"
+  - matrix-yaml-file: ".ci/snapshotBwcVersions"
+  - matrix-variable: BWC_VERSION
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files v$BWC_VERSION#bwcTest"

.ci/jobs.t/elastic+elasticsearch+intake.yml

@@ -42,7 +42,7 @@
               kill-phase-on: NEVER
               current-parameters: true
               git-revision: true
-            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc-snapshots
               kill-phase-on: NEVER
               current-parameters: true
               git-revision: true

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml

@@ -14,6 +14,7 @@
           type: label-expression
           name: os
           values:
+            - "almalinux-8-aarch64&&immutable"
             - "ubuntu-1804-aarch64&&immutable"
     builders:
       - inject:

.ci/jobs.t/elastic+elasticsearch+pull-request+bwc.yml → .ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml

@@ -1,9 +1,11 @@
 ---
 - job:
-    name: "elastic+elasticsearch+pull-request+bwc"
+    name: "elastic+elasticsearch+pull-request+bwc-snapshots"
     display-name: "elastic / elasticsearch - pull request bwc"
     description: "Testing of Elasticsearch pull requests - bwc"
-    workspace: "/dev/shm/elastic+elasticsearch+pull-request+bwc"
+    project-type: matrix
+    node: master
+    child-workspace: "/dev/shm/elastic+elasticsearch+pull-request+bwc"
     scm:
       - git:
           refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
@@ -23,6 +25,16 @@
           black-list-labels:
             - '>test-mute'
             - 'test-full-bwc'
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "general-purpose"
+      - axis:
+          type: yaml
+          filename: ".ci/snapshotBwcVersions"
+          name: "BWC_VERSION"
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'
@@ -34,4 +46,4 @@
             JAVA16_HOME=$HOME/.java/openjdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
-          $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed bwcTestSnapshots
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dignore.tests.seed v$BWC_VERSION#bwcTest

.ci/snapshotBwcVersions

@@ -0,0 +1,5 @@
+BWC_VERSION:
+  - "7.17.1"
+  - "8.0.1"
+  - "8.1.0"
+  - "8.2.0"

CHANGELOG.md

@@ -0,0 +1,3 @@
+# Elasticsearch Changlog
+
+Please see the [release notes](https://www.elastic.co/guide/en/elasticsearch/reference/current/es-release-notes.html) in the reference manual.

CONTRIBUTING.md

@@ -112,11 +112,11 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-JDK 16 is required to build Elasticsearch. You must have a JDK 16 installation
+JDK 17 is required to build Elasticsearch. You must have a JDK 17 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
-your JDK 16 installation. By default, tests use the same runtime as `JAVA_HOME`.
+your JDK 17 installation. By default, tests use the same runtime as `JAVA_HOME`.
 However, since Elasticsearch supports JDK 11, the build supports compiling with
-JDK 16 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
+JDK 17 and testing on a JDK 11 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 11 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 11.
 
@@ -151,9 +151,9 @@ and then run `curl` in another window like this:
 ### Importing the project into IntelliJ IDEA
 
 The minimum IntelliJ IDEA version required to import the Elasticsearch project is 2020.1
-Elasticsearch builds using Java 16. When importing into IntelliJ you will need
+Elasticsearch builds using Java 17. When importing into IntelliJ you will need
 to define an appropriate SDK. The convention is that **this SDK should be named
-"16"** so that the project import will detect it automatically. For more details
+"17"** so that the project import will detect it automatically. For more details
 on defining an SDK in IntelliJ please refer to [their documentation](https://www.jetbrains.com/help/idea/sdk.html#define-sdk).
 SDK definitions are global, so you can add the JDK from any project, or after
 project import. Importing with a missing JDK will still work, IntelliJ will

benchmarks/README.md

@@ -5,7 +5,7 @@ This directory contains the microbenchmark suite of Elasticsearch. It relies on
 ## Purpose
 
 We do not want to microbenchmark everything but the kitchen sink and should typically rely on our
-[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with
+[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/) with
 [Rally](http://github.com/elastic/rally). Microbenchmarks are intended to spot performance regressions in performance-critical components.
 The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR.
 

benchmarks/src/main/resources/org/elasticsearch/benchmark/xcontent/monitor_cluster_stats.json

@@ -1233,6 +1233,11 @@
             "total" : 0,
             "failed" : 0
           }
+        },
+        "archive" : {
+          "available" : false,
+          "enabled" : true,
+          "indices_count" : 0
         }
       }
     }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesLoader.java

@@ -43,7 +43,6 @@ protected static void loadBuildSrcVersion(Properties loadedProps, ProviderFactor
             );
         }
         String qualifier = providers.systemProperty("build.version_qualifier")
-                .forUseAtConfigurationTime()
                 .getOrElse("");
         if (qualifier.isEmpty() == false) {
             if (qualifier.matches("(alpha|beta|rc)\\d+") == false) {
@@ -52,7 +51,6 @@ protected static void loadBuildSrcVersion(Properties loadedProps, ProviderFactor
             elasticsearch += "-" + qualifier;
         }
         final String buildSnapshotSystemProperty = providers.systemProperty("build.snapshot")
-                .forUseAtConfigurationTime()
                 .getOrElse("true");
         switch (buildSnapshotSystemProperty) {
             case "true":

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesPlugin.java

@@ -32,6 +32,6 @@ public void apply(Project project) {
                 .registerIfAbsent("versions", VersionPropertiesBuildService.class, spec -> {
             spec.getParameters().getInfoPath().set(infoPath);
         });
-        project.getExtensions().add("versions", serviceProvider.forUseAtConfigurationTime().get().getProperties());
+        project.getExtensions().add("versions", serviceProvider.get().getProperties());
     }
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/ParallelDetector.java

@@ -78,7 +78,7 @@ public static int findDefaultParallel(Project project) {
     }
 
     private static boolean isMac(ProviderFactory providers) {
-        return providers.systemProperty("os.name").forUseAtConfigurationTime().getOrElse("").startsWith("Mac");
+        return providers.systemProperty("os.name").getOrElse("").startsWith("Mac");
     }
 
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

@@ -25,6 +25,7 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.provider.ListProperty;
 import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
@@ -68,6 +69,7 @@ public LicenseHeadersTask() {
      * constructor can write to it.
      */
     @InputFiles
+    @IgnoreEmptyDirectories
     @SkipWhenEmpty
     @PathSensitive(PathSensitivity.RELATIVE)
     public List<FileCollection> getJavaFiles() {

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=c9490e938b221daf0094982288e4038deed954a3f12fb54cbf270ddf4e37d879
+distributionSha256Sum=cd5c2958a107ee7f0722004a12d0f8559b4564c34daad7df06cffd4d12a426d0

build-tools-internal/performance/elasticsearch-build-tool-update.scenarios

@@ -69,10 +69,9 @@ precommit_master {
 
 single_project_branch {
     title = "single project (@testGitCommit@)"
-    cleanup-tasks = [":server:clean"]
-    tasks = [":server:spotlessApply", ":server:precommit"]
+    tasks = [":server:precommit"]
     gradle-args = ["--no-scan"]
-    apply-abi-change-to = "server/src/main/java/org/elasticsearch/Build.java"
+    apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java"
     run-using = cli // value can be "cli" or "tooling-api"
     daemon = warm // value can be "warm", "cold", or "none"
     warm-ups = 5
@@ -87,10 +86,9 @@ single_project_branch {
 
 single_project_master {
     title = "single project (master)"
-    cleanup-tasks = [":server:clean"]
-    tasks = [":server:spotlessApply", ":server:precommit"]
+    tasks = [":server:precommit"]
     gradle-args = ["--no-scan"]
-    apply-abi-change-to = "server/src/main/java/org/elasticsearch/Build.java"
+    apply-abi-change-to = "server/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java"
     run-using = cli // value can be "cli" or "tooling-api"
     daemon = warm // value can be "warm", "cold", or "none"
     warm-ups = 5

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy

@@ -30,7 +30,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
     def "yamlRestTestVxCompatTest does nothing when there are no tests"() {
         given:
-        addSubProject(":distribution:bwc:bugfix") << """
+        addSubProject(":distribution:bwc:maintenance") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -53,11 +53,11 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         result.task(transformTask).outcome == TaskOutcome.NO_SOURCE
     }
 
-    def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:bugfix"() {
+    def "yamlRestTestVxCompatTest executes and copies api and transforms tests from :bwc:maintenance"() {
         given:
         internalBuild()
 
-        addSubProject(":distribution:bwc:bugfix") << """
+        addSubProject(":distribution:bwc:maintenance") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -90,8 +90,8 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         String api = "foo.json"
         String test = "10_basic.yml"
         //add the compatible test and api files, these are the prior version's normal yaml rest tests
-        file("distribution/bwc/bugfix/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
-        file("distribution/bwc/bugfix/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
+        file("distribution/bwc/maintenance/checkoutDir/rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
+        file("distribution/bwc/maintenance/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
 
         when:
         def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest").build()
@@ -136,7 +136,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
     def "yamlRestTestVxCompatTest is wired into check and checkRestCompat"() {
         given:
 
-        addSubProject(":distribution:bwc:bugfix") << """
+        addSubProject(":distribution:bwc:maintenance") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -180,7 +180,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         given:
         internalBuild()
 
-        addSubProject(":distribution:bwc:bugfix") << """
+        addSubProject(":distribution:bwc:maintenance") << """
         configurations { checkout }
         artifacts {
             checkout(new File(projectDir, "checkoutDir"))
@@ -224,7 +224,7 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
 
         setupRestResources([], [])
 
-        file("distribution/bwc/bugfix/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
+        file("distribution/bwc/maintenance/checkoutDir/src/yamlRestTest/resources/rest-api-spec/test/test.yml" ) << """
         "one":
           - do:
               do_.some.key_to_replace:

build-tools-internal/src/main/groovy/elasticsearch.authenticated-testclusters.gradle

@@ -10,10 +10,8 @@ import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 def clusterCredentials = [
         username: providers.systemProperty('tests.rest.cluster.username')
-                .forUseAtConfigurationTime()
                 .getOrElse('test_admin'),
         password: providers.systemProperty('tests.rest.cluster.password')
-                .forUseAtConfigurationTime()
                 .getOrElse('x-pack-test-password')
 ]
 

build-tools-internal/src/main/groovy/elasticsearch.forbidden-dependencies.gradle

@@ -8,16 +8,15 @@
 
 // we do not want any of these dependencies on the compilation classpath
 // because they could then be used within Elasticsearch
-List<String> FORBIDDEN_DEPENDENCIES = [
-  'guava'
+List<String> FORBIDDEN_DEPENDENCY_GROUPS = [
+  'com.google.guava'
 ]
 
 Closure checkDeps = { Configuration configuration ->
   configuration.resolutionStrategy.eachDependency {
-    String artifactName = it.target.name
-    if (FORBIDDEN_DEPENDENCIES.contains(artifactName)) {
-      throw new GradleException("Dependency '${artifactName}' on configuration '${configuration.name}' is not allowed. " +
-        "If it is needed as a transitive depenency, try adding it to the runtime classpath")
+    if (FORBIDDEN_DEPENDENCY_GROUPS.contains(it.target.group)) {
+      throw new GradleException("Dependency '${it.target.group}:${it.target.name}' on configuration '${configuration.name}' is not allowed. " +
+        "If it is needed as a transitive dependency, try adding it to the runtime classpath")
     }
   }
 }

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -69,7 +69,7 @@ tasks.register('configureIdeCheckstyle') {
 }
 
 // Applying this stuff, particularly the idea-ext plugin, has a cost so avoid it unless we're running in the IDE
-if (providers.systemProperty('idea.active').forUseAtConfigurationTime().getOrNull() == 'true') {
+if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   project.apply(plugin: org.jetbrains.gradle.ext.IdeaExtPlugin)
 
   tasks.register('configureIdeaGradleJvm') {

build-tools-internal/src/main/groovy/elasticsearch.run.gradle

@@ -14,9 +14,9 @@ import org.elasticsearch.gradle.testclusters.RunTask
 // apply plugin: 'elasticsearch.internal-testclusters'
 
 testClusters.register("runTask") {
-    testDistribution = providers.systemProperty('run.distribution').orElse('default').forUseAtConfigurationTime().get()
-    if (providers.systemProperty('run.distribution').forUseAtConfigurationTime().getOrElse('default') == 'default') {
-      String licenseType = providers.systemProperty("run.license_type").forUseAtConfigurationTime().getOrElse("basic")
+    testDistribution = providers.systemProperty('run.distribution').orElse('default').get()
+    if (providers.systemProperty('run.distribution').getOrElse('default') == 'default') {
+      String licenseType = providers.systemProperty("run.license_type").getOrElse("basic")
       if (licenseType == 'trial') {
         setting 'xpack.ml.enabled', 'true'
         setting 'xpack.graph.enabled', 'true'

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java

@@ -84,25 +84,20 @@ public void apply(Project project) {
             String remoteRepo = remote.get();
             // for testing only we can override the base remote url
             String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
-                .forUseAtConfigurationTime()
                 .getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git");
             addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl));
         });
 
         TaskProvider<LoggedExec> fetchLatestTaskProvider = tasks.register("fetchLatest", LoggedExec.class, fetchLatest -> {
-            var gitFetchLatest = project.getProviders()
-                .systemProperty("tests.bwc.git_fetch_latest")
-                .forUseAtConfigurationTime()
-                .orElse("true")
-                .map(fetchProp -> {
-                    if ("true".equals(fetchProp)) {
-                        return true;
-                    }
-                    if ("false".equals(fetchProp)) {
-                        return false;
-                    }
-                    throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + fetchProp + "]");
-                });
+            var gitFetchLatest = project.getProviders().systemProperty("tests.bwc.git_fetch_latest").orElse("true").map(fetchProp -> {
+                if ("true".equals(fetchProp)) {
+                    return true;
+                }
+                if ("false".equals(fetchProp)) {
+                    return false;
+                }
+                throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + fetchProp + "]");
+            });
             fetchLatest.onlyIf(t -> project.getGradle().getStartParameter().isOffline() == false && gitFetchLatest.get());
             fetchLatest.dependsOn(addRemoteTaskProvider);
             fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get());

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java

@@ -23,7 +23,6 @@
 import java.io.File;
 
 import static org.elasticsearch.gradle.internal.conventions.GUtils.capitalize;
-import static org.gradle.api.internal.artifacts.ArtifactAttributes.ARTIFACT_FORMAT;
 
 /**
  * Provides a DSL and common configurations to define different types of
@@ -75,12 +74,14 @@ private void registerAndConfigureDistributionArchivesExtension(Project project)
                 sub.getArtifacts().add(DEFAULT_CONFIGURATION_NAME, distributionArchive.getArchiveTask());
                 var extractedConfiguration = sub.getConfigurations().create(EXTRACTED_CONFIGURATION_NAME);
                 extractedConfiguration.setCanBeResolved(false);
-                extractedConfiguration.getAttributes().attribute(ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
+                extractedConfiguration.getAttributes()
+                    .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
                 sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExpandedDistTask());
                 // The "composite" configuration is specifically used for resolving transformed artifacts in an included build
                 var compositeConfiguration = sub.getConfigurations().create(COMPOSITE_CONFIGURATION_NAME);
                 compositeConfiguration.setCanBeResolved(false);
-                compositeConfiguration.getAttributes().attribute(ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
+                compositeConfiguration.getAttributes()
+                    .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
                 compositeConfiguration.getAttributes().attribute(Attribute.of("composite", Boolean.class), true);
                 sub.getArtifacts().add(COMPOSITE_CONFIGURATION_NAME, distributionArchive.getArchiveTask());
                 sub.getTasks().register("extractedAssemble", task ->