Merge branch 'main' into extension-handshake

.ci/bwcVersions

@@ -17,3 +17,4 @@ BWC_VERSION:
   - "2.6.0"
   - "2.6.1"
   - "2.7.0"
+  - "2.8.0"

.github/ISSUE_TEMPLATE/failed_check.md

@@ -1,6 +1,6 @@
 ---
 title: '[AUTOCUT] Gradle Check Failure on push to {{ env.branch_name }}'
-labels: '>test-failure, bug'
+labels: '>test-failure, bug, autocut'
 ---
 
 Gradle check has failed on push of your commit {{ env.pr_from_sha }}.

CHANGELOG.md

@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
+- Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
 - Add connectToNodeAsExtension in TransportService ([#6866](https://github.com/opensearch-project/OpenSearch/pull/6866))
 
 ### Dependencies
@@ -17,7 +18,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `woodstox-core` from 6.3.0 to 6.3.1
 - Bump `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354))
 - Bump `azure-storage-common` from 12.18.0 to 12.18.1 ([#4164](https://github.com/opensearch-project/OpenSearch/pull/4664))
-- Bump `org.gradle.test-retry` from 1.4.0 to 1.4.1 ([#4411](https://github.com/opensearch-project/OpenSearch/pull/4411))
 - Bump `reactor-netty-core` from 1.0.19 to 1.0.22 ([#4447](https://github.com/opensearch-project/OpenSearch/pull/4447))
 - Bump `reactive-streams` from 1.0.3 to 1.0.4 ([#4488](https://github.com/opensearch-project/OpenSearch/pull/4488))
 - Bump `reactor-core` from 3.4.23 to 3.5.1 ([#5604](https://github.com/opensearch-project/OpenSearch/pull/5604))
@@ -37,13 +37,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.google.code.gson:gson` from 2.10 to 2.10.1
 - Bump `com.maxmind.geoip2:geoip2` from 4.0.0 to 4.0.1
 - Bump `com.networknt:json-schema-validator` from 1.0.76 to 1.0.78
-- Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.0.1
+- Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.0
 - Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.11 to 0.16.12
 - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0
 - Bump `org.apache.commons:commons-configuration2` from 2.8.0 to 2.9.0
 - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.2.0
-- Bump `net.minidev:json-smart` from 2.4.9 to 2.4.10
 - Bump `com.netflix.nebula.ospackage-base` from 11.0.0 to 11.2.0
+- Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0
+- Bump `com.google.protobuf:protobuf-java` from 3.22.2 to 3.22.3
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -52,7 +53,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
-- Changed `opensearch-env` to respect already set `OPENSEARCH_HOME` environment variable ([#6956](https://github.com/opensearch-project/OpenSearch/pull/6956/))
 
 ### Deprecated
 
@@ -76,62 +76,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
-- Avoid negative memory result in IndicesQueryCache stats calculation ([#6917](https://github.com/opensearch-project/OpenSearch/pull/6917))
 
 ### Security
 
 ## [Unreleased 2.x]
 ### Added
-- Add GeoTile and GeoHash Grid aggregations on GeoShapes. ([#5589](https://github.com/opensearch-project/OpenSearch/pull/5589))
-- Disallow multiple data paths for search nodes ([#6427](https://github.com/opensearch-project/OpenSearch/pull/6427))
-- [Segment Replication] Allocation and rebalancing based on average primary shard count per index ([#6422](https://github.com/opensearch-project/OpenSearch/pull/6422))
-- The truncation limit of the OpenSearchJsonLayout logger is now configurable ([#6569](https://github.com/opensearch-project/OpenSearch/pull/6569))
-- Add 'base_path' setting to File System Repository ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558))
-- Return success on DeletePits when no PITs exist. ([#6544](https://github.com/opensearch-project/OpenSearch/pull/6544))
-- Add node repurpose command for search nodes ([#6517](https://github.com/opensearch-project/OpenSearch/pull/6517))
-- Add wait_for_completion parameter to resize, open, and forcemerge APIs ([#6434](https://github.com/opensearch-project/OpenSearch/pull/6434))
-- [Segment Replication] Apply backpressure when replicas fall behind ([#6563](https://github.com/opensearch-project/OpenSearch/pull/6563))
-- [Remote Store] Integrate remote segment store in peer recovery flow ([#6664](https://github.com/opensearch-project/OpenSearch/pull/6664))
-- Enable sort optimization for all NumericTypes ([#6464](https://github.com/opensearch-project/OpenSearch/pull/6464)
-- Remove 'cluster_manager' role attachment when using 'node.master' deprecated setting ([#6331](https://github.com/opensearch-project/OpenSearch/pull/6331))
-- Add new cluster settings to ignore weighted round-robin routing and fallback to default behaviour. ([#6834](https://github.com/opensearch-project/OpenSearch/pull/6834))
-- Add experimental support for ZSTD compression. ([#3577](https://github.com/opensearch-project/OpenSearch/pull/3577))
-- [Segment Replication] Add point in time and scroll query compatibility. ([#6644](https://github.com/opensearch-project/OpenSearch/pull/6644))
+- [Extensions] Moving Extensions APIs to protobuf serialization. ([#6960](https://github.com/opensearch-project/OpenSearch/pull/6960))
 
 ### Dependencies
-- Bump `org.apache.logging.log4j:log4j-core` from 2.18.0 to 2.20.0 ([#6490](https://github.com/opensearch-project/OpenSearch/pull/6490))
-- Bump `com.azure:azure-storage-common` from 12.19.3 to 12.20.0 ([#6492](https://github.com/opensearch-project/OpenSearch/pull/6492)
-- Bump `snakeyaml` from 1.33 to 2.0 ([#6511](https://github.com/opensearch-project/OpenSearch/pull/6511))
-- Bump `io.projectreactor.netty:reactor-netty` from 1.1.3 to 1.1.4
-- Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.15.2 to 0.16.11
-- Bump `net.minidev:json-smart` from 2.4.8 to 2.4.9
-- Bump `com.google.protobuf:protobuf-java` to 3.22.2 ([#6994](https://github.com/opensearch-project/OpenSearch/pull/6994))
-- Bump Netty to 4.1.90.Final ([#6677](https://github.com/opensearch-project/OpenSearch/pull/6677)
-- Bump `com.diffplug.spotless` from 6.15.0 to 6.17.0
-- Bump `org.apache.zookeeper:zookeeper` from 3.8.0 to 3.8.1
-- Bump `net.minidev:json-smart` from 2.4.7 to 2.4.10
-- Bump `org.apache.maven:maven-model` from 3.6.2 to 3.9.1
-- Bump `org.codehaus.jettison:jettison` from 1.5.3 to 1.5.4 ([#6878](https://github.com/opensearch-project/OpenSearch/pull/6878))
-- Add `com.github.luben:zstd-jni:1.5.4-1` ([#3577](https://github.com/opensearch-project/OpenSearch/pull/3577))
-- Bump: Netty from 4.1.90.Final to 4.1.91.Final , ASM 9.4 to ASM 9.5, ByteBuddy 1.14.2 to 1.14.3 ([#6981](https://github.com/opensearch-project/OpenSearch/pull/6981))
-- Bump `com.azure:azure-storage-blob` from 12.15.0 to 12.21.1
 
 ### Changed
-- Require MediaType in Strings.toString API ([#6009](https://github.com/opensearch-project/OpenSearch/pull/6009))
-- [Refactor] XContent base classes from xcontent to core library ([#5902](https://github.com/opensearch-project/OpenSearch/pull/5902))
-- Added a new field type: flat_object ([#6507](https://github.com/opensearch-project/OpenSearch/pull/6507))
 
 ### Deprecated
-- Map, List, and Set in org.opensearch.common.collect ([#6609](https://github.com/opensearch-project/OpenSearch/pull/6609))
 
 ### Removed
 
 ### Fixed
-- Added depth check in doc parser for deep nested document ([#5199](https://github.com/opensearch-project/OpenSearch/pull/5199))
-- Added equals/hashcode for named DocValueFormat.DateTime inner class ([#6357](https://github.com/opensearch-project/OpenSearch/pull/6357))
-- Fixed bug for searchable snapshot to take 'base_path' of blob into account ([#6558](https://github.com/opensearch-project/OpenSearch/pull/6558))
-- Fix fuzziness validation ([#5805](https://github.com/opensearch-project/OpenSearch/pull/5805))
-- Fix GetSnapshots to not return non-existent snapshots with ignore_unavailable=true ([#6839](https://github.com/opensearch-project/OpenSearch/pull/6839))
 
 ### Security
 

benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java

@@ -107,27 +107,10 @@ public void setup() {
     /**
      * Stubbed out IndexInput that does nothing but report a fixed size
      */
-    private static class FixedSizeStubIndexInput extends CachedIndexInput {
-        private FixedSizeStubIndexInput() {
-            super(FixedSizeStubIndexInput.class.getSimpleName());
-        }
-
-        @Override
-        public boolean isClosed() {
-            return false;
-        }
-
-        @Override
-        public void close() {}
-
-        @Override
-        public long getFilePointer() {
-            throw new UnsupportedOperationException();
-        }
-
+    private static class FixedSizeStubIndexInput implements CachedIndexInput {
         @Override
-        public void seek(long pos) {
-            throw new UnsupportedOperationException();
+        public IndexInput getIndexInput() {
+            return null;
         }
 
         @Override
@@ -136,18 +119,13 @@ public long length() {
         }
 
         @Override
-        public IndexInput slice(String sliceDescription, long offset, long length) {
-            throw new UnsupportedOperationException();
+        public boolean isClosed() {
+            return false;
         }
 
         @Override
-        public byte readByte() {
-            throw new UnsupportedOperationException();
-        }
+        public void close() throws Exception {
 
-        @Override
-        public void readBytes(byte[] b, int offset, int len) {
-            throw new UnsupportedOperationException();
         }
     }
 }

build.gradle

@@ -54,7 +54,7 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.17.0" apply false
-  id "org.gradle.test-retry" version "1.5.1" apply false
+  id "org.gradle.test-retry" version "1.5.2" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }

buildSrc/build.gradle

@@ -61,7 +61,7 @@ Properties props = VersionPropertiesLoader.loadBuildSrcVersion(project.file('ver
 version = props.getProperty("opensearch")
 
 def generateVersionProperties = tasks.register("generateVersionProperties", WriteProperties) {
-  outputFile = "${buildDir}/version.properties"
+  destinationFile = file("${buildDir}/version.properties")
   comment = 'Generated version properties'
   properties(props)
 }
@@ -107,7 +107,7 @@ dependencies {
   api 'org.apache.ant:ant:1.10.13'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:9.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:20.2.0'
-  api 'com.netflix.nebula:gradle-info-plugin:12.0.1'
+  api 'com.netflix.nebula:gradle-info-plugin:12.1.0'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.11.0'
   api "net.java.dev.jna:jna:5.13.0"

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -35,6 +35,7 @@
 import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.info.GlobalBuildInfoPlugin;
+import org.opensearch.gradle.jvm.JvmTestSuiteHelper;
 import org.opensearch.gradle.test.ErrorReportingTestListener;
 import org.opensearch.gradle.util.Util;
 import org.gradle.api.Action;
@@ -223,7 +224,15 @@ public void execute(Task t) {
                 // Add the shadow JAR artifact itself
                 FileCollection shadowJar = project.files(project.getTasks().named("shadowJar"));
 
-                test.setClasspath(test.getClasspath().minus(mainRuntime).plus(shadowConfig).plus(shadowJar));
+                // See please https://docs.gradle.org/8.1/userguide/upgrading_version_8.html#test_task_default_classpath
+                test.setClasspath(
+                    JvmTestSuiteHelper.getDefaultTestSuite(project)
+                        .map(suite -> suite.getSources().getRuntimeClasspath())
+                        .orElseGet(() -> test.getClasspath())
+                        .minus(mainRuntime)
+                        .plus(shadowConfig)
+                        .plus(shadowJar)
+                );
             });
         });
     }

buildSrc/src/main/java/org/opensearch/gradle/jvm/JvmTestSuiteHelper.java

@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.jvm;
+
+import org.gradle.api.Project;
+import org.gradle.api.plugins.JvmTestSuitePlugin;
+import org.gradle.api.plugins.jvm.JvmTestSuite;
+import org.gradle.testing.base.TestSuite;
+import org.gradle.testing.base.TestingExtension;
+
+import java.util.Optional;
+
+public final class JvmTestSuiteHelper {
+    private JvmTestSuiteHelper() {}
+
+    /**
+     * Gets the default test suite. This method assumes the Java plugin is applied,
+     * adapted from {@link org.gradle.api.plugins.internal.JavaPluginHelper} since it is not
+     * available in Gradle releases predated 8.1.
+     */
+    public static Optional<JvmTestSuite> getDefaultTestSuite(Project project) {
+        TestingExtension testing = project.getExtensions().findByType(TestingExtension.class);
+        if (testing == null) {
+            return Optional.empty();
+        }
+
+        TestSuite defaultTestSuite = testing.getSuites().findByName(JvmTestSuitePlugin.DEFAULT_TEST_SUITE_NAME);
+        if (!(defaultTestSuite instanceof JvmTestSuite)) {
+            return Optional.empty();
+        }
+
+        return Optional.of((JvmTestSuite) defaultTestSuite);
+    }
+}

buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionsTasks.java

@@ -32,20 +32,26 @@
 package org.opensearch.gradle.precommit;
 
 import groovy.lang.Closure;
+
+import org.opensearch.gradle.jvm.JvmTestSuiteHelper;
 import org.opensearch.gradle.util.GradleUtils;
 import org.opensearch.gradle.util.Util;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Task;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
+import org.gradle.api.plugins.jvm.JvmTestSuite;
 import org.gradle.api.tasks.Classpath;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.OutputFile;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.testing.Test;
+import org.gradle.api.tasks.util.PatternFilterable;
+import org.gradle.api.tasks.util.PatternSet;
+import org.gradle.internal.Factory;
 
 import java.io.File;
 import java.io.IOException;
@@ -70,6 +76,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import javax.inject.Inject;
+
 public class TestingConventionsTasks extends DefaultTask {
 
     private static final String TEST_METHOD_PREFIX = "test";
@@ -85,13 +93,39 @@ public TestingConventionsTasks() {
         naming = getProject().container(TestingConventionRule.class);
     }
 
+    @Inject
+    protected Factory<PatternSet> getPatternSetFactory() {
+        throw new UnsupportedOperationException();
+    }
+
     @Input
     public Map<String, Set<File>> getClassFilesPerEnabledTask() {
         return getProject().getTasks()
             .withType(Test.class)
             .stream()
             .filter(Task::getEnabled)
-            .collect(Collectors.toMap(Task::getPath, task -> task.getCandidateClassFiles().getFiles()));
+            .collect(Collectors.toMap(Task::getPath, task -> {
+                // See please https://docs.gradle.org/8.1/userguide/upgrading_version_8.html#test_task_default_classpath
+                final JvmTestSuite jvmTestSuite = JvmTestSuiteHelper.getDefaultTestSuite(getProject()).orElse(null);
+                if (jvmTestSuite != null) {
+                    final PatternFilterable patternSet = getPatternSetFactory().create()
+                        .include(task.getIncludes())
+                        .exclude(task.getExcludes());
+
+                    final Set<File> files = jvmTestSuite.getSources()
+                        .getOutput()
+                        .getClassesDirs()
+                        .getAsFileTree()
+                        .matching(patternSet)
+                        .getFiles();
+
+                    if (!files.isEmpty()) {
+                        return files;
+                    }
+                }
+
+                return task.getCandidateClassFiles().getFiles();
+            }));
     }
 
     @Input

buildSrc/src/main/java/org/opensearch/gradle/test/RestIntegTestTask.java

@@ -33,6 +33,10 @@
 package org.opensearch.gradle.test;
 
 import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask;
+
+import groovy.lang.Closure;
+
+import org.gradle.api.Task;
 import org.gradle.api.tasks.CacheableTask;
 
 /**
@@ -41,4 +45,12 @@
  * conventional configured tasks of {@link RestIntegTestTask}
  */
 @CacheableTask
-public class RestIntegTestTask extends StandaloneRestIntegTestTask {}
+public class RestIntegTestTask extends StandaloneRestIntegTestTask implements TestSuiteConventionMappings {
+    @SuppressWarnings("rawtypes")
+    @Override
+    public Task configure(Closure closure) {
+        final Task t = super.configure(closure);
+        applyConventionMapping(getProject(), getConventionMapping());
+        return t;
+    }
+}