Merge branch 'opensearch-project:feature/segment-replication' into fe…

…ature/segment-replication

.ci/bwcVersions

@@ -39,4 +39,5 @@ BWC_VERSION:
   - "1.3.0"
   - "1.3.1"
   - "1.3.2"
-  - "1.4.0"
+  - "2.0.0"
+  - "2.1.0"

.github/workflows/version.yml

@@ -59,6 +59,7 @@ jobs:
           base: ${{ env.BASE }}
           branch: 'create-pull-request/patch-${{ env.BASE }}'
           commit-message: Incremented version to ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] Incremented version to ${{ env.NEXT_VERSION }}.'
           body: |
@@ -83,6 +84,7 @@ jobs:
           base: ${{ env.BASE_X }}
           branch: 'create-pull-request/patch-${{ env.BASE_X }}'
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] [${{ env.BASE_X }}] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |
@@ -107,6 +109,7 @@ jobs:
           base: main
           branch: 'create-pull-request/patch-main'
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
+          signoff: true
           delete-branch: true
           title: '[AUTO] [main] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |

TESTING.md

@@ -245,7 +245,7 @@ The YAML REST tests support all the options provided by the randomized runner, p
 
 -   `tests.rest.suite`: comma separated paths of the test suites to be run (by default loaded from /rest-api-spec/test). It is possible to run only a subset of the tests providing a sub-folder or even a single yaml file (the default /rest-api-spec/test prefix is optional when files are loaded from classpath) e.g. `-Dtests.rest.suite=index,get,create/10_with_id`
 
--   `tests.rest.blacklist`: comma separated globs that identify tests that are denylisted and need to be skipped e.g. `-Dtests.rest.blacklist=index/**/Index document,get/10_basic/**`
+-   `tests.rest.denylist`: comma separated globs that identify tests that are denylisted and need to be skipped e.g. `-Dtests.rest.denylist=index/**/Index document,get/10_basic/**`
 
 Java REST tests can be run with the "javaRestTest" task.
 

build.gradle

@@ -48,7 +48,8 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.3.0" apply false
+  id "com.diffplug.spotless" version "6.4.2" apply false
+  id "org.gradle.test-retry" version "1.3.1" apply false
 }
 
 apply from: 'gradle/build-complete.gradle'
@@ -232,7 +233,7 @@ allprojects {
   tasks.withType(JavaCompile).configureEach { JavaCompile compile ->
     // See please https://bugs.openjdk.java.net/browse/JDK-8209058
     if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
-        compile.options.compilerArgs << '-Werror'
+      compile.options.compilerArgs << '-Werror'
     }
     compile.options.compilerArgs << '-Xlint:auxiliaryclass'
     compile.options.compilerArgs << '-Xlint:cast'
@@ -386,6 +387,18 @@ gradle.projectsEvaluated {
   }
 }
 
+// test retry configuration
+subprojects {
+  apply plugin: "org.gradle.test-retry"
+  tasks.withType(Test).configureEach {
+    retry {
+      failOnPassedAfterRetry = false
+      maxRetries = 3
+      maxFailures = 10
+    }
+  }
+}
+
 // eclipse configuration
 allprojects {
   apply plugin: 'eclipse'
@@ -445,9 +458,9 @@ allprojects {
   tasks.named('eclipse') { dependsOn 'cleanEclipse', 'copyEclipseSettings' }
 
   afterEvaluate {
-     tasks.findByName("eclipseJdt")?.configure {
-         dependsOn 'copyEclipseSettings'
-     }
+    tasks.findByName("eclipseJdt")?.configure {
+      dependsOn 'copyEclipseSettings'
+    }
   }
 }
 

buildSrc/build.gradle

@@ -107,16 +107,16 @@ dependencies {
   api 'org.apache.ant:ant:1.10.12'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:7.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
-  api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
+  api 'com.netflix.nebula:gradle-info-plugin:11.3.3'
   api 'org.apache.rat:apache-rat:0.13'
   api 'commons-io:commons-io:2.7'
   api "net.java.dev.jna:jna:5.10.0"
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
-  api 'de.thetaphi:forbiddenapis:3.2'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.12'
+  api 'de.thetaphi:forbiddenapis:3.3'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.67'
-  api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson')}"
+  api 'com.networknt:json-schema-validator:1.0.68'
+  api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"

buildSrc/reaper/build.gradle

@@ -11,6 +11,9 @@
 
 apply plugin: 'java'
 
+targetCompatibility = JavaVersion.VERSION_11
+sourceCompatibility = JavaVersion.VERSION_11
+
 jar {
   archiveFileName = "${project.name}.jar"
   manifest {

buildSrc/src/main/groovy/org/opensearch/gradle/precommit/LicenseHeadersTask.groovy

@@ -35,7 +35,10 @@ import org.opensearch.gradle.AntTask
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.OutputFile
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
 import org.gradle.api.tasks.SkipWhenEmpty
 
 import java.nio.file.Files
@@ -78,6 +81,8 @@ class LicenseHeadersTask extends AntTask {
      */
     @InputFiles
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
+    @PathSensitive(PathSensitivity.RELATIVE)
     List<FileCollection> getJavaFiles() {
         return project.sourceSets.collect({it.allJava})
     }

buildSrc/src/main/java/org/opensearch/gradle/Version.java

@@ -77,7 +77,7 @@ public Version(int major, int minor, int revision) {
         // currently snapshot is not taken into account
         int id = major * 10000000 + minor * 100000 + revision * 1000;
         // identify if new OpenSearch version 1
-        this.id = major == 1 || major == 2 ? id ^ MASK : id;
+        this.id = major == 1 || major == 2 || major == 3 ? id ^ MASK : id;
     }
 
     private static int parseSuffixNumber(String substring) {

buildSrc/src/main/java/org/opensearch/gradle/precommit/FilePermissionsTask.java

@@ -46,8 +46,11 @@
 import org.gradle.api.GradleException;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.StopExecutionException;
 import org.gradle.api.tasks.TaskAction;
@@ -92,6 +95,8 @@ private static boolean isExecutableFile(File file) {
      */
     @InputFiles
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileCollection getFiles() {
         return GradleUtils.getJavaSourceSets(getProject())
             .stream()

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java

@@ -37,9 +37,12 @@
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.util.PatternFilterable;
@@ -100,6 +103,8 @@ public ForbiddenPatternsTask() {
 
     @InputFiles
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileCollection getFiles() {
         return getProject().getConvention()
             .getPlugin(JavaPluginConvention.class)

buildSrc/src/main/java/org/opensearch/gradle/precommit/LoggerUsageTask.java

@@ -37,6 +37,7 @@
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.PathSensitive;
 import org.gradle.api.tasks.PathSensitivity;
@@ -79,6 +80,7 @@ public void setClasspath(FileCollection classpath) {
     @InputFiles
     @PathSensitive(PathSensitivity.RELATIVE)
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
     public FileCollection getClassDirectories() {
         return getProject().getConvention()
             .getPlugin(JavaPluginConvention.class)

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -47,6 +47,7 @@
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Classpath;
 import org.gradle.api.tasks.CompileClasspath;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
@@ -195,6 +196,7 @@ public Set<String> getMissingClassExcludes() {
 
     @Classpath
     @SkipWhenEmpty
+    @IgnoreEmptyDirectories
     public Set<File> getJarsToScan() {
         // These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files,
         // or dependencies added as `files(...)`, we can't be sure if those are third party or not.
@@ -235,6 +237,7 @@ public void runThirdPartyAudit() throws IOException {
         Set<String> jdkJarHellClasses = runJdkJarHellCheck();
 
         if (missingClassExcludes != null) {
+            assertNoPointlessExclusions("are not missing", missingClassExcludes, missingClasses);
             long bogousExcludesCount = Stream.concat(missingClassExcludes.stream(), violationsExcludes.stream())
                 .filter(each -> missingClasses.contains(each) == false)
                 .filter(each -> violationsClasses.contains(each) == false)
@@ -245,7 +248,6 @@ public void runThirdPartyAudit() throws IOException {
                     "All excluded classes seem to have no issues. " + "This is sometimes an indication that the check silently failed"
                 );
             }
-            assertNoPointlessExclusions("are not missing", missingClassExcludes, missingClasses);
             missingClasses.removeAll(missingClassExcludes);
         }
         assertNoPointlessExclusions("have no violations", violationsExcludes, violationsClasses);

buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestApiTask.java

@@ -43,9 +43,12 @@
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.ListProperty;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.TaskAction;
@@ -112,8 +115,10 @@ public boolean isSkipHasRestTestCheck() {
         return skipHasRestTestCheck;
     }
 
+    @IgnoreEmptyDirectories
     @SkipWhenEmpty
     @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileTree getInputDir() {
         FileTree coreFileTree = null;
         boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests();

buildSrc/src/main/java/org/opensearch/gradle/test/rest/CopyRestTestsTask.java

@@ -43,9 +43,12 @@
 import org.gradle.api.file.FileTree;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.provider.ListProperty;
+import org.gradle.api.tasks.IgnoreEmptyDirectories;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.TaskAction;
@@ -104,8 +107,10 @@ String getSourceSetName() {
         return sourceSetName;
     }
 
+    @IgnoreEmptyDirectories
     @SkipWhenEmpty
     @InputFiles
+    @PathSensitive(PathSensitivity.RELATIVE)
     public FileTree getInputDir() {
         FileTree coreFileTree = null;
         if (includeCore.get().isEmpty() == false) {

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-6.6.1
+7.4.1

buildSrc/src/test/java/org/opensearch/gradle/DistributionDownloadPluginTests.java

@@ -53,11 +53,11 @@ public class DistributionDownloadPluginTests extends GradleUnitTestCase {
     private static Project packagesProject;
     private static Project bwcProject;
 
-    private static final Version BWC_MAJOR_VERSION = Version.fromString("5.0.0");
-    private static final Version BWC_MINOR_VERSION = Version.fromString("4.1.0");
-    private static final Version BWC_STAGED_VERSION = Version.fromString("4.0.0");
-    private static final Version BWC_BUGFIX_VERSION = Version.fromString("4.0.1");
-    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("3.90.1");
+    private static final Version BWC_MAJOR_VERSION = Version.fromString("6.0.0");
+    private static final Version BWC_MINOR_VERSION = Version.fromString("5.1.0");
+    private static final Version BWC_STAGED_VERSION = Version.fromString("5.0.0");
+    private static final Version BWC_BUGFIX_VERSION = Version.fromString("5.0.1");
+    private static final Version BWC_MAINTENANCE_VERSION = Version.fromString("4.90.1");
     private static final BwcVersions BWC_MINOR = new BwcVersions(
         new TreeSet<>(Arrays.asList(BWC_BUGFIX_VERSION, BWC_MINOR_VERSION, BWC_MAJOR_VERSION)),
         BWC_MAJOR_VERSION

buildSrc/src/test/java/org/opensearch/gradle/VersionTests.java

@@ -65,7 +65,7 @@ public void testRelaxedVersionParsing() {
 
     public void testCompareWithStringVersions() {
         // 1.10.2 is now rebased to OpenSearch version; so this needs to report
-        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 3.0.0", Version.fromString("1.10.20").after("3.0.0"));
+        assertTrue("OpenSearch 1.10.20 is not interpreted as after Legacy 6.0.0", Version.fromString("1.10.20").after("6.0.0"));
         assertTrue(
             "7.0.0-alpha1 should be equal to 7.0.0-alpha1",
             Version.fromString("7.0.0-alpha1").equals(Version.fromString("7.0.0-alpha1"))

buildSrc/src/testKit/thirdPartyAudit/build.gradle

@@ -40,7 +40,7 @@ repositories {
 }
 
 dependencies {
-  forbiddenApisCliJar 'de.thetaphi:forbiddenapis:3.2'
+  forbiddenApisCliJar 'de.thetaphi:forbiddenapis:3.3'
   jdkJarHell 'org.opensearch:opensearch-core:current'
   compileOnly "org.${project.properties.compileOnlyGroup}:${project.properties.compileOnlyVersion}"
   implementation "org.${project.properties.compileGroup}:${project.properties.compileVersion}"

buildSrc/version.properties

@@ -1,5 +1,5 @@
-opensearch        = 2.0.0
-lucene            = 9.1.0-snapshot-ea989fe8f30
+opensearch        = 3.0.0
+lucene            = 9.1.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.2+8
@@ -10,6 +10,7 @@ bundled_jdk = 17.0.2+8
 spatial4j         = 0.7
 jts               = 1.15.0
 jackson           = 2.13.2
+jackson_databind  = 2.13.2.2
 snakeyaml         = 1.26
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -22,18 +23,20 @@ jna               = 5.5.0
 netty             = 4.1.73.Final
 joda              = 2.10.12
 
+# client dependencies
+httpclient        = 4.5.13
+httpcore          = 4.4.12
+httpasyncclient   = 4.1.4
+commonslogging    = 1.2
+commonscodec      = 1.13
+
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
 bouncycastle=1.70
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
-httpclient        = 4.5.13
-httpcore          = 4.4.12
-httpasyncclient   = 4.1.4
-commonslogging    = 1.1.3
-commonscodec      = 1.13
 hamcrest          = 2.1
 mockito           = 4.3.1
 objenesis         = 3.2