Merge branch 'main' of https://github.com/opensearch-project/OpenSearch…

… into feature/sandboxing

Signed-off-by: Sarat Vemulapalli <[email protected]>

.ci/bwcVersions

@@ -77,4 +77,6 @@ BWC_VERSION:
   - "1.0.0"
   - "1.0.1"
   - "1.1.0"
+  - "1.1.1"
   - "1.2.0"
+  - "1.3.0"

.github/CODEOWNERS

@@ -0,0 +1,3 @@
+# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
+*   @opensearch-project/opensearch-core
+

.github/workflows/backport.yml

@@ -0,0 +1,16 @@
+name: Backport
+on:
+  pull_request:
+    types:
+      - closed
+      - labeled
+
+jobs:
+  backport:
+    runs-on: ubuntu-latest
+    name: Backport
+    steps:
+      - name: Backport
+        uses: tibdex/backport@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/dco.yml

@@ -0,0 +1,18 @@
+name: Developer Certificate of Origin Check
+
+on: [pull_request]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Get PR Commits
+        id: 'get-pr-commits'
+        uses: tim-actions/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: DCO Check
+        uses: tim-actions/[email protected]
+        with:
+          commits: ${{ steps.get-pr-commits.outputs.commits }}

.whitesource

@@ -0,0 +1,15 @@
+{
+  "scanSettings": {
+    "configMode": "LOCAL",
+    "configExternalURL": "",
+    "projectToken": "",
+    "baseBranches": []
+  },
+  "checkRunSettings": {
+    "vulnerableCheckRunConclusionLevel": "failure",
+    "displayMode": "diff"
+  },
+  "issueSettings": {
+    "minSeverityLevel": "LOW"
+  }
+}

CONTRIBUTING.md

@@ -20,7 +20,7 @@ OpenSearch is a community project that is built and maintained by people just li
 
 ## Ways to Contribute
 
-**Please note:** OpenSearch is a fork of [Elasticsearch 7.10.2](https://github.com/elastic/elasticsearch), and is work in progress. If you do find references to Elasticsearch (outside of attributions and copyrights!) please [open an issue](https://github.com/opensearch-project/OpenSearch/issues).
+**Please note:** OpenSearch is a fork of [Elasticsearch 7.10.2](https://github.com/elastic/elasticsearch). If you do find references to Elasticsearch (outside of attributions and copyrights!) please [open an issue](https://github.com/opensearch-project/OpenSearch/issues).
 
 ### Bug Reports
 

DEVELOPER_GUIDE.md

@@ -3,7 +3,7 @@
     - [Git Clone OpenSearch Repo](#git-clone-opensearch-repo)
     - [Install Prerequisites](#install-prerequisites)
       - [JDK 11](#jdk-11)
-      - [JDK 8 and 17](#jdk-8-and-17)
+      - [JDK 8 and 14](#jdk-8-and-14)
       - [Runtime JDK](#runtime-jdk)
       - [Windows](#windows)
       - [Docker](#docker)
@@ -44,6 +44,7 @@
     - [Aggregations](#aggregations)
     - [Distributed Framework](#distributed-framework)
   - [Submitting Changes](#submitting-changes)
+  - [Backports](#backports)
 
 # Developer Guide
 
@@ -63,9 +64,9 @@ OpenSearch builds using Java 11 at a minimum. This means you must have a JDK 11
 
 Download Java 11 from [here](https://adoptium.net/releases.html?variant=openjdk11).
 
-#### JDK 8 and 17
+#### JDK 8 and 14
 
-To run the full suite of tests, download and install [JDK 8](https://adoptium.net/releases.html?variant=openjdk8) and [17](https://adoptium.net/releases.html?variant=openjdk17) and set `JAVA8_HOME`, `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility).
+To run the full suite of tests, download and install [JDK 8](https://adoptium.net/releases.html?variant=openjdk8) and [JDK 14](https://jdk.java.net/archive/) and set `JAVA8_HOME`, `JAVA11_HOME`, and `JAVA14_HOME`. They are required by the [backwards compatibility test](./TESTING.md#testing-backwards-compatibility).
 
 #### Runtime JDK
 
@@ -408,3 +409,7 @@ Includes:
 ## Submitting Changes
 
 See [CONTRIBUTING](CONTRIBUTING.md).
+
+## Backports
+
+The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.

MAINTAINERS.md

@@ -7,7 +7,7 @@
 | Harold Wang | [harold-wang](https://github.com/harold-wang) | Amazon |
 | Himanshu Setia | [setiah](https://github.com/setiah) | Amazon |
 | Nick Knize | [nknize](https://github.com/nknize) | Amazon | 
-| Rabi Panda | [adnapibar](adnapibar) | Amazon |
+| Rabi Panda | [adnapibar](https://github.com/adnapibar) | Amazon |
 | Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon |
 | Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon |
 | Gopala Krishna Ambareesh | [krishna-ggk](https://github.com/krishna-ggk) |Amazon |

build.gradle

@@ -313,7 +313,7 @@ allprojects {
         }
       }
       boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)
-      project.configurations.compile.dependencies
+      project.configurations.implementation.dependencies
         .findAll()
         .toSorted(sortClosure)
         .each({ c -> depJavadocClosure(hasShadow, c) })

buildSrc/build.gradle

@@ -104,27 +104,27 @@ dependencies {
 
   api 'commons-codec:commons-codec:1.13'
   api 'org.apache.commons:commons-compress:1.21'
-  api 'org.apache.ant:ant:1.10.9'
+  api 'org.apache.ant:ant:1.10.12'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
   api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
-  api 'org.apache.rat:apache-rat:0.11'
+  api 'org.apache.rat:apache-rat:0.13'
+  api 'commons-io:commons-io:2.7'
   api "net.java.dev.jna:jna:5.5.0"
   api 'com.github.jengelman.gradle.plugins:shadow:6.0.0'
   api 'de.thetaphi:forbiddenapis:3.0'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.12.1'
   api 'org.apache.maven:maven-model:3.6.2'
   api 'com.networknt:json-schema-validator:1.0.36'
+  api 'com.fasterxml.jackson.core:jackson-databind:2.12.5'
 
-  compileOnly "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
-  testImplementation "com.puppycrawl.tools:checkstyle:${props.getProperty('checkstyle')}"
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testFixturesApi gradleApi()
   testFixturesApi gradleTestKit()
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
-  integTestImplementation('org.spockframework:spock-core:1.3-groovy-2.5') {
+  integTestImplementation('org.spockframework:spock-core:2.0-groovy-3.0') {
     exclude module: "groovy"
   }
 }
@@ -260,6 +260,9 @@ if (project != rootProject) {
    */
   afterEvaluate {
     generatePomFileForPluginMavenPublication.enabled = false
+    tasks.matching { it.name.contains("PluginMaven") }*.configure {
+      dependsOn("generatePomFileForNebulaPublication")
+    }
   }
 
   publishing.publications.named("nebula").configure {

buildSrc/src/integTest/java/org/opensearch/gradle/ExportOpenSearchBuildResourcesTaskIT.java

@@ -33,34 +33,11 @@
 package org.opensearch.gradle;
 
 import org.opensearch.gradle.test.GradleIntegrationTestCase;
-import org.gradle.testkit.runner.BuildResult;
 
 public class ExportOpenSearchBuildResourcesTaskIT extends GradleIntegrationTestCase {
 
     public static final String PROJECT_NAME = "opensearch-build-resources";
 
-    public void testUpToDateWithSourcesConfigured() {
-        getGradleRunner(PROJECT_NAME).withArguments("clean", "-s").build();
-
-        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("buildResources", "-s", "-i").build();
-        assertTaskSuccessful(result, ":buildResources");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
-
-        result = getGradleRunner(PROJECT_NAME).withArguments("buildResources", "-s", "-i").build();
-        assertTaskUpToDate(result, ":buildResources");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle.xml");
-        assertBuildFileExists(result, PROJECT_NAME, "build-tools-exported/checkstyle_suppressions.xml");
-    }
-
-    public void testOutputAsInput() {
-        BuildResult result = getGradleRunner(PROJECT_NAME).withArguments("clean", "sampleCopy", "-s", "-i").build();
-
-        assertTaskSuccessful(result, ":sampleCopy");
-        assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle.xml");
-        assertBuildFileExists(result, PROJECT_NAME, "sampleCopy/checkstyle_suppressions.xml");
-    }
-
     public void testIncorrectUsage() {
         assertOutputContains(
             getGradleRunner(PROJECT_NAME).withArguments("noConfigAfterExecution", "-s", "-i").buildAndFail().getOutput(),

buildSrc/src/main/groovy/org/opensearch/gradle/precommit/PrecommitTasks.groovy

@@ -40,7 +40,6 @@ class PrecommitTasks {
 
     static void create(Project project, boolean includeDependencyLicenses) {
 
-        project.pluginManager.apply(CheckstylePrecommitPlugin)
         project.pluginManager.apply(ForbiddenApisPrecommitPlugin)
         project.pluginManager.apply(JarHellPrecommitPlugin)
         project.pluginManager.apply(ForbiddenPatternsPrecommitPlugin)

buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java

@@ -174,7 +174,10 @@ public BwcVersions(SortedSet<Version> allVersions, Version currentVersionPropert
 
     private void assertNoOlderThanTwoMajors() {
         Set<Integer> majors = groupByMajor.keySet();
-        if (majors.size() != 2 && currentVersion.getMinor() != 0 && currentVersion.getRevision() != 0) {
+        // until OpenSearch 3.0 we will need to carry three major support
+        // (1, 7, 6) && (2, 1, 7) since OpenSearch 1.0 === Legacy 7.x
+        int numSupportedMajors = (currentVersion.getMajor() < 3) ? 3 : 2;
+        if (majors.size() != numSupportedMajors && currentVersion.getMinor() != 0 && currentVersion.getRevision() != 0) {
             throw new IllegalStateException("Expected exactly 2 majors in parsed versions but found: " + majors);
         }
     }

buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java

@@ -110,12 +110,17 @@ private void setupRepository(Project project, Jdk jdk) {
 
         if (jdk.getVendor().equals(VENDOR_ADOPTIUM)) {
             repoUrl = "https://github.com/adoptium/temurin" + jdk.getMajor() + "-binaries/releases/download/";
+            // JDK updates are suffixed with 'U' (fe OpenJDK17U), whereas GA releases are not (fe OpenJDK17).
+            // To distinguish between those, the GA releases have only major version component (fe 17+32),
+            // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
+            // version separator '.' should be enough.
             artifactPattern = "jdk-"
                 + jdk.getBaseVersion()
                 + "+"
                 + jdk.getBuild()
                 + "/OpenJDK"
                 + jdk.getMajor()
+                + (jdk.getBaseVersion().contains(".") ? "U" : "")
                 + "-jdk_[classifier]_[module]_hotspot_"
                 + jdk.getBaseVersion()
                 + "_"

buildSrc/src/main/java/org/opensearch/gradle/LoggedExec.java

@@ -82,7 +82,7 @@ public LoggedExec(FileSystemOperations fileSystemOperations) {
             doLast(new Action<Task>() {
                 @Override
                 public void execute(Task task) {
-                    if (LoggedExec.this.getExecResult().getExitValue() != 0) {
+                    if (LoggedExec.this.getExecutionResult().get().getExitValue() != 0) {
                         try {
                             LoggedExec.this.getLogger().error("Output for " + LoggedExec.this.getExecutable() + ":");
                             outputLogger.accept(LoggedExec.this.getLogger());
@@ -94,7 +94,7 @@ public void execute(Task task) {
                                 "Process '%s %s' finished with non-zero exit value %d",
                                 LoggedExec.this.getExecutable(),
                                 LoggedExec.this.getArgs(),
-                                LoggedExec.this.getExecResult().getExitValue()
+                                LoggedExec.this.getExecutionResult().get().getExitValue()
                             )
                         );
                     }

buildSrc/src/main/java/org/opensearch/gradle/OpenSearchTestBasePlugin.java

@@ -74,7 +74,7 @@ public void apply(Project project) {
         File heapdumpDir = new File(project.getBuildDir(), "heapdump");
 
         project.getTasks().withType(Test.class).configureEach(test -> {
-            File testOutputDir = new File(test.getReports().getJunitXml().getDestination(), "output");
+            File testOutputDir = new File(test.getReports().getJunitXml().getOutputLocation().getAsFile().get(), "output");
 
             ErrorReportingTestListener listener = new ErrorReportingTestListener(test.getTestLogging(), test.getLogger(), testOutputDir);
             test.getExtensions().add("errorReportingTestListener", listener);