Merge branch 'opensearch-project:main' into main

Signed-off-by: Andreas <[email protected]>
opensearch-project · Jan 27, 2022 · 969eb6f · 969eb6f
2 parents 01476b3 + 3ce377f
commit 969eb6f
Show file tree

Hide file tree

Showing 107 changed files with 2,045 additions and 1,040 deletions.
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -1,6 +1,6 @@
 name: Backport
 on:
-  pull_request:
+  pull_request_target:
     types:
       - closed
       - labeled
@@ -14,6 +14,7 @@ jobs:
     name: Backport
     steps:
       - name: Backport
-        uses: tibdex/backport@v1
+        uses: VachaShah/backport@v1.1.4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch_name: backport/backport-${{ github.event.number }}
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -7,6 +7,7 @@
       - [Runtime JDK](#runtime-jdk)
       - [Windows](#windows)
       - [Docker](#docker)
+    - [Build](#build)
     - [Run Tests](#run-tests)
     - [Run OpenSearch](#run-opensearch)
   - [Use an Editor](#use-an-editor)
@@ -82,6 +83,22 @@ Download and install [Docker](https://docs.docker.com/install/), required for bu
 
 On Windows, [use Docker Desktop 3.6](https://docs.docker.com/desktop/windows/release-notes/3.x/). See [OpenSearch#1425](https://github.com/opensearch-project/OpenSearch/issues/1425) for workarounds and issues with Docker Desktop 4.1.1.
 
+### Build
+
+To build all distributions of OpenSearch, run:
+
+```
+./gradlew assemble
+```
+
+To build a distribution to run on your local platform, run:
+
+```
+./gradlew localDistro
+```
+
+All distributions built will be under `distributions/archives`.
+
 ### Run Tests
 
 OpenSearch uses a Gradle wrapper for its build. Run `gradlew` on Unix systems, or `gradlew.bat` on Windows in the root of the repository.
@@ -270,7 +287,7 @@ Please follow these formatting guidelines:
 
 ### Editor / IDE Support
 
-IntelliJ IDEs can [import](https://blog.jetbrains.com/idea/2014/01/intellij-idea-13-importing-code-formatter-settings-from-eclipse/) the same settings file, and / or use the [Eclipse Code Formatter](https://plugins.jetbrains.com/plugin/6546-eclipse-code-formatter)
+IntelliJ IDEs can [import](https://blog.jetbrains.com/idea/2014/01/intellij-idea-13-importing-code-formatter-settings-from-eclipse/) the [settings file](buildSrc/formatterConfig.xml), and / or use the [Eclipse Code Formatter](https://plugins.jetbrains.com/plugin/6546-eclipse-code-formatter)
 plugin.
 
 You can also tell Spotless to [format a specific file](https://github.com/diffplug/spotless/tree/master/plugin-gradle#can-i-apply-spotless-to-specific-files) from the command line.
@@ -412,4 +429,4 @@ See [CONTRIBUTING](CONTRIBUTING.md).
 
 ## Backports
 
-The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.
+The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -19,7 +19,7 @@ slf4j             = 1.6.2
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.72.Final
+netty             = 4.1.73.Final
 joda              = 2.10.12
 
 # when updating this version, you need to ensure compatibility with:

diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.72.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.72.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.73.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+244a569c9aae973f6f485ac9801d79c1eca36daa
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.72.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.72.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.73.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+9496a30a349863a4c6fa10d5c36b4f3b495d3a31
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.72.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.72.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.73.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+1ceeac4429b9bd517dc05e376a144bbe6b6bd038
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.72.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.72.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.73.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+27731b58d741b6faa6a00fa3285e7a55cc47be01
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.72.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.72.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.73.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+1a2231c0074f88254865c3769a4b5842939ea04d
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.72.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.72.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.73.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+bfe83710f0c1739019613e81a06101020ca65def
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.72.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.72.Final.jar.sha1
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.73.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+abb155ddff196ccedfe85b810d4b9375ef85fcfa
diff --git a/.../internalClusterTest/java/org/opensearch/transport/netty4/OpenSearchLoggingHandlerIT.java b/.../internalClusterTest/java/org/opensearch/transport/netty4/OpenSearchLoggingHandlerIT.java
@@ -66,12 +66,13 @@ public void tearDown() throws Exception {
 
     @TestLogging(value = "org.opensearch.transport.netty4.OpenSearchLoggingHandler:trace,org.opensearch.transport.TransportLogger:trace", reason = "to ensure we log network events on TRACE level")
     public void testLoggingHandler() {
-        final String writePattern = ".*\\[length: \\d+"
+        final String writePattern = "^.*\\[length: \\d+"
             + ", request id: \\d+"
             + ", type: request"
-            + ", version: .*"
+            + ", version: [^,]+"
+            + ", header size: \\d+B"
             + ", action: cluster:monitor/nodes/hot_threads\\[n\\]\\]"
-            + " WRITE: \\d+B";
+            + " WRITE: \\d+B$";
         final MockLogAppender.LoggingExpectation writeExpectation = new MockLogAppender.PatternSeenEventExpectation(
             "hot threads request",
             TransportLogger.class.getCanonicalName(),
@@ -86,12 +87,12 @@ public void testLoggingHandler() {
             "*FLUSH*"
         );
 
-        final String readPattern = ".*\\[length: \\d+"
+        final String readPattern = "^.*\\[length: \\d+"
             + ", request id: \\d+"
             + ", type: request"
-            + ", version: .*"
+            + ", version: [^,]+"
             + ", action: cluster:monitor/nodes/hot_threads\\[n\\]\\]"
-            + " READ: \\d+B";
+            + " READ: \\d+B$";
 
         final MockLogAppender.LoggingExpectation readExpectation = new MockLogAppender.PatternSeenEventExpectation(
             "hot threads request",

diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.72.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.72.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.73.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+46137a5b01a5202059324cf4300443e53f11a38d
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.72.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.72.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.73.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+0eb145bc31fd32a20fd2a3e8b30736d2e0248b0c
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.72.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.72.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.73.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+cefa44d8f5dcaab21179d945f12b6c6d7325cce9
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.72.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.72.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.73.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+d1afa6876c3d3bdbdbe5127ddd495e6514d6e600
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.72.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.72.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.73.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+97cdf5fb97f8d961cfa3ffb05175009b90e5cfee
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.72.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.72.Final.jar.sha1
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.73.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+4701063d36f390e02da6da85c13e32a0e78349d2
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
@@ -61,8 +61,8 @@ dependencies {
   api 'com.google.api:api-common:1.8.1'
   api 'com.google.api:gax:1.54.0'
   api 'org.threeten:threetenbp:1.4.4'
-  api 'com.google.protobuf:protobuf-java-util:3.11.3'
-  api 'com.google.protobuf:protobuf-java:3.11.3'
+  api 'com.google.protobuf:protobuf-java-util:3.19.3'
+  api 'com.google.protobuf:protobuf-java:3.19.3'
   api 'com.google.code.gson:gson:2.8.9'
   api 'com.google.api.grpc:proto-google-common-protos:1.16.0'
   api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'

diff --git a/plugins/repository-gcs/licenses/protobuf-java-3.11.3.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-3.11.3.jar.sha1
diff --git a/plugins/repository-gcs/licenses/protobuf-java-3.19.3.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-3.19.3.jar.sha1
@@ -0,0 +1 @@
+4b57f1b1b9e281231c3fcfc039ce3021e29ff570
diff --git a/plugins/repository-gcs/licenses/protobuf-java-util-3.11.3.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-util-3.11.3.jar.sha1
diff --git a/plugins/repository-gcs/licenses/protobuf-java-util-3.19.3.jar.sha1 b/plugins/repository-gcs/licenses/protobuf-java-util-3.19.3.jar.sha1
@@ -0,0 +1 @@
+3e6812cbbb7e6faffa7b56438740dec510e1fc1a
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
@@ -67,7 +67,7 @@ dependencies {
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   api 'com.google.code.gson:gson:2.8.9'
   runtimeOnly 'com.google.guava:guava:30.1.1-jre'
-  api 'com.google.protobuf:protobuf-java:2.5.0'
+  api 'com.google.protobuf:protobuf-java:3.19.3'
   api 'commons-logging:commons-logging:1.1.3'
   api 'commons-cli:commons-cli:1.2'
   api "commons-codec:commons-codec:${versions.commonscodec}"
@@ -113,6 +113,19 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /hadoop-.*/, to: 'hadoop'
 }
 
+thirdPartyAudit {
+  ignoreViolations(
+    // uses internal java api: sun.misc.Unsafe
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil',
+    'com.google.protobuf.UnsafeUtil$1',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+  )
+}
+
 tasks.named("integTest").configure {
   it.dependsOn(project.tasks.named("bundlePlugin"))
 }
@@ -235,7 +248,7 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec
           )
         }
       }
-      
+
       if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_1_8) {
         jvmArgs += ["--add-opens", "java.security.jgss/sun.security.krb5=ALL-UNNAMED"]
       }

diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.72.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.72.Final.jar.sha1
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.73.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+75c5a0ddb28adcc9e4991c75678d4a85dfe4a0b3
diff --git a/plugins/repository-hdfs/licenses/protobuf-java-2.5.0.jar.sha1 b/plugins/repository-hdfs/licenses/protobuf-java-2.5.0.jar.sha1
diff --git a/plugins/repository-hdfs/licenses/protobuf-java-3.19.3.jar.sha1 b/plugins/repository-hdfs/licenses/protobuf-java-3.19.3.jar.sha1
@@ -0,0 +1 @@
+4b57f1b1b9e281231c3fcfc039ce3021e29ff570
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.72.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.72.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.73.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+244a569c9aae973f6f485ac9801d79c1eca36daa
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.72.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.72.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.73.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+9496a30a349863a4c6fa10d5c36b4f3b495d3a31
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.72.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.72.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.73.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+1ceeac4429b9bd517dc05e376a144bbe6b6bd038
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.72.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.72.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.73.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+27731b58d741b6faa6a00fa3285e7a55cc47be01
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.72.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.72.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.73.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+1a2231c0074f88254865c3769a4b5842939ea04d
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.72.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.72.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.73.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+bfe83710f0c1739019613e81a06101020ca65def
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.72.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.72.Final.jar.sha1
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.73.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.73.Final.jar.sha1
@@ -0,0 +1 @@
+abb155ddff196ccedfe85b810d4b9375ef85fcfa
diff --git a/qa/evil-tests/src/test/java/org/opensearch/cluster/routing/EvilSystemPropertyTests.java b/qa/evil-tests/src/test/java/org/opensearch/cluster/routing/EvilSystemPropertyTests.java