Merge branch 'main' into bump-apache-rat-0.16.1

Signed-off-by: Đỗ Trọng Hải <[email protected]>
opensearch-project · Oct 1, 2024 · e03523c · e03523c
2 parents dd8a0c3 + c46bf41
commit e03523c
Show file tree

Hide file tree

Showing 103 changed files with 4,309 additions and 493 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,18 +13,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967))
 - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430))
 - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923))
+- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718))
+- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976))
+- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611))
 
 ### Dependencies
 - Bump `com.azure:azure-identity` from 1.13.0 to 1.13.2 ([#15578](https://github.com/opensearch-project/OpenSearch/pull/15578))
 - Bump `protobuf` from 3.22.3 to 3.25.4 ([#15684](https://github.com/opensearch-project/OpenSearch/pull/15684))
-- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.1 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858), [#16134](https://github.com/opensearch-project/OpenSearch/pull/16134))
 - Bump `peter-evans/create-pull-request` from 6 to 7 ([#15863](https://github.com/opensearch-project/OpenSearch/pull/15863))
 - Bump `com.nimbusds:oauth2-oidc-sdk` from 11.9.1 to 11.19.1 ([#15862](https://github.com/opensearch-project/OpenSearch/pull/15862))
 - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945))
 - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946))
 - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011))
+- Bump `org.roaringbitmap:RoaringBitmap` from 1.2.1 to 1.3.0 ([#16040](https://github.com/opensearch-project/OpenSearch/pull/16040))
+- Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038))
 - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039))
 - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041))
+- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042))
+- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137))
+- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
+- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
 - Bump `org.apache.rat:apache-rat` from 0.15 to 0.16.1 ([#16121](https://github.com/opensearch-project/OpenSearch/pull/16121))
 
 ### Changed
@@ -42,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988))
 - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985))
 - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931))
+- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521))
 
 ### Security
 

diff --git a/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle b/buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle
@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.24.0"
+  implementation "org.apache.logging.log4j:log4j-core:2.24.1"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
+distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
@@ -39,9 +39,9 @@ opensearchplugin {
 }
 
 dependencies {
-  api('com.maxmind.geoip2:geoip2:4.2.0')
+  api('com.maxmind.geoip2:geoip2:4.2.1')
   // geoip2 dependencies:
-  api('com.maxmind.db:maxmind-db:3.1.0')
+  api('com.maxmind.db:maxmind-db:3.1.1')
   api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
 

diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1
diff --git a/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1 b/modules/ingest-geoip/licenses/geoip2-4.2.1.jar.sha1
@@ -0,0 +1 @@
+9dbf8a8bea88a33e88c46eb3f503721b4bd08b90
diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1
diff --git a/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1 b/modules/ingest-geoip/licenses/maxmind-db-3.1.1.jar.sha1
@@ -0,0 +1 @@
+84a2a5d322fef1f1e82f05095683c8d53902baf1
diff --git a/.../ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java b/.../ingest-geoip/src/test/java/org/opensearch/ingest/geoip/IngestGeoIpModulePluginTests.java
@@ -35,6 +35,7 @@
 import com.maxmind.geoip2.model.AbstractResponse;
 
 import org.opensearch.common.network.InetAddresses;
+import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.TestEnvironment;
 import org.opensearch.ingest.Processor;
@@ -126,6 +127,12 @@ public void testAllowListNotSpecified() throws IOException {
         }
     }
 
+    public void testSettingsRegistration() {
+        final IngestGeoIpModulePlugin plugin = new IngestGeoIpModulePlugin();
+        final List<Setting<?>> settings = plugin.getSettings();
+        assertTrue(settings.contains(IngestGeoIpModulePlugin.PROCESSORS_ALLOWLIST_SETTING));
+    }
+
     private void runAllowListTest(List<String> allowList) throws IOException {
         Settings.Builder settingsBuilder = Settings.builder();
         createDb(settingsBuilder);

diff --git a/...user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java b/...user-agent/src/main/java/org/opensearch/ingest/useragent/IngestUserAgentModulePlugin.java
@@ -44,6 +44,7 @@
 import java.nio.file.Path;
 import java.nio.file.PathMatcher;
 import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -152,6 +153,6 @@ static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigD
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Collections.singletonList(CACHE_SIZE_SETTING);
+        return Arrays.asList(CACHE_SIZE_SETTING, PROCESSORS_ALLOWLIST_SETTING);
     }
 }
diff --git a/...agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java b/...agent/src/test/java/org/opensearch/ingest/useragent/IngestUserAgentModulePluginTests.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.ingest.useragent;
 
+import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.TestEnvironment;
 import org.opensearch.ingest.Processor;
@@ -89,6 +90,12 @@ public void testAllowListNotSpecified() throws IOException {
         }
     }
 
+    public void testSettingsRegistration() {
+        final IngestUserAgentModulePlugin plugin = new IngestUserAgentModulePlugin();
+        final List<Setting<?>> settings = plugin.getSettings();
+        assertTrue(settings.contains(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING));
+    }
+
     private void runAllowListTest(List<String> allowList) throws IOException {
         final Settings settings = settingsBuilder.putList(IngestUserAgentModulePlugin.PROCESSORS_ALLOWLIST_SETTING.getKey(), allowList)
             .build();

diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
@@ -48,7 +48,7 @@ dependencies {
   api 'com.azure:azure-json:1.1.0'
   api 'com.azure:azure-xml:1.1.0'
   api 'com.azure:azure-storage-common:12.25.1'
-  api 'com.azure:azure-core-http-netty:1.15.3'
+  api 'com.azure:azure-core-http-netty:1.15.4'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"
@@ -63,7 +63,7 @@ dependencies {
   api "net.java.dev.jna:jna-platform:${versions.jna}"
   api 'com.microsoft.azure:msal4j:1.17.1'
   api 'com.nimbusds:oauth2-oidc-sdk:11.19.1'
-  api 'com.nimbusds:nimbus-jose-jwt:9.40'
+  api 'com.nimbusds:nimbus-jose-jwt:9.41.1'
   api 'com.nimbusds:content-type:2.3'
   api 'com.nimbusds:lang-tag:1.7'
   // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,

diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.3.jar.sha1
diff --git a/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1 b/plugins/repository-azure/licenses/azure-core-http-netty-1.15.4.jar.sha1
@@ -0,0 +1 @@
+489a38c9e6efb5ce01fbd276d8cb6c0e89000459
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.40.jar.sha1
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.41.1.jar.sha1
@@ -0,0 +1 @@
+35532a88e1b49a623ec97fd276cc208ea525b6bc
diff --git a/qa/remote-clusters/docker-compose.yml b/qa/remote-clusters/docker-compose.yml
@@ -72,3 +72,11 @@ services:
       - "9600"
     volumes:
       - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    depends_on:
+      - opensearch-2
+    healthcheck:
+      test: ["CMD", "timeout", "1", "bash", "-c", "cat < /dev/null > /dev/tcp/localhost/9600"]
+      interval: 2s
+      timeout: 1s
+      retries: 5
+      start_period: 15s
diff --git a/release-notes/opensearch.release-notes-2.17.1.md b/release-notes/opensearch.release-notes-2.17.1.md
@@ -0,0 +1,16 @@
+## 2024-10-01 Version 2.17.1 Release Notes
+
+## [2.17.1]
+### Added
+- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
+- Memory optimisations in _cluster/health API ([#15492](https://github.com/opensearch-project/OpenSearch/pull/15492))
+
+### Dependencies
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/40_range.yml
@@ -535,8 +535,8 @@ setup:
 ---
 "Double range profiler shows filter rewrite info":
   - skip:
-      version: " - 2.99.99"
-      reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.15.0)
+      version: " - 2.15.99"
+      reason: debug info for filter rewrite was added in 2.16.0
 
   - do:
       indices.create:

diff --git a/server/build.gradle b/server/build.gradle
@@ -127,7 +127,7 @@ dependencies {
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
 
   // https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap
-  implementation 'org.roaringbitmap:RoaringBitmap:1.2.1'
+  implementation 'org.roaringbitmap:RoaringBitmap:1.3.0'
 
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server

diff --git a/server/licenses/RoaringBitmap-1.2.1.jar.sha1 b/server/licenses/RoaringBitmap-1.2.1.jar.sha1
diff --git a/server/licenses/RoaringBitmap-1.3.0.jar.sha1 b/server/licenses/RoaringBitmap-1.3.0.jar.sha1
@@ -0,0 +1 @@
+a46ce7a2dc494da69700ab421f081b1583857f6d