[Feature/Identity] Merge remote-tracking branch 'origin/main' into feature/identity

.ci/bwcVersions

@@ -11,4 +11,5 @@ BWC_VERSION:
   - "2.3.1"
   - "2.4.0"
   - "2.4.1"
+  - "2.4.2"
   - "2.5.0"

.github/CODEOWNERS

@@ -1,3 +1 @@
-# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @opensearch-project/opensearch-core @reta
-
+*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @adnapibar @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @xuezhou25

MAINTAINERS.md

@@ -5,7 +5,6 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
-| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon |
 | Anas Alkouz | [anasalkouz](https://github.com/anasalkouz) | Amazon |
 | Andrew Ross   | [andrross](https://github.com/andrross)| Amazon |
 | Andriy Redko | [reta](https://github.com/reta) | Aiven |
@@ -22,8 +21,8 @@
 | Rishikesh Pasham | [Rishikesh1159](https://github.com/Rishikesh1159) | Amazon|
 | Ryan Bogan | [ryanbogan](https://github.com/ryanbogan) | Amazon |
 | Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon |
-| Shweta Thareja |[shwetathareja](https://github.com/shwetathareja) | Amazon |
-| Suraj Singh |[dreamer-89](https://github.com/dreamer-89) | Amazon |
+| Shweta Thareja | [shwetathareja](https://github.com/shwetathareja) | Amazon |
+| Suraj Singh | [dreamer-89](https://github.com/dreamer-89) | Amazon |
 | Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon |
 | Vacha Shah | [VachaShah](https://github.com/VachaShah) | Amazon |
 | Xue Zhou | [xuezhou25](https://github.com/xuezhou25) | Amazon |
@@ -32,6 +31,7 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
+| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon |
 | Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon |
 
 [This document](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).

TESTING.md

@@ -383,6 +383,10 @@ Use -Dtest.class and -Dtests.method to run a specific bwcTest test. For example
      -Dtests.class=org.opensearch.upgrades.RecoveryIT \
      -Dtests.method=testHistoryUUIDIsGenerated
 
+Use `-PcustomDistributionDownloadType=bundle` to run the bwcTest against the test cluster with latest CI distribution bundle set up for the specified version; this property is default to min and exclusive choices between `bundle` and `min`:
+
+    ./gradlew bwcTest -PcustomDistributionDownloadType=bundle
+
 When running `./gradlew check`, minimal bwc checks are also run against compatible versions that are not yet released.
 
 ## BWC Testing against a specific remote/branch

buildSrc/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   api localGroovy()
 
   api 'commons-codec:commons-codec:1.15'
-  api 'org.apache.commons:commons-compress:1.21'
+  api 'org.apache.commons:commons-compress:1.22'
   api 'org.apache.ant:ant:1.10.12'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:8.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.6.0'

buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java

@@ -73,6 +73,8 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
     private static final String SNAPSHOT_PATTERN_LAYOUT =
         "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier])-latest.[ext]";
+    private static final String BUNDLE_PATTERN_LAYOUT =
+        "/ci/dbc/distribution-build-opensearch/[revision]/latest/linux/x64/tar/dist/opensearch/[module]-[revision](-[classifier]).[ext]";
 
     private NamedDomainObjectContainer<OpenSearchDistribution> distributionsContainer;
     private NamedDomainObjectContainer<DistributionResolution> distributionsResolutionStrategiesContainer;
@@ -174,20 +176,39 @@ private static void setupDownloadServiceRepo(Project project) {
             return;
         }
         Object customDistributionUrl = project.findProperty("customDistributionUrl");
-        // checks if custom Distribution Url has been passed by user from plugins
+        Object customDistributionDownloadType = project.findProperty("customDistributionDownloadType");
+        // distributionDownloadType is default min if is not specified; download the distribution from CI if is bundle
+        String distributionDownloadType = customDistributionDownloadType != null
+            && customDistributionDownloadType.toString().equals("bundle") ? "bundle" : "min";
         if (customDistributionUrl != null) {
             addIvyRepo(project, DOWNLOAD_REPO_NAME, customDistributionUrl.toString(), FAKE_IVY_GROUP, "");
             addIvyRepo(project, SNAPSHOT_REPO_NAME, customDistributionUrl.toString(), FAKE_SNAPSHOT_IVY_GROUP, "");
-        } else {
-            addIvyRepo(
-                project,
-                DOWNLOAD_REPO_NAME,
-                "https://artifacts.opensearch.org",
-                FAKE_IVY_GROUP,
-                "/releases" + RELEASE_PATTERN_LAYOUT,
-                "/release-candidates" + RELEASE_PATTERN_LAYOUT
-            );
-            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
+            return;
+        }
+        switch (distributionDownloadType) {
+            case "bundle":
+                addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://ci.opensearch.org", FAKE_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://ci.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                break;
+            case "min":
+                addIvyRepo(
+                    project,
+                    DOWNLOAD_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_IVY_GROUP,
+                    "/releases" + RELEASE_PATTERN_LAYOUT,
+                    "/release-candidates" + RELEASE_PATTERN_LAYOUT
+                );
+                addIvyRepo(
+                    project,
+                    SNAPSHOT_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_SNAPSHOT_IVY_GROUP,
+                    SNAPSHOT_PATTERN_LAYOUT
+                );
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported property argument: " + distributionDownloadType);
         }
     }
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.5.0-snapshot-a4ef70f
+lucene            = 9.5.0-snapshot-d5cef1c
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 19.0.1+10
@@ -24,7 +24,7 @@ kotlin            = 1.7.10
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.12.1
 
-netty             = 4.1.84.Final
+netty             = 4.1.86.Final
 joda              = 2.10.13
 
 # client dependencies

modules/ingest-geoip/build.gradle

@@ -41,8 +41,6 @@ opensearchplugin {
 dependencies {
   api('com.maxmind.geoip2:geoip2:3.0.2')
   // geoip2 dependencies:
-  api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
   api('com.maxmind.db:maxmind-db:2.1.0')
 
   testImplementation 'org.elasticsearch:geolite2-databases:20191119'

modules/lang-expression/licenses/lucene-expressions-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+afe877ebf5ec4be7d17636b695015c449a523a3b

modules/transport-netty4/build.gradle

@@ -125,7 +125,8 @@ thirdPartyAudit {
     'com.aayushatharva.brotli4j.Brotli4jLoader',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
     'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
-    'com.aayushatharva.brotli4j.encoder.Encoders',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder',
     'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
     'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
     // classes are missing

modules/transport-netty4/licenses/netty-buffer-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+a66fa0ed2687eb33a2e53a17a6df61bfe3b3f2bd

modules/transport-netty4/licenses/netty-codec-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+ee126da926ea202da3b21eb737788ef83b1db772

modules/transport-netty4/licenses/netty-codec-http-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+23674593f004959ae002ec348626eecf677191ae

modules/transport-netty4/licenses/netty-codec-http2-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+ac0ca067e4118533ad1038776fcd9d5f3058b7d4

modules/transport-netty4/licenses/netty-common-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+1dceab4662a9cc93faf87b237bb41103b1bc7f0e

modules/transport-netty4/licenses/netty-handler-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+0bcb65230218286e6456b5d085cb42e67776eb70

modules/transport-netty4/licenses/netty-resolver-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+bad83d479f7bd8ea84eefd77c316435be4c97270

modules/transport-netty4/licenses/netty-transport-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+7c7739c41fd110c3576e9faace332ee957f27203

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.86.Final.jar.sha1

@@ -0,0 +1 @@
+5e0e7fc1c337485cabcf7971faefe692b76f93a2

modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4NioServerSocketChannel.java

@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import io.netty.channel.socket.InternetProtocolFamily;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.util.internal.SocketUtils;
+import io.netty.util.internal.logging.InternalLogger;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.spi.SelectorProvider;
+import java.util.List;
+
+public class Netty4NioServerSocketChannel extends NioServerSocketChannel {
+    private static final InternalLogger logger = InternalLoggerFactory.getInstance(Netty4NioServerSocketChannel.class);
+
+    public Netty4NioServerSocketChannel() {
+        super();
+    }
+
+    public Netty4NioServerSocketChannel(SelectorProvider provider) {
+        super(provider);
+    }
+
+    public Netty4NioServerSocketChannel(SelectorProvider provider, InternetProtocolFamily family) {
+        super(provider, family);
+    }
+
+    public Netty4NioServerSocketChannel(ServerSocketChannel channel) {
+        super(channel);
+    }
+
+    @Override
+    protected int doReadMessages(List<Object> buf) throws Exception {
+        SocketChannel ch = SocketUtils.accept(javaChannel());
+
+        try {
+            if (ch != null) {
+                buf.add(new Netty4NioSocketChannel(this, ch));
+                return 1;
+            }
+        } catch (Throwable t) {
+            logger.warn("Failed to create a new channel from an accepted socket.", t);
+
+            try {
+                ch.close();
+            } catch (Throwable t2) {
+                logger.warn("Failed to close a socket.", t2);
+            }
+        }
+
+        return 0;
+    }
+}

modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java

@@ -39,7 +39,6 @@
 import io.netty.buffer.UnpooledByteBufAllocator;
 import io.netty.channel.Channel;
 import io.netty.channel.ServerChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.Booleans;
@@ -181,7 +180,7 @@ public static Class<? extends ServerChannel> getServerChannelType() {
         if (ALLOCATOR instanceof NoDirectBuffers) {
             return CopyBytesServerSocketChannel.class;
         } else {
-            return NioServerSocketChannel.class;
+            return Netty4NioServerSocketChannel.class;
         }
     }
 

plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+670d8f48ea9cba542e263d3ec6c3e2a33accc561

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+b1f42bad26470c8ef88096e0c8564a74223c52c9

plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+f3cf74fa91da5133667f8916f93071fed231f2ee

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+e633c2d0cd677e4f1cef5aadc6bdc65e8e898d98

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+c12a2943e6f4977f15d489ac3e9802c5dfb3c4cc

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+5d257928a34e586a7de9fc7d4a013868f7a1db74

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0-snapshot-d5cef1c.jar.sha1

@@ -0,0 +1 @@
+3d88f80ad07421b9470cb44a6f5b67dd47047b13

plugins/discovery-ec2/build.gradle

@@ -46,8 +46,6 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
 restResources {

plugins/ingest-attachment/build.gradle

@@ -38,10 +38,10 @@ opensearchplugin {
 }
 
 versions << [
-  'tika'  : '2.5.0',
-  'pdfbox': '2.0.25',
-  'poi'   : '5.2.2',
-  'mime4j': '0.8.3'
+  'tika'  : '2.6.0',
+  'pdfbox': '2.0.27',
+  'poi'   : '5.2.3',
+  'mime4j': '0.8.8'
 ]
 
 dependencies {

plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.8.jar.sha1

@@ -0,0 +1 @@
+7330de23c52f71617cbec7f1d2760dae32e687cd

plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.8.jar.sha1

@@ -0,0 +1 @@
+e76715563a6bd150f84ccb0adb920aec8faf4779

plugins/ingest-attachment/licenses/fontbox-2.0.27.jar.sha1

@@ -0,0 +1 @@
+d08c064d18b2b149da937d15c0d1708cba03f29d

plugins/ingest-attachment/licenses/pdfbox-2.0.27.jar.sha1

@@ -0,0 +1 @@
+416a9dfce3714116bfdf793b15368df04266845f

plugins/ingest-attachment/licenses/poi-5.2.3.jar.sha1

@@ -0,0 +1 @@
+2fb22ae74ad5aea6af1a9c64b9542f2ccf348604

plugins/ingest-attachment/licenses/poi-ooxml-5.2.3.jar.sha1

@@ -0,0 +1 @@
+02efd11c940adb18c03eb9ce7ad88fc40ee6a196