merge main

Signed-off-by: Gao Binlong <[email protected]>

.github/workflows/check-compatibility.yml

@@ -36,7 +36,7 @@ jobs:
           echo "### Compatible components"   >> "${{ github.workspace }}/results.txt" && grep -e 'Compatible component'   $HOME/gradlew-check.out | sed -e 's/Compatible component: \[\(.*\)\]/- \1/'   >> "${{ github.workspace }}/results.txt"
 
       - name: Upload results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: results.txt
           path: ${{ github.workspace }}/results.txt
@@ -48,7 +48,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download results
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: results.txt
 

buildSrc/build.gradle

@@ -106,7 +106,7 @@ dependencies {
   api 'org.apache.commons:commons-compress:1.25.0'
   api 'org.apache.ant:ant:1.10.14'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
-  api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
+  api 'com.netflix.nebula:nebula-publishing-plugin:21.0.0'
   api 'com.netflix.nebula:gradle-info-plugin:12.1.6'
   api 'org.apache.rat:apache-rat:0.15'
   api 'commons-io:commons-io:2.15.1'

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.22.0"
+  implementation "org.apache.logging.log4j:log4j-core:2.22.1"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.5.0"
+  id "com.netflix.nebula.ospackage-base" version "11.6.0"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

distribution/src/config/jvm.options

@@ -38,12 +38,12 @@
 8-10:-XX:+UseCMSInitiatingOccupancyOnly
 
 ## G1GC Configuration
-# NOTE: G1 GC is only supported on JDK version 10 or later
-# to use G1GC, uncomment the next two lines and update the version on the
-# following three lines to your version of the JDK
-# 10:-XX:-UseConcMarkSweepGC
-# 10:-XX:-UseCMSInitiatingOccupancyOnly
+# NOTE: G1GC is the default GC for all JDKs 11 and newer
 11-:-XX:+UseG1GC
+# See https://github.com/elastic/elasticsearch/pull/46169 for the history
+# behind these settings, but the tl;dr is that default values can lead
+# to situations where heap usage grows enough to trigger a circuit breaker
+# before GC kicks in.
 11-:-XX:G1ReservePercent=25
 11-:-XX:InitiatingHeapOccupancyPercent=30
 

libs/core/build.gradle

@@ -36,45 +36,6 @@ base {
   archivesName = 'opensearch-core'
 }
 
-// we want to keep the JDKs in our IDEs set to JDK 8 until minimum JDK is bumped to 11 so we do not include this source set in our IDEs
-if (!isEclipse) {
-  sourceSets {
-    java11 {
-      java {
-        srcDirs = ['src/main/java11']
-      }
-    }
-  }
-
-  configurations {
-    java11Compile.extendsFrom(compile)
-  }
-
-  dependencies {
-    java11Implementation sourceSets.main.output
-  }
-
-  compileJava11Java {
-    sourceCompatibility = JavaVersion.VERSION_11
-    targetCompatibility = JavaVersion.VERSION_11
-  }
-
-  forbiddenApisJava11 {
-    if (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_11) {
-      targetCompatibility = JavaVersion.VERSION_11
-    }
-    replaceSignatureFiles 'jdk-signatures'
-  }
-
-  jar {
-    metaInf {
-      into 'versions/11'
-      from sourceSets.java11.output
-    }
-    manifest.attributes('Multi-Release': 'true')
-  }
-}
-
 dependencies {
   api project(':libs:opensearch-common')
 

modules/ingest-geoip/build.gradle

@@ -39,9 +39,9 @@ opensearchplugin {
 }
 
 dependencies {
-  api('com.maxmind.geoip2:geoip2:4.1.0')
+  api('com.maxmind.geoip2:geoip2:4.2.0')
   // geoip2 dependencies:
-  api('com.maxmind.db:maxmind-db:3.0.0')
+  api('com.maxmind.db:maxmind-db:3.1.0')
   api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
   api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
 

modules/ingest-geoip/licenses/geoip2-4.2.0.jar.sha1

@@ -0,0 +1 @@
+78ff932dc13ac41dd1f0fd9e7405a7f4ad815ce0

modules/ingest-geoip/licenses/maxmind-db-3.1.0.jar.sha1

@@ -0,0 +1 @@
+2008992ab45d61c7b28a18678b5df82272529da3

modules/lang-mustache/src/main/java/org/opensearch/script/mustache/SearchTemplateRequest.java

@@ -259,16 +259,25 @@ public void writeTo(StreamOutput out) throws IOException {
 
     @Override
     public String[] indices() {
+        if (request == null) {
+            return new String[0];
+        }
         return request.indices();
     }
 
     @Override
     public IndicesOptions indicesOptions() {
+        if (request == null) {
+            return SearchRequest.DEFAULT_INDICES_OPTIONS;
+        }
         return request.indicesOptions();
     }
 
     @Override
     public IndicesRequest indices(String... indices) {
+        if (request == null) {
+            return new SearchRequest(new String[0]).indices(indices);
+        }
         return request.indices(indices);
     }
 }

modules/lang-mustache/src/test/java/org/opensearch/script/mustache/SearchTemplateRequestTests.java

@@ -32,6 +32,7 @@
 
 package org.opensearch.script.mustache;
 
+import org.opensearch.action.search.SearchRequest;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.script.ScriptType;
 import org.opensearch.search.RandomSearchRequestGenerator;
@@ -110,4 +111,19 @@ public static SearchTemplateRequest createRandomRequest() {
         request.setRequest(RandomSearchRequestGenerator.randomSearchRequest(SearchSourceBuilder::searchSource));
         return request;
     }
+
+    public void testSimulatedSearchTemplateRequest() {
+        SearchTemplateRequest request = new SearchTemplateRequest();
+        request.setSimulate(true);
+
+        assertEquals(0, request.indices().length);
+        assertEquals(SearchRequest.DEFAULT_INDICES_OPTIONS, request.indicesOptions());
+        assertEquals(2, request.indices("index1", "index2").indices().length);
+
+        SearchTemplateRequest randomRequest = createRandomRequest();
+        int expectedIndicesLength = randomRequest.indices().length;
+        request.setSimulate(true);
+
+        assertEquals(expectedIndicesLength, randomRequest.indices().length);
+    }
 }

plugins/discovery-ec2/src/main/java/org/opensearch/discovery/ec2/AwsEc2ServiceImpl.java

@@ -114,7 +114,7 @@ protected String getFullEndpoint(String endpoint) {
         if (!Strings.hasText(endpoint)) {
             return null;
         }
-        if (endpoint.startsWith("http")) {
+        if (endpoint.startsWith("http://") || endpoint.startsWith("https://")) {
             return endpoint;
         }
 

plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/AwsEc2ServiceImplTests.java

@@ -211,6 +211,10 @@ public void testGetFullEndpointWithScheme() {
 
         String endpoint = awsEc2ServiceImpl.getFullEndpoint(clientSettings.endpoint);
         assertEquals("http://ec2.us-west-2.amazonaws.com", endpoint);
+
+        assertEquals("http://httpserver.example.com", awsEc2ServiceImpl.getFullEndpoint("http://httpserver.example.com"));
+
+        assertEquals("https://httpserver.example.com", awsEc2ServiceImpl.getFullEndpoint("https://httpserver.example.com"));
     }
 
     public void testGetFullEndpointWithoutScheme() {
@@ -222,6 +226,8 @@ public void testGetFullEndpointWithoutScheme() {
         String endpoint = awsEc2ServiceImpl.getFullEndpoint(clientSettings.endpoint);
         assertEquals("https://ec2.us-west-2.amazonaws.com", endpoint);
 
+        assertEquals("https://httpserver.example.com", awsEc2ServiceImpl.getFullEndpoint("httpserver.example.com"));
+
         assertNull(awsEc2ServiceImpl.getFullEndpoint(""));
     }
 }

plugins/ingest-attachment/build.gradle

@@ -57,7 +57,7 @@ dependencies {
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   // Other dependencies
   api 'org.tukaani:xz:1.9'
-  api 'commons-io:commons-io:2.15.0'
+  api 'commons-io:commons-io:2.15.1'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 
   // character set detection

plugins/ingest-attachment/licenses/commons-io-2.15.1.jar.sha1

@@ -0,0 +1 @@
+f11560da189ab563a5c8e351941415430e9304ea

plugins/repository-hdfs/build.gradle

@@ -70,13 +70,13 @@ dependencies {
   api 'com.google.code.gson:gson:2.10.1'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api 'commons-cli:commons-cli:1.5.0'
+  api 'commons-cli:commons-cli:1.6.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
   api 'org.apache.commons:commons-configuration2:2.9.0'
   api 'commons-io:commons-io:2.14.0'
-  api 'org.apache.commons:commons-lang3:3.13.0'
+  api 'org.apache.commons:commons-lang3:3.14.0'
   implementation 'com.google.re2j:re2j:1.7'
   api 'javax.servlet:servlet-api:2.5'
   api "org.slf4j:slf4j-api:${versions.slf4j}"

plugins/repository-hdfs/licenses/commons-cli-1.6.0.jar.sha1

@@ -0,0 +1 @@
+38166a23afb5bd5520f739b87b3be87f7f0fb96d

plugins/repository-hdfs/licenses/commons-lang3-3.14.0.jar.sha1

@@ -0,0 +1 @@
+1ed471194b02f2c6cb734a0cd6f6f107c673afae

rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/40_max_shard_size.yml

@@ -4,8 +4,8 @@
   # number_of_shards for the target index.
 
   - skip:
-      version: " - 2.99.99"
-      reason: "only available in 3.0+"
+      version: " - 2.4.99"
+      reason: "max_shard_size was introduced in 2.5.0"
       features: allowed_warnings
 
   - do:

server/build.gradle

@@ -57,45 +57,6 @@ sourceSets {
     }
   }
 }
-// we want to keep the JDKs in our IDEs set to JDK 8 until minimum JDK is bumped to 11 so we do not include this source set in our IDEs
-if (!isEclipse) {
-  sourceSets {
-    java11 {
-      java {
-        srcDirs = ['src/main/java11']
-      }
-    }
-  }
-
-  configurations {
-    java11Implementation.extendsFrom(api)
-  }
-
-  dependencies {
-    java11Implementation sourceSets.main.output
-  }
-
-  compileJava11Java {
-    sourceCompatibility = JavaVersion.VERSION_11
-    targetCompatibility = JavaVersion.VERSION_11
-  }
-
-  tasks.named('forbiddenApisJava11').configure {
-    doFirst {
-      if (BuildParams.runtimeJavaVersion < JavaVersion.VERSION_11) {
-        targetCompatibility = JavaVersion.VERSION_11
-      }
-    }
-  }
-
-  jar {
-    metaInf {
-      into 'versions/11'
-      from sourceSets.java11.output
-    }
-    manifest.attributes('Multi-Release': 'true')
-  }
-}
 
 dependencies {