Merge branch 'main' into feature/segment-replication

Signed-off-by: Marc Handalian <[email protected]>

.ci/bwcVersions

@@ -37,4 +37,5 @@ BWC_VERSION:
   - "1.2.4"
   - "1.2.5"
   - "1.3.0"
-  - "1.4.0"
+  - "1.3.1"
+  - "1.4.0"

README.md

@@ -6,6 +6,7 @@
 - [Security](#security)
 - [License](#license)
 - [Copyright](#copyright)
+- [Trademark](#trademark)
 
 ## Welcome!
 

benchmarks/src/main/java/org/opensearch/benchmark/routing/allocation/Allocators.java

@@ -111,7 +111,7 @@ public static DiscoveryNode newNode(String nodeId, Map<String, String> attribute
             nodeId,
             new TransportAddress(TransportAddress.META_ADDRESS, portGenerator.incrementAndGet()),
             attributes,
-            Sets.newHashSet(DiscoveryNodeRole.MASTER_ROLE, DiscoveryNodeRole.DATA_ROLE),
+            Sets.newHashSet(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE),
             Version.CURRENT
         );
     }

buildSrc/build.gradle

@@ -110,19 +110,19 @@ dependencies {
   api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
   api 'org.apache.rat:apache-rat:0.13'
   api 'commons-io:commons-io:2.7'
-  api "net.java.dev.jna:jna:5.5.0"
+  api "net.java.dev.jna:jna:5.10.0"
   api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
   api 'de.thetaphi:forbiddenapis:3.2'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.12'
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.36'
+  api 'com.networknt:json-schema-validator:1.0.67'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testFixturesApi gradleApi()
   testFixturesApi gradleTestKit()
-  testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
+  testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.32.0'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
   integTestImplementation('org.spockframework:spock-core:2.1-groovy-3.0') {
     exclude module: "groovy"

buildSrc/src/main/groovy/org/opensearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -123,7 +123,7 @@ class PluginBuildPlugin implements Plugin<Project> {
             naming.clear()
             naming {
                 Tests {
-                    baseClass 'org.apache.lucene.util.LuceneTestCase'
+                    baseClass 'org.apache.lucene.tests.util.LuceneTestCase'
                 }
                 IT {
                     baseClass 'org.opensearch.test.OpenSearchIntegTestCase'

buildSrc/src/main/groovy/org/opensearch/gradle/test/ClusterFormationTasks.groovy

@@ -153,7 +153,8 @@ class ClusterFormationTasks {
                     }
                     boolean supportsInitialMasterNodes = hasBwcNodes == false || config.bwcVersion.onOrAfter("7.0.0")
                     if (esConfig['discovery.type'] == null && config.getAutoSetInitialMasterNodes() && supportsInitialMasterNodes) {
-                        esConfig['cluster.initial_master_nodes'] = nodes.stream().map({ n ->
+                        // To promote inclusive language, the old setting name is deprecated in 2.0.0
+                        esConfig[node.nodeVersion.onOrAfter("2.0.0") ? 'cluster.initial_cluster_manager_nodes' : 'cluster.initial_master_nodes'] = nodes.stream().map({ n ->
                             if (n.config.settings['node.name'] == null) {
                                 return "node-" + n.nodeNum
                             } else {

buildSrc/src/main/java/org/opensearch/gradle/precommit/TestingConventionsPrecommitPlugin.java

@@ -43,7 +43,7 @@ public TaskProvider<? extends Task> createTask(Project project) {
             .register("testingConventions", TestingConventionsTasks.class);
         testingConventions.configure(t -> {
             TestingConventionRule testsRule = t.getNaming().maybeCreate("Tests");
-            testsRule.baseClass("org.apache.lucene.util.LuceneTestCase");
+            testsRule.baseClass("org.apache.lucene.tests.util.LuceneTestCase");
             TestingConventionRule itRule = t.getNaming().maybeCreate("IT");
             itRule.baseClass("org.opensearch.test.OpenSearchIntegTestCase");
             itRule.baseClass("org.opensearch.test.rest.OpenSearchRestTestCase");

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java

@@ -361,7 +361,12 @@ private void commonNodeConfig(OpenSearchNode node, String nodeNames, OpenSearchN
                 .collect(Collectors.toList())
                 .forEach(node.defaultConfig::remove);
             if (nodeNames != null && node.settings.getOrDefault("discovery.type", "anything").equals("single-node") == false) {
-                node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
+                // To promote inclusive language, the old setting name is deprecated n 2.0.0
+                if (node.getVersion().onOrAfter("2.0.0")) {
+                    node.defaultConfig.put("cluster.initial_cluster_manager_nodes", "[" + nodeNames + "]");
+                } else {
+                    node.defaultConfig.put("cluster.initial_master_nodes", "[" + nodeNames + "]");
+                }
             }
             node.defaultConfig.put("discovery.seed_providers", "file");
             node.defaultConfig.put("discovery.seed_hosts", "[]");

buildSrc/src/main/resources/forbidden/opensearch-test-signatures.txt

@@ -19,9 +19,9 @@ com.carrotsearch.randomizedtesting.annotations.Seed @ Don't commit hardcoded see
 com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded repeats
 
 org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
-org.apache.lucene.util.LuceneTestCase$Slow @ Don't write slow tests
+org.apache.lucene.tests.util.LuceneTestCase$Slow @ Don't write slow tests
 org.junit.Ignore @ Use AwaitsFix instead
-org.apache.lucene.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
+org.apache.lucene.tests.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
 com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!
 
 org.junit.Test @defaultMessage Just name your test method testFooBar

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 2.0.0
-lucene            = 9.0.0
+lucene            = 9.1.0-snapshot-ea989fe8f30
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.2+8
@@ -9,9 +9,9 @@ bundled_jdk = 17.0.2+8
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.12.6
+jackson           = 2.13.2
 snakeyaml         = 1.26
-icu4j             = 68.2
+icu4j             = 70.1
 supercsv          = 2.4.0
 log4j             = 2.17.1
 slf4j             = 1.6.2

client/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java

@@ -37,7 +37,7 @@
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.admin.indices.alias.Alias;
 import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;

client/rest/src/main/java/org/opensearch/client/Node.java

@@ -210,21 +210,21 @@ public Roles(final Set<String> roles) {
         }
 
         /**
-         * Teturns whether or not the node <strong>could</strong> be elected master.
+         * Returns whether or not the node <strong>could</strong> be elected master.
          */
         public boolean isMasterEligible() {
-            return roles.contains("master");
+            return roles.contains("master") || roles.contains("cluster_manager");
         }
 
         /**
-         * Teturns whether or not the node stores data.
+         * Returns whether or not the node stores data.
          */
         public boolean isData() {
             return roles.contains("data");
         }
 
         /**
-         * Teturns whether or not the node runs ingest pipelines.
+         * Returns whether or not the node runs ingest pipelines.
          */
         public boolean isIngest() {
             return roles.contains("ingest");

client/sniffer/licenses/jackson-core-2.13.2.jar.sha1

@@ -0,0 +1 @@
+0a6a0e0620d51833feffc67bccb51937b2345763

distribution/docker/docker-compose.yml

@@ -5,7 +5,7 @@ services:
     image: opensearch:test
     environment:
        - node.name=opensearch-1
-       - cluster.initial_master_nodes=opensearch-1,opensearch-2
+       - cluster.initial_cluster_manager_nodes=opensearch-1,opensearch-2
        - discovery.seed_hosts=opensearch-2:9300
        - cluster.name=opensearch
        - bootstrap.memory_lock=true
@@ -29,7 +29,7 @@ services:
     image: opensearch:test
     environment:
        - node.name=opensearch-2
-       - cluster.initial_master_nodes=opensearch-1,opensearch-2
+       - cluster.initial_cluster_manager_nodes=opensearch-1,opensearch-2
        - discovery.seed_hosts=opensearch-1:9300
        - cluster.name=opensearch
        - bootstrap.memory_lock=true

distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml

@@ -1,22 +1,40 @@
+---
+"Test cat nodes output - before 2.0.0":
+  - skip:
+      version: "2.0.0 - "
+      reason: "master is replaced by cluster_manager in 2.0.0"
+
+  - do:
+      cat.nodes:
+        v: true
+
+  - match:
+      $body: |
+        /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role              \s+  master   \s+   name  \n
+           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdhilmrstvw]{1,11}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+
 ---
 "Test cat nodes output":
+  - skip:
+      version: " - 1.4.99"
+      reason: "cluster_manager is introduced in 2.0.0"
 
   - do:
       cat.nodes: {}
 
   - match:
       $body: |
-        /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role                   master          name
-        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[cdhilmrstvw]{1,11}) \s+ [-*x]     \s+   (\S+\s?)+     \n)+  $/
+        /  #ip                          heap.percent        ram.percent     cpu         load_1m                load_5m                load_15m               node.role                   cluster_manager       name
+        ^  ((\d{1,3}\.){3}\d{1,3}  \s+  \d+            \s+  \d*         \s+ (-)?\d* \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)?\s+  ((-)?\d*(\.\d+)?)? \s+ (-|[cdhilmrstvw]{1,11}) \s+ [-*x]            \s+  (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:
         v: true
 
   - match:
       $body: |
-        /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role              \s+  master   \s+   name  \n
-           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdhilmrstvw]{1,11}) \s+  [-*x]    \s+   (\S+\s?)+     \n)+  $/
+        /^  ip                     \s+  heap\.percent   \s+  ram\.percent \s+ cpu      \s+ load_1m            \s+ load_5m            \s+ load_15m           \s+ node\.role              \s+  cluster_manager   \s+   name  \n
+           ((\d{1,3}\.){3}\d{1,3}  \s+  \d+             \s+  \d*          \s+ (-)?\d* \s+  ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ ((-)?\d*(\.\d+)?)? \s+ (-|[cdhilmrstvw]{1,11}) \s+  [-*x]             \s+   (\S+\s?)+     \n)+  $/
 
   - do:
       cat.nodes:

distribution/src/config/opensearch.yml

@@ -67,9 +67,9 @@ ${path.logs}
 #
 #discovery.seed_hosts: ["host1", "host2"]
 #
-# Bootstrap the cluster using an initial set of master-eligible nodes:
+# Bootstrap the cluster using an initial set of cluster-manager-eligible nodes:
 #
-#cluster.initial_master_nodes: ["node-1", "node-2"]
+#cluster.initial_cluster_manager_nodes: ["node-1", "node-2"]
 #
 # For more information, consult the discovery and cluster formation module documentation.
 #

distribution/tools/keystore-cli/src/test/java/org/opensearch/common/settings/KeyStoreCommandTestCase.java

@@ -43,7 +43,7 @@
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
 import org.opensearch.core.internal.io.IOUtils;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.cli.CommandTestCase;
 import org.opensearch.common.io.PathUtilsForTesting;
 import org.opensearch.env.Environment;

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java

@@ -35,7 +35,7 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.bouncycastle.bcpg.ArmoredOutputStream;
 import org.bouncycastle.bcpg.BCPGOutputStream;
 import org.bouncycastle.bcpg.HashAlgorithmTags;

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java

@@ -40,7 +40,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.LegacyESVersion;
 import org.opensearch.Version;
 import org.opensearch.cli.ExitCodes;

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/RemovePluginCommandTests.java

@@ -32,7 +32,7 @@
 
 package org.opensearch.plugins;
 
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.Version;
 import org.opensearch.cli.ExitCodes;
 import org.opensearch.cli.MockTerminal;

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.13.2.jar.sha1

@@ -0,0 +1 @@
+ec18851f1976d5b810ae1a5fcc32520d2d38f77a

distribution/tools/upgrade-cli/licenses/jackson-databind-2.13.2.jar.sha1

@@ -0,0 +1 @@
+926e48c451166a291f1ce6c6276d9abbefa7c00f

libs/core/src/test/java/org/opensearch/core/internal/io/IOUtilsTests.java

@@ -30,8 +30,8 @@
 
 package org.opensearch.core.internal.io;
 
-import org.apache.lucene.mockfile.FilterFileSystemProvider;
-import org.apache.lucene.mockfile.FilterPath;
+import org.apache.lucene.tests.mockfile.FilterFileSystemProvider;
+import org.apache.lucene.tests.mockfile.FilterPath;
 import org.apache.lucene.util.Constants;
 import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.io.PathUtils;

libs/x-content/licenses/jackson-core-2.13.2.jar.sha1

@@ -0,0 +1 @@
+0a6a0e0620d51833feffc67bccb51937b2345763

libs/x-content/licenses/jackson-dataformat-cbor-2.13.2.jar.sha1

@@ -0,0 +1 @@
+4fc77e1ec6922fc48bf1181e4b38f600dac222ff

libs/x-content/licenses/jackson-dataformat-smile-2.13.2.jar.sha1

@@ -0,0 +1 @@
+984bb22f310ebbedc967d206e672f8acf766a98e

libs/x-content/licenses/jackson-dataformat-yaml-2.13.2.jar.sha1

@@ -0,0 +1 @@
+5601496b5b6e43d947aeeffbffadb2b18961c731

modules/aggs-matrix-stats/src/test/java/org/opensearch/search/aggregations/matrix/stats/MatrixStatsAggregatorTests.java

@@ -36,7 +36,7 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;

modules/analysis-common/src/internalClusterTest/java/org/opensearch/analysis/common/QueryStringWithAnalyzersIT.java

@@ -73,7 +73,7 @@ public void testCustomWordDelimiterQueryString() {
                         .put("analysis.filter.custom_word_delimiter.split_on_numerics", "false")
                         .put("analysis.filter.custom_word_delimiter.stem_english_possessive", "false")
                 )
-                .addMapping("type1", "field1", "type=text,analyzer=my_analyzer", "field2", "type=text,analyzer=my_analyzer")
+                .setMapping("field1", "type=text,analyzer=my_analyzer", "field2", "type=text,analyzer=my_analyzer")
         );
 
         client().prepareIndex("test").setId("1").setSource("field1", "foo bar baz", "field2", "not needed").get();

modules/analysis-common/src/test/java/org/opensearch/analysis/common/ConcatenateGraphTokenFilterFactoryTests.java

@@ -8,8 +8,8 @@
 
 package org.opensearch.analysis.common;
 
-import org.apache.lucene.analysis.CannedTokenStream;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.tests.analysis.CannedTokenStream;
+import org.apache.lucene.tests.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;

modules/analysis-common/src/test/java/org/opensearch/analysis/common/FlattenGraphTokenFilterFactoryTests.java

@@ -32,8 +32,8 @@
 
 package org.opensearch.analysis.common;
 
-import org.apache.lucene.analysis.CannedTokenStream;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.tests.analysis.CannedTokenStream;
+import org.apache.lucene.tests.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.Index;