Integration test against recent OpenSearch versions

.github/actions/build-opensearch/action.yml

@@ -15,7 +15,7 @@ inputs:
   plugins_output_directory:
     description: The directory to output the plugins to
     default: ""
-outputs: 
+outputs:
     distribution:
       description: The path to the OpenSearch distribution
       value: ${{ steps.determine.outputs.distribution }}
@@ -36,11 +36,11 @@ runs:
           ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz
           ./opensearch/plugins/*/build/distributions/*.zip
         build_script: |
-          ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          ./gradlew --stacktrace :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
 
           PluginList=("analysis-icu"  "analysis-kuromoji"  "analysis-nori"  "analysis-phonetic"  "ingest-attachment"  "mapper-murmur3")
           for plugin in ${PluginList[*]}; do
-            ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
+            ./gradlew --stacktrace :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }}
           done
 
     - name: Determine OpenSearch distribution path and version
@@ -62,7 +62,7 @@ runs:
         cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }}
         cached_paths: |
           ./opensearch-security/build/distributions/opensearch-security-*.zip
-        build_script: ./gradlew assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+        build_script: ./gradlew --stacktrace assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
 
     - name: Restore or Build OpenSearch k-NN
       uses: ./client/.github/actions/cached-git-build
@@ -76,7 +76,7 @@ runs:
           ./opensearch-knn/build/distributions/opensearch-knn-*.zip
         build_script: |
           sudo apt-get install -y libopenblas-dev libomp-dev
-          ./gradlew buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
+          ./gradlew --stacktrace buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }}
           distributions=./build/distributions
           lib_dir=$distributions/lib
           mkdir $lib_dir

.github/workflows/integration-yaml-tests.yml

@@ -14,7 +14,9 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - 2.11.1
+          - 2.16.0
+          - 2.14.0
+          - 2.12.0
           - 2.10.0
           - 2.8.0
           - 2.6.0
@@ -85,7 +87,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        opensearch_ref: ['1.x', '2.x', 'main']
+        include:
+          - { opensearch_ref: '1.x', java_version: '11' }
+          - { opensearch_ref: '2.x', java_version: '17' }
+          - { opensearch_ref: 'main', java_version: '21' }
     steps:
       - name: Checkout Client
         uses: actions/checkout@v4
@@ -109,8 +114,8 @@ jobs:
       - name: Set up JDK
         uses: actions/setup-java@v4
         with:
-          java-version: 21
           distribution: 'temurin'
+          java-version: ${{ matrix.java_version }}
 
       - name: Restore or Build OpenSearch
         id: opensearch_build

.github/workflows/integration.yml

@@ -17,7 +17,9 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - 2.11.1
+          - 2.16.0
+          - 2.14.0
+          - 2.12.0
           - 2.10.0
           - 2.8.0
           - 2.6.0
@@ -48,7 +50,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-nuget-
 
-      - run: "./build.sh integrate $VERSION readonly,replicatedreadonly,writable random:test_only_one --report"
+      - run: "./build.sh integrate $VERSION readonly,multinode,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
         env:
@@ -67,10 +69,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        opensearch_ref:
-          - '1.x'
-          - '2.x'
-          - 'main'
+        include:
+          - { opensearch_ref: '1.x', java_version: '11' }
+          - { opensearch_ref: '2.x', java_version: '17' }
+          - { opensearch_ref: 'main', java_version: '21' }
 
     steps:
       - name: Checkout Client
@@ -98,6 +100,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-nuget-
 
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java_version }}
+
       - name: Restore or Build OpenSearch
         id: opensearch
         uses: ./client/.github/actions/build-opensearch
@@ -107,7 +115,7 @@ jobs:
           knn_plugin: true
           plugins_output_directory: ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }}
 
-      - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,replicatedreadonly,writable random:test_only_one --report"
+      - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,multinode,writable random:test_only_one --report"
         name: Integration Tests
         working-directory: client
         env:

abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs

@@ -60,13 +60,6 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo
 
 		protected EphemeralClusterComposer<TConfiguration> Composer { get; }
 
-		protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port)
-		{
-			base.ModifyNodeConfiguration(nodeConfiguration, port);
-
-			if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true");
-		}
-
 		public virtual ICollection<Uri> NodesUris(string hostName = null)
 		{
 			hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any()

abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs

@@ -29,74 +29,74 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text.RegularExpressions;
 using OpenSearch.OpenSearch.Ephemeral.Plugins;
 using OpenSearch.OpenSearch.Ephemeral.Tasks;
 using OpenSearch.OpenSearch.Managed.Configuration;
 using OpenSearch.Stack.ArtifactsApi;
-using OpenSearch.Stack.ArtifactsApi.Products;
 
-namespace OpenSearch.OpenSearch.Ephemeral
+namespace OpenSearch.OpenSearch.Ephemeral;
+
+public class EphemeralClusterConfiguration : ClusterConfiguration<EphemeralFileSystem>
 {
-	public class EphemeralClusterConfiguration : ClusterConfiguration<EphemeralFileSystem>
-	{
-		public EphemeralClusterConfiguration(OpenSearchVersion version, OpenSearchPlugins plugins = null,
-			int numberOfNodes = 1)
-			: this(version, ClusterFeatures.None, plugins, numberOfNodes)
-		{
-		}
-
-		public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures features,
-			OpenSearchPlugins plugins = null, int numberOfNodes = 1)
-			: base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName)
-		{
-			Features = features;
-
-			var pluginsList = plugins?.ToList() ?? new List<OpenSearchPlugin>();
-			Plugins = new OpenSearchPlugins(pluginsList);
-		}
-
-		private static string UniqueishSuffix => Guid.NewGuid().ToString("N").Substring(0, 6);
-		private static string EphemeralClusterName => $"ephemeral-cluster-{UniqueishSuffix}";
-
-		/// <summary>
-		///     The features supported by the cluster
-		/// </summary>
-		public ClusterFeatures Features { get; }
-
-		/// <summary>
-		///     The collection of plugins to install
-		/// </summary>
-		public OpenSearchPlugins Plugins { get; }
-
-		/// <summary>
-		///     Validates that the plugins to install can be installed on the target OpenSearch version.
-		///     This can be useful to fail early when subsequent operations are relying on installation
-		///     succeeding.
-		/// </summary>
-		public bool ValidatePluginsToInstall { get; } = true;
-
-		public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL);
-
-		public IList<IClusterComposeTask> AdditionalBeforeNodeStartedTasks { get; } = new List<IClusterComposeTask>();
-
-		public IList<IClusterComposeTask> AdditionalAfterStartedTasks { get; } = new List<IClusterComposeTask>();
-
-		/// <summary>
-		///     Expert level setting, skips all built-in validation tasks for cases where you need to guarantee your call is the
-		///     first call into the cluster
-		/// </summary>
-		public bool SkipBuiltInAfterStartTasks { get; set; }
-
-		/// <summary> Bootstrapping HTTP calls should attempt to auto route traffic through fiddler if its running </summary>
-		public bool HttpFiddlerAware { get; set; }
-
-		protected virtual string NodePrefix => "ephemeral";
-
-		public override string CreateNodeName(int? node)
-		{
-			var suffix = Guid.NewGuid().ToString("N").Substring(0, 6);
-			return $"{NodePrefix}-node-{suffix}{node}";
-		}
-	}
+    public EphemeralClusterConfiguration(OpenSearchVersion version, OpenSearchPlugins plugins = null,
+        int numberOfNodes = 1)
+        : this(version, ClusterFeatures.None, plugins, numberOfNodes)
+    {
+    }
+
+    public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures features,
+        OpenSearchPlugins plugins = null, int numberOfNodes = 1)
+        : base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName)
+    {
+        Features = features;
+
+        var pluginsList = plugins?.ToList() ?? [];
+        Plugins = new OpenSearchPlugins(pluginsList);
+
+        Add("plugins.security.disabled", (!EnableSsl).ToString().ToLowerInvariant());
+        if (EnableSsl) Add("plugins.security.audit.type", "debug");
+    }
+
+    private static string UniqueishSuffix => Guid.NewGuid().ToString("N").Substring(0, 6);
+    private static string EphemeralClusterName => $"ephemeral-cluster-{UniqueishSuffix}";
+
+    /// <summary>
+    ///     The features supported by the cluster
+    /// </summary>
+    public ClusterFeatures Features { get; }
+
+    /// <summary>
+    ///     The collection of plugins to install
+    /// </summary>
+    public OpenSearchPlugins Plugins { get; }
+
+    /// <summary>
+    ///     Validates that the plugins to install can be installed on the target OpenSearch version.
+    ///     This can be useful to fail early when subsequent operations are relying on installation
+    ///     succeeding.
+    /// </summary>
+    public bool ValidatePluginsToInstall { get; } = true;
+
+    public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL);
+
+    public IList<IClusterComposeTask> AdditionalBeforeNodeStartedTasks { get; } = new List<IClusterComposeTask>();
+
+    public IList<IClusterComposeTask> AdditionalAfterStartedTasks { get; } = new List<IClusterComposeTask>();
+
+    /// <summary>
+    ///     Expert level setting, skips all built-in validation tasks for cases where you need to guarantee your call is the
+    ///     first call into the cluster
+    /// </summary>
+    public bool SkipBuiltInAfterStartTasks { get; set; }
+
+    /// <summary> Bootstrapping HTTP calls should attempt to auto route traffic through fiddler if its running </summary>
+    public bool HttpFiddlerAware { get; set; }
+
+    protected virtual string NodePrefix => "ephemeral";
+
+    public override string CreateNodeName(int? node)
+    {
+        var suffix = Guid.NewGuid().ToString("N").Substring(0, 6);
+        return $"{NodePrefix}-node-{suffix}{node}";
+    }
 }

abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs

@@ -75,29 +75,32 @@ protected ClusterBase(TConfiguration clusterConfiguration)
 			ClusterConfiguration = clusterConfiguration;
 			ClusterMoniker = GetType().Name.Replace("Cluster", "");
 
-			NodeConfiguration Modify(NodeConfiguration n, int p)
-			{
-				ModifyNodeConfiguration(n, p);
-				return n;
-			}
-
-			var nodes =
-				(from port in Enumerable.Range(ClusterConfiguration.StartingPortNumber,
-						ClusterConfiguration.NumberOfNodes)
-					let config = new NodeConfiguration(clusterConfiguration, port, ClusterMoniker)
-					{
-						ShowOpenSearchOutputAfterStarted =
-							clusterConfiguration.ShowOpenSearchOutputAfterStarted,
-					}
-					let node = new OpenSearchNode(Modify(config, port))
-					{
-						AssumeStartedOnNotEnoughMasterPing = ClusterConfiguration.NumberOfNodes > 1,
-					}
-					select node).ToList();
-
-			var initialMasterNodes = string.Join(",", nodes.Select(n => n.NodeConfiguration.DesiredNodeName));
-			foreach (var node in nodes)
-				node.NodeConfiguration.InitialMasterNodes(initialMasterNodes);
+            var nodeConfigs = Enumerable.Range(ClusterConfiguration.StartingPortNumber, ClusterConfiguration.NumberOfNodes)
+                .Select(port => new NodeConfiguration(ClusterConfiguration, port, ClusterMoniker)
+                {
+                    ShowOpenSearchOutputAfterStarted = ClusterConfiguration.ShowOpenSearchOutputAfterStarted
+                })
+                .ToArray();
+
+            var initialClusterManagerNodes = string.Join(",", nodeConfigs.Select(n => n.DesiredNodeName));
+
+            var nodes = nodeConfigs
+                .Select(config =>
+                {
+                    if (nodeConfigs.Length > 1)
+                    {
+                        var otherNodes = nodeConfigs
+                            .Where(n => n != config)
+                            .Select(n => $"localhost:{(n.DesiredPort ?? 9200) + 100}");
+                        config.SeedHosts(string.Join(",", otherNodes));
+                    }
+
+                    config.InitialClusterManagerNodes(initialClusterManagerNodes);
+                    ModifyNodeConfiguration(config, config.DesiredPort ?? 9200);
+
+                    return new OpenSearchNode(config) { AssumeStartedOnNotEnoughClusterManagerPing = ClusterConfiguration.NumberOfNodes > 1 };
+                })
+                .ToArray();
 
 			Nodes = new ReadOnlyCollection<OpenSearchNode>(nodes);
 		}

abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs

@@ -90,11 +90,8 @@ public ClusterConfiguration(OpenSearchVersion version, Func<OpenSearchVersion, s
 			NumberOfNodes = numberOfNodes;
 
 			var fs = FileSystem;
-			Add("node.max_local_storage_nodes", numberOfNodes.ToString(CultureInfo.InvariantCulture), "1.0.0");
-
 			Add("cluster.name", clusterName);
 			Add("path.repo", fs.RepositoryPath);
-			Add("path.data", fs.DataPath);
 			var logsPathDefault = Path.Combine(fs.OpenSearchHome, "logs");
 			if (logsPathDefault != fs.LogsPath) Add("path.logs", fs.LogsPath);
 		}
@@ -122,7 +119,7 @@ public ClusterConfiguration(OpenSearchVersion version, Func<OpenSearchVersion, s
 		public bool CacheOpenSearchHomeInstallation { get; set; }
 
 		/// <summary>The node settings to apply to each started node</summary>
-		public NodeSettings DefaultNodeSettings { get; } = new NodeSettings();
+		public NodeSettings DefaultNodeSettings { get; } = new();
 
 		/// <summary>
 		///     Creates a node name