From 8a14655f28401bbb7bfd22f62bd64c0d832cb7b7 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 15 Aug 2024 12:35:49 +1200 Subject: [PATCH 1/3] Integration test against recent OpenSearch versions Signed-off-by: Thomas Farr --- .github/workflows/integration-yaml-tests.yml | 4 +++- .github/workflows/integration.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-yaml-tests.yml b/.github/workflows/integration-yaml-tests.yml index a01843fe07..fa2b9fcb38 100644 --- a/.github/workflows/integration-yaml-tests.yml +++ b/.github/workflows/integration-yaml-tests.yml @@ -14,7 +14,9 @@ jobs: fail-fast: false matrix: version: - - 2.11.1 + - 2.16.0 + - 2.14.0 + - 2.12.0 - 2.10.0 - 2.8.0 - 2.6.0 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index dd3895c1ae..9daf1a834a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -17,7 +17,9 @@ jobs: fail-fast: false matrix: version: - - 2.11.1 + - 2.16.0 + - 2.14.0 + - 2.12.0 - 2.10.0 - 2.8.0 - 2.6.0 From 6d6f54abacebe78c3b0807ca20f8b030694d05ff Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 15 Aug 2024 13:03:32 +1200 Subject: [PATCH 2/3] Fix multi-node tests Signed-off-by: Thomas Farr --- .github/actions/build-opensearch/action.yml | 10 +- .github/workflows/integration-yaml-tests.yml | 7 +- .github/workflows/integration.yml | 18 +- .../ClusterBase.cs | 49 +- .../Configuration/ClusterConfiguration.cs | 5 +- .../Configuration/NodeConfiguration.cs | 30 +- .../OpenSearchNode.cs | 4 +- .../XunitClusterBase.cs | 18 +- build/scripts/scripts.fsproj | 6 +- .../Generator/ApiEndpointFactory.cs | 3 + .../_Generated/Descriptors.Cluster.cs | 4 +- .../_Generated/Requests.Cluster.cs | 6 +- .../RequestParameters.Cluster.cs | 6 +- .../Clusters/ClientTestClusterBase.cs | 85 +- .../Clusters/ReadOnlyCluster.cs | 26 +- .../NodeSeeders/DefaultSeeder.cs | 773 +++++++++--------- .../CatSegmentReplicationApiTests.cs | 20 +- 17 files changed, 536 insertions(+), 534 deletions(-) diff --git a/.github/actions/build-opensearch/action.yml b/.github/actions/build-opensearch/action.yml index 8ad36ee266..9c3d640df0 100644 --- a/.github/actions/build-opensearch/action.yml +++ b/.github/actions/build-opensearch/action.yml @@ -15,7 +15,7 @@ inputs: plugins_output_directory: description: The directory to output the plugins to default: "" -outputs: +outputs: distribution: description: The path to the OpenSearch distribution value: ${{ steps.determine.outputs.distribution }} @@ -36,11 +36,11 @@ runs: ./opensearch/distribution/archives/linux-tar/build/distributions/opensearch-*.tar.gz ./opensearch/plugins/*/build/distributions/*.zip build_script: | - ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }} + ./gradlew --stacktrace :distribution:archives:linux-tar:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }} PluginList=("analysis-icu" "analysis-kuromoji" "analysis-nori" "analysis-phonetic" "ingest-attachment" "mapper-murmur3") for plugin in ${PluginList[*]}; do - ./gradlew :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }} + ./gradlew --stacktrace :plugins:$plugin:assemble -Dbuild.snapshot=${{ inputs.build_snapshot }} done - name: Determine OpenSearch distribution path and version @@ -62,7 +62,7 @@ runs: cache_key_suffix: ${{ inputs.build_snapshot == 'true' && '-snapshot' || '' }} cached_paths: | ./opensearch-security/build/distributions/opensearch-security-*.zip - build_script: ./gradlew assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }} + build_script: ./gradlew --stacktrace assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }} - name: Restore or Build OpenSearch k-NN uses: ./client/.github/actions/cached-git-build @@ -76,7 +76,7 @@ runs: ./opensearch-knn/build/distributions/opensearch-knn-*.zip build_script: | sudo apt-get install -y libopenblas-dev libomp-dev - ./gradlew buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }} + ./gradlew --stacktrace buildJniLib assemble -Dopensearch.version=${{ steps.determine.outputs.version }} -Dbuild.snapshot=${{ inputs.build_snapshot }} distributions=./build/distributions lib_dir=$distributions/lib mkdir $lib_dir diff --git a/.github/workflows/integration-yaml-tests.yml b/.github/workflows/integration-yaml-tests.yml index fa2b9fcb38..99296a0394 100644 --- a/.github/workflows/integration-yaml-tests.yml +++ b/.github/workflows/integration-yaml-tests.yml @@ -87,7 +87,10 @@ jobs: strategy: fail-fast: false matrix: - opensearch_ref: ['1.x', '2.x', 'main'] + include: + - { opensearch_ref: '1.x', java_version: '11' } + - { opensearch_ref: '2.x', java_version: '17' } + - { opensearch_ref: 'main', java_version: '21' } steps: - name: Checkout Client uses: actions/checkout@v4 @@ -111,8 +114,8 @@ jobs: - name: Set up JDK uses: actions/setup-java@v4 with: - java-version: 21 distribution: 'temurin' + java-version: ${{ matrix.java_version }} - name: Restore or Build OpenSearch id: opensearch_build diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9daf1a834a..822bb67924 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -50,7 +50,7 @@ jobs: restore-keys: | ${{ runner.os }}-nuget- - - run: "./build.sh integrate $VERSION readonly,replicatedreadonly,writable random:test_only_one --report" + - run: "./build.sh integrate $VERSION readonly,multinode,writable random:test_only_one --report" name: Integration Tests working-directory: client env: @@ -69,10 +69,10 @@ jobs: strategy: fail-fast: false matrix: - opensearch_ref: - - '1.x' - - '2.x' - - 'main' + include: + - { opensearch_ref: '1.x', java_version: '11' } + - { opensearch_ref: '2.x', java_version: '17' } + - { opensearch_ref: 'main', java_version: '21' } steps: - name: Checkout Client @@ -100,6 +100,12 @@ jobs: restore-keys: | ${{ runner.os }}-nuget- + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.java_version }} + - name: Restore or Build OpenSearch id: opensearch uses: ./client/.github/actions/build-opensearch @@ -109,7 +115,7 @@ jobs: knn_plugin: true plugins_output_directory: ${{ env.OPENSEARCH_PLUGINS_DIRECTORY }} - - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,replicatedreadonly,writable random:test_only_one --report" + - run: "./build.sh integrate $OPENSEARCH_VERSION readonly,multinode,writable random:test_only_one --report" name: Integration Tests working-directory: client env: diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs index 3cdb74e966..9abf6830d7 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs @@ -75,29 +75,32 @@ protected ClusterBase(TConfiguration clusterConfiguration) ClusterConfiguration = clusterConfiguration; ClusterMoniker = GetType().Name.Replace("Cluster", ""); - NodeConfiguration Modify(NodeConfiguration n, int p) - { - ModifyNodeConfiguration(n, p); - return n; - } - - var nodes = - (from port in Enumerable.Range(ClusterConfiguration.StartingPortNumber, - ClusterConfiguration.NumberOfNodes) - let config = new NodeConfiguration(clusterConfiguration, port, ClusterMoniker) - { - ShowOpenSearchOutputAfterStarted = - clusterConfiguration.ShowOpenSearchOutputAfterStarted, - } - let node = new OpenSearchNode(Modify(config, port)) - { - AssumeStartedOnNotEnoughMasterPing = ClusterConfiguration.NumberOfNodes > 1, - } - select node).ToList(); - - var initialMasterNodes = string.Join(",", nodes.Select(n => n.NodeConfiguration.DesiredNodeName)); - foreach (var node in nodes) - node.NodeConfiguration.InitialMasterNodes(initialMasterNodes); + var nodeConfigs = Enumerable.Range(ClusterConfiguration.StartingPortNumber, ClusterConfiguration.NumberOfNodes) + .Select(port => new NodeConfiguration(ClusterConfiguration, port, ClusterMoniker) + { + ShowOpenSearchOutputAfterStarted = ClusterConfiguration.ShowOpenSearchOutputAfterStarted + }) + .ToArray(); + + var initialClusterManagerNodes = string.Join(",", nodeConfigs.Select(n => n.DesiredNodeName)); + + var nodes = nodeConfigs + .Select(config => + { + if (nodeConfigs.Length > 1) + { + var otherNodes = nodeConfigs + .Where(n => n != config) + .Select(n => $"localhost:{(n.DesiredPort ?? 9200) + 100}"); + config.SeedHosts(string.Join(",", otherNodes)); + } + + config.InitialClusterManagerNodes(initialClusterManagerNodes); + ModifyNodeConfiguration(config, config.DesiredPort ?? 9200); + + return new OpenSearchNode(config) { AssumeStartedOnNotEnoughClusterManagerPing = ClusterConfiguration.NumberOfNodes > 1 }; + }) + .ToArray(); Nodes = new ReadOnlyCollection(nodes); } diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs index e74d8bd116..a9ae4cefa0 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs @@ -90,11 +90,8 @@ public ClusterConfiguration(OpenSearchVersion version, FuncThe node settings to apply to each started node - public NodeSettings DefaultNodeSettings { get; } = new NodeSettings(); + public NodeSettings DefaultNodeSettings { get; } = new(); /// /// Creates a node name diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs index 1e379e1470..4d067242e3 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs @@ -28,6 +28,7 @@ using System; using System.Globalization; +using System.IO; using OpenSearch.OpenSearch.Managed.FileSystem; using OpenSearch.Stack.ArtifactsApi; using ProcNet; @@ -49,12 +50,18 @@ public NodeConfiguration(IClusterConfiguration clusterConfigurat ClusterConfiguration = clusterConfiguration; DesiredPort = port; DesiredNodeName = CreateNodeName(port, nodePrefix) ?? clusterConfiguration.CreateNodeName(port); - Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings); - - if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName); - if (DesiredPort.HasValue) - Settings.Add("http.port", DesiredPort.Value.ToString(CultureInfo.InvariantCulture)); - } + Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings) + { + { "path.data", Path.Combine(ClusterConfiguration.FileSystem.DataPath, DesiredNodeName) } + }; + + if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName); + if (DesiredPort is { } desiredPort) + { + Settings.Add("http.port", desiredPort.ToString(CultureInfo.InvariantCulture)); + Settings.Add("transport.port", (desiredPort + 100).ToString(CultureInfo.InvariantCulture)); + } + } private IClusterConfiguration ClusterConfiguration { get; } @@ -87,10 +94,15 @@ public Action ModifyStartArguments public OpenSearchVersion Version => ClusterConfiguration.Version; public string[] CommandLineArguments => Settings.ToCommandLineArguments(Version); - public void InitialMasterNodes(string initialMasterNodes) => - Settings.Add("cluster.initial_master_nodes", initialMasterNodes, ">=1.0.0"); + public void SeedHosts(string seedHosts) => Settings.Add("discovery.seed_hosts", seedHosts); + + public void InitialClusterManagerNodes(string initialClusterManagerNodes) + { + Settings.Add("cluster.initial_master_nodes", initialClusterManagerNodes, ">=1.0.0 <2.0.0"); + Settings.Add("cluster.initial_cluster_manager_nodes", initialClusterManagerNodes, ">=2.0.0"); + } - public string AttributeKey(string attribute) + public string AttributeKey(string attribute) { var attr = "attr."; return $"node.{attr}{attribute}"; diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs index da8250901a..0ea0a83884 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs @@ -64,7 +64,7 @@ public OpenSearchNode(OpenSearchVersion version, string openSearchHome = null) /// doing the election. /// Useful to speed up starting multi node clusters /// - public bool AssumeStartedOnNotEnoughMasterPing { get; set; } + public bool AssumeStartedOnNotEnoughClusterManagerPing { get; set; } internal IConsoleLineHandler Writer { get; private set; } @@ -119,7 +119,7 @@ private static void AppendPathEnvVar(string name, string value) private bool AssumedStartedStateChecker(string section, string message) { - if (AssumeStartedOnNotEnoughMasterPing + if (AssumeStartedOnNotEnoughClusterManagerPing && section.Contains("ZenDiscovery") && message.Contains("not enough master nodes discovered during pinging")) return true; diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs index 2c7f5d15b7..d6576c84bd 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs @@ -33,21 +33,13 @@ namespace OpenSearch.OpenSearch.Xunit /// /// Base class for a cluster that integrates with Xunit tests /// - public abstract class XunitClusterBase : XunitClusterBase - { - protected XunitClusterBase(XunitClusterConfiguration configuration) : base(configuration) - { - } - } + public abstract class XunitClusterBase(XunitClusterConfiguration configuration) + : XunitClusterBase(configuration); /// /// Base class for a cluster that integrates with Xunit tests /// - public abstract class XunitClusterBase : EphemeralCluster - where TConfiguration : XunitClusterConfiguration - { - protected XunitClusterBase(TConfiguration configuration) : base(configuration) - { - } - } + public abstract class XunitClusterBase(TConfiguration configuration) + : EphemeralCluster(configuration) + where TConfiguration : XunitClusterConfiguration; } diff --git a/build/scripts/scripts.fsproj b/build/scripts/scripts.fsproj index e98ae8fc61..253f9523e5 100644 --- a/build/scripts/scripts.fsproj +++ b/build/scripts/scripts.fsproj @@ -32,15 +32,17 @@ license-header-fs.txt + + - + - + diff --git a/src/ApiGenerator/Generator/ApiEndpointFactory.cs b/src/ApiGenerator/Generator/ApiEndpointFactory.cs index 13d3b4887d..5d3f748e6c 100644 --- a/src/ApiGenerator/Generator/ApiEndpointFactory.cs +++ b/src/ApiGenerator/Generator/ApiEndpointFactory.cs @@ -289,6 +289,9 @@ private static string SanitizeDescription(this string description) { if (string.IsNullOrWhiteSpace(description)) return null; + description = Regex.Replace(description, "&", "&"); + description = Regex.Replace(description, "<", "<"); + description = Regex.Replace(description, ">", ">"); description = Regex.Replace(description, @"\s+", " "); if (!description.EndsWith('.')) description += '.'; diff --git a/src/OpenSearch.Client/_Generated/Descriptors.Cluster.cs b/src/OpenSearch.Client/_Generated/Descriptors.Cluster.cs index 426e0af0be..6150a2c27b 100644 --- a/src/OpenSearch.Client/_Generated/Descriptors.Cluster.cs +++ b/src/OpenSearch.Client/_Generated/Descriptors.Cluster.cs @@ -336,7 +336,7 @@ public ClusterHealthDescriptor WaitForActiveShards(string waitforactiveshards) = public ClusterHealthDescriptor WaitForEvents(WaitForEvents? waitforevents) => Qs("wait_for_events", waitforevents); - /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and + /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and <N. Alternatively, it is possible to use ge(N), le(N), gt(N) and lt(N) notation. public ClusterHealthDescriptor WaitForNodes(string waitfornodes) => Qs("wait_for_nodes", waitfornodes); @@ -350,7 +350,7 @@ public ClusterHealthDescriptor WaitForNoRelocatingShards( bool? waitfornorelocatingshards = true ) => Qs("wait_for_no_relocating_shards", waitfornorelocatingshards); - /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. green > yellow > red. By default, will not wait for any status. + /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. green > yellow > red. By default, will not wait for any status. public ClusterHealthDescriptor WaitForStatus(HealthStatus? waitforstatus) => Qs("wait_for_status", waitforstatus); } diff --git a/src/OpenSearch.Client/_Generated/Requests.Cluster.cs b/src/OpenSearch.Client/_Generated/Requests.Cluster.cs index 2132bd4bfa..60f7645108 100644 --- a/src/OpenSearch.Client/_Generated/Requests.Cluster.cs +++ b/src/OpenSearch.Client/_Generated/Requests.Cluster.cs @@ -458,8 +458,8 @@ public WaitForEvents? WaitForEvents } /// - /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and public string WaitForNodes { @@ -489,7 +489,7 @@ public bool? WaitForNoRelocatingShards /// /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, - /// i.e. green > yellow > red. By default, will not wait for any status. + /// i.e. green > yellow > red. By default, will not wait for any status. /// public HealthStatus? WaitForStatus { diff --git a/src/OpenSearch.Net/_Generated/Api/RequestParameters/RequestParameters.Cluster.cs b/src/OpenSearch.Net/_Generated/Api/RequestParameters/RequestParameters.Cluster.cs index 2c6f2b3b20..6415954714 100644 --- a/src/OpenSearch.Net/_Generated/Api/RequestParameters/RequestParameters.Cluster.cs +++ b/src/OpenSearch.Net/_Generated/Api/RequestParameters/RequestParameters.Cluster.cs @@ -367,8 +367,8 @@ public WaitForEvents? WaitForEvents } /// - /// The request waits until the specified number N of nodes is available. It also accepts >=N, <=N, >N and public string WaitForNodes { @@ -398,7 +398,7 @@ public bool? WaitForNoRelocatingShards /// /// One of green, yellow or red. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, - /// i.e. green > yellow > red. By default, will not wait for any status. + /// i.e. green > yellow > red. By default, will not wait for any status. /// public HealthStatus? WaitForStatus { diff --git a/tests/Tests.Core/ManagedOpenSearch/Clusters/ClientTestClusterBase.cs b/tests/Tests.Core/ManagedOpenSearch/Clusters/ClientTestClusterBase.cs index e5eb2d1e99..d36e53ceb7 100644 --- a/tests/Tests.Core/ManagedOpenSearch/Clusters/ClientTestClusterBase.cs +++ b/tests/Tests.Core/ManagedOpenSearch/Clusters/ClientTestClusterBase.cs @@ -39,57 +39,68 @@ using Tests.Core.ManagedOpenSearch.Tasks; using Tests.Domain.Extensions; -namespace Tests.Core.ManagedOpenSearch.Clusters +namespace Tests.Core.ManagedOpenSearch.Clusters; + +public abstract class ClientTestClusterBase(ClientTestClusterConfiguration configuration) + : XunitClusterBase(configuration), + IOpenSearchClientTestCluster { - public abstract class ClientTestClusterBase : XunitClusterBase, IOpenSearchClientTestCluster - { - protected ClientTestClusterBase() : this(new ClientTestClusterConfiguration()) { } + protected ClientTestClusterBase() : this(new ClientTestClusterConfiguration()) { } + + protected ClientTestClusterBase(params OpenSearchPlugin[] plugins) : this(new ClientTestClusterConfiguration(plugins)) { } - protected ClientTestClusterBase(params OpenSearchPlugin[] plugins) : this(new ClientTestClusterConfiguration(plugins)) { } + public IOpenSearchClient Client => this.GetOrAddClient(s => ConnectionSettings(s.ApplyDomainSettings())); - protected ClientTestClusterBase(ClientTestClusterConfiguration configuration) : base(configuration) { } + protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) => s; - public IOpenSearchClient Client => this.GetOrAddClient(s => ConnectionSettings(s.ApplyDomainSettings())); + protected sealed override void SeedCluster() + { + var clusterHealth = new ClusterHealthRequest + { + WaitForNodes = ClusterConfiguration.NumberOfNodes.ToString(), + WaitForStatus = HealthStatus.Green, + Level = ClusterHealthLevel.Shards + }; - protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) => s; + Client.Cluster.Health(clusterHealth).ShouldBeValid(); - protected sealed override void SeedCluster() - { - Client.Cluster.Health(new ClusterHealthRequest { WaitForStatus = HealthStatus.Green }); - SeedNode(); - Client.Cluster.Health(new ClusterHealthRequest { WaitForStatus = HealthStatus.Green }); - } + SeedNode(); - protected virtual void SeedNode() { } - } + Client.Cluster.Health(clusterHealth).ShouldBeValid(); + } - public class ClientTestClusterConfiguration : XunitClusterConfiguration - { - public ClientTestClusterConfiguration(params OpenSearchPlugin[] plugins) : this(numberOfNodes: 1, plugins: plugins) { } + protected virtual void SeedNode() { } +} + +public class ClientTestClusterConfiguration : XunitClusterConfiguration +{ + public ClientTestClusterConfiguration(params OpenSearchPlugin[] plugins) : this(numberOfNodes: 1, plugins: plugins) { } - public ClientTestClusterConfiguration(ClusterFeatures features = ClusterFeatures.SSL, int numberOfNodes = 1, - params OpenSearchPlugin[] plugins - ) - : base(TestClient.Configuration.OpenSearchVersion, features, new OpenSearchPlugins(plugins), numberOfNodes) - { - TestConfiguration = TestClient.Configuration; + public ClientTestClusterConfiguration(ClusterFeatures features = ClusterFeatures.SSL, int numberOfNodes = 1, + params OpenSearchPlugin[] plugins + ) + : base(TestClient.Configuration.OpenSearchVersion, features, new OpenSearchPlugins(plugins), numberOfNodes) + { + TestConfiguration = TestClient.Configuration; - ShowOpenSearchOutputAfterStarted = TestConfiguration.ShowOpenSearchOutputAfterStarted; - HttpFiddlerAware = true; + ShowOpenSearchOutputAfterStarted = TestConfiguration.ShowOpenSearchOutputAfterStarted; + HttpFiddlerAware = true; - CacheOpenSearchHomeInstallation = true; + CacheOpenSearchHomeInstallation = true; - Add(AttributeKey("testingcluster"), "true"); - Add(AttributeKey("gateway"), "true"); + Add(AttributeKey("testingcluster"), "true"); + Add(AttributeKey("gateway"), "true"); - Add($"script.disable_max_compilations_rate", "true"); + Add("cluster.routing.allocation.disk.watermark.low", "100%"); + Add("cluster.routing.allocation.disk.watermark.high", "100%"); + Add("cluster.routing.allocation.disk.watermark.flood_stage", "100%"); - Add($"script.allowed_types", "inline,stored"); + Add("script.disable_max_compilations_rate", "true"); + Add("script.allowed_types", "inline,stored"); - AdditionalBeforeNodeStartedTasks.Add(new WriteAnalysisFiles()); - } + AdditionalBeforeNodeStartedTasks.Add(new WriteAnalysisFiles()); + } - public string AnalysisFolder => Path.Combine(FileSystem.ConfigPath, "analysis"); - public TestConfigurationBase TestConfiguration { get; } - } + public string AnalysisFolder => Path.Combine(FileSystem.ConfigPath, "analysis"); + public TestConfigurationBase TestConfiguration { get; } } diff --git a/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs b/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs index 26186c3351..23fef6b367 100644 --- a/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs +++ b/tests/Tests.Core/ManagedOpenSearch/Clusters/ReadOnlyCluster.cs @@ -26,27 +26,17 @@ * under the License. */ -using OpenSearch.Client; using Tests.Core.ManagedOpenSearch.NodeSeeders; using static OpenSearch.Stack.ArtifactsApi.Products.OpenSearchPlugin; -namespace Tests.Core.ManagedOpenSearch.Clusters -{ - public class ReadOnlyCluster : ClientTestClusterBase - { - public ReadOnlyCluster() : base(Knn, MapperMurmur3, Security) { } - - protected override void SeedNode() => new DefaultSeeder(Client).SeedNode(); - } +namespace Tests.Core.ManagedOpenSearch.Clusters; - public class ReplicatedReadOnlyCluster : ClientTestClusterBase - { - public ReplicatedReadOnlyCluster() : base(new ClientTestClusterConfiguration(numberOfNodes: 2, plugins: new[] {Knn, MapperMurmur3, Security})) { } +public class ReadOnlyCluster : ClientTestClusterBase +{ + public ReadOnlyCluster() : base(Knn, MapperMurmur3, Security) { } - protected override void SeedNode() => new DefaultSeeder(Client, new IndexSettings - { - NumberOfShards = 2, - NumberOfReplicas = 1 - }).SeedNode(); - } + protected override void SeedNode() => new DefaultSeeder(Client).SeedNode(); } + +public class MultiNodeCluster() + : ClientTestClusterBase(new ClientTestClusterConfiguration(numberOfNodes: 2, plugins: [Security])); diff --git a/tests/Tests.Core/ManagedOpenSearch/NodeSeeders/DefaultSeeder.cs b/tests/Tests.Core/ManagedOpenSearch/NodeSeeders/DefaultSeeder.cs index 23cc132c64..45aa0c8f42 100644 --- a/tests/Tests.Core/ManagedOpenSearch/NodeSeeders/DefaultSeeder.cs +++ b/tests/Tests.Core/ManagedOpenSearch/NodeSeeders/DefaultSeeder.cs @@ -34,412 +34,385 @@ using Tests.Core.Extensions; using Tests.Domain; -namespace Tests.Core.ManagedOpenSearch.NodeSeeders +namespace Tests.Core.ManagedOpenSearch.NodeSeeders; + +public class DefaultSeeder { - public class DefaultSeeder - { - public const string CommitsAliasFilter = "commits-only"; - public const string ProjectsAliasFilter = "projects-only"; - - public const string ProjectsAliasName = "projects-alias"; - public const string TestsIndexTemplateName = "osc_tests"; - - public const string RemoteClusterName = "remote-cluster"; - - public const string PipelineName = "osc-pipeline"; - - private readonly IIndexSettings _defaultIndexSettings = new IndexSettings() - { - NumberOfShards = 2, - NumberOfReplicas = 0, - }; - - public DefaultSeeder(IOpenSearchClient client, IIndexSettings indexSettings) - { - Client = client; - IndexSettings = indexSettings ?? _defaultIndexSettings; - } - - public DefaultSeeder(IOpenSearchClient client) : this(client, null) { } - - private IOpenSearchClient Client { get; } - - private IIndexSettings IndexSettings { get; } - - public void SeedNode() - { - var alreadySeeded = false; - if (!TestClient.Configuration.ForceReseed && (alreadySeeded = AlreadySeeded())) return; - - var t = Task.Run(async () => await SeedNodeAsync(alreadySeeded).ConfigureAwait(false)); - - t.Wait(TimeSpan.FromSeconds(40)); - } - - public void SeedNodeNoData() - { - var alreadySeeded = false; - if (!TestClient.Configuration.ForceReseed && (alreadySeeded = AlreadySeeded())) return; - - var t = Task.Run(async () => await SeedNodeNoDataAsync(alreadySeeded).ConfigureAwait(false)); - - t.Wait(TimeSpan.FromSeconds(40)); - } - - // Sometimes we run against an manually started OpenSearch when - // writing tests to cut down on cluster startup times. - // If raw_fields exists assume this cluster is already seeded. - private bool AlreadySeeded() => Client.Indices.TemplateExists(TestsIndexTemplateName).Exists; - - private async Task SeedNodeAsync(bool alreadySeeded) - { - // Ensure a clean slate by deleting everything regardless of whether they may already exist - await DeleteIndicesAndTemplatesAsync(alreadySeeded).ConfigureAwait(false); - await ClusterSettingsAsync().ConfigureAwait(false); - await PutPipeline().ConfigureAwait(false); - // and now recreate everything - await CreateIndicesAndSeedIndexDataAsync().ConfigureAwait(false); - } - - private async Task SeedNodeNoDataAsync(bool alreadySeeded) - { - // Ensure a clean slate by deleting everything regardless of whether they may already exist - await DeleteIndicesAndTemplatesAsync(alreadySeeded).ConfigureAwait(false); - await ClusterSettingsAsync().ConfigureAwait(false); - // and now recreate everything - await CreateIndicesAsync().ConfigureAwait(false); - } - - public async Task ClusterSettingsAsync() - { - var clusterConfiguration = new Dictionary() - { - { "cluster.routing.use_adaptive_replica_selection", true } - }; - - clusterConfiguration += new RemoteClusterConfiguration - { - { RemoteClusterName, "127.0.0.1:9300" } - }; - - var putSettingsResponse = await Client.Cluster.PutSettingsAsync(new ClusterPutSettingsRequest - { - Transient = clusterConfiguration - }).ConfigureAwait(false); - - putSettingsResponse.ShouldBeValid(); - } - - public async Task PutPipeline() - { - var putProcessors = await Client.Ingest.PutPipelineAsync(PipelineName, pi => pi - .Description("A pipeline registered by the OSC test framework") - .Processors(pp => pp - .Set(s => s.Field(p => p.Metadata).Value(new { x = "y" })) - ) - ).ConfigureAwait(false); - putProcessors.ShouldBeValid(); - } - - - public async Task DeleteIndicesAndTemplatesAsync(bool alreadySeeded) - { - var tasks = new List - { - Client.Indices.DeleteAsync(typeof(Project)), - Client.Indices.DeleteAsync(typeof(Developer)), - Client.Indices.DeleteAsync(typeof(ProjectPercolation)) - }; - - if (alreadySeeded) - tasks.Add(Client.Indices.DeleteTemplateAsync(TestsIndexTemplateName)); - - await Task.WhenAll(tasks.ToArray()).ConfigureAwait(false); - } - - private async Task CreateIndicesAndSeedIndexDataAsync() - { - await CreateIndicesAsync().ConfigureAwait(false); - await SeedIndexDataAsync().ConfigureAwait(false); - } - - public async Task CreateIndicesAsync() - { - var indexTemplateResponse = await CreateIndexTemplateAsync().ConfigureAwait(false); - indexTemplateResponse.ShouldBeValid(); - - var tasks = new[] - { - CreateProjectIndexAsync(), - CreateDeveloperIndexAsync(), - CreatePercolatorIndexAsync(), - }; - await Task.WhenAll(tasks) - .ContinueWith(t => - { - foreach (var r in t.Result) - r.ShouldBeValid(); - }).ConfigureAwait(false); - } - - private async Task SeedIndexDataAsync() - { - var tasks = new Task[] - { - Client.IndexManyAsync(Project.Projects), - Client.IndexManyAsync(Developer.Developers), - Client.IndexDocumentAsync(new ProjectPercolation - { - Id = "1", - Query = new MatchAllQuery() - }), - Client.BulkAsync(b => b - .IndexMany( - CommitActivity.CommitActivities, - (d, c) => d.Document(c).Routing(c.ProjectName) - ) - ) }; - await Task.WhenAll(tasks).ConfigureAwait(false); - await Client.Indices.RefreshAsync(Indices.Index(typeof(Project), typeof(Developer), typeof(ProjectPercolation))).ConfigureAwait(false); - } - - private Task CreateIndexTemplateAsync() => Client.Indices.PutTemplateAsync( - new PutIndexTemplateRequest(TestsIndexTemplateName) - { - IndexPatterns = new[] { "*" }, - Settings = IndexSettings - }); - - private Task CreateDeveloperIndexAsync() => Client.Indices.CreateAsync(Infer.Index(), c => c - .Map(m => m - .AutoMap() - .Properties(DeveloperProperties) - ) - ); + public const string CommitsAliasFilter = "commits-only"; + public const string ProjectsAliasFilter = "projects-only"; + + public const string ProjectsAliasName = "projects-alias"; + public const string TestsIndexTemplateName = "osc_tests"; + + public const string PipelineName = "osc-pipeline"; + + private readonly IIndexSettings _defaultIndexSettings = new IndexSettings + { + NumberOfShards = 2, + NumberOfReplicas = 0, + }; + + public DefaultSeeder(IOpenSearchClient client, IIndexSettings indexSettings = null) + { + Client = client; + IndexSettings = indexSettings ?? _defaultIndexSettings; + } + + private IOpenSearchClient Client { get; } + + private IIndexSettings IndexSettings { get; } + + public void SeedNode() + { + var alreadySeeded = false; + if (!TestClient.Configuration.ForceReseed && (alreadySeeded = AlreadySeeded())) return; + + var t = Task.Run(async () => await SeedNodeAsync(alreadySeeded).ConfigureAwait(false)); + + t.Wait(TimeSpan.FromSeconds(40)); + } + + public void SeedNodeNoData() + { + var alreadySeeded = false; + if (!TestClient.Configuration.ForceReseed && (alreadySeeded = AlreadySeeded())) return; + + var t = Task.Run(async () => await SeedNodeNoDataAsync(alreadySeeded).ConfigureAwait(false)); + + t.Wait(TimeSpan.FromSeconds(40)); + } + + // Sometimes we run against an manually started OpenSearch when + // writing tests to cut down on cluster startup times. + // If raw_fields exists assume this cluster is already seeded. + private bool AlreadySeeded() => Client.Indices.TemplateExists(TestsIndexTemplateName).Exists; + + private async Task SeedNodeAsync(bool alreadySeeded) + { + // Ensure a clean slate by deleting everything regardless of whether they may already exist + await DeleteIndicesAndTemplatesAsync(alreadySeeded).ConfigureAwait(false); + await PutPipeline().ConfigureAwait(false); + // and now recreate everything + await CreateIndicesAndSeedIndexDataAsync().ConfigureAwait(false); + } + + private async Task SeedNodeNoDataAsync(bool alreadySeeded) + { + // Ensure a clean slate by deleting everything regardless of whether they may already exist + await DeleteIndicesAndTemplatesAsync(alreadySeeded).ConfigureAwait(false); + // and now recreate everything + await CreateIndicesAsync().ConfigureAwait(false); + } + + public async Task PutPipeline() + { + var putProcessors = await Client.Ingest.PutPipelineAsync(PipelineName, pi => pi + .Description("A pipeline registered by the OSC test framework") + .Processors(pp => pp + .Set(s => s.Field(p => p.Metadata).Value(new { x = "y" })) + ) + ).ConfigureAwait(false); + putProcessors.ShouldBeValid(); + } + + public async Task DeleteIndicesAndTemplatesAsync(bool alreadySeeded) + { + var tasks = new List + { + Client.Indices.DeleteAsync(typeof(Project)), + Client.Indices.DeleteAsync(typeof(Developer)), + Client.Indices.DeleteAsync(typeof(ProjectPercolation)) + }; + + if (alreadySeeded) + tasks.Add(Client.Indices.DeleteTemplateAsync(TestsIndexTemplateName)); + + await Task.WhenAll(tasks.ToArray()).ConfigureAwait(false); + } + + private async Task CreateIndicesAndSeedIndexDataAsync() + { + await CreateIndicesAsync().ConfigureAwait(false); + await SeedIndexDataAsync().ConfigureAwait(false); + } + + public async Task CreateIndicesAsync() + { + var indexTemplateResponse = await CreateIndexTemplateAsync().ConfigureAwait(false); + indexTemplateResponse.ShouldBeValid(); + + var tasks = new[] + { + CreateProjectIndexAsync(), + CreateDeveloperIndexAsync(), + CreatePercolatorIndexAsync(), + }; + await Task.WhenAll(tasks) + .ContinueWith(t => + { + foreach (var r in t.Result) + r.ShouldBeValid(); + }).ConfigureAwait(false); + } + + private async Task SeedIndexDataAsync() + { + var tasks = new Task[] + { + Client.IndexManyAsync(Project.Projects), + Client.IndexManyAsync(Developer.Developers), + Client.IndexDocumentAsync(new ProjectPercolation + { + Id = "1", + Query = new MatchAllQuery() + }), + Client.BulkAsync(b => b + .IndexMany( + CommitActivity.CommitActivities, + (d, c) => d.Document(c).Routing(c.ProjectName) + ) + ) + }; + await Task.WhenAll(tasks).ConfigureAwait(false); + await Client.Indices.RefreshAsync(Indices.Index(typeof(Project), typeof(Developer), typeof(ProjectPercolation))).ConfigureAwait(false); + } + + private Task CreateIndexTemplateAsync() => Client.Indices.PutTemplateAsync( + new PutIndexTemplateRequest(TestsIndexTemplateName) + { + IndexPatterns = new[] { "*" }, + Settings = IndexSettings + }); + + private Task CreateDeveloperIndexAsync() => Client.Indices.CreateAsync(Infer.Index(), c => c + .Map(m => m + .AutoMap() + .Properties(DeveloperProperties) + ) + ); #pragma warning disable 618 - private Task CreateProjectIndexAsync() => Client.Indices.CreateAsync(typeof(Project), c => c - .Settings(ProjectIndexSettings) - .Mappings(ProjectMappings) - .Aliases(aliases => aliases - .Alias(ProjectsAliasName) - .Alias(ProjectsAliasFilter, a => a - .Filter(f => f.Term(p => p.Join, Infer.Relation())) - ) - .Alias(CommitsAliasFilter, a => a - .Filter(f => f.Term(p => p.Join, Infer.Relation())) - ) - ) - ); + private Task CreateProjectIndexAsync() => Client.Indices.CreateAsync(typeof(Project), c => c + .Settings(ProjectIndexSettings) + .Mappings(ProjectMappings) + .Aliases(aliases => aliases + .Alias(ProjectsAliasName) + .Alias(ProjectsAliasFilter, a => a + .Filter(f => f.Term(p => p.Join, Infer.Relation())) + ) + .Alias(CommitsAliasFilter, a => a + .Filter(f => f.Term(p => p.Join, Infer.Relation())) + ) + ) + ); #pragma warning restore 618 #pragma warning disable 618 - public static ITypeMapping ProjectMappings(MappingsDescriptor map) => map - .Map(ProjectTypeMappings); + public static ITypeMapping ProjectMappings(MappingsDescriptor map) => map + .Map(ProjectTypeMappings); #pragma warning restore 618 - public static ITypeMapping ProjectTypeMappings(TypeMappingDescriptor mapping) - { - mapping - .RoutingField(r => r.Required()) - .AutoMap() - .Properties(ProjectProperties) - .Properties(props => props - .Object(o => o - .AutoMap() - .Name(p => p.Committer) - .Properties(DeveloperProperties) - .Dynamic() - ) - .Text(t => t - .Name(p => p.ProjectName) - .Index(false) - ) - ); - - return mapping; - } - - public static IndexSettingsDescriptor ProjectIndexSettings(IndexSettingsDescriptor settings) => - settings - .Analysis(ProjectAnalysisSettings) - .Setting("index.knn", true) - .Setting("index.knn.algo_param.ef_search", 100); - - public static IAnalysis ProjectAnalysisSettings(AnalysisDescriptor analysis) - { - analysis - .TokenFilters(tokenFilters => tokenFilters - .Shingle("shingle", shingle => shingle - .MinShingleSize(2) - .MaxShingleSize(4) - ) - ) - .Analyzers(analyzers => analyzers - .Custom("shingle", shingle => shingle - .Filters("shingle") - .Tokenizer("standard") - ) - ) - .Normalizers(analyzers => analyzers - .Custom("my_normalizer", n => n - .Filters("lowercase", "asciifolding") - ) - ); - return analysis; - } - - public static IndexSettingsDescriptor PercolatorIndexSettings(IndexSettingsDescriptor settings) => - ProjectIndexSettings(settings).AutoExpandReplicas("0-all"); - - private Task CreatePercolatorIndexAsync() => Client.Indices.CreateAsync(typeof(ProjectPercolation), c => c - .Settings(PercolatorIndexSettings) - .Map(m => m - .AutoMap() - .Properties(PercolatedQueryProperties) - ) - ); - - public static PropertiesDescriptor ProjectProperties(PropertiesDescriptor props) - where TProject : Project - { - props - .Join(j => j - .Name(n => n.Join) - .Relations(r => r - .Join() - ) - ) - .Keyword(d => d.Name(p => p.Type)) - .Keyword(s => s - .Name(p => p.Name) - .Store() - .Fields(fs => fs - .Text(ss => ss - .Name("standard") - .Analyzer("standard") - ) - .Completion(cm => cm - .Name("suggest") - ) - ) - ) - .Text(s => s - .Name(p => p.Description) - .Fielddata() - .Fields(f => f - .Text(t => t - .Name("shingle") - .Analyzer("shingle") - ) - ) - ) - .Date(d => d - .Store() - .Name(p => p.StartedOn) - ) - .Text(d => d - .Store() - .Name(p => p.DateString) - ) - .Keyword(d => d - .Name(p => p.State) - .Fields(fs => fs - .Text(st => st - .Name("offsets") - .IndexOptions(IndexOptions.Offsets) - ) - .Keyword(sk => sk - .Name("keyword") - ) - ) - ) - .Nested(mo => mo - .AutoMap() - .Name(p => p.Tags) - .Properties(TagProperties) - ) - .Object(o => o - .AutoMap() - .Name(p => p.LeadDeveloper) - .Properties(DeveloperProperties) - ) - .GeoPoint(g => g - .Name(p => p.LocationPoint) - ) - .GeoShape(g => g - .Name(p => p.LocationShape) - ) - .Completion(cm => cm - .Name(p => p.Suggest) - .Contexts(cx => cx - .Category(c => c - .Name("color") - ) - .GeoLocation(c => c - .Name("geo") - .Precision("1") - ) - ) - ) - .Scalar(p => p.NumberOfCommits, n => n.Store()) - .Scalar(p => p.NumberOfContributors, n => n.Store()) - .Object>(o => o - .Name(p => p.Metadata) - ) - .RankFeature(rf => rf - .Name(p => p.Rank) - .PositiveScoreImpact() - ) - .KnnVector(k => k - .Name(p => p.Vector) - .Dimension(2) - .Method(m => m - .Name("hnsw") - .SpaceType("l2") - .Engine("nmslib") - .Parameters(p => p - .Parameter("ef_construction", 128) - .Parameter("m", 24) - ) - )); - - return props; - } - - private static PropertiesDescriptor TagProperties(PropertiesDescriptor props) => props - .Keyword(s => s - .Name(p => p.Name) - .Fields(f => f - .Text(st => st - .Name("vectors") - .TermVector(TermVectorOption.WithPositionsOffsetsPayloads) - ) - ) - ); - - public static PropertiesDescriptor DeveloperProperties(PropertiesDescriptor props) => props - .Keyword(s => s - .Name(p => p.OnlineHandle) - ) - .Keyword(s => s - .Name(p => p.Gender) - ) - .Text(s => s - .Name(p => p.FirstName) - .TermVector(TermVectorOption.WithPositionsOffsetsPayloads) - ) - .Ip(s => s - .Name(p => p.IpAddress) - ) - .GeoPoint(g => g - .Name(p => p.Location) - ) - .Object(o => o - .Name(p => p.GeoIp) - ); - - public static PropertiesDescriptor PercolatedQueryProperties(PropertiesDescriptor props) => - ProjectProperties(props.Percolator(pp => pp.Name(n => n.Query))); - } + public static ITypeMapping ProjectTypeMappings(TypeMappingDescriptor mapping) + { + mapping + .RoutingField(r => r.Required()) + .AutoMap() + .Properties(ProjectProperties) + .Properties(props => props + .Object(o => o + .AutoMap() + .Name(p => p.Committer) + .Properties(DeveloperProperties) + .Dynamic() + ) + .Text(t => t + .Name(p => p.ProjectName) + .Index(false) + ) + ); + + return mapping; + } + + public static IndexSettingsDescriptor ProjectIndexSettings(IndexSettingsDescriptor settings) => + settings + .Analysis(ProjectAnalysisSettings) + .Setting("index.knn", true) + .Setting("index.knn.algo_param.ef_search", 100); + + public static IAnalysis ProjectAnalysisSettings(AnalysisDescriptor analysis) + { + analysis + .TokenFilters(tokenFilters => tokenFilters + .Shingle("shingle", shingle => shingle + .MinShingleSize(2) + .MaxShingleSize(4) + ) + ) + .Analyzers(analyzers => analyzers + .Custom("shingle", shingle => shingle + .Filters("shingle") + .Tokenizer("standard") + ) + ) + .Normalizers(analyzers => analyzers + .Custom("my_normalizer", n => n + .Filters("lowercase", "asciifolding") + ) + ); + return analysis; + } + + public static IndexSettingsDescriptor PercolatorIndexSettings(IndexSettingsDescriptor settings) => + ProjectIndexSettings(settings).AutoExpandReplicas("0-all"); + + private Task CreatePercolatorIndexAsync() => Client.Indices.CreateAsync(typeof(ProjectPercolation), c => c + .Settings(PercolatorIndexSettings) + .Map(m => m + .AutoMap() + .Properties(PercolatedQueryProperties) + ) + ); + + public static PropertiesDescriptor ProjectProperties(PropertiesDescriptor props) + where TProject : Project + { + props + .Join(j => j + .Name(n => n.Join) + .Relations(r => r + .Join() + ) + ) + .Keyword(d => d.Name(p => p.Type)) + .Keyword(s => s + .Name(p => p.Name) + .Store() + .Fields(fs => fs + .Text(ss => ss + .Name("standard") + .Analyzer("standard") + ) + .Completion(cm => cm + .Name("suggest") + ) + ) + ) + .Text(s => s + .Name(p => p.Description) + .Fielddata() + .Fields(f => f + .Text(t => t + .Name("shingle") + .Analyzer("shingle") + ) + ) + ) + .Date(d => d + .Store() + .Name(p => p.StartedOn) + ) + .Text(d => d + .Store() + .Name(p => p.DateString) + ) + .Keyword(d => d + .Name(p => p.State) + .Fields(fs => fs + .Text(st => st + .Name("offsets") + .IndexOptions(IndexOptions.Offsets) + ) + .Keyword(sk => sk + .Name("keyword") + ) + ) + ) + .Nested(mo => mo + .AutoMap() + .Name(p => p.Tags) + .Properties(TagProperties) + ) + .Object(o => o + .AutoMap() + .Name(p => p.LeadDeveloper) + .Properties(DeveloperProperties) + ) + .GeoPoint(g => g + .Name(p => p.LocationPoint) + ) + .GeoShape(g => g + .Name(p => p.LocationShape) + ) + .Completion(cm => cm + .Name(p => p.Suggest) + .Contexts(cx => cx + .Category(c => c + .Name("color") + ) + .GeoLocation(c => c + .Name("geo") + .Precision("1") + ) + ) + ) + .Scalar(p => p.NumberOfCommits, n => n.Store()) + .Scalar(p => p.NumberOfContributors, n => n.Store()) + .Object>(o => o + .Name(p => p.Metadata) + ) + .RankFeature(rf => rf + .Name(p => p.Rank) + .PositiveScoreImpact() + ) + .KnnVector(k => k + .Name(p => p.Vector) + .Dimension(2) + .Method(m => m + .Name("hnsw") + .SpaceType("l2") + .Engine("nmslib") + .Parameters(p => p + .Parameter("ef_construction", 128) + .Parameter("m", 24) + ) + )); + + return props; + } + + private static PropertiesDescriptor TagProperties(PropertiesDescriptor props) => props + .Keyword(s => s + .Name(p => p.Name) + .Fields(f => f + .Text(st => st + .Name("vectors") + .TermVector(TermVectorOption.WithPositionsOffsetsPayloads) + ) + ) + ); + + public static PropertiesDescriptor DeveloperProperties(PropertiesDescriptor props) => props + .Keyword(s => s + .Name(p => p.OnlineHandle) + ) + .Keyword(s => s + .Name(p => p.Gender) + ) + .Text(s => s + .Name(p => p.FirstName) + .TermVector(TermVectorOption.WithPositionsOffsetsPayloads) + ) + .Ip(s => s + .Name(p => p.IpAddress) + ) + .GeoPoint(g => g + .Name(p => p.Location) + ) + .Object(o => o + .Name(p => p.GeoIp) + ); + + public static PropertiesDescriptor PercolatedQueryProperties(PropertiesDescriptor props) => + ProjectProperties(props.Percolator(pp => pp.Name(n => n.Query))); } diff --git a/tests/Tests/Cat/CatSegmentReplication/CatSegmentReplicationApiTests.cs b/tests/Tests/Cat/CatSegmentReplication/CatSegmentReplicationApiTests.cs index 1c5aab5b9e..f1dac93704 100644 --- a/tests/Tests/Cat/CatSegmentReplication/CatSegmentReplicationApiTests.cs +++ b/tests/Tests/Cat/CatSegmentReplication/CatSegmentReplicationApiTests.cs @@ -7,6 +7,7 @@ using System; using System.Linq; +using System.Threading; using FluentAssertions; using OpenSearch.Client; using OpenSearch.Net; @@ -20,9 +21,9 @@ namespace Tests.Cat.CatSegmentReplication; [SkipVersion("<2.7.0", "/_cat/segment_replication was added in version 2.7.0")] public class CatSegmentReplicationApiTests - : ApiIntegrationTestBase, ICatSegmentReplicationRequest, CatSegmentReplicationDescriptor, CatSegmentReplicationRequest> + : ApiIntegrationTestBase, ICatSegmentReplicationRequest, CatSegmentReplicationDescriptor, CatSegmentReplicationRequest> { - public CatSegmentReplicationApiTests(ReplicatedReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + public CatSegmentReplicationApiTests(MultiNodeCluster cluster, EndpointUsage usage) : base(cluster, usage) { } private static readonly string IndexName = nameof(CatSegmentReplicationApiTests).ToLower(); @@ -45,9 +46,11 @@ protected override void ExpectResponse(CatResponse response.Records.Should().NotBeEmpty().And.AllSatisfy(r => r.ShardId.Should().StartWith($"[{IndexName}]")); protected override void IntegrationSetup(IOpenSearchClient client, CallUniqueValues values) - { + { var resp = client.Indices.Create(IndexName, d => d .Settings(s => s + .NumberOfShards(1) + .NumberOfReplicas(1) .Setting("index.replication.type", "SEGMENT"))); resp.ShouldBeValid(); @@ -56,9 +59,16 @@ protected override void IntegrationSetup(IOpenSearchClient client, CallUniqueVal .IndexMany(Enumerable.Range(0, 10).Select(i => new Doc { Id = i })) .Refresh(Refresh.WaitFor)); bulkResp.ShouldBeValid(); - } - protected override void IntegrationTeardown(IOpenSearchClient client, CallUniqueValues values) => client.Indices.Delete(IndexName); + var healthResp = client.Cluster.Health(IndexName, d => d + .WaitForStatus(HealthStatus.Green) + .WaitForNodes("2") + .Timeout("30s") + .Level(ClusterHealthLevel.Shards) + .WaitForNoInitializingShards() + .WaitForNoRelocatingShards()); + healthResp.ShouldBeValid(); + } public class Doc { From c62a5d8753c0f19cc3e8b509c11abf5591334dc9 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Fri, 16 Aug 2024 14:40:27 +1200 Subject: [PATCH 3/3] Use debug auditlog Signed-off-by: Thomas Farr --- .../EphemeralCluster.cs | 7 - .../EphemeralClusterConfiguration.cs | 128 +++++++++--------- 2 files changed, 64 insertions(+), 71 deletions(-) diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs index f0e3769c96..17bfc781b1 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs @@ -60,13 +60,6 @@ protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterCo protected EphemeralClusterComposer Composer { get; } - protected override void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) - { - base.ModifyNodeConfiguration(nodeConfiguration, port); - - if (!ClusterConfiguration.EnableSsl) nodeConfiguration.Add("plugins.security.disabled", "true"); - } - public virtual ICollection NodesUris(string hostName = null) { hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs index 77aaef02c4..644d1844c8 100644 --- a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs @@ -29,74 +29,74 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text.RegularExpressions; using OpenSearch.OpenSearch.Ephemeral.Plugins; using OpenSearch.OpenSearch.Ephemeral.Tasks; using OpenSearch.OpenSearch.Managed.Configuration; using OpenSearch.Stack.ArtifactsApi; -using OpenSearch.Stack.ArtifactsApi.Products; -namespace OpenSearch.OpenSearch.Ephemeral +namespace OpenSearch.OpenSearch.Ephemeral; + +public class EphemeralClusterConfiguration : ClusterConfiguration { - public class EphemeralClusterConfiguration : ClusterConfiguration - { - public EphemeralClusterConfiguration(OpenSearchVersion version, OpenSearchPlugins plugins = null, - int numberOfNodes = 1) - : this(version, ClusterFeatures.None, plugins, numberOfNodes) - { - } - - public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures features, - OpenSearchPlugins plugins = null, int numberOfNodes = 1) - : base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName) - { - Features = features; - - var pluginsList = plugins?.ToList() ?? new List(); - Plugins = new OpenSearchPlugins(pluginsList); - } - - private static string UniqueishSuffix => Guid.NewGuid().ToString("N").Substring(0, 6); - private static string EphemeralClusterName => $"ephemeral-cluster-{UniqueishSuffix}"; - - /// - /// The features supported by the cluster - /// - public ClusterFeatures Features { get; } - - /// - /// The collection of plugins to install - /// - public OpenSearchPlugins Plugins { get; } - - /// - /// Validates that the plugins to install can be installed on the target OpenSearch version. - /// This can be useful to fail early when subsequent operations are relying on installation - /// succeeding. - /// - public bool ValidatePluginsToInstall { get; } = true; - - public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL); - - public IList AdditionalBeforeNodeStartedTasks { get; } = new List(); - - public IList AdditionalAfterStartedTasks { get; } = new List(); - - /// - /// Expert level setting, skips all built-in validation tasks for cases where you need to guarantee your call is the - /// first call into the cluster - /// - public bool SkipBuiltInAfterStartTasks { get; set; } - - /// Bootstrapping HTTP calls should attempt to auto route traffic through fiddler if its running - public bool HttpFiddlerAware { get; set; } - - protected virtual string NodePrefix => "ephemeral"; - - public override string CreateNodeName(int? node) - { - var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); - return $"{NodePrefix}-node-{suffix}{node}"; - } - } + public EphemeralClusterConfiguration(OpenSearchVersion version, OpenSearchPlugins plugins = null, + int numberOfNodes = 1) + : this(version, ClusterFeatures.None, plugins, numberOfNodes) + { + } + + public EphemeralClusterConfiguration(OpenSearchVersion version, ClusterFeatures features, + OpenSearchPlugins plugins = null, int numberOfNodes = 1) + : base(version, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName) + { + Features = features; + + var pluginsList = plugins?.ToList() ?? []; + Plugins = new OpenSearchPlugins(pluginsList); + + Add("plugins.security.disabled", (!EnableSsl).ToString().ToLowerInvariant()); + if (EnableSsl) Add("plugins.security.audit.type", "debug"); + } + + private static string UniqueishSuffix => Guid.NewGuid().ToString("N").Substring(0, 6); + private static string EphemeralClusterName => $"ephemeral-cluster-{UniqueishSuffix}"; + + /// + /// The features supported by the cluster + /// + public ClusterFeatures Features { get; } + + /// + /// The collection of plugins to install + /// + public OpenSearchPlugins Plugins { get; } + + /// + /// Validates that the plugins to install can be installed on the target OpenSearch version. + /// This can be useful to fail early when subsequent operations are relying on installation + /// succeeding. + /// + public bool ValidatePluginsToInstall { get; } = true; + + public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL); + + public IList AdditionalBeforeNodeStartedTasks { get; } = new List(); + + public IList AdditionalAfterStartedTasks { get; } = new List(); + + /// + /// Expert level setting, skips all built-in validation tasks for cases where you need to guarantee your call is the + /// first call into the cluster + /// + public bool SkipBuiltInAfterStartTasks { get; set; } + + /// Bootstrapping HTTP calls should attempt to auto route traffic through fiddler if its running + public bool HttpFiddlerAware { get; set; } + + protected virtual string NodePrefix => "ephemeral"; + + public override string CreateNodeName(int? node) + { + var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); + return $"{NodePrefix}-node-{suffix}{node}"; + } }