diff --git a/abstractions/Directory.Build.props b/abstractions/Directory.Build.props new file mode 100644 index 0000000000..7b8bd0a7dd --- /dev/null +++ b/abstractions/Directory.Build.props @@ -0,0 +1,14 @@ + + + + canary + 0.1 + + latest + true + + + + + + \ No newline at end of file diff --git a/abstractions/OpenSearch.svg b/abstractions/OpenSearch.svg new file mode 100644 index 0000000000..ad4c18df2d --- /dev/null +++ b/abstractions/OpenSearch.svg @@ -0,0 +1 @@ + diff --git a/abstractions/README.md b/abstractions/README.md new file mode 100644 index 0000000000..2765381131 --- /dev/null +++ b/abstractions/README.md @@ -0,0 +1,79 @@ +![OpenSearch logo](OpenSearch.svg) + +- [OpenSearch .NET abstractions](#opensearch-net-abstractions) + - [OpenSearch.OpenSearch.Managed](#opensearchopensearchmanaged) + - [OpenSearch.OpenSearch.Ephemeral](#opensearchopensearchephemeral) + - [OpenSearch.OpenSearch.Xunit](#opensearchopensearchxunit) + - [OpenSearch.Stack.ArtifactsApi](#opensearchstackartifactsapi) +- [Project Resources](#project-resources) +- [Code of Conduct](#code-of-conduct) +- [Security](#security) +- [License](#license) +- [Copyright](#copyright) + +## Welcome! + +# OpenSearch .NET abstractions + +You've reached the home repository for several auxiliary projects from the .NET team within OpenSearch. + +Pre-release packages of these projects are available at [OpenSearch NuGet Package Repository](https://www.github.com/opensearch-project/packages). See [GitHub documentation](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-nuget-registry#installing-a-package) for instructions on how to access them. + +Current projects: + +### [OpenSearch.OpenSearch.Managed](src/OpenSearch.OpenSearch.Managed/README.md) + +Provides an easy to start/stop one or more OpenSearch instances that exists on disk already + +### [OpenSearch.OpenSearch.Ephemeral](src/OpenSearch.OpenSearch.Ephemeral/README.md) + +Bootstrap (download, install, configure) and run OpenSearch clusters with ease. +Started nodes are run in a new ephemeral location each time they are started and will clean up after they +are disposed. + +### [OpenSearch.OpenSearch.Xunit](src/OpenSearch.OpenSearch.Xunit/README.md) + +Write integration tests against OpenSearch. +Works with `.NET Core` and `.NET 4.6` and up. + +Supports `dotnet xunit`, `dotnet test`, `xunit.console.runner` and tests will be runnable in your IDE through VSTest and jetBrains Rider. + +### [OpenSearch.Stack.ArtifactsApi](src/OpenSearch.Stack.ArtifactsApi/README.md) + +Library to fetch the url and metadata for released artifacts. + +Supports: + +1. Snapshots builds + * `latest-MAJOR` where `MAJOR` is a single integer representing the major you want + * `latest` latest greatest + +2. Released versions + * `MAJOR.MINOR.PATH` where `MAJOR` is still supported as defined by the EOL policy of OpenSearch. + * Note if the version exists but is not yet released it will resolve as a build candidate + +## Project Resources + +* [Project Website](https://opensearch.org/) +* Need help? Try [Forums](https://discuss.opendistrocommunity.dev/) +* [Project Principles](https://opensearch.org/#principles) +* [Contributing to OpenSearch](CONTRIBUTING.md) +* [Maintainer Responsibilities](MAINTAINERS.md) +* [Release Management](RELEASING.md) +* [Admin Responsibilities](ADMINS.md) +* [Security](SECURITY.md) + +## Code of Conduct + +This project has adopted the [Amazon Open Source Code of Conduct](CODE_OF_CONDUCT.md). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq), or contact [opensource-codeofconduct@amazon.com](mailto:opensource-codeofconduct@amazon.com) with any additional questions or comments. + +## Security +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue. + +## License + +This project is licensed under the [Apache v2.0 License](LICENSE.txt). + +## Copyright + +Copyright OpenSearch Contributors. See [NOTICE](NOTICE.txt) for details. diff --git a/abstractions/build.bat b/abstractions/build.bat new file mode 100644 index 0000000000..b8df2a0857 --- /dev/null +++ b/abstractions/build.bat @@ -0,0 +1,2 @@ +@echo off +dotnet run --project build/scripts -- %* diff --git a/abstractions/build.sh b/abstractions/build.sh new file mode 100644 index 0000000000..8c4d9c1114 --- /dev/null +++ b/abstractions/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +dotnet run --project build/scripts -- "$@" diff --git a/abstractions/build/keys/keypair.snk b/abstractions/build/keys/keypair.snk new file mode 100644 index 0000000000..6de0112e8e Binary files /dev/null and b/abstractions/build/keys/keypair.snk differ diff --git a/abstractions/build/keys/public.snk b/abstractions/build/keys/public.snk new file mode 100644 index 0000000000..9c2e41d4a6 Binary files /dev/null and b/abstractions/build/keys/public.snk differ diff --git a/abstractions/build/scripts/CommandLine.fs b/abstractions/build/scripts/CommandLine.fs new file mode 100644 index 0000000000..bb984b2329 --- /dev/null +++ b/abstractions/build/scripts/CommandLine.fs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. +// +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +module CommandLine + +open Argu +open Microsoft.FSharp.Reflection + +type Arguments = + | [] Clean + | [] Build + + | [] PristineCheck + | [] GeneratePackages + | [] ValidatePackages + | [] GenerateReleaseNotes + | [] GenerateApiChanges + | [] Release + + | [] CreateReleaseOnGithub + | [] Publish + + | [] SingleTarget of bool + | [] Token of string +with + interface IArgParserTemplate with + member this.Usage = + match this with + | Clean _ -> "clean known output locations" + | Build _ -> "Run build and tests" + | Release _ -> "runs build, and create an validates the packages shy of publishing them" + | Publish _ -> "Runs the full release" + + | SingleTarget _ -> "Runs the provided sub command without running their dependencies" + | Token _ -> "Token to be used to authenticate with github" + + | PristineCheck + | GeneratePackages + | ValidatePackages + | GenerateReleaseNotes + | GenerateApiChanges + | CreateReleaseOnGithub + -> "Undocumented, dependent target" + member this.Name = + match FSharpValue.GetUnionFields(this, typeof) with + | case, _ -> case.Name.ToLowerInvariant() diff --git a/abstractions/build/scripts/Paths.fs b/abstractions/build/scripts/Paths.fs new file mode 100644 index 0000000000..d385b62526 --- /dev/null +++ b/abstractions/build/scripts/Paths.fs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. +// +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +module Paths + +open System +open System.IO + +let ToolName = "opensearch-net-abstractions" +let Repository = sprintf "opensearch/%s" ToolName +let MainTFM = "netstandard2.0" + +let ValidateAssemblyName = false +let IncludeGitHashInInformational = true +let GenerateApiChanges = false + +let Root = + let mutable dir = DirectoryInfo(".") + while dir.GetFiles("*.sln").Length = 0 do dir <- dir.Parent + Environment.CurrentDirectory <- dir.FullName + dir + +let RootRelative path = Path.GetRelativePath(Root.FullName, path) + +let Output = DirectoryInfo(Path.Combine(Root.FullName, "build", "output")) + +let ToolProject = DirectoryInfo(Path.Combine(Root.FullName, "src", ToolName)) diff --git a/abstractions/build/scripts/Program.fs b/abstractions/build/scripts/Program.fs new file mode 100644 index 0000000000..759b1d1726 --- /dev/null +++ b/abstractions/build/scripts/Program.fs @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. +// +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +module Program + +open Argu +open Bullseye +open ProcNet +open CommandLine + +[] +let main argv = + let parser = ArgumentParser.Create(programName = "./build.sh") + let parsed = + try + let parsed = parser.ParseCommandLine(inputs = argv, raiseOnUsage = true) + let arguments = parsed.GetSubCommand() + Some (parsed, arguments) + with e -> + printfn "%s" e.Message + None + + match parsed with + | None -> 2 + | Some (parsed, arguments) -> + + let target = arguments.Name + + Targets.Setup parsed arguments + let swallowTypes = [typeof; typeof] + + Targets.RunTargetsAndExit + ([target], (fun e -> swallowTypes |> List.contains (e.GetType()) ), ":") + 0 + diff --git a/abstractions/build/scripts/Targets.fs b/abstractions/build/scripts/Targets.fs new file mode 100644 index 0000000000..3b09cc92fd --- /dev/null +++ b/abstractions/build/scripts/Targets.fs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The OpenSearch Contributors require contributions made to +// this file be licensed under the Apache-2.0 license or a +// compatible open source license. +// +// Modifications Copyright OpenSearch Contributors. See +// GitHub history for details. +// +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +module Targets + +open Argu +open System +open System.IO +open Bullseye +open CommandLine +open Fake.Tools.Git +open ProcNet + + +let exec binary args = + let r = Proc.Exec (binary, args |> List.map (fun a -> sprintf "\"%s\"" a) |> List.toArray) + match r.HasValue with | true -> r.Value | false -> failwithf "invocation of `%s` timed out" binary + +let private restoreTools = lazy(exec "dotnet" ["tool"; "restore"]) +let private currentVersion = + lazy( + restoreTools.Value |> ignore + let r = Proc.Start("dotnet", "minver", "-d", "canary", "-m", "0.1") + let o = r.ConsoleOut |> Seq.find (fun l -> not(l.Line.StartsWith("MinVer:"))) + o.Line + ) +let private currentVersionInformational = + lazy( + match Paths.IncludeGitHashInInformational with + | false -> currentVersion.Value + | true -> sprintf "%s+%s" currentVersion.Value (Information.getCurrentSHA1( ".")) + ) + +let private clean (arguments:ParseResults) = + if (Paths.Output.Exists) then Paths.Output.Delete (true) + exec "dotnet" ["clean"] |> ignore + +let private build (arguments:ParseResults) = exec "dotnet" ["build"; "-c"; "Release"] |> ignore + +let private pristineCheck (arguments:ParseResults) = + match Information.isCleanWorkingCopy "." with + | true -> printfn "The checkout folder does not have pending changes, proceeding" + | _ -> failwithf "The checkout folder has pending changes, aborting" + +let private generatePackages (arguments:ParseResults) = + let output = Paths.RootRelative Paths.Output.FullName + exec "dotnet" ["pack"; "-c"; "Release"; "-o"; output] |> ignore + +let private validatePackages (arguments:ParseResults) = + let output = Paths.RootRelative <| Paths.Output.FullName + let nugetPackages = + Paths.Output.GetFiles("*.nupkg") |> Seq.sortByDescending(fun f -> f.CreationTimeUtc) + |> Seq.map (fun p -> Paths.RootRelative p.FullName) + + let appVeyorArgs = + if Fake.Core.Environment.environVarAsBool "APPVEYOR" then ["-r"; "true"] else [] + + let args = ["-v"; currentVersionInformational.Value; "-k"; "96c599bbe3e70f5d"; "-t"; output] @ appVeyorArgs + nugetPackages |> Seq.iter (fun p -> exec "dotnet" (["nupkg-validator"; p] @ args) |> ignore) + + +let private generateApiChanges (arguments:ParseResults) = + let output = Paths.RootRelative <| Paths.Output.FullName + let currentVersion = currentVersion.Value + let nugetPackages = + Paths.Output.GetFiles("*.nupkg") |> Seq.sortByDescending(fun f -> f.CreationTimeUtc) + |> Seq.map (fun p -> Path.GetFileNameWithoutExtension(Paths.RootRelative p.FullName).Replace("." + currentVersion, "")) + nugetPackages + |> Seq.iter(fun p -> + let outputFile = + let f = sprintf "breaking-changes-%s.md" p + Path.Combine(output, f) + let args = + [ + "assembly-differ" + (sprintf "previous-nuget|%s|%s|%s" p currentVersion Paths.MainTFM); + (sprintf "directory|src/%s/bin/Release/%s" p Paths.MainTFM); + "-a"; "true"; "--target"; p; "-f"; "github-comment"; "--output"; outputFile + ] + + exec "dotnet" args |> ignore + ) + +let private generateReleaseNotes (arguments:ParseResults) = + let currentVersion = currentVersion.Value + let output = + Paths.RootRelative <| Path.Combine(Paths.Output.FullName, sprintf "release-notes-%s.md" currentVersion) + let tokenArgs = + match arguments.TryGetResult Token with + | None -> [] + | Some token -> ["--token"; token;] + let releaseNotesArgs = + (Paths.Repository.Split("/") |> Seq.toList) + @ ["--version"; currentVersion + "--label"; "enhancement"; "New Features" + "--label"; "bug"; "Bug Fixes" + "--label"; "documentation"; "Docs Improvements" + ] @ tokenArgs + @ ["--output"; output] + + exec "dotnet" (["release-notes"] @ releaseNotesArgs) |> ignore + +let private createReleaseOnGithub (arguments:ParseResults) = + let currentVersion = currentVersion.Value + let tokenArgs = + match arguments.TryGetResult Token with + | None -> [] + | Some token -> ["--token"; token;] + let releaseNotes = Paths.RootRelative <| Path.Combine(Paths.Output.FullName, sprintf "release-notes-%s.md" currentVersion) + let breakingChanges = + let breakingChangesDocs = Paths.Output.GetFiles("breaking-changes-*.md") + breakingChangesDocs + |> Seq.map(fun f -> ["--body"; Paths.RootRelative f.FullName]) + |> Seq.collect id + |> Seq.toList + let releaseArgs = + (Paths.Repository.Split("/") |> Seq.toList) + @ ["create-release" + "--version"; currentVersion + "--body"; releaseNotes; + ] @ breakingChanges @ tokenArgs + + exec "dotnet" (["release-notes"] @ releaseArgs) |> ignore + +let private release (arguments:ParseResults) = printfn "release" + +let private publish (arguments:ParseResults) = printfn "publish" + +let Setup (parsed:ParseResults) (subCommand:Arguments) = + let step (name:string) action = Targets.Target(name, new Action(fun _ -> action(parsed))) + + let cmd (name:string) commandsBefore steps action = + let singleTarget = (parsed.TryGetResult SingleTarget |> Option.defaultValue false) + let deps = + match (singleTarget, commandsBefore) with + | (true, _) -> [] + | (_, Some d) -> d + | _ -> [] + let steps = steps |> Option.defaultValue [] + Targets.Target(name, deps @ steps, Action(action)) + + step Clean.Name clean + cmd Build.Name None (Some [Clean.Name]) <| fun _ -> build parsed + + step PristineCheck.Name pristineCheck + step GeneratePackages.Name generatePackages + step ValidatePackages.Name validatePackages + step GenerateReleaseNotes.Name generateReleaseNotes + step GenerateApiChanges.Name generateApiChanges + cmd Release.Name + (Some [PristineCheck.Name; Build.Name;]) + (Some [GeneratePackages.Name; ValidatePackages.Name; GenerateReleaseNotes.Name; GenerateApiChanges.Name]) + <| fun _ -> release parsed + + step CreateReleaseOnGithub.Name createReleaseOnGithub + cmd Publish.Name + (Some [Release.Name]) + (Some [CreateReleaseOnGithub.Name; ]) + <| fun _ -> publish parsed diff --git a/abstractions/build/scripts/scripts.fsproj b/abstractions/build/scripts/scripts.fsproj new file mode 100644 index 0000000000..b0377d43e3 --- /dev/null +++ b/abstractions/build/scripts/scripts.fsproj @@ -0,0 +1,22 @@ + + + + Exe + net5.0 + false + + + + + + + + + + + + + + + + diff --git a/abstractions/dotnet-tools.json b/abstractions/dotnet-tools.json new file mode 100644 index 0000000000..6da03022c9 --- /dev/null +++ b/abstractions/dotnet-tools.json @@ -0,0 +1,30 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "minver-cli": { + "version": "2.3.1", + "commands": [ + "minver" + ] + }, + "assembly-differ": { + "version": "0.13.0", + "commands": [ + "assembly-differ" + ] + }, + "release-notes": { + "version": "0.3.0", + "commands": [ + "release-notes" + ] + }, + "nupkg-validator": { + "version": "0.4.0", + "commands": [ + "nupkg-validator" + ] + } + } +} \ No newline at end of file diff --git a/abstractions/nuget-icon.png b/abstractions/nuget-icon.png new file mode 100644 index 0000000000..edbf28ee83 Binary files /dev/null and b/abstractions/nuget-icon.png differ diff --git a/abstractions/src/Directory.Build.props b/abstractions/src/Directory.Build.props new file mode 100644 index 0000000000..968df46aea --- /dev/null +++ b/abstractions/src/Directory.Build.props @@ -0,0 +1,39 @@ + + + + + OpenSearch Project and contributors + OpenSearch + Apache-2.0 + https://github.com/opensearch-project/opensearch-net-abstractions + Git + https://github.com/opensearch-project/opensearch-net-abstractions + https://github.com/opensearch-project/opensearch-net-abstractions/releases + + + true + ..\..\build\keys\keypair.snk + + $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb + true + nuget-icon.png + + + + + + nuget-icon.png + True + nuget-icon.png + + + True + LICENSE.txt + + + + + + + + diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterAuthentication.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterAuthentication.cs new file mode 100644 index 0000000000..5e041115cb --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterAuthentication.cs @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +namespace OpenSearch.OpenSearch.Ephemeral +{ + /// + /// Authentication credentials for the cluster + /// + public class ClusterAuthentication + { + /// + /// Administrator credentials + /// + public static Credentials Admin => new Credentials {Username = "admin", Role = "admin"}; + + /// + /// User credentials + /// + public static Credentials User => new Credentials {Username = "admin", Role = "admin"}; + + /// + /// Credentials for all users + /// + public static Credentials[] AllUsers { get; } = {Admin, User}; + + /// + /// Authentication credentials + /// + public class Credentials + { + public string Username { get; set; } + public string Role { get; set; } + public string Password => Username; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterFeatures.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterFeatures.cs new file mode 100644 index 0000000000..632270bf69 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/ClusterFeatures.cs @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; + +namespace OpenSearch.OpenSearch.Ephemeral +{ + /// + /// Hints to what features the cluster to be started should have. + /// It's up to the to actually bootstrap these features. + /// + [Flags] + public enum ClusterFeatures + { + /// + /// No features + /// + None = 1 << 0, + + /// + /// SSL/TLS for HTTP and Transport layers + /// + SSL = 1 << 3, + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs new file mode 100644 index 0000000000..cc95c7d9f7 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralCluster.cs @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using OpenSearch.OpenSearch.Managed; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Ephemeral +{ + public class EphemeralCluster : EphemeralCluster + { + public EphemeralCluster(OpenSearchVersion version, int numberOfNodes = 1) + : base(new EphemeralClusterConfiguration(version, ServerType.DEFAULT, ClusterFeatures.None, numberOfNodes: numberOfNodes)) + { + } + + public EphemeralCluster(EphemeralClusterConfiguration clusterConfiguration) : base(clusterConfiguration) + { + } + } + + public abstract class EphemeralCluster : ClusterBase, + IEphemeralCluster + where TConfiguration : EphemeralClusterConfiguration + { + protected EphemeralCluster(TConfiguration clusterConfiguration) : base(clusterConfiguration) => + Composer = new EphemeralClusterComposer(this); + + protected EphemeralClusterComposer Composer { get; } + + public virtual ICollection NodesUris(string hostName = null) + { + hostName = hostName ?? (ClusterConfiguration.HttpFiddlerAware && Process.GetProcessesByName("fiddler").Any() + ? "ipv4.fiddler" + : "localhost"); + var ssl = ClusterConfiguration.EnableSsl ? "s" : ""; + return Nodes + .Select(n => $"http{ssl}://{hostName}:{n.Port ?? 9200}") + .Distinct() + .Select(n => new Uri(n)) + .ToList(); + } + + public bool CachingAndCachedHomeExists() + { + if (!ClusterConfiguration.CacheOpenSearchHomeInstallation) return false; + var cachedOpenSearchHomeFolder = Path.Combine(FileSystem.LocalFolder, GetCacheFolderName()); + return Directory.Exists(cachedOpenSearchHomeFolder); + } + + public virtual string GetCacheFolderName() + { + var config = ClusterConfiguration; + + var sb = new StringBuilder(); + sb.Append(EphemeralClusterComposerBase.InstallationTasks.Count()); + sb.Append("-"); + if (config.EnableSsl) sb.Append("ssl"); + if (config.Plugins != null && config.Plugins.Count > 0) + { + sb.Append("-"); + foreach (var p in config.Plugins.OrderBy(p => p.SubProductName)) + sb.Append(p.SubProductName.ToLowerInvariant()); + } + + var name = sb.ToString(); + + return CalculateSha1(name, Encoding.UTF8); + } + + protected override void OnBeforeStart() + { + Composer.Install(); + Composer.OnBeforeStart(); + } + + protected override void OnDispose() => Composer.OnStop(); + + protected override void OnAfterStarted() => Composer.OnAfterStart(); + + protected override string SeeLogsMessage(string message) + { + var log = Path.Combine(FileSystem.LogsPath, $"{ClusterConfiguration.ClusterName}.log"); + if (!File.Exists(log) || ClusterConfiguration.ShowOpenSearchOutputAfterStarted) return message; + if (!Started) return message; + using (var fileStream = new FileStream(log, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + using (var textReader = new StreamReader(fileStream)) + { + var logContents = textReader.ReadToEnd(); + return message + $" contents of {log}:{Environment.NewLine}" + logContents; + } + } + + public static string CalculateSha1(string text, Encoding enc) + { + var buffer = enc.GetBytes(text); + var cryptoTransformSha1 = new SHA1CryptoServiceProvider(); + return BitConverter.ToString(cryptoTransformSha1.ComputeHash(buffer)) + .Replace("-", "").ToLowerInvariant().Substring(0, 12); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterComposer.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterComposer.cs new file mode 100644 index 0000000000..c245935598 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterComposer.cs @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using OpenSearch.OpenSearch.Ephemeral.Tasks; +using OpenSearch.OpenSearch.Ephemeral.Tasks.AfterNodeStoppedTasks; +using OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks; +using OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; +using OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks; +using OpenSearch.OpenSearch.Managed.FileSystem; + +namespace OpenSearch.OpenSearch.Ephemeral +{ + public class EphemeralClusterComposerBase + { + protected EphemeralClusterComposerBase() + { + } + + internal static IEnumerable InstallationTasks { get; } = new List + { + new PrintConfiguration(), + new CreateLocalApplicationDirectory(), + new CopyCachedOpenSearchInstallation(), + new EnsureJavaHomeEnvironmentVariableIsSet(), + new DownloadOpenSearchVersion(), + new UnzipOpenSearch(), + new SetOpenSearchBundledJdkJavaHome(), + new InstallPlugins(), + new InitialConfiguration() + }; + + protected static IEnumerable BeforeStart { get; } = new List + { + new CreateEphemeralDirectory(), + new CacheOpenSearchInstallation() + }; + + protected static IEnumerable NodeStoppedTasks { get; } = new List + { + new CleanUpDirectoriesAfterNodeStopped() + }; + + protected static IEnumerable AfterStartedTasks { get; } = new List + { + new ValidateRunningVersion(), + new ValidateClusterStateTask(), + new ValidatePluginsTask(), + }; + } + + + public class EphemeralClusterComposer : EphemeralClusterComposerBase + where TConfiguration : EphemeralClusterConfiguration + { + private readonly object _lock = new object(); + public EphemeralClusterComposer(IEphemeralCluster cluster) => Cluster = cluster; + + private IEphemeralCluster Cluster { get; } + + private bool NodeStarted { get; set; } + + public void OnStop() => Itterate(NodeStoppedTasks, (t, c, fs) => t.Run(c, NodeStarted), false); + + public void Install() => Itterate(InstallationTasks, (t, c, fs) => t.Run(c)); + + public void OnBeforeStart() + { + var tasks = new List(BeforeStart); + if (Cluster.ClusterConfiguration.AdditionalBeforeNodeStartedTasks != null) + tasks.AddRange(Cluster.ClusterConfiguration.AdditionalBeforeNodeStartedTasks); + + if (Cluster.ClusterConfiguration.PrintYamlFilesInConfigFolder) + tasks.Add(new PrintYamlContents()); + + Itterate(tasks, (t, c, fs) => t.Run(c)); + + NodeStarted = true; + } + + public void OnAfterStart() + { + if (Cluster.ClusterConfiguration.SkipBuiltInAfterStartTasks) return; + var tasks = new List(AfterStartedTasks); + if (Cluster.ClusterConfiguration.AdditionalAfterStartedTasks != null) + tasks.AddRange(Cluster.ClusterConfiguration.AdditionalAfterStartedTasks); + Itterate(tasks, (t, c, fs) => t.Run(c), false); + } + + private void Itterate(IEnumerable collection, + Action, INodeFileSystem> act, bool callOnStop = true) + { + lock (_lock) + { + var cluster = Cluster; + foreach (var task in collection) + try + { + act(task, cluster, cluster.FileSystem); + } + catch (Exception) + { + if (callOnStop) OnStop(); + throw; + } + } + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs new file mode 100644 index 0000000000..181a569829 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralClusterConfiguration.cs @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using OpenSearch.OpenSearch.Ephemeral.Plugins; +using OpenSearch.OpenSearch.Ephemeral.Tasks; +using OpenSearch.OpenSearch.Managed.Configuration; +using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; + +namespace OpenSearch.OpenSearch.Ephemeral +{ + public class EphemeralClusterConfiguration : ClusterConfiguration + { + public EphemeralClusterConfiguration(OpenSearchVersion version, ServerType serverType, OpenSearchPlugins plugins = null, + int numberOfNodes = 1) + : this(version, serverType, ClusterFeatures.None, plugins, numberOfNodes) + { + } + + public EphemeralClusterConfiguration(OpenSearchVersion version, ServerType serverType, ClusterFeatures features, + OpenSearchPlugins plugins = null, int numberOfNodes = 1) + : base(version, serverType, (v, s) => new EphemeralFileSystem(v, s), numberOfNodes, EphemeralClusterName) + { + Features = features; + + var pluginsList = plugins?.ToList() ?? new List(); + Plugins = new OpenSearchPlugins(pluginsList); + } + + private static string UniqueishSuffix => Guid.NewGuid().ToString("N").Substring(0, 6); + private static string EphemeralClusterName => $"ephemeral-cluster-{UniqueishSuffix}"; + + /// + /// The features supported by the cluster + /// + public ClusterFeatures Features { get; } + + /// + /// The collection of plugins to install + /// + public OpenSearchPlugins Plugins { get; } + + /// + /// Validates that the plugins to install can be installed on the target OpenSearch version. + /// This can be useful to fail early when subsequent operations are relying on installation + /// succeeding. + /// + public bool ValidatePluginsToInstall { get; } = true; + + public bool EnableSsl => Features.HasFlag(ClusterFeatures.SSL); + + public IList AdditionalBeforeNodeStartedTasks { get; } = new List(); + + public IList AdditionalAfterStartedTasks { get; } = new List(); + + /// + /// Expert level setting, skips all built-in validation tasks for cases where you need to guarantee your call is the + /// first call into the cluster + /// + public bool SkipBuiltInAfterStartTasks { get; set; } + + /// Bootstrapping HTTP calls should attempt to auto route traffic through fiddler if its running + public bool HttpFiddlerAware { get; set; } + + protected virtual string NodePrefix => "ephemeral"; + + public override string CreateNodeName(int? node) + { + var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); + return $"{NodePrefix}-node-{suffix}{node}"; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralFileSystem.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralFileSystem.cs new file mode 100644 index 0000000000..f27b6f6484 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/EphemeralFileSystem.cs @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.FileSystem; +using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; + +namespace OpenSearch.OpenSearch.Ephemeral +{ + public class EphemeralFileSystem : NodeFileSystem + { + public EphemeralFileSystem(OpenSearchVersion version, string clusterName) : base(version, + EphemeralHome(version, clusterName)) => ClusterName = clusterName; + + private string ClusterName { get; } + + public string TempFolder => Path.Combine(Path.GetTempPath(), SubFolder, Artifact.LocalFolderName, ClusterName); + + public override string ConfigPath => Path.Combine(TempFolder, "config"); + public override string LogsPath => Path.Combine(TempFolder, "logs"); + public override string RepositoryPath => Path.Combine(TempFolder, "repositories"); + public override string DataPath => Path.Combine(TempFolder, "data"); + + //certificates + public string CertificateFolderName => "node-certificates"; + public string CertificateNodeName => "node01"; + public string ClientCertificateName => "cn=John Doe,ou=example,o=com"; + public string ClientCertificateFilename => "john_doe"; + + public string CertificatesPath => Path.Combine(ConfigPath, CertificateFolderName); + + public string CaCertificate => Path.Combine(CertificatesPath, "ca", "ca") + ".crt"; + + public string NodePrivateKey => + Path.Combine(CertificatesPath, CertificateNodeName, CertificateNodeName) + ".key"; + + public string NodeCertificate => + Path.Combine(CertificatesPath, CertificateNodeName, CertificateNodeName) + ".crt"; + + public string ClientCertificate => + Path.Combine(CertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".crt"; + + public string ClientPrivateKey => + Path.Combine(CertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".key"; + + public string UnusedCertificateFolderName => $"unused-{CertificateFolderName}"; + public string UnusedCertificatesPath => Path.Combine(ConfigPath, UnusedCertificateFolderName); + public string UnusedCaCertificate => Path.Combine(UnusedCertificatesPath, "ca", "ca") + ".crt"; + + public string UnusedClientCertificate => + Path.Combine(UnusedCertificatesPath, ClientCertificateFilename, ClientCertificateFilename) + ".crt"; + + + protected static string EphemeralHome(OpenSearchVersion version, string clusterName) + { + var temp = Path.Combine(Path.GetTempPath(), SubFolder, + version.Artifact(Product.OpenSearch).LocalFolderName, clusterName); + return Path.Combine(temp, "home"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/IEphemeralCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/IEphemeralCluster.cs new file mode 100644 index 0000000000..b5d9f8e7cc --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/IEphemeralCluster.cs @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using OpenSearch.OpenSearch.Managed; + +namespace OpenSearch.OpenSearch.Ephemeral +{ + public interface IEphemeralCluster + { + ICollection NodesUris(string hostName = null); + string GetCacheFolderName(); + bool CachingAndCachedHomeExists(); + } + + public interface IEphemeralCluster : IEphemeralCluster, ICluster + where TConfiguration : EphemeralClusterConfiguration + { + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/OpenSearch.OpenSearch.Ephemeral.csproj b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/OpenSearch.OpenSearch.Ephemeral.csproj new file mode 100644 index 0000000000..a4ec93b16c --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/OpenSearch.OpenSearch.Ephemeral.csproj @@ -0,0 +1,15 @@ + + + + netstandard2.0;net461 + Provides an EphemeralCluster implementation that can download/bootstrap/run a throwaway customizable OpenSearch cluster + opensearch,opensearch,cluster,ephemeral + + + + + + + + + \ No newline at end of file diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs new file mode 100644 index 0000000000..3833aba1cf --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Plugins/OpenSearchPlugins.cs @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using OpenSearch.Stack.ArtifactsApi.Products; + +namespace OpenSearch.OpenSearch.Ephemeral.Plugins +{ + public class OpenSearchPlugins : ReadOnlyCollection + { + public OpenSearchPlugins(IList list) : base(list) + { + } + + public OpenSearchPlugins(params OpenSearchPlugin[] list) : base(list) + { + } + + public static OpenSearchPlugins Supported { get; } = + new OpenSearchPlugins(new List + { + OpenSearchPlugin.AnalysisIcu, + OpenSearchPlugin.AnalysisKuromoji, + OpenSearchPlugin.AnalysisPhonetic, + OpenSearchPlugin.AnalysisSmartCn, + OpenSearchPlugin.AnalysisStempel, + OpenSearchPlugin.AnalysisUkrainian, + OpenSearchPlugin.DiscoveryAzureClassic, + OpenSearchPlugin.DiscoveryEC2, + OpenSearchPlugin.DiscoveryFile, + OpenSearchPlugin.DiscoveryGCE, + OpenSearchPlugin.IngestAttachment, + OpenSearchPlugin.IngestGeoIp, + OpenSearchPlugin.IngestUserAgent, + OpenSearchPlugin.MapperAttachment, + OpenSearchPlugin.MapperMurmur3, + OpenSearchPlugin.MapperSize, + OpenSearchPlugin.RepositoryAzure, + OpenSearchPlugin.RepositoryGCS, + OpenSearchPlugin.RepositoryHDFS, + OpenSearchPlugin.RepositoryS3, + OpenSearchPlugin.StoreSMB, + OpenSearchPlugin.DeleteByQuery, + }); + + public override string ToString() => string.Join(", ", Items.Select(s => s.SubProductName)); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/README.md b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/README.md new file mode 100644 index 0000000000..39f5814d22 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/README.md @@ -0,0 +1,60 @@ +# OpenSearch.Managed.Ephemeral + +Bootstrap (download, install, configure) and run OpenSearch clusters with ease. +Started nodes are run in a new ephemeral location each time they are started and will clean up after they +are disposed. + + +## EphemeralCluster + +A `ClusterBase` implementation from `OpenSearch.Managed` that can: + +* download opensearch versions (stable releases, snapshots, build candidates) +* download opensearch plugins (stable releases, snapshots, build candidates) +* install opensearch and desired plugins in an ephemeral location. The source downloaded zips are cached +on disk (LocalAppData). +* Ships with builtin knowledge on how to enable SSL on the running cluster. +* Start opensearch using ephemeral locations for OPENSEARCH_HOME and conf/logs/data paths. + + +#### Examples: + +The easiest way to get started is by simply passing the version you want to be bootstrapped to `EphemeralCluster`. +`Start` starts the `OpenSearchNode`'s and waits for them to be started. The default overload waits `2 minutes`. + +```csharp +using (var cluster = new EphemeralCluster("1.0.0")) +{ + cluster.Start(); +} +``` + +If you want the full configuration possibilities inject a `EphemeralClusterConfiguration` instead: + + +```csharp +var plugins = new OpenSearchPlugins(OpenSearchPlugin.RepositoryAzure, OpenSearchPlugin.IngestAttachment); +var config = new EphemeralClusterConfiguration("1.0.0", ServerType.OpenSearch, ClusterFeatures.None, plugins, numberOfNodes: 2); +using (var cluster = new EphemeralCluster(config)) +{ + cluster.Start(); + + var nodes = cluster.NodesUris(); + var connectionPool = new StaticConnectionPool(nodes); + var settings = new ConnectionSettings(connectionPool).EnableDebugMode(); + var client = new OpenSearchClient(settings); + + Console.Write(client.CatPlugins().DebugInformation); +} +``` +Here we first create a `OpenSearchPlugins` collection of the plugins that we want to bootstrap. +Then we create an instance of `EphemeralClusterConfiguration` that dictates we want a 2 node cluster +running opensearch using the previous declared `plugins`. + +We then Start the node and after its up create a `OSC` client using the `NodeUris()` that the cluster +started. + +We call `/_cat/plugins` and write `OSC`'s debug information to the console. + +When the cluster exits the using block and disposes the cluster all nodes will be shutdown gracefully. + diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/SecurityRealms.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/SecurityRealms.cs new file mode 100644 index 0000000000..d776863306 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/SecurityRealms.cs @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +namespace OpenSearch.OpenSearch.Ephemeral +{ + public static class SecurityRealms + { + public const string FileRealm = "file1"; + + public const string PkiRealm = "pki1"; + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/AfterNodeStoppedTasks/CleanUpDirectoriesAfterNodeStopped.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/AfterNodeStoppedTasks/CleanUpDirectoriesAfterNodeStopped.cs new file mode 100644 index 0000000000..e4cd466796 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/AfterNodeStoppedTasks/CleanUpDirectoriesAfterNodeStopped.cs @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.AfterNodeStoppedTasks +{ + public class CleanUpDirectoriesAfterNodeStopped : IClusterTeardownTask + { + public void Run(IEphemeralCluster cluster, bool nodeStarted) + { + var fs = cluster.FileSystem; + var w = cluster.Writer; + var a = cluster.ClusterConfiguration.Artifact; + if (cluster.ClusterConfiguration.NoCleanupAfterNodeStopped) + { + w.WriteDiagnostic( + $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} skipping cleanup as requested on cluster configuration"); + return; + } + + DeleteDirectory(w, "cluster data", fs.DataPath); + DeleteDirectory(w, "cluster config", fs.ConfigPath); + DeleteDirectory(w, "cluster logs", fs.LogsPath); + DeleteDirectory(w, "repositories", fs.RepositoryPath); + var efs = fs as EphemeralFileSystem; + if (!string.IsNullOrWhiteSpace(efs?.TempFolder)) + DeleteDirectory(w, "cluster temp folder", efs.TempFolder); + + if (efs != null) + { + var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip); + if (extractedFolder != fs.OpenSearchHome) + DeleteDirectory(w, "ephemeral OPENSEARCH_HOME", fs.OpenSearchHome); + //if the node was not started delete the cached extractedFolder + if (!nodeStarted) + DeleteDirectory(w, "cached extracted folder - node failed to start", extractedFolder); + } + + //if the node did not start make sure we delete the cached folder as we can not assume its in a good state + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + if (cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation && !nodeStarted) + DeleteDirectory(w, "cached installation - node failed to start", cachedOpenSearchHomeFolder); + else + w.WriteDiagnostic( + $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} Leaving [cached folder] on disk: {{{cachedOpenSearchHomeFolder}}}"); + } + + private static void DeleteDirectory(IConsoleLineHandler w, string description, string path) + { + if (!Directory.Exists(path)) return; + w.WriteDiagnostic( + $"{{{nameof(CleanUpDirectoriesAfterNodeStopped)}}} attempting to delete [{description}]: {{{path}}}"); + Directory.Delete(path, true); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CacheOpenSearchInstallation.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CacheOpenSearchInstallation.cs new file mode 100644 index 0000000000..7bad124a23 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CacheOpenSearchInstallation.cs @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks +{ + public class CacheOpenSearchInstallation : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; + + var fs = cluster.FileSystem; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var cachedOpenSearchConfig = Path.Combine(cachedOpenSearchHomeFolder, "config"); + if (File.Exists(cachedOpenSearchConfig)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CacheOpenSearchInstallation)}}} cached home already exists [{cachedOpenSearchHomeFolder}]"); + return; + } + + var source = fs.OpenSearchHome; + var target = cachedOpenSearchHomeFolder; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CacheOpenSearchInstallation)}}} caching {{{source}}} to [{target}]"); + CopyFolder(source, target, false); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CreateEphemeralDirectory.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CreateEphemeralDirectory.cs new file mode 100644 index 0000000000..745f32ec45 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/CreateEphemeralDirectory.cs @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.OpenSearch.Managed.FileSystem; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks +{ + public class CreateEphemeralDirectory : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; + if (!(fs is EphemeralFileSystem f)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} unexpected IFileSystem implementation {{{fs.GetType()}}}"); + return; + } + + cluster.Writer?.WriteDiagnostic($"{{{nameof(CreateEphemeralDirectory)}}} creating {{{f.TempFolder}}}"); + + Directory.CreateDirectory(f.TempFolder); + + if (!Directory.Exists(f.ConfigPath)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} creating config folder {{{f.ConfigPath}}}"); + Directory.CreateDirectory(f.ConfigPath); + } + + CopyHomeConfigToEphemeralConfig(cluster, f, fs); + } + + private static void CopyHomeConfigToEphemeralConfig(IEphemeralCluster cluster, + EphemeralFileSystem ephemeralFileSystem, INodeFileSystem fs) + { + var target = ephemeralFileSystem.ConfigPath; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var cachedOpenSearchYaml = Path.Combine(cachedOpenSearchHomeFolder, "config", "opensearch.yaml"); + + var homeSource = + cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation && File.Exists(cachedOpenSearchYaml) + ? cachedOpenSearchHomeFolder + : fs.OpenSearchHome; + var source = Path.Combine(homeSource, "config"); + if (!Directory.Exists(source)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} source config {{{source}}} does not exist nothing to copy"); + return; + } + + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateEphemeralDirectory)}}} copying cached {{{source}}} as to [{target}]"); + CopyFolder(source, target); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/PrintYamlContents.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/PrintYamlContents.cs new file mode 100644 index 0000000000..c629294bee --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/BeforeStartNodeTasks/PrintYamlContents.cs @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using System.Linq; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.BeforeStartNodeTasks +{ + public class PrintYamlContents : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + var c = cluster.ClusterConfiguration; + var v = c.Version; + var fs = cluster.FileSystem; + + var files = Directory.GetFiles(fs.ConfigPath, "*.yml", SearchOption.AllDirectories); + foreach (var file in files) DumpFile(cluster, file); + } + + private static void DumpFile(IEphemeralCluster cluster, string configFile) + { + if (!File.Exists(configFile)) + { + cluster.Writer.WriteDiagnostic( + $"{{{nameof(PrintYamlContents)}}} skipped printing [{configFile}] as it does not exists"); + return; + } + + var fileName = Path.GetFileName(configFile); + cluster.Writer.WriteDiagnostic($"{{{nameof(PrintYamlContents)}}} printing [{configFile}]"); + var lines = File.ReadAllLines(configFile).ToList(); + foreach (var l in lines.Where(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith("#"))) + cluster.Writer.WriteDiagnostic($"{{{nameof(PrintYamlContents)}}} [{fileName}] {l}"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs new file mode 100644 index 0000000000..1ba2581c74 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/IClusterComposeTask.cs @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Security; +using System.Runtime.InteropServices; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using ICSharpCode.SharpZipLib.GZip; +using ICSharpCode.SharpZipLib.Tar; +using ProcNet; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks +{ + public interface IClusterComposeTask + { + void Run(IEphemeralCluster cluster); + } + + public interface IClusterTeardownTask + { + /// + /// Called when the cluster disposes, used to clean up after itself. + /// + /// The cluster configuration of the node that was started + /// Whether the cluster composer was successful in starting the node + void Run(IEphemeralCluster cluster, bool nodeStarted); + } + + public abstract class ClusterComposeTask : IClusterComposeTask + { + protected static bool IsWindows { get; } = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + protected static string BinarySuffix => IsWindows ? ".bat" : string.Empty; + public abstract void Run(IEphemeralCluster cluster); + + protected static void DownloadFile(string from, string to) + { + if (File.Exists(to)) return; + var http = new HttpClient(); + using (var stream = http.GetStreamAsync(new Uri(from)).GetAwaiter().GetResult()) + using (var fileStream = File.Create(to)) + { + stream.CopyTo(fileStream); + fileStream.Flush(); + } + } + + protected string GetResponseException(HttpResponseMessage m) => + $"Code: {m?.StatusCode} Reason: {m?.ReasonPhrase} Content: {GetResponseString(m)}"; + + protected string GetResponseString(HttpResponseMessage m) => + m?.Content?.ReadAsStringAsync().ConfigureAwait(false).GetAwaiter().GetResult() ?? string.Empty; + + protected HttpResponseMessage Get(IEphemeralCluster cluster, string path, + string query) => + Call(cluster, path, query, (c, u, t) => c.GetAsync(u, t)); + + protected HttpResponseMessage Post(IEphemeralCluster cluster, string path, + string query, string json) => + Call(cluster, path, query, + (c, u, t) => c.PostAsync(u, new StringContent(json, Encoding.UTF8, "application/json"), t)); + + private HttpResponseMessage Call( + IEphemeralCluster cluster, + string path, + string query, + Func> verb) + { + var q = string.IsNullOrEmpty(query) ? "pretty=true" : query + "&pretty=true"; + var statusUrl = new UriBuilder(cluster.NodesUris().First()) {Path = path, Query = q}.Uri; + + var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + var handler = new HttpClientHandler + { + AutomaticDecompression = + DecompressionMethods.Deflate | DecompressionMethods.GZip | DecompressionMethods.None, + }; + cluster.Writer.WriteDiagnostic( + $"{{{nameof(Call)}}} [{statusUrl}] SSL: {cluster.ClusterConfiguration.EnableSsl}"); + if (cluster.ClusterConfiguration.EnableSsl) + { +#if !NETSTANDARD + ServicePointManager.ServerCertificateValidationCallback += ServerCertificateValidationCallback; +#else + handler.ServerCertificateCustomValidationCallback += (m, c, cn, p) => true; +#endif + } + + using (var client = new HttpClient(handler) {Timeout = TimeSpan.FromSeconds(20)}) + { + if (cluster.ClusterConfiguration.EnableSsl) + { + var byteArray = + Encoding.ASCII.GetBytes( + $"{ClusterAuthentication.Admin.Username}:{ClusterAuthentication.Admin.Password}"); + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray)); + } + + try + { + var response = verb(client, statusUrl, tokenSource.Token).ConfigureAwait(false).GetAwaiter() + .GetResult(); + if (!response.IsSuccessStatusCode) + { + cluster.Writer.WriteDiagnostic( + $"{{{nameof(Call)}}} [{statusUrl}] Unsuccessful status code: [{(int) response.StatusCode}]"); + var body = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + foreach (var l in (body ?? string.Empty).Split('\n', '\r')) + cluster.Writer.WriteDiagnostic($"{{{nameof(Call)}}} [{statusUrl}] returned [{l}]"); + } + + return response; + } + catch (Exception e) + { + cluster.Writer.WriteError($"{{{nameof(Call)}}} [{statusUrl}] exception: {e}"); + throw; + } + finally + { +#if !NETSTANDARD + ServicePointManager.ServerCertificateValidationCallback -= ServerCertificateValidationCallback; +#endif + } + } + } + + private static bool ServerCertificateValidationCallback(object sender, X509Certificate certificate, + X509Chain chain, SslPolicyErrors sslpolicyerrors) => true; + + protected static void WriteFileIfNotExist(string fileLocation, string contents) + { + if (!File.Exists(fileLocation)) File.WriteAllText(fileLocation, contents); + } + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, null, arguments); + + protected static void ExecuteBinary(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, StartedHandler startedHandler, params string[] arguments) => + ExecuteBinaryInternal(config, writer, binary, description, startedHandler, arguments); + + private static void ExecuteBinaryInternal(EphemeralClusterConfiguration config, IConsoleLineHandler writer, + string binary, string description, StartedHandler startedHandler, params string[] arguments) + { + var command = $"{{{binary}}} {{{string.Join(" ", arguments)}}}"; + writer?.WriteDiagnostic($"{{{nameof(ExecuteBinary)}}} starting process [{description}] {command}"); + + var timeout = TimeSpan.FromSeconds(420); + var processStartArguments = new StartArguments(binary, arguments) + { + Environment = new Dictionary + { + {config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath}, + {"OPENSEARCH_HOME", config.FileSystem.OpenSearchHome}, + // Duplicate all env vars for ES_* prefix for backward compatibility with OpenDistro and ElasticSearch; + // unused env vars would be just ignored. + {config.FileSystem.ConfigEnvironmentVariableName.Replace("OPENSEARCH", "ES"), config.FileSystem.ConfigPath}, + {"ES_HOME", config.FileSystem.OpenSearchHome} + } + }; + + var result = startedHandler != null + ? Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter(), startedHandler) + : Proc.Start(processStartArguments, timeout, new ConsoleOutColorWriter()); + + if (!result.Completed) + throw new Exception($"Timeout while executing {description} exceeded {timeout}"); + + if (result.ExitCode != 0) + throw new Exception( + $"Expected exit code 0 but received ({result.ExitCode}) while executing {description}: {command}"); + + var errorOut = result.ConsoleOut.Where(c => c.Error).ToList(); + + if (errorOut.Any(e => + !string.IsNullOrWhiteSpace(e.Line) && !e.Line.Contains("usage of JAVA_HOME is deprecated")) && + !binary.Contains("plugin") && !binary.Contains("cert")) + throw new Exception( + $"Received error out with exitCode ({result.ExitCode}) while executing {description}: {command}"); + + writer?.WriteDiagnostic( + $"{{{nameof(ExecuteBinary)}}} finished process [{description}] {{{result.ExitCode}}}"); + } + + protected static void CopyFolder(string source, string target, bool overwrite = true) + { + foreach (var sourceDir in Directory.GetDirectories(source, "*", SearchOption.AllDirectories)) + { + var targetDir = sourceDir.Replace(source, target); + Directory.CreateDirectory(targetDir); + } + + foreach (var sourcePath in Directory.GetFiles(source, "*.*", SearchOption.AllDirectories)) + { + var targetPath = sourcePath.Replace(source, target); + if (!overwrite && File.Exists(targetPath)) continue; + File.Copy(sourcePath, targetPath, overwrite); + } + } + + protected static void Extract(string file, string toFolder) + { + if (file.EndsWith(".zip")) ExtractZip(file, toFolder); + else if (file.EndsWith(".tar.gz")) ExtractTarGz(file, toFolder); + else if (file.EndsWith(".tar")) ExtractTar(file, toFolder); + else throw new Exception("Can not extract:" + file); + } + + private static void ExtractTar(string file, string toFolder) + { + using (var inStream = File.OpenRead(file)) + using (var tarArchive = TarArchive.CreateInputTarArchive(inStream)) + tarArchive.ExtractContents(toFolder); + } + + private static void ExtractTarGz(string file, string toFolder) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + using (var inStream = File.OpenRead(file)) + using (var gzipStream = new GZipInputStream(inStream)) + using (var tarArchive = TarArchive.CreateInputTarArchive(gzipStream)) + tarArchive.ExtractContents(toFolder); + else + //SharpZipLib loses permissions when untarring + Proc.Exec("tar", "-xvf", file, "-C", toFolder); + } + + private static void ExtractZip(string file, string toFolder) => + ZipFile.ExtractToDirectory(file, toFolder); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CopyCachedOpenSearchInstallation.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CopyCachedOpenSearchInstallation.cs new file mode 100644 index 0000000000..444e278f87 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CopyCachedOpenSearchInstallation.cs @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class CopyCachedOpenSearchInstallation : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; + + var fs = cluster.FileSystem; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + if (!Directory.Exists(cachedOpenSearchHomeFolder)) return; + + var source = cachedOpenSearchHomeFolder; + var target = fs.OpenSearchHome; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CopyCachedOpenSearchInstallation)}}} using cached OPENSEARCH_HOME {{{source}}} and copying it to [{target}]"); + CopyFolder(source, target); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CreateLocalApplicationDirectory.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CreateLocalApplicationDirectory.cs new file mode 100644 index 0000000000..36f2029592 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/CreateLocalApplicationDirectory.cs @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class CreateLocalApplicationDirectory : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; + if (Directory.Exists(fs.LocalFolder)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateLocalApplicationDirectory)}}} already exists: {{{fs.LocalFolder}}}"); + return; + } + + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(CreateLocalApplicationDirectory)}}} creating {{{fs.LocalFolder}}}"); + + Directory.CreateDirectory(fs.LocalFolder); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs new file mode 100644 index 0000000000..3c7d411bb8 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/DownloadOpenSearchVersion.cs @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.Stack.ArtifactsApi.Products; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class DownloadOpenSearchVersion : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + if (cluster.CachingAndCachedHomeExists()) return; + + var fs = cluster.FileSystem; + var v = cluster.ClusterConfiguration.Version; + var a = cluster.ClusterConfiguration.Artifact; + var from = v.Artifact(Product.OpenSearch).DownloadUrl; + var to = Path.Combine(fs.LocalFolder, a.Archive); + if (File.Exists(to)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} {v} was already downloaded"); + return; + } + + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} downloading OpenSearch [{v}] from {{{from}}} {{{to}}}"); + DownloadFile(from, to); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(DownloadOpenSearchVersion)}}} downloaded OpenSearch [{v}] from {{{from}}} {{{to}}}"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/EnsureJavaHomeEnvironmentVariableIsSet.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/EnsureJavaHomeEnvironmentVariableIsSet.cs new file mode 100644 index 0000000000..b6f6fd4779 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/EnsureJavaHomeEnvironmentVariableIsSet.cs @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class EnsureJavaHomeEnvironmentVariableIsSet : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; + + var envVarName = cluster.ClusterConfiguration.JavaHomeEnvironmentVariable; + var javaHome = Environment.GetEnvironmentVariable(envVarName); + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var jdkFolder = Path.Combine(cachedOpenSearchHomeFolder, "jdk"); + if (Directory.Exists(jdkFolder)) + { + //prefer bundled jdk + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} [{envVarName}] is set to bundled jdk: {{{jdkFolder}}} "); + Environment.SetEnvironmentVariable("JAVA_HOME", jdkFolder); + } + else if (!string.IsNullOrWhiteSpace(javaHome)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} [{envVarName}] is set; clearing value for process to prefer bundled jdk..."); + Environment.SetEnvironmentVariable(envVarName, null); + } + else + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(EnsureJavaHomeEnvironmentVariableIsSet)}}} {envVarName} is not set proceeding or using default JDK"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs new file mode 100644 index 0000000000..8eb238ed6d --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfiguration.cs @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class InitialConfiguration : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + if (cluster.CachingAndCachedHomeExists()) return; + + if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.ElasticSearch) + { + cluster.Writer?.WriteDiagnostic($"{{{nameof(Run)}}} skipping for ElasticSearch"); + return; + } + + var fs = cluster.FileSystem; + var script = Path.Combine(fs.OpenSearchHome, "server-initial-config.sh"); + + if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenSearch) + File.WriteAllText(script, InitialConfigurationOpenSearch.GetConfigurationScript(cluster.ClusterConfiguration.Version)); + if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenDistro) + File.WriteAllText(script, InitialConfigurationOpenDistro.GetConfigurationScript()); + + cluster.Writer?.WriteDiagnostic($"{{{nameof(Run)}}} going to run [server-initial-config.sh]"); + + ExecuteBinary( + cluster.ClusterConfiguration, + cluster.Writer, + "/bin/bash", + "run initial cluster configuration", + script); + + if (!cluster.ClusterConfiguration.EnableSsl) + { + if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenSearch) + File.AppendAllText(Path.Combine(fs.OpenSearchHome, "config", "opensearch.yml"), "plugins.security.disabled: true"); + if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenDistro) + File.AppendAllText(Path.Combine(fs.OpenSearchHome, "config", "elasticsearch.yml"), "opendistro_security.disabled: true"); + } + + if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.ElasticSearch && cluster.ClusterConfiguration.EnableSsl) + throw new NotImplementedException("ElasticSearch with SSL is not supported"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenDistro.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenDistro.cs new file mode 100644 index 0000000000..cad9b6ca7f --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenDistro.cs @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + internal class InitialConfigurationOpenDistro + { + // Actually, it is content of file `opendistro-tar-install.sh` shipped + // in the tarball, but due to (1) this file might be changed and + // (2) we have to modify the file before execution, because it launches + // the server what we want to do on our own, it is decided to have a + // snapshot of this file. + // The script is taken from v.1.13.3, last 3 lines omitted. + public static string GetConfigurationScript() => +@"#!/bin/bash + +ES_HOME=`dirname $(realpath $0)`; cd $ES_HOME +ES_KNN_LIB_DIR=$ES_HOME/plugins/opendistro-knn/knn-lib +##Security Plugin +bash $ES_HOME/plugins/opendistro_security/tools/install_demo_configuration.sh -y -i -s + +##Perf Plugin +chmod 755 $ES_HOME/plugins/opendistro-performance-analyzer/pa_bin/performance-analyzer-agent +chmod 755 $ES_HOME/bin/performance-analyzer-agent-cli +echo ""done security"" +PA_AGENT_JAVA_OPTS=""-Dlog4j.configurationFile=$ES_HOME/plugins/opendistro-performance-analyzer/pa_config/log4j2.xml \ + -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ + -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m"" + +ES_MAIN_CLASS=""com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp"" \ +ES_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opendistro-performance-analyzer \ +ES_JAVA_OPTS=$PA_AGENT_JAVA_OPTS + +if ! grep -q '## OpenDistro Performance Analyzer' $ES_HOME/config/jvm.options; then + CLK_TCK=`/usr/bin/getconf CLK_TCK` + echo >> $ES_HOME/config/jvm.options + echo '## OpenDistro Performance Analyzer' >> $ES_HOME/config/jvm.options + echo ""-Dclk.tck=$CLK_TCK"" >> $ES_HOME/config/jvm.options + echo ""-Djdk.attach.allowAttachSelf=true"" >> $ES_HOME/config/jvm.options + echo ""-Djava.security.policy=$ES_HOME/plugins/opendistro-performance-analyzer/pa_config/es_security.policy"" >> $ES_HOME/config/jvm.options +fi +echo ""done plugins"" + +##Check KNN lib existence in ES TAR distribution +echo ""Checking kNN library"" +FILE=`ls $ES_KNN_LIB_DIR/libKNNIndex*.so` +if test -f ""$FILE""; then + echo ""FILE EXISTS $FILE"" +else + echo ""TEST FAILED OR FILE NOT EXIST $FILE"" +fi + +##Set KNN Dylib Path for macOS and *nix systems +if echo ""$OSTYPE"" | grep -qi ""darwin""; then + if echo ""$JAVA_LIBRARY_PATH"" | grep -q ""$ES_KNN_LIB_DIR""; then + echo ""KNN lib path has been set"" + else + export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$ES_KNN_LIB_DIR + echo ""KNN lib path not found, set new path"" + echo $JAVA_LIBRARY_PATH + fi +else + if echo ""$LD_LIBRARY_PATH"" | grep -q ""$ES_KNN_LIB_DIR""; then + echo ""KNN lib path has been set"" + else + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ES_KNN_LIB_DIR + echo ""KNN lib path not found, set new path"" + echo $LD_LIBRARY_PATH + fi +fi +"; + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenSearch.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenSearch.cs new file mode 100644 index 0000000000..cda04dac13 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InitialConfigurationOpenSearch.cs @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + internal class InitialConfigurationOpenSearch + { + // Actually, it is content of file `opensearch-tar-install.sh` shipped + // in the tarball, but due to (1) this file might be changed and + // (2) we have to modify the file before execution, because it launches + // the server what we want to do on our own, it is decided to have a + // snapshot of this file. + // The script is taken from v.1.2.4, last 3 lines omitted. + public static string GetConfigurationScript(OpenSearchVersion version) + { + if (version < (OpenSearchVersion)"2.0.0") + return +@"#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +OPENSEARCH_HOME=`dirname $(realpath $0)`; cd $OPENSEARCH_HOME +KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/knnlib +##Security Plugin +bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s + +##Perf Plugin +chmod 755 $OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_bin/performance-analyzer-agent +chmod 755 $OPENSEARCH_HOME/bin/performance-analyzer-agent-cli +echo ""done security"" +PA_AGENT_JAVA_OPTS=""-Dlog4j.configurationFile=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/log4j2.xml \ + -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ + -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m"" + +OPENSEARCH_MAIN_CLASS=""org.opensearch.performanceanalyzer.PerformanceAnalyzerApp"" \ +OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opensearch-performance-analyzer \ +OPENSEARCH_JAVA_OPTS=$PA_AGENT_JAVA_OPTS + +if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_HOME/config/jvm.options; then + CLK_TCK=`/usr/bin/getconf CLK_TCK` + echo >> $OPENSEARCH_HOME/config/jvm.options + echo '## OpenSearch Performance Analyzer' >> $OPENSEARCH_HOME/config/jvm.options + echo ""-Dclk.tck=$CLK_TCK"" >> $OPENSEARCH_HOME/config/jvm.options + echo ""-Djdk.attach.allowAttachSelf=true"" >> $OPENSEARCH_HOME/config/jvm.options + echo ""-Djava.security.policy=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy"" >> $OPENSEARCH_HOME/config/jvm.options +fi +echo ""done plugins"" + +##Set KNN Dylib Path for macOS and *nix systems +if echo ""$OSTYPE"" | grep -qi ""darwin""; then + if echo ""$JAVA_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then + echo ""k-NN libraries found in JAVA_LIBRARY_PATH"" + else + export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$KNN_LIB_DIR + echo ""k-NN libraries not found in JAVA_LIBRARY_PATH. Updating path to: $JAVA_LIBRARY_PATH."" + fi +else + if echo ""$LD_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then + echo ""k-NN libraries found in LD_LIBRARY_PATH"" + else + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KNN_LIB_DIR + echo ""k-NN libraries not found in LD_LIBRARY_PATH. Updating path to: $LD_LIBRARY_PATH."" + fi +fi +"; + return + //script from 2.0.0 + @"#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +export OPENSEARCH_HOME=`dirname $(realpath $0)` +export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config +cd $OPENSEARCH_HOME + +KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib +##Security Plugin +bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s + +echo ""done security"" +PA_AGENT_JAVA_OPTS=""-Dlog4j.configurationFile=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer/log4j2.xml \ + -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ + -XX:MaxRAM=400m"" + +OPENSEARCH_MAIN_CLASS=""org.opensearch.performanceanalyzer.PerformanceAnalyzerApp"" \ +OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opensearch-performance-analyzer \ +OPENSEARCH_JAVA_OPTS=$PA_AGENT_JAVA_OPTS + +if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_PATH_CONF/jvm.options; then + CLK_TCK=`/usr/bin/getconf CLK_TCK` + echo >> $OPENSEARCH_PATH_CONF/jvm.options + echo '## OpenSearch Performance Analyzer' >> $OPENSEARCH_PATH_CONF/jvm.options + echo ""-Dclk.tck=$CLK_TCK"" >> $OPENSEARCH_PATH_CONF/jvm.options + echo ""-Djdk.attach.allowAttachSelf=true"" >> $OPENSEARCH_PATH_CONF/jvm.options + echo ""-Djava.security.policy=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer/opensearch_security.policy"" >> $OPENSEARCH_PATH_CONF/jvm.options + echo ""--add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED"" >> $OPENSEARCH_PATH_CONF/jvm.options +fi +echo ""done plugins"" + +##Set KNN Dylib Path for macOS and *nix systems +if echo ""$OSTYPE"" | grep -qi ""darwin""; then + if echo ""$JAVA_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then + echo ""k-NN libraries found in JAVA_LIBRARY_PATH"" + else + export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$KNN_LIB_DIR + echo ""k-NN libraries not found in JAVA_LIBRARY_PATH. Updating path to: $JAVA_LIBRARY_PATH."" + fi +else + if echo ""$LD_LIBRARY_PATH"" | grep -q ""$KNN_LIB_DIR""; then + echo ""k-NN libraries found in LD_LIBRARY_PATH"" + else + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KNN_LIB_DIR + echo ""k-NN libraries not found in LD_LIBRARY_PATH. Updating path to: $LD_LIBRARY_PATH."" + fi +fi +"; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs new file mode 100644 index 0000000000..b8de7d94bf --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/InstallPlugins.cs @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.IO; +using System.Linq; +using OpenSearch.OpenSearch.Managed; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.OpenSearch.Managed.FileSystem; +using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class InstallPlugins : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + if (cluster.CachingAndCachedHomeExists()) return; + + var v = cluster.ClusterConfiguration.Version; + + var fs = cluster.FileSystem; + var requiredPlugins = cluster.ClusterConfiguration.Plugins; + + if (cluster.ClusterConfiguration.ValidatePluginsToInstall) + { + var invalidPlugins = requiredPlugins + .Where(p => !p.IsValid(v)) + .Select(p => p.SubProductName).ToList(); + if (invalidPlugins.Any()) + throw new OpenSearchCleanExitException( + $"Can not install the following plugins for version {v}: {string.Join(", ", invalidPlugins)} "); + } + + foreach (var plugin in requiredPlugins) + { + var includedByDefault = plugin.IsIncludedOutOfTheBox(v); + if (includedByDefault) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(Run)}}} SKIP plugin [{plugin.SubProductName}] shipped OOTB as of: {{{plugin.ShippedByDefaultAsOf}}}"); + continue; + } + + var validForCurrentVersion = plugin.IsValid(v); + if (!validForCurrentVersion) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(Run)}}} SKIP plugin [{plugin.SubProductName}] not valid for version: {{{v}}}"); + continue; + } + + var alreadyInstalled = AlreadyInstalled(fs, plugin.SubProductName); + if (alreadyInstalled) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(Run)}}} SKIP plugin [{plugin.SubProductName}] already installed"); + continue; + } + + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(Run)}}} attempting install [{plugin.SubProductName}] as it's not OOTB: {{{plugin.ShippedByDefaultAsOf}}} and valid for {v}: {{{plugin.IsValid(v)}}}"); + + if (!Directory.Exists(fs.ConfigPath)) Directory.CreateDirectory(fs.ConfigPath); + ExecuteBinary( + cluster.ClusterConfiguration, + cluster.Writer, + fs.PluginBinary, + $"install opensearch plugin: {plugin.SubProductName}", + "install --batch", GetPluginLocation(plugin, v)); + + CopyConfigDirectoryToHomeCacheConfigDirectory(cluster, plugin); + } + } + + private static string GetPluginLocation(OpenSearchPlugin plugin, OpenSearchVersion v) + { + // OpenSearch 1.0.0 artifacts were not published. The plugins are built in the workflow and used here. + if (v == "1.0.0") + // The environment variable is set in the integration workflow in + // https://github.com/opensearch-project/opensearch-net/blob/main/.github/workflows/integration.yml + return "file://" + Environment.GetEnvironmentVariable("plugins-directory") + $"/{plugin.SubProductName}-{v}.zip"; + else + return plugin.SubProductName; + } + + private static void CopyConfigDirectoryToHomeCacheConfigDirectory( + IEphemeralCluster cluster, OpenSearchPlugin plugin) + { + if (!cluster.ClusterConfiguration.CacheOpenSearchHomeInstallation) return; + var fs = cluster.FileSystem; + var cachedOpenSearchHomeFolder = Path.Combine(fs.LocalFolder, cluster.GetCacheFolderName()); + var configTarget = Path.Combine(cachedOpenSearchHomeFolder, "config"); + + var configPluginPath = Path.Combine(fs.ConfigPath, plugin.SubProductName); + var configPluginPathCached = Path.Combine(configTarget, plugin.SubProductName); + if (!Directory.Exists(configPluginPath) || Directory.Exists(configPluginPathCached)) return; + + Directory.CreateDirectory(configPluginPathCached); + CopyFolder(configPluginPath, configPluginPathCached); + } + + private static bool AlreadyInstalled(INodeFileSystem fileSystem, string folderName) + { + var pluginFolder = Path.Combine(fileSystem.OpenSearchHome, "plugins", folderName); + return Directory.Exists(pluginFolder); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/PrintConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/PrintConfiguration.cs new file mode 100644 index 0000000000..7a4caee531 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/PrintConfiguration.cs @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Linq; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using static OpenSearch.OpenSearch.Ephemeral.ClusterFeatures; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class PrintConfiguration : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + var c = cluster.ClusterConfiguration; + var version = c.Version; + + string F(ClusterFeatures feature) + { + return c.Features.HasFlag(feature) ? Enum.GetName(typeof(ClusterFeatures), feature) : string.Empty; + } + + var features = string.Join("|", + new[] { F(SSL)}.Where(v => !string.IsNullOrWhiteSpace(v))); + features = string.IsNullOrWhiteSpace(features) ? "None" : features; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} starting {{{version}}} with features [{features}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NumberOfNodes)}}} [{c.NumberOfNodes}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ClusterName)}}} [{c.ClusterName}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.EnableSsl)}}} [{c.EnableSsl}]"); + cluster.Writer?.WriteDiagnostic($"{{{nameof(PrintConfiguration)}}} {{{nameof(c.Plugins)}}} [{c.Plugins}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.CacheOpenSearchHomeInstallation)}}} [{c.CacheOpenSearchHomeInstallation}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ShowOpenSearchOutputAfterStarted)}}} [{c.ShowOpenSearchOutputAfterStarted}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.ValidatePluginsToInstall)}}} [{c.ValidatePluginsToInstall}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.PrintYamlFilesInConfigFolder)}}} [{c.PrintYamlFilesInConfigFolder}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.SkipBuiltInAfterStartTasks)}}} [{c.SkipBuiltInAfterStartTasks}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.HttpFiddlerAware)}}} [{c.HttpFiddlerAware}]"); + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(PrintConfiguration)}}} {{{nameof(c.NoCleanupAfterNodeStopped)}}} [{c.NoCleanupAfterNodeStopped}]"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/SetOpenSearchBundledJdkJavaHome.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/SetOpenSearchBundledJdkJavaHome.cs new file mode 100644 index 0000000000..673dea2801 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/SetOpenSearchBundledJdkJavaHome.cs @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class SetOpenSearchBundledJdkJavaHome : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + var fs = cluster.FileSystem; + var jdkFolder = Path.Combine(fs.OpenSearchHome, "jdk"); + if (Directory.Exists(jdkFolder)) + { + var envVarName = cluster.ClusterConfiguration.JavaHomeEnvironmentVariable; + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(SetOpenSearchBundledJdkJavaHome)}}} [{envVarName}] is set to bundled jdk: {{{jdkFolder}}} "); + Environment.SetEnvironmentVariable(envVarName, jdkFolder); + } + else + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(SetOpenSearchBundledJdkJavaHome)}}} [No bundled jdk found] looked in: {{{jdkFolder}}} "); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/UnzipOpenSearch.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/UnzipOpenSearch.cs new file mode 100644 index 0000000000..8854d061b9 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/InstallationTasks/UnzipOpenSearch.cs @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.IO; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks +{ + public class UnzipOpenSearch : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + if (cluster.CachingAndCachedHomeExists()) return; + + var fs = cluster.FileSystem; + var v = cluster.ClusterConfiguration.Version; + var a = cluster.ClusterConfiguration.Artifact; + if (Directory.Exists(fs.OpenSearchHome)) + { + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(UnzipOpenSearch)}}} skipping [{fs.OpenSearchHome}] already exists"); + return; + } + + var from = Path.Combine(fs.LocalFolder, a.Archive); + var extractedFolder = Path.Combine(fs.LocalFolder, a.FolderInZip); + if (!Directory.Exists(extractedFolder)) + { + cluster.Writer?.WriteDiagnostic($"{{{nameof(UnzipOpenSearch)}}} unzipping version [{v}] {{{from}}}"); + Extract(from, fs.LocalFolder); + + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(UnzipOpenSearch)}}} extracted version [{v}] to {{{fs.LocalFolder}}}"); + } + + if (extractedFolder == fs.OpenSearchHome) return; + + cluster.Writer?.WriteDiagnostic( + $"{{{nameof(UnzipOpenSearch)}}} Copying extracted folder {{{extractedFolder}}} => {fs.OpenSearchHome}"); + CopyFolder(extractedFolder, fs.OpenSearchHome); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateClusterStateTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateClusterStateTask.cs new file mode 100644 index 0000000000..6b98b880bc --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateClusterStateTask.cs @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Net; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks +{ + public class ValidateClusterStateTask : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + cluster.Writer.WriteDiagnostic( + $"{{{nameof(ValidateClusterStateTask)}}} waiting cluster to go into yellow health state"); + var healthyResponse = Get(cluster, "_cluster/health", "wait_for_status=yellow&timeout=20s"); + if (healthyResponse == null || healthyResponse.StatusCode != HttpStatusCode.OK) + throw new Exception( + $"Cluster health waiting for status yellow failed after 20s {GetResponseException(healthyResponse)}"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidatePluginsTask.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidatePluginsTask.cs new file mode 100644 index 0000000000..20207f7450 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidatePluginsTask.cs @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Linq; +using OpenSearch.OpenSearch.Ephemeral.Tasks.InstallationTasks; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks +{ + public class ValidatePluginsTask : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + var v = cluster.ClusterConfiguration.Version; + var requestPlugins = cluster.ClusterConfiguration.Plugins + .Where(p => p.IsValid(v)) + .Where(p => !p.IsIncludedOutOfTheBox(v)) + .Select(p => p.GetExistsMoniker(v)) + .ToList(); + if (!requestPlugins.Any()) return; + + cluster.Writer.WriteDiagnostic( + $"{{{nameof(ValidatePluginsTask)}}} validating the cluster is running the requested plugins"); + var catPlugins = Get(cluster, "_cat/plugins", "h=component"); + if (catPlugins == null || !catPlugins.IsSuccessStatusCode) + throw new Exception( + $"Calling _cat/plugins did not result in an OK response {GetResponseException(catPlugins)}"); + + var installedPlugins = GetResponseString(catPlugins) + .Split(new[] {'\n'}, StringSplitOptions.RemoveEmptyEntries).ToList(); + + var missingPlugins = requestPlugins.Except(installedPlugins).ToList(); + if (!missingPlugins.Any()) return; + + var missingString = string.Join(", ", missingPlugins); + var pluginsString = string.Join(", ", installedPlugins); + throw new Exception( + $"Already running opensearch missed the following plugin(s): {missingString} currently installed: {pluginsString}."); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs new file mode 100644 index 0000000000..f0f2da1377 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Ephemeral/Tasks/ValidationTasks/ValidateRunningVersion.cs @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks +{ + public class ValidateRunningVersion : ClusterComposeTask + { + public override void Run(IEphemeralCluster cluster) + { + void WriteDiagnostic(string message) => + cluster.Writer?.WriteDiagnostic($"{{{nameof(ValidateRunningVersion)}}} {message}"); + var requestedVersion = cluster.ClusterConfiguration.Version; + if (cluster.ClusterConfiguration.Artifact.ServerType == ServerType.OpenDistro) + //All supported version of OpenDistro are based on ElasticSearch v.7.10.2 + requestedVersion = OpenSearchVersion.From("7.10.2"); + + + WriteDiagnostic($"validating the cluster is running the requested version: {requestedVersion}"); + + HttpResponseMessage catNodes = null; + var retryCount = 4; + var initialRetryWait = 5; + foreach (var retry in Enumerable.Range(1, retryCount)) + { + catNodes = Get(cluster, "_cat/nodes", "h=version"); + if (catNodes.IsSuccessStatusCode) break; + var retryWait = TimeSpan.FromSeconds(initialRetryWait * retry); + WriteDiagnostic($"{catNodes.StatusCode} response for GET _cat/nodes. Waiting to retry #{retry}"); + Thread.Sleep(retryWait); + } + if (catNodes is not {IsSuccessStatusCode: true}) + { + throw new Exception( + $"Calling _cat/nodes for version checking did not result in an OK response {GetResponseException(catNodes)}"); + } + + var nodeVersions = GetResponseString(catNodes).Split(new[] {'\n'}, StringSplitOptions.RemoveEmptyEntries) + .ToList(); + + var anchorVersion = $"{requestedVersion.Major}.{requestedVersion.Minor}.{requestedVersion.Patch}"; + var allOnRequestedVersion = nodeVersions.All(v => v.Trim() == anchorVersion); + if (!allOnRequestedVersion) + { + throw new Exception( + $"Not all the running nodes in the cluster are on requested version: {anchorVersion} received: {string.Join(", ", nodeVersions)}"); + } + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.EphemeralTests/EphemeralClusterTests.cs b/abstractions/src/OpenSearch.OpenSearch.EphemeralTests/EphemeralClusterTests.cs new file mode 100644 index 0000000000..1e9e88b450 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.EphemeralTests/EphemeralClusterTests.cs @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Collections; +using System.Collections.Generic; +using OpenSearch.OpenSearch.Ephemeral; +using OpenSearch.Stack.ArtifactsApi; +using Xunit; + +namespace OpenSearch.OpenSearch.EphemeralTests +{ + public class EphemeralClusterTests + { + [Theory] + [ClassData(typeof(SampleClusters))] + public void TestEphemeralCluster(OpenSearchVersion version, ServerType serverType, ClusterFeatures features) + { + var numberOfNodes = 1; + var clusterConfiguration = + new EphemeralClusterConfiguration(version, serverType, features, null, numberOfNodes) + { + ShowOpenSearchOutputAfterStarted = true + }; + using var cluster = new EphemeralCluster(clusterConfiguration); + var timeout = new System.TimeSpan(0, 5, 30); + using (cluster.Start(timeout)) + { + Assert.True(cluster.Started, "OpenSearch cluster started"); + foreach (var n in cluster.Nodes) + { + n.SendControlC(); + Assert.True(n.WaitForCompletion(timeout), $"Failed to stop node {n.ProcessId}"); + } + } + } + + private class SampleClusters : IEnumerable + { + public IEnumerator GetEnumerator() + { + yield return new object[] {OpenSearchVersion.From("1.2.0"), ServerType.OpenSearch, ClusterFeatures.None}; + yield return new object[] + { + OpenSearchVersion.From("opendistro-latest"), ServerType.OpenDistro, ClusterFeatures.None + }; + yield return new object[] {OpenSearchVersion.From("1.2.0"), ServerType.OpenSearch, ClusterFeatures.SSL}; + yield return new object[] + { + OpenSearchVersion.From("opendistro-latest"), ServerType.OpenDistro, ClusterFeatures.SSL + }; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.EphemeralTests/OpenSearch.OpenSearch.EphemeralTests.csproj b/abstractions/src/OpenSearch.OpenSearch.EphemeralTests/OpenSearch.OpenSearch.EphemeralTests.csproj new file mode 100644 index 0000000000..d14069227a --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.EphemeralTests/OpenSearch.OpenSearch.EphemeralTests.csproj @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + false + net5.0 + false + + + diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs new file mode 100644 index 0000000000..5cf1fe92e7 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ClusterBase.cs @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Threading; +using OpenSearch.OpenSearch.Managed.Configuration; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.OpenSearch.Managed.FileSystem; +using OpenSearch.Stack.ArtifactsApi; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Managed +{ + public interface ICluster : IDisposable + where TConfiguration : IClusterConfiguration + { + string ClusterMoniker { get; } + TConfiguration ClusterConfiguration { get; } + INodeFileSystem FileSystem { get; } + bool Started { get; } + ReadOnlyCollection Nodes { get; } + IConsoleLineHandler Writer { get; } + + IDisposable Start(); + + IDisposable Start(TimeSpan waitForStarted); + + IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted); + } + + + public abstract class ClusterBase : ClusterBase + { + protected ClusterBase(ClusterConfiguration clusterConfiguration) : base(clusterConfiguration) + { + } + } + + public abstract class ClusterBase : ICluster + where TConfiguration : IClusterConfiguration + { + private Action _defaultConfigSelector = (n, i) => { }; + + protected ClusterBase(TConfiguration clusterConfiguration) + { + ClusterConfiguration = clusterConfiguration; + ClusterMoniker = GetType().Name.Replace("Cluster", ""); + + NodeConfiguration Modify(NodeConfiguration n, int p) + { + ModifyNodeConfiguration(n, p); + return n; + } + + var nodes = + (from port in Enumerable.Range(ClusterConfiguration.StartingPortNumber, + ClusterConfiguration.NumberOfNodes) + let config = new NodeConfiguration(clusterConfiguration, port, ClusterMoniker) + { + ShowOpenSearchOutputAfterStarted = + clusterConfiguration.ShowOpenSearchOutputAfterStarted, + } + let node = new OpenSearchNode(Modify(config, port)) + { + AssumeStartedOnNotEnoughMasterPing = ClusterConfiguration.NumberOfNodes > 1, + } + select node).ToList(); + + var initialMasterNodes = string.Join(",", nodes.Select(n => n.NodeConfiguration.DesiredNodeName)); + foreach (var node in nodes) + node.NodeConfiguration.InitialMasterNodes(initialMasterNodes); + + Nodes = new ReadOnlyCollection(nodes); + } + + /// + /// A short name to identify the cluster defaults to the subclass name with Cluster + /// removed + /// + public virtual string ClusterMoniker { get; } + + public TConfiguration ClusterConfiguration { get; } + public INodeFileSystem FileSystem => ClusterConfiguration.FileSystem; + + public ReadOnlyCollection Nodes { get; } + public bool Started { get; private set; } + public IConsoleLineHandler Writer { get; private set; } = NoopConsoleLineWriter.Instance; + + public IDisposable Start() => Start(TimeSpan.FromMinutes(2)); + + public IDisposable Start(TimeSpan waitForStarted) + { + var nodes = Nodes.Select(n => n.NodeConfiguration.DesiredNodeName).ToArray(); + var lineHighlightWriter = new LineHighlightWriter(nodes, LineOutParser.From(ClusterConfiguration.ServerType)); + return Start(lineHighlightWriter, waitForStarted); + } + + public IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted) + { + Writer = writer ?? NoopConsoleLineWriter.Instance; + + OnBeforeStart(); + + var subscriptions = new Subscriptions(); + foreach (var node in Nodes) subscriptions.Add(node.SubscribeLines(writer)); + + var waitHandles = Nodes.Select(w => w.StartedHandle).ToArray(); + if (!WaitHandle.WaitAll(waitHandles, waitForStarted)) + { + var nodeExceptions = Nodes.Select(n => n.LastSeenException).Where(e => e != null).ToList(); + writer?.WriteError( + $"{{{GetType().Name}.{nameof(Start)}}} cluster did not start after {waitForStarted}"); + throw new AggregateException($"Not all nodes started after waiting {waitForStarted}", nodeExceptions); + } + + Started = Nodes.All(n => n.NodeStarted); + if (!Started) + { + var nodeExceptions = Nodes.Select(n => n.LastSeenException).Where(e => e != null).ToList(); + var message = $"{{{GetType().Name}.{nameof(Start)}}} cluster did not start successfully"; + var seeLogsMessage = SeeLogsMessage(message); + writer?.WriteError(seeLogsMessage); + throw new AggregateException(seeLogsMessage, nodeExceptions); + } + + try + { + OnAfterStarted(); + SeedCluster(); + } + catch (Exception e) + { + writer?.WriteError(e.ToString()); + throw; + } + + return subscriptions; + } + + public void Dispose() + { + Started = false; + foreach (var node in Nodes) + node?.Dispose(); + + OnDispose(); + } + + protected virtual void ModifyNodeConfiguration(NodeConfiguration nodeConfiguration, int port) + { + } + + protected virtual void SeedCluster() + { + } + + + protected virtual string SeeLogsMessage(string message) + { + var log = Path.Combine(FileSystem.LogsPath, $"{ClusterConfiguration.ClusterName}.log"); + return $"{message} see {log} to diagnose the issue"; + } + + public void WaitForExit(TimeSpan waitForCompletion) + { + foreach (var node in Nodes) + node.WaitForCompletion(waitForCompletion); + } + + protected virtual void OnAfterStarted() + { + } + + protected virtual void OnBeforeStart() + { + } + + protected virtual void OnDispose() + { + } + + private class Subscriptions : IDisposable + { + private List Disposables { get; } = new List(); + + public void Dispose() + { + foreach (var d in Disposables) d.Dispose(); + } + + internal void Add(IDisposable disposable) => Disposables.Add(disposable); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs new file mode 100644 index 0000000000..d64ba11ff1 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/ClusterConfiguration.cs @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Globalization; +using System.IO; +using OpenSearch.OpenSearch.Managed.FileSystem; +using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; + +namespace OpenSearch.OpenSearch.Managed.Configuration +{ + public interface IClusterConfiguration where TFileSystem : INodeFileSystem + { + TFileSystem FileSystem { get; } + + string ClusterName { get; } + NodeSettings DefaultNodeSettings { get; } + OpenSearchVersion Version { get; } + ServerType ServerType { get; } + int NumberOfNodes { get; } + int StartingPortNumber { get; set; } + bool NoCleanupAfterNodeStopped { get; set; } + + bool ShowOpenSearchOutputAfterStarted { get; set; } + bool CacheOpenSearchHomeInstallation { get; set; } + + string CreateNodeName(int? node); + } + + public class ClusterConfiguration : ClusterConfiguration + { + public ClusterConfiguration(OpenSearchVersion version, ServerType serverType, string openSearchHome, int numberOfNodes = 1) + : base(version, serverType, (v, s) => new NodeFileSystem(v, openSearchHome), numberOfNodes, null) + { + } + + public ClusterConfiguration(OpenSearchVersion version, ServerType serverType, + Func fileSystem = null, int numberOfNodes = 1, + string clusterName = null) + : base(version, serverType, fileSystem ?? ((v, s) => new NodeFileSystem(v, s)), numberOfNodes, clusterName) + { + } + } + + public class ClusterConfiguration : IClusterConfiguration + where TFileSystem : INodeFileSystem + { + /// + /// Creates a new instance of a configuration for an OpenSearch cluster. + /// + /// The version of OpenSearch + /// + /// A delegate to create the instance of . + /// Passed the OpenSearch version and the Cluster name + /// + /// The number of nodes in the cluster + /// The name of the cluster + public ClusterConfiguration(OpenSearchVersion version, ServerType serverType, Func fileSystem, + int numberOfNodes = 1, string clusterName = null) + { + if (fileSystem == null) throw new ArgumentException(nameof(fileSystem)); + + ClusterName = clusterName; + Version = version; + ServerType = serverType; + Artifact = version.Artifact(Product.From("opensearch", serverType: serverType)); + FileSystem = fileSystem(Version, ClusterName); + NumberOfNodes = numberOfNodes; + + var fs = FileSystem; + Add("node.max_local_storage_nodes", numberOfNodes.ToString(CultureInfo.InvariantCulture), "1.0.0"); + + Add("cluster.name", clusterName); + Add("path.repo", fs.RepositoryPath); + Add("path.data", fs.DataPath); + var logsPathDefault = Path.Combine(fs.OpenSearchHome, "logs"); + if (logsPathDefault != fs.LogsPath) Add("path.logs", fs.LogsPath); + } + + public Artifact Artifact { get; } + + public string JavaHomeEnvironmentVariable => "JAVA_HOME"; + + /// Will print the contents of all the yaml files when starting the cluster up, great for debugging purposes + public bool PrintYamlFilesInConfigFolder { get; set; } + + public string ClusterName { get; } + public OpenSearchVersion Version { get; } + public ServerType ServerType { get; } + public TFileSystem FileSystem { get; } + public int NumberOfNodes { get; } + public int StartingPortNumber { get; set; } = 9200; + public bool NoCleanupAfterNodeStopped { get; set; } + + /// + /// Whether should continue to write output to console after it has started. + /// Defaults to true + /// + public bool ShowOpenSearchOutputAfterStarted { get; set; } = true; + + public bool CacheOpenSearchHomeInstallation { get; set; } + + /// The node settings to apply to each started node + public NodeSettings DefaultNodeSettings { get; } = new NodeSettings(); + + /// + /// Creates a node name + /// + public virtual string CreateNodeName(int? node) => + node.HasValue ? $"managed-opensearch-{node}" : " managed-opensearch"; + + /// + /// Calculates the quorum given the number of instances + /// + private static int Quorum(int instanceCount) => Math.Max(1, (int) Math.Floor((double) instanceCount / 2) + 1); + + /// + /// Creates a node attribute for the version of OpenSearch + /// + public string AttributeKey(string attribute) + { + var attr = "attr."; + return $"node.{attr}{attribute}"; + } + + /// + /// Adds a node setting to the default node settings + /// + protected void Add(string key, string value) + { + if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return; + DefaultNodeSettings.Add(key, value); + } + + /// + /// Adds a node setting to the default node settings only if the OpenSearch + /// version is in the range. + /// + protected void Add(string key, string value, string range) + { + if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) return; + if (string.IsNullOrWhiteSpace(range) || Version.InRange(range)) + DefaultNodeSettings.Add(key, value, range); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs new file mode 100644 index 0000000000..43edce5dda --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeConfiguration.cs @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Globalization; +using OpenSearch.OpenSearch.Managed.FileSystem; +using OpenSearch.Stack.ArtifactsApi; +using ProcNet; + +namespace OpenSearch.OpenSearch.Managed.Configuration +{ + public class NodeConfiguration + { + private Action _defaultStartArgs = s => { }; + + public NodeConfiguration(OpenSearchVersion version, int? port = null) : this(new ClusterConfiguration(version, ServerType.DEFAULT), + port) + { + } + + public NodeConfiguration(IClusterConfiguration clusterConfiguration, int? port = null, + string nodePrefix = null) + { + ClusterConfiguration = clusterConfiguration; + DesiredPort = port; + DesiredNodeName = CreateNodeName(port, nodePrefix) ?? clusterConfiguration.CreateNodeName(port); + Settings = new NodeSettings(clusterConfiguration.DefaultNodeSettings); + + if (!string.IsNullOrWhiteSpace(DesiredNodeName)) Settings.Add("node.name", DesiredNodeName); + if (DesiredPort.HasValue) + Settings.Add("http.port", DesiredPort.Value.ToString(CultureInfo.InvariantCulture)); + } + + private IClusterConfiguration ClusterConfiguration { get; } + + public int? DesiredPort { get; } + public string DesiredNodeName { get; } + + public Action ModifyStartArguments + { + get => _defaultStartArgs; + set => _defaultStartArgs = value ?? (s => { }); + } + + /// + /// Wheter should continue to write output to console after it has started. + /// Defaults to true but useful to turn of if it proofs to be too noisy + /// + public bool ShowOpenSearchOutputAfterStarted { get; set; } = true; + + /// + /// The expected duration of the shut down sequence for OpenSearch. After this wait duration a hard kill will occur. + /// + public TimeSpan WaitForShutdown { get; set; } = TimeSpan.FromSeconds(10); + + /// + /// Copy of . Add individual node settings here. + /// + public NodeSettings Settings { get; } + + public INodeFileSystem FileSystem => ClusterConfiguration.FileSystem; + public OpenSearchVersion Version => ClusterConfiguration.Version; + public ServerType ServerType => ClusterConfiguration.ServerType; + public string[] CommandLineArguments => Settings.ToCommandLineArguments(Version); + + public void InitialMasterNodes(string initialMasterNodes) => + Settings.Add("cluster.initial_master_nodes", initialMasterNodes, ">=1.0.0"); + + public string AttributeKey(string attribute) + { + var attr = "attr."; + return $"node.{attr}{attribute}"; + } + + public void Add(string key, string value) => Settings.Add(key, value); + + private string CreateNodeName(int? node, string prefix = null) + { + if (prefix == null) return null; + var suffix = Guid.NewGuid().ToString("N").Substring(0, 6); + return $"{prefix.Replace("Cluster", "").ToLowerInvariant()}-node-{suffix}{node}"; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSetting.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSetting.cs new file mode 100644 index 0000000000..23d1d28f4b --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSetting.cs @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +namespace OpenSearch.OpenSearch.Managed.Configuration +{ + public struct NodeSetting + { + public string Key { get; } + public string Value { get; } + + /// + /// Stores for which opensearch version range this setting is applicable + /// + public string VersionRange { get; } + + public NodeSetting(string key, string value, string range) + { + Key = key; + Value = value; + VersionRange = range; + } + + public override string ToString() => $"{Key}={Value}"; + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSettings.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSettings.cs new file mode 100644 index 0000000000..7d68b41f96 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/Configuration/NodeSettings.cs @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Managed.Configuration +{ + public class NodeSettings : List + { + public NodeSettings() + { + } + + public NodeSettings(NodeSettings settings) : base(settings) + { + } + + public void Add(string setting) + { + var s = setting.Split(new[] {'='}, 2, StringSplitOptions.RemoveEmptyEntries); + if (s.Length != 2) + throw new ArgumentException($"Can only add node settings in key=value from but received: {setting}"); + Add(new NodeSetting(s[0], s[1], null)); + } + + public void Add(string key, string value) => Add(new NodeSetting(key, value, null)); + + public void Add(string key, string value, string versionRange) => + Add(new NodeSetting(key, value, versionRange)); + + public string[] ToCommandLineArguments(OpenSearchVersion version) + { + var settingArgument = "-E "; + return this + //if a node setting is only applicable for a certain version make sure its filtered out + .Where(s => string.IsNullOrEmpty(s.VersionRange) || version.InRange(s.VersionRange)) + //allow additional settings to take precedence over already DefaultNodeSettings + //without relying on opensearch to dedup + .GroupBy(setting => setting.Key) + .Select(g => g.Last()) + .Select(s => s.Key.StartsWith(settingArgument) ? s.ToString() : $"{settingArgument}{s}") + .ToArray(); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ConsoleLineWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ConsoleLineWriter.cs new file mode 100644 index 0000000000..097428ce69 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ConsoleLineWriter.cs @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +{ + public class ConsoleLineWriter : IConsoleLineHandler + { + public void Handle(LineOut lineOut) + { + var w = lineOut.Error ? Console.Error : Console.Out; + w.WriteLine(lineOut); + } + + public void Handle(Exception e) => Console.Error.WriteLine(e); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ExceptionLineParser.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ExceptionLineParser.cs new file mode 100644 index 0000000000..9943b1f26d --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/ExceptionLineParser.cs @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Text.RegularExpressions; + +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +{ + public static class ExceptionLineParser + { + private static readonly Regex CauseRegex = new Regex(@"^(?.*?Exception:)(?.*?)$"); + + private static readonly Regex + LocRegex = new Regex(@"^(?\s*?at )(?.*?)\((?.*?)\)(?.*?)$"); + + public static bool TryParseCause(string line, out string cause, out string message) + { + cause = message = null; + if (string.IsNullOrEmpty(line)) return false; + var match = CauseRegex.Match(line); + if (!match.Success) return false; + cause = match.Groups["cause"].Value.Trim(); + message = match.Groups["message"].Value.Trim(); + return true; + } + + public static bool TryParseStackTrace(string line, out string at, out string method, out string file, + out string jar) + { + at = method = file = jar = null; + if (string.IsNullOrEmpty(line)) return false; + var match = LocRegex.Match(line); + if (!match.Success) return false; + at = match.Groups["at"].Value; + method = match.Groups["method"].Value.Trim(); + file = match.Groups["file"].Value.Trim(); + jar = match.Groups["jar"].Value.Trim(); + return true; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/IConsoleLineWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/IConsoleLineWriter.cs new file mode 100644 index 0000000000..480a7438aa --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/IConsoleLineWriter.cs @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +{ + public static class LineWriterExtensions + { + public static void WriteDiagnostic(this IConsoleLineHandler writer, string message) => + writer.Handle(Info(message)); + + public static void WriteDiagnostic(this IConsoleLineHandler writer, string message, string node) => + writer?.Handle(Info(node != null ? $"[{node}] {message}" : message)); + + public static void WriteError(this IConsoleLineHandler writer, string message) => writer.Handle(Error(message)); + + public static void WriteError(this IConsoleLineHandler writer, string message, string node) => + writer?.Handle(Error(node != null ? $"[{node}] {message}" : message)); + + private static string Format(bool error, string message) => + $"[{DateTime.UtcNow:yyyy-MM-ddThh:mm:ss,fff}][{(error ? "ERROR" : "INFO ")}][Managed OpenSearch\t] {message}"; + + private static LineOut Info(string message) => ConsoleOut.Out(Format(false, message)); + private static LineOut Error(string message) => ConsoleOut.ErrorOut(Format(true, message)); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineHighlightWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineHighlightWriter.cs new file mode 100644 index 0000000000..2ed2539370 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineHighlightWriter.cs @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +{ + public class LineHighlightWriter : IConsoleLineHandler + { + private readonly LineOutParser _lineOutParser; + + private static readonly ConsoleColor[] AvailableNodeColors = + { + ConsoleColor.DarkGreen, ConsoleColor.DarkBlue, ConsoleColor.DarkRed, ConsoleColor.DarkCyan, + ConsoleColor.DarkYellow, ConsoleColor.Blue, + }; + + private static readonly object Lock = new object(); + + private static readonly char[] Anchors = {'[', ']', '{', '}'}; + + private readonly Dictionary _nodeColors; + + public LineHighlightWriter() + { + } + + public LineHighlightWriter(IList nodes, LineOutParser lineOutParser) + { + _lineOutParser = lineOutParser ?? throw new NullReferenceException(nameof(lineOutParser)); + if (nodes == null) throw new NullReferenceException(nameof(nodes)); + + var colors = new Dictionary(); + for (var i = 0; i < nodes.Count; i++) + { + var color = i % AvailableNodeColors.Length; + colors.Add(nodes[i], AvailableNodeColors[color]); + } + + _nodeColors = colors; + } + + public void Handle(Exception e) + { + lock (Lock) + { + Console.BackgroundColor = ConsoleColor.Red; + Console.ForegroundColor = ConsoleColor.White; + Console.Error.WriteLine(e); + Console.ResetColor(); + } + } + + public void Handle(LineOut lineOut) + { + var parsed = _lineOutParser.TryParse(lineOut.Line, out var date, out var level, out var section, + out var node, out var message, out _); + if (parsed) Write(lineOut.Error, date, level, section, node, message, NodeColor); + else if (ExceptionLineParser.TryParseCause(lineOut.Line, out var cause, out var causeMessage)) + WriteCausedBy(lineOut.Error, cause, causeMessage); + + else if (ExceptionLineParser.TryParseStackTrace(lineOut.Line, out var at, out var method, out var file, + out var jar)) + WriteStackTraceLine(lineOut.Error, at, method, file, jar); + + else Write(lineOut.Error, lineOut.Line); + } + + private ConsoleColor NodeColor(string node) => + _nodeColors != null && _nodeColors.TryGetValue(node, out var color) ? color : AvailableNodeColors[0]; + + private static void Write(bool error, string date, string level, string section, string node, string message, + Func getNodeColor = null) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + WriteBlock(w, ConsoleColor.DarkGray, date); + WriteBlock(w, LevelColor(level), level, 5); + + if (!string.IsNullOrWhiteSpace(section)) + { + WriteBlock(w, ConsoleColor.DarkCyan, section, 25); + WriteSpace(w); + } + + WriteBlock(w, getNodeColor?.Invoke(node) ?? ConsoleColor.DarkGreen, node); + WriteSpace(w); + + var messageColor = error || level == "ERROR" ? ConsoleColor.Red : ConsoleColor.White; + WriteMessage(w, messageColor, message); + + Console.ResetColor(); + w.Flush(); + } + } + + private static void WriteCausedBy(bool error, string cause, string causeMessage) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + Write(w, ConsoleColor.DarkRed, cause); + WriteSpace(w); + Write(w, ConsoleColor.Red, causeMessage); + w.WriteLine(); + Console.ResetColor(); + w.Flush(); + } + } + + private static void WriteStackTraceLine(bool error, string at, string method, string file, string jar) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + Write(w, ConsoleColor.DarkGray, at); + Write(w, ConsoleColor.DarkBlue, method); + Write(w, ConsoleColor.DarkGray, "("); + Write(w, ConsoleColor.Blue, file); + Write(w, ConsoleColor.DarkGray, ")"); + WriteSpace(w); + Write(w, ConsoleColor.Gray, jar); + w.WriteLine(); + + Console.ResetColor(); + w.Flush(); + } + } + + private static void Write(bool error, string message) + { + lock (Lock) + { + var w = error ? Console.Error : Console.Out; + var messageColor = error ? ConsoleColor.Red : ConsoleColor.White; + WriteMessage(w, messageColor, message); + Console.ResetColor(); + w.Flush(); + } + } + + private static ConsoleColor LevelColor(string level) + { + switch (level ?? "") + { + case "WARN": return ConsoleColor.Yellow; + case "FATAL": + case "ERROR": + return ConsoleColor.Red; + case "DEBUG": + case "TRACE": + return ConsoleColor.DarkGray; + default: + return ConsoleColor.Cyan; + } + } + + private static IEnumerable Parts(string s) + { + int start = 0, index; + while ((index = s.IndexOfAny(Anchors, start)) != -1) + { + if (index - start > 0) + yield return s.Substring(start, index - start); + + yield return s.Substring(index, 1); + start = index + 1; + } + + if (start < s.Length) + yield return s.Substring(start); + } + + private static void WriteMessage(TextWriter w, ConsoleColor color, string message) + { + var insideSquareBracket = 0; + var insideCurlyBracket = 0; + foreach (var p in Parts(message)) + { + if (p.Length == 0) continue; + if (p[0] == '[') insideSquareBracket++; + else if (p[0] == ']') insideSquareBracket--; + else if (p[0] == '{') insideCurlyBracket++; + else if (p[0] == '}') insideCurlyBracket--; + + if (Anchors.Contains(p[0])) Console.ForegroundColor = ConsoleColor.DarkGray; + else if (insideSquareBracket > 0) Console.ForegroundColor = ConsoleColor.Yellow; + else if (insideCurlyBracket > 0) Console.ForegroundColor = ConsoleColor.Blue; + else Console.ForegroundColor = color; + + w.Write(p); + } + + Console.ResetColor(); + w.WriteLine(); + } + + private static void WriteSpace(TextWriter w) => w.Write(" "); + + private static void WriteBlock(TextWriter w, ConsoleColor color, string block, int? pad = null) + { + if (string.IsNullOrEmpty(block)) return; + var b = pad != null ? block.PadRight(pad.Value) : block; + Console.ForegroundColor = ConsoleColor.DarkGray; + w.Write("["); + Console.ForegroundColor = color; + w.Write(b); + Console.ForegroundColor = ConsoleColor.DarkGray; + w.Write("]"); + } + + private static void Write(TextWriter w, ConsoleColor color, string block, int? pad = null) + { + var b = pad != null ? block.PadRight(pad.Value) : block; + var original = Console.ForegroundColor; + Console.ForegroundColor = color; + w.Write(b); + Console.ForegroundColor = original; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineOutParser.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineOutParser.cs new file mode 100644 index 0000000000..183e228f3f --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/LineOutParser.cs @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Text.RegularExpressions; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +{ + public class LineOutParser + { + private LineOutParser() { } + + public static readonly LineOutParser OpenSearch = new(shortNamePrefix: "o.o", fullNamePrefix: "org.opensearch", + securityPluginName: "OpenSearchSecurityPlugin"); + + public static readonly LineOutParser OpenDistro = new(shortNamePrefix: "o.e", fullNamePrefix: "org.elastic", + securityPluginName: "OpenDistroSecurityPlugin"); + public static LineOutParser From(ServerType serverType) + { + switch (serverType) + { + case ServerType.OpenDistro: + case ServerType.ElasticSearch: + return LineOutParser.OpenDistro; + case ServerType.OpenSearch: + return LineOutParser.OpenSearch; + default: + throw new ApplicationException( + $"Unexpected value for {nameof(serverType)} -- {serverType}"); + } + } + +/* +[2016-09-26T11:43:17,314][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] initializing ... +[2016-09-26T11:43:17,470][INFO ][o.e.e.NodeEnvironment ] [readonly-node-a9c5f4] using [1] data paths, mounts [[BOOTCAMP (C:)]], net usable_space [27.7gb], net total_space [129.7gb], spins? [unknown], types [NTFS] +[2016-09-26T11:43:17,471][INFO ][o.e.e.NodeEnvironment ] [readonly-node-a9c5f4] heap size [1.9gb], compressed ordinary object pointers [true] +[2016-09-26T11:43:17,475][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] version[5.0.0-beta1], pid[13172], build[7eb6260/2016-09-20T23:10:37.942Z], OS[Windows 10/10.0/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_101/25.101-b13] +[2016-09-26T11:43:19,160][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [aggs-matrix-stats] +[2016-09-26T11:43:19,160][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [ingest-common] +[2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-expression] +[2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-groovy] +[2016-09-26T11:43:19,161][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-mustache] +[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [lang-painless] +[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [percolator] +[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [reindex] +[2016-09-26T11:43:19,162][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [transport-netty3] +[2016-09-26T11:43:19,163][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded module [transport-netty4] +[2016-09-26T11:43:19,163][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [ingest-attachment] +[2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [ingest-geoip] +[2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [mapper-attachments] +[2016-09-26T11:43:19,164][INFO ][o.e.p.PluginsService ] [readonly-node-a9c5f4] loaded plugin [mapper-murmur3] +[2016-09-26T11:43:19,374][WARN ][d.m.attachment ] [mapper-attachments] plugin has been deprecated and will be replaced by [ingest-attachment] plugin. +[2016-09-26T11:43:22,179][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] initialized +[2016-09-26T11:43:22,180][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] starting ... +*/ + private static readonly Regex ConsoleLineParser = + new Regex(@"\[(?.*?)\]\[(?.*?)\](?:\[(?
.*?)\])(?: \[(?.*?)\])? (?.+)"); + + private static readonly Regex PortParser = new Regex(@"bound_address(es)?(opensearch)? {.+\:(?\d+)}"); + + //[2016-09-26T11:43:17,475][INFO ][o.e.n.Node ] [readonly-node-a9c5f4] version[5.0.0-beta1], pid[13172], build[7eb6260/2016-09-20T23:10:37.942Z], OS[Windows 10/10.0/amd64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_101/25.101-b13] + private static readonly Regex InfoParser = + new Regex(@"version\[(?.*)\], pid\[(?.*)\], build\[(?.+)\]"); + + private LineOutParser(string shortNamePrefix, string fullNamePrefix, string securityPluginName) : this() + { + _shortNamePrefix = shortNamePrefix; + _fullNamePrefix = fullNamePrefix; + _securityPluginRegex = new Regex(Regex.Escape(securityPluginName)); + } + + private readonly Regex _securityPluginRegex; + private readonly string _shortNamePrefix; + private readonly string _fullNamePrefix ; + + public bool TryParse(string line, + out string date, out string level, out string section, out string node, out string message, + out bool started) + { + date = level = section = node = message = null; + started = false; + if (string.IsNullOrEmpty(line)) return false; + + var match = ConsoleLineParser.Match(line); + if (!match.Success) return false; + date = match.Groups["date"].Value.Trim(); + level = match.Groups["level"].Value.Trim(); + section = match.Groups["section"].Value.Trim().Replace(_fullNamePrefix + ".", ""); + node = match.Groups["node"].Value.Trim(); + message = match.Groups["message"].Value.Trim(); + started = TryGetStartedConfirmation(section, message); + return true; + } + + private bool TryGetStartedConfirmation(string section, string message) + { + return section == ShortName("n.Node") && message == "started"; + } + + public bool TryGetPortNumber(string section, string message, out int port) + { + port = 0; + var inHttpSection = + section == ShortName("h.HttpServer") + || section == "http" + || section == ShortName("h.AbstractHttpServerTransport") + || section == ShortName("h.n.Netty4HttpServerTransport") + || section == ShortName("x.s.t.n.SecurityNetty4HttpServerTransport"); + if (!inHttpSection) return false; + + if (string.IsNullOrWhiteSpace(message)) return false; + + var match = PortParser.Match(message); + if (!match.Success) return false; + + var portString = match.Groups["port"].Value.Trim(); + port = int.Parse(portString); + return true; + } + + public bool TryParseNodeInfo(string section, string message, out string version, out int? pid) + { + var inNodeSection = section == ShortName("n.Node") || section == "node"; + + version = null; + pid = null; + if (!inNodeSection) return false; + + var match = InfoParser.Match(message.Replace(Environment.NewLine, "")); + if (!match.Success) return false; + + version = match.Groups["version"].Value.Trim(); + pid = int.Parse(match.Groups["pid"].Value.Trim()); + return true; + } + + private string ShortName(string suffix) => $"{_shortNamePrefix}.{suffix}"; + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/NoopConsoleLineWriter.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/NoopConsoleLineWriter.cs new file mode 100644 index 0000000000..f707972c24 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/ConsoleWriters/NoopConsoleLineWriter.cs @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Managed.ConsoleWriters +{ + internal class NoopConsoleLineWriter : IConsoleLineHandler + { + public static NoopConsoleLineWriter Instance { get; } = new NoopConsoleLineWriter(); + + public void Handle(LineOut lineOut) + { + } + + public void Handle(Exception e) + { + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/INodeFileSystem.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/INodeFileSystem.cs new file mode 100644 index 0000000000..768d7478cb --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/INodeFileSystem.cs @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +namespace OpenSearch.OpenSearch.Managed.FileSystem +{ + /// + /// The file system for an OpenSearch node + /// + public interface INodeFileSystem + { + /// + /// The path to the script to start OpenSearch + /// + string Binary { get; } + + /// + /// The path to the script to manage plugins + /// + string PluginBinary { get; } + + /// + /// The path to the home directory + /// + string OpenSearchHome { get; } + + /// + /// The path to the config directory + /// + string ConfigPath { get; } + + /// + /// The path to the data directory + /// + string DataPath { get; } + + /// + /// The path to the logs directory + /// + string LogsPath { get; } + + /// + /// The path to the repository directory + /// + string RepositoryPath { get; } + + /// + /// The path to the directory in which this node resides + /// + string LocalFolder { get; } + + /// The config environment variable to use for this version + string ConfigEnvironmentVariableName { get; } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/NodeFileSystem.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/NodeFileSystem.cs new file mode 100644 index 0000000000..7059d0b4b2 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/FileSystem/NodeFileSystem.cs @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.IO; +using System.Runtime.InteropServices; +using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; + +namespace OpenSearch.OpenSearch.Managed.FileSystem +{ + /// + public class NodeFileSystem : INodeFileSystem + { + protected const string SubFolder = "OpenSearchManaged"; + + public NodeFileSystem(OpenSearchVersion version, string openSearchHome = null) + { + Version = version; + Artifact = version.Artifact(Product.OpenSearch); + LocalFolder = AppDataFolder(version); + OpenSearchHome = openSearchHome ?? + GetOpenSearchHomeVariable() ?? throw new ArgumentNullException(nameof(openSearchHome)); + + ConfigEnvironmentVariableName = "OPENSEARCH_PATH_CONF"; + } + + protected OpenSearchVersion Version { get; } + protected Artifact Artifact { get; } + + private static bool IsMono { get; } = Type.GetType("Mono.Runtime") != null; + + protected static string BinarySuffix => IsMono || Path.DirectorySeparatorChar == '/' ? "" : ".bat"; + + /// + public string Binary + { + get + { + if (Artifact.ServerType == ServerType.OpenSearch) + return Path.Combine(OpenSearchHome, "bin", "opensearch") + BinarySuffix; + return Path.Combine(OpenSearchHome, "bin", "elasticsearch") + BinarySuffix; + } + } + + /// + public string PluginBinary + { + get + { + if (Artifact.ServerType == ServerType.OpenSearch) + return Path.Combine(OpenSearchHome, "bin", "opensearch-plugin") + BinarySuffix; + return Path.Combine(OpenSearchHome, "bin", "elasticsearch-plugin") + BinarySuffix; + } + } + + /// + public string OpenSearchHome { get; } + + /// + public string LocalFolder { get; } + + /// + public virtual string ConfigPath => null; + + /// + public virtual string DataPath => null; + + /// + public virtual string LogsPath => null; + + /// + public virtual string RepositoryPath => null; + + public string ConfigEnvironmentVariableName { get; } + + protected static string AppDataFolder(OpenSearchVersion version) + { + var appData = GetApplicationDataDirectory(); + return Path.Combine(appData, SubFolder, version.Artifact(Product.OpenSearch).LocalFolderName); + } + + protected static string GetOpenSearchHomeVariable() => Environment.GetEnvironmentVariable("OPENSEARCH_HOME"); + + protected static string GetApplicationDataDirectory() => + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? Environment.GetEnvironmentVariable("LocalAppData") + : Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData, + Environment.SpecialFolderOption.Create); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearch.OpenSearch.Managed.csproj b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearch.OpenSearch.Managed.csproj new file mode 100644 index 0000000000..1778c51080 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearch.OpenSearch.Managed.csproj @@ -0,0 +1,19 @@ + + + + netstandard2.0;net461 + + Provides an observable OpenSearchNode abstraction that can be used to wrap an opensearch process. + Also ships with an cluster abstraction that can start one or more OpenSearchNode's + + opensearch,opensearch,cluster,observable,rx + + + + + + + + + + \ No newline at end of file diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCleanExitException.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCleanExitException.cs new file mode 100644 index 0000000000..4afd39c5b3 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCleanExitException.cs @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; + +namespace OpenSearch.OpenSearch.Managed +{ + public class OpenSearchCleanExitException : Exception + { + public OpenSearchCleanExitException(string message) : base(message) + { + } + + public OpenSearchCleanExitException(string message, Exception innerException) : base(message, innerException) + { + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCluster.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCluster.cs new file mode 100644 index 0000000000..0f38f45553 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchCluster.cs @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using OpenSearch.OpenSearch.Managed.Configuration; + +namespace OpenSearch.OpenSearch.Managed +{ + public class OpenSearchCluster : ClusterBase + { + public OpenSearchCluster(ClusterConfiguration clusterConfiguration) : base(clusterConfiguration) + { + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs new file mode 100644 index 0000000000..75fafc28f9 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/OpenSearchNode.cs @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using OpenSearch.OpenSearch.Managed.Configuration; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; +using OpenSearch.OpenSearch.Managed.FileSystem; +using OpenSearch.Stack.ArtifactsApi; +using ProcNet; +using ProcNet.Std; + +namespace OpenSearch.OpenSearch.Managed +{ + public class OpenSearchNode : ObservableProcess + { + private readonly ManualResetEvent _startedHandle = new ManualResetEvent(false); + + public OpenSearchNode(OpenSearchVersion version, string openSearchHome = null) + : this(new NodeConfiguration(new ClusterConfiguration(version, ServerType.DEFAULT, + (v, s) => new NodeFileSystem(v, openSearchHome)))) + { + } + + public OpenSearchNode(NodeConfiguration config) : base(StartArgs(config)) => NodeConfiguration = config; + public string Version { get; private set; } + public int? Port { get; private set; } + public bool NodeStarted { get; private set; } + public NodeConfiguration NodeConfiguration { get; } + + private int? JavaProcessId { get; set; } + public override int? ProcessId => JavaProcessId ?? base.ProcessId; + public int? HostProcessId => base.ProcessId; + + /// + /// Set this true if you want the node to go into assumed started state as soon as its waiting for more nodes to start + /// doing the election. + /// Useful to speed up starting multi node clusters + /// + public bool AssumeStartedOnNotEnoughMasterPing { get; set; } + + internal IConsoleLineHandler Writer { get; private set; } + + public Exception LastSeenException { get; set; } + public WaitHandle StartedHandle => _startedHandle; + + private static StartArguments StartArgs(NodeConfiguration config) + { + //var args = new[] {config.FileSystem.Binary}.Concat(config.CommandLineArguments); + + var startArguments = new StartArguments(config.FileSystem.Binary, config.CommandLineArguments) + { + SendControlCFirst = true, + Environment = EnvVars(config), + WaitForExit = config.WaitForShutdown, + WaitForStreamReadersTimeout = config.WaitForShutdown + }; + config.ModifyStartArguments(startArguments); + return startArguments; + } + + private static Dictionary EnvVars(NodeConfiguration config) + { + var environmentVariables = new Dictionary {{"OPENSEARCH_JAVA_OPTS", "-Xms1g -Xmx1g"}}; + if (!string.IsNullOrWhiteSpace(config.FileSystem.ConfigPath)) + environmentVariables.Add(config.FileSystem.ConfigEnvironmentVariableName, config.FileSystem.ConfigPath); + + if (!string.IsNullOrWhiteSpace(config.FileSystem.OpenSearchHome)) + environmentVariables.Add("OPENSEARCH_HOME", config.FileSystem.OpenSearchHome); + + // Duplicate all env vars for ES_* prefix for backward compatibility with OpenDistro and ElasticSearch; + // unused env vars would be just ignored. + environmentVariables.Add("ES_JAVA_OPTS", "-Xms1g -Xmx1g"); + if (!string.IsNullOrWhiteSpace(config.FileSystem.ConfigPath)) + environmentVariables.Add(config.FileSystem.ConfigEnvironmentVariableName.Replace("OPENSEARCH", "ES"), config.FileSystem.ConfigPath); + + if (!string.IsNullOrWhiteSpace(config.FileSystem.OpenSearchHome)) + environmentVariables.Add("ES_HOME", config.FileSystem.OpenSearchHome); + + return environmentVariables; + } + + private bool AssumedStartedStateChecker(string section, string message) + { + if (AssumeStartedOnNotEnoughMasterPing + && section.Contains("ZenDiscovery") + && message.Contains("not enough master nodes discovered during pinging")) + return true; + return false; + } + + public IDisposable Start() => Start(TimeSpan.FromMinutes(2)); + + public IDisposable Start(TimeSpan waitForStarted) => Start(new LineHighlightWriter(), waitForStarted); + + public IDisposable Start(IConsoleLineHandler writer, TimeSpan waitForStarted) + { + var node = NodeConfiguration.DesiredNodeName; + var subscription = SubscribeLines(writer); + if (WaitForStarted(waitForStarted)) return subscription; + subscription.Dispose(); + throw new OpenSearchCleanExitException( + $"Failed to start node: {node} before the configured timeout of: {waitForStarted}"); + } + + public IDisposable SubscribeLines() => SubscribeLines(new LineHighlightWriter()); + + public IDisposable SubscribeLines(IConsoleLineHandler writer) => + SubscribeLines(writer, delegate { }, delegate { }, delegate { }); + + public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext) => + SubscribeLines(writer, onNext, delegate { }, delegate { }); + + public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext, + Action onError) => + SubscribeLines(writer, onNext, onError, delegate { }); + + public IDisposable SubscribeLines(IConsoleLineHandler writer, Action onNext, Action onError, + Action onCompleted) + { + Writer = writer; + var node = NodeConfiguration.DesiredNodeName; + writer?.WriteDiagnostic($"OpenSearch location: [{Binary}]", node); + writer?.WriteDiagnostic($"Settings: {{{string.Join(" ", NodeConfiguration.CommandLineArguments)}}}", node); + + var envVarName = "JAVA_HOME"; + var javaHome = Environment.GetEnvironmentVariable(envVarName); + writer?.WriteDiagnostic($"{envVarName}: {{{javaHome}}}", node); + Process.StartInfo.Environment[envVarName] = javaHome; + + return SubscribeLines( + l => + { + writer?.Handle(l); + onNext?.Invoke(l); + }, + e => + { + LastSeenException = e; + writer?.Handle(e); + onError?.Invoke(e); + _startedHandle.Set(); + }, + () => + { + onCompleted?.Invoke(); + _startedHandle.Set(); + }); + } + + public bool WaitForStarted(TimeSpan timeout) => _startedHandle.WaitOne(timeout); + + protected override void OnBeforeSetCompletedHandle() + { + _startedHandle.Set(); + base.OnBeforeSetCompletedHandle(); + } + + protected override void OnBeforeWaitForEndOfStreamsError(TimeSpan waited) + { + // The wait for streams finished before streams were fully read. + // this usually indicates the process is still running. + // Proc will successfully kill the host but will leave the JavaProcess the bat file starts running + // The opensearch jar is closing down so won't leak but might prevent EphemeralClusterComposer to do its clean up. + // We do a hard kill on both here to make sure both processes are gone. + HardKill(HostProcessId); + HardKill(JavaProcessId); + } + + private static void HardKill(int? processId) + { + if (!processId.HasValue) return; + try + { + var p = Process.GetProcessById(processId.Value); + p.Kill(); + } + catch (Exception) + { + } + } + + protected override bool ContinueReadingFromProcessReaders() + { + if (!NodeStarted) return true; + return true; + + // some how if we return false here it leads to Task starvation in Proc and tests in e.g will OpenSearch.OpenSearch.Xunit will start + // to timeout. This makes little sense to me now, so leaving this performance optimization out for now. Hopefully another fresh look will yield + // to (not so) obvious. + //return this.NodeConfiguration.ShowOpenSearchOutputAfterStarted; + } + + protected override bool KeepBufferingLines(LineOut c) + { + var lineOutParser = LineOutParser.From(NodeConfiguration.ServerType); + //if the node is already started only keep buffering lines while we have a writer and the nodeconfiguration wants output after started + if (NodeStarted) + { + var keepBuffering = Writer != null && NodeConfiguration.ShowOpenSearchOutputAfterStarted; + if (!keepBuffering) CancelAsyncReads(); + return keepBuffering; + } + + var parsed = lineOutParser.TryParse(c?.Line, out _, out _, out var section, out _, out var message, + out var started); + + if (!parsed) return Writer != null; + + if (JavaProcessId == null && lineOutParser.TryParseNodeInfo(section, message, out var version, out var pid)) + { + JavaProcessId = pid; + Version = version; + } + + else if (lineOutParser.TryGetPortNumber(section, message, out var port)) + { + Port = port; + var dp = NodeConfiguration.DesiredPort; + if (dp.HasValue && Port != dp.Value) + throw new OpenSearchCleanExitException( + $"Node started on port {port} but {dp.Value} was requested"); + } + + if (!started) started = AssumedStartedStateChecker(section, message); + if (started) + { + if (!Port.HasValue) + throw new OpenSearchCleanExitException( + $"Node started but OpenSearchNode did not grab its port number"); + NodeStarted = true; + _startedHandle.Set(); + } + + // if we have dont a writer always return true + if (Writer != null) return true; + //otherwise only keep buffering if we are not started + return !started; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Managed/README.md b/abstractions/src/OpenSearch.OpenSearch.Managed/README.md new file mode 100644 index 0000000000..7b336efe43 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Managed/README.md @@ -0,0 +1,86 @@ +# OpenSearch.Managed +Provides an easy to start/stop one or more OpenSearch instances that exists on disk already + + +## OpenSearchNode + +A `Proc.ObservableProcess` implementation that starts an OpenSearch instance from the specified +location. It is able to optionally block untill the node goes into started state and it sniffs the output +to expose useful information such as the java process id, port number and others. + +Because its a subclass of `Proc.ObservableProcess` its completely reactive and allows you to seperate the act +of listening to the output and proxying stdout/err. + +#### Examples: + +All the examples assume the following are defined. `openSearchHome` points to a local folder where `OpenSearch` is installed/unzipped. + +```csharp +var version = "1.0.0"; +var openSearchHome = Environment.ExpandEnvironmentVariables($@"%LOCALAPPDATA%\OpenSearchManaged\{version}\opensearch-{version}"); +``` + +The easiest way to get going pass the `version` and `openSearchHome` to `OpenSearchNode`. +`OpenSearchNode` implements `IDisposable` and will try to shutdown gracefully when disposed. +Simply new'ing `OpenSearchNode` won't actually start the node. We need to explicitly call `Start()`. +`Start()` has several overloads but the default waits `2 minutes` for a started confirmation and proxies +the consoleout using `HighlightWriter` which pretty prints the opensearch output. + + +```csharp +using (var node = new OpenSearchNode(version, openSearchHome)) +{ + node.Start(); +} +``` + +`Start` is simply sugar over + +```csharp +using (var node = new OpenSearchNode(version, openSearchHome)) +{ + node.SubscribeLines(new HighlightWriter()); + + if (!node.WaitForStarted(TimeSpan.FromMinutes(2))) throw new Exception(); +} +``` + +As mentioned before `OpenSearchNode` is really an `IObservable` by virtue of being an +subclass of `Proc.ObservableProcess`. `SubscribeLines` is a specialized +`Subscribe` that buffers `CharactersOut` untill a line is formed and emits a `LineOut`. Overloads exists that +take additional `onNext/onError/onCompleted` handlers. + +A node can also be started using a `NodeConfiguration` + +```csharp +var clusterConfiguration = new ClusterConfiguration(version, ServerType.OpenSearch, openSearchHome); +var nodeConfiguration = new NodeConfiguration(clusterConfiguration, 9200) +{ + ShowOpenSearchOutputAfterStarted = false, + Settings = { "node.attr.x", "y" } +}; +using (var node = new OpenSearchNode(nodeConfiguration)) +{ + node.Start(); +} +``` + +Which exposes the full range of options e.g here `ShowOpenSearchOutputAfterStarted` will dispose +of the console out subscriptions as soon as we've parsed the started message to minimize the resources we consume. +`Settings` here allows you to pass opensearch node settings to use for the node. + +## OpenSearchCluster + +A simple abstraction that can can start and stop one or more `OpenSearchNodes` and wait for all of them to +be started + +```csharp +var clusterConfiguration = new ClusterConfiguration(version, ServerType.OpenSearch, openSearchHome, numberOfNodes: 2); +using (var cluster = new OpenSearchCluster(clusterConfiguration)) +{ + cluster.Start(); +} +``` + +`OpenSearchCluster` is simply a barebones `ClusterBase` implementation, which is more powerful then it seems +and serves as the base for `OpenSearch.Managed.Ephemeral`. diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearch.OpenSearch.Xunit.csproj b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearch.OpenSearch.Xunit.csproj new file mode 100644 index 0000000000..3892ea3700 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearch.OpenSearch.Xunit.csproj @@ -0,0 +1,15 @@ + + + + netstandard2.0;net461 + True + Provides an Xunit test framework allowing you to run integration tests against local ephemeral OpenSearch clusters + opensearch,opensearch,xunit,cluster,integration,test,ephemeral + + + + + + + + \ No newline at end of file diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitConfigurationAttribute.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitConfigurationAttribute.cs new file mode 100644 index 0000000000..f92538d0f8 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitConfigurationAttribute.cs @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; + +namespace OpenSearch.OpenSearch.Xunit +{ + /// + /// An assembly attribute that specifies the + /// for Xunit tests within the assembly. + /// + [AttributeUsage(AttributeTargets.Assembly)] + public class OpenSearchXunitConfigurationAttribute : Attribute + { + /// + /// Creates a new instance of + /// + /// + /// A type deriving from that specifies the run options + /// + public OpenSearchXunitConfigurationAttribute(Type type) + { + var options = Activator.CreateInstance(type) as OpenSearchXunitRunOptions; + Options = options ?? new OpenSearchXunitRunOptions(); + } + + /// + /// The run options + /// + public OpenSearchXunitRunOptions Options { get; } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunOptions.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunOptions.cs new file mode 100644 index 0000000000..7696cd1891 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunOptions.cs @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using OpenSearch.OpenSearch.Xunit.XunitPlumbing; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Xunit +{ + /// + /// The Xunit test runner options + /// + public class OpenSearchXunitRunOptions + { + /// + /// Informs the runner whether we expect to run integration tests. Defaults to true + /// + public bool RunIntegrationTests { get; set; } = true; + + /// + /// Setting this to true will assume the cluster that is currently running was started for the purpose of these tests + /// Defaults to false + /// + public bool IntegrationTestsMayUseAlreadyRunningNode { get; set; } = false; + + /// + /// Informs the runner whether unit tests will be run. Defaults to false. + /// If set to true and is false, the runner will run all the + /// tests in parallel with the maximum degree of parallelism + /// + public bool RunUnitTests { get; set; } + + /// + /// A global test filter that can be used to only run certain tests. + /// Accepts a comma separated list of filters + /// + public string TestFilter { get; set; } + + /// + /// A global cluster filter that can be used to only run certain cluster's tests. + /// Accepts a comma separated list of filters + /// + public string ClusterFilter { get; set; } + + /// + /// Informs the runner what version of OpenSearch is under test. Required for + /// to kick in + /// + public OpenSearchVersion Version { get; set; } + + /// + /// Called when the tests have finished running successfully + /// + /// Per cluster timings of the total test time, including starting OpenSearch + /// All collection of failed cluster, failed tests tuples + public virtual void OnTestsFinished(Dictionary runnerClusterTotals, + ConcurrentBag> runnerFailedCollections) + { + } + + /// + /// Called before tests run. An ideal place to perform actions such as writing information to + /// . + /// + public virtual void OnBeforeTestsRun() + { + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunner.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunner.cs new file mode 100644 index 0000000000..7b39749b6c --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/OpenSearchXunitRunner.cs @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using OpenSearch.OpenSearch.Ephemeral; + +namespace OpenSearch.OpenSearch.Xunit +{ + public static class OpenSearchXunitRunner + { + public static IEphemeralCluster CurrentCluster { get; internal set; } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/PrintXunitAfterStartedTask.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/PrintXunitAfterStartedTask.cs new file mode 100644 index 0000000000..30c4ac89a9 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/PrintXunitAfterStartedTask.cs @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using OpenSearch.OpenSearch.Ephemeral; +using OpenSearch.OpenSearch.Ephemeral.Tasks; +using OpenSearch.OpenSearch.Managed.ConsoleWriters; + +namespace OpenSearch.OpenSearch.Xunit +{ + /// + /// A task that writes a diagnostic message to indicate that tests will now run + /// + public class PrintXunitAfterStartedTask : ClusterComposeTask + { + /// + public override void Run(IEphemeralCluster cluster) + { + var name = cluster.GetType().Name; + cluster.Writer.WriteDiagnostic($"All good! kicking off [{name}] tests now"); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/README.md b/abstractions/src/OpenSearch.OpenSearch.Xunit/README.md new file mode 100644 index 0000000000..5177cd4268 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/README.md @@ -0,0 +1,104 @@ +# OpenSearch.OpenSearch.Xunit + +Write integration tests against OpenSearch. +Works with `.NET Core` and `.NET 4.6` and up. + +Supports `dotnet xunit`, `dotnet test`, `xunit.console.runner` and tests will be runnable in your IDE through VSTest and jetBrains Rider. + + +## Getting started + +**NOTE:** `OpenSearch.Xunit` supports both .NET core and Full Framework 4.6 and up. The getting started uses the new csproj +from core but you can also use a full framework project. + +### Create a class library project + +```xml + + + netcoreapp3.0 + + + + + + + + + + + +``` + +### Use OpenSearch.OpenSearch.Xunit's test framework + +Add the following Assembly attribute anywhere in your project. This informs Xunit to use our +test framework to orchestrate and discover the tests. + +```csharp +[assembly: Xunit.TestFrameworkAttribute("OpenSearch.OpenSearch.Xunit.Sdk.OpenSearchTestFramework", "OpenSearch.OpenSearch.Xunit")] +``` + +### Create a cluster + +This is the cluster that we'll write our integration test against. You can have multiple cluster. +`OpenSearch.OpenSearch.Xunit` will only ever start one cluster at a time and then run all tests belonging to that cluster. + +```csharp +/// Declare our cluster that we want to inject into our test classes +public class MyTestCluster : XunitClusterBase +{ + /// + /// We pass our configuration instance to the base class. + /// We only configure it to run version 1.0.0 here but lots of additional options are available. + /// + public MyTestCluster() : base(new XunitClusterConfiguration("1.0.0")) { } +} +``` + +### Create a test class + +```csharp +public class ExampleTest : IClusterFixture +{ + public ExampleTest(MyTestCluster cluster) + { + // This registers a single client for the whole clusters lifetime to be reused and shared. + // we do not expose Client on the passed cluster directly for two reasons + // + // 1) We do not want to prescribe how to new up the client + // + // 2) We do not want OpenSearch.Xunit to depend on OSC. OpenSearch.Xunit can start clusters + // and OSC Major.x is only tested and supported against OpenSearch Major.x. + // + this.Client = cluster.GetOrAddClient(c => + { + var nodes = cluster.NodesUris(); + var connectionPool = new StaticConnectionPool(nodes); + var settings = new ConnectionSettings(connectionPool) + .EnableDebugMode(); + return new OpenSearchClient(settings); + ); + } + + private OpenSearchClient Client { get; } + + /// [I] marks an integration test (like [Fact] would for plain Xunit) + [I] public void SomeTest() + { + var rootNodeInfo = this.Client.RootNodeInfo(); + + rootNodeInfo.Name.Should().NotBeNullOrEmpty(); + } +} + +``` + +### Run your integration tests! + +![jetBrains rider integration](ide-integration.png) + +Or on the command line using `dotnet test` + +![sample output](output.gif) + diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/ForEachAsyncExtensions.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/ForEachAsyncExtensions.cs new file mode 100644 index 0000000000..5e316b5858 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/ForEachAsyncExtensions.cs @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace OpenSearch.OpenSearch.Xunit.Sdk +{ + internal static class ForEachAsyncExtensions + { + internal static Task ForEachAsync(this IEnumerable source, int dop, Func body) => + Task.WhenAll( + from partition in Partitioner.Create(source).GetPartitions(dop) + select Task.Run(async delegate + { + using (partition) + while (partition.MoveNext()) + await body(partition.Current).ConfigureAwait(false); + })); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFramework.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFramework.cs new file mode 100644 index 0000000000..a1bb7d1564 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFramework.cs @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.Sdk +{ + public class OpenSearchTestFramework : XunitTestFramework + { + public OpenSearchTestFramework(IMessageSink messageSink) : base(messageSink) + { + } + + protected override ITestFrameworkDiscoverer CreateDiscoverer(IAssemblyInfo assemblyInfo) => + new OpenSearchTestFrameworkDiscoverer(assemblyInfo, SourceInformationProvider, DiagnosticMessageSink); + + protected override ITestFrameworkExecutor CreateExecutor(AssemblyName assemblyName) + { + var assembly = Assembly.Load(assemblyName); + var options = assembly.GetCustomAttribute()?.Options ?? + new OpenSearchXunitRunOptions(); + + return new TestFrameworkExecutor(assemblyName, SourceInformationProvider, DiagnosticMessageSink) + { + Options = options + }; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFrameworkDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFrameworkDiscoverer.cs new file mode 100644 index 0000000000..00dec3b5fc --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/OpenSearchTestFrameworkDiscoverer.cs @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.Sdk +{ + public class OpenSearchTestFrameworkDiscoverer : XunitTestFrameworkDiscoverer + { + public OpenSearchTestFrameworkDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, + IMessageSink diagnosticMessageSink, IXunitTestCollectionFactory collectionFactory = null) : base( + assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory) + { + var a = Assembly.Load(new AssemblyName(assemblyInfo.Name)); + var options = a.GetCustomAttribute()?.Options ?? + new OpenSearchXunitRunOptions(); + Options = options; + } + + /// + /// The options for + /// + public OpenSearchXunitRunOptions Options { get; } + + protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, + IMessageBus messageBus, ITestFrameworkDiscoveryOptions discoveryOptions) + { + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + return base.FindTestsForType(testClass, includeSourceInformation, messageBus, discoveryOptions); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestAssemblyRunner.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestAssemblyRunner.cs new file mode 100644 index 0000000000..eb43aa4e15 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestAssemblyRunner.cs @@ -0,0 +1,306 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using OpenSearch.OpenSearch.Ephemeral; +using OpenSearch.OpenSearch.Ephemeral.Tasks.ValidationTasks; +using OpenSearch.OpenSearch.Xunit.XunitPlumbing; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.Sdk +{ + internal class TestAssemblyRunner : XunitTestAssemblyRunner + { + private readonly Dictionary> _assemblyFixtureMappings = + new Dictionary>(); + + private readonly List, GroupedByCluster>> _grouped; + + public TestAssemblyRunner(ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) + { + var tests = OrderTestCollections(); + RunIntegrationTests = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests)); + IntegrationTestsMayUseAlreadyRunningNode = + executionOptions.GetValue(nameof(OpenSearchXunitRunOptions + .IntegrationTestsMayUseAlreadyRunningNode)); + RunUnitTests = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests)); + TestFilter = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.TestFilter)); + ClusterFilter = executionOptions.GetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter)); + + //bit side effecty, sets up _assemblyFixtureMappings before possibly letting xunit do its regular concurrency thing + _grouped = (from c in tests + let cluster = ClusterFixture(c.Item2.First().TestMethod.TestClass) + let testcase = new GroupedByCluster {Collection = c.Item1, TestCases = c.Item2, Cluster = cluster} + group testcase by testcase.Cluster + into g + orderby g.Count() descending + select g).ToList(); + } + + public ConcurrentBag Summaries { get; } = new ConcurrentBag(); + + public ConcurrentBag> FailedCollections { get; } = + new ConcurrentBag>(); + + public Dictionary ClusterTotals { get; } = new Dictionary(); + + private bool RunIntegrationTests { get; } + private bool IntegrationTestsMayUseAlreadyRunningNode { get; } + private bool RunUnitTests { get; } + private string TestFilter { get; } + private string ClusterFilter { get; } + + protected override Task RunTestCollectionAsync(IMessageBus b, ITestCollection c, + IEnumerable t, CancellationTokenSource s) + { + var aggregator = new ExceptionAggregator(Aggregator); + var fixtureObjects = new Dictionary(); + foreach (var kv in _assemblyFixtureMappings) fixtureObjects.Add(kv.Key, kv.Value); + return new TestCollectionRunner(fixtureObjects, c, t, DiagnosticMessageSink, b, TestCaseOrderer, aggregator, + s) + .RunAsync(); + } + + protected override async Task RunTestCollectionsAsync(IMessageBus messageBus, + CancellationTokenSource cancellationTokenSource) + { + //threading guess + var defaultMaxConcurrency = Environment.ProcessorCount * 4; + + if (RunUnitTests && !RunIntegrationTests) + return await UnitTestPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) + .ConfigureAwait(false); + + return await IntegrationPipeline(defaultMaxConcurrency, messageBus, cancellationTokenSource) + .ConfigureAwait(false); + } + + + private async Task UnitTestPipeline(int defaultMaxConcurrency, IMessageBus messageBus, + CancellationTokenSource ctx) + { + //make sure all clusters go in started state (won't actually start clusters in unit test mode) + //foreach (var g in this._grouped) g.Key?.Start(); + + var testFilters = CreateTestFilters(TestFilter); + await _grouped.SelectMany(g => g) + .ForEachAsync(defaultMaxConcurrency, + async g => { await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); }) + .ConfigureAwait(false); + //foreach (var g in this._grouped) g.Key?.Dispose(); + + return new RunSummary + { + Total = Summaries.Sum(s => s.Total), + Failed = Summaries.Sum(s => s.Failed), + Skipped = Summaries.Sum(s => s.Skipped) + }; + } + + private async Task IntegrationPipeline(int defaultMaxConcurrency, IMessageBus messageBus, + CancellationTokenSource ctx) + { + var testFilters = CreateTestFilters(TestFilter); + foreach (var group in _grouped) + { + OpenSearchXunitRunner.CurrentCluster = @group.Key; + if (@group.Key == null) + { + var testCount = @group.SelectMany(q => q.TestCases).Count(); + Console.WriteLine($" -> Several tests skipped because they have no cluster associated"); + Summaries.Add(new RunSummary {Total = testCount, Skipped = testCount}); + continue; + } + + var type = @group.Key.GetType(); + var clusterName = type.Name.Replace("Cluster", string.Empty) ?? "UNKNOWN"; + if (!MatchesClusterFilter(clusterName)) continue; + + var dop = @group.Key.ClusterConfiguration?.MaxConcurrency ?? defaultMaxConcurrency; + dop = dop <= 0 ? defaultMaxConcurrency : dop; + + var timeout = @group.Key.ClusterConfiguration?.Timeout ?? TimeSpan.FromMinutes(2); + + var skipReasons = @group.SelectMany(g => g.TestCases.Select(t => t.SkipReason)).ToList(); + var allSkipped = skipReasons.All(r => !string.IsNullOrWhiteSpace(r)); + if (allSkipped) + { + Console.WriteLine($" -> All tests from {clusterName} are skipped under the current configuration"); + Summaries.Add(new RunSummary {Total = skipReasons.Count, Skipped = skipReasons.Count}); + continue; + } + + ClusterTotals.Add(clusterName, Stopwatch.StartNew()); + + bool ValidateRunningVersion() + { + try + { + var t = new ValidateRunningVersion(); + t.Run(@group.Key); + return true; + } + catch (Exception) + { + return false; + } + } + + using (@group.Key) + { + if (!IntegrationTestsMayUseAlreadyRunningNode || !ValidateRunningVersion()) + @group.Key?.Start(timeout); + + await @group.ForEachAsync(dop, + async g => + { + await RunTestCollections(messageBus, ctx, g, testFilters).ConfigureAwait(false); + }) + .ConfigureAwait(false); + } + + ClusterTotals[clusterName].Stop(); + } + + return new RunSummary + { + Total = Summaries.Sum(s => s.Total), + Failed = Summaries.Sum(s => s.Failed), + Skipped = Summaries.Sum(s => s.Skipped) + }; + } + + private async Task RunTestCollections(IMessageBus messageBus, CancellationTokenSource ctx, GroupedByCluster g, + string[] testFilters) + { + var test = g.Collection.DisplayName.Replace("Test collection for", string.Empty).Trim(); + if (!MatchesATestFilter(test, testFilters)) return; + if (testFilters.Length > 0) Console.WriteLine(" -> " + test); + + try + { + var summary = await RunTestCollectionAsync(messageBus, g.Collection, g.TestCases, ctx) + .ConfigureAwait(false); + var type = g.Cluster?.GetType(); + var clusterName = type?.Name.Replace("Cluster", "") ?? "UNKNOWN"; + if (summary.Failed > 0) + FailedCollections.Add(Tuple.Create(clusterName, test)); + Summaries.Add(summary); + } + catch (TaskCanceledException) + { + // TODO: What should happen here? + } + } + + private static string[] CreateTestFilters(string testFilters) => + testFilters?.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() + ?? new string[] { }; + + private static bool MatchesATestFilter(string test, IReadOnlyCollection testFilters) + { + if (testFilters.Count == 0 || string.IsNullOrWhiteSpace(test)) return true; + return testFilters + .Any(filter => test.IndexOf(filter, StringComparison.OrdinalIgnoreCase) >= 0); + } + + private bool MatchesClusterFilter(string cluster) + { + if (string.IsNullOrWhiteSpace(cluster) || string.IsNullOrWhiteSpace(ClusterFilter)) return true; + return ClusterFilter + .Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries) + .Select(c => c.Trim()) + .Any(c => cluster.IndexOf(c, StringComparison.OrdinalIgnoreCase) >= 0); + } + + private IEphemeralCluster ClusterFixture(ITestClass testMethodTestClass) + { + var clusterType = GetClusterForClass(testMethodTestClass.Class); + if (clusterType == null) return null; + + if (_assemblyFixtureMappings.TryGetValue(clusterType, out var cluster)) return cluster; + Aggregator.Run(() => + { + var o = Activator.CreateInstance(clusterType); + cluster = o as IEphemeralCluster; + }); + _assemblyFixtureMappings.Add(clusterType, cluster); + return cluster; + } + + public static bool IsAnIntegrationTestClusterType(Type type) => + typeof(XunitClusterBase).GetTypeInfo().IsAssignableFrom(type.GetTypeInfo()) + || IsSubclassOfRawGeneric(typeof(XunitClusterBase<>), type); + + public static Type GetClusterForClass(ITypeInfo testClass) => + GetClusterFromClassClusterFixture(testClass) ?? GetClusterFromIntegrationAttribute(testClass); + + private static Type GetClusterFromClassClusterFixture(ITypeInfo testClass) => ( + from i in testClass.Interfaces + where i.IsGenericType + from a in i.GetGenericArguments() + select a.ToRuntimeType() + ).FirstOrDefault(IsAnIntegrationTestClusterType); + + private static Type GetClusterFromIntegrationAttribute(ITypeInfo testClass) => + testClass.GetCustomAttributes(typeof(IntegrationTestClusterAttribute)) + .FirstOrDefault()?.GetNamedArgument(nameof(IntegrationTestClusterAttribute.ClusterType)); + + private static bool IsSubclassOfRawGeneric(Type generic, Type toCheck) + { + while (toCheck != null && toCheck != typeof(object)) + { + var cur = toCheck.GetTypeInfo().IsGenericType ? toCheck.GetGenericTypeDefinition() : toCheck; + if (generic == cur) return true; + + toCheck = toCheck.GetTypeInfo().BaseType; + } + + return false; + } + + private class GroupedByCluster + { + public IEphemeralCluster Cluster { get; set; } + public ITestCollection Collection { get; set; } + public List TestCases { get; set; } + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestCollectionRunner.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestCollectionRunner.cs new file mode 100644 index 0000000000..b21de00e36 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestCollectionRunner.cs @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.Sdk +{ + internal class TestCollectionRunner : XunitTestCollectionRunner + { + private readonly Dictionary _assemblyFixtureMappings; + private readonly IMessageSink _diagnosticMessageSink; + + public TestCollectionRunner(Dictionary assemblyFixtureMappings, + ITestCollection testCollection, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + ITestCaseOrderer testCaseOrderer, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + : base(testCollection, testCases, diagnosticMessageSink, messageBus, testCaseOrderer, aggregator, + cancellationTokenSource) + { + _assemblyFixtureMappings = assemblyFixtureMappings; + _diagnosticMessageSink = diagnosticMessageSink; + } + + protected override Task RunTestClassAsync(ITestClass testClass, IReflectionTypeInfo @class, + IEnumerable testCases) + { + // whats this doing exactly?? + var combinedFixtures = new Dictionary(_assemblyFixtureMappings); + foreach (var kvp in CollectionFixtureMappings) + combinedFixtures[kvp.Key] = kvp.Value; + + // We've done everything we need, so hand back off to default Xunit implementation for class runner + return new XunitTestClassRunner(testClass, @class, testCases, _diagnosticMessageSink, MessageBus, + TestCaseOrderer, new ExceptionAggregator(Aggregator), CancellationTokenSource, combinedFixtures) + .RunAsync(); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestFrameworkExecutor.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestFrameworkExecutor.cs new file mode 100644 index 0000000000..bb787a0a90 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/Sdk/TestFrameworkExecutor.cs @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using OpenSearch.OpenSearch.Managed; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.Sdk +{ + internal class TestFrameworkExecutor : XunitTestFrameworkExecutor + { + public TestFrameworkExecutor(AssemblyName a, ISourceInformationProvider sip, IMessageSink d) : base(a, sip, d) + { + } + + public OpenSearchXunitRunOptions Options { get; set; } + + public override void RunAll(IMessageSink executionMessageSink, ITestFrameworkDiscoveryOptions discoveryOptions, + ITestFrameworkExecutionOptions executionOptions) + { + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + discoveryOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + + base.RunAll(executionMessageSink, discoveryOptions, executionOptions); + } + + + public override void RunTests(IEnumerable testCases, IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + { + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + executionOptions.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + base.RunTests(testCases, executionMessageSink, executionOptions); + } + + protected override async void RunTestCases(IEnumerable testCases, IMessageSink sink, + ITestFrameworkExecutionOptions options) + { + options.SetValue(nameof(OpenSearchXunitRunOptions.Version), Options.Version); + options.SetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests), Options.RunIntegrationTests); + options.SetValue(nameof(OpenSearchXunitRunOptions.IntegrationTestsMayUseAlreadyRunningNode), + Options.IntegrationTestsMayUseAlreadyRunningNode); + options.SetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests), Options.RunUnitTests); + options.SetValue(nameof(OpenSearchXunitRunOptions.TestFilter), Options.TestFilter); + options.SetValue(nameof(OpenSearchXunitRunOptions.ClusterFilter), Options.ClusterFilter); + try + { + using (var runner = + new TestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, sink, options)) + { + Options.OnBeforeTestsRun(); + await runner.RunAsync().ConfigureAwait(false); + Options.OnTestsFinished(runner.ClusterTotals, runner.FailedCollections); + } + } + catch (Exception e) + { + if (e is OpenSearchCleanExitException || e is AggregateException ae && + ae.Flatten().InnerException is OpenSearchCleanExitException) + sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, + new OpenSearchCleanExitException("Node failed to start", e))); + else + sink.OnMessage(new TestAssemblyCleanupFailure(Enumerable.Empty(), TestAssembly, e)); + throw; + } + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs new file mode 100644 index 0000000000..d59d6d0f82 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterBase.cs @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using OpenSearch.OpenSearch.Ephemeral; + +namespace OpenSearch.OpenSearch.Xunit +{ + /// + /// Base class for a cluster that integrates with Xunit tests + /// + public abstract class XunitClusterBase : XunitClusterBase + { + protected XunitClusterBase(XunitClusterConfiguration configuration) : base(configuration) + { + } + } + + /// + /// Base class for a cluster that integrates with Xunit tests + /// + public abstract class XunitClusterBase : EphemeralCluster + where TConfiguration : XunitClusterConfiguration + { + protected XunitClusterBase(TConfiguration configuration) : base(configuration) + { + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterConfiguration.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterConfiguration.cs new file mode 100644 index 0000000000..2781653b8c --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterConfiguration.cs @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using OpenSearch.OpenSearch.Ephemeral; +using OpenSearch.OpenSearch.Ephemeral.Plugins; +using OpenSearch.Stack.ArtifactsApi; + +namespace OpenSearch.OpenSearch.Xunit +{ + public class XunitClusterConfiguration : EphemeralClusterConfiguration + { + public XunitClusterConfiguration( + OpenSearchVersion version, + ServerType serverType, + ClusterFeatures features = ClusterFeatures.None, + OpenSearchPlugins plugins = null, + int numberOfNodes = 1) + : base(version, serverType, features, plugins, numberOfNodes) => + AdditionalAfterStartedTasks.Add(new PrintXunitAfterStartedTask()); + + /// + protected override string NodePrefix => "xunit"; + + /// + /// The maximum number of tests that can run concurrently against a cluster using this configuration. + /// + public int MaxConcurrency { get; set; } + + /// + /// The maximum amount of time a cluster can run using this configuration. + /// + public TimeSpan? Timeout { get; set; } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterExtensions.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterExtensions.cs new file mode 100644 index 0000000000..d426b3b658 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitClusterExtensions.cs @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Concurrent; +using OpenSearch.OpenSearch.Ephemeral; + +namespace OpenSearch.OpenSearch.Xunit +{ + /// + /// Extension methods for + /// + public static class XunitClusterExtensions + { + private static readonly ConcurrentDictionary Clients = + new ConcurrentDictionary(); + + /// + /// Gets a client for the cluster if one exists, or creates a new client if one doesn't. + /// + /// the cluster to create a client for + /// A delegate to create a client, given the cluster to create it for + /// the type of the client + /// An instance of a client + public static T GetOrAddClient(this IEphemeralCluster cluster, Func getOrAdd) => + (T) Clients.GetOrAdd(cluster, c => getOrAdd(c)); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IClusterFixture.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IClusterFixture.cs new file mode 100644 index 0000000000..3d58c90a99 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IClusterFixture.cs @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using OpenSearch.OpenSearch.Ephemeral; +using OpenSearch.OpenSearch.Managed; + +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +{ + // ReSharper disable once UnusedTypeParameter + // used by the runner to new() the proper cluster + public interface IClusterFixture + where TCluster : ICluster, new() + { + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs new file mode 100644 index 0000000000..e85ab89c35 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestClusterAttribute.cs @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using OpenSearch.OpenSearch.Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] + public class IntegrationTestClusterAttribute : Attribute + { + public IntegrationTestClusterAttribute(Type clusterType) + { + if (!TestAssemblyRunner.IsAnIntegrationTestClusterType(clusterType)) + throw new ArgumentException( + $"Cluster must be subclass of {nameof(XunitClusterBase)} or {nameof(XunitClusterBase)}<>"); + ClusterType = clusterType; + } + + public Type ClusterType { get; } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs new file mode 100644 index 0000000000..7ae53131b0 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/IntegrationTestDiscoverer.cs @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using OpenSearch.OpenSearch.Xunit.Sdk; +using OpenSearch.Stack.ArtifactsApi; +using SemVer; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using Enumerable = System.Linq.Enumerable; + +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +{ + /// + /// An Xunit test that should be skipped, and a reason why. + /// + public abstract class SkipTestAttributeBase : Attribute + { + /// + /// Whether the test should be skipped + /// + public abstract bool Skip { get; } + + /// + /// The reason why the test should be skipped + /// + public abstract string Reason { get; } + } + + /// + /// An Xunit integration test + /// + [XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.IntegrationTestDiscoverer", + "OpenSearch.OpenSearch.Xunit")] + public class I : FactAttribute + { + } + + /// + /// A test discoverer used to discover integration tests cases attached + /// to test methods that are attributed with attribute + /// + public class IntegrationTestDiscoverer : OpenSearchTestCaseDiscoverer + { + public IntegrationTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) + { + } + + /// + protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + out string skipReason) + { + skipReason = null; + var runIntegrationTests = + discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunIntegrationTests)); + if (!runIntegrationTests) return true; + + var cluster = TestAssemblyRunner.GetClusterForClass(testMethod.TestClass.Class); + if (cluster == null) + { + skipReason += + $"{testMethod.TestClass.Class.Name} does not define a cluster through IClusterFixture or {nameof(IntegrationTestClusterAttribute)}"; + return true; + } + + var openSearchVersion = + discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.Version)); + + // Skip if the version we are testing against is attributed to be skipped do not run the test nameof(SkipVersionAttribute.Ranges) + var skipVersionAttribute = Enumerable.FirstOrDefault(GetAttributes(testMethod)); + if (skipVersionAttribute != null) + { + var skipVersionRanges = + skipVersionAttribute.GetNamedArgument>(nameof(SkipVersionAttribute.Ranges)) ?? + new List(); + if (openSearchVersion == null && skipVersionRanges.Count > 0) + { + skipReason = $"{nameof(SkipVersionAttribute)} has ranges defined for this test but " + + $"no {nameof(OpenSearchXunitRunOptions.Version)} has been provided to {nameof(OpenSearchXunitRunOptions)}"; + return true; + } + + if (openSearchVersion != null) + { + var reason = skipVersionAttribute.GetNamedArgument(nameof(SkipVersionAttribute.Reason)); + for (var index = 0; index < skipVersionRanges.Count; index++) + { + var range = skipVersionRanges[index]; + // inrange takes prereleases into account + if (!openSearchVersion.InRange(range)) continue; + skipReason = + $"{nameof(SkipVersionAttribute)} has range {range} that {openSearchVersion} satisfies"; + if (!string.IsNullOrWhiteSpace(reason)) skipReason += $": {reason}"; + return true; + } + } + } + + var skipTests = GetAttributes(testMethod) + .FirstOrDefault(a => a.GetNamedArgument(nameof(SkipTestAttributeBase.Skip))); + + if (skipTests == null) return false; + + skipReason = skipTests.GetNamedArgument(nameof(SkipTestAttributeBase.Reason)); + return true; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/OpenSearchTestCaseDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/OpenSearchTestCaseDiscoverer.cs new file mode 100644 index 0000000000..5dca5c3022 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/OpenSearchTestCaseDiscoverer.cs @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +{ + /// + /// Base test discoverer used to discover tests cases attached + /// to test methods that are attributed with (or a subclass). + /// + public abstract class OpenSearchTestCaseDiscoverer : IXunitTestCaseDiscoverer + { + protected readonly IMessageSink DiagnosticMessageSink; + + protected OpenSearchTestCaseDiscoverer(IMessageSink diagnosticMessageSink) => + DiagnosticMessageSink = diagnosticMessageSink; + + /// + public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, + ITestMethod testMethod, IAttributeInfo factAttribute) => + SkipMethod(discoveryOptions, testMethod, out var skipReason) + ? string.IsNullOrEmpty(skipReason) + ? new IXunitTestCase[] { } + : new IXunitTestCase[] {new SkippingTestCase(skipReason, testMethod, null)} + : new[] + { + new XunitTestCase(DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), + discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod) + }; + + /// + /// Detemines whether a test method should be skipped, and the reason why + /// + /// The discovery options + /// The test method + /// The reason to skip + /// + protected virtual bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + out string skipReason) + { + skipReason = null; + return false; + } + + protected static TValue GetAttribute(ITestMethod testMethod, string propertyName) + where TAttribute : Attribute + { + var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + var attribute = classAttributes.Concat(methodAttributes).FirstOrDefault(); + return attribute == null ? default(TValue) : attribute.GetNamedArgument(propertyName); + } + + protected static IList GetAttributes(ITestMethod testMethod) + where TAttribute : Attribute + { + var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)) ?? + Enumerable.Empty(); + return classAttributes.Concat(methodAttributes).ToList(); + } + + protected static IEnumerable GetAttributes(ITestMethod testMethod, + string propertyName) + where TAttribute : Attribute + { + var classAttributes = testMethod.TestClass.Class.GetCustomAttributes(typeof(TAttribute)); + var methodAttributes = testMethod.Method.GetCustomAttributes(typeof(TAttribute)); + return classAttributes + .Concat(methodAttributes) + .Select(a => a.GetNamedArgument(propertyName)); + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs new file mode 100644 index 0000000000..b34f510f84 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkipVersionAttribute.cs @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using SemVer; + +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +{ + /// + /// An Xunit test that should be skipped for given OpenSearch versions, and a reason why. + /// + public class SkipVersionAttribute : Attribute + { + // ReSharper disable once UnusedParameter.Local + // reason is used to allow the test its used on to self document why its been put in place + public SkipVersionAttribute(string skipVersionRangesSeparatedByComma, string reason) + { + Reason = reason; + Ranges = string.IsNullOrEmpty(skipVersionRangesSeparatedByComma) + ? new List() + : skipVersionRangesSeparatedByComma.Split(',') + .Select(r => r.Trim()) + .Where(r => !string.IsNullOrWhiteSpace(r)) + .Select(r => new Range(r)) + .ToList(); + } + + /// + /// The reason why the test should be skipped + /// + public string Reason { get; } + + /// + /// The version ranges for which the test should be skipped + /// + public IList Ranges { get; } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkippingTestCase.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkippingTestCase.cs new file mode 100644 index 0000000000..bd234f3ef6 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/SkippingTestCase.cs @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +{ + public class SkippingTestCase : TestMethodTestCase, IXunitTestCase + { + /// Used for de-serialization. + public SkippingTestCase() + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The test method this test case belongs to. + /// The arguments for the test method. + public SkippingTestCase(string skipReason, ITestMethod testMethod, object[] testMethodArguments = null) + : base(TestMethodDisplay.ClassAndMethod, TestMethodDisplayOptions.None, testMethod, testMethodArguments) => + SkipReason = skipReason ?? "skipped"; + + public int Timeout => 0; + + /// + public Task RunAsync( + IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + object[] constructorArguments, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) => + new XunitTestCaseRunner( + this, + DisplayName, + SkipReason, + constructorArguments, + TestMethodArguments, + messageBus, + aggregator, + cancellationTokenSource).RunAsync(); + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs new file mode 100644 index 0000000000..eaa8bab6e3 --- /dev/null +++ b/abstractions/src/OpenSearch.OpenSearch.Xunit/XunitPlumbing/UnitTestDiscoverer.cs @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace OpenSearch.OpenSearch.Xunit.XunitPlumbing +{ + /// + /// An Xunit unit test + /// + [XunitTestCaseDiscoverer("OpenSearch.OpenSearch.Xunit.XunitPlumbing.UnitTestDiscoverer", + "OpenSearch.OpenSearch.Xunit")] + public class U : FactAttribute + { + } + + /// + /// A test discoverer used to discover unit tests cases attached + /// to test methods that are attributed with attribute + /// + public class UnitTestDiscoverer : OpenSearchTestCaseDiscoverer + { + public UnitTestDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) + { + } + + /// + protected override bool SkipMethod(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + out string skipReason) + { + skipReason = null; + var runUnitTests = discoveryOptions.GetValue(nameof(OpenSearchXunitRunOptions.RunUnitTests)); + if (!runUnitTests) return true; + + var skipTests = GetAttributes(testMethod) + .FirstOrDefault(a => a.GetNamedArgument(nameof(SkipTestAttributeBase.Skip))); + + if (skipTests == null) return false; + + skipReason = skipTests.GetNamedArgument(nameof(SkipTestAttributeBase.Reason)); + return true; + } + } +} diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/ide-integration.png b/abstractions/src/OpenSearch.OpenSearch.Xunit/ide-integration.png new file mode 100644 index 0000000000..d0923170e0 Binary files /dev/null and b/abstractions/src/OpenSearch.OpenSearch.Xunit/ide-integration.png differ diff --git a/abstractions/src/OpenSearch.OpenSearch.Xunit/output.gif b/abstractions/src/OpenSearch.OpenSearch.Xunit/output.gif new file mode 100644 index 0000000000..34f941dc22 Binary files /dev/null and b/abstractions/src/OpenSearch.OpenSearch.Xunit/output.gif differ diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs new file mode 100644 index 0000000000..9981a443d4 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Artifact.cs @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.IO; +using OpenSearch.Stack.ArtifactsApi.Products; +using OpenSearch.Stack.ArtifactsApi.Resolvers; +using Version = SemVer.Version; + +namespace OpenSearch.Stack.ArtifactsApi +{ + public class Artifact + { + private static readonly Uri BaseUri = new Uri("http://localhost"); + + internal Artifact(Product product, Version version, string downloadUrl, ArtifactBuildState state, + string buildHash) + { + ServerType = product.ServerType; + switch (ServerType) + { + case ServerType.OpenSearch: ProductName = "opensearch"; break; // actually = product.ProductName + case ServerType.OpenDistro: ProductName = "opendistroforelasticsearch"; break; + case ServerType.ElasticSearch: ProductName = "elasticsearch"; break; + } + Version = version; + DownloadUrl = product?.PatchDownloadUrl(downloadUrl); + State = state; + BuildHash = buildHash; + } + + internal Artifact(Product product, Version version, SearchPackage package, string buildHash = null) + { + ProductName = product.ProductName; + Version = version; + State = ArtifactBuildState.Snapshot; + DownloadUrl = product?.PatchDownloadUrl(package.DownloadUrl); + ShaUrl = package.ShaUrl; + AscUrl = package.AscUrl; + BuildHash = buildHash; + } + + public string LocalFolderName + { + get + { + var hashed = string.IsNullOrWhiteSpace(BuildHash) ? string.Empty : $"-build-{BuildHash}"; + switch (State) + { + case ArtifactBuildState.Released: + return $"{ProductName}-{Version}"; + case ArtifactBuildState.Snapshot: + return $"{ProductName}-{Version}{hashed}"; + case ArtifactBuildState.BuildCandidate: + return $"{ProductName}-{Version}{hashed}"; + default: + throw new ArgumentOutOfRangeException(nameof(State), $"{State} not expected here"); + } + } + } + + public string FolderInZip => $"{ProductName}-{Version}"; + + public string Archive + { + get + { + if (!Uri.TryCreate(DownloadUrl, UriKind.Absolute, out var uri)) + uri = new Uri(BaseUri, DownloadUrl); + + return Path.GetFileName(uri.LocalPath); + } + } + + // ReSharper disable UnusedAutoPropertyAccessor.Global + public ServerType ServerType { get; } + public string ProductName { get; } + public Version Version { get; } + public string DownloadUrl { get; } + + public ArtifactBuildState State { get; } + public string BuildHash { get; } + public string ShaUrl { get; } + + public string AscUrl { get; } + // ReSharper restore UnusedAutoPropertyAccessor.Global + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/ArtifactBuildState.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/ArtifactBuildState.cs new file mode 100644 index 0000000000..800ff77e70 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/ArtifactBuildState.cs @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +namespace OpenSearch.Stack.ArtifactsApi +{ + public enum ArtifactBuildState + { + Released, + Snapshot, + BuildCandidate + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearch.Stack.ArtifactsApi.csproj b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearch.Stack.ArtifactsApi.csproj new file mode 100644 index 0000000000..c228e377fd --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearch.Stack.ArtifactsApi.csproj @@ -0,0 +1,15 @@ + + + netstandard2.0;net461 + Provides a set of classes to resolve the location of OpenSearch stack products in various stages: released, snapshot and build candidates + opensearch,opensearch,stack,versioning,artifacts + + + + + + + + + + diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs new file mode 100644 index 0000000000..46eda636a1 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/OpenSearchVersion.cs @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Runtime.InteropServices; +using OpenSearch.Stack.ArtifactsApi.Platform; +using OpenSearch.Stack.ArtifactsApi.Products; +using OpenSearch.Stack.ArtifactsApi.Resolvers; +using SemVer; +using Version = SemVer.Version; + +namespace OpenSearch.Stack.ArtifactsApi +{ + public class OpenSearchVersion : Version, IComparable + { + private readonly ConcurrentDictionary _resolved = new(); + + protected OpenSearchVersion(string version, ArtifactBuildState state, string buildHash = null) : base(version) + { + ArtifactBuildState = state; + BuildHash = buildHash; + } + + public ArtifactBuildState ArtifactBuildState { get; } + private string BuildHash { get; } + + public int CompareTo(string other) + { + var v = (OpenSearchVersion)other; + return CompareTo(v); + } + + public Artifact Artifact(Product product) + { + var cacheKey = product.ToString(); + if (_resolved.TryGetValue(cacheKey, out var artifact)) + return artifact; + var currentPlatform = OsMonikers.CurrentPlatform(); + switch (ArtifactBuildState) + { + case ArtifactBuildState.Released: + ReleasedVersionResolver.TryResolve(product, this, currentPlatform, RuntimeInformation.OSArchitecture, out artifact); + break; + case ArtifactBuildState.Snapshot: + SnapshotApiResolver.TryResolve(product, this, currentPlatform, null, out artifact); + break; + case ArtifactBuildState.BuildCandidate: + StagingVersionResolver.TryResolve(product, this, BuildHash, out artifact); + break; + default: + throw new ArgumentOutOfRangeException(nameof(ArtifactBuildState), + $"{ArtifactBuildState} not expected here"); + } + + _resolved.TryAdd(cacheKey, artifact); + + return artifact; + } + + /// + /// Resolves an OpenSearch version using either format '$version' or '$ServerType-$version', where version could be 'x.y.z' or 'latest' or even 'latest-x' + /// + public static OpenSearchVersion From(string managedVersionString) + { + if (managedVersionString == null) + return null; + + var serverType = ServerType.DEFAULT; + var hasServerType = Enum.GetNames(typeof(ServerType)).Any(s => managedVersionString.StartsWith(s, StringComparison.InvariantCultureIgnoreCase)); + if (hasServerType) + { + var parts = managedVersionString.Split('-'); + serverType = (ServerType)Enum.Parse(typeof(ServerType), parts[0], true); + managedVersionString = string.Join("-", parts.Skip(1)); + } + + if (managedVersionString.StartsWith("latest") && serverType == ServerType.OpenDistro) + // No releases for OpenDistro after 1.13.3 - all deveploment afterwards goes to OpenSearch + managedVersionString = "1.13.3"; + if (managedVersionString.StartsWith("latest") && serverType == ServerType.ElasticSearch) + // OpenDistro (1.13.x - another not supported) and OpenSearch both based on Elasticsearch v.7.10.2, other versions are incompatible + managedVersionString = "7.10.2"; + + // TODO resolve `latest` and `latest-x` for OpenSearch + + return new OpenSearchVersion(managedVersionString, ArtifactBuildState.Released, ""); + } + + internal static bool TryParseBuildCandidate(string passedVersion, out string version, out string gitHash) + { + version = null; + gitHash = null; + var tokens = passedVersion.Split(':'); + if (tokens.Length < 2) + return false; + version = tokens[1].Trim(); + gitHash = tokens[0].Trim(); + return true; + } + + public bool InRange(string range) + { + var versionRange = new Range(range); + return InRange(versionRange); + } + + public bool InRange(Range versionRange) + { + var satisfied = versionRange.IsSatisfied(this); + if (satisfied) + return true; + + //Semver can only match snapshot version with ranges on the same major and minor + //anything else fails but we want to know e.g 1.0.0-SNAPSHOT satisfied by 1.0.0; + var wholeVersion = $"{Major}.{Minor}.{Patch}"; + return versionRange.IsSatisfied(wholeVersion); + } + + + public static implicit operator OpenSearchVersion(string version) => From(version); + + public static bool operator <(OpenSearchVersion first, string second) => first < (OpenSearchVersion)second; + public static bool operator >(OpenSearchVersion first, string second) => first > (OpenSearchVersion)second; + + public static bool operator <(string first, OpenSearchVersion second) => (OpenSearchVersion)first < second; + public static bool operator >(string first, OpenSearchVersion second) => (OpenSearchVersion)first > second; + + public static bool operator <=(OpenSearchVersion first, string second) => first <= (OpenSearchVersion)second; + public static bool operator >=(OpenSearchVersion first, string second) => first >= (OpenSearchVersion)second; + + public static bool operator <=(string first, OpenSearchVersion second) => (OpenSearchVersion)first <= second; + public static bool operator >=(string first, OpenSearchVersion second) => (OpenSearchVersion)first >= second; + + public static bool operator ==(OpenSearchVersion first, string second) => first == (OpenSearchVersion)second; + public static bool operator !=(OpenSearchVersion first, string second) => first != (OpenSearchVersion)second; + + + public static bool operator ==(string first, OpenSearchVersion second) => (OpenSearchVersion)first == second; + public static bool operator !=(string first, OpenSearchVersion second) => (OpenSearchVersion)first != second; + + // ReSharper disable once UnusedMember.Local + private bool Equals(OpenSearchVersion other) => base.Equals(other); + public override bool Equals(object obj) => base.Equals(obj); + + public override int GetHashCode() => base.GetHashCode(); + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Platform/OsMonikers.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Platform/OsMonikers.cs new file mode 100644 index 0000000000..2bf89b59a3 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Platform/OsMonikers.cs @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +namespace OpenSearch.Stack.ArtifactsApi.Platform +{ + internal static class OsMonikers + { + public static readonly string Windows = "windows"; + public static readonly string Linux = "linux"; + public static readonly string OSX = "darwin"; + + public static string From(OSPlatform platform) + { + if (platform == OSPlatform.Windows) return Windows; + if (platform == OSPlatform.Linux) return Linux; + if (platform == OSPlatform.OSX) return OSX; + return "unknown"; + } + + public static OSPlatform CurrentPlatform() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return OSPlatform.Windows; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return OSPlatform.OSX; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return OSPlatform.Linux; + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net-abstractions"); + } + + /// + /// Maps from current os and architecture to a suffix used for tagging platform-specific OpenSearch releases. + /// + /// + public static string GetOpenSearchPlatformMoniker(OSPlatform platform, Architecture architecture) + { + if (platform == OSPlatform.Windows) + { + if (architecture == Architecture.X64) return "windows-x64"; + if (architecture == Architecture.X86) return "windows-x86"; + } + + if (platform == OSPlatform.OSX) + { + if (architecture == Architecture.X64) return "macos-x64"; + if (architecture == Architecture.Arm64) return "macos-arm64"; + + } + + if (platform == OSPlatform.Linux) + { + if (architecture == Architecture.X64) return "linux-x64"; + if (architecture == Architecture.Arm64) return "linux-arm64"; + } + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net-abstractions"); + } + + public static string GetPlatformArchiveExtension(OSPlatform platform) + { + if (platform == OSPlatform.Linux) return "tar.gz"; + if (platform == OSPlatform.OSX) return "tar.gz"; + if (platform == OSPlatform.Windows) return "zip"; + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net-abstractions"); + + } + public static string CurrentPlatformArchiveExtension() + { + var platform = CurrentPlatform(); + return GetPlatformArchiveExtension(platform); + } + + public static string CurrentPlatformPackageSuffix() + { + var intelX86Suffix = "x86_64"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return $"{Windows}-{intelX86Suffix}"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return $"{OSX}-{intelX86Suffix}"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return $"{Linux}-{intelX86Suffix}"; + + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net-abstractions"); + } + + internal static string CurrentPlatformSearchFilter() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return "zip"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) return "tar"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "tar"; + + throw new Exception( + $"{RuntimeInformation.OSDescription} is currently not supported please open an issue @opensearch-project/opensearch-net-abstractions"); + } + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs new file mode 100644 index 0000000000..e8ff1e5554 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/OpenSearchPlugin.cs @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; + +namespace OpenSearch.Stack.ArtifactsApi.Products +{ + /// An OpenSearch plugin + public class OpenSearchPlugin : SubProduct + { + public OpenSearchPlugin(string plugin, Func isValid = null, + Func listName = null) + : base(plugin, isValid, listName) + { + PlatformDependent = false; + PatchDownloadUrl = s => + { + //Temporary correct plugin download urls as reported by the snapshot API as it currently has a bug + var correct = $"downloads/opensearch-plugins/{plugin}"; + return !s.Contains(correct) ? s.Replace("downloads/opensearch", correct) : s; + }; + } // ReSharper disable InconsistentNaming + public static OpenSearchPlugin AnalysisIcu { get; } = new OpenSearchPlugin("analysis-icu"); + public static OpenSearchPlugin AnalysisKuromoji { get; } = new OpenSearchPlugin("analysis-kuromoji"); + public static OpenSearchPlugin AnalysisPhonetic { get; } = new OpenSearchPlugin("analysis-phonetic"); + public static OpenSearchPlugin AnalysisSmartCn { get; } = new OpenSearchPlugin("analysis-smartcn"); + public static OpenSearchPlugin AnalysisStempel { get; } = new OpenSearchPlugin("analysis-stempel"); + public static OpenSearchPlugin AnalysisUkrainian { get; } = new OpenSearchPlugin("analysis-ukrainian"); + + public static OpenSearchPlugin DiscoveryAzureClassic { get; } = + new OpenSearchPlugin("discovery-azure-classic"); + + public static OpenSearchPlugin DiscoveryEC2 { get; } = new OpenSearchPlugin("discovery-ec2"); + public static OpenSearchPlugin DiscoveryFile { get; } = new OpenSearchPlugin("discovery-file"); + public static OpenSearchPlugin DiscoveryGCE { get; } = new OpenSearchPlugin("discovery-gce"); + + public static OpenSearchPlugin IngestAttachment { get; } = + new OpenSearchPlugin("ingest-attachment", version => version >= "1.0.0"); + + public static OpenSearchPlugin IngestGeoIp { get; } = + new OpenSearchPlugin("ingest-geoip", version => version >= "1.0.0") + { + ShippedByDefaultAsOf = "1.0.0" + }; + + public static OpenSearchPlugin IngestUserAgent { get; } = + new OpenSearchPlugin("ingest-user-agent", version => version >= "1.0.0") + { + ShippedByDefaultAsOf = "1.0.0" + }; + + public static OpenSearchPlugin MapperAttachment { get; } = new OpenSearchPlugin("mapper-attachments"); + public static OpenSearchPlugin MapperMurmur3 { get; } = new OpenSearchPlugin("mapper-murmur3"); + public static OpenSearchPlugin MapperSize { get; } = new OpenSearchPlugin("mapper-size"); + + public static OpenSearchPlugin RepositoryAzure { get; } = new OpenSearchPlugin("repository-azure"); + public static OpenSearchPlugin RepositoryGCS { get; } = new OpenSearchPlugin("repository-gcs"); + public static OpenSearchPlugin RepositoryHDFS { get; } = new OpenSearchPlugin("repository-hdfs"); + public static OpenSearchPlugin RepositoryS3 { get; } = new OpenSearchPlugin("repository-s3"); + + public static OpenSearchPlugin StoreSMB { get; } = new OpenSearchPlugin("store-smb"); + + public static OpenSearchPlugin DeleteByQuery { get; } = + new OpenSearchPlugin("delete-by-query", version => version < "1.0.0"); + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs new file mode 100644 index 0000000000..e0d9187a41 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/Product.cs @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System.Collections.Concurrent; +using OpenSearch.Stack.ArtifactsApi.Platform; + +namespace OpenSearch.Stack.ArtifactsApi.Products +{ + public class Product + { + private static readonly ConcurrentDictionary CachedProducts = + new ConcurrentDictionary(); + + private static readonly OpenSearchVersion DefaultIncludePlatformSuffix = OpenSearchVersion.From("1.0.0"); + + private Product(string productName) => ProductName = productName; + + protected Product(string productName, SubProduct relatedProduct, ServerType serverType, + OpenSearchVersion platformVersionSuffixAfter = null) : this(productName) + { + SubProduct = relatedProduct; + ServerType = serverType; + PlatformSuffixAfter = platformVersionSuffixAfter ?? DefaultIncludePlatformSuffix; + } + + public SubProduct SubProduct { get; } + + public string Moniker => SubProduct?.SubProductName ?? ProductName; + + public string Extension => PlatformDependent ? OsMonikers.CurrentPlatformArchiveExtension() : "zip"; + + public string ProductName { get; } + + public ServerType ServerType { get; } + + public bool PlatformDependent => SubProduct?.PlatformDependent ?? true; + + public OpenSearchVersion PlatformSuffixAfter { get; } + + public static Product OpenSearch { get; } = From("opensearch"); + + public static Product OpenSearchDashboards { get; } = From("opensearch-dashboards", platformInZipAfter: "1.0.0"); + + public static Product From(string product, SubProduct subProduct = null, ServerType serverType = ServerType.DEFAULT, + OpenSearchVersion platformInZipAfter = null) => + CachedProducts.GetOrAdd(subProduct == null ? $"{product}-{serverType}" : $"{product}/{subProduct.SubProductName}", + k => new Product(product, subProduct, serverType, platformInZipAfter)); + + public static Product OpenSearchPlugin(OpenSearchPlugin plugin) => From("opensearch-plugins", plugin); + + public override string ToString() => + SubProduct != null ? $"{ProductName}/{SubProduct.SubProductName}" : ProductName; + + public string PatchDownloadUrl(string downloadUrl) => SubProduct?.PatchDownloadUrl(downloadUrl) ?? downloadUrl; + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs new file mode 100644 index 0000000000..39a51e50e2 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Products/SubProduct.cs @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; + +namespace OpenSearch.Stack.ArtifactsApi.Products +{ + public class SubProduct + { + private readonly Func _getExistsMoniker; + + private readonly Func _isValid; + + public SubProduct(string subProject, Func isValid = null, + Func listName = null) + { + SubProductName = subProject; + _isValid = isValid ?? (v => true); + _getExistsMoniker = listName ?? (v => subProject); + } + + public string SubProductName { get; } + + public OpenSearchVersion ShippedByDefaultAsOf { get; set; } + + /// + /// Temporary, snapshot API reports bad plugin download urls + /// + public Func PatchDownloadUrl { get; set; } = s => s; + + public bool PlatformDependent { get; protected set; } + + /// what moniker to use when asserting the sub product is already present + public string GetExistsMoniker(OpenSearchVersion version) => _getExistsMoniker(version); + + /// Whether the sub project is included in the distribution out of the box for the given version + public bool IsIncludedOutOfTheBox(OpenSearchVersion version) => + ShippedByDefaultAsOf != null && version >= ShippedByDefaultAsOf; + + /// Whether the subProject is valid for the given version + public bool IsValid(OpenSearchVersion version) => IsIncludedOutOfTheBox(version) || _isValid(version); + + public override string ToString() => SubProductName; + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/README.md b/abstractions/src/OpenSearch.Stack.ArtifactsApi/README.md new file mode 100644 index 0000000000..fed65b8b53 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/README.md @@ -0,0 +1,57 @@ +# OpenSearch.Stack.ArtifactsApi + +Library to fetch the url and metadata for released artifacts. + +Supports: + +1. Snapshots builds + * `latest-MAJOR` where `MAJOR` is a single integer representing the major you want a snapshot for + * `latest` latest greatest + +3. Released versions + * `MAJOR.MINOR.PATH` where `MAJOR` is still supported as defined by the EOL policy of OpenSearch. + * Note if the version exists but is not yet released it will resolve as a build candidate + + +## Usage + +First create an opensearch version + +```csharp +var version = OpenSearchVersion.From(versionString); +``` + +Where `versionString` is a string in the aforementioned formats. `version.ArtifactBuildState` represents the type of version parsed. + +```csharp +var version = OpenSearchVersion.From(versionString); +``` + +To go from a version to an artifact do the following + +```csharp +var product = Product.From("opensearch"); +var artifact = version.Artifact(product); +``` +By first creating a `product` we can then pass that `product` to `version.Artifact` to get an artifact to that product's version. + +A product can be a main product such as `opensearch` or a related product e.g + +```csharp +var product = Product.From("opensearch-plugins", "analysis-icu"); +var artifact = version.Artifact(product); +``` + +To aid with discoverability we ship with some statics so you do not need to guess the right monikers. + +```csharp +Product.OpenSearch; +Product.OpenSearchDashboards; +Product.OpenSearchPlugin(OpenSearchPlugin.AnalysisIcu); +``` + + + + + + diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ApiResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ApiResolver.cs new file mode 100644 index 0000000000..bf60576aec --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ApiResolver.cs @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Authentication; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +namespace OpenSearch.Stack.ArtifactsApi.Resolvers +{ + public static class ApiResolver + { + // TODO: update string when working on artifacts API + private const string ArtifactsApiUrl = "https://artifacts-api.opensearch.org/v1/"; + + private static readonly ConcurrentDictionary Releases = new ConcurrentDictionary(); + + private static HttpClient HttpClient { get; } = + new HttpClient(new HttpClientHandler {SslProtocols = SslProtocols.Tls12}) + { + BaseAddress = new Uri(ArtifactsApiUrl) + }; + + private static Regex BuildHashRegex { get; } = + // TODO: update string when working on artifacts API + new Regex(@"https://(?:snapshots|staging).opensearch.org/(\d+\.\d+\.\d+-([^/]+)?)"); + + public static string FetchJson(string path) + { + using (var stream = HttpClient.GetStreamAsync(path).GetAwaiter().GetResult()) + using (var fileStream = new StreamReader(stream)) + return fileStream.ReadToEnd(); + } + + public static bool IsReleasedVersion(string version) + { + if (Releases.TryGetValue(version, out var released)) return released; + var versionPath = "https://github.com/opensearch-project/opensearch/releases/tag/" + version; + var message = new HttpRequestMessage {Method = HttpMethod.Head, RequestUri = new Uri(versionPath)}; + + using (var response = HttpClient.SendAsync(message).GetAwaiter().GetResult()) + { + released = response.IsSuccessStatusCode; + Releases.TryAdd(version, released); + return released; + } + } + + public static string LatestBuildHash(string version) + { + var json = FetchJson($"search/{version}/msi"); + try + { + // if packages is empty it turns into an array[] otherwise its a dictionary :/ + var packages = JsonSerializer.Deserialize(json).Packages; + if (packages.Count == 0) + throw new Exception("Can not get build hash for: " + version); + return GetBuildHash(packages.First().Value.DownloadUrl); + } + catch + { + throw new Exception("Can not get build hash for: " + version); + } + } + + public static string GetBuildHash(string url) + { + var tokens = BuildHashRegex.Split(url).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray(); + if (tokens.Length < 2) + throw new Exception("Can not parse build hash from: " + url); + + return tokens[1]; + } + } + + internal class ArtifactsVersionsResponse + { + [JsonPropertyName("versions")] public List Versions { get; set; } + } + + internal class ArtifactsSearchResponse + { + [JsonPropertyName("packages")] public Dictionary Packages { get; set; } + } + + internal class SearchPackage + { + [JsonPropertyName("url")] public string DownloadUrl { get; set; } + [JsonPropertyName("sha_url")] public string ShaUrl { get; set; } + [JsonPropertyName("asc_url")] public string AscUrl { get; set; } + [JsonPropertyName("type")] public string Type { get; set; } + [JsonPropertyName("architecture")] public string Architecture { get; set; } + [JsonPropertyName("os")] public string[] OperatingSystem { get; set; } + [JsonPropertyName("classifier")] public string Classifier { get; set; } + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs new file mode 100644 index 0000000000..3b80e1d74b --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/ReleasedVersionResolver.cs @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Runtime.InteropServices; +using OpenSearch.Stack.ArtifactsApi.Platform; +using OpenSearch.Stack.ArtifactsApi.Products; +using Version = SemVer.Version; + +namespace OpenSearch.Stack.ArtifactsApi.Resolvers +{ + public static class ReleasedVersionResolver + { + private const string ArtifactsUrl = "https://artifacts.opensearch.org"; + + public static bool TryResolve(Product product, Version version, OSPlatform platform, Architecture architecture, out Artifact artifact) + { + var downloadUrl = ""; + switch (product.ServerType) + { + case ServerType.OpenSearch: + var productMoniker = product.Moniker; + var platformMoniker = OsMonikers.GetOpenSearchPlatformMoniker(platform, architecture); + var downloadPath = $"{ArtifactsUrl}/releases/bundle/{product}/{version}"; + var extension = OsMonikers.GetPlatformArchiveExtension(platform); + var archive = $"{productMoniker}-{version}-{platformMoniker}.{extension}"; + downloadUrl = $"{downloadPath}/{archive}"; + break; + case ServerType.OpenDistro: + if (platform == OSPlatform.Linux) + { + // Only version 1.13.x supported, because it is based on elasticsearch 7.10.2: https://opendistro.github.io/for-elasticsearch-docs/version-history/ + if (version < new Version("1.13.0")) + throw new ArgumentOutOfRangeException($"This OpenDistro version {version} is not supported, only 1.13.0 to 1.13.3 are supported"); + downloadUrl = $"https://d3g5vo6xdbdb9a.cloudfront.net/tarball/opendistro-elasticsearch/opendistroforelasticsearch-{version}-linux-x64.{product.Extension}"; + } + else if (platform == OSPlatform.Windows) + { + // No Windows package for version < 1.3.0 + if (version < new Version("1.13.0")) + throw new ArgumentOutOfRangeException($"This OpenDistro version {version} is not supported, only 1.13.0 to 1.13.3 are supported"); + downloadUrl = $"https://d3g5vo6xdbdb9a.cloudfront.net/downloads/odfe-windows/ode-windows-zip/opendistroforelasticsearch-{version}-windows-x64.{product.Extension}"; + } + else + throw new ArgumentOutOfRangeException($"OS {platform} is not supported by OpenDistro"); + break; + case ServerType.ElasticSearch: + // Only Elasticsearch v.7.10.2 is supported + if (version != new Version("7.10.2")) + throw new ArgumentOutOfRangeException($"Elasticsearch version {version} is not supported, the only supported version is 7.10.2"); + if (platform == OSPlatform.Windows) downloadUrl = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.2-windows-x86_64.zip"; + if (platform == OSPlatform.Linux) downloadUrl = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.2-linux-x86_64.tar.gz"; + if (platform == OSPlatform.OSX) downloadUrl = "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.10.2-darwin-x86_64.tar.gz"; + break; + } + + artifact = new Artifact(product, version, downloadUrl, ArtifactBuildState.Released, null); + return true; + } + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/SnapshotApiResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/SnapshotApiResolver.cs new file mode 100644 index 0000000000..e773bbdd25 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/SnapshotApiResolver.cs @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using OpenSearch.Stack.ArtifactsApi.Platform; +using OpenSearch.Stack.ArtifactsApi.Products; +using SemVer; +using Version = SemVer.Version; + +namespace OpenSearch.Stack.ArtifactsApi.Resolvers +{ + public static class SnapshotApiResolver + { + public static readonly System.Lazy> AvailableVersions = + new System.Lazy>(LoadVersions, LazyThreadSafetyMode.ExecutionAndPublication); + + private static Regex PackageProductRegex { get; } = + new Regex(@"(.*?)-(\d+\.\d+\.\d+(?:-(?:SNAPSHOT|alpha\d+|beta\d+|rc\d+))?)"); + + private static Version IncludeOsMoniker { get; } = new Version("1.0.0"); + + public static Version LatestReleaseOrSnapshot => AvailableVersions.Value.OrderByDescending(v => v).First(); + + public static bool TryResolve(Product product, Version version, OSPlatform os, string filters, + out Artifact artifact) + { + artifact = null; + var p = product.SubProduct?.SubProductName ?? product.ProductName; + var query = p; + if (product.PlatformDependent && version > product.PlatformSuffixAfter) + query += $",{OsMonikers.From(os)}"; + else if (product.PlatformDependent) + query += $",{OsMonikers.CurrentPlatformSearchFilter()}"; + if (!string.IsNullOrWhiteSpace(filters)) + query += $",{filters}"; + + var packages = new Dictionary(); + try + { + var json = ApiResolver.FetchJson($"search/{version}/{query}"); + // if packages is empty it turns into an array[] otherwise its a dictionary :/ + packages = JsonSerializer.Deserialize(json).Packages; + } + catch + { + } + + if (packages == null || packages.Count == 0) return false; + var list = packages + .OrderByDescending(k => k.Value.Classifier == null ? 1 : 0) + .ToArray(); + + var ext = OsMonikers.CurrentPlatformArchiveExtension(); + var shouldEndWith = $"{version}.{ext}"; + if (product.PlatformDependent && version > product.PlatformSuffixAfter) + shouldEndWith = $"{version}-{OsMonikers.CurrentPlatformPackageSuffix()}.{ext}"; + foreach (var kv in list) + { + if (product.PlatformDependent && !kv.Key.EndsWith(shouldEndWith)) continue; + + + var tokens = PackageProductRegex.Split(kv.Key).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray(); + if (tokens.Length < 2) continue; + + if (!tokens[0].Equals(p, StringComparison.CurrentCultureIgnoreCase)) continue; + if (!tokens[1].Equals(version.ToString(), StringComparison.CurrentCultureIgnoreCase)) continue; + var buildHash = ApiResolver.GetBuildHash(kv.Value.DownloadUrl); + artifact = new Artifact(product, version, kv.Value, buildHash); + } + + return false; + } + + + private static IReadOnlyCollection LoadVersions() + { + var json = ApiResolver.FetchJson("versions"); + var versions = JsonSerializer.Deserialize(json).Versions; + + return new List(versions.Select(v => new Version(v))); + } + + public static Version LatestSnapshotForMajor(int major) + { + var range = new Range($"~{major}"); + return AvailableVersions.Value + .Reverse() + .FirstOrDefault(v => + v.PreRelease == "SNAPSHOT" && range.IsSatisfied(v.ToString().Replace("-SNAPSHOT", ""))); + } + + public static Version LatestReleaseOrSnapshotForMajor(int major) + { + var range = new Range($"~{major}"); + return AvailableVersions.Value + .Reverse() + .FirstOrDefault(v => range.IsSatisfied(v.ToString().Replace("-SNAPSHOT", ""))); + } + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/StagingVersionResolver.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/StagingVersionResolver.cs new file mode 100644 index 0000000000..06dd757c39 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/Resolvers/StagingVersionResolver.cs @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +using System; +using OpenSearch.Stack.ArtifactsApi.Platform; +using OpenSearch.Stack.ArtifactsApi.Products; +using Version = SemVer.Version; + +namespace OpenSearch.Stack.ArtifactsApi.Resolvers +{ + public static class StagingVersionResolver + { + public static bool TryResolve(Product product, Version version, string buildHash, out Artifact artifact) + { + artifact = null; + // Tests are not supposed to run on any staging server except OpenSearch + if (product.ServerType != ServerType.OpenSearch) + throw new ArgumentOutOfRangeException($"Tests are not supposed to run on staging {product.ServerType} regardless of version"); + + // Only Linux and only x64 is supported now + var downloadUrl = $"https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/{version}/latest/linux/x64/dist/opensearch/opensearch-{version}-linux-x64.{product.Extension}"; + artifact = new Artifact(product, version, downloadUrl, ArtifactBuildState.BuildCandidate, buildHash); + return true; + } + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApi/ServerType.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApi/ServerType.cs new file mode 100644 index 0000000000..5f41261996 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApi/ServerType.cs @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +namespace OpenSearch.Stack.ArtifactsApi +{ + public enum ServerType + { + OpenSearch, + OpenDistro, + ElasticSearch, + DEFAULT = OpenSearch + } +} diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApiTests/OpenSearch.Stack.ArtifactsApiTests.csproj b/abstractions/src/OpenSearch.Stack.ArtifactsApiTests/OpenSearch.Stack.ArtifactsApiTests.csproj new file mode 100644 index 0000000000..9a1e672dce --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApiTests/OpenSearch.Stack.ArtifactsApiTests.csproj @@ -0,0 +1,25 @@ + + + + + net5 + enable + + false + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/abstractions/src/OpenSearch.Stack.ArtifactsApiTests/ReleasedVersionResolverTests.cs b/abstractions/src/OpenSearch.Stack.ArtifactsApiTests/ReleasedVersionResolverTests.cs new file mode 100644 index 0000000000..21d5389582 --- /dev/null +++ b/abstractions/src/OpenSearch.Stack.ArtifactsApiTests/ReleasedVersionResolverTests.cs @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +* +* Modifications Copyright OpenSearch Contributors. See +* GitHub history for details. +* +* Licensed to Elasticsearch B.V. under one or more contributor +* license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright +* ownership. Elasticsearch B.V. licenses this file to you under +* the Apache License, Version 2.0 (the "License"); you may +* not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + + +using System; +using System.Net; +using System.Runtime.InteropServices; +using OpenSearch.Stack.ArtifactsApi; +using OpenSearch.Stack.ArtifactsApi.Products; +using OpenSearch.Stack.ArtifactsApi.Resolvers; +using Xunit; +using Xunit.Abstractions; + +namespace OpenSearch.Stack.ArtifactsApiTests +{ + public class ReleasedVersionResolverTests + { + public ReleasedVersionResolverTests(ITestOutputHelper traceSink) => _traceSink = traceSink ?? throw new NullReferenceException(nameof(traceSink)); + + private readonly ITestOutputHelper _traceSink; + [Fact] + public void Does_Resolver_Construct_Valid_DownloadUrl_Test() + { + var testCases = new[] + { + new {Product = Product.OpenSearch, Version = "1.2.3", Platform = OSPlatform.Linux, Architecture = Architecture.X64}, + new {Product = Product.OpenSearch, Version = "1.2.4", Platform = OSPlatform.Linux, Architecture = Architecture.Arm64}, + new {Product = Product.OpenSearch, Version = "1.0.0", Platform = OSPlatform.Linux, Architecture = Architecture.Arm64} + }; + foreach (var testCase in testCases) + { + var version = OpenSearchVersion.From(testCase.Version); + var resolveSucceeded = ReleasedVersionResolver.TryResolve(testCase.Product, version, testCase.Platform, + testCase.Architecture, out var artifact); + Assert.True(resolveSucceeded); + _traceSink.WriteLine($"Checking URL {artifact.DownloadUrl}"); + var downloadRequest = WebRequest.CreateHttp(new Uri(artifact.DownloadUrl)); + downloadRequest.Timeout = 3 * 1000; // Timeout of 3 seconds. + // HTTP HEAD request can be used to check if a web resource exists without getting its content. + // See here for more details https://hc.apache.org/httpclient-legacy/methods/head.html + downloadRequest.Method = "HEAD"; + var response = (HttpWebResponse)downloadRequest.GetResponse(); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + } + } + } +}