From 3e4c55ee0ee4635519532b9b6e20b8def5f6dfb9 Mon Sep 17 00:00:00 2001 From: Robert E Date: Sun, 18 Aug 2024 17:30:56 +1000 Subject: [PATCH] Try running K8S tests using local kind --- .../Helpers/CalamariResult.cs | 4 +- source/Calamari.Tests/Calamari.Tests.csproj | 9 + .../KubernetesClusterOneTimeSetUp.cs | 39 +++ .../KubernetesCommandTests.cs | 159 ++++++++++ ...etesContextScriptWrapperLiveFixtureBase.cs | 9 +- ...netesContextScriptWrapperLiveFixtureEks.cs | 288 ------------------ .../KubernetesLiveStatusCheckTests.cs | 185 +++++++++++ .../Tools/IToolDownloader.cs | 271 ++++++++++++++++ .../Tools/KindDownloader.cs | 41 +++ .../Tools/KubeCtlDownloader.cs | 38 +++ .../Tools/KubernetesClusterInstaller.cs | 226 ++++++++++++++ .../Tools/KubernetesTestsGlobalContext.cs | 44 +++ .../Tools/docker-desktop-network-routing.yml | 8 + .../KubernetesFixtures/Tools/kind-config.yml | 8 + 14 files changed, 1033 insertions(+), 296 deletions(-) create mode 100644 source/Calamari.Tests/KubernetesFixtures/KubernetesClusterOneTimeSetUp.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/KubernetesCommandTests.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/KubernetesLiveStatusCheckTests.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/Tools/IToolDownloader.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/Tools/KindDownloader.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/Tools/KubeCtlDownloader.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesClusterInstaller.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesTestsGlobalContext.cs create mode 100644 source/Calamari.Tests/KubernetesFixtures/Tools/docker-desktop-network-routing.yml create mode 100644 source/Calamari.Tests/KubernetesFixtures/Tools/kind-config.yml diff --git a/source/Calamari.Testing/Helpers/CalamariResult.cs b/source/Calamari.Testing/Helpers/CalamariResult.cs index 6b4c30073..2f9c918aa 100644 --- a/source/Calamari.Testing/Helpers/CalamariResult.cs +++ b/source/Calamari.Testing/Helpers/CalamariResult.cs @@ -140,14 +140,14 @@ public void AssertNoOutput(string expectedOutput) public void AssertOutput(string expectedOutput) { - var allOutput = string.Join(Environment.NewLine, captured.Infos); + var allOutput = string.Join(Environment.NewLine, captured.AllMessages); Assert.That(allOutput, Does.Contain(expectedOutput)); } public void AssertOutputContains(string expected) { - var allOutput = string.Join(Environment.NewLine, captured.Infos); + var allOutput = string.Join(Environment.NewLine, captured.AllMessages); allOutput.Should().Contain(expected); } diff --git a/source/Calamari.Tests/Calamari.Tests.csproj b/source/Calamari.Tests/Calamari.Tests.csproj index 0aeed4500..bc5d3d437 100644 --- a/source/Calamari.Tests/Calamari.Tests.csproj +++ b/source/Calamari.Tests/Calamari.Tests.csproj @@ -11,6 +11,7 @@ win-x64;linux-x64;osx-x64;linux-arm;linux-arm64 net462;net6.0 + 8 $(DefineConstants);NETCORE;AWS;AZURE_CORE;JAVA_SUPPORT @@ -167,6 +168,14 @@ PreserveNewest + + + PreserveNewest + + + + PreserveNewest + diff --git a/source/Calamari.Tests/KubernetesFixtures/KubernetesClusterOneTimeSetUp.cs b/source/Calamari.Tests/KubernetesFixtures/KubernetesClusterOneTimeSetUp.cs new file mode 100644 index 000000000..3e6d0b534 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/KubernetesClusterOneTimeSetUp.cs @@ -0,0 +1,39 @@ +#if NETCORE +using System; +using System.Threading; +using System.Threading.Tasks; +using Calamari.Tests.KubernetesFixtures.Tools; +using NUnit.Framework; + +namespace Calamari.Tests.KubernetesFixtures +{ + [SetUpFixture] + public class KubernetesClusterOneTimeSetUp + { + KubernetesClusterInstaller installer; + [OneTimeSetUp] + public async Task OneTimeSetUp() + { + var toolDownloader = new RequiredToolDownloader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger); + var (kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None); + + installer = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger); + await installer.Install(); + + KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath); + KubernetesTestsGlobalContext.Instance.KubeConfigPath = installer.KubeConfigPath; + + var details = installer.ExtractLoginDetails(); + KubernetesTestsGlobalContext.Instance.ClusterUser = details.ClusterUser; + KubernetesTestsGlobalContext.Instance.ClusterEndpoint = details.ClusterEndpoint; + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + installer.Dispose(); + KubernetesTestsGlobalContext.Instance.Dispose(); + } + } +} +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/KubernetesCommandTests.cs b/source/Calamari.Tests/KubernetesFixtures/KubernetesCommandTests.cs new file mode 100644 index 000000000..6473cf415 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/KubernetesCommandTests.cs @@ -0,0 +1,159 @@ +#if NETCORE +using System; +using System.Collections.Generic; +using System.IO; +using Calamari.Commands; +using Calamari.Common.Features.Scripts; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Variables; +using Calamari.Kubernetes; +using Calamari.Kubernetes.Commands; +using Calamari.Testing.Helpers; +using Calamari.Tests.Helpers; +using Calamari.Tests.KubernetesFixtures.Tools; +using JetBrains.Annotations; +using NUnit.Framework; + +namespace Calamari.Tests.KubernetesFixtures +{ + internal static class VariablesExtensionMethods + { + public static void SetInlineScriptVariables(this IVariables variables, string bashScript, string powershellScript) + { + variables.Set(Deployment.SpecialVariables.Action.Script.ScriptBodyBySyntax(ScriptSyntax.Bash), bashScript); + variables.Set(Deployment.SpecialVariables.Action.Script.ScriptBodyBySyntax(ScriptSyntax.PowerShell), powershellScript); + } + + public static void SetAuthenticationDetails(this IVariables variables) + { + variables.Set(SpecialVariables.ClientCertificate, "UserCert"); + variables.Set(SpecialVariables.CertificatePem("UserCert"), System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(KubernetesTestsGlobalContext.Instance.ClusterUser.ClientCertPem))); + variables.Set(SpecialVariables.PrivateKeyPem("UserCert"), System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(KubernetesTestsGlobalContext.Instance.ClusterUser.ClientCertKey))); + } + + public static void SetClusterDetails(this IVariables variables) + { + variables.Set(SpecialVariables.ClusterUrl, KubernetesTestsGlobalContext.Instance.ClusterEndpoint.ClusterUrl); + variables.Set(SpecialVariables.CertificateAuthority, "ClientCert"); + variables.Set(SpecialVariables.CertificatePem("ClientCert"), System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(KubernetesTestsGlobalContext.Instance.ClusterEndpoint.ClusterCert))); + } + + public static void SetVariablesForKubernetesResourceStatusCheck(this IVariables variables, int timeout = 30, string deploymentWait = "wait") + { + variables.Set("Octopus.Action.Kubernetes.ResourceStatusCheck", "True"); + variables.Set(SpecialVariables.DeploymentWait, deploymentWait); + variables.Set("Octopus.Action.Kubernetes.DeploymentTimeout", timeout.ToString()); + variables.Set("Octopus.Action.Kubernetes.PrintVerboseKubectlOutputOnError", "True"); + } + } + + [TestFixture] + public class KubernetesCommandTests : CalamariFixture + { + [Test] + public void GivenInvalidYaml_ShouldFail() + { + var variables = new CalamariVariables(); + variables.SetClusterDetails(); + variables.SetAuthenticationDetails(); + variables.SetVariablesForKubernetesResourceStatusCheck(); + var ns = CreateString(); + variables.Set(SpecialVariables.Namespace, ns); + + const string failToDeploymentResource = @"apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + invalid-spec: this-is-not-valid + containers: + - name: nginx + image: nginx"; + + var output = ExecuteRawYamlCommand(variables, failToDeploymentResource); + + output.AssertFailure(); + output.AssertOutputContains("Pod in version \"v1\" cannot be handled as a Pod: strict decoding error: unknown field \"spec.invalid-spec\""); + //$"error: error parsing {fileName}: error converting YAML to JSON: yaml: line 7: could not find expected ':'"; + } + + [Test] + public void GivenKubectlScript_ShouldExecute() + { + var variables = new CalamariVariables(); + variables.SetClusterDetails(); + variables.SetAuthenticationDetails(); + variables.Set(SpecialVariables.Namespace, "default"); + + var sampleMessage = CreateString(); + var cmd = $"echo \"{sampleMessage}\"{Environment.NewLine}kubectl cluster-info"; + variables.SetInlineScriptVariables(cmd,cmd); + var output = ExecuteCommand(variables, RunScriptCommand.Name, null); + + output.AssertSuccess(); + output.AssertOutputContains(sampleMessage); + output.AssertOutputContains($"Kubernetes control plane is running at {KubernetesTestsGlobalContext.Instance.ClusterEndpoint.ClusterUrl}"); + } + + CalamariResult ExecuteRawYamlCommand(CalamariVariables variables, string yaml) + { + variables.Set(SpecialVariables.CustomResourceYamlFileName, "**/*.{yml,yaml}"); + using var workingDirectory = TemporaryDirectory.Create(); + CreateResourceYamlFile(workingDirectory.DirectoryPath, DeploymentFileName, yaml); + variables.Set(SpecialVariables.CustomResourceYamlFileName, DeploymentFileName); + variables.Set(KnownVariables.OriginalPackageDirectoryPath, workingDirectory.DirectoryPath); + + var output = ExecuteCommand(variables, KubernetesApplyRawYamlCommand.Name, workingDirectory.DirectoryPath); + return output; + } + + private static string CreateResourceYamlFile(string directory, string fileName, string content) + { + var pathToCustomResource = Path.Combine(directory, fileName); + File.WriteAllText(pathToCustomResource, content); + return pathToCustomResource; + } + private Func CreateAddCustomResourceFileFunc(IVariables variables, string yamlContent) + { + return directory => + { + CreateResourceYamlFile(directory, DeploymentFileName, yamlContent); + if (!variables.IsSet(SpecialVariables.CustomResourceYamlFileName)) + { + variables.Set(SpecialVariables.CustomResourceYamlFileName, DeploymentFileName); + } + return null; + }; + } + + string CreateString() + { + return $"Test{Guid.NewGuid().ToString("N").Substring(0, 10)}".ToLower(); + } + + + CalamariResult ExecuteCommand(IVariables variables, string command, [CanBeNull] string workingDirectory) + { + using var variablesFile = new TemporaryFile(Path.GetTempFileName()); + variables.Save(variablesFile.FilePath); + + var calamariCommand = Calamari().Action(command) + .Argument("variables", variablesFile.FilePath) + .WithEnvironmentVariables(new Dictionary()) + .OutputToLog(true); + + if (workingDirectory != null) + { + calamariCommand = calamariCommand.WithWorkingDirectory(workingDirectory); + } + + return InvokeInProcess(calamariCommand, variables); + } + + + private const string DeploymentFileName = "customresource.yml"; + + } + +} +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureBase.cs b/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureBase.cs index 588e34cc4..6b74636a1 100644 --- a/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureBase.cs +++ b/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureBase.cs @@ -6,21 +6,18 @@ using Calamari.Commands; using Calamari.Common.Features.Discovery; using Calamari.Common.Features.Scripts; -using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.ServiceMessages; using Calamari.Common.Plumbing.Variables; +using Calamari.Deployment; using Calamari.Kubernetes.Commands; using Calamari.Testing.Helpers; using Calamari.Tests.Helpers; using FluentAssertions; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; using NUnit.Framework; using KubernetesSpecialVariables = Calamari.Kubernetes.SpecialVariables; -using SpecialVariables = Calamari.Deployment.SpecialVariables; namespace Calamari.Tests.KubernetesFixtures { @@ -153,9 +150,9 @@ private void SetInlineScriptVariables(string script) private void SetInlineScriptVariables(string bashScript, string powershellScript) { variables.Set(SpecialVariables.Action.Script.ScriptBodyBySyntax(ScriptSyntax.Bash), - bashScript); + bashScript); variables.Set(SpecialVariables.Action.Script.ScriptBodyBySyntax(ScriptSyntax.PowerShell), - powershellScript); + powershellScript); } private CalamariResult ExecuteCommand(string command, string workingDirectory, string packagePath) diff --git a/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureEks.cs b/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureEks.cs index ad1b5fbcc..14ee72ac1 100644 --- a/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureEks.cs +++ b/source/Calamari.Tests/KubernetesFixtures/KubernetesContextScriptWrapperLiveFixtureEks.cs @@ -1,28 +1,16 @@ #if NETCORE using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; -using Assent; using Calamari.Aws.Kubernetes.Discovery; -using Calamari.CloudAccounts; -using Calamari.Common.FeatureToggles; -using Calamari.Common.Plumbing.Variables; using Calamari.Deployment; using Calamari.Kubernetes.Commands; using Calamari.Testing; using Calamari.Testing.Helpers; -using Calamari.Tests.AWS; -using Calamari.Tests.Helpers; using FluentAssertions; using Newtonsoft.Json.Linq; using NUnit.Framework; -using SharpCompress.Archives.Zip; -using SharpCompress.Common; -using File = System.IO.File; using KubernetesSpecialVariables = Calamari.Kubernetes.SpecialVariables; @@ -32,40 +20,6 @@ namespace Calamari.Tests.KubernetesFixtures [Category(TestCategory.RunOnceOnWindowsAndLinux)] public class KubernetesContextScriptWrapperLiveFixtureEks: KubernetesContextScriptWrapperLiveFixture { - private const string ResourcePackageFileName = "package.1.0.0.zip"; - private const string DeploymentFileName = "customresource.yml"; - private const string DeploymentFileName2 = "myapp-deployment.yml"; - private const string ServiceFileName = "myapp-service.yml"; - private const string ConfigMapFileName = "myapp-configmap1.yml"; - private const string ConfigMapFileName2 = "myapp-configmap2.yml"; - - private const string SimpleDeploymentResourceType = "Deployment"; - private const string SimpleDeploymentResourceName = "nginx-deployment"; - private const string SimpleDeploymentResource = - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-deployment\nspec:\n selector:\n matchLabels:\n app: nginx\n replicas: 3\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: nginx:1.14.2\n ports:\n - containerPort: 80"; - - private const string SimpleDeployment2ResourceName = "nginx-deployment"; - private const string SimpleDeploymentResource2 = - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-deployment2\nspec:\n selector:\n matchLabels:\n app: nginx2\n replicas: 1\n template:\n metadata:\n labels:\n app: nginx2\n spec:\n containers:\n - name: nginx2\n image: nginx:1.14.2\n ports:\n - containerPort: 81\n"; - - private const string InvalidDeploymentResource = - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-deployment\nspec:\nbad text here\n selector:\n matchLabels:\n app: nginx\n replicas: 3\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: nginx:1.14.2\n ports:\n - containerPort: 80\n"; - - private const string FailToDeploymentResource = - "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx-deployment\nspec:\n selector:\n matchLabels:\n app: nginx\n replicas: 3\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: nginx-bad-container-name:1.14.2\n ports:\n - containerPort: 80\n"; - - private const string SimpleServiceResourceName = "nginx-service"; - private const string SimpleService = - "apiVersion: v1\nkind: Service\nmetadata:\n name: nginx-service\nspec:\n selector:\n app.kubernetes.io/name: nginx\n ports:\n - protocol: TCP\n port: 80\n targetPort: 9376"; - - private const string SimpleConfigMapResourceName = "game-demo"; - private const string SimpleConfigMap = - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: game-demo\ndata:\n player_initial_lives: '3'\n ui_properties_file_name: 'user-interface.properties'\n game.properties: |\n enemy.types=aliens,monsters\n player.maximum-lives=5\n user-interface.properties: |\n color.good=purple\n color.bad=yellow\n allow.textmode=true"; - - private const string SimpleConfigMap2ResourceName = "game-demo2"; - private const string SimpleConfigMap2 = - "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: game-demo2\ndata:\n player_initial_lives: '1'\n ui_properties_file_name: 'user-interface.properties'\n game.properties: |\n enemy.types=blobs,foxes\n player.maximum-lives=10\n user-interface.properties: |\n color.good=orange\n color.bad=pink\n allow.textmode=false"; - string awsAccessKey; string awsSecretKey; @@ -129,143 +83,6 @@ protected override async Task> GetEnvironmentVars(Can }; } - [Test] - [TestCase(true)] - [TestCase(false)] - public void DeployRawYaml_WithRawYamlDeploymentScriptOrCommand_OutputShouldIndicateSuccessfulDeployment(bool usePackage) - { - SetupAndRunKubernetesRawYamlDeployment(usePackage, SimpleDeploymentResource); - - var rawLogs = Log.Messages.Select(m => m.FormattedMessage).ToArray(); - - AssertObjectStatusMonitoringStarted(rawLogs, (SimpleDeploymentResourceType, SimpleDeploymentResourceName)); - - var objectStatusUpdates = Log.Messages.GetServiceMessagesOfType("k8s-status"); - - objectStatusUpdates.Where(m => m.Properties["status"] == "Successful").Should().HaveCount(6); - - rawLogs.Should().ContainSingle(m => - m.Contains("Resource status check completed successfully because all resources are deployed successfully")); - } - - private static void AssertObjectStatusMonitoringStarted(string[] rawLogs, params (string Type, string Name)[] resources) - { - var resourceStatusCheckLog = "Resource Status Check: 1 new resources have been added:"; - var idx = Array.IndexOf(rawLogs, resourceStatusCheckLog); - foreach (var (i, type, name) in resources.Select((t, i) => (i, t.Type, t.Name))) - { - rawLogs[idx + i + 1].Should().Be($" - {type}/{name} in namespace calamari-testing"); - } - } - - [Test] - [TestCase(true)] - [TestCase(false)] - public void DeployRawYaml_WithInvalidYaml_OutputShouldIndicateFailure(bool usePackage) - { - SetupAndRunKubernetesRawYamlDeployment(usePackage, InvalidDeploymentResource, shouldSucceed: false); - - var rawLogs = Log.Messages.Select(m => m.FormattedMessage).Where(m => !m.StartsWith("##octopus") && m != string.Empty).ToArray(); - - var fileName = usePackage ? $"deployments{Path.DirectorySeparatorChar}{DeploymentFileName}" : DeploymentFileName; - var parsingErrorLog = - $"error: error parsing {fileName}: error converting YAML to JSON: yaml: line 7: could not find expected ':'"; - rawLogs.Should() - .ContainSingle(l => l == parsingErrorLog); - } - - [Test] - [TestCase(true)] - [TestCase(false)] - public void DeployRawYaml_WithYamlThatWillNotSucceed_OutputShouldIndicateFailure(bool usePackage) - { - SetupAndRunKubernetesRawYamlDeployment(usePackage, FailToDeploymentResource, shouldSucceed: false); - - var rawLogs = Log.Messages.Select(m => m.FormattedMessage).ToArray(); - - AssertObjectStatusMonitoringStarted(rawLogs, (SimpleDeploymentResourceType, SimpleDeploymentResourceName)); - - rawLogs.Should().ContainSingle(l => - l == - "Resource status check terminated because the timeout has been reached but some resources are still in progress"); - } - - [Test] - public void DeployRawYaml_WithMultipleYamlFilesGlobPatterns_YamlFilesAppliedInCorrectBatches() - { - SetVariablesToAuthoriseWithAmazonAccount(); - - SetVariablesForKubernetesResourceStatusCheck(30); - - SetVariablesForRawYamlCommand($@"deployments/**/* - services/{ServiceFileName} - configmaps/*.yml"); - - string CreatePackageWithMultipleYamlFiles(string directory) - { - var packageToPackage = CreatePackageWithFiles(ResourcePackageFileName, directory, - ("deployments", DeploymentFileName, SimpleDeploymentResource), - (Path.Combine("deployments", "subfolder"), DeploymentFileName2, SimpleDeploymentResource2), - ("services", ServiceFileName, SimpleService), - ("services", "EmptyYamlFile.yml", ""), - ("configmaps", ConfigMapFileName, SimpleConfigMap), - ("configmaps", ConfigMapFileName2, SimpleConfigMap2), - (Path.Combine("configmaps","subfolder"), "InvalidJSONNotUsed.yml", InvalidDeploymentResource)); - return packageToPackage; - } - - ExecuteCommandAndVerifyResult(KubernetesApplyRawYamlCommand.Name, CreatePackageWithMultipleYamlFiles); - - var rawLogs = Log.Messages.Select(m => m.FormattedMessage).Where(l => !l.StartsWith("##octopus")).ToArray(); - - // We take the logs starting from when Calamari starts applying batches - // to when the last k8s resource is created and compare them in an assent test. - var startIndex = Array.FindIndex(rawLogs, l => l.StartsWith("Applying Batch #1")); - var endIndex = - Array.FindLastIndex(rawLogs, l => l == "Resource Status Check: 2 new resources have been added:") + 2; - var assentLogs = rawLogs.Skip(startIndex) - .Take(endIndex + 1 - startIndex) - .Where(l => !l.StartsWith("##octopus")).ToArray(); - var batch3Index = Array.FindIndex(assentLogs, l => l.StartsWith("Applying Batch #3")); - - // In this case the two config maps have been loaded in reverse order - // This can happen as Directory.EnumerateFiles() does not behave the - // same on all platforms. - // We'll flip them back the right way before performing the Assent Test. - if (assentLogs[batch3Index + 1].Contains("myapp-configmap1.yml")) - { - var configMap1Idx = batch3Index + 1; - var configMap2Idx = Array.FindIndex(assentLogs, l => l.Contains("myapp-configmap2.yml")); - var endIdx = Array.FindLastIndex(assentLogs, l => l == "Created Resources:") - 1; - InPlaceSwap(assentLogs, configMap1Idx, configMap2Idx, endIdx); - } - - // We need to replace the backslash with forward slash because - // the slash comes out differently on windows machines. - var assentString = string.Join('\n', assentLogs).Replace("\\", "/"); - this.Assent(assentString, configuration: AssentConfiguration.DefaultWithPostfix("ApplyingBatches")); - - var resources = new[] - { - (Name: SimpleDeploymentResourceName, Label: "Deployment1"), - (Name: SimpleDeployment2ResourceName,Label: "Deployment2"), - (Name: SimpleServiceResourceName, Label: "Service1"), - (Name: SimpleConfigMapResourceName, Label: "ConfigMap1"), - (Name: SimpleConfigMap2ResourceName, Label: "ConfigMap3") - }; - - var statusMessages = Log.Messages.GetServiceMessagesOfType("k8s-status"); - - foreach (var (name, label) in resources) - { - // Check that each deployed resource has a "Successful" status reported. - statusMessages.Should().Contain(m => m.Properties["name"] == name && m.Properties["status"] == "Successful"); - } - - rawLogs.Should().ContainSingle(m => - m.Contains("Resource status check completed successfully because all resources are deployed successfully")); - } - [Test] [TestCase(true)] [TestCase(false)] @@ -590,22 +407,6 @@ public void DiscoverKubernetesClusterWithInvalidAccountCredentials() "Unable to authorise credentials, see verbose log for details."); } - private void SetupAndRunKubernetesRawYamlDeployment(bool usePackage, string resource, bool shouldSucceed = true) - { - SetVariablesToAuthoriseWithAmazonAccount(); - - SetVariablesForKubernetesResourceStatusCheck(shouldSucceed ? 30 : 5); - - SetVariablesForRawYamlCommand("**/*.{yml,yaml}"); - - ExecuteCommandAndVerifyResult(KubernetesApplyRawYamlCommand.Name, - usePackage - ? CreateAddPackageFunc(resource) - : CreateAddCustomResourceFileFunc(resource), - shouldSucceed); - - } - private void SetVariablesToAuthoriseWithAmazonAccount() { const string account = "eks_account"; @@ -621,95 +422,6 @@ private void SetVariablesToAuthoriseWithAmazonAccount() variables.Set("Octopus.Action.Kubernetes.CertificateAuthority", certificateAuthority); variables.Set($"{certificateAuthority}.CertificatePem", eksClusterCaCertificate); } - - private void SetVariablesForRawYamlCommand(string globPaths) - { - variables.Set("Octopus.Action.KubernetesContainers.Namespace", "nginx-2"); - variables.Set(KnownVariables.Package.JsonConfigurationVariablesTargets, "**/*.{yml,yaml}"); - variables.Set(KubernetesSpecialVariables.CustomResourceYamlFileName, globPaths); - } - - private void SetVariablesForKubernetesResourceStatusCheck(int timeout) - { - variables.Set("Octopus.Action.Kubernetes.ResourceStatusCheck", "True"); - variables.Set("Octopus.Action.KubernetesContainers.DeploymentWait", "NoWait"); - variables.Set("Octopus.Action.Kubernetes.DeploymentTimeout", timeout.ToString()); - variables.Set("Octopus.Action.Kubernetes.PrintVerboseKubectlOutputOnError", "True"); - } - - private static string CreateResourceYamlFile(string directory, string fileName, string content) - { - var pathToCustomResource = Path.Combine(directory, fileName); - File.WriteAllText(pathToCustomResource, content); - return pathToCustomResource; - } - - private Func CreateAddCustomResourceFileFunc(string yamlContent) - { - return directory => - { - CreateResourceYamlFile(directory, DeploymentFileName, yamlContent); - if (!variables.IsSet(KubernetesSpecialVariables.CustomResourceYamlFileName)) - { - variables.Set(KubernetesSpecialVariables.CustomResourceYamlFileName, DeploymentFileName); - } - return null; - }; - } - - private Func CreateAddPackageFunc(string yamlContent) - { - return directory => - { - var pathInPackage = Path.Combine("deployments", DeploymentFileName); - var pathToPackage = CreatePackageWithFiles(ResourcePackageFileName, directory, - ("deployments", DeploymentFileName, yamlContent)); - if (!variables.IsSet(KubernetesSpecialVariables.CustomResourceYamlFileName)) - { - variables.Set(KubernetesSpecialVariables.CustomResourceYamlFileName, pathInPackage); - } - return pathToPackage; - }; - } - - private string CreatePackageWithFiles(string packageFileName, string currentDirectory, - params (string directory, string fileName, string content)[] files) - { - var pathToPackage = Path.Combine(currentDirectory, packageFileName); - using (var archive = ZipArchive.Create()) - { - var readStreams = new List(); - foreach (var (directory, fileName, content) in files) - { - var pathInPackage = Path.Combine(directory, fileName); - var readStream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - readStreams.Add(readStream); - archive.AddEntry(pathInPackage, readStream); - } - - using (var writeStream = File.OpenWrite(pathToPackage)) - { - archive.SaveTo(writeStream, CompressionType.Deflate); - } - - foreach (var readStream in readStreams) - { - readStream.Dispose(); - } - } - - return pathToPackage; - } - - private void InPlaceSwap(string[] array, int section1StartIndex, int section2StartIndex, int endIndex) - { - var length = endIndex + 1 - section1StartIndex; - var section2TempIndex = section2StartIndex - section1StartIndex; - var temp = new string[length]; - Array.Copy(array, section1StartIndex, temp, 0, length); - Array.Copy(temp, section2TempIndex, array, section1StartIndex, temp.Length - section2TempIndex); - Array.Copy(temp, 0, array, section1StartIndex + temp.Length - section2TempIndex, section2TempIndex); - } } } #endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/KubernetesLiveStatusCheckTests.cs b/source/Calamari.Tests/KubernetesFixtures/KubernetesLiveStatusCheckTests.cs new file mode 100644 index 000000000..50012c154 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/KubernetesLiveStatusCheckTests.cs @@ -0,0 +1,185 @@ +#if NETCORE +using System; +using System.Collections.Generic; +using System.IO; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Common.Plumbing.Variables; +using Calamari.Kubernetes; +using Calamari.Kubernetes.Commands; +using Calamari.Testing.Helpers; +using Calamari.Tests.Helpers; +using JetBrains.Annotations; +using NUnit.Framework; + +namespace Calamari.Tests.KubernetesFixtures +{ + [TestFixture] + public class KubernetesLiveStatusCheckTests : CalamariFixture + { + [Test] + public void GivenUndeployableResource_ShouldFail() + { + var variables = new CalamariVariables(); + variables.SetClusterDetails(); + variables.SetAuthenticationDetails(); + variables.SetVariablesForKubernetesResourceStatusCheck( 5); + var ns = CreateString(); + variables.Set(SpecialVariables.Namespace, ns); + + var undeployableYaml = @"apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx-bad-container-name:1.14.2"; + + var output = ExecuteRawYamlCommand(variables, undeployableYaml); + + output.AssertFailure(); + output.AssertOutputContains($"Resource Status Check: the following resources are still in progress by the end of the timeout:{Environment.NewLine}" + + $" - Pod/nginx in namespace {ns}"); + output.AssertOutputContains("Resource status check terminated because the timeout has been reached but some resources are still in progress"); + } + + [Test] + public void GivenValidResource_ShouldSucceed() + { + var variables = new CalamariVariables(); + variables.SetClusterDetails(); + variables.SetAuthenticationDetails(); + variables.SetVariablesForKubernetesResourceStatusCheck(); + var ns = CreateString(); + variables.Set(SpecialVariables.Namespace, ns); + var deployableYaml = @"apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx:1.14.2"; + + var output = ExecuteRawYamlCommand(variables, deployableYaml); + + output.AssertSuccess(); + output.AssertOutputContains($"Resource Status Check: 1 new resources have been added:{Environment.NewLine}" + + $" - Pod/nginx in namespace {ns}"); + } + + [Test] + public void GivenJob_WhenNoWait_ShouldCompleteWithoutWaitingForJob() + { + var yaml = @"apiVersion: batch/v1 +kind: Job +metadata: + name: sleep +spec: + template: + spec: + containers: + - name: alpine + image: alpine + command: [""/bin/sh"",""-c""] + args: [""sleep 10; echo fail; exit 1""] + command: [""sleep"", ""10""] + restartPolicy: Never"; + var variables = new CalamariVariables(); + variables.SetClusterDetails(); + variables.SetAuthenticationDetails(); + variables.SetVariablesForKubernetesResourceStatusCheck(timeout: 5); + var ns = CreateString(); + variables.Set(SpecialVariables.WaitForJobs, "false"); + variables.Set(SpecialVariables.Namespace, ns); + + + var output = ExecuteRawYamlCommand(variables, yaml); + output.AssertSuccess(); + + //output.AssertServiceMessage("k8s-status"); + } + + [Test] + public void GivenJob_WhenWaitAndJobCompletesAfterTaskTimeout_ShouldTimeout() + { + //var jobSleep = 10; + var yaml = @"apiVersion: batch/v1 +kind: Job +metadata: + name: sleep +spec: + template: + spec: + containers: + - name: alpine + image: alpine + command: [""/bin/sh"",""-c""] + args: [""sleep 10; echo fail; exit 1""] + restartPolicy: Never"; + var variables = new CalamariVariables(); + variables.SetClusterDetails(); + variables.SetAuthenticationDetails(); + variables.SetVariablesForKubernetesResourceStatusCheck( timeout: 5); + variables.Set(SpecialVariables.WaitForJobs, "true"); + + var ns = CreateString(); + variables.Set(SpecialVariables.Namespace, ns); + + + var output = ExecuteRawYamlCommand(variables, yaml); + output.AssertFailure(); + output.AssertOutputContains("Resource status check terminated because the timeout has been reached but some resources are still in progress"); + /*[60] = {string} "Resource Status Check: the following resources are still in progress by the end of the timeout:" + [61] = {string} " - Job/sleep in namespace testff41f51700" + [62] = {string} "Resource status check terminated because the timeout has been reached but some resources are still in progress"*/ + } + + CalamariResult ExecuteRawYamlCommand(CalamariVariables variables, string yaml) + { + const string deploymentFileName = "customresource.yml"; + variables.Set(SpecialVariables.CustomResourceYamlFileName, "**/*.{yml,yaml}"); + using var workingDirectory = TemporaryDirectory.Create(); + CreateResourceYamlFile(workingDirectory.DirectoryPath, deploymentFileName, yaml); + variables.Set(SpecialVariables.CustomResourceYamlFileName, deploymentFileName); + variables.Set(KnownVariables.OriginalPackageDirectoryPath, workingDirectory.DirectoryPath); + + var output = ExecuteCommand(variables, KubernetesApplyRawYamlCommand.Name, workingDirectory.DirectoryPath); + return output; + } + + static string CreateResourceYamlFile(string directory, string fileName, string content) + { + var pathToCustomResource = Path.Combine(directory, fileName); + File.WriteAllText(pathToCustomResource, content); + return pathToCustomResource; + } + + string CreateString() + { + return $"Test{Guid.NewGuid().ToString("N").Substring(0, 10)}".ToLower(); + } + + CalamariResult ExecuteCommand(IVariables variables, string command, [CanBeNull] string workingDirectory) + { + using var variablesFile = new TemporaryFile(Path.GetTempFileName()); + variables.Save(variablesFile.FilePath); + + var calamariCommand = Calamari().Action(command) + .Argument("variables", variablesFile.FilePath) + .WithEnvironmentVariables(new Dictionary()) + .OutputToLog(true); + + if (workingDirectory != null) + { + calamariCommand = calamariCommand.WithWorkingDirectory(workingDirectory); + } + + return InvokeInProcess(calamariCommand, variables); + } + + + + } +} +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/Tools/IToolDownloader.cs b/source/Calamari.Tests/KubernetesFixtures/Tools/IToolDownloader.cs new file mode 100644 index 000000000..1291a73ae --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Tools/IToolDownloader.cs @@ -0,0 +1,271 @@ +#if NETCORE + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Tests.KubernetesFixtures.Tools; +using Octopus.CoreUtilities.Extensions; +using Serilog; + +namespace Calamari.Tests.KubernetesFixtures.Tools +{ + public interface IToolDownloader + { + Task Download(string targetDirectory, CancellationToken cancellationToken); + } + + + /// + /// Copied as is from the octopus server repo. + /// + public static class OctopusPackageDownloader + { + public static async Task DownloadPackage(string downloadUrl, string filePath, ILogger logger, CancellationToken cancellationToken = default) + { + var exceptions = new List(); + for (int i = 0; i < 5; i++) + { + try + { + await AttemptToDownloadPackage(downloadUrl, filePath, logger, cancellationToken); + return; + } + catch (Exception e) + { + exceptions.Add(e); + } + } + + throw new AggregateException(exceptions); + } + + static async Task AttemptToDownloadPackage(string downloadUrl, string filePath, ILogger logger, CancellationToken cancellationToken) + { + var totalTime = Stopwatch.StartNew(); + var totalRead = 0L; + string expectedHash = null; + try + { + using (var client = new HttpClient()) + { + // This appears to be the time it takes to do a single read/write, not the entire download. + client.Timeout = TimeSpan.FromSeconds(20); + using (var response = await client.GetAsync(downloadUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken)) + { + response.EnsureSuccessStatusCode(); + var totalLength = response.Content.Headers.ContentLength; + expectedHash = TryGetExpectedHashFromHeaders(response, expectedHash); + + logger.Information($"Downloading {downloadUrl} ({totalLength} bytes)"); + + var sw = new Stopwatch(); + sw.Start(); + using (Stream contentStream = await response.Content.ReadAsStreamAsync(), + fileStream = new FileStream( + filePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + 8192, + true)) + { + + var buffer = new byte[8192]; + + var read = await contentStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken); + while (read != 0) + { + await fileStream.WriteAsync(buffer, 0, read, cancellationToken); + + if (totalLength.HasValue && sw.ElapsedMilliseconds >= TimeSpan.FromSeconds(7).TotalMilliseconds) + { + var percentRead = totalRead * 1.0 / totalLength.Value * 100; + logger.Information($"Downloading Completed {percentRead}%"); + sw.Reset(); + sw.Start(); + } + + read = await contentStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken); + totalRead += read; + } + + totalTime.Stop(); + + logger.Information("Download Finished in {totalTime}ms", totalTime.ElapsedMilliseconds); + } + } + } + } + catch (Exception e) + { + throw new Exception($"Failure to download: {downloadUrl}. After {totalTime.Elapsed.TotalSeconds} seconds we only downloaded, {totalRead}", e); + } + + ValidateDownload(filePath, expectedHash); + } + + static string TryGetExpectedHashFromHeaders(HttpResponseMessage response, string expectedHash) + { + if (response.Headers.TryGetValues("x-amz-meta-sha256", out var expectedHashs)) + { + expectedHash = expectedHashs.FirstOrDefault(); + } + + return expectedHash; + } + + static void ValidateDownload(string filePath, string expectedHash) + { + if (!expectedHash.IsNullOrEmpty()) + { + using (var sha256 = SHA256.Create()) + { + var fileBytes = File.ReadAllBytes(filePath); + var hash = sha256.ComputeHash(fileBytes); + var computedHash = BitConverter.ToString(hash).Replace("-", ""); + if (!computedHash.Equals(expectedHash, StringComparison.OrdinalIgnoreCase)) + { + throw new Exception($"Computed SHA256 ({computedHash}) hash of file does not match expected ({expectedHash})." + $"Downloaded file may be corrupt. File size {((long)fileBytes.Length)}"); + } + } + } + } + } + + public abstract class ToolDownloader : IToolDownloader + { + readonly OperatingSystem os; + + protected ILogger Logger { get; } + protected string ExecutableName { get; } + + protected ToolDownloader(string executableName, ILogger logger) + { + ExecutableName = executableName; + Logger = logger; + + os = GetOperationSystem(); + + //we assume that windows always has .exe suffixed + if (os is OperatingSystem.Windows) + { + ExecutableName += ".exe"; + } + } + + public async Task Download(string targetDirectory, CancellationToken cancellationToken) + { + var downloadUrl = BuildDownloadUrl(RuntimeInformation.ProcessArchitecture, os); + + //we download to a random file name + var downloadFilePath = Path.Combine(targetDirectory, Guid.NewGuid().ToString("N")); + + Logger.Information("Downloading {DownloadUrl} to {DownloadFilePath}", downloadUrl, downloadFilePath); + await OctopusPackageDownloader.DownloadPackage(downloadUrl, downloadFilePath, Logger, cancellationToken); + + downloadFilePath = PostDownload(targetDirectory, downloadFilePath, RuntimeInformation.ProcessArchitecture, os); + + //if this is not running on windows, chmod the tool to be executable + if (os != OperatingSystem.Windows) + { + var exitCode = SilentProcessRunner.ExecuteCommand( + "chmod", + $"+x {downloadFilePath}", + targetDirectory, + new Dictionary(), + (x) => Logger.Information(x), + (m) => Logger.Error(m)); + + if (exitCode.ExitCode != 0) + { + Logger.Error("Error running chmod against executable {ExecutablePath}", downloadFilePath); + } + } + + return downloadFilePath; + } + + protected abstract string BuildDownloadUrl(Architecture processArchitecture, OperatingSystem operatingSystem); + + protected virtual string PostDownload(string downloadDirectory, string downloadFilePath, Architecture processArchitecture, OperatingSystem operatingSystem) + { + var targetFilename = Path.Combine(downloadDirectory, ExecutableName); + File.Move(downloadFilePath, targetFilename); + + return targetFilename; + } + + static OperatingSystem GetOperationSystem() + { + if (PlatformDetection.IsRunningOnWindows) + { + return OperatingSystem.Windows; + } + + if (PlatformDetection.IsRunningOnNix) + { + return OperatingSystem.Nix; + } + + if (PlatformDetection.IsRunningOnMac) + { + return OperatingSystem.Mac; + } + + throw new InvalidOperationException("Unsupported OS"); + } + } + + public static class PlatformDetection + { + public static bool IsRunningOnNix => RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + public static bool IsRunningOnWindows => RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + public static bool IsRunningOnMac => RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + } + + public enum OperatingSystem + { + Windows, + Nix, + Mac + } + + public class RequiredToolDownloader + { + readonly TemporaryDirectory temporaryDirectory; + + readonly KindDownloader kindDownloader; + // readonly HelmDownloader helmDownloader; + readonly KubeCtlDownloader kubeCtlDownloader; + + public RequiredToolDownloader(TemporaryDirectory temporaryDirectory, ILogger logger) + { + this.temporaryDirectory = temporaryDirectory; + + kindDownloader = new KindDownloader(logger); + //helmDownloader = new HelmDownloader(logger); + kubeCtlDownloader = new KubeCtlDownloader(logger); + } + + public async Task<(string KindExePath, string HelmExePath, string KubeCtlPath)> DownloadRequiredTools(CancellationToken cancellationToken) + { + var kindExePathTask = kindDownloader.Download(temporaryDirectory.DirectoryPath, cancellationToken); + var helmExePathTask = kindExePathTask; //helmDownloader.Download(temporaryDirectory.DirectoryPath, cancellationToken); + var kubeCtlExePathTask = kubeCtlDownloader.Download(temporaryDirectory.DirectoryPath, cancellationToken); + + await Task.WhenAll(kindExePathTask, helmExePathTask, kubeCtlExePathTask); + + return (kindExePathTask.Result, helmExePathTask.Result, kubeCtlExePathTask.Result); + } + } +} +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/Tools/KindDownloader.cs b/source/Calamari.Tests/KubernetesFixtures/Tools/KindDownloader.cs new file mode 100644 index 000000000..49ae89cd3 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Tools/KindDownloader.cs @@ -0,0 +1,41 @@ +#if NETCORE +using System; +using System.Runtime.InteropServices; +using Serilog; + +namespace Calamari.Tests.KubernetesFixtures.Tools +{ + public class KindDownloader : ToolDownloader + { + const string LatestKindVersion = "v0.22.0"; + + public KindDownloader(ILogger logger) + : base("kind", logger) + { + } + + protected override string BuildDownloadUrl(Architecture processArchitecture, OperatingSystem operatingSystem) + { + var architecture = processArchitecture == Architecture.Arm64 ? "arm64" : "amd64"; + var osName = GetOsName(operatingSystem); + + return $"https://github.com/kubernetes-sigs/kind/releases/download/{LatestKindVersion}/kind-{osName}-{architecture}"; + } + + static string GetOsName(OperatingSystem operatingSystem) + { + switch (operatingSystem) + { + case OperatingSystem.Windows: + return "windows"; + case OperatingSystem.Nix: + return "linux"; + case OperatingSystem.Mac: + return "darwin"; + default: + throw new ArgumentOutOfRangeException(nameof(operatingSystem), operatingSystem, null); + } + } + } +} +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/Tools/KubeCtlDownloader.cs b/source/Calamari.Tests/KubernetesFixtures/Tools/KubeCtlDownloader.cs new file mode 100644 index 000000000..ab959d336 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Tools/KubeCtlDownloader.cs @@ -0,0 +1,38 @@ +#if NETCORE +using System; +using System.Runtime.InteropServices; +using Serilog; + +namespace Calamari.Tests.KubernetesFixtures.Tools +{ + public class KubeCtlDownloader : ToolDownloader + { + public const string LatestKubeCtlVersion = "v1.29.3"; + + public KubeCtlDownloader(ILogger logger) + : base("kubectl", logger) + { } + + protected override string BuildDownloadUrl(Architecture processArchitecture, OperatingSystem operatingSystem) + { + var architecture = processArchitecture == Architecture.Arm64 ? "arm64" : "amd64"; + var osName = GetOsName(operatingSystem); + + var extension = operatingSystem is OperatingSystem.Windows + ? ".exe" + : null; + + return $"https://dl.k8s.io/release/{LatestKubeCtlVersion}/bin/{osName}/{architecture}/kubectl{extension}"; + } + + static string GetOsName(OperatingSystem operatingSystem) + => operatingSystem switch + { + OperatingSystem.Windows => "windows", + OperatingSystem.Nix => "linux", + OperatingSystem.Mac => "darwin", + _ => throw new ArgumentOutOfRangeException(nameof(operatingSystem), operatingSystem, null) + }; + } +} +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesClusterInstaller.cs b/source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesClusterInstaller.cs new file mode 100644 index 000000000..5367d09e3 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesClusterInstaller.cs @@ -0,0 +1,226 @@ +#if NETCORE +#nullable enable +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Calamari.Common.Features.Processes; +using Calamari.Common.Plumbing.Commands; +using Calamari.Common.Plumbing.FileSystem; +using Calamari.Kubernetes; +using Octostache; +using Serilog; +using Serilog.Configuration; +using Serilog.Core; +using Serilog.Events; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; + +namespace Calamari.Tests.KubernetesFixtures.Tools +{ + + public static class StringBuilderLogEventSinkExtensions + { + public static LoggerConfiguration StringBuilder(this LoggerSinkConfiguration configuration, StringBuilder stringBuilder, IFormatProvider? formatProvider = null) + => configuration.Sink(new StringBuilderLogEventSink(stringBuilder, formatProvider)); + } + + public class StringBuilderLogEventSink : ILogEventSink + { + readonly StringBuilder stringBuilder; + readonly IFormatProvider? formatProvider; + + public StringBuilderLogEventSink(StringBuilder stringBuilder, IFormatProvider? formatProvider) + { + this.stringBuilder = stringBuilder; + this.formatProvider = formatProvider; + } + + public void Emit(LogEvent logEvent) + { + var message = logEvent.RenderMessage(formatProvider); + stringBuilder.AppendLine(message); + } + } + + public class KubernetesClusterInstaller + { + readonly string clusterName; + readonly string kubeConfigName; + + readonly TemporaryDirectory tempDir; + readonly string kindExePath; + //readonly string helmExePath; + readonly string kubeCtlPath; + readonly ILogger logger; + + public string KubeConfigPath => Path.Combine(tempDir.DirectoryPath, kubeConfigName); + public string ClusterName => clusterName; + + public KubernetesClusterInstaller(TemporaryDirectory tempDirectory, + string kindExePath, + string helmExePath, + string kubeCtlPath, + ILogger logger) + { + tempDir = tempDirectory; + this.kindExePath = kindExePath; + // this.helmExePath = helmExePath; + this.kubeCtlPath = kubeCtlPath; + this.logger = logger; + + clusterName = $"tentacleint-{DateTime.Now:yyyyMMddhhmmss}"; + kubeConfigName = $"{clusterName}.config"; + } + + public async Task Install() + { + var configFilePath = await WriteFileToTemporaryDirectory("kind-config.yml"); + + var sw = new Stopwatch(); + sw.Restart(); + var exitCode = SilentProcessRunner.ExecuteCommand( + kindExePath, + //we give the cluster a unique name + $"create cluster --name={clusterName} --config=\"{configFilePath}\" --kubeconfig=\"{kubeConfigName}\"", + tempDir.DirectoryPath, + new Dictionary(), + (m) => logger.Debug(m), + (m) => logger.Error(m)); + + sw.Stop(); + + if (exitCode.ExitCode != 0) + { + logger.Error("Failed to create Kind Kubernetes cluster {ClusterName}", clusterName); + throw new InvalidOperationException($"Failed to create Kind Kubernetes cluster {clusterName}"); + } + + logger.Information("Test cluster kubeconfig path: {Path:l}", KubeConfigPath); + + logger.Information("Created Kind Kubernetes cluster {ClusterName} in {ElapsedTime}", clusterName, sw.Elapsed); + + await SetLocalhostRouting(); + + ExtractLoginDetails(); + //await InstallNfsCsiDriver(); + } + + public (ClusterEndpoint ClusterEndpoint, ClusterUser ClusterUser) ExtractLoginDetails() + { + var config = Path.Combine(tempDir.DirectoryPath,$"{clusterName}.config"); + + var deserializer = new DeserializerBuilder().Build(); + var data = deserializer.Deserialize(File.ReadAllText(config)); + + var cluster = data["clusters"][0]["cluster"]; + var clusterCert = cluster["certificate-authority-data"].ToString(); + var clusterUrl = cluster["server"].ToString(); + + var user = data["users"][0]["user"]; + var clientCertPem = user["client-certificate-data"].ToString(); + var clientCertKey = user["client-key-data"].ToString(); + + return (new ClusterEndpoint(clusterUrl, clusterCert), new ClusterUser(clientCertPem, clientCertKey)); + } + + async Task SetLocalhostRouting() + { + var filename = PlatformDetection.IsRunningOnNix ? "linux-network-routing.yml" : "docker-desktop-network-routing.yml"; + + var manifestFilePath = await WriteFileToTemporaryDirectory(filename, "manifest.yml"); + + var sb = new StringBuilder(); + var sprLogger = new LoggerConfiguration() + .WriteTo.Logger(logger) + .WriteTo.StringBuilder(sb) + .CreateLogger(); + + var exitCode = SilentProcessRunner.ExecuteCommand( + kubeCtlPath, + //we give the cluster a unique name + $"apply -n default -f \"{manifestFilePath}\" --kubeconfig=\"{KubeConfigPath}\"", + tempDir.DirectoryPath, + new Dictionary(), + (m) => logger.Debug(m), + (m) => logger.Information(m)); + + if (exitCode.ExitCode != 0) + { + logger.Error("Failed to apply localhost routing to cluster {ClusterName}", clusterName); + throw new InvalidOperationException($"Failed to apply localhost routing to cluster {clusterName}. Logs: {sb}"); + } + } + + async Task WriteFileToTemporaryDirectory(string resourceFileName, string? outputFilename = null) + { + await using var resourceStream = Assembly.GetExecutingAssembly().GetManifestResourceStreamFromPartialName(resourceFileName); + + var filePath = Path.Combine(tempDir.DirectoryPath, outputFilename ?? resourceFileName); + await using var file = File.Create(filePath); + + resourceStream.Seek(0, SeekOrigin.Begin); + await resourceStream.CopyToAsync(file); + + return filePath; + } + + public void Dispose() + { + var exitCode = SilentProcessRunner.ExecuteCommand( + kindExePath, + //delete the cluster for this test run + $"delete cluster --name={clusterName}", + tempDir.DirectoryPath, + new Dictionary(), + (m) => logger.Debug(m), + (m) => logger.Information(m)); + + + if (exitCode.ExitCode != 0) + { + logger.Error("Failed to delete Kind kubernetes cluster {ClusterName}", clusterName); + } + } + } + + public static class AssemblyExtensions + { + public static Stream GetManifestResourceStreamFromPartialName(this Assembly assembly, string filename) + { + var manifests = assembly.GetManifestResourceNames(); + var valuesFileName = manifests.Single(n => n.Contains(filename, StringComparison.OrdinalIgnoreCase)); + return assembly.GetManifestResourceStream(valuesFileName)!; + } + } +} + +public struct ClusterUser +{ + public ClusterUser(string clientCertPem, string clientCertKey) + { + ClientCertPem = clientCertPem; + ClientCertKey = clientCertKey; + } + + public string ClientCertPem { get; } + public string ClientCertKey { get; } +} +public struct ClusterEndpoint +{ + public ClusterEndpoint(string clusterUrl, string clusterCert) + { + ClusterCert = clusterCert; + ClusterUrl = clusterUrl; + } + + public string ClusterCert { get; } + public string ClusterUrl { get; } +} + +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesTestsGlobalContext.cs b/source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesTestsGlobalContext.cs new file mode 100644 index 000000000..1530637e0 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Tools/KubernetesTestsGlobalContext.cs @@ -0,0 +1,44 @@ +#if NETCORE +using System; +using Calamari.Common.Plumbing.FileSystem; +using Serilog; + +namespace Calamari.Tests.KubernetesFixtures.Tools +{ + public class KubernetesTestsGlobalContext : IDisposable + { + public static KubernetesTestsGlobalContext Instance { get; } = new KubernetesTestsGlobalContext(); + + public TemporaryDirectory TemporaryDirectory { get; } + + public ILogger Logger { get; } + + public string KubeConfigPath { get; set; } = ""; + + public string HelmExePath { get; private set; } = null!; + public string KubeCtlExePath { get; private set; }= null!; + + public ClusterEndpoint ClusterEndpoint { get; set; } + public ClusterUser ClusterUser { get; set; } + KubernetesTestsGlobalContext() + { + TemporaryDirectory = TemporaryDirectory.Create();; + + Logger = Log.Logger; + } + + public void Dispose() + { + TemporaryDirectory.Dispose(); + } + + public void SetToolExePaths(string helmExePath, string kubeCtlPath) + { + HelmExePath = helmExePath; + KubeCtlExePath = kubeCtlPath; + } + + + } +} +#endif \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/Tools/docker-desktop-network-routing.yml b/source/Calamari.Tests/KubernetesFixtures/Tools/docker-desktop-network-routing.yml new file mode 100644 index 000000000..3efd80fd9 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Tools/docker-desktop-network-routing.yml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + name: dockerhost + namespace: default +spec: + type: ExternalName + externalName: host.docker.internal \ No newline at end of file diff --git a/source/Calamari.Tests/KubernetesFixtures/Tools/kind-config.yml b/source/Calamari.Tests/KubernetesFixtures/Tools/kind-config.yml new file mode 100644 index 000000000..37eb950b6 --- /dev/null +++ b/source/Calamari.Tests/KubernetesFixtures/Tools/kind-config.yml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 + +nodes: + - role: control-plane + image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245 + - role: worker + image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245