From 24df9db37ee1844e1cf8e987884bd799e0fb82a0 Mon Sep 17 00:00:00 2001 From: Kevin Tchang Date: Thu, 19 Dec 2024 15:09:26 +1100 Subject: [PATCH] Update Kubernetes Client package to v15 (#1051) --- .../KubernetesAgentIntegrationTest.cs | 59 ++---- .../KubernetesAgentMetricsIntegrationTest.cs | 2 +- .../KubernetesClusterOneTimeSetUp.cs | 31 ++++ ...ubernetesScriptServiceV1IntegrationTest.cs | 12 +- .../KubernetesClientCompatibilityTests.cs | 173 ++++++++++++++++++ .../KubernetesClusterOneTimeSetUp.cs | 62 ------- .../KubernetesTestsGlobalContext.cs | 7 + ...ntacle.Kubernetes.Tests.Integration.csproj | 15 +- .../KindConfiguration/kind-config-v1-28.yaml | 8 + .../KindConfiguration/kind-config-v1-29.yaml | 8 + .../kind-config-v1-30.yaml} | 0 .../KindConfiguration/kind-config-v1-31.yaml | 8 + .../Setup/KubernetesClusterInstaller.cs | 16 +- .../Setup/SetupHelpers.cs | 79 ++++++++ .../Octopus.Tentacle/Octopus.Tentacle.csproj | 4 +- 15 files changed, 359 insertions(+), 125 deletions(-) rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/{ => KubernetesAgent}/KubernetesAgentIntegrationTest.cs (61%) rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/{ => KubernetesAgent}/KubernetesAgentMetricsIntegrationTest.cs (98%) create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/{ => KubernetesAgent}/KubernetesScriptServiceV1IntegrationTest.cs (97%) create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs delete mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/{kind-config.yaml => KindConfiguration/kind-config-v1-30.yaml} (100%) create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs similarity index 61% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs index 32cb98e4c..3f8b40771 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs @@ -1,18 +1,13 @@ -using Halibut; -using Halibut.Diagnostics; -using Halibut.Diagnostics.LogCreators; -using Halibut.Logging; +using System; +using Halibut; using Octopus.Tentacle.Client; -using Octopus.Tentacle.Client.Retries; -using Octopus.Tentacle.Client.Scripts; using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.Contracts.Observability; using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; using Octopus.Tentacle.Kubernetes.Tests.Integration.Tooling; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Common.Logging; -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; public abstract class KubernetesAgentIntegrationTest { @@ -23,9 +18,7 @@ public abstract class KubernetesAgentIntegrationTest protected ILogger? Logger { get; private set; } protected KubernetesAgentInstaller KubernetesAgentInstaller => kubernetesAgentInstaller ?? throw new InvalidOperationException("Expected kubernetesAgentInstaller to be set"); - - protected HalibutRuntime ServerHalibutRuntime { get; private set; } = null!; - + protected TentacleClient TentacleClient { get; private set; } = null!; protected CancellationToken CancellationToken { get; private set; } @@ -34,6 +27,8 @@ public abstract class KubernetesAgentIntegrationTest protected readonly IDictionary CustomHelmValues = new Dictionary(); + HalibutRuntime serverHalibutRuntime; + string? agentThumbprint; [OneTimeSetUp] @@ -54,12 +49,13 @@ public async Task OneTimeSetUp() KubernetesTestsGlobalContext.Instance.Logger); //create a new server halibut runtime - var listeningPort = BuildServerHalibutRuntimeAndListen(); + serverHalibutRuntime = SetupHelpers.BuildServerHalibutRuntime(); + var listeningPort = serverHalibutRuntime.Listen(); agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag, CustomHelmValues); //trust the generated cert thumbprint - ServerHalibutRuntime.Trust(agentThumbprint); + serverHalibutRuntime.Trust(agentThumbprint); } [SetUp] @@ -76,7 +72,7 @@ public void SetUp() CancellationToken = cancellationTokenSource.Token; //each test should get its own tentacle client, so it gets its own builders - BuildTentacleClient(); + TentacleClient = SetupHelpers.BuildTentacleClient(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, serverHalibutRuntime, ConfigureTentacleServiceDecoratorBuilder); } [TearDown] @@ -91,45 +87,14 @@ public async Task TearDown() cancellationTokenSource?.Dispose(); } - protected virtual TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) => builder; - - void BuildTentacleClient() - { - var endpoint = new ServiceEndPoint(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, ServerHalibutRuntime.TimeoutsAndLimits); - - var retrySettings = new RpcRetrySettings(true, TimeSpan.FromMinutes(2)); - var clientOptions = new TentacleClientOptions(retrySettings); - - TentacleClient.CacheServiceWasNotFoundResponseMessages(ServerHalibutRuntime); - - var builder = new TentacleServiceDecoratorBuilder(); - ConfigureTentacleServiceDecoratorBuilder(builder); - - TentacleClient = new TentacleClient( - endpoint, - ServerHalibutRuntime, - new PollingTentacleScriptObserverBackoffStrategy(), - new NoTentacleClientObserver(), - clientOptions, - builder.Build()); - } - - int BuildServerHalibutRuntimeAndListen() + protected virtual void ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) { - var serverHalibutRuntimeBuilder = new HalibutRuntimeBuilder() - .WithServerCertificate(TestCertificates.Server) - .WithHalibutTimeoutsAndLimits(HalibutTimeoutsAndLimits.RecommendedValues()) - .WithLogFactory(new TestContextLogCreator("Server", LogLevel.Trace).ToCachingLogFactory()); - - ServerHalibutRuntime = serverHalibutRuntimeBuilder.Build(); - - return ServerHalibutRuntime.Listen(); } [OneTimeTearDown] public async Task OneTimeTearDown() { - await ServerHalibutRuntime.DisposeAsync(); + await serverHalibutRuntime.DisposeAsync(); kubernetesAgentInstaller?.Dispose(); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs similarity index 98% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs index ebd7b8a2a..724f5269f 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs @@ -6,7 +6,7 @@ using Octopus.Tentacle.Diagnostics; using Octopus.Tentacle.Kubernetes.Diagnostics; -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; public class KubernetesAgentMetricsIntegrationTest : KubernetesAgentIntegrationTest { diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs new file mode 100644 index 000000000..3854e7c53 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs @@ -0,0 +1,31 @@ +using System; +using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; + +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; + +[SetUpFixture] +public class KubernetesClusterOneTimeSetUp +{ + KubernetesClusterInstaller? installer; + + [OneTimeSetUp] + public async Task OneTimeSetUp() + { + var toolDownloader = new RequiredToolDownloader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger); + var (kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None); + + installer = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger); + await installer.InstallLatestSupported(); + + KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, installer); + KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath); + KubernetesTestsGlobalContext.Instance.KubeConfigPath = installer.KubeConfigPath; + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + installer?.Dispose(); + KubernetesTestsGlobalContext.Instance.Dispose(); + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs similarity index 97% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs index 22970ed6d..16593ccb7 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs @@ -1,32 +1,28 @@ +using System; using FluentAssertions; using FluentAssertions.Execution; -using Octopus.Client.Model.Endpoints; using Octopus.Tentacle.Client.Scripts.Models; using Octopus.Tentacle.Client.Scripts.Models.Builders; using Octopus.Tentacle.CommonTestUtils; using Octopus.Tentacle.CommonTestUtils.Diagnostics; using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.Capabilities; using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Kubernetes.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; [TestFixture] public class KubernetesScriptServiceV1IntegrationTest : KubernetesAgentIntegrationTest { IRecordedMethodUsages recordedMethodUsages = null!; - protected override TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) + protected override void ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) { builder.RecordMethodUsages(out var recordedUsages); recordedMethodUsages = recordedUsages; - - return builder; } [Test] diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs new file mode 100644 index 000000000..a80f45c14 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs @@ -0,0 +1,173 @@ +using System; +using FluentAssertions; +using Halibut; +using Octopus.Tentacle.Client; +using Octopus.Tentacle.Client.Scripts.Models; +using Octopus.Tentacle.Client.Scripts.Models.Builders; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; +using Octopus.Tentacle.Kubernetes.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; +using Octopus.Tentacle.Tests.Integration.Common.Logging; + +namespace Octopus.Tentacle.Kubernetes.Tests.Integration; + +[TestFixture] +public class KubernetesClientCompatibilityTests +{ + static readonly object[] TestClusterVersions = + [ + new object[] {new ClusterVersion(1, 31)}, + new object[] {new ClusterVersion(1, 30)}, + new object[] {new ClusterVersion(1, 29)}, + new object[] {new ClusterVersion(1, 28)}, + ]; + + KubernetesTestsGlobalContext? testContext; + ILogger logger = null!; + TemporaryDirectory toolsTemporaryDirectory; + string kindExePath; + string helmExePath; + string kubeCtlPath; + KubernetesClusterInstaller? clusterInstaller; + KubernetesAgentInstaller? kubernetesAgentInstaller; + HalibutRuntime serverHalibutRuntime = null!; + string? agentThumbprint; + TraceLogFileLogger? traceLogFileLogger; + CancellationToken cancellationToken; + CancellationTokenSource? cancellationTokenSource; + TentacleClient tentacleClient = null!; + IRecordedMethodUsages recordedMethodUsages = null!; + + [OneTimeSetUp] + public async Task OneTimeSetup() + { + logger = new SerilogLoggerBuilder().Build(); + toolsTemporaryDirectory = new TemporaryDirectory(); + var toolDownloader = new RequiredToolDownloader(toolsTemporaryDirectory, logger); + (kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + toolsTemporaryDirectory.Dispose(); + } + + [TearDown] + public async Task TearDown() + { + if (traceLogFileLogger is not null) await traceLogFileLogger.DisposeAsync(); + if (cancellationTokenSource is not null) + { + await cancellationTokenSource.CancelAsync(); + cancellationTokenSource.Dispose(); + } + clusterInstaller?.Dispose(); + testContext?.Dispose(); + + traceLogFileLogger = null; + cancellationTokenSource = null; + clusterInstaller = null; + testContext = null; + } + + [Test] + [TestCaseSource(nameof(TestClusterVersions))] + public async Task RunSimpleScript(ClusterVersion clusterVersion) + { + await SetUp(clusterVersion); + + // Arrange + var logs = new List(); + var scriptCompleted = false; + + var builder = new ExecuteKubernetesScriptCommandBuilder(LoggingUtils.CurrentTestHash()) + .WithScriptBody(script => script + .Print("Hello World") + .PrintNTimesWithDelay("Yep", 30, TimeSpan.FromMilliseconds(100))); + + var command = builder.Build(); + + // Act + var result = await tentacleClient.ExecuteScript(command, StatusReceived, ScriptCompleted, new InMemoryLog(), cancellationToken); + + // Assert + logs.Should().Contain(po => po.Text.StartsWith("[POD EVENT]")); // Verify that we are receiving some pod events + logs.Should().Contain(po => po.Source == ProcessOutputSource.StdOut && po.Text == "Hello World"); + scriptCompleted.Should().BeTrue(); + result.ExitCode.Should().Be(0); + result.State.Should().Be(ProcessState.Complete); + + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.StartScriptAsync)).Started.Should().Be(1); + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.GetStatusAsync)).Started.Should().BeGreaterThan(1); + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.CompleteScriptAsync)).Started.Should().Be(1); + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.CancelScriptAsync)).Started.Should().Be(0); + + return; + + void StatusReceived(ScriptExecutionStatus status) + { + logs.AddRange(status.Logs); + } + + Task ScriptCompleted(CancellationToken ct) + { + scriptCompleted = true; + return Task.CompletedTask; + } + } + + async Task SetUp(ClusterVersion clusterVersion) + { + testContext = new KubernetesTestsGlobalContext(logger); + + await SetupCluster(clusterVersion); + + kubernetesAgentInstaller = new KubernetesAgentInstaller( + testContext.TemporaryDirectory, + testContext.HelmExePath, + testContext.KubeCtlExePath, + testContext.KubeConfigPath, + testContext.Logger); + + //create a new server halibut runtime + serverHalibutRuntime = SetupHelpers.BuildServerHalibutRuntime(); + var listeningPort = serverHalibutRuntime.Listen(); + + agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, testContext.TentacleImageAndTag, new Dictionary()); + + //trust the generated cert thumbprint + serverHalibutRuntime.Trust(agentThumbprint); + + traceLogFileLogger = new TraceLogFileLogger(LoggingUtils.CurrentTestHash()); + logger = new SerilogLoggerBuilder() + .SetTraceLogFileLogger(traceLogFileLogger) + .Build() + .ForContext(GetType()); + + cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5)); + cancellationToken = cancellationTokenSource.Token; + + tentacleClient = SetupHelpers.BuildTentacleClient(kubernetesAgentInstaller.SubscriptionId, agentThumbprint, serverHalibutRuntime, builder => + { + builder.RecordMethodUsages(out var recordedUsages); + recordedMethodUsages = recordedUsages; + }); + } + + async Task SetupCluster(ClusterVersion clusterVersion) + { + clusterInstaller = new KubernetesClusterInstaller(testContext.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, testContext.Logger); + await clusterInstaller.Install(clusterVersion); + + testContext.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, clusterInstaller); + testContext.SetToolExePaths(helmExePath, kubeCtlPath); + testContext.KubeConfigPath = clusterInstaller.KubeConfigPath; + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs deleted file mode 100644 index cf6fe017a..000000000 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs +++ /dev/null @@ -1,62 +0,0 @@ -using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; -using Octopus.Tentacle.Util; - -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; - -[SetUpFixture] -public class KubernetesClusterOneTimeSetUp -{ - KubernetesClusterInstaller? installer; - - [OneTimeSetUp] - public async Task OneTimeSetUp() - { - var toolDownloader = new RequiredToolDownloader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger); - var (kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None); - - installer = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger); - await installer.Install(); - - KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = GetTentacleImageAndTag(kindExePath); - KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath); - KubernetesTestsGlobalContext.Instance.KubeConfigPath = installer.KubeConfigPath; - } - - string? GetTentacleImageAndTag(string kindExePath) - { - if (installer == null) - { - throw new InvalidOperationException("Expected installer to be set"); - } - //By default, we don't override the values in the helm chart. This is useful if you are just writing new tests and not changing Tentacle code. - string? imageAndTag = null; - if (TeamCityDetection.IsRunningInTeamCity()) - { - //In TeamCity, use the tag of the currently building code - var tag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageTag"); - imageAndTag = $"docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle:{tag}"; - } - else if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"))) - { - imageAndTag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"); - } - else if(bool.TryParse(Environment.GetEnvironmentVariable("KubernetesAgentTests_UseLatestLocalImage"), out var useLocal) && useLocal) - { - //if we should use the latest locally build image, load the tag from docker and load it into kind - var imageLoader = new DockerImageLoader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger, kindExePath); - imageAndTag = imageLoader.LoadMostRecentImageIntoKind(installer.ClusterName); - } - - if(imageAndTag is not null) - KubernetesTestsGlobalContext.Instance.Logger.Information("Using tentacle image: {ImageAndTag}", imageAndTag); - - return imageAndTag; - } - - [OneTimeTearDown] - public void OneTimeTearDown() - { - installer?.Dispose(); - KubernetesTestsGlobalContext.Instance.Dispose(); - } -} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs index 9f7dc9e37..3df570c04 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs @@ -16,6 +16,13 @@ public class KubernetesTestsGlobalContext : IDisposable public string HelmExePath { get; private set; } = null!; public string KubeCtlExePath { get; private set; }= null!; public string? TentacleImageAndTag { get; set; } + + internal KubernetesTestsGlobalContext(ILogger logger) + { + TemporaryDirectory = new TemporaryDirectory(); + + Logger = logger; + } KubernetesTestsGlobalContext() { diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj index 07f0b3d27..412854765 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj @@ -52,14 +52,23 @@ PreserveNewest - - PreserveNewest - PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml new file mode 100644 index 000000000..d6d495524 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 + +nodes: + - role: control-plane + image: kindest/node:v1.28.15@sha256:a7c05c7ae043a0b8c818f5a06188bc2c4098f6cb59ca7d1856df00375d839251 + - role: worker + image: kindest/node:v1.28.15@sha256:a7c05c7ae043a0b8c818f5a06188bc2c4098f6cb59ca7d1856df00375d839251 diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml new file mode 100644 index 000000000..92fd734be --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 + +nodes: + - role: control-plane + image: kindest/node:v1.29.12@sha256:62c0672ba99a4afd7396512848d6fc382906b8f33349ae68fb1dbfe549f70dec + - role: worker + image: kindest/node:v1.29.12@sha256:62c0672ba99a4afd7396512848d6fc382906b8f33349ae68fb1dbfe549f70dec diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-30.yaml similarity index 100% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-30.yaml diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml new file mode 100644 index 000000000..b5b111128 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 + +nodes: + - role: control-plane + image: kindest/node:v1.31.0@sha256:25a3504b2b340954595fa7a6ed1575ef2edadf5abd83c0776a4308b64bf47c93 + - role: worker + image: kindest/node:v1.31.0@sha256:25a3504b2b340954595fa7a6ed1575ef2edadf5abd83c0776a4308b64bf47c93 diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs index 98318efb7..2e8ccb583 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs @@ -20,6 +20,8 @@ public class KubernetesClusterInstaller readonly string kubeCtlPath; readonly ILogger logger; + readonly ClusterVersion latestSupportedClusterVersion = new(1, 31); + public string KubeConfigPath => Path.Combine(tempDir.DirectoryPath, kubeConfigName); public string ClusterName => clusterName; @@ -35,9 +37,19 @@ public KubernetesClusterInstaller(TemporaryDirectory tempDirectory, string kindE kubeConfigName = $"{clusterName}.config"; } - public async Task Install() + public Task InstallLatestSupported() + { + return InstallCluster(latestSupportedClusterVersion); + } + + public Task Install(ClusterVersion clusterVersion) + { + return InstallCluster(clusterVersion); + } + + async Task InstallCluster(ClusterVersion clusterVersion) { - var configFilePath = await WriteFileToTemporaryDirectory("kind-config.yaml"); + var configFilePath = await WriteFileToTemporaryDirectory($"kind-config-v{clusterVersion.Major}-{clusterVersion.Minor}.yaml"); var sw = new Stopwatch(); sw.Restart(); diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs new file mode 100644 index 000000000..f5ed3dca0 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs @@ -0,0 +1,79 @@ +using Halibut; +using Halibut.Diagnostics; +using Halibut.Diagnostics.LogCreators; +using Halibut.Logging; +using Octopus.Tentacle.Client; +using Octopus.Tentacle.Client.Retries; +using Octopus.Tentacle.Client.Scripts; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.Contracts.Observability; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Common.Logging; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; + +static class SetupHelpers +{ + public static HalibutRuntime BuildServerHalibutRuntime() + { + var serverHalibutRuntimeBuilder = new HalibutRuntimeBuilder() + .WithServerCertificate(TestCertificates.Server) + .WithHalibutTimeoutsAndLimits(HalibutTimeoutsAndLimits.RecommendedValues()) + .WithLogFactory(new TestContextLogCreator("Server", LogLevel.Trace).ToCachingLogFactory()); + + return serverHalibutRuntimeBuilder.Build(); + } + + public static TentacleClient BuildTentacleClient(Uri uri, string? thumbprint, HalibutRuntime halibutRuntime, Action tentacleServiceDecoratorBuilderAction) + { + var endpoint = new ServiceEndPoint(uri, thumbprint, halibutRuntime.TimeoutsAndLimits); + + var retrySettings = new RpcRetrySettings(true, TimeSpan.FromMinutes(2)); + var clientOptions = new TentacleClientOptions(retrySettings); + + TentacleClient.CacheServiceWasNotFoundResponseMessages(halibutRuntime); + + var builder = new TentacleServiceDecoratorBuilder(); + tentacleServiceDecoratorBuilderAction(builder); + + return new TentacleClient( + endpoint, + halibutRuntime, + new PollingTentacleScriptObserverBackoffStrategy(), + new NoTentacleClientObserver(), + clientOptions, + builder.Build()); + } + + public static string? GetTentacleImageAndTag(string kindExePath, KubernetesClusterInstaller clusterInstaller) + { + if (clusterInstaller == null) + { + throw new InvalidOperationException("Expected installer to be set"); + } + //By default, we don't override the values in the helm chart. This is useful if you are just writing new tests and not changing Tentacle code. + string? imageAndTag = null; + if (TeamCityDetection.IsRunningInTeamCity()) + { + //In TeamCity, use the tag of the currently building code + var tag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageTag"); + imageAndTag = $"docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle:{tag}"; + } + else if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"))) + { + imageAndTag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"); + } + else if(bool.TryParse(Environment.GetEnvironmentVariable("KubernetesAgentTests_UseLatestLocalImage"), out var useLocal) && useLocal) + { + //if we should use the latest locally build image, load the tag from docker and load it into kind + var imageLoader = new DockerImageLoader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger, kindExePath); + imageAndTag = imageLoader.LoadMostRecentImageIntoKind(clusterInstaller.ClusterName); + } + + if(imageAndTag is not null) + KubernetesTestsGlobalContext.Instance.Logger.Information("Using tentacle image: {ImageAndTag}", imageAndTag); + + return imageAndTag; + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Octopus.Tentacle.csproj b/source/Octopus.Tentacle/Octopus.Tentacle.csproj index fb648dd8b..6247dc364 100644 --- a/source/Octopus.Tentacle/Octopus.Tentacle.csproj +++ b/source/Octopus.Tentacle/Octopus.Tentacle.csproj @@ -46,10 +46,10 @@ $(DefineConstants);HTTP_CLIENT_SUPPORTS_SSL_OPTIONS;REQUIRES_EXPLICIT_LOG_CONFIG;REQUIRES_CODE_PAGE_PROVIDER;USER_INTERACTIVE_DOES_NOT_WORK;DEFAULT_PROXY_IS_NOT_AVAILABLE;HAS_NULLABLE_REF_TYPES - + - +