From 8d49964ff4340b8d3d3907769c10003cacbdcdcb Mon Sep 17 00:00:00 2001 From: Kevin Tchang Date: Wed, 18 Dec 2024 09:12:04 +1100 Subject: [PATCH 1/4] update client version --- .../KubernetesAgentIntegrationTest.cs | 5 +++-- .../KubernetesAgentMetricsIntegrationTest.cs | 2 +- .../KubernetesClusterOneTimeSetUp.cs | 5 +++-- .../KubernetesScriptServiceV1IntegrationTest.cs | 8 +++----- .../KubernetesClientCompatibilityTests.cs | 9 +++++++++ .../Octopus.Tentacle.Kubernetes.Tests.Integration.csproj | 9 ++++++--- .../kind-config-v1-30.yaml} | 0 .../Setup/KindConfiguration/kind-config-v1-31.yaml | 8 ++++++++ .../Setup/KubernetesClusterInstaller.cs | 2 +- source/Octopus.Tentacle/Octopus.Tentacle.csproj | 4 ++-- 10 files changed, 36 insertions(+), 16 deletions(-) rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/{ => KubernetesAgent}/KubernetesAgentIntegrationTest.cs (98%) rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/{ => KubernetesAgent}/KubernetesAgentMetricsIntegrationTest.cs (98%) rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/{ => KubernetesAgent}/KubernetesClusterOneTimeSetUp.cs (95%) rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/{ => KubernetesAgent}/KubernetesScriptServiceV1IntegrationTest.cs (98%) create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs rename source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/{kind-config.yaml => KindConfiguration/kind-config-v1-30.yaml} (100%) create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs similarity index 98% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs index 32cb98e4c..ff8f1034d 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs @@ -1,4 +1,5 @@ -using Halibut; +using System; +using Halibut; using Halibut.Diagnostics; using Halibut.Diagnostics.LogCreators; using Halibut.Logging; @@ -12,7 +13,7 @@ using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Common.Logging; -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; public abstract class KubernetesAgentIntegrationTest { diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs similarity index 98% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs index ebd7b8a2a..724f5269f 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgentMetricsIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentMetricsIntegrationTest.cs @@ -6,7 +6,7 @@ using Octopus.Tentacle.Diagnostics; using Octopus.Tentacle.Kubernetes.Diagnostics; -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; public class KubernetesAgentMetricsIntegrationTest : KubernetesAgentIntegrationTest { diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs similarity index 95% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs index cf6fe017a..e7cca8c71 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClusterOneTimeSetUp.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs @@ -1,7 +1,8 @@ -using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; +using System; +using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; using Octopus.Tentacle.Util; -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; [SetUpFixture] public class KubernetesClusterOneTimeSetUp diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs similarity index 98% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs index 22970ed6d..305027f99 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesScriptServiceV1IntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs @@ -1,19 +1,17 @@ +using System; using FluentAssertions; using FluentAssertions.Execution; -using Octopus.Client.Model.Endpoints; using Octopus.Tentacle.Client.Scripts.Models; using Octopus.Tentacle.Client.Scripts.Models.Builders; using Octopus.Tentacle.CommonTestUtils; using Octopus.Tentacle.CommonTestUtils.Diagnostics; using Octopus.Tentacle.Contracts; -using Octopus.Tentacle.Contracts.Capabilities; using Octopus.Tentacle.Contracts.ClientServices; -using Octopus.Tentacle.Contracts.KubernetesScriptServiceV1; -using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Kubernetes.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; -namespace Octopus.Tentacle.Kubernetes.Tests.Integration; +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; [TestFixture] public class KubernetesScriptServiceV1IntegrationTest : KubernetesAgentIntegrationTest diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs new file mode 100644 index 000000000..9b44c6b42 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs @@ -0,0 +1,9 @@ +using System; + +namespace Octopus.Tentacle.Kubernetes.Tests.Integration; + +[TestFixture] +public class KubernetesClientCompatibilityTests +{ + +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj index 07f0b3d27..9d05e2665 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj @@ -52,14 +52,17 @@ PreserveNewest - - PreserveNewest - PreserveNewest + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-30.yaml similarity index 100% rename from source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml rename to source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-30.yaml diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml new file mode 100644 index 000000000..b5b111128 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-31.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 + +nodes: + - role: control-plane + image: kindest/node:v1.31.0@sha256:25a3504b2b340954595fa7a6ed1575ef2edadf5abd83c0776a4308b64bf47c93 + - role: worker + image: kindest/node:v1.31.0@sha256:25a3504b2b340954595fa7a6ed1575ef2edadf5abd83c0776a4308b64bf47c93 diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs index 98318efb7..f07224dc7 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs @@ -37,7 +37,7 @@ public KubernetesClusterInstaller(TemporaryDirectory tempDirectory, string kindE public async Task Install() { - var configFilePath = await WriteFileToTemporaryDirectory("kind-config.yaml"); + var configFilePath = await WriteFileToTemporaryDirectory("kind-config-v1-31.yaml"); var sw = new Stopwatch(); sw.Restart(); diff --git a/source/Octopus.Tentacle/Octopus.Tentacle.csproj b/source/Octopus.Tentacle/Octopus.Tentacle.csproj index fb648dd8b..6247dc364 100644 --- a/source/Octopus.Tentacle/Octopus.Tentacle.csproj +++ b/source/Octopus.Tentacle/Octopus.Tentacle.csproj @@ -46,10 +46,10 @@ $(DefineConstants);HTTP_CLIENT_SUPPORTS_SSL_OPTIONS;REQUIRES_EXPLICIT_LOG_CONFIG;REQUIRES_CODE_PAGE_PROVIDER;USER_INTERACTIVE_DOES_NOT_WORK;DEFAULT_PROXY_IS_NOT_AVAILABLE;HAS_NULLABLE_REF_TYPES - + - + From 8568485e64a6ddfbae8fdc8473f062fe76b4e10d Mon Sep 17 00:00:00 2001 From: Kevin Tchang Date: Wed, 18 Dec 2024 15:13:08 +1100 Subject: [PATCH 2/4] add compatibility tests --- .../KubernetesAgentIntegrationTest.cs | 54 ++----- .../KubernetesClusterOneTimeSetUp.cs | 34 +--- ...ubernetesScriptServiceV1IntegrationTest.cs | 4 +- .../KubernetesClientCompatibilityTests.cs | 149 ++++++++++++++++++ ...ntacle.Kubernetes.Tests.Integration.csproj | 6 + .../KindConfiguration/kind-config-v1-28.yaml | 8 + .../KindConfiguration/kind-config-v1-29.yaml | 8 + .../Setup/KubernetesClusterInstaller.cs | 16 +- .../Setup/SetupHelpers.cs | 79 ++++++++++ 9 files changed, 275 insertions(+), 83 deletions(-) create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml create mode 100644 source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs index ff8f1034d..3f8b40771 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesAgentIntegrationTest.cs @@ -1,13 +1,7 @@ using System; using Halibut; -using Halibut.Diagnostics; -using Halibut.Diagnostics.LogCreators; -using Halibut.Logging; using Octopus.Tentacle.Client; -using Octopus.Tentacle.Client.Retries; -using Octopus.Tentacle.Client.Scripts; using Octopus.Tentacle.CommonTestUtils; -using Octopus.Tentacle.Contracts.Observability; using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; using Octopus.Tentacle.Kubernetes.Tests.Integration.Tooling; using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; @@ -24,9 +18,7 @@ public abstract class KubernetesAgentIntegrationTest protected ILogger? Logger { get; private set; } protected KubernetesAgentInstaller KubernetesAgentInstaller => kubernetesAgentInstaller ?? throw new InvalidOperationException("Expected kubernetesAgentInstaller to be set"); - - protected HalibutRuntime ServerHalibutRuntime { get; private set; } = null!; - + protected TentacleClient TentacleClient { get; private set; } = null!; protected CancellationToken CancellationToken { get; private set; } @@ -35,6 +27,8 @@ public abstract class KubernetesAgentIntegrationTest protected readonly IDictionary CustomHelmValues = new Dictionary(); + HalibutRuntime serverHalibutRuntime; + string? agentThumbprint; [OneTimeSetUp] @@ -55,12 +49,13 @@ public async Task OneTimeSetUp() KubernetesTestsGlobalContext.Instance.Logger); //create a new server halibut runtime - var listeningPort = BuildServerHalibutRuntimeAndListen(); + serverHalibutRuntime = SetupHelpers.BuildServerHalibutRuntime(); + var listeningPort = serverHalibutRuntime.Listen(); agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag, CustomHelmValues); //trust the generated cert thumbprint - ServerHalibutRuntime.Trust(agentThumbprint); + serverHalibutRuntime.Trust(agentThumbprint); } [SetUp] @@ -77,7 +72,7 @@ public void SetUp() CancellationToken = cancellationTokenSource.Token; //each test should get its own tentacle client, so it gets its own builders - BuildTentacleClient(); + TentacleClient = SetupHelpers.BuildTentacleClient(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, serverHalibutRuntime, ConfigureTentacleServiceDecoratorBuilder); } [TearDown] @@ -92,45 +87,14 @@ public async Task TearDown() cancellationTokenSource?.Dispose(); } - protected virtual TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) => builder; - - void BuildTentacleClient() - { - var endpoint = new ServiceEndPoint(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, ServerHalibutRuntime.TimeoutsAndLimits); - - var retrySettings = new RpcRetrySettings(true, TimeSpan.FromMinutes(2)); - var clientOptions = new TentacleClientOptions(retrySettings); - - TentacleClient.CacheServiceWasNotFoundResponseMessages(ServerHalibutRuntime); - - var builder = new TentacleServiceDecoratorBuilder(); - ConfigureTentacleServiceDecoratorBuilder(builder); - - TentacleClient = new TentacleClient( - endpoint, - ServerHalibutRuntime, - new PollingTentacleScriptObserverBackoffStrategy(), - new NoTentacleClientObserver(), - clientOptions, - builder.Build()); - } - - int BuildServerHalibutRuntimeAndListen() + protected virtual void ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) { - var serverHalibutRuntimeBuilder = new HalibutRuntimeBuilder() - .WithServerCertificate(TestCertificates.Server) - .WithHalibutTimeoutsAndLimits(HalibutTimeoutsAndLimits.RecommendedValues()) - .WithLogFactory(new TestContextLogCreator("Server", LogLevel.Trace).ToCachingLogFactory()); - - ServerHalibutRuntime = serverHalibutRuntimeBuilder.Build(); - - return ServerHalibutRuntime.Listen(); } [OneTimeTearDown] public async Task OneTimeTearDown() { - await ServerHalibutRuntime.DisposeAsync(); + await serverHalibutRuntime.DisposeAsync(); kubernetesAgentInstaller?.Dispose(); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs index e7cca8c71..62d9b989b 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs @@ -1,6 +1,5 @@ using System; using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; -using Octopus.Tentacle.Util; namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent; @@ -18,42 +17,11 @@ public async Task OneTimeSetUp() installer = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger); await installer.Install(); - KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = GetTentacleImageAndTag(kindExePath); + KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, installer); KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath); KubernetesTestsGlobalContext.Instance.KubeConfigPath = installer.KubeConfigPath; } - string? GetTentacleImageAndTag(string kindExePath) - { - if (installer == null) - { - throw new InvalidOperationException("Expected installer to be set"); - } - //By default, we don't override the values in the helm chart. This is useful if you are just writing new tests and not changing Tentacle code. - string? imageAndTag = null; - if (TeamCityDetection.IsRunningInTeamCity()) - { - //In TeamCity, use the tag of the currently building code - var tag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageTag"); - imageAndTag = $"docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle:{tag}"; - } - else if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"))) - { - imageAndTag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"); - } - else if(bool.TryParse(Environment.GetEnvironmentVariable("KubernetesAgentTests_UseLatestLocalImage"), out var useLocal) && useLocal) - { - //if we should use the latest locally build image, load the tag from docker and load it into kind - var imageLoader = new DockerImageLoader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger, kindExePath); - imageAndTag = imageLoader.LoadMostRecentImageIntoKind(installer.ClusterName); - } - - if(imageAndTag is not null) - KubernetesTestsGlobalContext.Instance.Logger.Information("Using tentacle image: {ImageAndTag}", imageAndTag); - - return imageAndTag; - } - [OneTimeTearDown] public void OneTimeTearDown() { diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs index 305027f99..16593ccb7 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesScriptServiceV1IntegrationTest.cs @@ -18,13 +18,11 @@ public class KubernetesScriptServiceV1IntegrationTest : KubernetesAgentIntegrati { IRecordedMethodUsages recordedMethodUsages = null!; - protected override TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) + protected override void ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) { builder.RecordMethodUsages(out var recordedUsages); recordedMethodUsages = recordedUsages; - - return builder; } [Test] diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs index 9b44c6b42..41d73c2f3 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs @@ -1,9 +1,158 @@ using System; +using FluentAssertions; +using Halibut; +using Octopus.Tentacle.Client; +using Octopus.Tentacle.Client.Scripts.Models; +using Octopus.Tentacle.Client.Scripts.Models.Builders; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.CommonTestUtils.Diagnostics; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ClientServices; +using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; +using Octopus.Tentacle.Kubernetes.Tests.Integration.Util; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies; +using Octopus.Tentacle.Tests.Integration.Common.Logging; namespace Octopus.Tentacle.Kubernetes.Tests.Integration; [TestFixture] public class KubernetesClientCompatibilityTests { + static readonly object[] TestClusterVersions = + [ + new object[] {new ClusterVersion(1, 31)}, + new object[] {new ClusterVersion(1, 30)}, + new object[] {new ClusterVersion(1, 29)}, + new object[] {new ClusterVersion(1, 28)}, + ]; + string kindExePath; + string helmExePath; + string kubeCtlPath; + KubernetesClusterInstaller clusterInstaller = null!; + KubernetesAgentInstaller? kubernetesAgentInstaller; + HalibutRuntime serverHalibutRuntime = null!; + string? agentThumbprint; + TraceLogFileLogger? traceLogFileLogger; + CancellationToken cancellationToken; + CancellationTokenSource? cancellationTokenSource; + ILogger? logger; + TentacleClient tentacleClient = null!; + IRecordedMethodUsages recordedMethodUsages = null!; + + [OneTimeSetUp] + public async Task OneTimeSetup() + { + var toolDownloader = new RequiredToolDownloader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger); + (kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None); + } + + [TearDown] + public async Task TearDown() + { + if (traceLogFileLogger is not null) await traceLogFileLogger.DisposeAsync(); + + if (cancellationTokenSource is not null) + { + await cancellationTokenSource.CancelAsync(); + cancellationTokenSource.Dispose(); + } + + clusterInstaller.Dispose(); + KubernetesTestsGlobalContext.Instance.Dispose(); + } + + [Test] + [TestCaseSource(nameof(TestClusterVersions))] + public async Task RunSimpleScript(ClusterVersion clusterVersion) + { + await SetUp(clusterVersion); + + // Arrange + var logs = new List(); + var scriptCompleted = false; + + var builder = new ExecuteKubernetesScriptCommandBuilder(LoggingUtils.CurrentTestHash()) + .WithScriptBody(script => script + .Print("Hello World") + .PrintNTimesWithDelay("Yep", 30, TimeSpan.FromMilliseconds(100))); + + var command = builder.Build(); + + // Act + var result = await tentacleClient.ExecuteScript(command, StatusReceived, ScriptCompleted, new InMemoryLog(), cancellationToken); + + // Assert + logs.Should().Contain(po => po.Text.StartsWith("[POD EVENT]")); // Verify that we are receiving some pod events + logs.Should().Contain(po => po.Source == ProcessOutputSource.StdOut && po.Text == "Hello World"); + scriptCompleted.Should().BeTrue(); + result.ExitCode.Should().Be(0); + result.State.Should().Be(ProcessState.Complete); + + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.StartScriptAsync)).Started.Should().Be(1); + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.GetStatusAsync)).Started.Should().BeGreaterThan(1); + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.CompleteScriptAsync)).Started.Should().Be(1); + recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.CancelScriptAsync)).Started.Should().Be(0); + + return; + + void StatusReceived(ScriptExecutionStatus status) + { + logs.AddRange(status.Logs); + } + + Task ScriptCompleted(CancellationToken ct) + { + scriptCompleted = true; + return Task.CompletedTask; + } + } + + async Task SetUp(ClusterVersion clusterVersion) + { + await SetupCluster(clusterVersion); + + kubernetesAgentInstaller = new KubernetesAgentInstaller( + KubernetesTestsGlobalContext.Instance.TemporaryDirectory, + KubernetesTestsGlobalContext.Instance.HelmExePath, + KubernetesTestsGlobalContext.Instance.KubeCtlExePath, + KubernetesTestsGlobalContext.Instance.KubeConfigPath, + KubernetesTestsGlobalContext.Instance.Logger); + + //create a new server halibut runtime + serverHalibutRuntime = SetupHelpers.BuildServerHalibutRuntime(); + var listeningPort = serverHalibutRuntime.Listen(); + + agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag, new Dictionary()); + + //trust the generated cert thumbprint + serverHalibutRuntime.Trust(agentThumbprint); + + traceLogFileLogger = new TraceLogFileLogger(LoggingUtils.CurrentTestHash()); + logger = new SerilogLoggerBuilder() + .SetTraceLogFileLogger(traceLogFileLogger) + .Build() + .ForContext(GetType()); + + cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5)); + cancellationToken = cancellationTokenSource.Token; + + tentacleClient = SetupHelpers.BuildTentacleClient(kubernetesAgentInstaller.SubscriptionId, agentThumbprint, serverHalibutRuntime, builder => + { + builder.RecordMethodUsages(out var recordedUsages); + recordedMethodUsages = recordedUsages; + }); + } + + async Task SetupCluster(ClusterVersion clusterVersion) + { + clusterInstaller = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger); + await clusterInstaller.Install(clusterVersion); + + KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, clusterInstaller); + KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath); + KubernetesTestsGlobalContext.Instance.KubeConfigPath = clusterInstaller.KubeConfigPath; + } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj index 9d05e2665..412854765 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Octopus.Tentacle.Kubernetes.Tests.Integration.csproj @@ -57,6 +57,12 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml new file mode 100644 index 000000000..d6d495524 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-28.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 + +nodes: + - role: control-plane + image: kindest/node:v1.28.15@sha256:a7c05c7ae043a0b8c818f5a06188bc2c4098f6cb59ca7d1856df00375d839251 + - role: worker + image: kindest/node:v1.28.15@sha256:a7c05c7ae043a0b8c818f5a06188bc2c4098f6cb59ca7d1856df00375d839251 diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml new file mode 100644 index 000000000..92fd734be --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KindConfiguration/kind-config-v1-29.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 + +nodes: + - role: control-plane + image: kindest/node:v1.29.12@sha256:62c0672ba99a4afd7396512848d6fc382906b8f33349ae68fb1dbfe549f70dec + - role: worker + image: kindest/node:v1.29.12@sha256:62c0672ba99a4afd7396512848d6fc382906b8f33349ae68fb1dbfe549f70dec diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs index f07224dc7..28724fb06 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs @@ -20,6 +20,8 @@ public class KubernetesClusterInstaller readonly string kubeCtlPath; readonly ILogger logger; + readonly ClusterVersion latestSupportedClusterVersion = new(1, 31); + public string KubeConfigPath => Path.Combine(tempDir.DirectoryPath, kubeConfigName); public string ClusterName => clusterName; @@ -35,9 +37,19 @@ public KubernetesClusterInstaller(TemporaryDirectory tempDirectory, string kindE kubeConfigName = $"{clusterName}.config"; } - public async Task Install() + public Task Install() + { + return InstallCluster(latestSupportedClusterVersion); + } + + public Task Install(ClusterVersion clusterVersion) + { + return InstallCluster(clusterVersion); + } + + async Task InstallCluster(ClusterVersion clusterVersion) { - var configFilePath = await WriteFileToTemporaryDirectory("kind-config-v1-31.yaml"); + var configFilePath = await WriteFileToTemporaryDirectory($"kind-config-v{clusterVersion.Major}-{clusterVersion.Minor}.yaml"); var sw = new Stopwatch(); sw.Restart(); diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs new file mode 100644 index 000000000..f5ed3dca0 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/SetupHelpers.cs @@ -0,0 +1,79 @@ +using Halibut; +using Halibut.Diagnostics; +using Halibut.Diagnostics.LogCreators; +using Halibut.Logging; +using Octopus.Tentacle.Client; +using Octopus.Tentacle.Client.Retries; +using Octopus.Tentacle.Client.Scripts; +using Octopus.Tentacle.CommonTestUtils; +using Octopus.Tentacle.Contracts.Observability; +using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators; +using Octopus.Tentacle.Tests.Integration.Common.Logging; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup; + +static class SetupHelpers +{ + public static HalibutRuntime BuildServerHalibutRuntime() + { + var serverHalibutRuntimeBuilder = new HalibutRuntimeBuilder() + .WithServerCertificate(TestCertificates.Server) + .WithHalibutTimeoutsAndLimits(HalibutTimeoutsAndLimits.RecommendedValues()) + .WithLogFactory(new TestContextLogCreator("Server", LogLevel.Trace).ToCachingLogFactory()); + + return serverHalibutRuntimeBuilder.Build(); + } + + public static TentacleClient BuildTentacleClient(Uri uri, string? thumbprint, HalibutRuntime halibutRuntime, Action tentacleServiceDecoratorBuilderAction) + { + var endpoint = new ServiceEndPoint(uri, thumbprint, halibutRuntime.TimeoutsAndLimits); + + var retrySettings = new RpcRetrySettings(true, TimeSpan.FromMinutes(2)); + var clientOptions = new TentacleClientOptions(retrySettings); + + TentacleClient.CacheServiceWasNotFoundResponseMessages(halibutRuntime); + + var builder = new TentacleServiceDecoratorBuilder(); + tentacleServiceDecoratorBuilderAction(builder); + + return new TentacleClient( + endpoint, + halibutRuntime, + new PollingTentacleScriptObserverBackoffStrategy(), + new NoTentacleClientObserver(), + clientOptions, + builder.Build()); + } + + public static string? GetTentacleImageAndTag(string kindExePath, KubernetesClusterInstaller clusterInstaller) + { + if (clusterInstaller == null) + { + throw new InvalidOperationException("Expected installer to be set"); + } + //By default, we don't override the values in the helm chart. This is useful if you are just writing new tests and not changing Tentacle code. + string? imageAndTag = null; + if (TeamCityDetection.IsRunningInTeamCity()) + { + //In TeamCity, use the tag of the currently building code + var tag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageTag"); + imageAndTag = $"docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle:{tag}"; + } + else if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"))) + { + imageAndTag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag"); + } + else if(bool.TryParse(Environment.GetEnvironmentVariable("KubernetesAgentTests_UseLatestLocalImage"), out var useLocal) && useLocal) + { + //if we should use the latest locally build image, load the tag from docker and load it into kind + var imageLoader = new DockerImageLoader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger, kindExePath); + imageAndTag = imageLoader.LoadMostRecentImageIntoKind(clusterInstaller.ClusterName); + } + + if(imageAndTag is not null) + KubernetesTestsGlobalContext.Instance.Logger.Information("Using tentacle image: {ImageAndTag}", imageAndTag); + + return imageAndTag; + } +} \ No newline at end of file From 374ca7cfb34d6dc46a37c518e2e5bc35eadb8c54 Mon Sep 17 00:00:00 2001 From: Kevin Tchang Date: Wed, 18 Dec 2024 15:54:24 +1100 Subject: [PATCH 3/4] fix tests --- .../KubernetesClientCompatibilityTests.cs | 49 ++++++++++++------- .../KubernetesTestsGlobalContext.cs | 7 +++ 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs index 41d73c2f3..a80f45c14 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesClientCompatibilityTests.cs @@ -26,41 +26,54 @@ public class KubernetesClientCompatibilityTests new object[] {new ClusterVersion(1, 29)}, new object[] {new ClusterVersion(1, 28)}, ]; - + + KubernetesTestsGlobalContext? testContext; + ILogger logger = null!; + TemporaryDirectory toolsTemporaryDirectory; string kindExePath; string helmExePath; string kubeCtlPath; - KubernetesClusterInstaller clusterInstaller = null!; + KubernetesClusterInstaller? clusterInstaller; KubernetesAgentInstaller? kubernetesAgentInstaller; HalibutRuntime serverHalibutRuntime = null!; string? agentThumbprint; TraceLogFileLogger? traceLogFileLogger; CancellationToken cancellationToken; CancellationTokenSource? cancellationTokenSource; - ILogger? logger; TentacleClient tentacleClient = null!; IRecordedMethodUsages recordedMethodUsages = null!; [OneTimeSetUp] public async Task OneTimeSetup() { - var toolDownloader = new RequiredToolDownloader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger); + logger = new SerilogLoggerBuilder().Build(); + toolsTemporaryDirectory = new TemporaryDirectory(); + var toolDownloader = new RequiredToolDownloader(toolsTemporaryDirectory, logger); (kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None); } + [OneTimeTearDown] + public void OneTimeTearDown() + { + toolsTemporaryDirectory.Dispose(); + } + [TearDown] public async Task TearDown() { if (traceLogFileLogger is not null) await traceLogFileLogger.DisposeAsync(); - if (cancellationTokenSource is not null) { await cancellationTokenSource.CancelAsync(); cancellationTokenSource.Dispose(); } + clusterInstaller?.Dispose(); + testContext?.Dispose(); - clusterInstaller.Dispose(); - KubernetesTestsGlobalContext.Instance.Dispose(); + traceLogFileLogger = null; + cancellationTokenSource = null; + clusterInstaller = null; + testContext = null; } [Test] @@ -111,20 +124,22 @@ Task ScriptCompleted(CancellationToken ct) async Task SetUp(ClusterVersion clusterVersion) { + testContext = new KubernetesTestsGlobalContext(logger); + await SetupCluster(clusterVersion); kubernetesAgentInstaller = new KubernetesAgentInstaller( - KubernetesTestsGlobalContext.Instance.TemporaryDirectory, - KubernetesTestsGlobalContext.Instance.HelmExePath, - KubernetesTestsGlobalContext.Instance.KubeCtlExePath, - KubernetesTestsGlobalContext.Instance.KubeConfigPath, - KubernetesTestsGlobalContext.Instance.Logger); + testContext.TemporaryDirectory, + testContext.HelmExePath, + testContext.KubeCtlExePath, + testContext.KubeConfigPath, + testContext.Logger); //create a new server halibut runtime serverHalibutRuntime = SetupHelpers.BuildServerHalibutRuntime(); var listeningPort = serverHalibutRuntime.Listen(); - agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag, new Dictionary()); + agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, testContext.TentacleImageAndTag, new Dictionary()); //trust the generated cert thumbprint serverHalibutRuntime.Trust(agentThumbprint); @@ -148,11 +163,11 @@ async Task SetUp(ClusterVersion clusterVersion) async Task SetupCluster(ClusterVersion clusterVersion) { - clusterInstaller = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger); + clusterInstaller = new KubernetesClusterInstaller(testContext.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, testContext.Logger); await clusterInstaller.Install(clusterVersion); - KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, clusterInstaller); - KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath); - KubernetesTestsGlobalContext.Instance.KubeConfigPath = clusterInstaller.KubeConfigPath; + testContext.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, clusterInstaller); + testContext.SetToolExePaths(helmExePath, kubeCtlPath); + testContext.KubeConfigPath = clusterInstaller.KubeConfigPath; } } \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs index 9f7dc9e37..3df570c04 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesTestsGlobalContext.cs @@ -16,6 +16,13 @@ public class KubernetesTestsGlobalContext : IDisposable public string HelmExePath { get; private set; } = null!; public string KubeCtlExePath { get; private set; }= null!; public string? TentacleImageAndTag { get; set; } + + internal KubernetesTestsGlobalContext(ILogger logger) + { + TemporaryDirectory = new TemporaryDirectory(); + + Logger = logger; + } KubernetesTestsGlobalContext() { From 2532dd6dccb7cafda00c886c6dc6b18eef255561 Mon Sep 17 00:00:00 2001 From: Kevin Tchang Date: Thu, 19 Dec 2024 14:25:22 +1100 Subject: [PATCH 4/4] rename --- .../KubernetesAgent/KubernetesClusterOneTimeSetUp.cs | 2 +- .../Setup/KubernetesClusterInstaller.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs index 62d9b989b..3854e7c53 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/KubernetesAgent/KubernetesClusterOneTimeSetUp.cs @@ -15,7 +15,7 @@ public async Task OneTimeSetUp() var (kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None); installer = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger); - await installer.Install(); + await installer.InstallLatestSupported(); KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, installer); KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath); diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs index 28724fb06..2e8ccb583 100644 --- a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs +++ b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/KubernetesClusterInstaller.cs @@ -37,7 +37,7 @@ public KubernetesClusterInstaller(TemporaryDirectory tempDirectory, string kindE kubeConfigName = $"{clusterName}.config"; } - public Task Install() + public Task InstallLatestSupported() { return InstallCluster(latestSupportedClusterVersion); }