Skip to content

Commit

Permalink
add compatibility tests
Browse files Browse the repository at this point in the history
  • Loading branch information
kevjt committed Dec 18, 2024
1 parent 8d49964 commit 8568485
Show file tree
Hide file tree
Showing 9 changed files with 275 additions and 83 deletions.
Original file line number Diff line number Diff line change
@@ -1,13 +1,7 @@
using System;
using Halibut;
using Halibut.Diagnostics;
using Halibut.Diagnostics.LogCreators;
using Halibut.Logging;
using Octopus.Tentacle.Client;
using Octopus.Tentacle.Client.Retries;
using Octopus.Tentacle.Client.Scripts;
using Octopus.Tentacle.CommonTestUtils;
using Octopus.Tentacle.Contracts.Observability;
using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup;
using Octopus.Tentacle.Kubernetes.Tests.Integration.Tooling;
using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
Expand All @@ -24,9 +18,7 @@ public abstract class KubernetesAgentIntegrationTest
protected ILogger? Logger { get; private set; }

protected KubernetesAgentInstaller KubernetesAgentInstaller => kubernetesAgentInstaller ?? throw new InvalidOperationException("Expected kubernetesAgentInstaller to be set");

protected HalibutRuntime ServerHalibutRuntime { get; private set; } = null!;


protected TentacleClient TentacleClient { get; private set; } = null!;

protected CancellationToken CancellationToken { get; private set; }
Expand All @@ -35,6 +27,8 @@ public abstract class KubernetesAgentIntegrationTest

protected readonly IDictionary<string, string> CustomHelmValues = new Dictionary<string, string>();

HalibutRuntime serverHalibutRuntime;

string? agentThumbprint;

[OneTimeSetUp]
Expand All @@ -55,12 +49,13 @@ public async Task OneTimeSetUp()
KubernetesTestsGlobalContext.Instance.Logger);

//create a new server halibut runtime
var listeningPort = BuildServerHalibutRuntimeAndListen();
serverHalibutRuntime = SetupHelpers.BuildServerHalibutRuntime();
var listeningPort = serverHalibutRuntime.Listen();

agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag, CustomHelmValues);

//trust the generated cert thumbprint
ServerHalibutRuntime.Trust(agentThumbprint);
serverHalibutRuntime.Trust(agentThumbprint);
}

[SetUp]
Expand All @@ -77,7 +72,7 @@ public void SetUp()
CancellationToken = cancellationTokenSource.Token;

//each test should get its own tentacle client, so it gets its own builders
BuildTentacleClient();
TentacleClient = SetupHelpers.BuildTentacleClient(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, serverHalibutRuntime, ConfigureTentacleServiceDecoratorBuilder);
}

[TearDown]
Expand All @@ -92,45 +87,14 @@ public async Task TearDown()
cancellationTokenSource?.Dispose();
}

protected virtual TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder) => builder;

void BuildTentacleClient()
{
var endpoint = new ServiceEndPoint(KubernetesAgentInstaller.SubscriptionId, agentThumbprint, ServerHalibutRuntime.TimeoutsAndLimits);

var retrySettings = new RpcRetrySettings(true, TimeSpan.FromMinutes(2));
var clientOptions = new TentacleClientOptions(retrySettings);

TentacleClient.CacheServiceWasNotFoundResponseMessages(ServerHalibutRuntime);

var builder = new TentacleServiceDecoratorBuilder();
ConfigureTentacleServiceDecoratorBuilder(builder);

TentacleClient = new TentacleClient(
endpoint,
ServerHalibutRuntime,
new PollingTentacleScriptObserverBackoffStrategy(),
new NoTentacleClientObserver(),
clientOptions,
builder.Build());
}

int BuildServerHalibutRuntimeAndListen()
protected virtual void ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder)
{
var serverHalibutRuntimeBuilder = new HalibutRuntimeBuilder()
.WithServerCertificate(TestCertificates.Server)
.WithHalibutTimeoutsAndLimits(HalibutTimeoutsAndLimits.RecommendedValues())
.WithLogFactory(new TestContextLogCreator("Server", LogLevel.Trace).ToCachingLogFactory());

ServerHalibutRuntime = serverHalibutRuntimeBuilder.Build();

return ServerHalibutRuntime.Listen();
}

[OneTimeTearDown]
public async Task OneTimeTearDown()
{
await ServerHalibutRuntime.DisposeAsync();
await serverHalibutRuntime.DisposeAsync();
kubernetesAgentInstaller?.Dispose();
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
using System;
using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup;
using Octopus.Tentacle.Util;

namespace Octopus.Tentacle.Kubernetes.Tests.Integration.KubernetesAgent;

Expand All @@ -18,42 +17,11 @@ public async Task OneTimeSetUp()
installer = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger);
await installer.Install();

KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = GetTentacleImageAndTag(kindExePath);
KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, installer);
KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath);
KubernetesTestsGlobalContext.Instance.KubeConfigPath = installer.KubeConfigPath;
}

string? GetTentacleImageAndTag(string kindExePath)
{
if (installer == null)
{
throw new InvalidOperationException("Expected installer to be set");
}
//By default, we don't override the values in the helm chart. This is useful if you are just writing new tests and not changing Tentacle code.
string? imageAndTag = null;
if (TeamCityDetection.IsRunningInTeamCity())
{
//In TeamCity, use the tag of the currently building code
var tag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageTag");
imageAndTag = $"docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle:{tag}";
}
else if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag")))
{
imageAndTag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag");
}
else if(bool.TryParse(Environment.GetEnvironmentVariable("KubernetesAgentTests_UseLatestLocalImage"), out var useLocal) && useLocal)
{
//if we should use the latest locally build image, load the tag from docker and load it into kind
var imageLoader = new DockerImageLoader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger, kindExePath);
imageAndTag = imageLoader.LoadMostRecentImageIntoKind(installer.ClusterName);
}

if(imageAndTag is not null)
KubernetesTestsGlobalContext.Instance.Logger.Information("Using tentacle image: {ImageAndTag}", imageAndTag);

return imageAndTag;
}

[OneTimeTearDown]
public void OneTimeTearDown()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,11 @@ public class KubernetesScriptServiceV1IntegrationTest : KubernetesAgentIntegrati
{
IRecordedMethodUsages recordedMethodUsages = null!;

protected override TentacleServiceDecoratorBuilder ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder)
protected override void ConfigureTentacleServiceDecoratorBuilder(TentacleServiceDecoratorBuilder builder)
{
builder.RecordMethodUsages<IAsyncClientKubernetesScriptServiceV1>(out var recordedUsages);

recordedMethodUsages = recordedUsages;

return builder;
}

[Test]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,158 @@
using System;
using FluentAssertions;
using Halibut;
using Octopus.Tentacle.Client;
using Octopus.Tentacle.Client.Scripts.Models;
using Octopus.Tentacle.Client.Scripts.Models.Builders;
using Octopus.Tentacle.CommonTestUtils;
using Octopus.Tentacle.CommonTestUtils.Diagnostics;
using Octopus.Tentacle.Contracts;
using Octopus.Tentacle.Contracts.ClientServices;
using Octopus.Tentacle.Kubernetes.Tests.Integration.Setup;
using Octopus.Tentacle.Kubernetes.Tests.Integration.Util;
using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators;
using Octopus.Tentacle.Tests.Integration.Common.Builders.Decorators.Proxies;
using Octopus.Tentacle.Tests.Integration.Common.Logging;

namespace Octopus.Tentacle.Kubernetes.Tests.Integration;

[TestFixture]
public class KubernetesClientCompatibilityTests
{
static readonly object[] TestClusterVersions =
[
new object[] {new ClusterVersion(1, 31)},
new object[] {new ClusterVersion(1, 30)},
new object[] {new ClusterVersion(1, 29)},
new object[] {new ClusterVersion(1, 28)},
];

string kindExePath;
string helmExePath;
string kubeCtlPath;
KubernetesClusterInstaller clusterInstaller = null!;
KubernetesAgentInstaller? kubernetesAgentInstaller;
HalibutRuntime serverHalibutRuntime = null!;
string? agentThumbprint;
TraceLogFileLogger? traceLogFileLogger;
CancellationToken cancellationToken;
CancellationTokenSource? cancellationTokenSource;
ILogger? logger;
TentacleClient tentacleClient = null!;
IRecordedMethodUsages recordedMethodUsages = null!;

[OneTimeSetUp]
public async Task OneTimeSetup()
{
var toolDownloader = new RequiredToolDownloader(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, KubernetesTestsGlobalContext.Instance.Logger);
(kindExePath, helmExePath, kubeCtlPath) = await toolDownloader.DownloadRequiredTools(CancellationToken.None);
}

[TearDown]
public async Task TearDown()
{
if (traceLogFileLogger is not null) await traceLogFileLogger.DisposeAsync();

if (cancellationTokenSource is not null)
{
await cancellationTokenSource.CancelAsync();
cancellationTokenSource.Dispose();
}

clusterInstaller.Dispose();
KubernetesTestsGlobalContext.Instance.Dispose();
}

[Test]
[TestCaseSource(nameof(TestClusterVersions))]
public async Task RunSimpleScript(ClusterVersion clusterVersion)
{
await SetUp(clusterVersion);

// Arrange
var logs = new List<ProcessOutput>();
var scriptCompleted = false;

var builder = new ExecuteKubernetesScriptCommandBuilder(LoggingUtils.CurrentTestHash())
.WithScriptBody(script => script
.Print("Hello World")
.PrintNTimesWithDelay("Yep", 30, TimeSpan.FromMilliseconds(100)));

var command = builder.Build();

// Act
var result = await tentacleClient.ExecuteScript(command, StatusReceived, ScriptCompleted, new InMemoryLog(), cancellationToken);

// Assert
logs.Should().Contain(po => po.Text.StartsWith("[POD EVENT]")); // Verify that we are receiving some pod events
logs.Should().Contain(po => po.Source == ProcessOutputSource.StdOut && po.Text == "Hello World");
scriptCompleted.Should().BeTrue();
result.ExitCode.Should().Be(0);
result.State.Should().Be(ProcessState.Complete);

recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.StartScriptAsync)).Started.Should().Be(1);
recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.GetStatusAsync)).Started.Should().BeGreaterThan(1);
recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.CompleteScriptAsync)).Started.Should().Be(1);
recordedMethodUsages.For(nameof(IAsyncClientKubernetesScriptServiceV1.CancelScriptAsync)).Started.Should().Be(0);

return;

void StatusReceived(ScriptExecutionStatus status)
{
logs.AddRange(status.Logs);
}

Task ScriptCompleted(CancellationToken ct)
{
scriptCompleted = true;
return Task.CompletedTask;
}
}

async Task SetUp(ClusterVersion clusterVersion)
{
await SetupCluster(clusterVersion);

kubernetesAgentInstaller = new KubernetesAgentInstaller(
KubernetesTestsGlobalContext.Instance.TemporaryDirectory,
KubernetesTestsGlobalContext.Instance.HelmExePath,
KubernetesTestsGlobalContext.Instance.KubeCtlExePath,
KubernetesTestsGlobalContext.Instance.KubeConfigPath,
KubernetesTestsGlobalContext.Instance.Logger);

//create a new server halibut runtime
serverHalibutRuntime = SetupHelpers.BuildServerHalibutRuntime();
var listeningPort = serverHalibutRuntime.Listen();

agentThumbprint = await kubernetesAgentInstaller.InstallAgent(listeningPort, KubernetesTestsGlobalContext.Instance.TentacleImageAndTag, new Dictionary<string, string>());

//trust the generated cert thumbprint
serverHalibutRuntime.Trust(agentThumbprint);

traceLogFileLogger = new TraceLogFileLogger(LoggingUtils.CurrentTestHash());
logger = new SerilogLoggerBuilder()
.SetTraceLogFileLogger(traceLogFileLogger)
.Build()
.ForContext(GetType());

cancellationTokenSource = new CancellationTokenSource();
cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(5));
cancellationToken = cancellationTokenSource.Token;

tentacleClient = SetupHelpers.BuildTentacleClient(kubernetesAgentInstaller.SubscriptionId, agentThumbprint, serverHalibutRuntime, builder =>
{
builder.RecordMethodUsages<IAsyncClientKubernetesScriptServiceV1>(out var recordedUsages);
recordedMethodUsages = recordedUsages;
});
}

async Task SetupCluster(ClusterVersion clusterVersion)
{
clusterInstaller = new KubernetesClusterInstaller(KubernetesTestsGlobalContext.Instance.TemporaryDirectory, kindExePath, helmExePath, kubeCtlPath, KubernetesTestsGlobalContext.Instance.Logger);
await clusterInstaller.Install(clusterVersion);

KubernetesTestsGlobalContext.Instance.TentacleImageAndTag = SetupHelpers.GetTentacleImageAndTag(kindExePath, clusterInstaller);
KubernetesTestsGlobalContext.Instance.SetToolExePaths(helmExePath, kubeCtlPath);
KubernetesTestsGlobalContext.Instance.KubeConfigPath = clusterInstaller.KubeConfigPath;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</EmbeddedResource>
<None Remove="Setup\teamcity-network-routing.yaml" />
<EmbeddedResource Include="Setup\KindConfiguration\kind-config-v1-28.yaml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="Setup\KindConfiguration\kind-config-v1-29.yaml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="Setup\KindConfiguration\kind-config-v1-30.yaml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</EmbeddedResource>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4

nodes:
- role: control-plane
image: kindest/node:v1.28.15@sha256:a7c05c7ae043a0b8c818f5a06188bc2c4098f6cb59ca7d1856df00375d839251
- role: worker
image: kindest/node:v1.28.15@sha256:a7c05c7ae043a0b8c818f5a06188bc2c4098f6cb59ca7d1856df00375d839251
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4

nodes:
- role: control-plane
image: kindest/node:v1.29.12@sha256:62c0672ba99a4afd7396512848d6fc382906b8f33349ae68fb1dbfe549f70dec
- role: worker
image: kindest/node:v1.29.12@sha256:62c0672ba99a4afd7396512848d6fc382906b8f33349ae68fb1dbfe549f70dec
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ public class KubernetesClusterInstaller
readonly string kubeCtlPath;
readonly ILogger logger;

readonly ClusterVersion latestSupportedClusterVersion = new(1, 31);

public string KubeConfigPath => Path.Combine(tempDir.DirectoryPath, kubeConfigName);
public string ClusterName => clusterName;

Expand All @@ -35,9 +37,19 @@ public KubernetesClusterInstaller(TemporaryDirectory tempDirectory, string kindE
kubeConfigName = $"{clusterName}.config";
}

public async Task Install()
public Task Install()
{
return InstallCluster(latestSupportedClusterVersion);
}

public Task Install(ClusterVersion clusterVersion)
{
return InstallCluster(clusterVersion);
}

async Task InstallCluster(ClusterVersion clusterVersion)
{
var configFilePath = await WriteFileToTemporaryDirectory("kind-config-v1-31.yaml");
var configFilePath = await WriteFileToTemporaryDirectory($"kind-config-v{clusterVersion.Major}-{clusterVersion.Minor}.yaml");

var sw = new Stopwatch();
sw.Restart();
Expand Down
Loading

0 comments on commit 8568485

Please sign in to comment.