From 4666cec566d57f064a4f248d44a3d607f49c5a5e Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Thu, 16 Nov 2023 10:38:08 +1100 Subject: [PATCH 01/27] Add Kubernetes Client SDK --- source/Octopus.Tentacle/Octopus.Tentacle.csproj | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/source/Octopus.Tentacle/Octopus.Tentacle.csproj b/source/Octopus.Tentacle/Octopus.Tentacle.csproj index aa55b4a63..7cca31bdd 100644 --- a/source/Octopus.Tentacle/Octopus.Tentacle.csproj +++ b/source/Octopus.Tentacle/Octopus.Tentacle.csproj @@ -40,6 +40,12 @@ $(DefineConstants);HTTP_CLIENT_SUPPORTS_SSL_OPTIONS;REQUIRES_EXPLICIT_LOG_CONFIG;REQUIRES_CODE_PAGE_PROVIDER;USER_INTERACTIVE_DOES_NOT_WORK;DEFAULT_PROXY_IS_NOT_AVAILABLE;HAS_NULLABLE_REF_TYPES + + + + + + @@ -55,6 +61,9 @@ + + + @@ -113,4 +122,8 @@ + + + + From 8e74d6ed9b47c58eafcb982101a7887558dd2e27 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Thu, 16 Nov 2023 12:47:43 +1100 Subject: [PATCH 02/27] Support executing scripts via Kubernetes Job --- .../KubernetesJobScriptExecutionContext.cs | 15 ++ .../Commands/ExecuteScriptCommand.cs | 117 +++++++++ ...us.Tentacle.Kubernetes.ScriptRunner.csproj | 20 ++ .../Program.cs | 10 + .../IKubernetesClientConfigProvider.cs | 9 + ...InClusterKubernetesClientConfigProvider.cs | 13 + .../Kubernetes/KubernetesClusterService.cs | 32 +++ .../Kubernetes/KubernetesJobService.cs | 89 +++++++ .../Kubernetes/KubernetesJobsConfig.cs | 15 ++ .../Kubernetes/KubernetesModule.cs | 19 ++ .../Kubernetes/KubernetesService.cs | 14 ++ ...alMachineKubernetesClientConfigProvider.cs | 18 ++ .../Octopus.Tentacle/Octopus.Tentacle.csproj | 4 - source/Octopus.Tentacle/Program.cs | 2 + .../Kubernetes/KubernetesJobScriptExecutor.cs | 39 +++ .../Kubernetes/RunningKubernetesJobScript.cs | 224 ++++++++++++++++++ .../Scripts/ScriptExecutorFactory.cs | 12 +- .../Util/ScriptLogExtensions.cs | 9 + source/Tentacle.sln | 41 ++++ 19 files changed, 696 insertions(+), 6 deletions(-) create mode 100644 source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs create mode 100644 source/Octopus.Tentacle.Kubernetes.ScriptRunner/Commands/ExecuteScriptCommand.cs create mode 100644 source/Octopus.Tentacle.Kubernetes.ScriptRunner/Octopus.Tentacle.Kubernetes.ScriptRunner.csproj create mode 100644 source/Octopus.Tentacle.Kubernetes.ScriptRunner/Program.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/IKubernetesClientConfigProvider.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/InClusterKubernetesClientConfigProvider.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/KubernetesJobsConfig.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/KubernetesService.cs create mode 100644 source/Octopus.Tentacle/Kubernetes/LocalMachineKubernetesClientConfigProvider.cs create mode 100644 source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs create mode 100644 source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs diff --git a/source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs b/source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs new file mode 100644 index 000000000..08b8c1a73 --- /dev/null +++ b/source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs @@ -0,0 +1,15 @@ +using Newtonsoft.Json; + +namespace Octopus.Tentacle.Contracts.ScriptServiceV3Alpha +{ + public class KubernetesJobScriptExecutionContext : IScriptExecutionContext + { + [JsonConstructor] + public KubernetesJobScriptExecutionContext(string containerImage) + { + ContainerImage = containerImage; + } + + public string ContainerImage { get; } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Commands/ExecuteScriptCommand.cs b/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Commands/ExecuteScriptCommand.cs new file mode 100644 index 000000000..a15229c4c --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Commands/ExecuteScriptCommand.cs @@ -0,0 +1,117 @@ +using System.CommandLine; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Diagnostics; +using Octopus.Tentacle.Scripts; +using Octopus.Tentacle.Util; + +namespace Octopus.Tentacle.Kubernetes.ScriptRunner.Commands; + +public class ExecuteScriptCommand : RootCommand +{ + readonly IShell shell; + + public ExecuteScriptCommand() + : base("Executes the script found in the work directory for the script ticket") + { + if (PlatformDetection.IsRunningOnWindows) + shell = new PowerShell(); + else + shell = new Bash(); + + var scriptPathOption = new Option( + name: "--script", + description: "The path to the script file to execute"); + AddOption(scriptPathOption); + + var scriptArgsOption = new Option( + name: "--args", + description: "The arguments to be passed to the script") + { + AllowMultipleArgumentsPerToken = true + }; + AddOption(scriptArgsOption); + + var logToConsoleOption = new Option( + name: "--logToConsole", + description: "If true, also writes the script logs to the console"); + AddOption(logToConsoleOption); + + this.SetHandler(async context => + { + var scriptPath = context.ParseResult.GetValueForOption(scriptPathOption); + var scriptArgs = context.ParseResult.GetValueForOption(scriptArgsOption); + var logToConsole = context.ParseResult.GetValueForOption(logToConsoleOption); + var token = context.GetCancellationToken(); + var exitCode = await ExecuteScript(scriptPath!, scriptArgs,logToConsole, token); + context.ExitCode = exitCode; + }); + } + + async Task ExecuteScript(string scriptPath, string[]? scriptArgs, bool logToConsole, CancellationToken cancellationToken) + { + await Task.CompletedTask; + + // we get erroneously left "s sometimes in k8s, so strip these + scriptPath = scriptPath.Trim('"'); + + var workingDirectory = Path.GetDirectoryName(scriptPath); + var scriptTicket = workingDirectory!.Split(Path.DirectorySeparatorChar).Last(); + + var workspace = new BashScriptWorkspace( + new ScriptTicket(scriptTicket), + workingDirectory, + new OctopusPhysicalFileSystem(new SystemLog()), + new SensitiveValueMasker()); + + var log = workspace.CreateLog(); + var logWriter = log.CreateWriter(); + + using var writer = logToConsole + ? new ConsoleLogWriterWrapper(logWriter) + : logWriter; + + scriptArgs ??= Array.Empty(); + + try + { + var exitCode = SilentProcessRunner.ExecuteCommand( + shell.GetFullPath(), + shell.FormatCommandArguments(scriptPath, scriptArgs, false), + workingDirectory, + output => writer.WriteOutput(ProcessOutputSource.Debug, output), + output => writer.WriteOutput(ProcessOutputSource.StdOut, output), + output => writer.WriteOutput(ProcessOutputSource.StdErr, output), + cancellationToken); + + return exitCode; + } + catch (Exception ex) + { + writer.WriteOutput(ProcessOutputSource.StdErr, "An exception was thrown when invoking " + shell.GetFullPath() + ": " + ex.Message); + writer.WriteOutput(ProcessOutputSource.StdErr, ex.ToString()); + + return ScriptExitCodes.PowershellInvocationErrorExitCode; + } + } + + class ConsoleLogWriterWrapper : IScriptLogWriter + { + readonly IScriptLogWriter writer; + + public ConsoleLogWriterWrapper(IScriptLogWriter writer) + { + this.writer = writer; + } + + public void WriteOutput(ProcessOutputSource source, string message) + { + Console.WriteLine($"[{source}] {message}"); + writer.WriteOutput(source, message); + } + + public void Dispose() + { + writer.Dispose(); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Octopus.Tentacle.Kubernetes.ScriptRunner.csproj b/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Octopus.Tentacle.Kubernetes.ScriptRunner.csproj new file mode 100644 index 000000000..b8dd0b67b --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Octopus.Tentacle.Kubernetes.ScriptRunner.csproj @@ -0,0 +1,20 @@ + + + + Exe + net6.0 + enable + enable + bin + ../../_build/$(AssemblyName)/$(TargetFramework)/$(RuntimeIdentifier) + + + + + + + + + + + diff --git a/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Program.cs b/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Program.cs new file mode 100644 index 000000000..9e2269770 --- /dev/null +++ b/source/Octopus.Tentacle.Kubernetes.ScriptRunner/Program.cs @@ -0,0 +1,10 @@ +using System.CommandLine; +using Octopus.Tentacle.Diagnostics; +using Octopus.Tentacle.Kubernetes.ScriptRunner.Commands; + +//Add the NLog appender (for use with the SystemLog) +Log.Appenders.Add(new NLogAppender()); + +var executeScriptCommand = new ExecuteScriptCommand(); + +return await executeScriptCommand.InvokeAsync(args); \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesClientConfigProvider.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesClientConfigProvider.cs new file mode 100644 index 000000000..02f9cf264 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/IKubernetesClientConfigProvider.cs @@ -0,0 +1,9 @@ +using k8s; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesClientConfigProvider + { + KubernetesClientConfiguration Get(); + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/InClusterKubernetesClientConfigProvider.cs b/source/Octopus.Tentacle/Kubernetes/InClusterKubernetesClientConfigProvider.cs new file mode 100644 index 000000000..20b8b7191 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/InClusterKubernetesClientConfigProvider.cs @@ -0,0 +1,13 @@ +using System; +using k8s; + +namespace Octopus.Tentacle.Kubernetes +{ + class InClusterKubernetesClientConfigProvider : IKubernetesClientConfigProvider + { + public KubernetesClientConfiguration Get() + { + return KubernetesClientConfiguration.InClusterConfig(); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs new file mode 100644 index 000000000..b46cd333e --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesClusterService.cs @@ -0,0 +1,32 @@ +using System; +using System.Threading.Tasks; +using k8s; +using Nito.AsyncEx; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesClusterService + { + Task GetClusterVersion(); + } + + public class KubernetesClusterService : KubernetesService, IKubernetesClusterService + { + readonly AsyncLazy lazyVersion; + public KubernetesClusterService(IKubernetesClientConfigProvider configProvider) + : base(configProvider) + { + //As the cluster version isn't going to change without restarting, we just cache the version in an AsyncLazy + lazyVersion = new AsyncLazy(async () => + { + var versionInfo = await Client.Version.GetCodeAsync(); + + //the git version is in the format "vX.Y.Z" so we trim the "v" from the front + return Version.Parse(versionInfo.GitVersion.Substring(1)); + }); + } + + public async Task GetClusterVersion() + => await lazyVersion; + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs new file mode 100644 index 000000000..e727d1103 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs @@ -0,0 +1,89 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using k8s; +using k8s.Autorest; +using k8s.Models; +using Octopus.Tentacle.Contracts; + +namespace Octopus.Tentacle.Kubernetes +{ + public interface IKubernetesJobService + { + Task TryGet(ScriptTicket scriptTicket, CancellationToken cancellationToken); + string BuildJobName(ScriptTicket scriptTicket); + Task CreateJob(V1Job job, CancellationToken cancellationToken); + void Delete(ScriptTicket scriptTicket); + Task Watch(ScriptTicket scriptTicket, Func onChange, Action onError, CancellationToken cancellationToken); + } + + public class KubernetesJobService : KubernetesService, IKubernetesJobService + { + public KubernetesJobService(IKubernetesClientConfigProvider configProvider) + : base(configProvider) + { + } + + public async Task TryGet(ScriptTicket scriptTicket, CancellationToken cancellationToken) + { + var jobName = BuildJobName(scriptTicket); + + try + { + return await Client.ReadNamespacedJobStatusAsync(jobName, KubernetesJobsConfig.Namespace, cancellationToken: cancellationToken); + } + catch (HttpOperationException opException) + { + if (opException.Response.StatusCode == HttpStatusCode.NotFound) + return null; + + //if there is some other error, just throw the exception + throw; + } + } + + public async Task Watch(ScriptTicket scriptTicket, Func onChange, Action onError, CancellationToken cancellationToken) + { + var jobName = BuildJobName(scriptTicket); + + using var response = Client.BatchV1.ListNamespacedJobWithHttpMessagesAsync( + KubernetesJobsConfig.Namespace, + //only list this job + fieldSelector: $"metadata.name=={jobName}", + watch: true, + timeoutSeconds: 1800, //same as the TTL of the job itself + cancellationToken: cancellationToken); + + await foreach (var (type, item) in response.WatchAsync(onError, cancellationToken: cancellationToken)) + { + //we are only watching for modifications + if (type != WatchEventType.Modified) + continue; + + var stopWatching = onChange(item); + if (stopWatching) + break; + } + } + + public string BuildJobName(ScriptTicket scriptTicket) => $"octopus-{scriptTicket.TaskId}".ToLowerInvariant(); + + public async Task CreateJob(V1Job job, CancellationToken cancellationToken) + { + await Client.CreateNamespacedJobAsync(job, KubernetesJobsConfig.Namespace, cancellationToken: cancellationToken); + } + + public void Delete(ScriptTicket scriptTicket) + { + try + { + Client.DeleteNamespacedJob(BuildJobName(scriptTicket), KubernetesJobsConfig.Namespace); + } + catch + { + //we are comfortable silently consuming this as the jobs have a TTL that will clean it up anyway + } + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesJobsConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesJobsConfig.cs new file mode 100644 index 000000000..abcc71a22 --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesJobsConfig.cs @@ -0,0 +1,15 @@ +using System; + +namespace Octopus.Tentacle.Kubernetes +{ + public static class KubernetesJobsConfig + { + public static string Namespace => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SNAMESPACE") + ?? throw new InvalidOperationException("Unable to determine Kubernetes namespace. An environment variable 'OCTOPUS__TENTACLE__K8SNAMESPACE' must be defined."); + + public static bool UseJobs => bool.TryParse(Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SUSEJOBS"), out var useJobs) && useJobs; + + public static string ServiceAccountName => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SSERVICEACCOUNTNAME") + ?? throw new InvalidOperationException("Unable to determine Kubernetes Job service account name. An environment variable 'OCTOPUS__TENTACLE__K8SSERVICEACCOUNTNAME' must be defined."); + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs new file mode 100644 index 000000000..93f3ffebe --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesModule.cs @@ -0,0 +1,19 @@ +using Autofac; + +namespace Octopus.Tentacle.Kubernetes +{ + public class KubernetesModule : Module + { + protected override void Load(ContainerBuilder builder) + { + builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As().SingleInstance(); + +#if DEBUG + builder.RegisterType().As().SingleInstance(); +#else + builder.RegisterType().As().SingleInstance(); +#endif + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs new file mode 100644 index 000000000..a2ab272dd --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesService.cs @@ -0,0 +1,14 @@ +using k8sClient = k8s.Kubernetes; + +namespace Octopus.Tentacle.Kubernetes +{ + public abstract class KubernetesService + { + protected k8sClient Client { get; } + + protected KubernetesService(IKubernetesClientConfigProvider configProvider) + { + Client = new k8sClient(configProvider.Get()); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/LocalMachineKubernetesClientConfigProvider.cs b/source/Octopus.Tentacle/Kubernetes/LocalMachineKubernetesClientConfigProvider.cs new file mode 100644 index 000000000..746d19ddf --- /dev/null +++ b/source/Octopus.Tentacle/Kubernetes/LocalMachineKubernetesClientConfigProvider.cs @@ -0,0 +1,18 @@ +using System; +using k8s; + +namespace Octopus.Tentacle.Kubernetes +{ + class LocalMachineKubernetesClientConfigProvider : IKubernetesClientConfigProvider + { + public KubernetesClientConfiguration Get() + { +#if DEBUG + var kubeConfigEnvVar = Environment.GetEnvironmentVariable("KUBECONFIG"); + return KubernetesClientConfiguration.BuildConfigFromConfigFile(kubeConfigEnvVar); +#else + throw new NotSupportedException("Local machine configuration is only supported when debugging."); +#endif + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Octopus.Tentacle.csproj b/source/Octopus.Tentacle/Octopus.Tentacle.csproj index 7cca31bdd..0c5b0a644 100644 --- a/source/Octopus.Tentacle/Octopus.Tentacle.csproj +++ b/source/Octopus.Tentacle/Octopus.Tentacle.csproj @@ -122,8 +122,4 @@ - - - - diff --git a/source/Octopus.Tentacle/Program.cs b/source/Octopus.Tentacle/Program.cs index f5b46abe1..47da04ff8 100644 --- a/source/Octopus.Tentacle/Program.cs +++ b/source/Octopus.Tentacle/Program.cs @@ -7,6 +7,7 @@ using Octopus.Tentacle.Communications; using Octopus.Tentacle.Configuration; using Octopus.Tentacle.Diagnostics; +using Octopus.Tentacle.Kubernetes; using Octopus.Tentacle.Maintenance; using Octopus.Tentacle.Properties; using Octopus.Tentacle.Services; @@ -56,6 +57,7 @@ public override IContainer BuildContainer(StartUpInstanceRequest startUpInstance builder.RegisterModule(new ServicesModule()); builder.RegisterModule(new VersioningModule(GetType().Assembly)); builder.RegisterModule(new MaintenanceModule()); + builder.RegisterModule(); builder.RegisterCommand("create-instance", "Registers a new instance of the Tentacle service"); builder.RegisterCommand("delete-instance", "Deletes an instance of the Tentacle service"); diff --git a/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs b/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs new file mode 100644 index 000000000..a3160c62b --- /dev/null +++ b/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs @@ -0,0 +1,39 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Octopus.Diagnostics; +using Octopus.Tentacle.Configuration.Instances; +using Octopus.Tentacle.Contracts.ScriptServiceV3Alpha; +using Octopus.Tentacle.Kubernetes; + +namespace Octopus.Tentacle.Scripts.Kubernetes +{ + public class KubernetesJobScriptExecutor : IScriptExecutor + { + readonly IKubernetesJobService jobService; + readonly IApplicationInstanceSelector appInstanceSelector; + readonly ISystemLog log; + + public KubernetesJobScriptExecutor(IKubernetesJobService jobService, IApplicationInstanceSelector appInstanceSelector, ISystemLog log) + { + this.jobService = jobService; + this.appInstanceSelector = appInstanceSelector; + this.log = log; + } + + public IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore? scriptStateStore, CancellationToken cancellationToken) + { + if (command.ExecutionContext is not KubernetesJobScriptExecutionContext kubernetesJobScriptExecutionContext) + throw new InvalidOperationException("The ExecutionContext must be of type KubernetesJobScriptExecutionContext"); + + var runningScript = new RunningKubernetesJobScript(workspace, workspace.CreateLog(), command.ScriptTicket, command.TaskId, cancellationToken, log, jobService, appInstanceSelector, kubernetesJobScriptExecutionContext); + + Task.Run(async () => + { + await runningScript.Execute(cancellationToken); + }, cancellationToken); + + return runningScript; + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs new file mode 100644 index 000000000..331cd1570 --- /dev/null +++ b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using k8s.Models; +using Octopus.Diagnostics; +using Octopus.Tentacle.Configuration.Instances; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Contracts.ScriptServiceV3Alpha; +using Octopus.Tentacle.Kubernetes; +using Octopus.Tentacle.Util; +using Octopus.Tentacle.Variables; + +namespace Octopus.Tentacle.Scripts.Kubernetes +{ + public class RunningKubernetesJobScript : IRunningScript + { + readonly IScriptWorkspace workspace; + readonly ScriptTicket scriptTicket; + readonly string taskId; + readonly ILog log; + readonly IKubernetesJobService jobService; + readonly KubernetesJobScriptExecutionContext executionContext; + readonly CancellationToken scriptCancellationToken; + readonly string? instanceName; + + public int ExitCode { get; private set; } + public ProcessState State { get; private set; } + public IScriptLog ScriptLog { get; } + + public RunningKubernetesJobScript(IScriptWorkspace workspace, + IScriptLog scriptLog, + ScriptTicket scriptTicket, + string taskId, + CancellationToken scriptCancellationToken, + ILog log, + IKubernetesJobService jobService, + IApplicationInstanceSelector appInstanceSelector, + KubernetesJobScriptExecutionContext executionContext) + { + this.workspace = workspace; + this.scriptTicket = scriptTicket; + this.taskId = taskId; + this.log = log; + this.jobService = jobService; + this.executionContext = executionContext; + this.scriptCancellationToken = scriptCancellationToken; + ScriptLog = scriptLog; + instanceName = appInstanceSelector.Current.InstanceName; + } + + public async Task Execute(CancellationToken taskCancellationToken) + { + var exitCode = -1; + + var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(scriptCancellationToken, taskCancellationToken); + var cancellationToken = linkedCancellationTokenSource.Token; + try + { + using var writer = ScriptLog.CreateWriter(); + try + { + using (ScriptIsolationMutex.Acquire(workspace.IsolationLevel, + workspace.ScriptMutexAcquireTimeout, + workspace.ScriptMutexName ?? nameof(RunningKubernetesJobScript), + message => writer.WriteOutput(ProcessOutputSource.StdOut, message), + taskId, + cancellationToken, + log)) + { + //create the k8s job + await CreateJob(writer, cancellationToken); + + State = ProcessState.Running; + + //we now need to monitor the resulting pod status + exitCode = await CheckIfPodHasCompleted(cancellationToken); + } + } + catch (OperationCanceledException) + { + writer.WriteOutput(ProcessOutputSource.StdOut, "Script execution canceled."); + exitCode = ScriptExitCodes.CanceledExitCode; + } + catch (TimeoutException) + { + writer.WriteOutput(ProcessOutputSource.StdOut, "Script execution timed out."); + exitCode = ScriptExitCodes.TimeoutExitCode; + } + } + catch (Exception) + { + exitCode = ScriptExitCodes.FatalExitCode; + } + finally + { + ExitCode = exitCode; + State = ProcessState.Complete; + } + } + + public void Complete() + { + jobService.Delete(scriptTicket); + } + + async Task CheckIfPodHasCompleted(CancellationToken cancellationToken) + { + var resultStatusCode = 0; + await jobService.Watch(scriptTicket, job => + { + var firstCondition = job.Status?.Conditions?.FirstOrDefault(); + switch (firstCondition) + { + case { Status: "True", Type: "Complete" }: + resultStatusCode = 0; + return true; + case { Status: "True", Type: "Failed" }: + resultStatusCode = 1; + return true; + default: + //continue watching + return false; + } + }, ex => + { + log.Error(ex); + resultStatusCode = 0; + }, cancellationToken); + + return resultStatusCode; + } + + async Task CreateJob(IScriptLogWriter writer, CancellationToken cancellationToken) + { + var scriptName = Path.GetFileName(workspace.BootstrapScriptFilePath); + + var jobName = jobService.BuildJobName(scriptTicket); + + var job = new V1Job + { + ApiVersion = "batch/v1", + Kind = "Job", + Metadata = new V1ObjectMeta + { + Name = jobName, + NamespaceProperty = KubernetesJobsConfig.Namespace, + Labels = new Dictionary + { + ["octopus.com/serverTaskId"] = taskId, + ["octopus.com/scriptTicketId"] = scriptTicket.TaskId + } + }, + Spec = new V1JobSpec + { + Template = new V1PodTemplateSpec + { + Spec = new V1PodSpec + { + Containers = new List + { + new() + { + Name = jobName, + Image = executionContext.ContainerImage, + Command = new List { "dotnet" }, + Args = new List + { + "/data/tentacle-app/source/Octopus.Tentacle.Kubernetes.ScriptRunner/bin/net6.0/Octopus.Tentacle.Kubernetes.ScriptRunner.dll", + "--script", + $"\"/data/tentacle-home/{instanceName}/Work/{scriptTicket.TaskId}/{scriptName}\"", + "--logToConsole" + }.Concat( + (workspace.ScriptArguments ?? Array.Empty()) + .SelectMany(arg => new[] + { + "--args", + $"\"{arg}\"" + }) + ).ToList(), + VolumeMounts = new List + { + new("/data/tentacle-home", "tentacle-home"), + new("/data/tentacle-app", "tentacle-app"), + }, + Env = new List + { + new(EnvironmentVariables.TentacleHome, $"/data/tentacle-home/{instanceName}"), + new(EnvironmentVariables.TentacleVersion, Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleVersion)), + new(EnvironmentVariables.TentacleCertificateSignatureAlgorithm, Environment.GetEnvironmentVariable(EnvironmentVariables.TentacleCertificateSignatureAlgorithm)), + new("OCTOPUS_RUNNING_IN_CONTAINER", "Y") + } + } + }, + ServiceAccountName = KubernetesJobsConfig.ServiceAccountName, + RestartPolicy = "Never", + Volumes = new List + { + new() + { + Name = "tentacle-home", + PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource("tentacle-home-pv-claim") + }, + new() + { + Name = "tentacle-app", + PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource("tentacle-app-pv-claim"), + }, + } + } + }, + BackoffLimit = 0, //we never want to rerun if it fails + TtlSecondsAfterFinished = 1800 //30min + } + }; + + writer.WriteVerbose($"Executing script in Kubernetes Job '{job.Name()}'"); + + await jobService.CreateJob(job, cancellationToken); + } + } +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs b/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs index 7a8101f65..83baf40de 100644 --- a/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs +++ b/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs @@ -1,19 +1,27 @@ using System; +using Octopus.Tentacle.Kubernetes; +using Octopus.Tentacle.Scripts.Kubernetes; namespace Octopus.Tentacle.Scripts { class ScriptExecutorFactory : IScriptExecutorFactory { readonly Lazy shellScriptExecutor; + readonly Lazy kubernetesJobScriptExecutor; - public ScriptExecutorFactory(Lazy shellScriptExecutor) + public ScriptExecutorFactory(Lazy shellScriptExecutor, Lazy kubernetesJobScriptExecutor) { this.shellScriptExecutor = shellScriptExecutor; + this.kubernetesJobScriptExecutor = kubernetesJobScriptExecutor; } public IScriptExecutor GetExecutor() { - return shellScriptExecutor.Value; + return KubernetesJobsConfig.UseJobs switch + { + true => kubernetesJobScriptExecutor.Value, + false => shellScriptExecutor.Value + }; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Util/ScriptLogExtensions.cs b/source/Octopus.Tentacle/Util/ScriptLogExtensions.cs index be62fa05d..a1ca61059 100644 --- a/source/Octopus.Tentacle/Util/ScriptLogExtensions.cs +++ b/source/Octopus.Tentacle/Util/ScriptLogExtensions.cs @@ -1,4 +1,6 @@ using System; +using Octopus.Tentacle.Contracts; +using Octopus.Tentacle.Scripts; namespace Octopus.Tentacle.Util { @@ -17,5 +19,12 @@ public static void WriteVerbose(this Action log, string message) log(message); log("##octopus[stdout-default]"); } + + public static void WriteVerbose(this IScriptLogWriter writer, string message) + { + writer.WriteOutput(ProcessOutputSource.StdOut, "##octopus[stdout-verbose]"); + writer.WriteOutput(ProcessOutputSource.StdOut, message); + writer.WriteOutput(ProcessOutputSource.StdOut, "##octopus[stdout-default]"); + } } } \ No newline at end of file diff --git a/source/Tentacle.sln b/source/Tentacle.sln index eacf70fbe..4d0918a9a 100644 --- a/source/Tentacle.sln +++ b/source/Tentacle.sln @@ -45,6 +45,7 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octopus.Tentacle.CommonTestUtils", "Octopus.Tentacle.CommonTestUtils\Octopus.Tentacle.CommonTestUtils.csproj", "{D0F2DBE7-017B-4AF1-9CA7-8C746220C6F0}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octopus.Tentacle.Client.Tests", "Octopus.Tentacle.Client.Tests\Octopus.Tentacle.Client.Tests.csproj", "{48C4D03B-F044-4089-942E-20DF102059F0}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Octopus.Tentacle.Kubernetes.ScriptRunner", "Octopus.Tentacle.Kubernetes.ScriptRunner\Octopus.Tentacle.Kubernetes.ScriptRunner.csproj", "{3C1A8996-0125-4551-AA83-C61CF38FEF09}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -255,6 +256,46 @@ Global {48C4D03B-F044-4089-942E-20DF102059F0}.Release-net6.0-win-x64|Any CPU.Build.0 = Debug|Any CPU {48C4D03B-F044-4089-942E-20DF102059F0}.Release-net6.0-win-x86|Any CPU.ActiveCfg = Debug|Any CPU {48C4D03B-F044-4089-942E-20DF102059F0}.Release-net6.0-win-x86|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net48-win|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net48-win|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-arm|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-arm|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-arm64|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-arm64|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-musl-x64|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-musl-x64|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-x64|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-linux-x64|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-osx-arm64|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-osx-arm64|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-osx-x64|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-osx-x64|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-win-x64|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-win-x64|Any CPU.Build.0 = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-win-x86|Any CPU.ActiveCfg = Debug|Any CPU + {EBB212E4-C1B3-411E-98B8-F7BFF8DDD15F}.Release-net6.0-win-x86|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release|Any CPU.Build.0 = Release|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net48-win|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net48-win|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-arm|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-arm|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-arm64|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-arm64|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-musl-x64|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-musl-x64|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-x64|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-linux-x64|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-osx-arm64|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-osx-arm64|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-osx-x64|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-osx-x64|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-win-x64|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-win-x64|Any CPU.Build.0 = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-win-x86|Any CPU.ActiveCfg = Debug|Any CPU + {3C1A8996-0125-4551-AA83-C61CF38FEF09}.Release-net6.0-win-x86|Any CPU.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From b8d66f42da157d569fb71e810d77ef2dcec83d4e Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Thu, 16 Nov 2023 18:14:13 +1100 Subject: [PATCH 03/27] Continued changes --- .../KubernetesJobScriptExecutionContext.cs | 4 +-- ...netesJobsConfig.cs => KubernetesConfig.cs} | 2 +- .../Kubernetes/KubernetesJobService.cs | 8 ++--- .../Scripts/IScriptExecutor.cs | 1 + .../Kubernetes/KubernetesJobScriptExecutor.cs | 11 +++---- .../Kubernetes/RunningKubernetesJobScript.cs | 30 ++++++++++++++++--- .../Scripts/LocalShellScriptExecutor.cs | 7 ++--- .../Scripts/ScriptExecutorFactory.cs | 2 +- .../Services/Scripts/ScriptServiceV3Alpha.cs | 3 ++ 9 files changed, 47 insertions(+), 21 deletions(-) rename source/Octopus.Tentacle/Kubernetes/{KubernetesJobsConfig.cs => KubernetesConfig.cs} (94%) diff --git a/source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs b/source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs index 08b8c1a73..373054483 100644 --- a/source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs +++ b/source/Octopus.Tentacle.Contracts/ScriptServiceV3Alpha/KubernetesJobScriptExecutionContext.cs @@ -5,11 +5,11 @@ namespace Octopus.Tentacle.Contracts.ScriptServiceV3Alpha public class KubernetesJobScriptExecutionContext : IScriptExecutionContext { [JsonConstructor] - public KubernetesJobScriptExecutionContext(string containerImage) + public KubernetesJobScriptExecutionContext(string? containerImage) { ContainerImage = containerImage; } - public string ContainerImage { get; } + public string? ContainerImage { get; } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesJobsConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs similarity index 94% rename from source/Octopus.Tentacle/Kubernetes/KubernetesJobsConfig.cs rename to source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs index abcc71a22..2f6ba7bb6 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesJobsConfig.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs @@ -2,7 +2,7 @@ namespace Octopus.Tentacle.Kubernetes { - public static class KubernetesJobsConfig + public static class KubernetesConfig { public static string Namespace => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SNAMESPACE") ?? throw new InvalidOperationException("Unable to determine Kubernetes namespace. An environment variable 'OCTOPUS__TENTACLE__K8SNAMESPACE' must be defined."); diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs index e727d1103..a56321f69 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs @@ -31,7 +31,7 @@ public KubernetesJobService(IKubernetesClientConfigProvider configProvider) try { - return await Client.ReadNamespacedJobStatusAsync(jobName, KubernetesJobsConfig.Namespace, cancellationToken: cancellationToken); + return await Client.ReadNamespacedJobStatusAsync(jobName, KubernetesConfig.Namespace, cancellationToken: cancellationToken); } catch (HttpOperationException opException) { @@ -48,7 +48,7 @@ public async Task Watch(ScriptTicket scriptTicket, Func onChange, A var jobName = BuildJobName(scriptTicket); using var response = Client.BatchV1.ListNamespacedJobWithHttpMessagesAsync( - KubernetesJobsConfig.Namespace, + KubernetesConfig.Namespace, //only list this job fieldSelector: $"metadata.name=={jobName}", watch: true, @@ -71,14 +71,14 @@ public async Task Watch(ScriptTicket scriptTicket, Func onChange, A public async Task CreateJob(V1Job job, CancellationToken cancellationToken) { - await Client.CreateNamespacedJobAsync(job, KubernetesJobsConfig.Namespace, cancellationToken: cancellationToken); + await Client.CreateNamespacedJobAsync(job, KubernetesConfig.Namespace, cancellationToken: cancellationToken); } public void Delete(ScriptTicket scriptTicket) { try { - Client.DeleteNamespacedJob(BuildJobName(scriptTicket), KubernetesJobsConfig.Namespace); + Client.DeleteNamespacedJob(BuildJobName(scriptTicket), KubernetesConfig.Namespace); } catch { diff --git a/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs b/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs index fdc6b6b89..b03522664 100644 --- a/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs +++ b/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs @@ -5,6 +5,7 @@ namespace Octopus.Tentacle.Scripts { public interface IScriptExecutor { + bool ValidateExecutionContext(IScriptExecutionContext executionContext); IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore? scriptStateStore, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs b/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs index a3160c62b..a562ea291 100644 --- a/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs +++ b/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs @@ -11,22 +11,23 @@ namespace Octopus.Tentacle.Scripts.Kubernetes public class KubernetesJobScriptExecutor : IScriptExecutor { readonly IKubernetesJobService jobService; + readonly IKubernetesClusterService clusterService; readonly IApplicationInstanceSelector appInstanceSelector; readonly ISystemLog log; - public KubernetesJobScriptExecutor(IKubernetesJobService jobService, IApplicationInstanceSelector appInstanceSelector, ISystemLog log) + public KubernetesJobScriptExecutor(IKubernetesJobService jobService, IKubernetesClusterService clusterService, IApplicationInstanceSelector appInstanceSelector, ISystemLog log) { this.jobService = jobService; + this.clusterService = clusterService; this.appInstanceSelector = appInstanceSelector; this.log = log; } + public bool ValidateExecutionContext(IScriptExecutionContext executionContext) => executionContext is KubernetesJobScriptExecutionContext; + public IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore? scriptStateStore, CancellationToken cancellationToken) { - if (command.ExecutionContext is not KubernetesJobScriptExecutionContext kubernetesJobScriptExecutionContext) - throw new InvalidOperationException("The ExecutionContext must be of type KubernetesJobScriptExecutionContext"); - - var runningScript = new RunningKubernetesJobScript(workspace, workspace.CreateLog(), command.ScriptTicket, command.TaskId, cancellationToken, log, jobService, appInstanceSelector, kubernetesJobScriptExecutionContext); + var runningScript = new RunningKubernetesJobScript(workspace, workspace.CreateLog(), command.ScriptTicket, command.TaskId, cancellationToken, log, jobService, clusterService, appInstanceSelector, (KubernetesJobScriptExecutionContext)command.ExecutionContext); Task.Run(async () => { diff --git a/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs index 331cd1570..2ac6d00a8 100644 --- a/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs +++ b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs @@ -22,6 +22,7 @@ public class RunningKubernetesJobScript : IRunningScript readonly string taskId; readonly ILog log; readonly IKubernetesJobService jobService; + readonly IKubernetesClusterService kubernetesClusterService; readonly KubernetesJobScriptExecutionContext executionContext; readonly CancellationToken scriptCancellationToken; readonly string? instanceName; @@ -30,13 +31,15 @@ public class RunningKubernetesJobScript : IRunningScript public ProcessState State { get; private set; } public IScriptLog ScriptLog { get; } - public RunningKubernetesJobScript(IScriptWorkspace workspace, + public RunningKubernetesJobScript( + IScriptWorkspace workspace, IScriptLog scriptLog, ScriptTicket scriptTicket, string taskId, CancellationToken scriptCancellationToken, ILog log, IKubernetesJobService jobService, + IKubernetesClusterService kubernetesClusterService, IApplicationInstanceSelector appInstanceSelector, KubernetesJobScriptExecutionContext executionContext) { @@ -45,6 +48,7 @@ public RunningKubernetesJobScript(IScriptWorkspace workspace, this.taskId = taskId; this.log = log; this.jobService = jobService; + this.kubernetesClusterService = kubernetesClusterService; this.executionContext = executionContext; this.scriptCancellationToken = scriptCancellationToken; ScriptLog = scriptLog; @@ -146,7 +150,7 @@ async Task CreateJob(IScriptLogWriter writer, CancellationToken cancellationToke Metadata = new V1ObjectMeta { Name = jobName, - NamespaceProperty = KubernetesJobsConfig.Namespace, + NamespaceProperty = KubernetesConfig.Namespace, Labels = new Dictionary { ["octopus.com/serverTaskId"] = taskId, @@ -164,7 +168,7 @@ async Task CreateJob(IScriptLogWriter writer, CancellationToken cancellationToke new() { Name = jobName, - Image = executionContext.ContainerImage, + Image = executionContext.ContainerImage ?? await GetDefaultContainer(), Command = new List { "dotnet" }, Args = new List { @@ -194,7 +198,7 @@ async Task CreateJob(IScriptLogWriter writer, CancellationToken cancellationToke } } }, - ServiceAccountName = KubernetesJobsConfig.ServiceAccountName, + ServiceAccountName = KubernetesConfig.ServiceAccountName, RestartPolicy = "Never", Volumes = new List { @@ -220,5 +224,23 @@ async Task CreateJob(IScriptLogWriter writer, CancellationToken cancellationToke await jobService.CreateJob(job, cancellationToken); } + + static readonly List KnownLatestContainerTags = new List + { + new(1, 26, 3), + new(1, 27, 3), + new(1, 28, 2), + }; + + async Task GetDefaultContainer() + { + var clusterVersion = await kubernetesClusterService.GetClusterVersion(); + + var tagVersion = KnownLatestContainerTags.FirstOrDefault(tag => tag.Major == clusterVersion.Major && tag.Minor == clusterVersion.Minor); + + var tag = tagVersion?.ToString(3) ?? "latest"; + + return $"octopuslabs/k8s-workertools:{tag}"; + } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs b/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs index fb4522900..672dc4913 100644 --- a/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs +++ b/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs @@ -16,12 +16,11 @@ public LocalShellScriptExecutor(IShell shell, ISystemLog log) this.log = log; } + public bool ValidateExecutionContext(IScriptExecutionContext executionContext) => executionContext is LocalShellScriptExecutionContext; + public IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore? scriptStateStore, CancellationToken cancellationToken) { - if (command.ExecutionContext is not LocalShellScriptExecutionContext) - throw new InvalidOperationException($"Cannot execute start script command as the execution context is not of type {nameof(LocalShellScriptExecutionContext)}."); - - var runningScript = new RunningScript(shell, workspace, scriptStateStore, workspace.CreateLog(), command.TaskId, cancellationToken, log); + var runningScript = new RunningScript(shell, workspace, scriptStateStore, workspace.CreateLog(), command.TaskId, cancellationToken, log); var thread = new Thread(runningScript.Execute) { Name = $"Executing {shell.Name} script for " + command.ScriptTicket.TaskId }; thread.Start(); diff --git a/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs b/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs index 83baf40de..052d5c9da 100644 --- a/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs +++ b/source/Octopus.Tentacle/Scripts/ScriptExecutorFactory.cs @@ -17,7 +17,7 @@ public ScriptExecutorFactory(Lazy shellScriptExecutor, public IScriptExecutor GetExecutor() { - return KubernetesJobsConfig.UseJobs switch + return KubernetesConfig.UseJobs switch { true => kubernetesJobScriptExecutor.Value, false => shellScriptExecutor.Value diff --git a/source/Octopus.Tentacle/Services/Scripts/ScriptServiceV3Alpha.cs b/source/Octopus.Tentacle/Services/Scripts/ScriptServiceV3Alpha.cs index a440497ff..aed72ca3d 100644 --- a/source/Octopus.Tentacle/Services/Scripts/ScriptServiceV3Alpha.cs +++ b/source/Octopus.Tentacle/Services/Scripts/ScriptServiceV3Alpha.cs @@ -72,6 +72,9 @@ public async Task StartScriptAsync(StartScriptComma } var executor = scriptExecutorFactory.GetExecutor(); + if(!executor.ValidateExecutionContext(command.ExecutionContext)) + throw new InvalidOperationException($"The execution context type {command.ExecutionContext.GetType().Name} cannot be used with script executor {executor.GetType().Name}."); + var process = executor.ExecuteOnBackgroundThread(command, workspace, runningScript.ScriptStateStore, runningScript.CancellationToken); runningScript.Process = process; From 548af9d7ab3bcee778df0d6930506d1c83d96711 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Mon, 20 Nov 2023 09:47:31 +1100 Subject: [PATCH 04/27] Add ScriptStateStore support to k8s jobs --- .../Kubernetes/KubernetesJobService.cs | 4 +- .../Scripts/IScriptExecutor.cs | 2 +- .../Kubernetes/KubernetesJobScriptExecutor.cs | 4 +- .../Kubernetes/RunningKubernetesJobScript.cs | 65 +++++++++++++++++-- .../Scripts/LocalShellScriptExecutor.cs | 2 +- 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs index a56321f69..bad0401f2 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesJobService.cs @@ -55,13 +55,13 @@ public async Task Watch(ScriptTicket scriptTicket, Func onChange, A timeoutSeconds: 1800, //same as the TTL of the job itself cancellationToken: cancellationToken); - await foreach (var (type, item) in response.WatchAsync(onError, cancellationToken: cancellationToken)) + await foreach (var (type, job) in response.WatchAsync(onError, cancellationToken: cancellationToken)) { //we are only watching for modifications if (type != WatchEventType.Modified) continue; - var stopWatching = onChange(item); + var stopWatching = onChange(job); if (stopWatching) break; } diff --git a/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs b/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs index b03522664..919995197 100644 --- a/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs +++ b/source/Octopus.Tentacle/Scripts/IScriptExecutor.cs @@ -6,6 +6,6 @@ namespace Octopus.Tentacle.Scripts public interface IScriptExecutor { bool ValidateExecutionContext(IScriptExecutionContext executionContext); - IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore? scriptStateStore, CancellationToken cancellationToken); + IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore scriptStateStore, CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs b/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs index a562ea291..4913fd5ff 100644 --- a/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs +++ b/source/Octopus.Tentacle/Scripts/Kubernetes/KubernetesJobScriptExecutor.cs @@ -25,9 +25,9 @@ public KubernetesJobScriptExecutor(IKubernetesJobService jobService, IKubernetes public bool ValidateExecutionContext(IScriptExecutionContext executionContext) => executionContext is KubernetesJobScriptExecutionContext; - public IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore? scriptStateStore, CancellationToken cancellationToken) + public IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore scriptStateStore, CancellationToken cancellationToken) { - var runningScript = new RunningKubernetesJobScript(workspace, workspace.CreateLog(), command.ScriptTicket, command.TaskId, cancellationToken, log, jobService, clusterService, appInstanceSelector, (KubernetesJobScriptExecutionContext)command.ExecutionContext); + var runningScript = new RunningKubernetesJobScript(workspace, workspace.CreateLog(), command.ScriptTicket, command.TaskId, cancellationToken, log, scriptStateStore, jobService, clusterService, appInstanceSelector, (KubernetesJobScriptExecutionContext)command.ExecutionContext); Task.Run(async () => { diff --git a/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs index 2ac6d00a8..9b38f8802 100644 --- a/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs +++ b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs @@ -21,6 +21,7 @@ public class RunningKubernetesJobScript : IRunningScript readonly ScriptTicket scriptTicket; readonly string taskId; readonly ILog log; + readonly IScriptStateStore stateStore; readonly IKubernetesJobService jobService; readonly IKubernetesClusterService kubernetesClusterService; readonly KubernetesJobScriptExecutionContext executionContext; @@ -38,6 +39,7 @@ public RunningKubernetesJobScript( string taskId, CancellationToken scriptCancellationToken, ILog log, + IScriptStateStore stateStore, IKubernetesJobService jobService, IKubernetesClusterService kubernetesClusterService, IApplicationInstanceSelector appInstanceSelector, @@ -47,6 +49,7 @@ public RunningKubernetesJobScript( this.scriptTicket = scriptTicket; this.taskId = taskId; this.log = log; + this.stateStore = stateStore; this.jobService = jobService; this.kubernetesClusterService = kubernetesClusterService; this.executionContext = executionContext; @@ -78,6 +81,7 @@ public async Task Execute(CancellationToken taskCancellationToken) await CreateJob(writer, cancellationToken); State = ProcessState.Running; + RecordScriptHasStarted(writer); //we now need to monitor the resulting pod status exitCode = await CheckIfPodHasCompleted(cancellationToken); @@ -100,15 +104,18 @@ public async Task Execute(CancellationToken taskCancellationToken) } finally { - ExitCode = exitCode; - State = ProcessState.Complete; + try + { + RecordScriptHasCompleted(exitCode); + } + finally + { + ExitCode = exitCode; + State = ProcessState.Complete; + } } } - public void Complete() - { - jobService.Delete(scriptTicket); - } async Task CheckIfPodHasCompleted(CancellationToken cancellationToken) { @@ -236,11 +243,57 @@ async Task GetDefaultContainer() { var clusterVersion = await kubernetesClusterService.GetClusterVersion(); + //find the highest tag for this cluster version var tagVersion = KnownLatestContainerTags.FirstOrDefault(tag => tag.Major == clusterVersion.Major && tag.Minor == clusterVersion.Minor); var tag = tagVersion?.ToString(3) ?? "latest"; return $"octopuslabs/k8s-workertools:{tag}"; } + + void RecordScriptHasStarted(IScriptLogWriter writer) + { + try + { + var scriptState = stateStore.Load(); + scriptState.Start(); + stateStore.Save(scriptState); + } + catch (Exception ex) + { + try + { + writer.WriteOutput(ProcessOutputSource.StdOut, $"Warning: An exception occurred saving the ScriptState: {ex.Message}"); + writer.WriteOutput(ProcessOutputSource.StdOut, ex.ToString()); + } + catch + { + //we don't care about errors here + } + } + } + + void RecordScriptHasCompleted(int exitCode) + { + try + { + var scriptState = stateStore.Load(); + scriptState.Complete(exitCode); + stateStore.Save(scriptState); + } + catch (Exception ex) + { + try + { + using var writer = ScriptLog.CreateWriter(); + writer.WriteOutput(ProcessOutputSource.StdOut, $"Warning: An exception occurred saving the ScriptState: {ex.Message}"); + writer.WriteOutput(ProcessOutputSource.StdOut, ex.ToString()); + } + catch + { + //we don't care about errors here + } + } + } } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs b/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs index 672dc4913..95a2a7708 100644 --- a/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs +++ b/source/Octopus.Tentacle/Scripts/LocalShellScriptExecutor.cs @@ -18,7 +18,7 @@ public LocalShellScriptExecutor(IShell shell, ISystemLog log) public bool ValidateExecutionContext(IScriptExecutionContext executionContext) => executionContext is LocalShellScriptExecutionContext; - public IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore? scriptStateStore, CancellationToken cancellationToken) + public IRunningScript ExecuteOnBackgroundThread(StartScriptCommandV3Alpha command, IScriptWorkspace workspace, ScriptStateStore scriptStateStore, CancellationToken cancellationToken) { var runningScript = new RunningScript(shell, workspace, scriptStateStore, workspace.CreateLog(), command.TaskId, cancellationToken, log); From 4b2cc939c38f48b3f5f6703d1d2e0760b9491240 Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Mon, 20 Nov 2023 13:45:03 +1100 Subject: [PATCH 05/27] Add support for supplying volume info via env vars --- .../Kubernetes/KubernetesConfig.cs | 3 ++ .../Kubernetes/RunningKubernetesJobScript.cs | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs index 2f6ba7bb6..d4de7c75a 100644 --- a/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs +++ b/source/Octopus.Tentacle/Kubernetes/KubernetesConfig.cs @@ -11,5 +11,8 @@ public static class KubernetesConfig public static string ServiceAccountName => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SSERVICEACCOUNTNAME") ?? throw new InvalidOperationException("Unable to determine Kubernetes Job service account name. An environment variable 'OCTOPUS__TENTACLE__K8SSERVICEACCOUNTNAME' must be defined."); + + public static string JobVolumeYaml => Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__K8SJOBVOLUMEYAML") + ?? throw new InvalidOperationException("Unable to determine Kubernetes Job volume yaml. An environment variable 'OCTOPUS__TENTACLE__K8SJOBVOLUMEYAML' must be defined."); } } \ No newline at end of file diff --git a/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs index 9b38f8802..bc5df28a5 100644 --- a/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs +++ b/source/Octopus.Tentacle/Scripts/Kubernetes/RunningKubernetesJobScript.cs @@ -150,6 +150,8 @@ async Task CreateJob(IScriptLogWriter writer, CancellationToken cancellationToke var jobName = jobService.BuildJobName(scriptTicket); + var volumes = k8s.KubernetesYaml.Deserialize>(KubernetesConfig.JobVolumeYaml); + var job = new V1Job { ApiVersion = "batch/v1", @@ -207,19 +209,20 @@ async Task CreateJob(IScriptLogWriter writer, CancellationToken cancellationToke }, ServiceAccountName = KubernetesConfig.ServiceAccountName, RestartPolicy = "Never", - Volumes = new List - { - new() - { - Name = "tentacle-home", - PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource("tentacle-home-pv-claim") - }, - new() - { - Name = "tentacle-app", - PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource("tentacle-app-pv-claim"), - }, - } + Volumes = volumes + // new List + // { + // new() + // { + // Name = "tentacle-home", + // PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource("tentacle-home-pv-claim") + // }, + // new() + // { + // Name = "tentacle-app", + // PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource("tentacle-app-pv-claim"), + // }, + // } } }, BackoffLimit = 0, //we never want to rerun if it fails From 7f998e84872eaa253e3ea565ed1a220e415d747e Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Mon, 20 Nov 2023 15:00:01 +1100 Subject: [PATCH 06/27] Set up for running in WSL --- ...topus.Tentacle (Run Agent - Linux).run.xml | 29 +++++++++++++++++++ .../Octopus.Manager.Tentacle.csproj | 2 +- .../Instances/ApplicationInstanceStore.cs | 10 +++++-- .../Util/OctopusPhysicalFileSystem.cs | 7 ++++- .../Util/PlatformDetection.cs | 9 ++++++ 5 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml diff --git a/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml b/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml new file mode 100644 index 000000000..dc00177cd --- /dev/null +++ b/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml @@ -0,0 +1,29 @@ + + + + \ No newline at end of file diff --git a/source/Octopus.Manager.Tentacle/Octopus.Manager.Tentacle.csproj b/source/Octopus.Manager.Tentacle/Octopus.Manager.Tentacle.csproj index 8546926f5..32946c52a 100644 --- a/source/Octopus.Manager.Tentacle/Octopus.Manager.Tentacle.csproj +++ b/source/Octopus.Manager.Tentacle/Octopus.Manager.Tentacle.csproj @@ -130,7 +130,7 @@ - + diff --git a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceStore.cs b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceStore.cs index 9d641ec29..52259cf8b 100644 --- a/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceStore.cs +++ b/source/Octopus.Tentacle/Configuration/Instances/ApplicationInstanceStore.cs @@ -29,8 +29,14 @@ public ApplicationInstanceStore( machineConfigurationHomeDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Octopus"); - if (!PlatformDetection.IsRunningOnWindows) + var customMachineHomeDirectory = Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__MACHINEHOMEDIRECTORY"); + //if there is a custom environment variable, respect that first + if (!string.IsNullOrWhiteSpace(customMachineHomeDirectory)) + machineConfigurationHomeDirectory = customMachineHomeDirectory; + else if (!PlatformDetection.IsRunningOnWindows) machineConfigurationHomeDirectory = "/etc/octopus"; + else + machineConfigurationHomeDirectory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), "Octopus"); } public ApplicationInstanceRecord LoadInstanceDetails(string? instanceName) @@ -234,4 +240,4 @@ public Instance(string name, string configurationFilePath) public string ConfigurationFilePath { get; set; } } } -} +} \ No newline at end of file diff --git a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs index e2076225d..daa90f990 100644 --- a/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs +++ b/source/Octopus.Tentacle/Util/OctopusPhysicalFileSystem.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Octopus.Diagnostics; +using Octopus.Tentacle.Kubernetes; using Polly; namespace Octopus.Tentacle.Util @@ -259,7 +260,11 @@ public void EnsureDiskHasEnoughFreeSpace(string directoryPath, long requiredSpac if (!Path.IsPathRooted(directoryPath)) return; - + + //We can't perform this check in Kubernetes due to how drives are mounted and reported (always returns 0 byte sized drives) + if(PlatformDetection.Kubernetes.IsRunningInKubernetes) + return; + var driveInfo = SafelyGetDriveInfo(directoryPath); var required = requiredSpaceInBytes < 0 ? 0 : (ulong)requiredSpaceInBytes; diff --git a/source/Octopus.Tentacle/Util/PlatformDetection.cs b/source/Octopus.Tentacle/Util/PlatformDetection.cs index 3d7b219d7..f7e821b79 100644 --- a/source/Octopus.Tentacle/Util/PlatformDetection.cs +++ b/source/Octopus.Tentacle/Util/PlatformDetection.cs @@ -8,5 +8,14 @@ public static class PlatformDetection public static bool IsRunningOnNix => RuntimeInformation.IsOSPlatform(OSPlatform.Linux); public static bool IsRunningOnWindows => RuntimeInformation.IsOSPlatform(OSPlatform.Windows); public static bool IsRunningOnMac => RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + + public static class Kubernetes + { + /// + /// Indicates if the Tentacle is running inside a Kubernetes cluster. + /// + public static bool IsRunningInKubernetes => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")) || + (bool.TryParse(Environment.GetEnvironmentVariable("OCTOPUS__TENTACLE__FORCEK8S"), out var b) && b); + } } } \ No newline at end of file From 4d8b21dcd910251d8cc80e0e89df03dd925b3e7d Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Tue, 21 Nov 2023 09:03:30 +1100 Subject: [PATCH 07/27] Adjust run config --- global.json | 2 +- ...topus.Tentacle (Run Agent - Linux).run.xml | 33 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/global.json b/global.json index 68eeba67e..9bdaba8ed 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "6.0.300", + "version": "6.0.414", "rollForward": "latestFeature" } } \ No newline at end of file diff --git a/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml b/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml index dc00177cd..e6a65fcbe 100644 --- a/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml +++ b/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml @@ -1,16 +1,16 @@  + + \ No newline at end of file From 4a4670f660610cf6a016868841082fc33db8ac6c Mon Sep 17 00:00:00 2001 From: Alastair Pitts Date: Tue, 21 Nov 2023 15:38:49 +1100 Subject: [PATCH 08/27] Fix a number of issues with running k8s scripts --- .../Octopus.Tentacle (Run Agent - Linux).run.xml | 5 +++-- .../Integration/ScriptServiceV3AlphaFixture.cs | 9 +++++++++ source/Octopus.Tentacle/ExternalInit.cs | 4 ++++ .../Kubernetes/KubernetesClusterService.cs | 13 +++++++------ .../Kubernetes/RunningKubernetesJobScript.cs | 3 ++- source/Octopus.Tentacle/Services/ServicesModule.cs | 2 ++ 6 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 source/Octopus.Tentacle/ExternalInit.cs diff --git a/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml b/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml index e6a65fcbe..66a03948b 100644 --- a/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml +++ b/source/.run/Octopus.Tentacle (Run Agent - Linux).run.xml @@ -32,12 +32,13 @@