Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Encrypt pod logs in bootstrap runner and decrypt in Tentacle #1047

Merged
merged 8 commits into from
Dec 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,16 @@ package main

import (
"bufio"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path"
"sync"
)

Expand All @@ -30,6 +36,12 @@ func main() {
args := os.Args[2:]
cmd := exec.Command("bash", args[0:]...)
cmd.Dir = workspacePath

gcm, err := CreateCipher(workspacePath)
if err != nil {
panic(err)
}

stdOutCmdReader, _ := cmd.StdoutPipe()
stdErrCmdReader, _ := cmd.StderrPipe()

Expand All @@ -39,14 +51,14 @@ func main() {
doneStd := make(chan bool)
doneErr := make(chan bool)

go reader(stdOutScanner, "stdout", &doneStd, &lineCounter)
go reader(stdErrScanner, "stderr", &doneErr, &lineCounter)
go reader(stdOutScanner, "stdout", &doneStd, &lineCounter, gcm)
go reader(stdErrScanner, "stderr", &doneErr, &lineCounter, gcm)

Write("stdout", "##octopus[stdout-verbose]", &lineCounter)
Write("stdout", "Kubernetes Script Pod started", &lineCounter)
Write("stdout", "##octopus[stdout-default]", &lineCounter)
Write("stdout", "##octopus[stdout-verbose]", &lineCounter, gcm)
Write("stdout", "Kubernetes Script Pod started", &lineCounter, gcm)
Write("stdout", "##octopus[stdout-default]", &lineCounter, gcm)

err := cmd.Start()
err = cmd.Start()

// Wait for output buffering first
<-doneStd
Expand All @@ -66,29 +78,73 @@ func main() {

exitCode := cmd.ProcessState.ExitCode()

Write("stdout", "##octopus[stdout-verbose]", &lineCounter)
Write("stdout", "Kubernetes Script Pod completed", &lineCounter)
Write("stdout", "##octopus[stdout-default]", &lineCounter)
Write("stdout", "##octopus[stdout-verbose]", &lineCounter, gcm)
Write("stdout", "Kubernetes Script Pod completed", &lineCounter, gcm)
Write("stdout", "##octopus[stdout-default]", &lineCounter, gcm)

Write("debug", fmt.Sprintf("EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>%d", exitCode), &lineCounter)
Write("debug", fmt.Sprintf("EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>%d", exitCode), &lineCounter, gcm)

os.Exit(exitCode)
}

func reader(scanner *bufio.Scanner, stream string, done *chan bool, counter *SafeCounter) {
func reader(scanner *bufio.Scanner, stream string, done *chan bool, counter *SafeCounter, gcm cipher.AEAD) {
for scanner.Scan() {
Write(stream, scanner.Text(), counter)
Write(stream, scanner.Text(), counter, gcm)
}
*done <- true
}

func Write(stream string, text string, counter *SafeCounter) {
func Write(stream string, text string, counter *SafeCounter, gcm cipher.AEAD) {
//Use a mutex to prevent race conditions updating the line number
//https://go.dev/tour/concurrency/9
counter.Mutex.Lock()

fmt.Printf("|%d|%s|%s\n", counter.Value, stream, text)
nonce := make([]byte, gcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
panic(err)
}

ciphertext := gcm.Seal(nonce, nonce, []byte(text), nil)

fmt.Printf("|%d|%s|%x\n", counter.Value, stream, ciphertext)
counter.Value++

counter.Mutex.Unlock()
}

func CreateCipher(workspaceDir string) (cipher.AEAD, error) {
// Read the key from the file
fileBytes, err := os.ReadFile(path.Join(workspaceDir, "keyfile"))
if err != nil {
return nil, err
}

//the key is encoded in the file in Base64
key := make([]byte, base64.StdEncoding.DecodedLen(len(fileBytes)))
length, err := base64.StdEncoding.Decode(key, fileBytes)
if err != nil {
return nil, err
}

// use the decoded length to slice the array to the correct length (removes padding bytes)
key = key[:length]

// Ensure the key length is valid for AES (16, 24, or 32 bytes for AES-128, AES-192, or AES-256)
keyLength := len(key)
if keyLength != 16 && keyLength != 24 && keyLength != 32 {
return nil, fmt.Errorf("invalid key size: %d bytes. Key must be 16, 24, or 32 bytes", keyLength)
}
// Create the AES cipher
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}

//we specify a known 12 byte nonce size so we can easily retrieve it in Tentacle
gcm, err := cipher.NewGCMWithNonceSize(block, 12)
if err != nil {
return nil, err
}

return gcm, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@
<PackageReference Include="NuGet.Packaging.Core" Version="3.6.0-octopus-58692" />
<PackageReference Include="NuGet.Packaging.Core.Types" Version="3.6.0-octopus-58692" />
<PackageReference Include="NuGet.Versioning" Version="3.6.0-octopus-58692" />
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
<PackageReference Include="TaskScheduler" Version="2.7.2" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' != 'net8.0-windows'">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ public async Task OneTimeSetUp()
var tag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageTag");
imageAndTag = $"docker.packages.octopushq.com/octopusdeploy/kubernetes-agent-tentacle:{tag}";
}
else if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag")))
{
imageAndTag = Environment.GetEnvironmentVariable("KubernetesAgentTests_ImageAndTag");
}
else if(bool.TryParse(Environment.GetEnvironmentVariable("KubernetesAgentTests_UseLatestLocalImage"), out var useLocal) && useLocal)
{
//if we should use the latest locally build image, load the tag from docker and load it into kind
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ static string GetChartVersion()
if (tentacleImageAndTag is null)
return null;

var parts = tentacleImageAndTag.Split(":");
var parts = tentacleImageAndTag.Split(':',2,StringSplitOptions.TrimEntries);
var repo = parts[0];
var tag = parts[1];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling;

public class HelmDownloader : ToolDownloader
{
const string LatestVersion = "v3.14.3";
const string LatestVersion = "v3.16.3";
public HelmDownloader( ILogger logger)
: base("helm", logger)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling
{
public class KindDownloader : ToolDownloader
{
const string LatestKindVersion = "v0.22.0";
const string LatestKindVersion = "v0.25.0";

public KindDownloader(ILogger logger)
: base("kind", logger)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling;

public class KubeCtlDownloader : ToolDownloader
{
public const string LatestKubeCtlVersion = "v1.29.3";
public const string LatestKubeCtlVersion = "v1.30.6";

public KubeCtlDownloader(ILogger logger)
: base("kubectl", logger)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@ apiVersion: kind.x-k8s.io/v1alpha4

nodes:
- role: control-plane
image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
image: kindest/node:v1.30.6@sha256:b6d08db72079ba5ae1f4a88a09025c0a904af3b52387643c285442afb05ab994
- role: worker
image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
image: kindest/node:v1.30.6@sha256:b6d08db72079ba5ae1f4a88a09025c0a904af3b52387643c285442afb05ab994
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
using Octopus.Tentacle.Configuration.Crypto;
using Octopus.Tentacle.Configuration.Instances;
using Octopus.Tentacle.Kubernetes;
using Octopus.Tentacle.Kubernetes.Configuration;
using Octopus.Tentacle.Kubernetes.Crypto;
using Octopus.Tentacle.Util;

namespace Octopus.Tentacle.Tests.Configuration
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using NUnit.Framework;
using Octopus.Tentacle.Contracts;
using Octopus.Tentacle.Kubernetes;
using Octopus.Tentacle.Kubernetes.Crypto;
using Octopus.Tentacle.Tests.Support;
using Octopus.Time;

Expand All @@ -26,6 +27,7 @@ public class KubernetesOrphanedPodCleanerTests
DateTimeOffset startTime;
ITentacleScriptLogProvider scriptLogProvider;
IScriptPodSinceTimeStore scriptPodSinceTimeStore;
IScriptPodLogEncryptionKeyProvider scriptPodLogEncryptionKeyProvider;

[SetUp]
public void Setup()
Expand All @@ -37,10 +39,11 @@ public void Setup()
clock = new FixedClock(startTime);
scriptLogProvider = Substitute.For<ITentacleScriptLogProvider>();
scriptPodSinceTimeStore = Substitute.For<IScriptPodSinceTimeStore>();
scriptPodLogEncryptionKeyProvider = Substitute.For<IScriptPodLogEncryptionKeyProvider>();
monitor = Substitute.For<IKubernetesPodStatusProvider>();
scriptTicket = new ScriptTicket(Guid.NewGuid().ToString());

cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore);
cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider);

overCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan + 1.Minutes();
underCutoff = cleaner.CompletedPodConsideredOrphanedAfterTimeSpan - 1.Minutes();
Expand Down Expand Up @@ -71,6 +74,7 @@ public async Task OrphanedPodCleanedUpIfOver10MinutesHavePassed()
await podService.Received().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
scriptLogProvider.Received().Delete(scriptTicket);
scriptPodSinceTimeStore.Received().Delete(scriptTicket);
scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
}

[TestCase(TrackedScriptPodPhase.Succeeded, true)]
Expand All @@ -95,12 +99,14 @@ public async Task OrphanedPodOnlyCleanedUpWhenNotRunning(TrackedScriptPodPhase p
await podService.Received().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
scriptLogProvider.Received().Delete(scriptTicket);
scriptPodSinceTimeStore.Received().Delete(scriptTicket);
scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
}
else
{
await podService.DidNotReceiveWithAnyArgs().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
scriptLogProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
scriptPodSinceTimeStore.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
scriptPodLogEncryptionKeyProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
}

TrackedScriptPodState CreateState(TrackedScriptPodPhase phase)
Expand Down Expand Up @@ -136,6 +142,7 @@ public async Task OrphanedPodNotCleanedUpIfOnly9MinutesHavePassed()
//Assert
await podService.DidNotReceiveWithAnyArgs().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
scriptLogProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
scriptPodLogEncryptionKeyProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
}

[Test]
Expand All @@ -157,6 +164,7 @@ public async Task OrphanedPodNotCleanedUpIfPodCleanupIsDisabled()
await podService.DidNotReceive().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
scriptLogProvider.Received().Delete(scriptTicket);
scriptPodSinceTimeStore.Received().Delete(scriptTicket);
scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
}

[TestCase(1, false)]
Expand All @@ -167,7 +175,7 @@ public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int c
Environment.SetEnvironmentVariable("OCTOPUS__K8STENTACLE__PODSCONSIDEREDORPHANEDAFTERMINUTES", "2");

// We need to reinitialise the sut after changing the env var value
cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore);
cleaner = new KubernetesOrphanedPodCleaner(monitor, podService, log, clock, scriptLogProvider, scriptPodSinceTimeStore, scriptPodLogEncryptionKeyProvider);
var pods = new List<ITrackedScriptPod>
{
CreatePod(TrackedScriptPodState.Succeeded(0, startTime))
Expand All @@ -184,12 +192,14 @@ public async Task EnvironmentVariableDictatesWhenPodsAreConsideredOrphaned(int c
await podService.Received().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
scriptLogProvider.Received().Delete(scriptTicket);
scriptPodSinceTimeStore.Received().Delete(scriptTicket);
scriptPodLogEncryptionKeyProvider.Received().Delete(scriptTicket);
}
else
{
await podService.DidNotReceiveWithAnyArgs().DeleteIfExists(scriptTicket, Arg.Any<CancellationToken>());
scriptLogProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
scriptPodSinceTimeStore.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
scriptPodLogEncryptionKeyProvider.DidNotReceiveWithAnyArgs().Delete(scriptTicket);
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
using System.Text;
using FluentAssertions;
using NUnit.Framework;
using Octopus.Tentacle.Kubernetes;
using Octopus.Tentacle.Kubernetes.Crypto;

namespace Octopus.Tentacle.Tests.Kubernetes
{
[TestFixture]
public class PodLogEncryptionProviderTests
{
IPodLogEncryptionProvider sut;

public static readonly byte[] Nonce =
{
12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1
};

[SetUp]
public void SetUp()
{
var key = Encoding.UTF8.GetBytes("qwertyuioplkjhgfdsazxcvbnmqwertd");
sut = PodLogEncryptionProvider.Create(key);
}

[TestCase("531f69408fcc09129a42b46b93c7c14fe7c36fec74ac77929b4e1f29b6b0c1e7cfe78055ceee24fbca5f9097501b5cb548f78928b5", "##octopus[stdout-verbose]")]
[TestCase("4463ba9b0672e65fdabcd99681ee20e5f003a59ce2c1b1751303e6399d5515ae4a715ed47ce5b7c5727ca66a127e485cd22de2d1dad76d8f704922dae0036d99c4a7e151498043b9d8", "EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>0")]
public void Decrypt_ShouldProduceCorrectOutput(string encryptedMessage, string expectedDecryptedMessage)
{
var result = sut.Decrypt(encryptedMessage);

result.Should().Be(expectedDecryptedMessage);
}

[TestCase("a cool message", "0c0b0a090807060504030201416ce819a896623cee784f92807857b9be36bd6075461789031a3d29aef1")]
[TestCase("##octopus[stdout-verbose]", "0c0b0a090807060504030201036fe415b3953224f8504f8783723ca65dca38cd3eab365c4d10d4efad161112e96cd449c6a52348ef")]
[TestCase("EOS-075CD4F0-8C76-491D-BA76-0879D35E9CFE<<>>0", "0c0b0a0908070605040302016503d85bf7cd7712cf3f7ac3ca250ae5469169866d80687b51b3cede4af296732090b9a9577055be3ced266ceb9eecd094a74f8df65c95ab33d9f5170c")]
public void Encrypt_ShouldProduceCorrectOutput(string plaintextMessage, string expectedEncryptedMessage)
{
var result = sut.Encrypt(plaintextMessage, Nonce);

result.Should().Be(expectedEncryptedMessage);
}
}
}
Loading