Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use static AKS cluster for authentication tests #1325

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ namespace Calamari.Tests.KubernetesFixtures
{
[TestFixture]
[Category(TestCategory.RunOnceOnWindowsAndLinux)]
public class KubernetesContextScriptWrapperLiveFixtureAks: KubernetesContextScriptWrapperLiveFixture
public class KubernetesContextScriptWrapperLiveFixtureAks : KubernetesContextScriptWrapperLiveFixture
{
string aksClusterHost;
string aksClusterClientCertificate;
Expand All @@ -31,7 +31,10 @@ public class KubernetesContextScriptWrapperLiveFixtureAks: KubernetesContextScri
string azurermResourceGroup;
string aksPodServiceAccountToken;
string azureSubscriptionId;

string azureSubscriptionClientId;
string azureSubscriptionPassword;
string azureSubscriptionTenantId;

static readonly CancellationTokenSource CancellationTokenSource = new CancellationTokenSource();
readonly CancellationToken cancellationToken = CancellationTokenSource.Token;

Expand All @@ -41,7 +44,7 @@ protected override IEnumerable<string> ToolsToAddToPath(InstallTools tools)
{
yield return tools.KubeloginExecutable;
}

protected override async Task InstallOptionalTools(InstallTools tools)
{
await tools.InstallKubelogin();
Expand All @@ -61,15 +64,20 @@ protected override void ExtractVariablesFromTerraformOutput(JObject jsonOutput)
protected override async Task<Dictionary<string, string>> GetEnvironmentVars(CancellationToken cancellationToken)
{
azureSubscriptionId = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionId, cancellationToken);
azureSubscriptionTenantId = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken);
azureSubscriptionPassword = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken);
azureSubscriptionClientId = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken);
Comment on lines +67 to +69
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Setting these up front, otherwise when you run locally 1password sometimes asks for authentication halfway through a long test run when running locally.


return new Dictionary<string, string>()
{
{ "ARM_SUBSCRIPTION_ID", azureSubscriptionId},
{ "ARM_CLIENT_ID", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken) },
{ "ARM_CLIENT_SECRET", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken) },
{ "ARM_TENANT_ID", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken) },
{ "TF_VAR_aks_client_id", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken) },
{ "TF_VAR_aks_client_secret", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken) },
{ "ARM_SUBSCRIPTION_ID", azureSubscriptionId },
{ "ARM_CLIENT_ID", azureSubscriptionClientId },
{ "ARM_CLIENT_SECRET", azureSubscriptionPassword },
{ "ARM_TENANT_ID", azureSubscriptionTenantId },
{ "TF_VAR_aks_client_id", azureSubscriptionClientId },
{ "TF_VAR_aks_client_secret", azureSubscriptionPassword },
{ "TF_VAR_test_namespace", TestNamespace },
{ "TF_VAR_static_resource_prefix", StaticTestResourcePrefix }
};
}

Expand Down Expand Up @@ -103,16 +111,17 @@ public void AuthorisingWithPodServiceAccountToken(bool runAsScript)
[Test]
[TestCase(true)]
[TestCase(false)]
public async Task AuthorisingWithAzureServicePrincipal(bool runAsScript)
public void AuthorisingWithAzureServicePrincipal(bool runAsScript)
{
variables.Set(Deployment.SpecialVariables.Account.AccountType, "AzureServicePrincipal");
variables.Set("Octopus.Action.Kubernetes.AksClusterResourceGroup", azurermResourceGroup);
variables.Set(SpecialVariables.AksClusterName, aksClusterName);
variables.Set("Octopus.Action.Kubernetes.AksAdminLogin", Boolean.FalseString);
variables.Set("Octopus.Action.Azure.SubscriptionId", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionId, cancellationToken));
variables.Set("Octopus.Action.Azure.TenantId", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken));
variables.Set("Octopus.Action.Azure.Password", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken));
variables.Set("Octopus.Action.Azure.ClientId", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken));
variables.Set("Octopus.Action.Azure.SubscriptionId", azureSubscriptionId);
variables.Set("Octopus.Action.Azure.ClientId", azureSubscriptionClientId);
variables.Set("Octopus.Action.Azure.Password", azureSubscriptionPassword);
variables.Set("Octopus.Action.Azure.TenantId", azureSubscriptionTenantId);

if (runAsScript)
{
DeployWithKubectlTestScriptAndVerifyResult();
Expand Down Expand Up @@ -166,7 +175,7 @@ public void UnreachableK8Cluster_ShouldExecuteTargetScript()
[Test]
[TestCase(false)]
[TestCase(true)]
public async Task DiscoverKubernetesClusterWithAzureServicePrincipalAccount(bool setHealthCheckContainer)
public void DiscoverKubernetesClusterWithAzureServicePrincipalAccount(bool setHealthCheckContainer)
{
var scope = new TargetDiscoveryScope("TestSpace",
"Staging",
Expand All @@ -177,13 +186,13 @@ public async Task DiscoverKubernetesClusterWithAzureServicePrincipalAccount(bool
setHealthCheckContainer ? new FeedImage("MyImage:with-tag", "Feeds-123") : null);

var account = new AzureServicePrincipalAccount(
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionId, cancellationToken),
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken),
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken),
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken),
null,
null,
null);
azureSubscriptionId,
azureSubscriptionClientId,
azureSubscriptionTenantId,
azureSubscriptionPassword,
null,
null,
null);

var authenticationDetails =
new AccountAuthenticationDetails<AzureServicePrincipalAccount>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ public class KubernetesContextScriptWrapperLiveFixtureAksLocalAccessDisabled : K
string aksClusterName;
string azurermResourceGroup;
string azureSubscriptionId;

string azureSubscriptionClientId;
string azureSubscriptionPassword;
string azureSubscriptionTenantId;

static readonly CancellationTokenSource CancellationTokenSource = new CancellationTokenSource();
readonly CancellationToken cancellationToken = CancellationTokenSource.Token;

Expand All @@ -49,31 +52,37 @@ protected override void ExtractVariablesFromTerraformOutput(JObject jsonOutput)
protected override async Task<Dictionary<string, string>> GetEnvironmentVars(CancellationToken cancellationToken)
{
azureSubscriptionId = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionId, cancellationToken);
azureSubscriptionClientId = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken);
azureSubscriptionPassword = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken);
azureSubscriptionTenantId = await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken);

return new Dictionary<string, string>()
{
{ "ARM_SUBSCRIPTION_ID", azureSubscriptionId },
{ "ARM_CLIENT_ID", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken) },
{ "ARM_CLIENT_SECRET", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken) },
{ "ARM_TENANT_ID", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken) },
{ "TF_VAR_aks_client_id", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken) },
{ "TF_VAR_aks_client_secret", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken) },
{ "ARM_CLIENT_ID", azureSubscriptionClientId },
{ "ARM_CLIENT_SECRET", azureSubscriptionPassword },
{ "ARM_TENANT_ID", azureSubscriptionTenantId },
{ "TF_VAR_aks_client_id", azureSubscriptionClientId },
{ "TF_VAR_aks_client_secret", azureSubscriptionPassword },
{ "TF_VAR_test_namespace", TestNamespace },
{ "TF_VAR_static_resource_prefix", StaticTestResourcePrefix }
};
}

[Test]
[TestCase(true)]
[TestCase(false)]
public async Task AuthorisingWithAzureServicePrincipal(bool runAsScript)
public void AuthorisingWithAzureServicePrincipal(bool runAsScript)
{
variables.Set(SpecialVariables.Account.AccountType, "AzureServicePrincipal");
variables.Set("Octopus.Action.Kubernetes.AksClusterResourceGroup", azurermResourceGroup);
variables.Set(Kubernetes.SpecialVariables.AksClusterName, aksClusterName);
variables.Set("Octopus.Action.Kubernetes.AksAdminLogin", Boolean.FalseString);
variables.Set("Octopus.Action.Azure.SubscriptionId", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionId, cancellationToken));
variables.Set("Octopus.Action.Azure.TenantId", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken));
variables.Set("Octopus.Action.Azure.Password", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken));
variables.Set("Octopus.Action.Azure.ClientId", await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken));
variables.Set("Octopus.Action.Azure.SubscriptionId", azureSubscriptionId);
variables.Set("Octopus.Action.Azure.ClientId", azureSubscriptionClientId);
variables.Set("Octopus.Action.Azure.Password", azureSubscriptionPassword);
variables.Set("Octopus.Action.Azure.TenantId", azureSubscriptionTenantId);

if (runAsScript)
{
DeployWithKubectlTestScriptAndVerifyResult();
Expand All @@ -87,7 +96,7 @@ public async Task AuthorisingWithAzureServicePrincipal(bool runAsScript)
[Test]
[TestCase(false)]
[TestCase(true)]
public async Task DiscoverKubernetesClusterWithAzureServicePrincipalAccount(bool setHealthCheckContainer)
public void DiscoverKubernetesClusterWithAzureServicePrincipalAccount(bool setHealthCheckContainer)
{
var scope = new TargetDiscoveryScope("TestSpace",
"Staging",
Expand All @@ -98,10 +107,10 @@ public async Task DiscoverKubernetesClusterWithAzureServicePrincipalAccount(bool
setHealthCheckContainer ? new FeedImage("MyImage:with-tag", "Feeds-123") : null);

var account = new AzureServicePrincipalAccount(
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionId, cancellationToken),
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionClientId, cancellationToken),
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionTenantId, cancellationToken),
await ExternalVariables.Get(ExternalVariable.AzureSubscriptionPassword, cancellationToken),
azureSubscriptionId,
azureSubscriptionClientId,
azureSubscriptionTenantId,
azureSubscriptionPassword,
null,
null,
null);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
provider "kubernetes" {
alias = "aks"
host = azurerm_kubernetes_cluster.default.kube_config.0.host
cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)
client_certificate = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)
client_key = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_key)
host = data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.host
cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.cluster_ca_certificate)
client_certificate = base64decode(data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.client_certificate)
client_key = base64decode(data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.client_key)
}
Original file line number Diff line number Diff line change
@@ -1,37 +1,8 @@
resource "azurerm_resource_group" "default" {
name = "${random_pet.prefix.id}-rg"
location = "Australia East"
data "azurerm_resource_group" "default" {
name = "${var.static_resource_prefix}-rg"
}

resource "azurerm_kubernetes_cluster" "default" {
name = "${random_pet.prefix.id}-aks"
resource_group_name = azurerm_resource_group.default.name
location = "Australia East"
dns_prefix = "${random_pet.prefix.id}-k8s"
kubernetes_version = "1.28"

tags = {
octopus-environment = "Staging"
octopus-role = "discovery-role"
source = "calamari-e2e-tests"
}

default_node_pool {
name = "default"
vm_size = "Standard_B2s"
node_count = 1
os_disk_size_gb = 30
}

identity {
type = "SystemAssigned"
}

role_based_access_control_enabled = true
local_account_disabled = true

azure_active_directory_role_based_access_control {
managed = true
azure_rbac_enabled = true
}
}
data "azurerm_kubernetes_cluster" "local_access_disabled" {
name = "${var.static_resource_prefix}-aks-no-local"
resource_group_name = data.azurerm_resource_group.default.name
}
Original file line number Diff line number Diff line change
@@ -1,14 +1,3 @@
variable "test_namespace" {
variable "static_resource_prefix" {
type = string
}

variable "aks_client_id" {
type = string
}

variable "aks_client_secret" {
type = string
sensitive = true
}

resource "random_pet" "prefix" {}
}
Original file line number Diff line number Diff line change
@@ -1,41 +1,41 @@
output "aks_cluster_host" {
description = "Endpoint for AKS control plane."
value = azurerm_kubernetes_cluster.default.kube_config.0.host
value = data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.host
sensitive = true
}


output "aks_cluster_username" {
value = azurerm_kubernetes_cluster.default.kube_config.0.username
value = data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.username
sensitive = true
}

output "aks_cluster_password" {
value = azurerm_kubernetes_cluster.default.kube_config.0.password
value = data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.password
sensitive = true
}

output "aks_cluster_client_certificate" {
value = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)
value = base64decode(data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.client_certificate)
sensitive = true
}

output "aks_cluster_client_key" {
value = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_key)
value = base64decode(data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.client_key)
sensitive = true
}

output "aks_cluster_ca_certificate" {
value = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)
value = base64decode(data.azurerm_kubernetes_cluster.local_access_disabled.kube_config.0.cluster_ca_certificate)
sensitive = true
}

output "aks_cluster_name" {
description = "AKS name."
value = azurerm_kubernetes_cluster.default.name
value = data.azurerm_kubernetes_cluster.local_access_disabled.name
}

output "aks_rg_name" {
description = "RG name."
value = azurerm_resource_group.default.name
}
value = data.azurerm_resource_group.default.name
}
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
provider "kubernetes" {
alias = "aks"
host = azurerm_kubernetes_cluster.default.kube_config.0.host
cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)
client_certificate = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)
client_key = base64decode(azurerm_kubernetes_cluster.default.kube_config.0.client_key)
host = data.azurerm_kubernetes_cluster.default.kube_config.0.host
cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate)
client_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate)
client_key = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key)
}

resource "kubernetes_namespace" "default" {
provider = kubernetes.aks
metadata {
name = var.test_namespace
name = "${var.test_namespace}-${random_pet.prefix.id}"
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,32 +1,8 @@
resource "azurerm_resource_group" "default" {
name = "${random_pet.prefix.id}-rg"
location = "Australia East"
data "azurerm_resource_group" "default" {
name = "${var.static_resource_prefix}-rg"
}

resource "azurerm_kubernetes_cluster" "default" {
name = "${random_pet.prefix.id}-aks"
resource_group_name = azurerm_resource_group.default.name
location = "Australia East"
dns_prefix = "${random_pet.prefix.id}-k8s"
kubernetes_version = "1.28"

tags = {
octopus-environment = "Staging"
octopus-role = "discovery-role"
source = "calamari-e2e-tests"
}

default_node_pool {
name = "default"
vm_size = "Standard_B2s"
node_count = 1
os_disk_size_gb = 30
}

role_based_access_control_enabled = true

service_principal {
client_id = var.aks_client_id
client_secret = var.aks_client_secret
}
data "azurerm_kubernetes_cluster" "default" {
name = "${var.static_resource_prefix}-aks"
resource_group_name = data.azurerm_resource_group.default.name
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,8 @@ variable "test_namespace" {
type = string
}

variable "aks_client_id" {
variable "static_resource_prefix" {
type = string
}

variable "aks_client_secret" {
type = string
sensitive = true
}

resource "random_pet" "prefix" {}
Loading