Skip to content

Commit

Permalink
Add initial support for air-gapped environments
Browse files Browse the repository at this point in the history
  • Loading branch information
markusewalker committed Jan 9, 2025
1 parent 74cb5b7 commit 271b514
Show file tree
Hide file tree
Showing 59 changed files with 1,776 additions and 164 deletions.
2 changes: 2 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ ARG AWS_PROVIDER_VERSION
ARG RANCHER2_KEY_PATH
ARG RKE_KEY_PATH
ARG SANITY_KEY_PATH
ARG AIRGAP_KEY_PATH

ENV QASE_TEST_RUN_ID=${QASE_TEST_RUN_ID}
ENV TERRAFORM_VERSION=${TERRAFORM_VERSION}
Expand All @@ -36,6 +37,7 @@ ENV AWS_PROVIDER_VERSION=${AWS_PROVIDER_VERSION}
ENV RANCHER2_KEY_PATH=${RANCHER2_KEY_PATH}
ENV RKE_KEY_PATH=${RKE_KEY_PATH}
ENV SANITY_KEY_PATH=${SANITY_KEY_PATH}
ENV AIRGAP_KEY_PATH=${AIRGAP_KEY_PATH}

RUN wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -q && apt-get update > /dev/null && apt-get install unzip > /dev/null && \
unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip > /dev/null && \
Expand Down
2 changes: 2 additions & 0 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ AWS_PROVIDER_VERSION="${AWS_PROVIDER_VERSION:-}"
RANCHER2_KEY_PATH="${RANCHER2_KEY_PATH:-}"
RKE_KEY_PATH="${RKE_KEY_PATH:-}"
SANITY_KEY_PATH="${SANITY_KEY_PATH:-}"
AIRGAP_KEY_PATH="${AIRGAP_KEY_PATH:-}"

TRIM_JOB_NAME=$(basename "$JOB_NAME")

Expand All @@ -33,6 +34,7 @@ while [[ 3 -gt $count ]]; do
--build-arg RANCHER2_KEY_PATH="$RANCHER2_KEY_PATH" \
--build-arg RKE_KEY_PATH="$RKE_KEY_PATH" \
--build-arg SANITY_KEY_PATH="$SANITY_KEY_PATH" \
--build-arg AIRGAP_KEY_PATH="$AIRGAP_KEY_PATH" \
--build-arg EXTERNAL_ENCODED_VPN="$EXTERNAL_ENCODED_VPN" \
--build-arg VPN_ENCODED_LOGIN="$VPN_ENCODED_LOGIN" \
-t tfp-automation-validation-"${TRIM_JOB_NAME}""${BUILD_NUMBER}"
Expand Down
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ type PrivateRegistries struct {
}

type Standalone struct {
AirgapInternalFQDN string `json:"airgapInternalFQDN,omitempty" yaml:"airgapInternalFQDN,omitempty"`
BootstrapPassword string `json:"bootstrapPassword,omitempty" yaml:"bootstrapPassword,omitempty"`
CertManagerVersion string `json:"certManagerVersion,omitempty" yaml:"certManagerVersion,omitempty"`
RancherChartVersion string `json:"rancherChartVersion,omitempty" yaml:"rancherChartVersion,omitempty"`
Expand Down
2 changes: 2 additions & 0 deletions defaults/modules/modules.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,6 @@ const (
VsphereRKE1 = "vsphere_rke1"
VsphereRKE2 = "vsphere_rke2"
VsphereK3s = "vsphere_k3s"
AirgapRKE2 = "airgap_rke2"
AirgapK3S = "airgap_k3s"
)
30 changes: 30 additions & 0 deletions framework/airgapSetup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package framework

import (
"testing"

"github.com/gruntwork-io/terratest/modules/logger"
"github.com/gruntwork-io/terratest/modules/terraform"
"github.com/rancher/tfp-automation/config"
resources "github.com/rancher/tfp-automation/framework/set/resources/airgap"
)

// AirgapSetup is a function that will set the Terraform configuration and return the Terraform options.
func AirgapSetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) {
keyPath := resources.KeyPath()

var terratestLogger logger.Logger
if terratestConfig.TFLogging {
terratestLogger = *logger.Default
} else {
terratestLogger = *logger.Discard
}

terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
TerraformDir: keyPath,
NoColor: true,
Logger: &terratestLogger,
})

return terraformOptions, keyPath
}
21 changes: 21 additions & 0 deletions framework/cleanup/airgap/cleanup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package airgap

import (
"testing"

"github.com/gruntwork-io/terratest/modules/terraform"
"github.com/rancher/shepherd/clients/rancher"
"github.com/rancher/shepherd/pkg/config"
"github.com/rancher/tfp-automation/defaults/configs"
)

// ConfigAirgapCleanup is a function that will run terraform destroy and cleanup Terraform resources.
func ConfigAirgapCleanup(t *testing.T, terraformOptions *terraform.Options) {
rancherConfig := new(rancher.Config)
config.LoadConfig(configs.Rancher, rancherConfig)

if *rancherConfig.Cleanup {
terraform.Destroy(t, terraformOptions)
ConfigAirgapCleanupTF()
}
}
48 changes: 48 additions & 0 deletions framework/cleanup/airgap/cleanupAirgapConfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package airgap

import (
"os"

"github.com/rancher/tfp-automation/defaults/configs"
resources "github.com/rancher/tfp-automation/framework/set/resources/airgap"
"github.com/sirupsen/logrus"
)

// ConfigAirgapCleanupTF is a function that will cleanup the main.tf file and terraform.tfstate files.
func ConfigAirgapCleanupTF() error {
keyPath := resources.KeyPath()

file, err := os.Create(keyPath + configs.MainTF)
if err != nil {
logrus.Errorf("Failed to overwrite main.tf file. Error: %v", err)
return err
}

defer file.Close()

_, err = file.WriteString("// Leave blank - main.tf will be set during testing")
if err != nil {
logrus.Errorf("Failed to write to main.tf file. Error: %v", err)
return err
}

delete_files := [3]string{configs.TFState, configs.TFStateBackup, configs.TFLockHCL}

for _, delete_file := range delete_files {
delete_file = keyPath + delete_file
err = os.Remove(delete_file)

if err != nil {
logrus.Errorf("Failed to delete terraform.tfstate, terraform.tfstate.backup, and terraform.lock.hcl files. Error: %v", err)
return err
}
}

err = os.RemoveAll(keyPath + configs.TerraformFolder)
if err != nil {
logrus.Errorf("Failed to delete .terraform folder. Error: %v", err)
return err
}

return nil
}
File renamed without changes.
4 changes: 2 additions & 2 deletions framework/rkeSetup.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

// RKESetup is a function that will set the Terraform configuration and return the Terraform options.
func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) *terraform.Options {
func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) {
keyPath := resources.KeyPath()

var terratestLogger logger.Logger
Expand All @@ -26,5 +26,5 @@ func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestCo
Logger: &terratestLogger,
})

return terraformOptions
return terraformOptions, keyPath
}
4 changes: 2 additions & 2 deletions framework/sanitySetup.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

// SanitySetup is a function that will set the Terraform configuration and return the Terraform options.
func SanitySetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) *terraform.Options {
func SanitySetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) {
keyPath := resources.KeyPath()

var terratestLogger logger.Logger
Expand All @@ -26,5 +26,5 @@ func SanitySetup(t *testing.T, terraformConfig *config.TerraformConfig, terrates
Logger: &terratestLogger,
})

return terraformOptions
return terraformOptions, keyPath
}
81 changes: 52 additions & 29 deletions framework/set/defaults/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@ const (
Resource = "resource"
ResourceKind = "kind"
ResourceName = "name"
Namespace = "namespace"
Triggers = "triggers"

DependsOn = "depends_on"
GenerateName = "generate_name"
Defaults = "defaults"
Index = "index"
File = "file"
Airgap = "airgap"
Custom = "custom"

Rancher2Source = "rancher/rancher2"
Expand All @@ -21,6 +24,7 @@ const (
CloudCredential = "rancher2_cloud_credential"
Cluster = "rancher2_cluster"
ClusterV2 = "rancher2_cluster_v2"
SecretV2 = "rancher2_secret_v2"

RkeConfig = "rke_config"
KubernetesVersion = "kubernetes_version"
Expand Down Expand Up @@ -58,23 +62,25 @@ const (

LogFile = "log_file"

Ami = "ami"
Count = "count"
InstanceType = "instance_type"
SubnetId = "subnet_id"
VpcId = "vpc_id"
VpcSecurityGroupIds = "vpc_security_group_ids"
KeyName = "key_name"
AwsInstance = "aws_instance"
Name = "Name"
Nodes = "nodes"
RootBlockDevice = "root_block_device"
Tags = "tags"
VolumeSize = "volume_size"
Timeout = "timeout"
Ami = "ami"
AssociatePublicIPAddress = "associate_public_ip_address"
Count = "count"
InstanceType = "instance_type"
SubnetId = "subnet_id"
VpcId = "vpc_id"
VpcSecurityGroupIds = "vpc_security_group_ids"
KeyName = "key_name"
AwsInstance = "aws_instance"
Name = "Name"
Nodes = "nodes"
RootBlockDevice = "root_block_device"
Tags = "tags"
VolumeSize = "volume_size"
Timeout = "timeout"

Locals = "locals"
RoleFlags = "role_flags"
AllFlags = "--etcd --controlplane --worker"
EtcdRoleFlag = "--etcd"
ControlPlaneRoleFlag = "--controlplane"
WorkerRoleFlag = "--worker"
Expand All @@ -97,6 +103,7 @@ const (
User = "user"
Self = "self"
PublicIp = "public_ip"
PrivateIp = "private_ip"
Length = "length"

ApiUrl = "api_url"
Expand All @@ -120,23 +127,39 @@ const (
TokenKey = "token_key"
Version = "version"

DefaultAction = "default_action"
HealthCheck = "health_check"
LoadBalancer = "aws_lb"
LoadBalancerARN = "load_balancer_arn"
LoadBalancerListener = "aws_lb_listener"
LoadBalancerType = "load_balancer_type"
LoadBalancerTargetGroup = "aws_lb_target_group"
LoadBalancerTargetGroupAttachment = "aws_lb_target_group_attachment"
Port = "port"
Route53Record = "aws_route53_record"
Route53Zone = "aws_route53_zone"
Subnets = "subnets"
DefaultAction = "default_action"
HealthCheck = "health_check"

InternalLoadBalancer = "aws_internal_lb"
LoadBalancer = "aws_lb"

LoadBalancerARN = "load_balancer_arn"
LoadBalancerInternalListerner = "aws_internal_lb_listener"
LoadBalancerListener = "aws_lb_listener"

LoadBalancerType = "load_balancer_type"
LoadBalancerTargetGroup = "aws_lb_target_group"

LoadBalancreInternalTargetGroupAttachment = "aws_internal_lb_target_group_attachment"
LoadBalancerTargetGroupAttachment = "aws_lb_target_group_attachment"

Port = "port"
Route53InternalRecord = "aws_internal_route53_record"
Route53Record = "aws_route53_record"
Route53Zone = "aws_route53_zone"
Subnets = "subnets"

InternalTargetGroup80Attachment = "aws_internal_tg_attachment_80_server"
InternalTargetGroup443Attachment = "aws_internal_tg_attachment_443_server"
InternalTargetGroup6443Attachment = "aws_internal_tg_attachment_6443_server"
InternalTargetGroup9345Attachment = "aws_internal_tg_attachment_9345_server"
TargetGroup80Attachment = "aws_tg_attachment_80_server"
TargetGroup443Attachment = "aws_tg_attachment_443_server"
TargetGroup6443Attachment = "aws_tg_attachment_6443_server"
TargetGroup9345Attachment = "aws_tg_attachment_9345_server"
TargetGroupARN = "target_group_arn"
TargetGroupPrefix = "aws_tg_"
TargetID = "target_id"

TargetGroupARN = "target_group_arn"
TargetGroupInternalPrefix = "aws_internal_tg_"
TargetGroupPrefix = "aws_tg_"
TargetID = "target_id"
)
32 changes: 32 additions & 0 deletions framework/set/provisioning/airgap/copyScriptToBastion.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package airgap

import (
"os"
"path/filepath"

"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/rancher/tfp-automation/framework/set/defaults"
"github.com/zclconf/go-cty/cty"
)

// copyScript is a function that will copy the register-nodes.sh script to the bastion node
func copyScript(provisionerBlockBody *hclwrite.Body) error {
userDir, err := os.UserHomeDir()
if err != nil {
return nil
}

nodesScriptPath := filepath.Join(userDir, "go/src/github.com/rancher/tfp-automation/framework/set/provisioning/airgap/register-nodes.sh")

nodesScriptContent, err := os.ReadFile(nodesScriptPath)
if err != nil {
return nil
}

provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
cty.StringVal("echo '" + string(nodesScriptContent) + "' > /tmp/register-nodes.sh"),
cty.StringVal("chmod +x /tmp/register-nodes.sh"),
}))

return nil
}
34 changes: 34 additions & 0 deletions framework/set/provisioning/airgap/createRegistrySecret.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package airgap

import (
"github.com/hashicorp/hcl/v2/hclwrite"
"github.com/rancher/tfp-automation/config"
"github.com/rancher/tfp-automation/framework/set/defaults"
"github.com/zclconf/go-cty/cty"
)

const (
clusterID = "cluster_id"
localCluster = "local"
namespace = "fleet-default"
password = "password"
secretType = "kubernetes.io/basic-auth"
username = "username"
)

// createRegistrySecret is a function that will set the airgap RKE2/K3s cluster configurations in the main.tf file.
func createRegistrySecret(terraformConfig *config.TerraformConfig, clusterName string, rootBody *hclwrite.Body) {
secretBlock := rootBody.AppendNewBlock(defaults.Resource, []string{defaults.SecretV2, clusterName})
secretBlockBody := secretBlock.Body()

secretBlockBody.SetAttributeValue(clusterID, cty.StringVal(localCluster))
secretBlockBody.SetAttributeValue(defaults.ResourceName, cty.StringVal(terraformConfig.PrivateRegistries.AuthConfigSecretName))
secretBlockBody.SetAttributeValue(defaults.Namespace, cty.StringVal(namespace))
secretBlockBody.SetAttributeValue(defaults.Type, cty.StringVal(secretType))

dataBlock := secretBlockBody.AppendNewBlock(defaults.Data+" =", nil)
configBlockBody := dataBlock.Body()

configBlockBody.SetAttributeValue(password, cty.StringVal(terraformConfig.PrivateRegistries.Password))
configBlockBody.SetAttributeValue(username, cty.StringVal(terraformConfig.PrivateRegistries.Username))
}
Loading

0 comments on commit 271b514

Please sign in to comment.