Add initial support for air-gapped environments

Dockerfile

@@ -24,6 +24,7 @@ ARG AWS_PROVIDER_VERSION
 ARG RANCHER2_KEY_PATH
 ARG RKE_KEY_PATH
 ARG SANITY_KEY_PATH
+ARG AIRGAP_KEY_PATH
 
 ENV QASE_TEST_RUN_ID=${QASE_TEST_RUN_ID}
 ENV TERRAFORM_VERSION=${TERRAFORM_VERSION}
@@ -36,6 +37,7 @@ ENV AWS_PROVIDER_VERSION=${AWS_PROVIDER_VERSION}
 ENV RANCHER2_KEY_PATH=${RANCHER2_KEY_PATH}
 ENV RKE_KEY_PATH=${RKE_KEY_PATH}
 ENV SANITY_KEY_PATH=${SANITY_KEY_PATH}
+ENV AIRGAP_KEY_PATH=${AIRGAP_KEY_PATH}
 
 RUN wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -q && apt-get update > /dev/null && apt-get install unzip > /dev/null && \
     unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip > /dev/null && \ 

build.sh

@@ -14,6 +14,7 @@ AWS_PROVIDER_VERSION="${AWS_PROVIDER_VERSION:-}"
 RANCHER2_KEY_PATH="${RANCHER2_KEY_PATH:-}"
 RKE_KEY_PATH="${RKE_KEY_PATH:-}"
 SANITY_KEY_PATH="${SANITY_KEY_PATH:-}"
+AIRGAP_KEY_PATH="${AIRGAP_KEY_PATH:-}"
 
 TRIM_JOB_NAME=$(basename "$JOB_NAME")
 
@@ -33,6 +34,7 @@ while [[ 3 -gt $count ]]; do
                                                                     --build-arg RANCHER2_KEY_PATH="$RANCHER2_KEY_PATH" \
                                                                     --build-arg RKE_KEY_PATH="$RKE_KEY_PATH" \
                                                                     --build-arg SANITY_KEY_PATH="$SANITY_KEY_PATH" \
+                                                                    --build-arg AIRGAP_KEY_PATH="$AIRGAP_KEY_PATH" \
                                                                     --build-arg EXTERNAL_ENCODED_VPN="$EXTERNAL_ENCODED_VPN" \
                                                                     --build-arg VPN_ENCODED_LOGIN="$VPN_ENCODED_LOGIN" \
                                                                     -t tfp-automation-validation-"${TRIM_JOB_NAME}""${BUILD_NUMBER}"

config/config.go

@@ -101,6 +101,7 @@ type PrivateRegistries struct {
 }
 
 type Standalone struct {
+	AirgapInternalFQDN       string `json:"airgapInternalFQDN,omitempty" yaml:"airgapInternalFQDN,omitempty"`
 	BootstrapPassword        string `json:"bootstrapPassword,omitempty" yaml:"bootstrapPassword,omitempty"`
 	CertManagerVersion       string `json:"certManagerVersion,omitempty" yaml:"certManagerVersion,omitempty"`
 	RancherChartVersion      string `json:"rancherChartVersion,omitempty" yaml:"rancherChartVersion,omitempty"`

defaults/modules/modules.go

@@ -17,4 +17,6 @@ const (
 	VsphereRKE1   = "vsphere_rke1"
 	VsphereRKE2   = "vsphere_rke2"
 	VsphereK3s    = "vsphere_k3s"
+	AirgapRKE2    = "airgap_rke2"
+	AirgapK3S     = "airgap_k3s"
 )

framework/airgapSetup.go

@@ -0,0 +1,30 @@
+package framework
+
+import (
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/logger"
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/rancher/tfp-automation/config"
+	resources "github.com/rancher/tfp-automation/framework/set/resources/airgap"
+)
+
+// AirgapSetup is a function that will set the Terraform configuration and return the Terraform options.
+func AirgapSetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) {
+	keyPath := resources.KeyPath()
+
+	var terratestLogger logger.Logger
+	if terratestConfig.TFLogging {
+		terratestLogger = *logger.Default
+	} else {
+		terratestLogger = *logger.Discard
+	}
+
+	terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{
+		TerraformDir: keyPath,
+		NoColor:      true,
+		Logger:       &terratestLogger,
+	})
+
+	return terraformOptions, keyPath
+}

framework/cleanup/airgap/cleanup.go

@@ -0,0 +1,21 @@
+package airgap
+
+import (
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+	"github.com/rancher/shepherd/clients/rancher"
+	"github.com/rancher/shepherd/pkg/config"
+	"github.com/rancher/tfp-automation/defaults/configs"
+)
+
+// ConfigAirgapCleanup is a function that will run terraform destroy and cleanup Terraform resources.
+func ConfigAirgapCleanup(t *testing.T, terraformOptions *terraform.Options) {
+	rancherConfig := new(rancher.Config)
+	config.LoadConfig(configs.Rancher, rancherConfig)
+
+	if *rancherConfig.Cleanup {
+		terraform.Destroy(t, terraformOptions)
+		ConfigAirgapCleanupTF()
+	}
+}

framework/cleanup/airgap/cleanupAirgapConfig.go

@@ -0,0 +1,48 @@
+package airgap
+
+import (
+	"os"
+
+	"github.com/rancher/tfp-automation/defaults/configs"
+	resources "github.com/rancher/tfp-automation/framework/set/resources/airgap"
+	"github.com/sirupsen/logrus"
+)
+
+// ConfigAirgapCleanupTF is a function that will cleanup the main.tf file and terraform.tfstate files.
+func ConfigAirgapCleanupTF() error {
+	keyPath := resources.KeyPath()
+
+	file, err := os.Create(keyPath + configs.MainTF)
+	if err != nil {
+		logrus.Errorf("Failed to overwrite main.tf file. Error: %v", err)
+		return err
+	}
+
+	defer file.Close()
+
+	_, err = file.WriteString("// Leave blank - main.tf will be set during testing")
+	if err != nil {
+		logrus.Errorf("Failed to write to main.tf file. Error: %v", err)
+		return err
+	}
+
+	delete_files := [3]string{configs.TFState, configs.TFStateBackup, configs.TFLockHCL}
+
+	for _, delete_file := range delete_files {
+		delete_file = keyPath + delete_file
+		err = os.Remove(delete_file)
+
+		if err != nil {
+			logrus.Errorf("Failed to delete terraform.tfstate, terraform.tfstate.backup, and terraform.lock.hcl files. Error: %v", err)
+			return err
+		}
+	}
+
+	err = os.RemoveAll(keyPath + configs.TerraformFolder)
+	if err != nil {
+		logrus.Errorf("Failed to delete .terraform folder. Error: %v", err)
+		return err
+	}
+
+	return nil
+}

framework/rkeSetup.go

@@ -10,7 +10,7 @@ import (
 )
 
 // RKESetup is a function that will set the Terraform configuration and return the Terraform options.
-func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) *terraform.Options {
+func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) {
 	keyPath := resources.KeyPath()
 
 	var terratestLogger logger.Logger
@@ -26,5 +26,5 @@ func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestCo
 		Logger:       &terratestLogger,
 	})
 
-	return terraformOptions
+	return terraformOptions, keyPath
 }

framework/sanitySetup.go

@@ -10,7 +10,7 @@ import (
 )
 
 // SanitySetup is a function that will set the Terraform configuration and return the Terraform options.
-func SanitySetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) *terraform.Options {
+func SanitySetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) {
 	keyPath := resources.KeyPath()
 
 	var terratestLogger logger.Logger
@@ -26,5 +26,5 @@ func SanitySetup(t *testing.T, terraformConfig *config.TerraformConfig, terrates
 		Logger:       &terratestLogger,
 	})
 
-	return terraformOptions
+	return terraformOptions, keyPath
 }

framework/set/defaults/defaults.go

@@ -5,12 +5,15 @@ const (
 	Resource     = "resource"
 	ResourceKind = "kind"
 	ResourceName = "name"
+	Namespace    = "namespace"
+	Triggers     = "triggers"
 
 	DependsOn    = "depends_on"
 	GenerateName = "generate_name"
 	Defaults     = "defaults"
 	Index        = "index"
 	File         = "file"
+	Airgap       = "airgap"
 	Custom       = "custom"
 
 	Rancher2Source      = "rancher/rancher2"
@@ -21,6 +24,7 @@ const (
 	CloudCredential             = "rancher2_cloud_credential"
 	Cluster                     = "rancher2_cluster"
 	ClusterV2                   = "rancher2_cluster_v2"
+	SecretV2                    = "rancher2_secret_v2"
 
 	RkeConfig                           = "rke_config"
 	KubernetesVersion                   = "kubernetes_version"
@@ -57,23 +61,25 @@ const (
 
 	LogFile = "log_file"
 
-	Ami                 = "ami"
-	Count               = "count"
-	InstanceType        = "instance_type"
-	SubnetId            = "subnet_id"
-	VpcId               = "vpc_id"
-	VpcSecurityGroupIds = "vpc_security_group_ids"
-	KeyName             = "key_name"
-	AwsInstance         = "aws_instance"
-	Name                = "Name"
-	Nodes               = "nodes"
-	RootBlockDevice     = "root_block_device"
-	Tags                = "tags"
-	VolumeSize          = "volume_size"
-	Timeout             = "timeout"
+	Ami                      = "ami"
+	AssociatePublicIPAddress = "associate_public_ip_address"
+	Count                    = "count"
+	InstanceType             = "instance_type"
+	SubnetId                 = "subnet_id"
+	VpcId                    = "vpc_id"
+	VpcSecurityGroupIds      = "vpc_security_group_ids"
+	KeyName                  = "key_name"
+	AwsInstance              = "aws_instance"
+	Name                     = "Name"
+	Nodes                    = "nodes"
+	RootBlockDevice          = "root_block_device"
+	Tags                     = "tags"
+	VolumeSize               = "volume_size"
+	Timeout                  = "timeout"
 
 	Locals                   = "locals"
 	RoleFlags                = "role_flags"
+	AllFlags                 = "--etcd --controlplane --worker"
 	EtcdRoleFlag             = "--etcd"
 	ControlPlaneRoleFlag     = "--controlplane"
 	WorkerRoleFlag           = "--worker"
@@ -96,6 +102,7 @@ const (
 	User          = "user"
 	Self          = "self"
 	PublicIp      = "public_ip"
+	PrivateIp     = "private_ip"
 	Length        = "length"
 
 	ApiUrl            = "api_url"
@@ -119,23 +126,39 @@ const (
 	TokenKey          = "token_key"
 	Version           = "version"
 
-	DefaultAction                     = "default_action"
-	HealthCheck                       = "health_check"
-	LoadBalancer                      = "aws_lb"
-	LoadBalancerARN                   = "load_balancer_arn"
-	LoadBalancerListener              = "aws_lb_listener"
-	LoadBalancerType                  = "load_balancer_type"
-	LoadBalancerTargetGroup           = "aws_lb_target_group"
-	LoadBalancerTargetGroupAttachment = "aws_lb_target_group_attachment"
-	Port                              = "port"
-	Route53Record                     = "aws_route53_record"
-	Route53Zone                       = "aws_route53_zone"
-	Subnets                           = "subnets"
+	DefaultAction = "default_action"
+	HealthCheck   = "health_check"
+
+	InternalLoadBalancer = "aws_internal_lb"
+	LoadBalancer         = "aws_lb"
+
+	LoadBalancerARN               = "load_balancer_arn"
+	LoadBalancerInternalListerner = "aws_internal_lb_listener"
+	LoadBalancerListener          = "aws_lb_listener"
+
+	LoadBalancerType        = "load_balancer_type"
+	LoadBalancerTargetGroup = "aws_lb_target_group"
+
+	LoadBalancreInternalTargetGroupAttachment = "aws_internal_lb_target_group_attachment"
+	LoadBalancerTargetGroupAttachment         = "aws_lb_target_group_attachment"
+
+	Port                  = "port"
+	Route53InternalRecord = "aws_internal_route53_record"
+	Route53Record         = "aws_route53_record"
+	Route53Zone           = "aws_route53_zone"
+	Subnets               = "subnets"
+
+	InternalTargetGroup80Attachment   = "aws_internal_tg_attachment_80_server"
+	InternalTargetGroup443Attachment  = "aws_internal_tg_attachment_443_server"
+	InternalTargetGroup6443Attachment = "aws_internal_tg_attachment_6443_server"
+	InternalTargetGroup9345Attachment = "aws_internal_tg_attachment_9345_server"
 	TargetGroup80Attachment           = "aws_tg_attachment_80_server"
 	TargetGroup443Attachment          = "aws_tg_attachment_443_server"
 	TargetGroup6443Attachment         = "aws_tg_attachment_6443_server"
 	TargetGroup9345Attachment         = "aws_tg_attachment_9345_server"
-	TargetGroupARN                    = "target_group_arn"
-	TargetGroupPrefix                 = "aws_tg_"
-	TargetID                          = "target_id"
+
+	TargetGroupARN            = "target_group_arn"
+	TargetGroupInternalPrefix = "aws_internal_tg_"
+	TargetGroupPrefix         = "aws_tg_"
+	TargetID                  = "target_id"
 )

framework/set/provisioning/airgap/copyScriptToBastion.go

@@ -0,0 +1,32 @@
+package airgap
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/hcl/v2/hclwrite"
+	"github.com/rancher/tfp-automation/framework/set/defaults"
+	"github.com/zclconf/go-cty/cty"
+)
+
+// copyScript is a function that will copy the register-nodes.sh script to the bastion node
+func copyScript(provisionerBlockBody *hclwrite.Body) error {
+	userDir, err := os.UserHomeDir()
+	if err != nil {
+		return nil
+	}
+
+	nodesScriptPath := filepath.Join(userDir, "go/src/github.com/rancher/tfp-automation/framework/set/provisioning/airgap/register-nodes.sh")
+
+	nodesScriptContent, err := os.ReadFile(nodesScriptPath)
+	if err != nil {
+		return nil
+	}
+
+	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
+		cty.StringVal("echo '" + string(nodesScriptContent) + "' > /tmp/register-nodes.sh"),
+		cty.StringVal("chmod +x /tmp/register-nodes.sh"),
+	}))
+
+	return nil
+}

framework/set/provisioning/airgap/createRegistrySecret.go

@@ -0,0 +1,34 @@
+package airgap
+
+import (
+	"github.com/hashicorp/hcl/v2/hclwrite"
+	"github.com/rancher/tfp-automation/config"
+	"github.com/rancher/tfp-automation/framework/set/defaults"
+	"github.com/zclconf/go-cty/cty"
+)
+
+const (
+	clusterID    = "cluster_id"
+	localCluster = "local"
+	namespace    = "fleet-default"
+	password     = "password"
+	secretType   = "kubernetes.io/basic-auth"
+	username     = "username"
+)
+
+// createRegistrySecret is a function that will set the airgap RKE2/K3s cluster configurations in the main.tf file.
+func createRegistrySecret(terraformConfig *config.TerraformConfig, clusterName string, rootBody *hclwrite.Body) {
+	secretBlock := rootBody.AppendNewBlock(defaults.Resource, []string{defaults.SecretV2, clusterName})
+	secretBlockBody := secretBlock.Body()
+
+	secretBlockBody.SetAttributeValue(clusterID, cty.StringVal(localCluster))
+	secretBlockBody.SetAttributeValue(defaults.ResourceName, cty.StringVal(terraformConfig.PrivateRegistries.AuthConfigSecretName))
+	secretBlockBody.SetAttributeValue(defaults.Namespace, cty.StringVal(namespace))
+	secretBlockBody.SetAttributeValue(defaults.Type, cty.StringVal(secretType))
+
+	dataBlock := secretBlockBody.AppendNewBlock(defaults.Data+" =", nil)
+	configBlockBody := dataBlock.Body()
+
+	configBlockBody.SetAttributeValue(password, cty.StringVal(terraformConfig.PrivateRegistries.Password))
+	configBlockBody.SetAttributeValue(username, cty.StringVal(terraformConfig.PrivateRegistries.Username))
+}