diff --git a/Dockerfile b/Dockerfile index 1b6ed14f..859405ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ ARG RANCHER2_KEY_PATH ARG RKE_KEY_PATH ARG SANITY_KEY_PATH ARG AIRGAP_KEY_PATH +ARG REGISTRY_KEY_PATH ENV QASE_TEST_RUN_ID=${QASE_TEST_RUN_ID} ENV TERRAFORM_VERSION=${TERRAFORM_VERSION} @@ -38,6 +39,7 @@ ENV RANCHER2_KEY_PATH=${RANCHER2_KEY_PATH} ENV RKE_KEY_PATH=${RKE_KEY_PATH} ENV SANITY_KEY_PATH=${SANITY_KEY_PATH} ENV AIRGAP_KEY_PATH=${AIRGAP_KEY_PATH} +ENV REGISTRY_KEY_PATH=${REGISTRY_KEY_PATH} RUN wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -q && apt-get update > /dev/null && apt-get install unzip > /dev/null && \ unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip > /dev/null && \ diff --git a/build.sh b/build.sh index 78c50da8..6ea2b1c8 100755 --- a/build.sh +++ b/build.sh @@ -15,6 +15,7 @@ RANCHER2_KEY_PATH="${RANCHER2_KEY_PATH:-}" RKE_KEY_PATH="${RKE_KEY_PATH:-}" SANITY_KEY_PATH="${SANITY_KEY_PATH:-}" AIRGAP_KEY_PATH="${AIRGAP_KEY_PATH:-}" +REGISTRY_KEY_PATH="${REGISTRY_KEY_PATH:-}" TRIM_JOB_NAME=$(basename "$JOB_NAME") @@ -35,6 +36,7 @@ while [[ 3 -gt $count ]]; do --build-arg RKE_KEY_PATH="$RKE_KEY_PATH" \ --build-arg SANITY_KEY_PATH="$SANITY_KEY_PATH" \ --build-arg AIRGAP_KEY_PATH="$AIRGAP_KEY_PATH" \ + --build-arg REGISTRY_KEY_PATH="$REGISTRY_KEY_PATH" \ --build-arg EXTERNAL_ENCODED_VPN="$EXTERNAL_ENCODED_VPN" \ --build-arg VPN_ENCODED_LOGIN="$VPN_ENCODED_LOGIN" \ -t tfp-automation-validation-"${TRIM_JOB_NAME}""${BUILD_NUMBER}" diff --git a/config/config.go b/config/config.go index 9760fc4c..9e736d6f 100644 --- a/config/config.go +++ b/config/config.go @@ -50,6 +50,13 @@ var WorkerNodePool = Nodepool{ Quantity: 1, } +var AllRolesNodePool = Nodepool{ + Etcd: true, + Controlplane: true, + Worker: true, + Quantity: 1, +} + var ScaleUpEtcdNodePool = Nodepool{ Etcd: true, Controlplane: false, diff --git a/config/nodeproviders/aws/awsConfig.go b/config/nodeproviders/aws/awsConfig.go index 1cf76e4a..c2eae2c3 100644 --- a/config/nodeproviders/aws/awsConfig.go +++ b/config/nodeproviders/aws/awsConfig.go @@ -1,21 +1,22 @@ package aws type Config struct { - AMI string `json:"ami,omitempty" yaml:"ami,omitempty"` - AWSInstanceType string `json:"awsInstanceType,omitempty" yaml:"awsInstanceType,omitempty"` - AWSKeyName string `json:"awsKeyName,omitempty" yaml:"awsKeyName,omitempty"` - AWSRootSize int64 `json:"awsRootSize,omitempty" yaml:"awsRootSize,omitempty"` - AWSSecurityGroupNames []string `json:"awsSecurityGroupNames,omitempty" yaml:"awsSecurityGroupNames,omitempty"` - AWSSecurityGroups []string `json:"awsSecurityGroups,omitempty" yaml:"awsSecurityGroups,omitempty"` - AWSSubnetID string `json:"awsSubnetID,omitempty" yaml:"awsSubnetID,omitempty"` - AWSSubnets []string `json:"awsSubnets,omitempty" yaml:"awsSubnets,omitempty"` - AWSVpcID string `json:"awsVpcID,omitempty" yaml:"awsVpcID,omitempty"` - AWSRoute53Zone string `json:"awsRoute53Zone,omitempty" yaml:"awsRoute53Zone,omitempty"` - AWSZoneLetter string `json:"awsZoneLetter,omitempty" yaml:"awsZoneLetter,omitempty"` - PrivateAccess bool `json:"privateAccess,omitempty" yaml:"privateAccess,omitempty"` - PublicAccess bool `json:"publicAccess,omitempty" yaml:"publicAccess,omitempty"` - RegistryRootSize int64 `json:"registryRootSize,omitempty" yaml:"registryRootSize,omitempty"` - Region string `json:"region,omitempty" yaml:"region,omitempty"` - AWSUser string `json:"awsUser,omitempty" yaml:"awsUser,omitempty"` - Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` + AMI string `json:"ami,omitempty" yaml:"ami,omitempty"` + AWSInstanceType string `json:"awsInstanceType,omitempty" yaml:"awsInstanceType,omitempty"` + AWSKeyName string `json:"awsKeyName,omitempty" yaml:"awsKeyName,omitempty"` + AWSRootSize int64 `json:"awsRootSize,omitempty" yaml:"awsRootSize,omitempty"` + AWSSecurityGroupNames []string `json:"awsSecurityGroupNames,omitempty" yaml:"awsSecurityGroupNames,omitempty"` + AWSSecurityGroups []string `json:"awsSecurityGroups,omitempty" yaml:"awsSecurityGroups,omitempty"` + AWSSubnetID string `json:"awsSubnetID,omitempty" yaml:"awsSubnetID,omitempty"` + AWSSubnets []string `json:"awsSubnets,omitempty" yaml:"awsSubnets,omitempty"` + AWSVpcID string `json:"awsVpcID,omitempty" yaml:"awsVpcID,omitempty"` + AWSRoute53Zone string `json:"awsRoute53Zone,omitempty" yaml:"awsRoute53Zone,omitempty"` + AWSZoneLetter string `json:"awsZoneLetter,omitempty" yaml:"awsZoneLetter,omitempty"` + PrivateAccess bool `json:"privateAccess,omitempty" yaml:"privateAccess,omitempty"` + PublicAccess bool `json:"publicAccess,omitempty" yaml:"publicAccess,omitempty"` + RegistryRootSize int64 `json:"registryRootSize,omitempty" yaml:"registryRootSize,omitempty"` + Region string `json:"region,omitempty" yaml:"region,omitempty"` + StandaloneSecurityGroupNames []string `json:"standaloneSecurityGroupNames,omitempty" yaml:"standaloneSecurityGroupNames,omitempty"` + AWSUser string `json:"awsUser,omitempty" yaml:"awsUser,omitempty"` + Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"` } diff --git a/framework/airgapSetup.go b/framework/airgapSetup.go deleted file mode 100644 index f40f788d..00000000 --- a/framework/airgapSetup.go +++ /dev/null @@ -1,30 +0,0 @@ -package framework - -import ( - "testing" - - "github.com/gruntwork-io/terratest/modules/logger" - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/rancher/tfp-automation/config" - resources "github.com/rancher/tfp-automation/framework/set/resources/airgap" -) - -// AirgapSetup is a function that will set the Terraform configuration and return the Terraform options. -func AirgapSetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) { - keyPath := resources.KeyPath() - - var terratestLogger logger.Logger - if terratestConfig.TFLogging { - terratestLogger = *logger.Default - } else { - terratestLogger = *logger.Discard - } - - terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ - TerraformDir: keyPath, - NoColor: true, - Logger: &terratestLogger, - }) - - return terraformOptions, keyPath -} diff --git a/framework/cleanup/airgap/cleanup.go b/framework/cleanup/airgap/cleanup.go deleted file mode 100644 index bb708f5b..00000000 --- a/framework/cleanup/airgap/cleanup.go +++ /dev/null @@ -1,21 +0,0 @@ -package airgap - -import ( - "testing" - - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/rancher/shepherd/clients/rancher" - "github.com/rancher/shepherd/pkg/config" - "github.com/rancher/tfp-automation/defaults/configs" -) - -// ConfigAirgapCleanup is a function that will run terraform destroy and cleanup Terraform resources. -func ConfigAirgapCleanup(t *testing.T, terraformOptions *terraform.Options) { - rancherConfig := new(rancher.Config) - config.LoadConfig(configs.Rancher, rancherConfig) - - if *rancherConfig.Cleanup { - terraform.Destroy(t, terraformOptions) - ConfigAirgapCleanupTF() - } -} diff --git a/framework/cleanup/airgap/cleanupAirgapConfig.go b/framework/cleanup/airgap/cleanupAirgapConfig.go deleted file mode 100644 index 750aca75..00000000 --- a/framework/cleanup/airgap/cleanupAirgapConfig.go +++ /dev/null @@ -1,48 +0,0 @@ -package airgap - -import ( - "os" - - "github.com/rancher/tfp-automation/defaults/configs" - resources "github.com/rancher/tfp-automation/framework/set/resources/airgap" - "github.com/sirupsen/logrus" -) - -// ConfigAirgapCleanupTF is a function that will cleanup the main.tf file and terraform.tfstate files. -func ConfigAirgapCleanupTF() error { - keyPath := resources.KeyPath() - - file, err := os.Create(keyPath + configs.MainTF) - if err != nil { - logrus.Errorf("Failed to overwrite main.tf file. Error: %v", err) - return err - } - - defer file.Close() - - _, err = file.WriteString("// Leave blank - main.tf will be set during testing") - if err != nil { - logrus.Errorf("Failed to write to main.tf file. Error: %v", err) - return err - } - - delete_files := [3]string{configs.TFState, configs.TFStateBackup, configs.TFLockHCL} - - for _, delete_file := range delete_files { - delete_file = keyPath + delete_file - err = os.Remove(delete_file) - - if err != nil { - logrus.Errorf("Failed to delete terraform.tfstate, terraform.tfstate.backup, and terraform.lock.hcl files. Error: %v", err) - return err - } - } - - err = os.RemoveAll(keyPath + configs.TerraformFolder) - if err != nil { - logrus.Errorf("Failed to delete .terraform folder. Error: %v", err) - return err - } - - return nil -} diff --git a/framework/cleanup/rke/cleanup.go b/framework/cleanup/cleanup.go similarity index 64% rename from framework/cleanup/rke/cleanup.go rename to framework/cleanup/cleanup.go index daebdefa..2e148317 100644 --- a/framework/cleanup/rke/cleanup.go +++ b/framework/cleanup/cleanup.go @@ -1,4 +1,4 @@ -package rke +package cleanup import ( "testing" @@ -9,13 +9,13 @@ import ( "github.com/rancher/tfp-automation/defaults/configs" ) -// ConfigRKECleanup is a function that will run terraform destroy and cleanup Terraform resources. -func ConfigRKECleanup(t *testing.T, terraformOptions *terraform.Options) { +// Cleanup is a function that will run terraform destroy and cleanup Terraform resources. +func Cleanup(t *testing.T, terraformOptions *terraform.Options, keyPath string) { rancherConfig := new(rancher.Config) config.LoadConfig(configs.Rancher, rancherConfig) if *rancherConfig.Cleanup { terraform.Destroy(t, terraformOptions) - ConfigRKECleanupTF() + TFFilesCleanup(keyPath) } } diff --git a/framework/cleanup/rancher2/cleanupConfigTF.go b/framework/cleanup/cleanupConfig.go similarity index 79% rename from framework/cleanup/rancher2/cleanupConfigTF.go rename to framework/cleanup/cleanupConfig.go index 513c0f3c..19919ee1 100644 --- a/framework/cleanup/rancher2/cleanupConfigTF.go +++ b/framework/cleanup/cleanupConfig.go @@ -1,21 +1,17 @@ -package rancher2 +package cleanup import ( "os" "github.com/rancher/tfp-automation/defaults/configs" - resources "github.com/rancher/tfp-automation/framework/set/resources/rancher2" "github.com/sirupsen/logrus" ) -// ConfigCleanupTF is a function that will cleanup the main.tf file and terraform.tfstate files. -func ConfigCleanupTF() error { - keyPath := resources.SetKeyPath() - +// TFFilesCleanup is a function that will cleanup the main.tf file and terraform.tfstate files. +func TFFilesCleanup(keyPath string) error { file, err := os.Create(keyPath + configs.MainTF) if err != nil { logrus.Errorf("Failed to overwrite main.tf file. Error: %v", err) - return err } @@ -24,7 +20,6 @@ func ConfigCleanupTF() error { _, err = file.WriteString("// Leave blank - main.tf will be set during testing") if err != nil { logrus.Errorf("Failed to write to main.tf file. Error: %v", err) - return err } @@ -36,7 +31,6 @@ func ConfigCleanupTF() error { if err != nil { logrus.Errorf("Failed to delete terraform.tfstate, terraform.tfstate.backup, and terraform.lock.hcl files. Error: %v", err) - return err } } @@ -44,7 +38,6 @@ func ConfigCleanupTF() error { err = os.RemoveAll(keyPath + configs.TerraformFolder) if err != nil { logrus.Errorf("Failed to delete .terraform folder. Error: %v", err) - return err } diff --git a/framework/cleanup/rancher2/cleanup.go b/framework/cleanup/rancher2/cleanup.go deleted file mode 100644 index b69b2344..00000000 --- a/framework/cleanup/rancher2/cleanup.go +++ /dev/null @@ -1,21 +0,0 @@ -package rancher2 - -import ( - "testing" - - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/rancher/shepherd/clients/rancher" - "github.com/rancher/shepherd/pkg/config" - "github.com/rancher/tfp-automation/defaults/configs" -) - -// ConfigCleanup is a function that will run terraform destroy and cleanup Terraform resources. -func ConfigCleanup(t *testing.T, terraformOptions *terraform.Options) { - rancherConfig := new(rancher.Config) - config.LoadConfig(configs.Rancher, rancherConfig) - - if *rancherConfig.Cleanup { - terraform.Destroy(t, terraformOptions) - ConfigCleanupTF() - } -} diff --git a/framework/cleanup/rke/cleanupRKEConfig.go b/framework/cleanup/rke/cleanupRKEConfig.go deleted file mode 100644 index 65f67a29..00000000 --- a/framework/cleanup/rke/cleanupRKEConfig.go +++ /dev/null @@ -1,48 +0,0 @@ -package rke - -import ( - "os" - - "github.com/rancher/tfp-automation/defaults/configs" - resources "github.com/rancher/tfp-automation/framework/set/resources/rke" - "github.com/sirupsen/logrus" -) - -// ConfigRKECleanupTF is a function that will cleanup the main.tf file and terraform.tfstate files. -func ConfigRKECleanupTF() error { - keyPath := resources.KeyPath() - - file, err := os.Create(keyPath + configs.MainTF) - if err != nil { - logrus.Errorf("Failed to overwrite main.tf file. Error: %v", err) - return err - } - - defer file.Close() - - _, err = file.WriteString("// Leave blank - main.tf will be set during testing") - if err != nil { - logrus.Errorf("Failed to write to main.tf file. Error: %v", err) - return err - } - - delete_files := [4]string{configs.TFState, configs.TFStateBackup, configs.TFLockHCL, configs.RKEDebugLog} - - for _, delete_file := range delete_files { - delete_file = keyPath + delete_file - err = os.Remove(delete_file) - - if err != nil { - logrus.Errorf("Failed to delete terraform.tfstate, terraform.tfstate.backup, and terraform.lock.hcl files. Error: %v", err) - return err - } - } - - err = os.RemoveAll(keyPath + configs.TerraformFolder) - if err != nil { - logrus.Errorf("Failed to delete .terraform folder. Error: %v", err) - return err - } - - return nil -} diff --git a/framework/cleanup/sanity/cleanup.go b/framework/cleanup/sanity/cleanup.go deleted file mode 100644 index 3f457c1f..00000000 --- a/framework/cleanup/sanity/cleanup.go +++ /dev/null @@ -1,21 +0,0 @@ -package sanity - -import ( - "testing" - - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/rancher/shepherd/clients/rancher" - "github.com/rancher/shepherd/pkg/config" - "github.com/rancher/tfp-automation/defaults/configs" -) - -// StandaloneConfigCleanup is a function that will run terraform destroy and cleanup Terraform resources. -func StandaloneConfigCleanup(t *testing.T, terraformOptions *terraform.Options) { - rancherConfig := new(rancher.Config) - config.LoadConfig(configs.Rancher, rancherConfig) - - if *rancherConfig.Cleanup { - terraform.Destroy(t, terraformOptions) - StandaloneConfigCleanupTF() - } -} diff --git a/framework/cleanup/sanity/cleanupStandaloneConfig.go b/framework/cleanup/sanity/cleanupStandaloneConfig.go deleted file mode 100644 index 2722f0fa..00000000 --- a/framework/cleanup/sanity/cleanupStandaloneConfig.go +++ /dev/null @@ -1,48 +0,0 @@ -package sanity - -import ( - "os" - - "github.com/rancher/tfp-automation/defaults/configs" - resources "github.com/rancher/tfp-automation/framework/set/resources/sanity" - "github.com/sirupsen/logrus" -) - -// StandaloneConfigCleanupTF is a function that will cleanup the main.tf file and terraform.tfstate files. -func StandaloneConfigCleanupTF() error { - keyPath := resources.KeyPath() - - file, err := os.Create(keyPath + configs.MainTF) - if err != nil { - logrus.Errorf("Failed to overwrite main.tf file. Error: %v", err) - return err - } - - defer file.Close() - - _, err = file.WriteString("// Leave blank - main.tf will be set during testing") - if err != nil { - logrus.Errorf("Failed to write to main.tf file. Error: %v", err) - return err - } - - delete_files := [3]string{configs.TFState, configs.TFStateBackup, configs.TFLockHCL} - - for _, delete_file := range delete_files { - delete_file = keyPath + delete_file - err = os.Remove(delete_file) - - if err != nil { - logrus.Errorf("Failed to delete terraform.tfstate, terraform.tfstate.backup, and terraform.lock.hcl files. Error: %v", err) - return err - } - } - - err = os.RemoveAll(keyPath + configs.TerraformFolder) - if err != nil { - logrus.Errorf("Failed to delete .terraform folder. Error: %v", err) - return err - } - - return nil -} diff --git a/framework/rancher2Setup.go b/framework/rancher2Setup.go deleted file mode 100644 index 52068522..00000000 --- a/framework/rancher2Setup.go +++ /dev/null @@ -1,36 +0,0 @@ -package framework - -import ( - "testing" - - "github.com/gruntwork-io/terratest/modules/logger" - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/rancher/shepherd/clients/rancher" - "github.com/rancher/tfp-automation/config" - setFramework "github.com/rancher/tfp-automation/framework/set" - resources "github.com/rancher/tfp-automation/framework/set/resources/rancher2" - "github.com/stretchr/testify/require" -) - -// Rancher2Setup is a function that will set the Terraform configuration and return the Terraform options. -func Rancher2Setup(t *testing.T, rancherConfig *rancher.Config, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) *terraform.Options { - keyPath := resources.SetKeyPath() - - err := setFramework.ConfigTF(nil, rancherConfig, terraformConfig, terratestConfig, "", "", "", "", "", nil) - require.NoError(t, err) - - var terratestLogger logger.Logger - if terratestConfig.TFLogging { - terratestLogger = *logger.Default - } else { - terratestLogger = *logger.Discard - } - - terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ - TerraformDir: keyPath, - NoColor: true, - Logger: &terratestLogger, - }) - - return terraformOptions -} diff --git a/framework/sanitySetup.go b/framework/sanitySetup.go deleted file mode 100644 index cdd74151..00000000 --- a/framework/sanitySetup.go +++ /dev/null @@ -1,30 +0,0 @@ -package framework - -import ( - "testing" - - "github.com/gruntwork-io/terratest/modules/logger" - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/rancher/tfp-automation/config" - resources "github.com/rancher/tfp-automation/framework/set/resources/sanity" -) - -// SanitySetup is a function that will set the Terraform configuration and return the Terraform options. -func SanitySetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) { - keyPath := resources.KeyPath() - - var terratestLogger logger.Logger - if terratestConfig.TFLogging { - terratestLogger = *logger.Default - } else { - terratestLogger = *logger.Discard - } - - terraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ - TerraformDir: keyPath, - NoColor: true, - Logger: &terratestLogger, - }) - - return terraformOptions, keyPath -} diff --git a/framework/set/provisioning/airgap/setConfig.go b/framework/set/provisioning/airgap/setConfig.go index 8406d91e..fcb93fbe 100644 --- a/framework/set/provisioning/airgap/setConfig.go +++ b/framework/set/provisioning/airgap/setConfig.go @@ -12,6 +12,7 @@ import ( "github.com/rancher/tfp-automation/framework/set/provisioning/airgap/nullresource" "github.com/rancher/tfp-automation/framework/set/provisioning/custom/locals" v2 "github.com/rancher/tfp-automation/framework/set/provisioning/custom/rke2k3s" + rke2k3s "github.com/rancher/tfp-automation/framework/set/provisioning/nodedriver/rke2k3s" airgap "github.com/rancher/tfp-automation/framework/set/resources/airgap/aws" "github.com/rancher/tfp-automation/framework/set/resources/sanity/aws" "github.com/sirupsen/logrus" @@ -32,7 +33,7 @@ func SetAirgapRKE2K3s(rancherConfig *rancher.Config, terraformConfig *config.Ter rootBody.AppendNewline() if terraformConfig.StandaloneRegistry.Authenticated { - createRegistrySecret(terraformConfig, clusterName, rootBody) + rke2k3s.CreateRegistrySecret(terraformConfig, clusterName, rootBody) rootBody.AppendNewline() } diff --git a/framework/set/provisioning/custom/rke2k3s/setRancher2ClusterV2.go b/framework/set/provisioning/custom/rke2k3s/setRancher2ClusterV2.go index 4408ec08..1b3e4702 100644 --- a/framework/set/provisioning/custom/rke2k3s/setRancher2ClusterV2.go +++ b/framework/set/provisioning/custom/rke2k3s/setRancher2ClusterV2.go @@ -30,6 +30,11 @@ func SetRancher2ClusterV2(rootBody *hclwrite.Body, terraformConfig *config.Terra } if terraformConfig.PrivateRegistries != nil { + if terraformConfig.PrivateRegistries.Username != "" { + rootBody.AppendNewline() + v2.CreateRegistrySecret(terraformConfig, clusterName, rootBody) + } + v2.SetMachineSelectorConfig(rkeConfigBlockBody, terraformConfig) registryBlock := rkeConfigBlockBody.AppendNewBlock(defaults.PrivateRegistries, nil) diff --git a/framework/set/provisioning/nodedriver/rke1/setPrivateRegistry.go b/framework/set/provisioning/nodedriver/rke1/setPrivateRegistry.go index a4dbcb37..f9df3102 100644 --- a/framework/set/provisioning/nodedriver/rke1/setPrivateRegistry.go +++ b/framework/set/provisioning/nodedriver/rke1/setPrivateRegistry.go @@ -9,8 +9,11 @@ import ( // setRKE1PrivateRegistryConfig is a function that will set the private registry configurations in the main.tf file. func setRKE1PrivateRegistryConfig(registryBlockBody *hclwrite.Body, terraformConfig *config.TerraformConfig) error { registryBlockBody.SetAttributeValue(privateRegistryURL, cty.StringVal(terraformConfig.PrivateRegistries.URL)) - registryBlockBody.SetAttributeValue(privateRegistryUsername, cty.StringVal(terraformConfig.PrivateRegistries.Username)) - registryBlockBody.SetAttributeValue(privateRegistryPassword, cty.StringVal(terraformConfig.PrivateRegistries.Password)) + + if terraformConfig.StandaloneRegistry.Authenticated { + registryBlockBody.SetAttributeValue(privateRegistryUsername, cty.StringVal(terraformConfig.PrivateRegistries.Username)) + registryBlockBody.SetAttributeValue(privateRegistryPassword, cty.StringVal(terraformConfig.PrivateRegistries.Password)) + } return nil } diff --git a/framework/set/provisioning/airgap/createRegistrySecret.go b/framework/set/provisioning/nodedriver/rke2k3s/createRegistrySecret.go similarity index 90% rename from framework/set/provisioning/airgap/createRegistrySecret.go rename to framework/set/provisioning/nodedriver/rke2k3s/createRegistrySecret.go index 42d420c2..9484e996 100644 --- a/framework/set/provisioning/airgap/createRegistrySecret.go +++ b/framework/set/provisioning/nodedriver/rke2k3s/createRegistrySecret.go @@ -1,4 +1,4 @@ -package airgap +package rke2k3s import ( "github.com/hashicorp/hcl/v2/hclwrite" @@ -16,8 +16,8 @@ const ( username = "username" ) -// createRegistrySecret is a function that will set the airgap RKE2/K3s cluster configurations in the main.tf file. -func createRegistrySecret(terraformConfig *config.TerraformConfig, clusterName string, rootBody *hclwrite.Body) { +// CreateRegistrySecret is a function that will set the airgap RKE2/K3s cluster configurations in the main.tf file. +func CreateRegistrySecret(terraformConfig *config.TerraformConfig, clusterName string, rootBody *hclwrite.Body) { secretBlock := rootBody.AppendNewBlock(defaults.Resource, []string{defaults.SecretV2, clusterName}) secretBlockBody := secretBlock.Body() diff --git a/framework/set/provisioning/nodedriver/rke2k3s/setConfig.go b/framework/set/provisioning/nodedriver/rke2k3s/setConfig.go index bc05d64c..95a3577e 100644 --- a/framework/set/provisioning/nodedriver/rke2k3s/setConfig.go +++ b/framework/set/provisioning/nodedriver/rke2k3s/setConfig.go @@ -133,6 +133,11 @@ func SetRKE2K3s(client *rancher.Client, terraformConfig *config.TerraformConfig, } if terraformConfig.PrivateRegistries != nil && strings.Contains(terraformConfig.Module, modules.EC2) { + if terraformConfig.PrivateRegistries.Username != "" { + rootBody.AppendNewline() + CreateRegistrySecret(terraformConfig, clusterName, rootBody) + } + SetMachineSelectorConfig(rkeConfigBlockBody, terraformConfig) registryBlock := rkeConfigBlockBody.AppendNewBlock(defaults.PrivateRegistries, nil) diff --git a/framework/set/provisioning/nodedriver/rke2k3s/setPrivateRegistry.go b/framework/set/provisioning/nodedriver/rke2k3s/setPrivateRegistry.go index fec11a84..d7564c77 100644 --- a/framework/set/provisioning/nodedriver/rke2k3s/setPrivateRegistry.go +++ b/framework/set/provisioning/nodedriver/rke2k3s/setPrivateRegistry.go @@ -29,7 +29,7 @@ func SetPrivateRegistryConfig(registryBlockBody *hclwrite.Body, terraformConfig configBlockBody.SetAttributeValue(hostname, cty.StringVal(terraformConfig.PrivateRegistries.URL)) - if terraformConfig.StandaloneRegistry.Authenticated { + if terraformConfig.PrivateRegistries.Username != "" { configBlockBody.SetAttributeValue(authConfigSecretName, cty.StringVal(terraformConfig.PrivateRegistries.AuthConfigSecretName)) } @@ -42,9 +42,5 @@ func SetPrivateRegistryConfig(registryBlockBody *hclwrite.Body, terraformConfig mirrorsBlockBody.SetAttributeValue(hostname, cty.StringVal(terraformConfig.PrivateRegistries.URL)) - if !terraformConfig.StandaloneRegistry.Authenticated { - mirrorsBlockBody.SetAttributeValue(endpoints, cty.ListVal([]cty.Value{cty.StringVal("http://" + terraformConfig.PrivateRegistries.URL + ":5000")})) - } - return nil } diff --git a/framework/set/resources/airgap/aws/instances.go b/framework/set/resources/airgap/aws/instances.go index 5beaa849..30c28c14 100644 --- a/framework/set/resources/airgap/aws/instances.go +++ b/framework/set/resources/airgap/aws/instances.go @@ -20,7 +20,7 @@ func CreateAirgappedAWSInstances(rootBody *hclwrite.Body, terraformConfig *confi configBlockBody.SetAttributeValue(defaults.InstanceType, cty.StringVal(terraformConfig.AWSConfig.AWSInstanceType)) configBlockBody.SetAttributeValue(defaults.SubnetId, cty.StringVal(terraformConfig.AWSConfig.AWSSubnetID)) - awsSecGroupsExpression := fmt.Sprintf(`["%s"]`, terraformConfig.AWSConfig.AWSSecurityGroupNames[0]) + awsSecGroupsExpression := fmt.Sprintf(`["%s"]`, terraformConfig.AWSConfig.StandaloneSecurityGroupNames[0]) awsSecGroupsList := hclwrite.Tokens{ {Type: hclsyntax.TokenIdent, Bytes: []byte(awsSecGroupsExpression)}, } diff --git a/framework/set/resources/airgap/createMainTF.go b/framework/set/resources/airgap/createMainTF.go index f5beea79..7504423f 100644 --- a/framework/set/resources/airgap/createMainTF.go +++ b/framework/set/resources/airgap/createMainTF.go @@ -11,7 +11,7 @@ import ( "github.com/rancher/tfp-automation/framework/set/resources/airgap/aws" "github.com/rancher/tfp-automation/framework/set/resources/airgap/rancher" "github.com/rancher/tfp-automation/framework/set/resources/airgap/rke2" - "github.com/rancher/tfp-automation/framework/set/resources/registries" + registry "github.com/rancher/tfp-automation/framework/set/resources/registries/createRegistry" "github.com/sirupsen/logrus" ) @@ -20,6 +20,8 @@ const ( rke2ServerTwo = "rke2_server2" rke2ServerThree = "rke2_server3" + nonAuthRegistry = "non_auth_registry" + registryPublicDNS = "registry_public_dns" rke2BastionPublicDNS = "rke2_bastion_public_dns" rke2ServerOnePrivateIP = "rke2_server1_private_ip" @@ -57,12 +59,12 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str file = OpenFile(file, keyPath) if terraformConfig.StandaloneRegistry.Authenticated { - file, err = registries.CreateAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, registryPublicDNS) + file, err = registry.CreateAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, registryPublicDNS) if err != nil { return "", err } } else { - file, err = registries.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, registryPublicDNS) + file, err = registry.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, registryPublicDNS, nonAuthRegistry) if err != nil { return "", err } diff --git a/framework/set/resources/airgap/rke2/createAirgapCluster.go b/framework/set/resources/airgap/rke2/createAirgapCluster.go index 55d11cbc..99abd0c6 100644 --- a/framework/set/resources/airgap/rke2/createAirgapCluster.go +++ b/framework/set/resources/airgap/rke2/createAirgapCluster.go @@ -11,6 +11,7 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/framework/set/defaults" "github.com/rancher/tfp-automation/framework/set/resources/sanity/rke2" + sanity "github.com/rancher/tfp-automation/framework/set/resources/sanity/rke2" "github.com/sirupsen/logrus" "github.com/zclconf/go-cty/cty" ) @@ -80,37 +81,10 @@ func CreateAirgapRKE2Cluster(file *os.File, newFile *hclwrite.File, rootBody *hc return file, nil } -// CreateNullResource is a helper function that will create the null_resource for the RKE2 cluster. -func CreateNullResource(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, instance, host string) (*hclwrite.Body, *hclwrite.Body) { - nullResourceBlock := rootBody.AppendNewBlock(defaults.Resource, []string{defaults.NullResource, host}) - nullResourceBlockBody := nullResourceBlock.Body() - - provisionerBlock := nullResourceBlockBody.AppendNewBlock(defaults.Provisioner, []string{defaults.RemoteExec}) - provisionerBlockBody := provisionerBlock.Body() - - connectionBlock := provisionerBlockBody.AppendNewBlock(defaults.Connection, nil) - connectionBlockBody := connectionBlock.Body() - - connectionBlockBody.SetAttributeValue(defaults.Host, cty.StringVal(instance)) - connectionBlockBody.SetAttributeValue(defaults.Type, cty.StringVal(defaults.Ssh)) - connectionBlockBody.SetAttributeValue(defaults.User, cty.StringVal(terraformConfig.AWSConfig.AWSUser)) - - keyPathExpression := defaults.File + `("` + terraformConfig.PrivateKeyPath + `")` - keyPath := hclwrite.Tokens{ - {Type: hclsyntax.TokenIdent, Bytes: []byte(keyPathExpression)}, - } - - connectionBlockBody.SetAttributeRaw(defaults.PrivateKey, keyPath) - - rootBody.AppendNewline() - - return nullResourceBlockBody, provisionerBlockBody -} - // createAirgappedRKE2Server is a helper function that will create the RKE2 server. func createAirgappedRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2Token, registryPublicDNS string, script []byte) { - nullResourceBlockBody, provisionerBlockBody := CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOne) + nullResourceBlockBody, provisionerBlockBody := sanity.CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOne) command := "bash -c '/tmp/init-server.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + rke2ServerOnePrivateIP + " " + rke2Token + " " + registryPublicDNS + " " + terraformConfig.Standalone.RancherImage + " " + @@ -148,7 +122,7 @@ func addAirgappedRKE2ServerNodes(rootBody *hclwrite.Body, terraformConfig *confi for i, instance := range instances { host := hosts[i] - nullResourceBlockBody, provisionerBlockBody := CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, host) + nullResourceBlockBody, provisionerBlockBody := sanity.CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, host) command := "bash -c '/tmp/add-servers.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + rke2ServerOnePrivateIP + " " + instance + " " + rke2Token + " " + registryPublicDNS + " " + diff --git a/framework/set/resources/registries/createMainTF.go b/framework/set/resources/registries/createMainTF.go new file mode 100644 index 00000000..2fbd0c9f --- /dev/null +++ b/framework/set/resources/registries/createMainTF.go @@ -0,0 +1,121 @@ +package registries + +import ( + "os" + "sync" + "testing" + + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/rancher/tfp-automation/config" + registry "github.com/rancher/tfp-automation/framework/set/resources/registries/createRegistry" + "github.com/rancher/tfp-automation/framework/set/resources/registries/rancher" + "github.com/rancher/tfp-automation/framework/set/resources/registries/rke2" + "github.com/rancher/tfp-automation/framework/set/resources/sanity" + "github.com/rancher/tfp-automation/framework/set/resources/sanity/aws" + "github.com/sirupsen/logrus" +) + +const ( + authRegistryPublicDNS = "auth_registry_public_dns" + nonAuthRegistryPublicDNS = "non_auth_registry_public_dns" + globalRegistryPublicDNS = "global_registry_public_dns" + + authRegistry = "auth_registry" + nonAuthRegistry = "non_auth_registry" + globalRegistry = "global_registry" + + rke2ServerOne = "rke2_server1" + rke2ServerTwo = "rke2_server2" + rke2ServerThree = "rke2_server3" + rke2ServerOnePublicDNS = "rke2_server1_public_dns" + rke2ServerOnePrivateIP = "rke2_server1_private_ip" + rke2ServerTwoPublicDNS = "rke2_server2_public_dns" + rke2ServerThreePublicDNS = "rke2_server3_public_dns" + + terraformConst = "terraform" +) + +// CreateMainTF is a helper function that will create the main.tf file for creating an Airgapped-Rancher server. +func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath string, terraformConfig *config.TerraformConfig, + terratest *config.TerratestConfig) (string, string, string, error) { + var file *os.File + file = sanity.OpenFile(file, keyPath) + defer file.Close() + + newFile := hclwrite.NewEmptyFile() + rootBody := newFile.Body() + + tfBlock := rootBody.AppendNewBlock(terraformConst, nil) + tfBlockBody := tfBlock.Body() + + instances := []string{rke2ServerOne, rke2ServerTwo, rke2ServerThree, authRegistry, nonAuthRegistry, globalRegistry} + file, err := aws.CreateAWSResources(file, newFile, tfBlockBody, rootBody, terraformConfig, terratest, instances) + if err != nil { + return "", "", "", err + } + + terraform.InitAndApply(t, terraformOptions) + + authRegistryPublicDNS := terraform.Output(t, terraformOptions, authRegistryPublicDNS) + nonAuthRegistryPublicDNS := terraform.Output(t, terraformOptions, nonAuthRegistryPublicDNS) + globalRegistryPublicDNS := terraform.Output(t, terraformOptions, globalRegistryPublicDNS) + rke2ServerOnePublicDNS := terraform.Output(t, terraformOptions, rke2ServerOnePublicDNS) + rke2ServerOnePrivateIP := terraform.Output(t, terraformOptions, rke2ServerOnePrivateIP) + rke2ServerTwoPublicDNS := terraform.Output(t, terraformOptions, rke2ServerTwoPublicDNS) + rke2ServerThreePublicDNS := terraform.Output(t, terraformOptions, rke2ServerThreePublicDNS) + + // Will create the authenticated registry, unauthenticated registry, and global registry in parallel using goroutines. + var wg sync.WaitGroup + wg.Add(3) + + go func() { + defer wg.Done() + file = sanity.OpenFile(file, keyPath) + file, err = registry.CreateAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, authRegistryPublicDNS) + if err != nil { + logrus.Fatalf("Error creating authenticated registry: %v", err) + } + }() + + go func() { + defer wg.Done() + file = sanity.OpenFile(file, keyPath) + file, err = registry.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, nonAuthRegistryPublicDNS, nonAuthRegistry) + if err != nil { + logrus.Fatalf("Error creating unauthenticated registry: %v", err) + } + }() + + go func() { + defer wg.Done() + file = sanity.OpenFile(file, keyPath) + file, err = registry.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, globalRegistryPublicDNS, globalRegistry) + if err != nil { + logrus.Fatalf("Error creating global registry: %v", err) + } + + terraform.InitAndApply(t, terraformOptions) + }() + + wg.Wait() + + file = sanity.OpenFile(file, keyPath) + file, err = rke2.CreateRKE2Cluster(file, newFile, rootBody, terraformConfig, rke2ServerOnePublicDNS, rke2ServerOnePrivateIP, + rke2ServerTwoPublicDNS, rke2ServerThreePublicDNS, globalRegistryPublicDNS) + if err != nil { + return "", "", "", err + } + + terraform.InitAndApply(t, terraformOptions) + + file = sanity.OpenFile(file, keyPath) + file, err = rancher.CreateRancher(file, newFile, rootBody, terraformConfig, rke2ServerOnePublicDNS, globalRegistryPublicDNS) + if err != nil { + return "", "", "", err + } + + terraform.InitAndApply(t, terraformOptions) + + return authRegistryPublicDNS, nonAuthRegistryPublicDNS, globalRegistryPublicDNS, nil +} diff --git a/framework/set/resources/registries/createRegistry.go b/framework/set/resources/registries/createRegistry.go index 1110a5ec..c6aab712 100644 --- a/framework/set/resources/registries/createRegistry.go +++ b/framework/set/resources/registries/createRegistry.go @@ -12,11 +12,6 @@ import ( "github.com/zclconf/go-cty/cty" ) -const ( - authRegistry = "auth_registry" - nonAuthRegistry = "non_auth_registry" -) - // CreateAuthenticatedRegistry is a helper function that will create an authenticated registry. func CreateAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2AuthRegistryPublicDNS string) (*os.File, error) { @@ -63,7 +58,7 @@ func CreateAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootBody // CreateNonAuthenticatedRegistry is a helper function that will create a non-authenticated registry. func CreateNonAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, - rke2NonAuthRegistryPublicDNS string) (*os.File, error) { + rke2NonAuthRegistryPublicDNS, registryType string) (*os.File, error) { userDir, err := os.UserHomeDir() if err != nil { return nil, err @@ -76,7 +71,7 @@ func CreateNonAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootB return nil, err } - _, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2NonAuthRegistryPublicDNS, nonAuthRegistry) + _, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2NonAuthRegistryPublicDNS, registryType) command := "bash -c '/tmp/non-auth-registry.sh " + terraformConfig.StandaloneRegistry.RegistryName + " " + rke2NonAuthRegistryPublicDNS + " " + terraformConfig.Standalone.RancherTagVersion + " " + diff --git a/framework/set/resources/registries/auth-registry.sh b/framework/set/resources/registries/createRegistry/auth-registry.sh similarity index 100% rename from framework/set/resources/registries/auth-registry.sh rename to framework/set/resources/registries/createRegistry/auth-registry.sh diff --git a/framework/set/resources/registries/createRegistry/createRegistry.go b/framework/set/resources/registries/createRegistry/createRegistry.go new file mode 100644 index 00000000..c9f12203 --- /dev/null +++ b/framework/set/resources/registries/createRegistry/createRegistry.go @@ -0,0 +1,105 @@ +package createRegistry + +import ( + "os" + "path/filepath" + + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/rancher/tfp-automation/config" + "github.com/rancher/tfp-automation/framework/set/defaults" + "github.com/rancher/tfp-automation/framework/set/resources/sanity/rke2" + "github.com/sirupsen/logrus" + "github.com/zclconf/go-cty/cty" +) + +const ( + authRegistry = "auth_registry" + nonAuthRegistry = "non_auth_registry" +) + +// CreateAuthenticatedRegistry is a helper function that will create an authenticated registry. +func CreateAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, + rke2AuthRegistryPublicDNS string) (*os.File, error) { + userDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + registryScriptPath := filepath.Join(userDir, "go/src/github.com/rancher/tfp-automation/framework/set/resources/registries/createRegistry/auth-registry.sh") + + registryScriptContent, err := os.ReadFile(registryScriptPath) + if err != nil { + return nil, err + } + + _, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2AuthRegistryPublicDNS, authRegistry) + + command := "bash -c '/tmp/auth-registry.sh " + terraformConfig.StandaloneRegistry.RegistryUsername + " " + + terraformConfig.StandaloneRegistry.RegistryPassword + " " + terraformConfig.StandaloneRegistry.RegistryName + " " + + rke2AuthRegistryPublicDNS + " " + terraformConfig.Standalone.RancherTagVersion + " " + + terraformConfig.StandaloneRegistry.AssetsPath + " " + terraformConfig.Standalone.OSUser + " " + + terraformConfig.Standalone.RancherImage + + if terraformConfig.Standalone.StagingRancherAgentImage != "" { + command += " " + terraformConfig.Standalone.StagingRancherAgentImage + } + + command += "'" + + provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{ + cty.StringVal("echo '" + string(registryScriptContent) + "' > /tmp/auth-registry.sh"), + cty.StringVal("chmod +x /tmp/auth-registry.sh"), + cty.StringVal(command), + })) + + _, err = file.Write(newFile.Bytes()) + if err != nil { + logrus.Infof("Failed to append configurations to main.tf file. Error: %v", err) + return nil, err + } + + return file, nil +} + +// CreateNonAuthenticatedRegistry is a helper function that will create a non-authenticated registry. +func CreateNonAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, + rke2NonAuthRegistryPublicDNS, registryType string) (*os.File, error) { + userDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + registryScriptPath := filepath.Join(userDir, "go/src/github.com/rancher/tfp-automation/framework/set/resources/registries/createRegistry/non-auth-registry.sh") + + registryScriptContent, err := os.ReadFile(registryScriptPath) + if err != nil { + return nil, err + } + + _, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2NonAuthRegistryPublicDNS, registryType) + + command := "bash -c '/tmp/non-auth-registry.sh " + terraformConfig.StandaloneRegistry.RegistryName + " " + + rke2NonAuthRegistryPublicDNS + " " + terraformConfig.Standalone.RancherTagVersion + " " + + terraformConfig.StandaloneRegistry.AssetsPath + " " + terraformConfig.Standalone.OSUser + " " + + terraformConfig.Standalone.RancherImage + + if terraformConfig.Standalone.StagingRancherAgentImage != "" { + command += " " + terraformConfig.Standalone.StagingRancherAgentImage + } + + command += "'" + + provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{ + cty.StringVal("echo '" + string(registryScriptContent) + "' > /tmp/non-auth-registry.sh"), + cty.StringVal("chmod +x /tmp/non-auth-registry.sh"), + cty.StringVal(command), + })) + + _, err = file.Write(newFile.Bytes()) + if err != nil { + logrus.Infof("Failed to append configurations to main.tf file. Error: %v", err) + return nil, err + } + + return file, nil +} diff --git a/framework/set/resources/registries/non-auth-registry.sh b/framework/set/resources/registries/createRegistry/non-auth-registry.sh similarity index 100% rename from framework/set/resources/registries/non-auth-registry.sh rename to framework/set/resources/registries/createRegistry/non-auth-registry.sh diff --git a/framework/set/resources/registries/keyPath.go b/framework/set/resources/registries/keyPath.go new file mode 100644 index 00000000..535bfff6 --- /dev/null +++ b/framework/set/resources/registries/keyPath.go @@ -0,0 +1,29 @@ +package registries + +import ( + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +const ( + mainTfKeyPath = "REGISTRY_KEY_PATH" +) + +// KeyPath is a function that will set the path to the key file. +func KeyPath() string { + userDir, err := os.UserHomeDir() + if err != nil { + return "" + } + + mainTfDirPath := os.Getenv(mainTfKeyPath) + if mainTfDirPath == "" { + logrus.Fatalf("Expected env var not set: %s", mainTfKeyPath) + } + + keyPath := filepath.Join(userDir, mainTfDirPath) + + return keyPath +} diff --git a/framework/set/resources/registries/rancher/createRancher.go b/framework/set/resources/registries/rancher/createRancher.go new file mode 100644 index 00000000..176d7fd0 --- /dev/null +++ b/framework/set/resources/registries/rancher/createRancher.go @@ -0,0 +1,60 @@ +package rancher + +import ( + "os" + "path/filepath" + + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/rancher/tfp-automation/config" + "github.com/rancher/tfp-automation/framework/set/defaults" + "github.com/rancher/tfp-automation/framework/set/resources/sanity/rke2" + "github.com/sirupsen/logrus" + "github.com/zclconf/go-cty/cty" +) + +const ( + installRancher = "install_rancher" +) + +// CreateRancher is a function that will set the Rancher configurations in the main.tf file. +func CreateRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, + rke2ServerOnePublicDNS, registryPublicDNS string) (*os.File, error) { + userDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + scriptPath := filepath.Join(userDir, "go/src/github.com/rancher/tfp-automation/framework/set/resources/registries/rancher/setup.sh") + + scriptContent, err := os.ReadFile(scriptPath) + if err != nil { + return nil, err + } + + _, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2ServerOnePublicDNS, installRancher) + + command := "bash -c '/tmp/setup.sh " + terraformConfig.Standalone.RancherRepo + " " + terraformConfig.Standalone.RancherChartRepository + " " + + terraformConfig.Standalone.Type + " " + terraformConfig.Standalone.CertManagerVersion + " " + + terraformConfig.Standalone.RancherHostname + " " + terraformConfig.Standalone.RancherTagVersion + " " + + terraformConfig.Standalone.BootstrapPassword + " " + terraformConfig.Standalone.RancherImage + " " + registryPublicDNS + + if terraformConfig.Standalone.StagingRancherAgentImage != "" { + command += " " + terraformConfig.Standalone.StagingRancherAgentImage + } + + command += "'" + + provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{ + cty.StringVal("printf '" + string(scriptContent) + "' > /tmp/setup.sh"), + cty.StringVal("chmod +x /tmp/setup.sh"), + cty.StringVal(command), + })) + + _, err = file.Write(newFile.Bytes()) + if err != nil { + logrus.Infof("Failed to append configurations to main.tf file. Error: %v", err) + return nil, err + } + + return file, nil +} diff --git a/framework/set/resources/registries/rancher/setup.sh b/framework/set/resources/registries/rancher/setup.sh new file mode 100755 index 00000000..0fd79ed5 --- /dev/null +++ b/framework/set/resources/registries/rancher/setup.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +RANCHER_REPO=$1 +RANCHER_CHART_REPO=$2 +TYPE=$3 +CERT_MANAGER_VERSION=$4 +HOSTNAME=$5 +RANCHER_TAG_VERSION=$6 +BOOTSTRAP_PASSWORD=$7 +RANCHER_IMAGE=$8 +REGISTRY=$9 +STAGING_RANCHER_AGENT_IMAGE=${10} + +set -ex + +echo "Installing kubectl" +curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl +mkdir -p ~/.kube +rm kubectl + +echo "Installing Helm" +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod +x get_helm.sh +./get_helm.sh +rm get_helm.sh + +echo "Adding Helm chart repo" +helm repo add ${RANCHER_REPO} ${RANCHER_CHART_REPO}${TYPE} + +echo "Installing cert manager" +kubectl create ns cattle-system +kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/${CERT_MANAGER_VERSION}/cert-manager.crds.yaml +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm upgrade --install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version ${CERT_MANAGER_VERSION} +kubectl get pods --namespace cert-manager + +echo "Waiting 1 minute for Rancher" +sleep 60 + +echo "Installing Rancher" +if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then + helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \ + --set hostname=${HOSTNAME} \ + --set rancherImageTag=${RANCHER_TAG_VERSION} \ + --set rancherImage=${REGISTRY}/${RANCHER_IMAGE} \ + --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \ + --set "extraEnv[0].value=${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \ + --set systemDefaultRegistry=${REGISTRY} \ + --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel +else + helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \ + --set hostname=${HOSTNAME} \ + --set rancherImage=${REGISTRY}/${RANCHER_IMAGE} \ + --set rancherImageTag=${RANCHER_TAG_VERSION} \ + --set systemDefaultRegistry=${REGISTRY} \ + --set bootstrapPassword=${BOOTSTRAP_PASSWORD} +fi + +echo "Waiting for Rancher to be rolled out" +kubectl -n cattle-system rollout status deploy/rancher +kubectl -n cattle-system get deploy rancher + +echo "Waiting 3 minutes for Rancher to be ready to deploy downstream clusters" +sleep 180 \ No newline at end of file diff --git a/framework/set/resources/registries/rke2/add-servers.sh b/framework/set/resources/registries/rke2/add-servers.sh new file mode 100755 index 00000000..7624a054 --- /dev/null +++ b/framework/set/resources/registries/rke2/add-servers.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +USER=$1 +GROUP=$2 +K8S_VERSION=$3 +RKE2_SERVER_IP=$4 +RKE2_TOKEN=$5 +RANCHER_IMAGE=$6 +RANCHER_TAG_VERSION=$7 +REGISTRY=$8 +STAGING_RANCHER_AGENT_IMAGE=${9} + +set -e + +sudo mkdir -p /etc/rancher/rke2 +sudo touch /etc/rancher/rke2/config.yaml + +echo "server: https://${RKE2_SERVER_IP}:9345 +token: ${RKE2_TOKEN} +tls-san: + - ${RKE2_SERVER_IP}" | sudo tee /etc/rancher/rke2/config.yaml > /dev/null + +sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF +mirrors: + docker.io: + endpoint: + - "https://${REGISTRY}" +configs: + "${REGISTRY}": + tls: + insecure_skip_verify: true +EOF + +curl -sfL https://get.rke2.io --output install.sh +sudo chmod +x install.sh +sudo INSTALL_RKE2_VERSION=${K8S_VERSION} INSTALL_RKE2_TYPE='server' ./install.sh + +sudo systemctl enable rke2-server +sudo systemctl start rke2-server + +sudo tee /etc/docker/daemon.json > /dev/null << EOF +{ + "insecure-registries" : [ "${REGISTRY}" ] +} +EOF + +sudo systemctl restart docker && sudo systemctl daemon-reload + +if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then + sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION} + sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION} + + sudo systemctl restart rke2-server +fi \ No newline at end of file diff --git a/framework/set/resources/registries/rke2/createCluster.go b/framework/set/resources/registries/rke2/createCluster.go new file mode 100644 index 00000000..8820e45b --- /dev/null +++ b/framework/set/resources/registries/rke2/createCluster.go @@ -0,0 +1,115 @@ +package rke2 + +import ( + "os" + "path/filepath" + + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" + namegen "github.com/rancher/shepherd/pkg/namegenerator" + "github.com/rancher/tfp-automation/config" + "github.com/rancher/tfp-automation/framework/set/defaults" + sanity "github.com/rancher/tfp-automation/framework/set/resources/sanity/rke2" + "github.com/sirupsen/logrus" + "github.com/zclconf/go-cty/cty" +) + +const ( + rke2ServerOne = "rke2_server1" + rke2ServerTwo = "rke2_server2" + rke2ServerThree = "rke2_server3" + token = "token" +) + +// CreateRKE2Cluster is a helper function that will create the RKE2 cluster. +func CreateRKE2Cluster(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, + rke2ServerOnePublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPublicDNS, rke2ServerThreePublicDNS, + registryPublicDNS string) (*os.File, error) { + userDir, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + serverScriptPath := filepath.Join(userDir, "go/src/github.com/rancher/tfp-automation/framework/set/resources/registries/rke2/init-server.sh") + newServersScriptPath := filepath.Join(userDir, "go/src/github.com/rancher/tfp-automation/framework/set/resources/registries/rke2/add-servers.sh") + + serverOneScriptContent, err := os.ReadFile(serverScriptPath) + if err != nil { + return nil, err + } + + newServersScriptContent, err := os.ReadFile(newServersScriptPath) + if err != nil { + return nil, err + } + + rke2Token := namegen.AppendRandomString(token) + + createRKE2Server(rootBody, terraformConfig, rke2ServerOnePublicDNS, rke2ServerOnePrivateIP, rke2Token, registryPublicDNS, serverOneScriptContent) + addRKE2ServerNodes(rootBody, terraformConfig, rke2ServerOnePrivateIP, rke2ServerTwoPublicDNS, rke2ServerThreePublicDNS, rke2Token, registryPublicDNS, newServersScriptContent) + + _, err = file.Write(newFile.Bytes()) + if err != nil { + logrus.Infof("Failed to append configurations to main.tf file. Error: %v", err) + return nil, err + } + + return file, nil +} + +// createRKE2Server is a helper function that will create the RKE2 server. +func createRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2ServerOnePublicDNS, rke2ServerOnePrivateIP, + rke2Token, registryPublicDNS string, script []byte) { + _, provisionerBlockBody := sanity.CreateNullResource(rootBody, terraformConfig, rke2ServerOnePublicDNS, rke2ServerOne) + + command := "bash -c '/tmp/init-server.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + + terraformConfig.Standalone.RKE2Version + " " + rke2ServerOnePrivateIP + " " + rke2Token + " " + + terraformConfig.Standalone.RancherImage + " " + terraformConfig.Standalone.RancherTagVersion + " " + registryPublicDNS + + if terraformConfig.Standalone.StagingRancherAgentImage != "" { + command += " " + terraformConfig.Standalone.StagingRancherAgentImage + } + + command += "'" + + provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{ + cty.StringVal("printf '" + string(script) + "' > /tmp/init-server.sh"), + cty.StringVal("chmod +x /tmp/init-server.sh"), + cty.StringVal(command), + })) +} + +// addRKE2ServerNodes is a helper function that will add additional RKE2 server nodes to the initial RKE2 server. +func addRKE2ServerNodes(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2ServerOnePrivateIP, rke2ServerTwoPublicDNS, + rke2ServerThreePublicDNS, rke2Token, registryPublicDNS string, script []byte) { + instances := []string{rke2ServerTwoPublicDNS, rke2ServerThreePublicDNS} + hosts := []string{rke2ServerTwo, rke2ServerThree} + + for i, instance := range instances { + host := hosts[i] + nullResourceBlockBody, provisionerBlockBody := sanity.CreateNullResource(rootBody, terraformConfig, instance, host) + + command := "bash -c '/tmp/add-servers.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + + terraformConfig.Standalone.RKE2Version + " " + rke2ServerOnePrivateIP + " " + rke2Token + " " + + terraformConfig.Standalone.RancherImage + " " + terraformConfig.Standalone.RancherTagVersion + " " + registryPublicDNS + + if terraformConfig.Standalone.StagingRancherAgentImage != "" { + command += " " + terraformConfig.Standalone.StagingRancherAgentImage + } + + command += "'" + + provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{ + cty.StringVal("printf '" + string(script) + "' > /tmp/add-servers.sh"), + cty.StringVal("chmod +x /tmp/add-servers.sh"), + cty.StringVal(command), + })) + + dependsOnServer := `[` + defaults.NullResource + `.` + rke2ServerOne + `]` + server := hclwrite.Tokens{ + {Type: hclsyntax.TokenIdent, Bytes: []byte(dependsOnServer)}, + } + + nullResourceBlockBody.SetAttributeRaw(defaults.DependsOn, server) + } +} diff --git a/framework/set/resources/registries/rke2/init-server.sh b/framework/set/resources/registries/rke2/init-server.sh new file mode 100755 index 00000000..2ca2702e --- /dev/null +++ b/framework/set/resources/registries/rke2/init-server.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +USER=$1 +GROUP=$2 +K8S_VERSION=$3 +RKE2_SERVER_IP=$4 +RKE2_TOKEN=$5 +RANCHER_IMAGE=$6 +RANCHER_TAG_VERSION=$7 +REGISTRY=$8 +STAGING_RANCHER_AGENT_IMAGE=${9} + +set -e + +sudo mkdir -p /etc/rancher/rke2 +sudo touch /etc/rancher/rke2/config.yaml + +echo "token: ${RKE2_TOKEN} +tls-san: + - ${RKE2_SERVER_IP}" | sudo tee /etc/rancher/rke2/config.yaml > /dev/null + +sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF +mirrors: + docker.io: + endpoint: + - "https://${REGISTRY}" +configs: + "${REGISTRY}": + tls: + insecure_skip_verify: true +EOF + +curl -sfL https://get.rke2.io --output install.sh +sudo chmod +x install.sh +sudo INSTALL_RKE2_VERSION=${K8S_VERSION} INSTALL_RKE2_TYPE='server' ./install.sh + +sudo systemctl enable rke2-server +sudo systemctl start rke2-server + +sudo tee /etc/docker/daemon.json > /dev/null << EOF +{ + "insecure-registries" : [ "${REGISTRY}" ] +} +EOF + +sudo systemctl restart docker && sudo systemctl daemon-reload + +if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then + sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION} + sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION} + + sudo systemctl restart rke2-server +fi + +sudo mkdir -p /home/${USER}/.kube +sudo cp /etc/rancher/rke2/rke2.yaml /home/${USER}/.kube/config +sudo chown -R ${USER}:${GROUP} /home/${USER}/.kube \ No newline at end of file diff --git a/framework/set/resources/sanity/aws/instances.go b/framework/set/resources/sanity/aws/instances.go index 06247c74..d849a1a7 100644 --- a/framework/set/resources/sanity/aws/instances.go +++ b/framework/set/resources/sanity/aws/instances.go @@ -25,7 +25,14 @@ func CreateAWSInstances(rootBody *hclwrite.Body, terraformConfig *config.Terrafo configBlockBody.SetAttributeValue(defaults.InstanceType, cty.StringVal(terraformConfig.AWSConfig.AWSInstanceType)) configBlockBody.SetAttributeValue(defaults.SubnetId, cty.StringVal(terraformConfig.AWSConfig.AWSSubnetID)) - awsSecGroupsExpression := fmt.Sprintf(`["%s"]`, terraformConfig.AWSConfig.AWSSecurityGroupNames[0]) + var awsSecGroupsExpression string + + if terraformConfig.Standalone != nil { + awsSecGroupsExpression = fmt.Sprintf(`["%s"]`, terraformConfig.AWSConfig.StandaloneSecurityGroupNames[0]) + } else { + awsSecGroupsExpression = fmt.Sprintf(`["%s"]`, terraformConfig.AWSConfig.AWSSecurityGroupNames[0]) + } + awsSecGroupsList := hclwrite.Tokens{ {Type: hclsyntax.TokenIdent, Bytes: []byte(awsSecGroupsExpression)}, } diff --git a/framework/set/resources/sanity/rancher/createRancher.go b/framework/set/resources/sanity/rancher/createRancher.go index cd0ec516..2155e60f 100644 --- a/framework/set/resources/sanity/rancher/createRancher.go +++ b/framework/set/resources/sanity/rancher/createRancher.go @@ -17,7 +17,8 @@ const ( ) // CreateRancher is a function that will set the Rancher configurations in the main.tf file. -func CreateRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2ServerOnePublicDNS string) (*os.File, error) { +func CreateRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, + rke2ServerOnePublicDNS string) (*os.File, error) { userDir, err := os.UserHomeDir() if err != nil { return nil, err diff --git a/framework/set/resources/sanity/rke2/createCluster.go b/framework/set/resources/sanity/rke2/createCluster.go index 66d996f3..fc5b4fba 100644 --- a/framework/set/resources/sanity/rke2/createCluster.go +++ b/framework/set/resources/sanity/rke2/createCluster.go @@ -87,11 +87,13 @@ func createRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.Terraform rke2Token string, script []byte) { _, provisionerBlockBody := CreateNullResource(rootBody, terraformConfig, rke2ServerOnePublicDNS, rke2ServerOne) + command := "bash -c '/tmp/init-server.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + + terraformConfig.Standalone.RKE2Version + " " + rke2ServerOnePrivateIP + " " + rke2Token + "'" + provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{ cty.StringVal("printf '" + string(script) + "' > /tmp/init-server.sh"), cty.StringVal("chmod +x /tmp/init-server.sh"), - cty.StringVal("bash -c '/tmp/init-server.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + - terraformConfig.Standalone.RKE2Version + " " + rke2ServerOnePrivateIP + " " + rke2Token + "'"), + cty.StringVal(command), })) } @@ -105,11 +107,13 @@ func addRKE2ServerNodes(rootBody *hclwrite.Body, terraformConfig *config.Terrafo host := hosts[i] nullResourceBlockBody, provisionerBlockBody := CreateNullResource(rootBody, terraformConfig, instance, host) + command := "bash -c '/tmp/add-servers.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + + terraformConfig.Standalone.RKE2Version + " " + rke2ServerOnePrivateIP + " " + rke2Token + "'" + provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{ cty.StringVal("printf '" + string(script) + "' > /tmp/add-servers.sh"), cty.StringVal("chmod +x /tmp/add-servers.sh"), - cty.StringVal("bash -c '/tmp/add-servers.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " + - terraformConfig.Standalone.RKE2Version + " " + rke2ServerOnePrivateIP + " " + rke2Token + "'"), + cty.StringVal(command), })) dependsOnServer := `[` + defaults.NullResource + `.` + rke2ServerOne + `]` diff --git a/framework/rkeSetup.go b/framework/setup.go similarity index 57% rename from framework/rkeSetup.go rename to framework/setup.go index bc39e915..de270137 100644 --- a/framework/rkeSetup.go +++ b/framework/setup.go @@ -6,13 +6,10 @@ import ( "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/rancher/tfp-automation/config" - resources "github.com/rancher/tfp-automation/framework/set/resources/rke" ) -// RKESetup is a function that will set the Terraform configuration and return the Terraform options. -func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig) (*terraform.Options, string) { - keyPath := resources.KeyPath() - +// Setup is a function that will set the Terraform configuration and return the Terraform options. +func Setup(t *testing.T, terraformConfig *config.TerraformConfig, terratestConfig *config.TerratestConfig, keyPath string) *terraform.Options { var terratestLogger logger.Logger if terratestConfig.TFLogging { terratestLogger = *logger.Default @@ -26,5 +23,5 @@ func RKESetup(t *testing.T, terraformConfig *config.TerraformConfig, terratestCo Logger: &terratestLogger, }) - return terraformOptions, keyPath + return terraformOptions } diff --git a/modules/registries/main.tf b/modules/registries/main.tf new file mode 100644 index 00000000..6ffc8ec0 --- /dev/null +++ b/modules/registries/main.tf @@ -0,0 +1 @@ +// Leave blank - main.tf will be set during testing \ No newline at end of file diff --git a/modules/registries/outputs.tf b/modules/registries/outputs.tf new file mode 100644 index 00000000..f8a80b8b --- /dev/null +++ b/modules/registries/outputs.tf @@ -0,0 +1,27 @@ +output "auth_registry_public_dns" { + value = aws_instance.auth_registry.public_dns +} + +output "non_auth_registry_public_dns" { + value = aws_instance.non_auth_registry.public_dns +} + +output "global_registry_public_dns" { + value = aws_instance.global_registry.public_dns +} + +output "rke2_server1_public_dns" { + value = aws_instance.rke2_server1.public_dns +} + +output "rke2_server1_private_ip" { + value = aws_instance.rke2_server1.private_ip +} + +output "rke2_server2_public_dns" { + value = aws_instance.rke2_server2.public_dns +} + +output "rke2_server3_public_dns" { + value = aws_instance.rke2_server3.public_dns +} \ No newline at end of file diff --git a/tests/airgap/README.md b/tests/airgap/README.md index 78c7a8fc..86a1243f 100644 --- a/tests/airgap/README.md +++ b/tests/airgap/README.md @@ -71,6 +71,7 @@ terraform: awsUser: "" registryRootSize: 500 sshConnectionType: "ssh" + standaloneSecurityGroupNames: [""] timeout: "5m" ################################### # STANDALONE CONFIG - RANCHER SETUP diff --git a/tests/airgap/airgap_provisioning_test.go b/tests/airgap/airgap_provisioning_test.go index a38a54be..b1044d82 100644 --- a/tests/airgap/airgap_provisioning_test.go +++ b/tests/airgap/airgap_provisioning_test.go @@ -12,9 +12,10 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - airgapCleanup "github.com/rancher/tfp-automation/framework/cleanup/airgap" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/airgap" resources "github.com/rancher/tfp-automation/framework/set/resources/airgap" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -35,7 +36,8 @@ type TfpAirgapProvisioningTestSuite struct { } func (a *TfpAirgapProvisioningTestSuite) TearDownSuite() { - airgapCleanup.ConfigAirgapCleanup(a.T(), a.standaloneTerraformOptions) + keyPath := airgap.KeyPath() + cleanup.Cleanup(a.T(), a.standaloneTerraformOptions, keyPath) } func (a *TfpAirgapProvisioningTestSuite) SetupSuite() { @@ -45,7 +47,8 @@ func (a *TfpAirgapProvisioningTestSuite) SetupSuite() { a.terratestConfig = new(config.TerratestConfig) ranchFrame.LoadConfig(config.TerratestConfigurationFileKey, a.terratestConfig) - standaloneTerraformOptions, keyPath := framework.AirgapSetup(a.T(), a.terraformConfig, a.terratestConfig) + keyPath := airgap.KeyPath() + standaloneTerraformOptions := framework.Setup(a.T(), a.terraformConfig, a.terratestConfig, keyPath) a.standaloneTerraformOptions = standaloneTerraformOptions registry, err := resources.CreateMainTF(a.T(), a.standaloneTerraformOptions, keyPath, a.terraformConfig, a.terratestConfig) @@ -80,7 +83,8 @@ func (a *TfpAirgapProvisioningTestSuite) TfpSetupSuite(terratestConfig *config.T rancherConfig.AdminToken = userToken.Token - terraformOptions := framework.Rancher2Setup(a.T(), a.rancherConfig, terraformConfig, terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(a.T(), terraformConfig, terratestConfig, keyPath) a.terraformOptions = terraformOptions } @@ -109,7 +113,8 @@ func (a *TfpAirgapProvisioningTestSuite) TestTfpAirgapProvisioning() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() a.Run((tt.name), func() { - defer cleanup.ConfigCleanup(a.T(), a.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(a.T(), a.terraformOptions, keyPath) provisioning.Provision(a.T(), a.client, a.rancherConfig, &terraformConfig, &terratestConfig, testUser, testPassword, clusterName, poolName, a.terraformOptions, nil) provisioning.VerifyCluster(a.T(), a.client, clusterName, &terraformConfig, &terratestConfig) @@ -126,8 +131,8 @@ func (a *TfpAirgapProvisioningTestSuite) TestTfpAirgapUpgrading() { name string module string }{ - {"RKE2", "airgap_rke2"}, - {"K3S", "airgap_k3s"}, + {"Upgrading RKE2", "airgap_rke2"}, + {"Upgrading K3S", "airgap_k3s"}, } for _, tt := range tests { @@ -146,7 +151,8 @@ func (a *TfpAirgapProvisioningTestSuite) TestTfpAirgapUpgrading() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() a.Run((tt.name), func() { - defer cleanup.ConfigCleanup(a.T(), a.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(a.T(), a.terraformOptions, keyPath) provisioning.Provision(a.T(), a.client, a.rancherConfig, &terraformConfig, &terratestConfig, testUser, testPassword, clusterName, poolName, a.terraformOptions, nil) provisioning.VerifyCluster(a.T(), a.client, clusterName, &terraformConfig, &terratestConfig) diff --git a/tests/extensions/provisioning/forceCleanup.go b/tests/extensions/provisioning/forceCleanup.go index 6ce506b8..b24ce81c 100644 --- a/tests/extensions/provisioning/forceCleanup.go +++ b/tests/extensions/provisioning/forceCleanup.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/gruntwork-io/terratest/modules/terraform" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" resources "github.com/rancher/tfp-automation/framework/set/resources/rancher2" ) @@ -18,7 +18,7 @@ func ForceCleanup(t *testing.T) error { }) terraform.Destroy(t, terraformOptions) - cleanup.ConfigCleanupTF() + cleanup.TFFilesCleanup(keyPath) return nil } diff --git a/tests/rancher2/nodescaling/scale_hosted_test.go b/tests/rancher2/nodescaling/scale_hosted_test.go index e1f4e317..b7e6be27 100644 --- a/tests/rancher2/nodescaling/scale_hosted_test.go +++ b/tests/rancher2/nodescaling/scale_hosted_test.go @@ -11,7 +11,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -52,7 +53,8 @@ func (s *ScaleHostedTestSuite) SetupSuite() { s.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(s.T(), s.rancherConfig, s.terraformConfig, s.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(s.T(), s.terraformConfig, s.terratestConfig, keyPath) s.terraformOptions = terraformOptions } @@ -69,7 +71,8 @@ func (s *ScaleHostedTestSuite) TestTfpScaleHosted() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() s.Run((tt.name), func() { - defer cleanup.ConfigCleanup(s.T(), s.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(s.T(), s.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(s.T(), s.client) require.NoError(s.T(), err) diff --git a/tests/rancher2/nodescaling/scale_test.go b/tests/rancher2/nodescaling/scale_test.go index 5b0a462a..090824a8 100644 --- a/tests/rancher2/nodescaling/scale_test.go +++ b/tests/rancher2/nodescaling/scale_test.go @@ -10,7 +10,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -51,7 +52,8 @@ func (s *ScaleTestSuite) SetupSuite() { s.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(s.T(), s.rancherConfig, s.terraformConfig, s.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(s.T(), s.terraformConfig, s.terratestConfig, keyPath) s.terraformOptions = terraformOptions provisioning.GetK8sVersion(s.T(), s.client, s.terratestConfig, s.terraformConfig, configs.DefaultK8sVersion) @@ -90,7 +92,8 @@ func (s *ScaleTestSuite) TestTfpScale() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() s.Run((tt.name), func() { - defer cleanup.ConfigCleanup(s.T(), s.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(s.T(), s.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(s.T(), s.client) require.NoError(s.T(), err) @@ -130,7 +133,8 @@ func (s *ScaleTestSuite) TestTfpScaleDynamicInput() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() s.Run((tt.name), func() { - defer cleanup.ConfigCleanup(s.T(), s.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(s.T(), s.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(s.T(), s.client) require.NoError(s.T(), err) diff --git a/tests/rancher2/provisioning/README.md b/tests/rancher2/provisioning/README.md index 19b7f57e..6568fda2 100644 --- a/tests/rancher2/provisioning/README.md +++ b/tests/rancher2/provisioning/README.md @@ -58,7 +58,7 @@ terraform: ami: "" awsInstanceType: "" region: "" - awsSecurityGroups: + awsSecurityGroupNames: - "" awsSubnetID: "" awsVpcID: "" diff --git a/tests/rancher2/provisioning/provision_hosted_test.go b/tests/rancher2/provisioning/provision_hosted_test.go index ddb1b8ea..ea16bcf1 100644 --- a/tests/rancher2/provisioning/provision_hosted_test.go +++ b/tests/rancher2/provisioning/provision_hosted_test.go @@ -10,7 +10,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -51,7 +52,8 @@ func (p *ProvisionHostedTestSuite) SetupSuite() { p.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(p.T(), p.rancherConfig, p.terraformConfig, p.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(p.T(), p.terraformConfig, p.terratestConfig, keyPath) p.terraformOptions = terraformOptions } @@ -68,7 +70,8 @@ func (p *ProvisionHostedTestSuite) TestTfpProvisionHosted() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() p.Run((tt.name), func() { - defer cleanup.ConfigCleanup(p.T(), p.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(p.T(), p.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(p.T(), p.client) require.NoError(p.T(), err) diff --git a/tests/rancher2/provisioning/provision_test.go b/tests/rancher2/provisioning/provision_test.go index 4b2b50cb..7bddb349 100644 --- a/tests/rancher2/provisioning/provision_test.go +++ b/tests/rancher2/provisioning/provision_test.go @@ -10,7 +10,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + cleanup "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -51,7 +52,8 @@ func (p *ProvisionTestSuite) SetupSuite() { p.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(p.T(), p.rancherConfig, p.terraformConfig, p.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(p.T(), p.terraformConfig, p.terratestConfig, keyPath) p.terraformOptions = terraformOptions provisioning.GetK8sVersion(p.T(), p.client, p.terratestConfig, p.terraformConfig, configs.DefaultK8sVersion) @@ -77,7 +79,8 @@ func (p *ProvisionTestSuite) TestTfpProvision() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() p.Run((tt.name), func() { - defer cleanup.ConfigCleanup(p.T(), p.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(p.T(), p.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(p.T(), p.client) require.NoError(p.T(), err) @@ -105,7 +108,8 @@ func (p *ProvisionTestSuite) TestTfpProvisionDynamicInput() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() p.Run((tt.name), func() { - defer cleanup.ConfigCleanup(p.T(), p.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(p.T(), p.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(p.T(), p.client) require.NoError(p.T(), err) diff --git a/tests/rancher2/psact/psact_test.go b/tests/rancher2/psact/psact_test.go index f07c7593..59f05c16 100644 --- a/tests/rancher2/psact/psact_test.go +++ b/tests/rancher2/psact/psact_test.go @@ -10,7 +10,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -51,7 +52,8 @@ func (p *PSACTTestSuite) SetupSuite() { p.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(p.T(), p.rancherConfig, p.terraformConfig, p.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(p.T(), p.terraformConfig, p.terratestConfig, keyPath) p.terraformOptions = terraformOptions provisioning.GetK8sVersion(p.T(), p.client, p.terratestConfig, p.terraformConfig, configs.DefaultK8sVersion) @@ -80,7 +82,8 @@ func (p *PSACTTestSuite) TestTfpPSACT() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() p.Run((tt.name), func() { - defer cleanup.ConfigCleanup(p.T(), p.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(p.T(), p.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(p.T(), p.client) require.NoError(p.T(), err) diff --git a/tests/rancher2/rbac/auth_test.go b/tests/rancher2/rbac/auth_test.go index 8b8f3a33..c239315e 100644 --- a/tests/rancher2/rbac/auth_test.go +++ b/tests/rancher2/rbac/auth_test.go @@ -11,7 +11,8 @@ import ( "github.com/rancher/tfp-automation/defaults/authproviders" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" rb "github.com/rancher/tfp-automation/tests/extensions/rbac" "github.com/stretchr/testify/require" @@ -52,7 +53,8 @@ func (r *AuthConfigTestSuite) SetupSuite() { r.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(r.T(), r.rancherConfig, r.terraformConfig, r.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(r.T(), r.terraformConfig, r.terratestConfig, keyPath) r.terraformOptions = terraformOptions } @@ -71,7 +73,8 @@ func (r *AuthConfigTestSuite) TestTfpAuthConfig() { authConfig := *r.terraformConfig authConfig.AuthProvider = tt.authProvider r.Run((tt.name), func() { - defer cleanup.ConfigCleanup(r.T(), r.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(r.T(), r.terraformOptions, keyPath) rb.AuthConfig(r.T(), &authConfig, r.terraformOptions) }) @@ -95,7 +98,8 @@ func (r *AuthConfigTestSuite) TestTfpAuthConfigDynamicInput() { for _, tt := range tests { r.Run((tt.name), func() { - defer cleanup.ConfigCleanup(r.T(), r.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(r.T(), r.terraformOptions, keyPath) rb.AuthConfig(r.T(), r.terraformConfig, r.terraformOptions) }) diff --git a/tests/rancher2/rbac/rbac_test.go b/tests/rancher2/rbac/rbac_test.go index 496070d6..3cd5f864 100644 --- a/tests/rancher2/rbac/rbac_test.go +++ b/tests/rancher2/rbac/rbac_test.go @@ -10,7 +10,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" rb "github.com/rancher/tfp-automation/tests/extensions/rbac" @@ -52,7 +53,8 @@ func (r *RBACTestSuite) SetupSuite() { r.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(r.T(), r.rancherConfig, r.terraformConfig, r.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(r.T(), r.terraformConfig, r.terratestConfig, keyPath) r.terraformOptions = terraformOptions provisioning.GetK8sVersion(r.T(), r.client, r.terratestConfig, r.terraformConfig, configs.DefaultK8sVersion) @@ -78,7 +80,8 @@ func (r *RBACTestSuite) TestTfpRBAC() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() r.Run((tt.name), func() { - defer cleanup.ConfigCleanup(r.T(), r.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(r.T(), r.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(r.T(), r.client) require.NoError(r.T(), err) diff --git a/tests/rancher2/resources/build_module_test.go b/tests/rancher2/resources/build_module_test.go index bbee266c..52ffb8dc 100644 --- a/tests/rancher2/resources/build_module_test.go +++ b/tests/rancher2/resources/build_module_test.go @@ -3,7 +3,8 @@ package tests import ( "testing" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + cleanup "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -14,7 +15,8 @@ type BuildModuleTestSuite struct { } func (r *BuildModuleTestSuite) TestBuildModule() { - defer cleanup.ConfigCleanupTF() + keyPath := rancher2.SetKeyPath() + defer cleanup.TFFilesCleanup(keyPath) err := provisioning.BuildModule(r.T()) require.NoError(r.T(), err) diff --git a/tests/rancher2/snapshot/snapshot_restore_test.go b/tests/rancher2/snapshot/snapshot_restore_test.go index 6da927d0..8107ccb4 100644 --- a/tests/rancher2/snapshot/snapshot_restore_test.go +++ b/tests/rancher2/snapshot/snapshot_restore_test.go @@ -11,7 +11,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -52,7 +53,8 @@ func (s *SnapshotRestoreTestSuite) SetupSuite() { s.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(s.T(), s.rancherConfig, s.terraformConfig, s.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(s.T(), s.terraformConfig, s.terratestConfig, keyPath) s.terraformOptions = terraformOptions provisioning.GetK8sVersion(s.T(), s.client, s.terratestConfig, s.terraformConfig, configs.DefaultK8sVersion) @@ -89,7 +91,8 @@ func (s *SnapshotRestoreTestSuite) TestTfpSnapshotRestore() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() s.Run(tt.name, func() { - defer cleanup.ConfigCleanup(s.T(), s.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(s.T(), s.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(s.T(), s.client) require.NoError(s.T(), err) @@ -123,7 +126,8 @@ func (s *SnapshotRestoreTestSuite) TestTfpSnapshotRestoreDynamicInput() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() s.Run((tt.name), func() { - defer cleanup.ConfigCleanup(s.T(), s.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(s.T(), s.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(s.T(), s.client) require.NoError(s.T(), err) diff --git a/tests/rancher2/upgrading/kubernetes_hosted_test.go b/tests/rancher2/upgrading/kubernetes_hosted_test.go index 5cf47fbc..6fbeeb75 100644 --- a/tests/rancher2/upgrading/kubernetes_hosted_test.go +++ b/tests/rancher2/upgrading/kubernetes_hosted_test.go @@ -11,7 +11,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -52,7 +53,8 @@ func (k *KubernetesUpgradeHostedTestSuite) SetupSuite() { k.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(k.T(), k.rancherConfig, k.terraformConfig, k.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(k.T(), k.terraformConfig, k.terratestConfig, keyPath) k.terraformOptions = terraformOptions } @@ -69,7 +71,8 @@ func (k *KubernetesUpgradeHostedTestSuite) TestTfpKubernetesUpgradeHosted() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() k.Run((tt.name), func() { - defer cleanup.ConfigCleanup(k.T(), k.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(k.T(), k.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(k.T(), k.client) require.NoError(k.T(), err) diff --git a/tests/rancher2/upgrading/kubernetes_upgrade_test.go b/tests/rancher2/upgrading/kubernetes_upgrade_test.go index 054d9be8..6946ef73 100644 --- a/tests/rancher2/upgrading/kubernetes_upgrade_test.go +++ b/tests/rancher2/upgrading/kubernetes_upgrade_test.go @@ -10,7 +10,8 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" "github.com/stretchr/testify/require" @@ -51,7 +52,8 @@ func (k *KubernetesUpgradeTestSuite) SetupSuite() { k.terratestConfig = terratestConfig - terraformOptions := framework.Rancher2Setup(k.T(), k.rancherConfig, k.terraformConfig, k.terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(k.T(), k.terraformConfig, k.terratestConfig, keyPath) k.terraformOptions = terraformOptions provisioning.GetK8sVersion(k.T(), k.client, k.terratestConfig, k.terraformConfig, configs.SecondHighestVersion) @@ -76,7 +78,8 @@ func (k *KubernetesUpgradeTestSuite) TestTfpKubernetesUpgrade() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() k.Run((tt.name), func() { - defer cleanup.ConfigCleanup(k.T(), k.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(k.T(), k.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(k.T(), k.client) require.NoError(k.T(), err) @@ -107,7 +110,8 @@ func (k *KubernetesUpgradeTestSuite) TestTfpKubernetesUpgradeDynamicInput() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() k.Run((tt.name), func() { - defer cleanup.ConfigCleanup(k.T(), k.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(k.T(), k.terraformOptions, keyPath) adminClient, err := provisioning.FetchAdminClient(k.T(), k.client) require.NoError(k.T(), err) diff --git a/tests/registries/README.md b/tests/registries/README.md new file mode 100644 index 00000000..169dd492 --- /dev/null +++ b/tests/registries/README.md @@ -0,0 +1,119 @@ +# Private Registry Tests + +In the tfp-automation private registries test, the following workflow is followed: + +1. Setup Rancher HA utilizing Terraform resources + specified provider infrastructure. A global registry is set as the system default registry while an authenticated and non-authenticated registry are created. +2. Provision downstream RKE1 / RKE2 / K3S clusters - done using the global registry, authenticated registry and non-authenticated registry. +3. Perform post-cluster provisioning checks +4. Cleanup resources (Terraform explicitly needs to call its cleanup method so that each test doesn't experience caching issues) + +Please see below for more details for your config. Please note that the config can be in either JSON or YAML (all examples are illustrated in YAML). + +## Table of Contents +1. [Getting Started](#Getting-Started) +2. [Local Qase Reporting](#Local-Qase-Reporting) + +## Getting Started +The config is split up into multiple parts. Think of the parts as follows: +- Standalone config for setting up Rancher +- Standalone config for setting up private registry +- Custom cluster config for provisioning downstream clusters +- Rancher config + +In no particular order, see an example below: + +```yaml +####################### +# RANCHER CONFIG +####################### +rancher: + host: "" # REQUIRED - fill out with the expected Rancher server URL + adminPassword: "" # REQUIRED - this is the same as the bootstrapPassword below, make sure they match + adminToken: "" # REQUIRED - leave this field empty as shown + insecure: true # REQUIRED - leave this as true +####################### +# TERRAFORM CONFIG +####################### +terraform: + cloudCredentialName: "" # REQUIRED - fill with desired value + defaultClusterRoleForProjectMembers: "true" # REQUIRED - leave value as true + enableNetworkPolicy: false # REQUIRED - values are true or false - can leave as false + hostnamePrefix: "" # REQUIRED - fill with desired value + machineConfigName: "" # REQUIRED - fill with desired value + module: "" # REQUIRED - leave this field empty as shown + networkPlugin: "" # REQUIRED - fill with desired value + nodeTemplateName: "" # REQUIRED - fill with desired value + privateKeyPath: "" # REQUIRED - specify private key that will be used to access created instances + privateRegistries: + authConfigSecretName: "" # REQUIRED (authenticated registry only) - specify the name of the secret you wanted created + insecure: true + url: "" # LEAVE BLANK - will be set during the test + systemDefaultRegistry: "" # LEAVE BLANK - will be set during the test + username: "" # REQUIRED (authenticated registry only) - username of the private registry + password: "" # REQUIRED (authenticated registry only) - password of the private registry + ######################## + # INFRASTRUCTURE SETUP + ######################## + awsCredentials: + awsAccessKey: "" + awsSecretKey: "" + awsConfig: + ami: "" + awsKeyName: "" + awsInstanceType: "" + awsSecurityGroupNames: [""] + awsSubnetID: "" + awsVpcID: "" + awsZoneLetter: "" + awsRootSize: 100 + awsRoute53Zone: "" + region: "" + awsUser: "" + registryRootSize: 500 + sshConnectionType: "ssh" + standaloneSecurityGroupNames: [""] + timeout: "5m" + ################################### + # STANDALONE CONFIG - RANCHER SETUP + ################################### + standalone: + bootstrapPassword: "" # REQUIRED - this is the same as the adminPassword above, make sure they match + certManagerVersion: "" # REQUIRED - (e.g. v1.15.3) + osGroup: "" # REQUIRED - fill with group of the instance created + osUser: "" # REQUIRED - fill with username of the instance created + rancherChartRepository: "" # REQUIRED - fill with desired value. Must end with a trailing / + rancherHostname: "" # REQUIRED - fill with desired value + rancherImage: "" # REQUIRED - fill with desired value + rancherRepo: "" # REQUIRED - fill with desired value + rancherTagVersion: "" # REQUIRED - fill with desired value + type: "" # REQUIRED - fill with desired value + stagingRancherAgentImage: "" # OPTIONAL - fill out only if you are using staging registry + rke2Version: "" # REQUIRED - fill with desired RKE2 k8s value (i.e. v1.30.6+rke2r1) + #################################### + # STANDALONE CONFIG - REGISTRY SETUP + #################################### + standaloneRegistry: + assetsPath: "" # REQUIRED - ensure that you end with a trailing `/` + registryName: "" # REQUIRED (authenticated registry only) + registryPassword: "" # REQUIRED (authenticated registry only) + registryUsername: "" # REQUIRED (authenticated registry only) +``` + +Before running, be sure to run the following commands: + +`export RANCHER2_KEY_PATH="///go/src/github.com/rancher/tfp-automation/modules/rancher2"; export REGISTRY_KEY_PATH="///go/src/github.com/rancher/tfp-automation/modules/registries"; export RANCHER2_PROVIDER_VERSION=""; export CATTLE_TEST_CONFIG=; export LOCALS_PROVIDER_VERSION=""; export AWS_PROVIDER_VERSION=""` + +See the below examples on how to run the tests: + +`gotestsum --format standard-verbose --packages=github.com/rancher/tfp-automation/tests/registries --junitfile results.xml --jsonfile results.json -- -timeout=600m -v -run "TestTfpRegistriesTestSuite$"` + +If the specified test passes immediately without warning, try adding the -count=1 flag to get around this issue. This will avoid previous results from interfering with the new test run. + +## Local Qase Reporting +If you are planning to report to Qase locally, then you will need to have the following done: +1. The `terratest` block in your config file must have `localQaseReporting: true`. +2. The working shell session must have the following two environmental variables set: + - `QASE_AUTOMATION_TOKEN=""` + - `QASE_TEST_RUN_ID=""` +3. Append `./reporter` to the end of the `gotestsum` command. See an example below:: + - `gotestsum --format standard-verbose --packages=github.com/rancher/tfp-automation/tests/airgap --junitfile results.xml --jsonfile results.json -- -timeout=120m -v -run TestTfpAirgapProvisioningTestSuite$";/path/to/tfp-automation/reporter` \ No newline at end of file diff --git a/tests/registries/registries_test.go b/tests/registries/registries_test.go new file mode 100644 index 00000000..af71bf77 --- /dev/null +++ b/tests/registries/registries_test.go @@ -0,0 +1,235 @@ +package registries + +import ( + "testing" + + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/rancher/shepherd/clients/rancher" + management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" + "github.com/rancher/shepherd/extensions/token" + ranchFrame "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tfp-automation/config" + "github.com/rancher/tfp-automation/defaults/configs" + "github.com/rancher/tfp-automation/framework" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" + "github.com/rancher/tfp-automation/framework/set/resources/registries" + qase "github.com/rancher/tfp-automation/pipeline/qase/results" + "github.com/rancher/tfp-automation/tests/extensions/provisioning" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type TfpRegistriesTestSuite struct { + suite.Suite + client *rancher.Client + session *session.Session + rancherConfig *rancher.Config + terraformConfig *config.TerraformConfig + terratestConfig *config.TerratestConfig + standaloneTerraformOptions *terraform.Options + terraformOptions *terraform.Options + adminUser *management.User + authRegistry string + nonAuthRegistry string + globalRegistry string +} + +func (r *TfpRegistriesTestSuite) TearDownSuite() { + keyPath := registries.KeyPath() + cleanup.Cleanup(r.T(), r.standaloneTerraformOptions, keyPath) +} + +func (r *TfpRegistriesTestSuite) SetupSuite() { + r.terraformConfig = new(config.TerraformConfig) + ranchFrame.LoadConfig(config.TerraformConfigurationFileKey, r.terraformConfig) + + r.terratestConfig = new(config.TerratestConfig) + ranchFrame.LoadConfig(config.TerratestConfigurationFileKey, r.terratestConfig) + + keyPath := registries.KeyPath() + standaloneTerraformOptions := framework.Setup(r.T(), r.terraformConfig, r.terratestConfig, keyPath) + r.standaloneTerraformOptions = standaloneTerraformOptions + + authRegistry, nonAuthRegistry, globalRegistry, err := registries.CreateMainTF(r.T(), r.standaloneTerraformOptions, keyPath, r.terraformConfig, r.terratestConfig) + require.NoError(r.T(), err) + + r.authRegistry = authRegistry + r.nonAuthRegistry = nonAuthRegistry + r.globalRegistry = globalRegistry +} + +func (r *TfpRegistriesTestSuite) TfpSetupSuite(terratestConfig *config.TerratestConfig, terraformConfig *config.TerraformConfig) { + testSession := session.NewSession() + r.session = testSession + + rancherConfig := new(rancher.Config) + ranchFrame.LoadConfig(configs.Rancher, rancherConfig) + + r.rancherConfig = rancherConfig + + adminUser := &management.User{ + Username: "admin", + Password: rancherConfig.AdminPassword, + } + + r.adminUser = adminUser + + userToken, err := token.GenerateUserToken(adminUser, r.rancherConfig.Host) + require.NoError(r.T(), err) + + client, err := rancher.NewClient(userToken.Token, testSession) + require.NoError(r.T(), err) + + r.client = client + + rancherConfig.AdminToken = userToken.Token + + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(r.T(), terraformConfig, terratestConfig, keyPath) + r.terraformOptions = terraformOptions +} + +func (r *TfpRegistriesTestSuite) TestTfpGlobalRegistry() { + nodeRolesAll := config.AllRolesNodePool + nodeRolesDedicated := []config.Nodepool{config.EtcdNodePool, config.ControlPlaneNodePool, config.WorkerNodePool} + + tests := []struct { + name string + module string + nodeRoles []config.Nodepool + }{ + {"Global RKE1", "ec2_rke1", nodeRolesDedicated}, + {"Global RKE2", "ec2_rke2", nodeRolesDedicated}, + {"Global K3S", "ec2_k3s", []config.Nodepool{nodeRolesAll}}, + } + + for _, tt := range tests { + terratestConfig := *r.terratestConfig + terraformConfig := *r.terraformConfig + terratestConfig.Nodepools = tt.nodeRoles + + terraformConfig.Module = tt.module + terraformConfig.PrivateRegistries.SystemDefaultRegistry = r.globalRegistry + terraformConfig.PrivateRegistries.URL = r.globalRegistry + terraformConfig.PrivateRegistries.Password = "" + terraformConfig.PrivateRegistries.Username = "" + terraformConfig.StandaloneRegistry.Authenticated = false + + r.TfpSetupSuite(&terratestConfig, &terraformConfig) + + provisioning.GetK8sVersion(r.T(), r.client, &terratestConfig, &terraformConfig, configs.DefaultK8sVersion) + + tt.name = tt.name + " Kubernetes version: " + terratestConfig.KubernetesVersion + testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() + + r.Run((tt.name), func() { + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(r.T(), r.terraformOptions, keyPath) + + provisioning.Provision(r.T(), r.client, r.rancherConfig, &terraformConfig, &terratestConfig, testUser, testPassword, clusterName, poolName, r.terraformOptions, nil) + provisioning.VerifyCluster(r.T(), r.client, clusterName, &terraformConfig, &terratestConfig) + }) + } + + if r.terratestConfig.LocalQaseReporting { + qase.ReportTest() + } +} + +func (r *TfpRegistriesTestSuite) TestTfpAuthenticatedRegistry() { + nodeRolesAll := config.AllRolesNodePool + nodeRolesDedicated := []config.Nodepool{config.EtcdNodePool, config.ControlPlaneNodePool, config.WorkerNodePool} + + tests := []struct { + name string + module string + nodeRoles []config.Nodepool + }{ + {"Auth RKE1", "ec2_rke1", nodeRolesDedicated}, + {"Auth RKE2", "ec2_rke2", nodeRolesDedicated}, + {"Auth K3S", "ec2_k3s", []config.Nodepool{nodeRolesAll}}, + } + + for _, tt := range tests { + terratestConfig := *r.terratestConfig + terraformConfig := *r.terraformConfig + terratestConfig.Nodepools = tt.nodeRoles + terraformConfig.Module = tt.module + + terraformConfig.PrivateRegistries.SystemDefaultRegistry = r.authRegistry + terraformConfig.PrivateRegistries.URL = r.authRegistry + terraformConfig.StandaloneRegistry.Authenticated = true + + r.TfpSetupSuite(&terratestConfig, &terraformConfig) + + provisioning.GetK8sVersion(r.T(), r.client, &terratestConfig, &terraformConfig, configs.DefaultK8sVersion) + + tt.name = tt.name + " Kubernetes version: " + terratestConfig.KubernetesVersion + testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() + + r.Run((tt.name), func() { + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(r.T(), r.terraformOptions, keyPath) + + provisioning.Provision(r.T(), r.client, r.rancherConfig, &terraformConfig, &terratestConfig, testUser, testPassword, clusterName, poolName, r.terraformOptions, nil) + provisioning.VerifyCluster(r.T(), r.client, clusterName, &terraformConfig, &terratestConfig) + }) + } + + if r.terratestConfig.LocalQaseReporting { + qase.ReportTest() + } +} + +func (r *TfpRegistriesTestSuite) TestTfpNonAuthenticatedRegistry() { + nodeRolesAll := config.AllRolesNodePool + nodeRolesDedicated := []config.Nodepool{config.EtcdNodePool, config.ControlPlaneNodePool, config.WorkerNodePool} + + tests := []struct { + name string + module string + nodeRoles []config.Nodepool + }{ + {"Non Auth RKE1", "ec2_rke1", nodeRolesDedicated}, + {"Non Auth RKE2", "ec2_rke2", nodeRolesDedicated}, + {"Non Auth K3S", "ec2_k3s", []config.Nodepool{nodeRolesAll}}, + } + + for _, tt := range tests { + terratestConfig := *r.terratestConfig + terraformConfig := *r.terraformConfig + terratestConfig.Nodepools = tt.nodeRoles + + terraformConfig.Module = tt.module + terraformConfig.PrivateRegistries.SystemDefaultRegistry = r.nonAuthRegistry + terraformConfig.PrivateRegistries.URL = r.nonAuthRegistry + terraformConfig.PrivateRegistries.Password = "" + terraformConfig.PrivateRegistries.Username = "" + terraformConfig.StandaloneRegistry.Authenticated = false + + r.TfpSetupSuite(&terratestConfig, &terraformConfig) + + provisioning.GetK8sVersion(r.T(), r.client, &terratestConfig, &terraformConfig, configs.DefaultK8sVersion) + + tt.name = tt.name + " Kubernetes version: " + terratestConfig.KubernetesVersion + testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() + + r.Run((tt.name), func() { + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(r.T(), r.terraformOptions, keyPath) + + provisioning.Provision(r.T(), r.client, r.rancherConfig, &terraformConfig, &terratestConfig, testUser, testPassword, clusterName, poolName, r.terraformOptions, nil) + provisioning.VerifyCluster(r.T(), r.client, clusterName, &terraformConfig, &terratestConfig) + }) + } + + if r.terratestConfig.LocalQaseReporting { + qase.ReportTest() + } +} + +func TestTfpRegistriesTestSuite(t *testing.T) { + suite.Run(t, new(TfpRegistriesTestSuite)) +} diff --git a/tests/rke/README.md b/tests/rke/README.md index 2f531376..5aa9f62b 100644 --- a/tests/rke/README.md +++ b/tests/rke/README.md @@ -42,7 +42,6 @@ terraform: awsKeyName: "" awsInstanceType: "" region: "us-east-2" - awsSecurityGroupNames: [""] awsSubnetID: "" awsVpcID: "" awsZoneLetter: "" @@ -52,9 +51,10 @@ terraform: prefix: "" awsUser: "" sshConnectionType: "ssh" + standaloneSecurityGroupNames: [""] timeout: "5m" standalone: - rke1User: "" + osUser: "" ``` Before running, be sure to run the following commands: diff --git a/tests/rke/rke_provider_test.go b/tests/rke/rke_provider_test.go index 9beff75b..abe3e0c2 100644 --- a/tests/rke/rke_provider_test.go +++ b/tests/rke/rke_provider_test.go @@ -9,7 +9,7 @@ import ( "github.com/rancher/shepherd/pkg/session" "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rke" + "github.com/rancher/tfp-automation/framework/cleanup" rke "github.com/rancher/tfp-automation/framework/set/resources/rke" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/stretchr/testify/suite" @@ -26,7 +26,8 @@ type RKEProviderTestSuite struct { } func (t *RKEProviderTestSuite) TearDownSuite() { - cleanup.ConfigRKECleanup(t.T(), t.terraformOptions) + keyPath := rke.KeyPath() + cleanup.Cleanup(t.T(), t.terraformOptions, keyPath) } func (t *RKEProviderTestSuite) TestCreateRKECluster() { @@ -36,7 +37,8 @@ func (t *RKEProviderTestSuite) TestCreateRKECluster() { t.terratestConfig = new(config.TerratestConfig) ranchFrame.LoadConfig(config.TerratestConfigurationFileKey, t.terratestConfig) - terraformOptions, keyPath := framework.RKESetup(t.T(), t.terraformConfig, t.terratestConfig) + keyPath := rke.KeyPath() + terraformOptions := framework.Setup(t.T(), t.terraformConfig, t.terratestConfig, keyPath) t.terraformOptions = terraformOptions rke.CreateRKEMainTF(t.T(), t.terraformOptions, keyPath, t.terraformConfig, t.terratestConfig) diff --git a/tests/sanity/README.md b/tests/sanity/README.md index 0742f070..c7214432 100644 --- a/tests/sanity/README.md +++ b/tests/sanity/README.md @@ -62,7 +62,6 @@ terraform: ami: "" awsKeyName: "" awsInstanceType: "" - awsSecurityGroupNames: [""] awsSubnetID: "" awsVpcID: "" awsZoneLetter: "" @@ -72,6 +71,7 @@ terraform: prefix: "" awsUser: "" sshConnectionType: "ssh" + standaloneSecurityGroupNames: [""] timeout: "5m" ################################### # STANDALONE CONFIG - RANCHER SETUP @@ -89,7 +89,7 @@ terraform: type: "" # REQUIRED - fill with desired value rke2User: "" # REQUIRED - fill with username of the instance created stagingRancherAgentImage: "" # OPTIONAL - fill out only if you are using staging registry - rke2Version: "" # REQUIRED - fill with desired RKE2 k8s value you wish the local cluster to be + rke2Version: "" # REQUIRED - fill with desired RKE2 k8s value (i.e. v1.30.6+rke2r1) ``` Before running, be sure to run the following commands: diff --git a/tests/sanity/tfp_automation_sanity_test.go b/tests/sanity/tfp_automation_sanity_test.go index b3b9cf7c..db1406f7 100644 --- a/tests/sanity/tfp_automation_sanity_test.go +++ b/tests/sanity/tfp_automation_sanity_test.go @@ -12,8 +12,9 @@ import ( "github.com/rancher/tfp-automation/config" "github.com/rancher/tfp-automation/defaults/configs" "github.com/rancher/tfp-automation/framework" - cleanup "github.com/rancher/tfp-automation/framework/cleanup/rancher2" - standaloneCleanup "github.com/rancher/tfp-automation/framework/cleanup/sanity" + "github.com/rancher/tfp-automation/framework/cleanup" + "github.com/rancher/tfp-automation/framework/set/resources/rancher2" + "github.com/rancher/tfp-automation/framework/set/resources/sanity" resources "github.com/rancher/tfp-automation/framework/set/resources/sanity" qase "github.com/rancher/tfp-automation/pipeline/qase/results" "github.com/rancher/tfp-automation/tests/extensions/provisioning" @@ -34,7 +35,8 @@ type TfpSanityTestSuite struct { } func (t *TfpSanityTestSuite) TearDownSuite() { - standaloneCleanup.StandaloneConfigCleanup(t.T(), t.standaloneTerraformOptions) + keyPath := sanity.KeyPath() + cleanup.Cleanup(t.T(), t.standaloneTerraformOptions, keyPath) } func (t *TfpSanityTestSuite) SetupSuite() { @@ -44,7 +46,8 @@ func (t *TfpSanityTestSuite) SetupSuite() { t.terratestConfig = new(config.TerratestConfig) ranchFrame.LoadConfig(config.TerratestConfigurationFileKey, t.terratestConfig) - standaloneTerraformOptions, keyPath := framework.SanitySetup(t.T(), t.terraformConfig, t.terratestConfig) + keyPath := sanity.KeyPath() + standaloneTerraformOptions := framework.Setup(t.T(), t.terraformConfig, t.terratestConfig, keyPath) t.standaloneTerraformOptions = standaloneTerraformOptions resources.CreateMainTF(t.T(), t.standaloneTerraformOptions, keyPath, t.terraformConfig, t.terratestConfig) @@ -76,7 +79,8 @@ func (t *TfpSanityTestSuite) TfpSetupSuite(terratestConfig *config.TerratestConf rancherConfig.AdminToken = userToken.Token - terraformOptions := framework.Rancher2Setup(t.T(), t.rancherConfig, terraformConfig, terratestConfig) + keyPath := rancher2.SetKeyPath() + terraformOptions := framework.Setup(t.T(), terraformConfig, terratestConfig, keyPath) t.terraformOptions = terraformOptions provisioning.GetK8sVersion(t.T(), t.client, terratestConfig, terraformConfig, configs.DefaultK8sVersion) @@ -107,7 +111,8 @@ func (t *TfpSanityTestSuite) TestTfpProvisioningSanity() { testUser, testPassword, clusterName, poolName := configs.CreateTestCredentials() t.Run((tt.name), func() { - defer cleanup.ConfigCleanup(t.T(), t.terraformOptions) + keyPath := rancher2.SetKeyPath() + defer cleanup.Cleanup(t.T(), t.terraformOptions, keyPath) provisioning.Provision(t.T(), t.client, t.rancherConfig, &terraformConfig, &terratestConfig, testUser, testPassword, clusterName, poolName, t.terraformOptions, nil) provisioning.VerifyCluster(t.T(), t.client, clusterName, &terraformConfig, &terratestConfig)