Add infrastructure folder and Prime support for registries

markusewalker · Jan 31, 2025 · ba16c40 · ba16c40
1 parent 250ee33
commit ba16c40
Show file tree

Hide file tree

Showing 29 changed files with 839 additions and 114 deletions.
diff --git a/config/config.go b/config/config.go
@@ -123,6 +123,7 @@ type Standalone struct {
 	Repo                           string `json:"repo,omitempty" yaml:"repo,omitempty"`
 	OSUser                         string `json:"osUser,omitempty" yaml:"osUser,omitempty"`
 	OSGroup                        string `json:"osGroup,omitempty" yaml:"osGroup,omitempty"`
+	PrimeRancherAgentImage         string `json:"primeRancherAgentImage,omitempty" yaml:"primeRancherAgentImage,omitempty"`
 	RKE2Version                    string `json:"rke2Version,omitempty" yaml:"rke2Version,omitempty"`
 	StagingRancherAgentImage       string `json:"stagingRancherAgentImage,omitempty" yaml:"stagingRancherAgentImage,omitempty"`
 	UpgradedRancherChartRepository string `json:"upgradedRancherChartRepository,omitempty" yaml:"upgradedRancherChartRepository,omitempty"`

diff --git a/framework/set/resources/airgap/createMainTF.go b/framework/set/resources/airgap/createMainTF.go
@@ -33,7 +33,7 @@ const (
 
 // CreateMainTF is a helper function that will create the main.tf file for creating an Airgapped-Rancher server.
 func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath string, terraformConfig *config.TerraformConfig,
-	terratest *config.TerratestConfig) (string, error) {
+	terratestConfig *config.TerratestConfig) (string, error) {
 	var file *os.File
 	file = OpenFile(file, keyPath)
 	defer file.Close()
@@ -44,7 +44,7 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str
 	tfBlock := rootBody.AppendNewBlock(terraformConst, nil)
 	tfBlockBody := tfBlock.Body()
 
-	file, err := aws.CreateAWSResources(file, newFile, tfBlockBody, rootBody, terraformConfig, terratest)
+	file, err := aws.CreateAWSResources(file, newFile, tfBlockBody, rootBody, terraformConfig, terratestConfig)
 	if err != nil {
 		return "", err
 	}
@@ -58,16 +58,9 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str
 	rke2ServerThreePrivateIP := terraform.Output(t, terraformOptions, rke2ServerThreePrivateIP)
 
 	file = OpenFile(file, keyPath)
-	if terraformConfig.StandaloneRegistry.Authenticated {
-		file, err = registry.CreateAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, registryPublicDNS)
-		if err != nil {
-			return "", err
-		}
-	} else {
-		file, err = registry.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, registryPublicDNS, nonAuthRegistry)
-		if err != nil {
-			return "", err
-		}
+	file, err = registry.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, registryPublicDNS, nonAuthRegistry)
+	if err != nil {
+		return "", err
 	}
 
 	terraform.InitAndApply(t, terraformOptions)

diff --git a/framework/set/resources/airgap/rancher/setupAirgapRancher.go b/framework/set/resources/airgap/rancher/setupAirgapRancher.go
@@ -43,6 +43,10 @@ func CreateAirgapRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwri
 		command += " " + terraformConfig.Standalone.StagingRancherAgentImage + " " + registryPublicDNS
 	}
 
+	if terraformConfig.Standalone.PrimeRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.PrimeRancherAgentImage
+	}
+
 	command += "'"
 
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{

diff --git a/framework/set/resources/airgap/rke2/add-servers.sh b/framework/set/resources/airgap/rke2/add-servers.sh
@@ -8,9 +8,8 @@ RKE2_TOKEN=$5
 REGISTRY=$6
 RANCHER_IMAGE=$7
 RANCHER_TAG_VERSION=$8
-REGISTRY_USERNAME=${9:-}
-REGISTRY_PASSWORD=${10:-}
-STAGING_RANCHER_AGENT_IMAGE=${11}
+STAGING_RANCHER_AGENT_IMAGE=${9:-}
+PRIME_RANCHER_AGENT_IMAGE=${10:-}
 PEM_FILE=/home/$USER/airgap.pem
 
 set -e
@@ -41,23 +40,7 @@ EOF
 
 setupRegistry() {
   sudo mkdir -p /etc/rancher/rke2
-
-  if [ -n "${REGISTRY_USERNAME}" ]; then
-    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
-mirrors:
-  docker.io:
-    endpoint:
-      - "https://${REGISTRY}"
-configs:
-  "${REGISTRY}":
-    auth:
-      username: "${REGISTRY_USERNAME}"
-      password: "${REGISTRY_PASSWORD}"
-    tls:
-      insecure_skip_verify: true
-EOF
-  else
-    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
+  sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
 mirrors:
   docker.io:
     endpoint:
@@ -67,7 +50,6 @@ configs:
     tls:
       insecure_skip_verify: true
 EOF
-  fi
 }
 
 setupDockerDaemon() {
@@ -88,17 +70,21 @@ runSSH "${RKE2_NEW_SERVER_IP}" "sudo INSTALL_RKE2_ARTIFACT_PATH=/home/${USER} sh
 runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl enable rke2-server"
 runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl start rke2-server"
 
-if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ] || [ -n "$PRIME_RANCHER_AGENT_IMAGE" ]; then
   setupDaemonFunction=$(declare -f setupDockerDaemon)
   runSSH "${RKE2_NEW_SERVER_IP}" "${setupDaemonFunction}; setupDockerDaemon"
   runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl restart docker && sudo systemctl daemon-reload"
 
-  if [ -n "$REGISTRY_USERNAME" ]; then
-    runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker login https://${REGISTRY} -u ${REGISTRY_USERNAME} -p ${REGISTRY_PASSWORD}"
+  runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION}"
+
+  if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+    runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
+  fi
+
+  if [ -n "$PRIME_RANCHER_AGENT_IMAGE" ]; then
+    runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker pull ${REGISTRY}/${PRIME_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
   fi
 
-  runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION}"
-  runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
   runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl restart rke2-server"
 fi
 

diff --git a/framework/set/resources/airgap/rke2/createAirgapCluster.go b/framework/set/resources/airgap/rke2/createAirgapCluster.go
@@ -90,14 +90,14 @@ func createAirgappedRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.
 		rke2ServerOnePrivateIP + " " + rke2Token + " " + registryPublicDNS + " " + terraformConfig.Standalone.RancherImage + " " +
 		terraformConfig.Standalone.RancherTagVersion
 
-	if terraformConfig.PrivateRegistries.Username != "" {
-		command += " " + terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password
-	}
-
 	if terraformConfig.Standalone.StagingRancherAgentImage != "" {
 		command += " " + terraformConfig.Standalone.StagingRancherAgentImage
 	}
 
+	if terraformConfig.Standalone.PrimeRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.PrimeRancherAgentImage
+	}
+
 	command += "'"
 
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
@@ -128,14 +128,14 @@ func addAirgappedRKE2ServerNodes(rootBody *hclwrite.Body, terraformConfig *confi
 			rke2ServerOnePrivateIP + " " + instance + " " + rke2Token + " " + registryPublicDNS + " " +
 			terraformConfig.Standalone.RancherImage + " " + terraformConfig.Standalone.RancherTagVersion
 
-		if terraformConfig.PrivateRegistries.Username != "" {
-			command += " " + terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password
-		}
-
 		if terraformConfig.Standalone.StagingRancherAgentImage != "" {
 			command += " " + terraformConfig.Standalone.StagingRancherAgentImage
 		}
 
+		if terraformConfig.Standalone.PrimeRancherAgentImage != "" {
+			command += " " + terraformConfig.Standalone.PrimeRancherAgentImage
+		}
+
 		command += "'"
 
 		provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{

diff --git a/framework/set/resources/airgap/rke2/init-server.sh b/framework/set/resources/airgap/rke2/init-server.sh
@@ -7,9 +7,8 @@ RKE2_TOKEN=$4
 REGISTRY=$5
 RANCHER_IMAGE=$6
 RANCHER_TAG_VERSION=$7
-REGISTRY_USERNAME=${8:-}
-REGISTRY_PASSWORD=${9:-}
-STAGING_RANCHER_AGENT_IMAGE=${10}
+STAGING_RANCHER_AGENT_IMAGE=${8:-}
+PRIME_RANCHER_AGENT_IMAGE=${9:-}
 PEM_FILE=/home/$USER/airgap.pem
 
 set -e
@@ -39,23 +38,7 @@ EOF
 
 setupRegistry() {
   sudo mkdir -p /etc/rancher/rke2
-
-  if [ -n "${REGISTRY_USERNAME}" ]; then
-    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
-mirrors:
-  docker.io:
-    endpoint:
-      - "https://${REGISTRY}"
-configs:
-  "${REGISTRY}":
-    auth:
-      username: "${REGISTRY_USERNAME}"
-      password: "${REGISTRY_PASSWORD}"
-    tls:
-      insecure_skip_verify: true
-EOF
-  else
-    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
+  sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
 mirrors:
   docker.io:
     endpoint:
@@ -65,7 +48,6 @@ configs:
     tls:
       insecure_skip_verify: true
 EOF
-  fi
 }
 
 setupDockerDaemon() {
@@ -88,17 +70,21 @@ runSSH "${RKE2_SERVER_ONE_IP}" "sudo INSTALL_RKE2_ARTIFACT_PATH=/home/${USER} sh
 runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl enable rke2-server"
 runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl start rke2-server"
 
-if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ] || [ -n "$PRIME_RANCHER_AGENT_IMAGE" ]; then
   setupDaemonFunction=$(declare -f setupDockerDaemon)
   runSSH "${RKE2_SERVER_ONE_IP}" "${setupDaemonFunction}; setupDockerDaemon"
   runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl restart docker && sudo systemctl daemon-reload"
-
-  if [ -n "$REGISTRY_USERNAME" ]; then
-    runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker login https://${REGISTRY} -u ${REGISTRY_USERNAME} -p ${REGISTRY_PASSWORD}"
-  fi
 
   runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION}"
-  runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
+
+  if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+    runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
+  fi
+
+  if [ -n "$PRIME_RANCHER_AGENT_IMAGE" ]; then
+    runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker pull ${REGISTRY}/${PRIME_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
+  fi
+
   runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl restart rke2-server"
 fi
 

diff --git a/framework/set/resources/registries/createMainTF.go b/framework/set/resources/registries/createMainTF.go
@@ -67,10 +67,14 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str
 
 	// Will create the authenticated registry, unauthenticated registry, and global registry in parallel using goroutines.
 	var wg sync.WaitGroup
+	var mutex sync.Mutex
 	wg.Add(3)
 
 	go func() {
 		defer wg.Done()
+		mutex.Lock()
+		defer mutex.Unlock()
+
 		file = sanity.OpenFile(file, keyPath)
 		file, err = registry.CreateAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, authRegistryPublicDNS)
 		if err != nil {
@@ -80,6 +84,9 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str
 
 	go func() {
 		defer wg.Done()
+		mutex.Lock()
+		defer mutex.Unlock()
+
 		file = sanity.OpenFile(file, keyPath)
 		file, err = registry.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, nonAuthRegistryPublicDNS, nonAuthRegistry)
 		if err != nil {
@@ -89,15 +96,18 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str
 
 	go func() {
 		defer wg.Done()
+		mutex.Lock()
+		defer mutex.Unlock()
+
 		file = sanity.OpenFile(file, keyPath)
 		file, err = registry.CreateNonAuthenticatedRegistry(file, newFile, rootBody, terraformConfig, globalRegistryPublicDNS, globalRegistry)
 		if err != nil {
 			logrus.Fatalf("Error creating global registry: %v", err)
 		}
-
-		terraform.InitAndApply(t, terraformOptions)
 	}()
 
+	terraform.InitAndApply(t, terraformOptions)
+
 	wg.Wait()
 
 	file = sanity.OpenFile(file, keyPath)

diff --git a/framework/set/resources/registries/createRegistry/auth-registry.sh b/framework/set/resources/registries/createRegistry/auth-registry.sh
@@ -8,7 +8,8 @@ RANCHER_VERSION=$5
 ASSET_DIR=$6
 USER=$7
 RANCHER_IMAGE=$8
-STAGING_RANCHER_AGENT_IMAGE=${9}
+STAGING_RANCHER_AGENT_IMAGE=${9:-}
+PRIME_RANCHER_AGENT_IMAGE=${10:-}
 
 set -e
 
@@ -58,6 +59,11 @@ if [ ! -z "${STAGING_RANCHER_AGENT_IMAGE}" ]; then
     sudo sed -i "s|rancher/rancher:|${RANCHER_IMAGE}:|g" /home/${USER}/rancher-images.txt
     sudo sed -i "s|rancher/rancher-agent:|${STAGING_RANCHER_AGENT_IMAGE}:|g" /home/${USER}/rancher-images.txt
 fi
+
+if [[ ! -z "${PRIME_RANCHER_AGENT_IMAGE}" ]]; then
+    sudo sed -i "s|rancher/rancher:|${RANCHER_IMAGE}:|g" /home/${USER}/rancher-images.txt
+    sudo sed -i "s|rancher/rancher-agent:|${PRIME_RANCHER_AGENT_IMAGE}:|g" /home/${USER}/rancher-images.txt
+fi
 
 echo "Saving the images..."
 sudo /home/${USER}/rancher-save-images.sh --image-list /home/${USER}/rancher-images.txt

diff --git a/framework/set/resources/registries/createRegistry/createRegistry.go b/framework/set/resources/registries/createRegistry/createRegistry.go
@@ -44,6 +44,10 @@ func CreateAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootBody
 		command += " " + terraformConfig.Standalone.StagingRancherAgentImage
 	}
 
+	if terraformConfig.Standalone.PrimeRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.PrimeRancherAgentImage
+	}
+
 	command += "'"
 
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{

diff --git a/framework/set/resources/registries/createRegistry/non-auth-registry.sh b/framework/set/resources/registries/createRegistry/non-auth-registry.sh
@@ -6,7 +6,8 @@ RANCHER_VERSION=$3
 ASSET_DIR=$4
 USER=$5
 RANCHER_IMAGE=$6
-STAGING_RANCHER_AGENT_IMAGE=${7}
+STAGING_RANCHER_AGENT_IMAGE=${7:-}
+PRIME_RANCHER_AGENT_IMAGE=${8:-}
 
 set -e
 
@@ -39,6 +40,11 @@ if [ ! -z "${STAGING_RANCHER_AGENT_IMAGE}" ]; then
     sudo sed -i "s|rancher/rancher:|${RANCHER_IMAGE}:|g" /home/${USER}/rancher-images.txt
     sudo sed -i "s|rancher/rancher-agent:|${STAGING_RANCHER_AGENT_IMAGE}:|g" /home/${USER}/rancher-images.txt
 fi
+
+if [[ ! -z "${PRIME_RANCHER_AGENT_IMAGE}" ]]; then
+    sudo sed -i "s|rancher/rancher:|${RANCHER_IMAGE}:|g" /home/${USER}/rancher-images.txt
+    sudo sed -i "s|rancher/rancher-agent:|${PRIME_RANCHER_AGENT_IMAGE}:|g" /home/${USER}/rancher-images.txt
+fi
 
 echo "Saving the images..."
 sudo /home/${USER}/rancher-save-images.sh --image-list /home/${USER}/rancher-images.txt

diff --git a/framework/set/resources/registries/rancher/createRancher.go b/framework/set/resources/registries/rancher/createRancher.go
@@ -42,6 +42,10 @@ func CreateRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Bod
 		command += " " + terraformConfig.Standalone.StagingRancherAgentImage
 	}
 
+	if terraformConfig.Standalone.PrimeRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.PrimeRancherAgentImage
+	}
+
 	command += "'"
 
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{

diff --git a/framework/set/resources/registries/rancher/setup.sh b/framework/set/resources/registries/rancher/setup.sh
@@ -48,6 +48,7 @@ if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
                                                                                  --set "extraEnv[0].value=${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \
                                                                                  --set systemDefaultRegistry=${REGISTRY} \
                                                                                  --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel
+
 else
     helm upgrade --install rancher rancher-${REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
                                                                                  --set hostname=${HOSTNAME} \

diff --git a/framework/set/resources/registries/rke2/add-servers.sh b/framework/set/resources/registries/rke2/add-servers.sh
@@ -8,7 +8,8 @@ RKE2_TOKEN=$5
 RANCHER_IMAGE=$6
 RANCHER_TAG_VERSION=$7
 REGISTRY=$8
-STAGING_RANCHER_AGENT_IMAGE=${9}
+STAGING_RANCHER_AGENT_IMAGE=${9:-}
+PRIME_RANCHER_AGENT_IMAGE=${10:-}
 
 set -e
 
@@ -46,9 +47,16 @@ EOF
 
 sudo systemctl restart docker && sudo systemctl daemon-reload
 
-if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ] || [ -n "$PRIME_RANCHER_AGENT_IMAGE" ]; then
   sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION}
-  sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}
+
+  if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+    sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}
+  fi
+
+  if [ -n "$PRIME_RANCHER_AGENT_IMAGE" ]; then
+    sudo docker pull ${REGISTRY}/${PRIME_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}
+  fi
 
   sudo systemctl restart rke2-server
 fi