Fix issue with rancher staging images not pulling

rancher · Jan 21, 2025 · 09ad204 · 09ad204
1 parent 4ac54c0
commit 09ad204
Show file tree

Hide file tree

Showing 12 changed files with 73 additions and 37 deletions.
diff --git a/config/config.go b/config/config.go
@@ -107,12 +107,12 @@ type Standalone struct {
 	RancherChartVersion      string `json:"rancherChartVersion,omitempty" yaml:"rancherChartVersion,omitempty"`
 	RancherChartRepository   string `json:"rancherChartRepository,omitempty" yaml:"rancherChartRepository,omitempty"`
 	RancherHostname          string `json:"rancherHostname,omitempty" yaml:"rancherHostname,omitempty"`
+	RancherImage             string `json:"rancherImage,omitempty" yaml:"rancherImage,omitempty"`
 	RancherRepo              string `json:"rancherRepo,omitempty" yaml:"rancherRepo,omitempty"`
 	RancherTagVersion        string `json:"rancherTagVersion,omitempty" yaml:"rancherTagVersion,omitempty"`
 	OSUser                   string `json:"osUser,omitempty" yaml:"osUser,omitempty"`
 	OSGroup                  string `json:"osGroup,omitempty" yaml:"osGroup,omitempty"`
 	RKE2Version              string `json:"rke2Version,omitempty" yaml:"rke2Version,omitempty"`
-	StagingRancherImage      string `json:"stagingRancherImage,omitempty" yaml:"stagingRancherImage,omitempty"`
 	StagingRancherAgentImage string `json:"stagingRancherAgentImage,omitempty" yaml:"stagingRancherAgentImage,omitempty"`
 	Type                     string `json:"type,omitempty" yaml:"type,omitempty"`
 }

diff --git a/framework/set/resources/airgap/createMainTF.go b/framework/set/resources/airgap/createMainTF.go
@@ -71,7 +71,7 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str
 	terraform.InitAndApply(t, terraformOptions)
 
 	file = OpenFile(file, keyPath)
-	file, err = rke2.CreateAirgapRKE2Cluster(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP)
+	file, err = rke2.CreateAirgapRKE2Cluster(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS, registryPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP)
 	if err != nil {
 		return "", err
 	}

diff --git a/framework/set/resources/airgap/rancher/setup.sh b/framework/set/resources/airgap/rancher/setup.sh
@@ -8,7 +8,7 @@ HOSTNAME=$5
 INTERNAL_FQDN=$6
 RANCHER_TAG_VERSION=$7
 BOOTSTRAP_PASSWORD=$8
-STAGING_RANCHER_IMAGE=${9}
+RANCHER_IMAGE=$9
 STAGING_RANCHER_AGENT_IMAGE=${10}
 
 set -ex
@@ -34,20 +34,21 @@ echo "Waiting 1 minute for Rancher"
 sleep 60
 
 echo "Installing Rancher"
-if [ -z "$STAGING_RANCHER_IMAGE" ]; then
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
     helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
                                                                                  --set hostname=${HOSTNAME} \
                                                                                  --set rancherImageTag=${RANCHER_TAG_VERSION} \
-                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD}
+                                                                                 --set rancherImage=${RANCHER_IMAGE} \
+                                                                                 --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \
+                                                                                 --set "extraEnv[0].value=${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \
+                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel
 
 else
     helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
                                                                                  --set hostname=${HOSTNAME} \
+                                                                                 --set rancherImage=${RANCHER_IMAGE} \
                                                                                  --set rancherImageTag=${RANCHER_TAG_VERSION} \
-                                                                                 --set rancherImage=${STAGING_RANCHER_IMAGE} \
-                                                                                 --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \                                                                                --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \
-                                                                                 --set "extraEnv[0].value=${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \
-                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel
+                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD}
 fi
 
 echo "Waiting for Rancher to be rolled out"

diff --git a/framework/set/resources/airgap/rancher/setupAirgapRancher.go b/framework/set/resources/airgap/rancher/setupAirgapRancher.go
@@ -36,10 +36,11 @@ func CreateAirgapRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwri
 	command := "bash -c '/tmp/setup.sh " + terraformConfig.Standalone.RancherRepo + " " + terraformConfig.Standalone.RancherChartRepository + " " +
 		terraformConfig.Standalone.Type + " " + terraformConfig.Standalone.CertManagerVersion + " " +
 		terraformConfig.Standalone.RancherHostname + " " + " " + terraformConfig.Standalone.AirgapInternalFQDN + " " +
-		terraformConfig.Standalone.RancherTagVersion + " " + terraformConfig.Standalone.BootstrapPassword
+		terraformConfig.Standalone.RancherTagVersion + " " + terraformConfig.Standalone.BootstrapPassword + " " +
+		terraformConfig.Standalone.RancherImage
 
-	if terraformConfig.Standalone.StagingRancherImage != "" {
-		command += " " + terraformConfig.Standalone.StagingRancherImage + " " + terraformConfig.Standalone.StagingRancherAgentImage
+	if terraformConfig.Standalone.StagingRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.StagingRancherAgentImage
 	}
 
 	command += "'"

diff --git a/framework/set/resources/airgap/rke2/createAirgapCluster.go b/framework/set/resources/airgap/rke2/createAirgapCluster.go
@@ -25,7 +25,7 @@ const (
 
 // CreateAirgapRKE2Cluster is a helper function that will create the RKE2 cluster.
 func CreateAirgapRKE2Cluster(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig,
-	rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP string) (*os.File, error) {
+	rke2BastionPublicDNS, registryPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP string) (*os.File, error) {
 	userDir, err := os.UserHomeDir()
 	if err != nil {
 		return nil, err
@@ -68,8 +68,8 @@ func CreateAirgapRKE2Cluster(file *os.File, newFile *hclwrite.File, rootBody *hc
 
 	rke2Token := namegen.AppendRandomString(token)
 
-	createAirgappedRKE2Server(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2Token, serverOneScriptContent)
-	addAirgappedRKE2ServerNodes(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP, rke2Token, newServersScriptContent)
+	createAirgappedRKE2Server(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2Token, registryPublicDNS, serverOneScriptContent)
+	addAirgappedRKE2ServerNodes(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP, rke2Token, registryPublicDNS, newServersScriptContent)
 
 	_, err = file.Write(newFile.Bytes())
 	if err != nil {
@@ -109,14 +109,14 @@ func CreateNullResource(rootBody *hclwrite.Body, terraformConfig *config.Terrafo
 
 // createAirgappedRKE2Server is a helper function that will create the RKE2 server.
 func createAirgappedRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP,
-	rke2Token string, script []byte) {
+	rke2Token, registryPublicDNS string, script []byte) {
 	nullResourceBlockBody, provisionerBlockBody := CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOne)
 
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
 		cty.StringVal("printf '" + string(script) + "' > /tmp/init-server.sh"),
 		cty.StringVal("chmod +x /tmp/init-server.sh"),
 		cty.StringVal("bash -c '/tmp/init-server.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " +
-			rke2ServerOnePrivateIP + " " + rke2Token + " " + terraformConfig.PrivateRegistries.URL + " " +
+			rke2ServerOnePrivateIP + " " + rke2Token + " " + registryPublicDNS + " " +
 			terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password + "'"),
 	}))
 
@@ -130,7 +130,7 @@ func createAirgappedRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.
 
 // addAirgappedRKE2ServerNodes is a helper function that will add additional RKE2 server nodes to the initial RKE2 airgapped server.
 func addAirgappedRKE2ServerNodes(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPublicDNS,
-	rke2ServerThreePublicDNS, rke2Token string, script []byte) {
+	rke2ServerThreePublicDNS, rke2Token, registryPublicDNS string, script []byte) {
 	instances := []string{rke2ServerTwoPublicDNS, rke2ServerThreePublicDNS}
 	hosts := []string{rke2ServerTwo, rke2ServerThree}
 
@@ -142,7 +142,7 @@ func addAirgappedRKE2ServerNodes(rootBody *hclwrite.Body, terraformConfig *confi
 			cty.StringVal("printf '" + string(script) + "' > /tmp/add-servers.sh"),
 			cty.StringVal("chmod +x /tmp/add-servers.sh"),
 			cty.StringVal("bash -c '/tmp/add-servers.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " +
-				rke2ServerOnePrivateIP + " " + instance + " " + rke2Token + " " + terraformConfig.PrivateRegistries.URL + " " +
+				rke2ServerOnePrivateIP + " " + instance + " " + rke2Token + " " + registryPublicDNS + " " +
 				terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password + "'"),
 		}))
 

diff --git a/framework/set/resources/registries/auth-registry.sh b/framework/set/resources/registries/auth-registry.sh
@@ -7,6 +7,8 @@ HOST=$4
 RANCHER_VERSION=$5
 ASSET_DIR=$6
 USER=$7
+RANCHER_IMAGE=$8
+STAGING_RANCHER_AGENT_IMAGE=${9}
 
 set -e
 
@@ -51,6 +53,11 @@ sudo chmod +x /home/${USER}/rancher-save-images.sh && sudo chmod +x /home/${USER
 sudo sed -i "s/docker save/# docker save /g" /home/${USER}/rancher-save-images.sh
 sudo sed -i "s/docker load/# docker load /g" /home/${USER}/rancher-load-images.sh
 sudo sed -i '/mirrored-prometheus-windows-exporter/d' /home/${USER}/rancher-images.txt
+
+if [ ! -z "${STAGING_RANCHER_AGENT_IMAGE}" ]; then
+    sudo sed -i "s|rancher/rancher:|${RANCHER_IMAGE}:|g" /home/${USER}/rancher-images.txt
+    sudo sed -i "s|rancher/rancher-agent:|${STAGING_RANCHER_AGENT_IMAGE}:|g" /home/${USER}/rancher-images.txt
+fi
 
 echo "Saving the images..."
 sudo /home/${USER}/rancher-save-images.sh --image-list /home/${USER}/rancher-images.txt

diff --git a/framework/set/resources/registries/createRegistry.go b/framework/set/resources/registries/createRegistry.go
@@ -34,13 +34,22 @@ func CreateAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootBody
 
 	_, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2AuthRegistryPublicDNS, authRegistry)
 
+	command := "bash -c '/tmp/auth-registry.sh " + terraformConfig.StandaloneRegistry.RegistryUsername + " " +
+		terraformConfig.StandaloneRegistry.RegistryPassword + " " + terraformConfig.StandaloneRegistry.RegistryName + " " +
+		rke2AuthRegistryPublicDNS + " " + terraformConfig.Standalone.RancherTagVersion + " " +
+		terraformConfig.StandaloneRegistry.AssetsPath + " " + terraformConfig.Standalone.OSUser + " " +
+		terraformConfig.Standalone.RancherImage
+
+	if terraformConfig.Standalone.StagingRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.StagingRancherAgentImage
+	}
+
+	command += "'"
+
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
 		cty.StringVal("echo '" + string(registryScriptContent) + "' > /tmp/auth-registry.sh"),
 		cty.StringVal("chmod +x /tmp/auth-registry.sh"),
-		cty.StringVal("bash -c '/tmp/auth-registry.sh " + terraformConfig.StandaloneRegistry.RegistryUsername + " " +
-			terraformConfig.StandaloneRegistry.RegistryPassword + " " + terraformConfig.StandaloneRegistry.RegistryName + " " +
-			rke2AuthRegistryPublicDNS + " " + terraformConfig.Standalone.RancherTagVersion + " " +
-			terraformConfig.StandaloneRegistry.AssetsPath + " " + terraformConfig.Standalone.OSUser + "'"),
+		cty.StringVal(command),
 	}))
 
 	_, err = file.Write(newFile.Bytes())
@@ -69,12 +78,21 @@ func CreateNonAuthenticatedRegistry(file *os.File, newFile *hclwrite.File, rootB
 
 	_, provisionerBlockBody := rke2.CreateNullResource(rootBody, terraformConfig, rke2NonAuthRegistryPublicDNS, nonAuthRegistry)
 
+	command := "bash -c '/tmp/non-auth-registry.sh " + terraformConfig.StandaloneRegistry.RegistryName + " " +
+		rke2NonAuthRegistryPublicDNS + " " + terraformConfig.Standalone.RancherTagVersion + " " +
+		terraformConfig.StandaloneRegistry.AssetsPath + " " + terraformConfig.Standalone.OSUser + " " +
+		terraformConfig.Standalone.RancherImage
+
+	if terraformConfig.Standalone.StagingRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.StagingRancherAgentImage
+	}
+
+	command += "'"
+
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
 		cty.StringVal("echo '" + string(registryScriptContent) + "' > /tmp/non-auth-registry.sh"),
 		cty.StringVal("chmod +x /tmp/non-auth-registry.sh"),
-		cty.StringVal("bash -c '/tmp/non-auth-registry.sh " + terraformConfig.StandaloneRegistry.RegistryName + " " +
-			rke2NonAuthRegistryPublicDNS + " " + terraformConfig.Standalone.RancherTagVersion + " " +
-			terraformConfig.StandaloneRegistry.AssetsPath + " " + terraformConfig.Standalone.OSUser + "'"),
+		cty.StringVal(command),
 	}))
 
 	_, err = file.Write(newFile.Bytes())

diff --git a/framework/set/resources/registries/non-auth-registry.sh b/framework/set/resources/registries/non-auth-registry.sh
@@ -5,6 +5,8 @@ HOST=$2
 RANCHER_VERSION=$3
 ASSET_DIR=$4
 USER=$5
+RANCHER_IMAGE=$6
+STAGING_RANCHER_AGENT_IMAGE=${7}
 
 HOST="${HOST}:5000"
 
@@ -26,6 +28,11 @@ sudo chmod +x /home/${USER}/rancher-save-images.sh && sudo chmod +x /home/${USER
 sudo sed -i "s/docker save/# docker save /g" /home/${USER}/rancher-save-images.sh
 sudo sed -i "s/docker load/# docker load /g" /home/${USER}/rancher-load-images.sh
 sudo sed -i '/mirrored-prometheus-windows-exporter/d' /home/${USER}/rancher-images.txt
+
+if [ ! -z "${STAGING_RANCHER_AGENT_IMAGE}" ]; then
+    sudo sed -i "s|rancher/rancher:|${RANCHER_IMAGE}:|g" /home/${USER}/rancher-images.txt
+    sudo sed -i "s|rancher/rancher-agent:|${STAGING_RANCHER_AGENT_IMAGE}:|g" /home/${USER}/rancher-images.txt
+fi
 
 echo "Saving the images..."
 sudo /home/${USER}/rancher-save-images.sh --image-list /home/${USER}/rancher-images.txt

diff --git a/framework/set/resources/sanity/rancher/createRancher.go b/framework/set/resources/sanity/rancher/createRancher.go
@@ -37,8 +37,8 @@ func CreateRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Bod
 		terraformConfig.Standalone.RancherHostname + " " + terraformConfig.Standalone.RancherTagVersion + " " +
 		terraformConfig.Standalone.BootstrapPassword
 
-	if terraformConfig.Standalone.StagingRancherImage != "" {
-		command += " " + terraformConfig.Standalone.StagingRancherImage + " " + terraformConfig.Standalone.StagingRancherAgentImage
+	if terraformConfig.Standalone.StagingRancherAgentImage != "" {
+		command += terraformConfig.Standalone.StagingRancherAgentImage
 	}
 
 	command += "'"

diff --git a/framework/set/resources/sanity/rancher/setup.sh b/framework/set/resources/sanity/rancher/setup.sh
@@ -7,7 +7,7 @@ CERT_MANAGER_VERSION=$4
 HOSTNAME=$5
 RANCHER_TAG_VERSION=$6
 BOOTSTRAP_PASSWORD=$7
-STAGING_RANCHER_IMAGE=${8}
+RANCHER_IMAGE=$8
 STAGING_RANCHER_AGENT_IMAGE=${9}
 
 set -ex
@@ -39,19 +39,21 @@ echo "Waiting 1 minute for Rancher"
 sleep 60
 
 echo "Installing Rancher"
-if [ -z "$STAGING_RANCHER_IMAGE" ]; then
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
     helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
                                                                                  --set hostname=${HOSTNAME} \
                                                                                  --set rancherImageTag=${RANCHER_TAG_VERSION} \
-                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD}
+                                                                                 --set rancherImage=${RANCHER_IMAGE} \
+                                                                                 --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \
+                                                                                 --set "extraEnv[0].value=${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \
+                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel
+
 else
     helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
                                                                                  --set hostname=${HOSTNAME} \
+                                                                                 --set rancherImage=${RANCHER_IMAGE} \
                                                                                  --set rancherImageTag=${RANCHER_TAG_VERSION} \
-                                                                                 --set rancherImage=${STAGING_RANCHER_IMAGE} \
-                                                                                 --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \
-                                                                                 --set "extraEnv[0].value=${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \
-                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel
+                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD}
 fi
 
 echo "Waiting for Rancher to be rolled out"

diff --git a/tests/airgap/README.md b/tests/airgap/README.md
@@ -84,10 +84,10 @@ terraform:
     rancherChartVersion: ""                       # REQUIRED - fill with desired value
     rancherChartRepository: ""                    # REQUIRED - fill with desired value. Must end with a trailing /
     rancherHostname: ""                           # REQUIRED - fill with desired value
+    rancherImage: ""                              # REQUIRED - fill with desired value
     rancherRepo: ""                               # REQUIRED - fill with desired value
     rancherTagVersion: ""                         # REQUIRED - fill with desired value
     type: ""                                      # REQUIRED - fill with desired value
-    stagingRancherImage: ""                       # OPTIONAL - fill out only if you are using staging registry
     stagingRancherAgentImage: ""                  # OPTIONAL - fill out only if you are using staging registry
     rke2Version: ""                               # REQUIRED - the format MUST be in `v1.xx.x` (i.e. v1.31.3)
   ####################################

diff --git a/tests/sanity/README.md b/tests/sanity/README.md
@@ -82,12 +82,12 @@ terraform:
     rancherChartVersion: ""                       # REQUIRED - fill with desired value
     rancherChartRepository: ""                    # REQUIRED - fill with desired value. Must end with a trailing /
     rancherHostname: ""                           # REQUIRED - fill with desired value
+    rancherImage: ""                              # REQUIRED - fill with desired value
     rancherRepo: ""                               # REQUIRED - fill with desired value
     rancherTagVersion: ""                         # REQUIRED - fill with desired value
     rke2Group: ""                                 # REQUIRED - fill with group of the instance created
     type: ""                                      # REQUIRED - fill with desired value
     rke2User: ""                                  # REQUIRED - fill with username of the instance created
-    stagingRancherImage: ""                       # OPTIONAL - fill out only if you are using staging registry
     stagingRancherAgentImage: ""                  # OPTIONAL - fill out only if you are using staging registry
     rke2Version: ""                               # REQUIRED - fill with desired RKE2 k8s value you wish the local cluster to be
 ```