Merge pull request #80 from markusewalker/markus/fix-staging-registry

Fix issue with rancher staging images not pulling in airgap environments
rancher · Jan 23, 2025 · 1947888 · 1947888
2 parents 4ac54c0 + 21837bf
commit 1947888
Show file tree

Hide file tree

Showing 16 changed files with 206 additions and 62 deletions.
diff --git a/config/config.go b/config/config.go
@@ -107,12 +107,12 @@ type Standalone struct {
 	RancherChartVersion      string `json:"rancherChartVersion,omitempty" yaml:"rancherChartVersion,omitempty"`
 	RancherChartRepository   string `json:"rancherChartRepository,omitempty" yaml:"rancherChartRepository,omitempty"`
 	RancherHostname          string `json:"rancherHostname,omitempty" yaml:"rancherHostname,omitempty"`
+	RancherImage             string `json:"rancherImage,omitempty" yaml:"rancherImage,omitempty"`
 	RancherRepo              string `json:"rancherRepo,omitempty" yaml:"rancherRepo,omitempty"`
 	RancherTagVersion        string `json:"rancherTagVersion,omitempty" yaml:"rancherTagVersion,omitempty"`
 	OSUser                   string `json:"osUser,omitempty" yaml:"osUser,omitempty"`
 	OSGroup                  string `json:"osGroup,omitempty" yaml:"osGroup,omitempty"`
 	RKE2Version              string `json:"rke2Version,omitempty" yaml:"rke2Version,omitempty"`
-	StagingRancherImage      string `json:"stagingRancherImage,omitempty" yaml:"stagingRancherImage,omitempty"`
 	StagingRancherAgentImage string `json:"stagingRancherAgentImage,omitempty" yaml:"stagingRancherAgentImage,omitempty"`
 	Type                     string `json:"type,omitempty" yaml:"type,omitempty"`
 }

diff --git a/framework/set/resources/airgap/createMainTF.go b/framework/set/resources/airgap/createMainTF.go
@@ -71,15 +71,15 @@ func CreateMainTF(t *testing.T, terraformOptions *terraform.Options, keyPath str
 	terraform.InitAndApply(t, terraformOptions)
 
 	file = OpenFile(file, keyPath)
-	file, err = rke2.CreateAirgapRKE2Cluster(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP)
+	file, err = rke2.CreateAirgapRKE2Cluster(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS, registryPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP)
 	if err != nil {
 		return "", err
 	}
 
 	terraform.InitAndApply(t, terraformOptions)
 
 	file = OpenFile(file, keyPath)
-	file, err = rancher.CreateAirgapRancher(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS)
+	file, err = rancher.CreateAirgapRancher(file, newFile, rootBody, terraformConfig, rke2BastionPublicDNS, registryPublicDNS)
 	if err != nil {
 		return "", err
 	}

diff --git a/framework/set/resources/airgap/rancher/setup.sh b/framework/set/resources/airgap/rancher/setup.sh
@@ -8,8 +8,9 @@ HOSTNAME=$5
 INTERNAL_FQDN=$6
 RANCHER_TAG_VERSION=$7
 BOOTSTRAP_PASSWORD=$8
-STAGING_RANCHER_IMAGE=${9}
+RANCHER_IMAGE=$9
 STAGING_RANCHER_AGENT_IMAGE=${10}
+REGISTRY=${11}
 
 set -ex
 
@@ -34,20 +35,21 @@ echo "Waiting 1 minute for Rancher"
 sleep 60
 
 echo "Installing Rancher"
-if [ -z "$STAGING_RANCHER_IMAGE" ]; then
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
     helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
                                                                                  --set hostname=${HOSTNAME} \
                                                                                  --set rancherImageTag=${RANCHER_TAG_VERSION} \
-                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD}
+                                                                                 --set rancherImage=${REGISTRY}/${RANCHER_IMAGE} \
+                                                                                 --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \
+                                                                                 --set "extraEnv[0].value=${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \
+                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel
 
 else
     helm upgrade --install rancher ${RANCHER_REPO}/rancher --namespace cattle-system --set global.cattle.psp.enabled=false \
                                                                                  --set hostname=${HOSTNAME} \
+                                                                                 --set rancherImage=${RANCHER_IMAGE} \
                                                                                  --set rancherImageTag=${RANCHER_TAG_VERSION} \
-                                                                                 --set rancherImage=${STAGING_RANCHER_IMAGE} \
-                                                                                 --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \                                                                                --set 'extraEnv[0].name=CATTLE_AGENT_IMAGE' \
-                                                                                 --set "extraEnv[0].value=${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}" \
-                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD} --devel
+                                                                                 --set bootstrapPassword=${BOOTSTRAP_PASSWORD}
 fi
 
 echo "Waiting for Rancher to be rolled out"

diff --git a/framework/set/resources/airgap/rancher/setupAirgapRancher.go b/framework/set/resources/airgap/rancher/setupAirgapRancher.go
@@ -18,7 +18,7 @@ const (
 
 // CreateAirgapRancher is a function that will set the airgap Rancher configurations in the main.tf file.
 func CreateAirgapRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig,
-	rke2BastionPublicDNS string) (*os.File, error) {
+	rke2BastionPublicDNS, registryPublicDNS string) (*os.File, error) {
 	userDir, err := os.UserHomeDir()
 	if err != nil {
 		return nil, err
@@ -36,10 +36,11 @@ func CreateAirgapRancher(file *os.File, newFile *hclwrite.File, rootBody *hclwri
 	command := "bash -c '/tmp/setup.sh " + terraformConfig.Standalone.RancherRepo + " " + terraformConfig.Standalone.RancherChartRepository + " " +
 		terraformConfig.Standalone.Type + " " + terraformConfig.Standalone.CertManagerVersion + " " +
 		terraformConfig.Standalone.RancherHostname + " " + " " + terraformConfig.Standalone.AirgapInternalFQDN + " " +
-		terraformConfig.Standalone.RancherTagVersion + " " + terraformConfig.Standalone.BootstrapPassword
+		terraformConfig.Standalone.RancherTagVersion + " " + terraformConfig.Standalone.BootstrapPassword + " " +
+		terraformConfig.Standalone.RancherImage
 
-	if terraformConfig.Standalone.StagingRancherImage != "" {
-		command += " " + terraformConfig.Standalone.StagingRancherImage + " " + terraformConfig.Standalone.StagingRancherAgentImage
+	if terraformConfig.Standalone.StagingRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.StagingRancherAgentImage + " " + registryPublicDNS
 	}
 
 	command += "'"

diff --git a/framework/set/resources/airgap/rke2/add-servers.sh b/framework/set/resources/airgap/rke2/add-servers.sh
@@ -6,8 +6,11 @@ RKE2_SERVER_ONE_IP=$3
 RKE2_NEW_SERVER_IP=$4
 RKE2_TOKEN=$5
 REGISTRY=$6
-REGISTRY_USERNAME=$7
-REGISTRY_PASSWORD=$8
+RANCHER_IMAGE=$7
+RANCHER_TAG_VERSION=$8
+REGISTRY_USERNAME=${9:-}
+REGISTRY_PASSWORD=${10:-}
+STAGING_RANCHER_AGENT_IMAGE=${11}
 PEM_FILE=/home/$USER/airgap.pem
 
 set -e
@@ -38,7 +41,9 @@ EOF
 
 setupRegistry() {
   sudo mkdir -p /etc/rancher/rke2
-  sudo tee /etc/rancher/rke2/registries.yaml > /dev/null << EOF
+
+  if [ -n "${REGISTRY_USERNAME}" ]; then
+    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
 mirrors:
   docker.io:
     endpoint:
@@ -50,6 +55,26 @@ configs:
       password: "${REGISTRY_PASSWORD}"
     tls:
       insecure_skip_verify: true
+EOF
+  else
+    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
+mirrors:
+  docker.io:
+    endpoint:
+      - "https://${REGISTRY}"
+configs:
+  "${REGISTRY}":
+    tls:
+      insecure_skip_verify: true
+EOF
+  fi
+}
+
+setupDockerDaemon() {
+  sudo tee -a /etc/docker/daemon.json > /dev/null << EOF
+{
+  "insecure-registries" : [ "${REGISTRY}" ]
+}
 EOF
 }
 
@@ -63,4 +88,18 @@ runSSH "${RKE2_NEW_SERVER_IP}" "sudo INSTALL_RKE2_ARTIFACT_PATH=/home/${USER} sh
 runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl enable rke2-server"
 runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl start rke2-server"
 
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+  setupDaemonFunction=$(declare -f setupDockerDaemon)
+  runSSH "${RKE2_NEW_SERVER_IP}" "${setupDaemonFunction}; setupDockerDaemon"
+  runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl restart docker && sudo systemctl daemon-reload"
+
+  if [ -n "$REGISTRY_USERNAME" ]; then
+    runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker login https://${REGISTRY} -u ${REGISTRY_USERNAME} -p ${REGISTRY_PASSWORD}"
+  fi
+
+  runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION}"
+  runSSH "${RKE2_NEW_SERVER_IP}" "sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
+  runSSH "${RKE2_NEW_SERVER_IP}" "sudo systemctl restart rke2-server"
+fi
+
 kubectl get nodes
diff --git a/framework/set/resources/airgap/rke2/createAirgapCluster.go b/framework/set/resources/airgap/rke2/createAirgapCluster.go
@@ -25,7 +25,7 @@ const (
 
 // CreateAirgapRKE2Cluster is a helper function that will create the RKE2 cluster.
 func CreateAirgapRKE2Cluster(file *os.File, newFile *hclwrite.File, rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig,
-	rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP string) (*os.File, error) {
+	rke2BastionPublicDNS, registryPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP string) (*os.File, error) {
 	userDir, err := os.UserHomeDir()
 	if err != nil {
 		return nil, err
@@ -68,8 +68,8 @@ func CreateAirgapRKE2Cluster(file *os.File, newFile *hclwrite.File, rootBody *hc
 
 	rke2Token := namegen.AppendRandomString(token)
 
-	createAirgappedRKE2Server(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2Token, serverOneScriptContent)
-	addAirgappedRKE2ServerNodes(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP, rke2Token, newServersScriptContent)
+	createAirgappedRKE2Server(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2Token, registryPublicDNS, serverOneScriptContent)
+	addAirgappedRKE2ServerNodes(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPrivateIP, rke2ServerThreePrivateIP, rke2Token, registryPublicDNS, newServersScriptContent)
 
 	_, err = file.Write(newFile.Bytes())
 	if err != nil {
@@ -109,15 +109,27 @@ func CreateNullResource(rootBody *hclwrite.Body, terraformConfig *config.Terrafo
 
 // createAirgappedRKE2Server is a helper function that will create the RKE2 server.
 func createAirgappedRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP,
-	rke2Token string, script []byte) {
+	rke2Token, registryPublicDNS string, script []byte) {
 	nullResourceBlockBody, provisionerBlockBody := CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, rke2ServerOne)
 
+	command := "bash -c '/tmp/init-server.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " +
+		rke2ServerOnePrivateIP + " " + rke2Token + " " + registryPublicDNS + " " + terraformConfig.Standalone.RancherImage + " " +
+		terraformConfig.Standalone.RancherTagVersion
+
+	if terraformConfig.PrivateRegistries.Username != "" {
+		command += " " + terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password
+	}
+
+	if terraformConfig.Standalone.StagingRancherAgentImage != "" {
+		command += " " + terraformConfig.Standalone.StagingRancherAgentImage
+	}
+
+	command += "'"
+
 	provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
 		cty.StringVal("printf '" + string(script) + "' > /tmp/init-server.sh"),
 		cty.StringVal("chmod +x /tmp/init-server.sh"),
-		cty.StringVal("bash -c '/tmp/init-server.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " +
-			rke2ServerOnePrivateIP + " " + rke2Token + " " + terraformConfig.PrivateRegistries.URL + " " +
-			terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password + "'"),
+		cty.StringVal(command),
 	}))
 
 	dependsOnServer := `[` + defaults.NullResource + `.` + rke2Bastion + `]`
@@ -130,20 +142,32 @@ func createAirgappedRKE2Server(rootBody *hclwrite.Body, terraformConfig *config.
 
 // addAirgappedRKE2ServerNodes is a helper function that will add additional RKE2 server nodes to the initial RKE2 airgapped server.
 func addAirgappedRKE2ServerNodes(rootBody *hclwrite.Body, terraformConfig *config.TerraformConfig, rke2BastionPublicDNS, rke2ServerOnePrivateIP, rke2ServerTwoPublicDNS,
-	rke2ServerThreePublicDNS, rke2Token string, script []byte) {
+	rke2ServerThreePublicDNS, rke2Token, registryPublicDNS string, script []byte) {
 	instances := []string{rke2ServerTwoPublicDNS, rke2ServerThreePublicDNS}
 	hosts := []string{rke2ServerTwo, rke2ServerThree}
 
 	for i, instance := range instances {
 		host := hosts[i]
 		nullResourceBlockBody, provisionerBlockBody := CreateNullResource(rootBody, terraformConfig, rke2BastionPublicDNS, host)
 
+		command := "bash -c '/tmp/add-servers.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " +
+			rke2ServerOnePrivateIP + " " + instance + " " + rke2Token + " " + registryPublicDNS + " " +
+			terraformConfig.Standalone.RancherImage + " " + terraformConfig.Standalone.RancherTagVersion
+
+		if terraformConfig.PrivateRegistries.Username != "" {
+			command += " " + terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password
+		}
+
+		if terraformConfig.Standalone.StagingRancherAgentImage != "" {
+			command += " " + terraformConfig.Standalone.StagingRancherAgentImage
+		}
+
+		command += "'"
+
 		provisionerBlockBody.SetAttributeValue(defaults.Inline, cty.ListVal([]cty.Value{
 			cty.StringVal("printf '" + string(script) + "' > /tmp/add-servers.sh"),
 			cty.StringVal("chmod +x /tmp/add-servers.sh"),
-			cty.StringVal("bash -c '/tmp/add-servers.sh " + terraformConfig.Standalone.OSUser + " " + terraformConfig.Standalone.OSGroup + " " +
-				rke2ServerOnePrivateIP + " " + instance + " " + rke2Token + " " + terraformConfig.PrivateRegistries.URL + " " +
-				terraformConfig.PrivateRegistries.Username + " " + terraformConfig.PrivateRegistries.Password + "'"),
+			cty.StringVal(command),
 		}))
 
 		dependsOnServer := `[` + defaults.NullResource + `.` + rke2ServerOne + `]`

diff --git a/framework/set/resources/airgap/rke2/init-server.sh b/framework/set/resources/airgap/rke2/init-server.sh
@@ -5,8 +5,11 @@ GROUP=$2
 RKE2_SERVER_ONE_IP=$3
 RKE2_TOKEN=$4
 REGISTRY=$5
-REGISTRY_USERNAME=$6
-REGISTRY_PASSWORD=$7
+RANCHER_IMAGE=$6
+RANCHER_TAG_VERSION=$7
+REGISTRY_USERNAME=${8:-}
+REGISTRY_PASSWORD=${9:-}
+STAGING_RANCHER_AGENT_IMAGE=${10}
 PEM_FILE=/home/$USER/airgap.pem
 
 set -e
@@ -36,7 +39,9 @@ EOF
 
 setupRegistry() {
   sudo mkdir -p /etc/rancher/rke2
-  sudo tee /etc/rancher/rke2/registries.yaml > /dev/null << EOF
+
+  if [ -n "${REGISTRY_USERNAME}" ]; then
+    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
 mirrors:
   docker.io:
     endpoint:
@@ -48,6 +53,26 @@ configs:
       password: "${REGISTRY_PASSWORD}"
     tls:
       insecure_skip_verify: true
+EOF
+  else
+    sudo tee -a /etc/rancher/rke2/registries.yaml > /dev/null << EOF
+mirrors:
+  docker.io:
+    endpoint:
+      - "https://${REGISTRY}"
+configs:
+  "${REGISTRY}":
+    tls:
+      insecure_skip_verify: true
+EOF
+  fi
+}
+
+setupDockerDaemon() {
+  sudo tee /etc/docker/daemon.json > /dev/null << EOF
+{
+  "insecure-registries" : [ "${REGISTRY}" ]
+}
 EOF
 }
 
@@ -63,6 +88,20 @@ runSSH "${RKE2_SERVER_ONE_IP}" "sudo INSTALL_RKE2_ARTIFACT_PATH=/home/${USER} sh
 runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl enable rke2-server"
 runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl start rke2-server"
 
+if [ -n "$STAGING_RANCHER_AGENT_IMAGE" ]; then
+  setupDaemonFunction=$(declare -f setupDockerDaemon)
+  runSSH "${RKE2_SERVER_ONE_IP}" "${setupDaemonFunction}; setupDockerDaemon"
+  runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl restart docker && sudo systemctl daemon-reload"
+
+  if [ -n "$REGISTRY_USERNAME" ]; then
+    runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker login https://${REGISTRY} -u ${REGISTRY_USERNAME} -p ${REGISTRY_PASSWORD}"
+  fi
+
+  runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker pull ${REGISTRY}/${RANCHER_IMAGE}:${RANCHER_TAG_VERSION}"
+  runSSH "${RKE2_SERVER_ONE_IP}" "sudo docker pull ${REGISTRY}/${STAGING_RANCHER_AGENT_IMAGE}:${RANCHER_TAG_VERSION}"
+  runSSH "${RKE2_SERVER_ONE_IP}" "sudo systemctl restart rke2-server"
+fi
+
 runSSH "${RKE2_SERVER_ONE_IP}" "sudo mkdir -p /home/${USER}/.kube"
 runSSH "${RKE2_SERVER_ONE_IP}" "sudo cp /etc/rancher/rke2/rke2.yaml /home/${USER}/.kube/config"
 runSSH "${RKE2_SERVER_ONE_IP}" "sudo chown -R ${USER}:${GROUP} /home/${USER}/.kube"

diff --git a/framework/set/resources/registries/auth-registry.sh b/framework/set/resources/registries/auth-registry.sh
@@ -7,6 +7,8 @@ HOST=$4
 RANCHER_VERSION=$5
 ASSET_DIR=$6
 USER=$7
+RANCHER_IMAGE=$8
+STAGING_RANCHER_AGENT_IMAGE=${9}
 
 set -e
 
@@ -51,6 +53,11 @@ sudo chmod +x /home/${USER}/rancher-save-images.sh && sudo chmod +x /home/${USER
 sudo sed -i "s/docker save/# docker save /g" /home/${USER}/rancher-save-images.sh
 sudo sed -i "s/docker load/# docker load /g" /home/${USER}/rancher-load-images.sh
 sudo sed -i '/mirrored-prometheus-windows-exporter/d' /home/${USER}/rancher-images.txt
+
+if [ ! -z "${STAGING_RANCHER_AGENT_IMAGE}" ]; then
+    sudo sed -i "s|rancher/rancher:|${RANCHER_IMAGE}:|g" /home/${USER}/rancher-images.txt
+    sudo sed -i "s|rancher/rancher-agent:|${STAGING_RANCHER_AGENT_IMAGE}:|g" /home/${USER}/rancher-images.txt
+fi
 
 echo "Saving the images..."
 sudo /home/${USER}/rancher-save-images.sh --image-list /home/${USER}/rancher-images.txt