🌱 test/framework: improve the way we inject the ci-artifacts script

test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh → test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl

@@ -22,6 +22,22 @@ set -o nounset
 set -o pipefail
 set -o errexit
 
+function retry {
+  attempt=0
+  max_attempts=$${1}
+  interval=$${2}
+  shift; shift
+  until [[ $${attempt} -ge "$${max_attempts}" ]] ; do
+    attempt=$((attempt+1))
+    set +e
+    eval "$*" && return || echo "failed $${attempt} times: $*"
+    set -e
+    sleep "$${interval}"
+  done
+  echo "error: reached max attempts at retry($*)"
+  return 1
+}
+
 [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
 
 USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false}
@@ -31,19 +47,6 @@ if [ ! "${USE_CI_ARTIFACTS}" = true ]; then
   exit 0
 fi
 
-GSUTIL=gsutil
-
-if ! command -v $${GSUTIL} >/dev/null; then
-  apt-get update
-  apt-get install -y apt-transport-https ca-certificates gnupg curl
-  echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
-  curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
-  apt-get update
-  apt-get install -y google-cloud-sdk
-fi
-
-$${GSUTIL} version
-
 # This test installs release packages or binaries that are a result of the CI and release builds.
 # It runs '... --version' commands to verify that the binaries are correctly installed
 # and finally uninstalls the packages.
@@ -62,14 +65,21 @@ if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
   CI_DIR=/tmp/k8s-ci
   mkdir -p "$${CI_DIR}"
   declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
+  {{- if .IsControlPlaneMachine }}
   declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
+  {{- else }}
+  declare -a CONTAINERS_TO_TEST=("kube-proxy")
+  {{- end }}
   CONTAINER_EXT="tar"
   echo "* testing CI version $${KUBERNETES_VERSION}"
   # Check for semver
   if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-    CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
+    CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
     VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
-    DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+    export DEBIAN_FRONTEND=noninteractive
+    # sometimes the network is not immediately available, so we have to retry the apt-get update
+    retry 10 5 "apt-get update"
+    apt-get install -y apt-transport-https ca-certificates curl
     curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
     echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list
     apt-get update
@@ -78,24 +88,28 @@ if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
     PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
     for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
       echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
-      DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
+      apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
     done
   else
-    CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
+    CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
     for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+      # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
+      # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
       echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
-      $${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}"
+      wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
       chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
       mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
     done
     systemctl restart kubelet
   fi
   for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
+    # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release
+    # e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar
     echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
-    $${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+    wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
     $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
     $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
-    $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+    $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
   done
 fi
 echo "* checking binary versions"

test/framework/kubernetesversions/data/kustomization.yaml

@@ -2,7 +2,19 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: default
 resources:
-  - ci-artifacts-source-template.yaml
+- ci-artifacts-source-template.yaml
 patchesStrategicMerge:
-  - kustomizeversions.yaml
-  - platform-kustomization.yaml
+- platform-kustomization.yaml
+patchesJson6902:
+- path: kubeadmcontrolplane-patch.yaml
+  target:
+    group: controlplane.cluster.x-k8s.io
+    kind: KubeadmControlPlane
+    name: ".*-control-plane"
+    version: v1alpha4
+- path: kubeadmconfigtemplate-patch.yaml
+  target:
+    group: bootstrap.cluster.x-k8s.io
+    kind: KubeadmConfigTemplate
+    name: ".*-md-0"
+    version: v1alpha4

test/framework/kubernetesversions/template.go

@@ -18,27 +18,30 @@ limitations under the License.
 package kubernetesversions
 
 import (
+	"bytes"
 	_ "embed"
 	"errors"
+	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
+	"strings"
+	"text/template"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4"
-	kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/yaml"
 )
 
-const yamlSeparator = "\n---\n"
-
 var (
-	//go:embed data/kustomization.yaml
-	kustomizationYamlBytes []byte
-
-	//go:embed data/debian_injection_script.envsubst.sh
+	//go:embed data/debian_injection_script.envsubst.sh.tpl
 	debianInjectionScriptBytes string
+
+	debianInjectionScriptTemplate = template.Must(template.New("").Parse(debianInjectionScriptBytes))
+
+	//go:embed data/kustomization.yaml
+	kustomizationYAMLBytes string
 )
 
 type GenerateCIArtifactsInjectedTemplateForDebianInput struct {
@@ -88,18 +91,34 @@ func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjec
 
 	kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml")
 
-	if err := os.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomizationYamlBytes, 0o600); err != nil {
+	if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), []byte(kustomizationYAMLBytes), 0o600); err != nil {
 		return "", err
 	}
 
-	kustomizeVersions, err := generateKustomizeVersionsYaml(input.KubeadmControlPlaneName, input.KubeadmConfigTemplateName, input.KubeadmConfigName)
+	var debianInjectionScriptControlPlaneBytes bytes.Buffer
+	if err := debianInjectionScriptTemplate.Execute(&debianInjectionScriptControlPlaneBytes, map[string]bool{"IsControlPlaneMachine": true}); err != nil {
+		return "", err
+	}
+	patch, err := generateInjectScriptJSONPatch(input.SourceTemplate, "KubeadmControlPlane", input.KubeadmControlPlaneName, "/spec/kubeadmConfigSpec", "/usr/local/bin/ci-artifacts.sh", debianInjectionScriptControlPlaneBytes.String())
 	if err != nil {
 		return "", err
 	}
+	if err := os.WriteFile(path.Join(overlayDir, "kubeadmcontrolplane-patch.yaml"), patch, 0o600); err != nil {
+		return "", err
+	}
 
-	if err := os.WriteFile(path.Join(overlayDir, "kustomizeversions.yaml"), kustomizeVersions, 0o600); err != nil {
+	var debianInjectionScriptWorkerBytes bytes.Buffer
+	if err := debianInjectionScriptTemplate.Execute(&debianInjectionScriptWorkerBytes, map[string]bool{"IsControlPlaneMachine": false}); err != nil {
 		return "", err
 	}
+	patch, err = generateInjectScriptJSONPatch(input.SourceTemplate, "KubeadmConfigTemplate", input.KubeadmConfigTemplateName, "/spec/template/spec", "/usr/local/bin/ci-artifacts.sh", debianInjectionScriptWorkerBytes.String())
+	if err != nil {
+		return "", err
+	}
+	if err := os.WriteFile(path.Join(overlayDir, "kubeadmconfigtemplate-patch.yaml"), patch, 0o600); err != nil {
+		return "", err
+	}
+
 	if err := os.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil {
 		return "", err
 	}
@@ -117,91 +136,89 @@ func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjec
 	return kustomizedTemplate, nil
 }
 
-func generateKustomizeVersionsYaml(kcpName, kubeadmTemplateName, kubeadmConfigName string) ([]byte, error) {
-	kcp := generateKubeadmControlPlane(kcpName)
-	kubeadm := generateKubeadmConfigTemplate(kubeadmTemplateName)
-	kcpYaml, err := yaml.Marshal(kcp)
-	if err != nil {
-		return nil, err
-	}
-	kubeadmYaml, err := yaml.Marshal(kubeadm)
-	if err != nil {
-		return nil, err
-	}
-	fileStr := string(kcpYaml) + yamlSeparator + string(kubeadmYaml)
-	if kubeadmConfigName == "" {
-		return []byte(fileStr), nil
-	}
+type jsonPatch struct {
+	Op    string      `json:"op"`
+	Path  string      `json:"path"`
+	Value interface{} `json:"value"`
+}
 
-	kubeadmConfig := generateKubeadmConfig(kubeadmConfigName)
-	kubeadmConfigYaml, err := yaml.Marshal(kubeadmConfig)
+// generateInjectScriptJSONPatch generates a JSON patch which injects a script
+// * objectKind: is the kind of the object we want to inject the script into
+// * objectName: is the name of the object we want to inject the script into
+// * jsonPatchPathPrefix: is the prefix of the 'files' and `preKubeadmCommands` arrays where we append the script
+// * scriptPath: is the path where the script will be stored at
+// * scriptContent: content of the script.
+func generateInjectScriptJSONPatch(sourceTemplate []byte, objectKind, objectName, jsonPatchPathPrefix, scriptPath, scriptContent string) ([]byte, error) {
+	filesPathExists, preKubeadmCommandsPathExists, err := checkIfArraysAlreadyExist(sourceTemplate, objectKind, objectName, jsonPatchPathPrefix)
 	if err != nil {
 		return nil, err
 	}
-	fileStr = fileStr + yamlSeparator + string(kubeadmConfigYaml)
 
-	return []byte(fileStr), nil
-}
-
-func generateKubeadmConfigTemplate(name string) *cabpkv1.KubeadmConfigTemplate {
-	kubeadmSpec := generateKubeadmConfigSpec()
-	return &cabpkv1.KubeadmConfigTemplate{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "KubeadmConfigTemplate",
-			APIVersion: cabpkv1.GroupVersion.String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
+	var patches []jsonPatch
+	if !filesPathExists {
+		patches = append(patches, jsonPatch{
+			Op:    "add",
+			Path:  fmt.Sprintf("%s/files", jsonPatchPathPrefix),
+			Value: []interface{}{},
+		})
+	}
+	patches = append(patches, jsonPatch{
+		Op:   "add",
+		Path: fmt.Sprintf("%s/files/-", jsonPatchPathPrefix),
+		Value: map[string]string{
+			"content":     scriptContent,
+			"owner":       "root:root",
+			"path":        scriptPath,
+			"permissions": "0750",
 		},
-		Spec: cabpkv1.KubeadmConfigTemplateSpec{
-			Template: cabpkv1.KubeadmConfigTemplateResource{
-				Spec: *kubeadmSpec,
-			},
-		},
-	}
+	})
+	if !preKubeadmCommandsPathExists {
+		patches = append(patches, jsonPatch{
+			Op:    "add",
+			Path:  fmt.Sprintf("%s/preKubeadmCommands", jsonPatchPathPrefix),
+			Value: []string{},
+		})
+	}
+	patches = append(patches, jsonPatch{
+		Op:    "add",
+		Path:  fmt.Sprintf("%s/preKubeadmCommands/-", jsonPatchPathPrefix),
+		Value: scriptPath,
+	})
+
+	return yaml.Marshal(patches)
 }
 
-func generateKubeadmConfig(name string) *cabpkv1.KubeadmConfig {
-	kubeadmSpec := generateKubeadmConfigSpec()
-	return &cabpkv1.KubeadmConfig{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "KubeadmConfig",
-			APIVersion: kcpv1.GroupVersion.String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-		},
-		Spec: *kubeadmSpec,
-	}
-}
-
-func generateKubeadmControlPlane(name string) *kcpv1.KubeadmControlPlane {
-	kubeadmSpec := generateKubeadmConfigSpec()
-	return &kcpv1.KubeadmControlPlane{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "KubeadmControlPlane",
-			APIVersion: kcpv1.GroupVersion.String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-		},
-		Spec: kcpv1.KubeadmControlPlaneSpec{
-			KubeadmConfigSpec: *kubeadmSpec,
-			Version:           "${KUBERNETES_VERSION}",
-		},
-	}
-}
-
-func generateKubeadmConfigSpec() *cabpkv1.KubeadmConfigSpec {
-	return &cabpkv1.KubeadmConfigSpec{
-		Files: []cabpkv1.File{
-			{
-				Path:        "/usr/local/bin/ci-artifacts.sh",
-				Content:     debianInjectionScriptBytes,
-				Owner:       "root:root",
-				Permissions: "0750",
-			},
-		},
-		PreKubeadmCommands: []string{"/usr/local/bin/ci-artifacts.sh"},
-	}
+// checkIfArraysAlreadyExist check is the 'files' and 'preKubeadmCommands' arrays already exist below jsonPatchPathPrefix.
+func checkIfArraysAlreadyExist(sourceTemplate []byte, objectKind, objectName, jsonPatchPathPrefix string) (bool, bool, error) {
+	yamlDocs := strings.Split(string(sourceTemplate), "---")
+	for _, yamlDoc := range yamlDocs {
+		if yamlDoc == "" {
+			continue
+		}
+		var obj unstructured.Unstructured
+		if err := yaml.Unmarshal([]byte(yamlDoc), &obj); err != nil {
+			return false, false, err
+		}
+
+		if obj.GetKind() != objectKind {
+			continue
+		}
+		if obj.GetName() != objectName {
+			continue
+		}
+
+		pathSplit := strings.Split(strings.TrimPrefix(jsonPatchPathPrefix, "/"), "/")
+		filesPath := append(pathSplit, "files")
+		preKubeadmCommandsPath := append(pathSplit, "preKubeadmCommands")
+		_, filesPathExists, err := unstructured.NestedFieldCopy(obj.Object, filesPath...)
+		if err != nil {
+			return false, false, err
+		}
+		_, preKubeadmCommandsPathExists, err := unstructured.NestedFieldCopy(obj.Object, preKubeadmCommandsPath...)
+		if err != nil {
+			return false, false, err
+		}
+		return filesPathExists, preKubeadmCommandsPathExists, nil
+	}
+	return false, false, fmt.Errorf("could not find document with kind %q and name %q", objectKind, objectName)
 }