diff --git a/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh b/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl similarity index 67% rename from test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh rename to test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl index 4ef2ee123729..5dcd1c5fe21c 100644 --- a/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh +++ b/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl @@ -22,6 +22,22 @@ set -o nounset set -o pipefail set -o errexit +function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 +} + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" USE_CI_ARTIFACTS=${USE_CI_ARTIFACTS:=false} @@ -31,19 +47,6 @@ if [ ! "${USE_CI_ARTIFACTS}" = true ]; then exit 0 fi -GSUTIL=gsutil - -if ! command -v $${GSUTIL} >/dev/null; then - apt-get update - apt-get install -y apt-transport-https ca-certificates gnupg curl - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | $${SUDO} tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | $${SUDO} apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - - apt-get update - apt-get install -y google-cloud-sdk -fi - -$${GSUTIL} version - # This test installs release packages or binaries that are a result of the CI and release builds. # It runs '... --version' commands to verify that the binaries are correctly installed # and finally uninstalls the packages. @@ -62,14 +65,21 @@ if [[ "$${KUBERNETES_VERSION}" != "" ]]; then CI_DIR=/tmp/k8s-ci mkdir -p "$${CI_DIR}" declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + {{- if .IsControlPlaneMachine }} declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + {{- else }} + declare -a CONTAINERS_TO_TEST=("kube-proxy") + {{- end }} CONTAINER_EXT="tar" echo "* testing CI version $${KUBERNETES_VERSION}" # Check for semver if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - CI_URL="gs://kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + CI_URL="https://storage.googleapis.com/kubernetes-release/release/$${KUBERNETES_VERSION}/bin/linux/amd64" VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" - DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - echo 'deb https://apt.kubernetes.io/ kubernetes-xenial main' >/etc/apt/sources.list.d/kubernetes.list apt-get update @@ -78,24 +88,28 @@ if [[ "$${KUBERNETES_VERSION}" != "" ]]; then PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" - DEBIAN_FRONTEND=noninteractive apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-get install -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" done else - CI_URL="gs://kubernetes-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" - $${GSUTIL} cp "$${CI_URL}/$${CI_PACKAGE}" "$${CI_DIR}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" chmod +x "$${CI_DIR}/$${CI_PACKAGE}" mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" done systemctl restart kubelet fi for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/kubernetes-release?project=kubernetes-release + # e.g.: https://storage.googleapis.com/kubernetes-release/release/v1.20.4/bin/linux/amd64/kube-apiserver.tar echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" - $${GSUTIL} cp "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "k8s.gcr.io/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" - $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/kubernetes-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "k8s.gcr.io/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" done fi echo "* checking binary versions" diff --git a/test/framework/kubernetesversions/data/kustomization.yaml b/test/framework/kubernetesversions/data/kustomization.yaml index e5db20f339ab..65b2222b012d 100644 --- a/test/framework/kubernetesversions/data/kustomization.yaml +++ b/test/framework/kubernetesversions/data/kustomization.yaml @@ -2,7 +2,19 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ci-artifacts-source-template.yaml +- ci-artifacts-source-template.yaml patchesStrategicMerge: - - kustomizeversions.yaml - - platform-kustomization.yaml +- platform-kustomization.yaml +patchesJson6902: +- path: kubeadmcontrolplane-patch.yaml + target: + group: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: ".*-control-plane" + version: v1alpha4 +- path: kubeadmconfigtemplate-patch.yaml + target: + group: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + name: ".*-md-0" + version: v1alpha4 diff --git a/test/framework/kubernetesversions/template.go b/test/framework/kubernetesversions/template.go index 3c6bdf6e5f88..2ac581e7b75d 100644 --- a/test/framework/kubernetesversions/template.go +++ b/test/framework/kubernetesversions/template.go @@ -18,27 +18,30 @@ limitations under the License. package kubernetesversions import ( + "bytes" _ "embed" "errors" + "fmt" + "io/ioutil" "os" "os/exec" "path" + "strings" + "text/template" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4" - kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/yaml" ) -const yamlSeparator = "\n---\n" - var ( - //go:embed data/kustomization.yaml - kustomizationYamlBytes []byte - - //go:embed data/debian_injection_script.envsubst.sh + //go:embed data/debian_injection_script.envsubst.sh.tpl debianInjectionScriptBytes string + + debianInjectionScriptTemplate = template.Must(template.New("").Parse(debianInjectionScriptBytes)) + + //go:embed data/kustomization.yaml + kustomizationYAMLBytes string ) type GenerateCIArtifactsInjectedTemplateForDebianInput struct { @@ -88,18 +91,34 @@ func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjec kustomizedTemplate := path.Join(templateDir, "cluster-template-conformance-ci-artifacts.yaml") - if err := os.WriteFile(path.Join(overlayDir, "kustomization.yaml"), kustomizationYamlBytes, 0o600); err != nil { + if err := ioutil.WriteFile(path.Join(overlayDir, "kustomization.yaml"), []byte(kustomizationYAMLBytes), 0o600); err != nil { return "", err } - kustomizeVersions, err := generateKustomizeVersionsYaml(input.KubeadmControlPlaneName, input.KubeadmConfigTemplateName, input.KubeadmConfigName) + var debianInjectionScriptControlPlaneBytes bytes.Buffer + if err := debianInjectionScriptTemplate.Execute(&debianInjectionScriptControlPlaneBytes, map[string]bool{"IsControlPlaneMachine": true}); err != nil { + return "", err + } + patch, err := generateInjectScriptJSONPatch(input.SourceTemplate, "KubeadmControlPlane", input.KubeadmControlPlaneName, "/spec/kubeadmConfigSpec", "/usr/local/bin/ci-artifacts.sh", debianInjectionScriptControlPlaneBytes.String()) if err != nil { return "", err } + if err := os.WriteFile(path.Join(overlayDir, "kubeadmcontrolplane-patch.yaml"), patch, 0o600); err != nil { + return "", err + } - if err := os.WriteFile(path.Join(overlayDir, "kustomizeversions.yaml"), kustomizeVersions, 0o600); err != nil { + var debianInjectionScriptWorkerBytes bytes.Buffer + if err := debianInjectionScriptTemplate.Execute(&debianInjectionScriptWorkerBytes, map[string]bool{"IsControlPlaneMachine": false}); err != nil { return "", err } + patch, err = generateInjectScriptJSONPatch(input.SourceTemplate, "KubeadmConfigTemplate", input.KubeadmConfigTemplateName, "/spec/template/spec", "/usr/local/bin/ci-artifacts.sh", debianInjectionScriptWorkerBytes.String()) + if err != nil { + return "", err + } + if err := os.WriteFile(path.Join(overlayDir, "kubeadmconfigtemplate-patch.yaml"), patch, 0o600); err != nil { + return "", err + } + if err := os.WriteFile(path.Join(overlayDir, "ci-artifacts-source-template.yaml"), input.SourceTemplate, 0o600); err != nil { return "", err } @@ -117,91 +136,89 @@ func GenerateCIArtifactsInjectedTemplateForDebian(input GenerateCIArtifactsInjec return kustomizedTemplate, nil } -func generateKustomizeVersionsYaml(kcpName, kubeadmTemplateName, kubeadmConfigName string) ([]byte, error) { - kcp := generateKubeadmControlPlane(kcpName) - kubeadm := generateKubeadmConfigTemplate(kubeadmTemplateName) - kcpYaml, err := yaml.Marshal(kcp) - if err != nil { - return nil, err - } - kubeadmYaml, err := yaml.Marshal(kubeadm) - if err != nil { - return nil, err - } - fileStr := string(kcpYaml) + yamlSeparator + string(kubeadmYaml) - if kubeadmConfigName == "" { - return []byte(fileStr), nil - } +type jsonPatch struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value"` +} - kubeadmConfig := generateKubeadmConfig(kubeadmConfigName) - kubeadmConfigYaml, err := yaml.Marshal(kubeadmConfig) +// generateInjectScriptJSONPatch generates a JSON patch which injects a script +// * objectKind: is the kind of the object we want to inject the script into +// * objectName: is the name of the object we want to inject the script into +// * jsonPatchPathPrefix: is the prefix of the 'files' and `preKubeadmCommands` arrays where we append the script +// * scriptPath: is the path where the script will be stored at +// * scriptContent: content of the script. +func generateInjectScriptJSONPatch(sourceTemplate []byte, objectKind, objectName, jsonPatchPathPrefix, scriptPath, scriptContent string) ([]byte, error) { + filesPathExists, preKubeadmCommandsPathExists, err := checkIfArraysAlreadyExist(sourceTemplate, objectKind, objectName, jsonPatchPathPrefix) if err != nil { return nil, err } - fileStr = fileStr + yamlSeparator + string(kubeadmConfigYaml) - return []byte(fileStr), nil -} - -func generateKubeadmConfigTemplate(name string) *cabpkv1.KubeadmConfigTemplate { - kubeadmSpec := generateKubeadmConfigSpec() - return &cabpkv1.KubeadmConfigTemplate{ - TypeMeta: metav1.TypeMeta{ - Kind: "KubeadmConfigTemplate", - APIVersion: cabpkv1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, + var patches []jsonPatch + if !filesPathExists { + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/files", jsonPatchPathPrefix), + Value: []interface{}{}, + }) + } + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/files/-", jsonPatchPathPrefix), + Value: map[string]string{ + "content": scriptContent, + "owner": "root:root", + "path": scriptPath, + "permissions": "0750", }, - Spec: cabpkv1.KubeadmConfigTemplateSpec{ - Template: cabpkv1.KubeadmConfigTemplateResource{ - Spec: *kubeadmSpec, - }, - }, - } + }) + if !preKubeadmCommandsPathExists { + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/preKubeadmCommands", jsonPatchPathPrefix), + Value: []string{}, + }) + } + patches = append(patches, jsonPatch{ + Op: "add", + Path: fmt.Sprintf("%s/preKubeadmCommands/-", jsonPatchPathPrefix), + Value: scriptPath, + }) + + return yaml.Marshal(patches) } -func generateKubeadmConfig(name string) *cabpkv1.KubeadmConfig { - kubeadmSpec := generateKubeadmConfigSpec() - return &cabpkv1.KubeadmConfig{ - TypeMeta: metav1.TypeMeta{ - Kind: "KubeadmConfig", - APIVersion: kcpv1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: *kubeadmSpec, - } -} - -func generateKubeadmControlPlane(name string) *kcpv1.KubeadmControlPlane { - kubeadmSpec := generateKubeadmConfigSpec() - return &kcpv1.KubeadmControlPlane{ - TypeMeta: metav1.TypeMeta{ - Kind: "KubeadmControlPlane", - APIVersion: kcpv1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: kcpv1.KubeadmControlPlaneSpec{ - KubeadmConfigSpec: *kubeadmSpec, - Version: "${KUBERNETES_VERSION}", - }, - } -} - -func generateKubeadmConfigSpec() *cabpkv1.KubeadmConfigSpec { - return &cabpkv1.KubeadmConfigSpec{ - Files: []cabpkv1.File{ - { - Path: "/usr/local/bin/ci-artifacts.sh", - Content: debianInjectionScriptBytes, - Owner: "root:root", - Permissions: "0750", - }, - }, - PreKubeadmCommands: []string{"/usr/local/bin/ci-artifacts.sh"}, - } +// checkIfArraysAlreadyExist check is the 'files' and 'preKubeadmCommands' arrays already exist below jsonPatchPathPrefix. +func checkIfArraysAlreadyExist(sourceTemplate []byte, objectKind, objectName, jsonPatchPathPrefix string) (bool, bool, error) { + yamlDocs := strings.Split(string(sourceTemplate), "---") + for _, yamlDoc := range yamlDocs { + if yamlDoc == "" { + continue + } + var obj unstructured.Unstructured + if err := yaml.Unmarshal([]byte(yamlDoc), &obj); err != nil { + return false, false, err + } + + if obj.GetKind() != objectKind { + continue + } + if obj.GetName() != objectName { + continue + } + + pathSplit := strings.Split(strings.TrimPrefix(jsonPatchPathPrefix, "/"), "/") + filesPath := append(pathSplit, "files") + preKubeadmCommandsPath := append(pathSplit, "preKubeadmCommands") + _, filesPathExists, err := unstructured.NestedFieldCopy(obj.Object, filesPath...) + if err != nil { + return false, false, err + } + _, preKubeadmCommandsPathExists, err := unstructured.NestedFieldCopy(obj.Object, preKubeadmCommandsPath...) + if err != nil { + return false, false, err + } + return filesPathExists, preKubeadmCommandsPathExists, nil + } + return false, false, fmt.Errorf("could not find document with kind %q and name %q", objectKind, objectName) }