aws · eks-distro-bot · Jan 8, 2025 · Jan 7, 2025
diff --git a/Makefile b/Makefile
@@ -216,7 +216,7 @@ generate-staging-buildspec: | ensure-locale
 	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "emissary-ingress_emissary" "$(BASE_DIRECTORY)/projects/emissary-ingress/emissary/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC"
 	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "goharbor_harbor" "$(BASE_DIRECTORY)/projects/goharbor/harbor/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC"
 	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_upgrader" "$(BASE_DIRECTORY)/projects/aws/upgrader/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC"
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/tools/version-tracker/buildspecs/upgrade.yml" "$(BASE_DIRECTORY)/buildspecs/upgrade-buildspec.yml" true EXCLUDE_FROM_UPGRADE_BUILDSPEC UPGRADE_BUILDSPECS false buildspecs/upgrade-eks-distro-buildspec.yml true
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/tools/version-tracker/buildspecs/upgrade.yml" "$(BASE_DIRECTORY)/buildspecs/upgrade-buildspec.yml" true EXCLUDE_FROM_UPGRADE_BUILDSPEC UPGRADE_BUILDSPECS false "buildspecs/upgrade-eks-distro-buildspec.yml,buildspecs/upgrade-eks-distro-build-tooling-buildspec.yml" true
 
 .PHONY: generate
 generate: generate-project-list generate-staging-buildspec

diff --git a/build/lib/generate_staging_buildspec.sh b/build/lib/generate_staging_buildspec.sh
@@ -25,7 +25,7 @@ SKIP_DEPEND_ON="${5:-false}"
 EXCLUDE_VAR="${6:-EXCLUDE_FROM_STAGING_BUILDSPEC}"
 BUILDSPECS_VAR="${7:-BUILDSPECS}"
 FAST_FAIL="${8:-true}"
-ADD_FINAL_STAGE="${9:-}"
+FINAL_STAGE_BUILDSPECS_CSV="${9:-}"
 NO_DEPS_FOR_FINAL_STAGE="${10:-false}"
 
 SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
@@ -85,7 +85,7 @@ for project in "${PROJECTS[@]}"; do
         if [[ $(realpath $buildspec) == $(realpath $DEFAULT_BUILDSPEC_FILE) ]]; then
             buildspec_field=""
         fi
-        
+
         DEPEND_ON=""
         # something other than empty string since some overrides are empty strings
         PROJECT_DEPENDENCIES="false"
@@ -107,17 +107,17 @@ for project in "${PROJECTS[@]}"; do
                 break
             fi
         done
-               
+
         PREVIOUS_SPEC_IDENTIFIERS=""
-        
+
         BUILDSPEC_NAME=$(basename $buildspec .yml)
         BUILDSPEC_IDENTIFIER_OVERRIDE="$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_IDENTIFIER_OVERRIDE)"
         if [[ -n "$BUILDSPEC_IDENTIFIER_OVERRIDE" ]]; then
             IDENTIFIER="$BUILDSPEC_IDENTIFIER_OVERRIDE"
         elif [[ "${BUILDSPEC_NAME}" != *buildspec* ]]; then
             IDENTIFIER+="_${BUILDSPEC_NAME//-/_}"
         fi
-       
+
         echo "Adding: $IDENTIFIER"
 
         if [ "$PROJECT_DEPENDENCIES" = "false" ]; then
@@ -146,7 +146,7 @@ for project in "${PROJECTS[@]}"; do
                 fi
 
                 DEP_IDENTIFIER=${DEP_ORG//-/_}_${DEP_REPO//-/_}
-                
+
                 if [ -n "${DEP_RELEASE_BRANCH}" ]; then
                     DEP_IDENTIFIER=${DEP_ORG//-/_}_${DEP_REPO//-/_}_${DEP_RELEASE_BRANCH//[-\/]/_}
                 fi
@@ -170,7 +170,7 @@ for project in "${PROJECTS[@]}"; do
         if [[ -z "$BUILDSPEC_VARS_KEYS" ]]; then
             BUILDSPEC_VARS_KEYS=$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_VARS_KEYS)
         fi
-        
+
         BUILDSPEC_PLATFORM=$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_PLATFORM)
         if [[ -z "$BUILDSPEC_PLATFORM" ]]; then
             BUILDSPEC_PLATFORM=$(make_var $PROJECT_PATH BUILDSPEC_PLATFORM)
@@ -192,13 +192,13 @@ for project in "${PROJECTS[@]}"; do
                     BUILDSPEC_VARS_KEYS="RELEASE_BRANCH"
                     BUILDSPEC_VARS_VALUES="SUPPORTED_K8S_VERSIONS"
                 fi
-            elif [[ "${BUILDSPEC_VARS_KEYS}" = "IMAGE_PLATFORMS" ]]; then            
+            elif [[ "${BUILDSPEC_VARS_KEYS}" = "IMAGE_PLATFORMS" ]]; then
                 BUILDSPEC_VARS_KEYS=""
             fi
         fi
 
         if [[ "$BUILDSPECS_VAR" == "UPGRADE_BUILDSPECS" ]]; then
-            if [[ "${IDENTIFIER}" = "kubernetes_sigs_image_builder" ]]; then            
+            if [[ "${IDENTIFIER}" = "kubernetes_sigs_image_builder" ]]; then
                 BUILDSPEC_VARS_KEYS=""
             elif [[ "true" == "$(make_var $PROJECT_PATH HAS_RELEASE_BRANCHES)" ]]; then
                 if [[ "false" == "$(make_var $PROJECT_PATH BINARIES_ARE_RELEASE_BRANCHED)" ]]; then
@@ -220,15 +220,15 @@ for project in "${PROJECTS[@]}"; do
                 fi
             fi
             VARS=(${BUILDSPEC_VARS_VALUES// / })
-            
-            # Note: only support 1 or 2 vars for now since that is all we need for kind + image-builder 
+
+            # Note: only support 1 or 2 vars for now since that is all we need for kind + image-builder
             if [ ${#VARS[@]} -eq 1 ]; then
                 VALUES_1=$(make_var $PROJECT_PATH ${VARS[0]})
                 ARR_1=(${VALUES_1// / })
-                
-                for val1 in "${ARR_1[@]}"; do                
+
+                for val1 in "${ARR_1[@]}"; do
                     IDENTIFIER_WITH_VAL=${IDENTIFIER}_${val1//[-\/]/_}
-                    
+
                     # If building on one binary platform assume we want to run on a specific arch instance
                     ARCH_TYPE="\"type\":\"$BUILDSPEC_PLATFORM\",\"compute-type\":\"$BUILDSPEC_COMPUTE_TYPE\","
                     if [ "${KEYS[0]}" = "BINARY_PLATFORMS" ] || [ "${KEYS[0]}" = "IMAGE_PLATFORMS" ]; then
@@ -239,7 +239,7 @@ for project in "${PROJECTS[@]}"; do
                         fi
                     fi
 
-                    EXTRA_VARS=""    
+                    EXTRA_VARS=""
                     if [ "${KEYS[0]}" = "IMAGE_PLATFORMS" ]; then
                         EXTRA_VARS+=",\"BINARY_PLATFORMS\":\"${val1}\",\"IMAGE_TAG_SUFFIX\":\"-${val1#linux/}\""
                         HAS_HELM_CHART=$(make_var $PROJECT_PATH HAS_HELM_CHART)
@@ -285,7 +285,7 @@ for project in "${PROJECTS[@]}"; do
                         PREVIOUS_SPEC_IDENTIFIERS+="$IDENTIFIER_WITH_VAL "
                         yq eval -i -P \
                             ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER_WITH_VAL\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\",\"${KEYS[1]}\":\"$val2\"$EXTRA_VARS}}}]" \
-                            $STAGING_BUILDSPEC_FILE 
+                            $STAGING_BUILDSPEC_FILE
                     done
                 done
             fi
@@ -294,21 +294,28 @@ for project in "${PROJECTS[@]}"; do
             PREVIOUS_SPEC_IDENTIFIERS+="$IDENTIFIER "
             yq eval -i -P \
                 ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL}}}]" \
-                $STAGING_BUILDSPEC_FILE 
+                $STAGING_BUILDSPEC_FILE
         fi
-
     done
 done
 
-if [ -n "${ADD_FINAL_STAGE}" ]; then
-    ARCH_TYPE="\"type\":\"ARM_CONTAINER\",\"compute-type\":\"BUILD_GENERAL1_SMALL\""
-    DEPEND_ON=",\"depend-on\":[${ALL_PROJECT_IDS%?}]"
-    if [ "$NO_DEPS_FOR_FINAL_STAGE" = "true" ]; then
-        DEPEND_ON=""
-    fi
-    yq eval -i -P \
-        ".batch.build-graph += [{\"identifier\":\"final_stage\",\"buildspec\":\"$ADD_FINAL_STAGE\",\"env\":{$ARCH_TYPE}$DEPEND_ON}]" \
-        $STAGING_BUILDSPEC_FILE 
+if [ -n "${FINAL_STAGE_BUILDSPECS_CSV}" ]; then
+    IFS=',' read -r -a FINAL_STAGE_BUILDSPECS <<< "$FINAL_STAGE_BUILDSPECS_CSV"
+    for i in "${!FINAL_STAGE_BUILDSPECS[@]}";
+        do
+            IDENTIFIER="final_stage"
+            if [[ "$i" -gt 0 ]]; then
+                IDENTIFIER="final_stage_$((i+1))"
+            fi
+            ARCH_TYPE="\"type\":\"ARM_CONTAINER\",\"compute-type\":\"BUILD_GENERAL1_SMALL\""
+            DEPEND_ON=",\"depend-on\":[${ALL_PROJECT_IDS%?}]"
+            if [ "$NO_DEPS_FOR_FINAL_STAGE" = "true" ]; then
+                DEPEND_ON=""
+            fi
+            yq eval -i -P \
+                ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",\"buildspec\":\"${FINAL_STAGE_BUILDSPECS[$i]}\",\"env\":{$ARCH_TYPE}$DEPEND_ON}]" \
+                $STAGING_BUILDSPEC_FILE
+        done
 fi
 
 HEAD_COMMENT=$(cat $BASE_DIRECTORY/hack/boilerplate.yq.txt)

diff --git a/buildspecs/upgrade-eks-distro-build-tooling-buildspec.yml b/buildspecs/upgrade-eks-distro-build-tooling-buildspec.yml
@@ -0,0 +1,14 @@
+version: 0.2
+
+env:
+  secrets-manager:
+    GITHUB_TOKEN: "github-eks-distro-pr-bot:github-token"
+
+phases:
+  pre_build:
+    commands:
+    - ./build/lib/setup.sh
+
+  build:
+    commands:
+    - make upgrade -C tools/version-tracker PROJECT=aws/eks-distro-build-tooling VERBOSITY=6
diff --git a/tools/version-tracker/buildspecs/upgrade.yml b/tools/version-tracker/buildspecs/upgrade.yml
@@ -497,6 +497,11 @@ batch:
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
+    - identifier: final_stage_2
+      buildspec: buildspecs/upgrade-eks-distro-build-tooling-buildspec.yml
+      env:
+        type: ARM_CONTAINER
+        compute-type: BUILD_GENERAL1_SMALL
 version: 0.2
 env:
   secrets-manager:

diff --git a/tools/version-tracker/pkg/commands/display/display.go b/tools/version-tracker/pkg/commands/display/display.go
@@ -44,7 +44,7 @@ func Run(displayOptions *types.DisplayOptions) error {
 	// Get base repository owner environment variable if set.
 	baseRepoOwner := os.Getenv(constants.BaseRepoOwnerEnvvar)
 	if baseRepoOwner == "" {
-		baseRepoOwner = constants.DefaultBaseRepoOwner
+		baseRepoOwner = constants.AWSOrgName
 	}
 
 	// Clone the eks-anywhere-build-tooling repository.

diff --git a/tools/version-tracker/pkg/commands/listprojects/listprojects.go b/tools/version-tracker/pkg/commands/listprojects/listprojects.go
@@ -30,7 +30,7 @@ func Run() error {
 	// Get base repository owner environment variable if set.
 	baseRepoOwner := os.Getenv(constants.BaseRepoOwnerEnvvar)
 	if baseRepoOwner == "" {
-		baseRepoOwner = constants.DefaultBaseRepoOwner
+		baseRepoOwner = constants.AWSOrgName
 	}
 
 	// Clone the eks-anywhere-build-tooling repository.

diff --git a/tools/version-tracker/pkg/commands/upgrade/upgrade.go b/tools/version-tracker/pkg/commands/upgrade/upgrade.go
@@ -146,6 +146,43 @@ func Run(upgradeOptions *types.UpgradeOptions) error {
 		if isUpdated {
 			updatedFiles = append(updatedFiles, constants.EKSDistroLatestReleasesFile)
 		}
+	} else if isEKSDistroBuildToolingUpgrade(projectName) {
+		headBranchName = fmt.Sprintf("update-eks-distro-base-image-tag-files-%s", branchName)
+		baseBranchName = branchName
+		commitMessage = "Bump EKS Distro base image tag files to latest"
+
+		// Checkout a new branch to keep track of version upgrade chaneges.
+		err = git.Checkout(worktree, headBranchName, true)
+		if err != nil {
+			return fmt.Errorf("checking out worktree at branch %s: %v", headBranchName, err)
+		}
+
+		// Reset current worktree to get a clean index.
+		err = git.ResetToHEAD(worktree, headCommit)
+		if err != nil {
+			return fmt.Errorf("resetting new branch to [origin/%s] HEAD: %v", branchName, err)
+		}
+
+		eksDistroBaseTagFilesGlobPattern := filepath.Join(buildToolingRepoPath, constants.EKSDistroBaseTagFilesPattern)
+		eksDistroBaseTagFilesGlob, err := filepath.Glob(eksDistroBaseTagFilesGlobPattern)
+		if err != nil {
+			return fmt.Errorf("finding filenames matching EKS Distro Base tag file pattern [%s]: %v", constants.EKSDistroBaseTagFilesPattern, err)
+		}
+
+		updatedPackages, isUpdated, err := updateEKSDistroBaseImageTagFiles(client, buildToolingRepoPath, eksDistroBaseTagFilesGlob)
+		if err != nil {
+			return fmt.Errorf("updating EKS Distro base tag files: %v", err)
+		}
+		if isUpdated {
+			pullRequestBody = fmt.Sprintf(constants.EKSDistroBuildToolingUpgradePullRequestBody, updatedPackages)
+			for _, tagFile := range eksDistroBaseTagFilesGlob {
+				tagFileRelativePath, err := filepath.Rel(buildToolingRepoPath, tagFile)
+				if err != nil {
+					return fmt.Errorf("getting relative path for tag file: %v", err)
+				}
+				updatedFiles = append(updatedFiles, tagFileRelativePath)
+			}
+		}
 	} else {
 		// Validate if the project name provided exists in the repository.
 		projectPath := filepath.Join("projects", projectName)
@@ -531,6 +568,56 @@ func updateEKSDistroReleasesFile(buildToolingRepoPath string) (bool, error) {
 	return isUpdated, nil
 }
 
+func updateEKSDistroBaseImageTagFiles(client *gogithub.Client, buildToolingRepoPath string, tagFileGlob []string) (string, bool, error) {
+	var updatedPackages string
+	var isUpdated bool
+
+	eksDistroBaseTagYAMLContents, err := github.GetFileContents(client, constants.AWSOrgName, constants.EKSDistroBuildToolingRepoName, constants.EKSDistroBaseTagsYAMLFile, "main")
+	if err != nil {
+		return "", false, fmt.Errorf("getting contents of EKS Distro Base tag file: %v", err)
+	}
+
+	var eksDistroBaseTagYAMLMap map[string]interface{}
+	err = yaml.Unmarshal(eksDistroBaseTagYAMLContents, &eksDistroBaseTagYAMLMap)
+	if err != nil {
+		return "", false, fmt.Errorf("unmarshalling EKS Distro Base tag file: %v", err)
+	}
+
+	for _, tagFile := range tagFileGlob {
+		tagFileContents, err := os.ReadFile(tagFile)
+		if err != nil {
+			return "", false, fmt.Errorf("reading tag file: %v", err)
+		}
+		tagFileName := filepath.Base(tagFile)
+		imageName := strings.TrimSuffix(tagFileName, constants.TagFileSuffix)
+
+		tagFileKey := strings.ReplaceAll(strings.ToLower(imageName), "_", "-")
+		osFolder := "2"
+		osKey := "al2"
+		if strings.HasSuffix(tagFileKey, "al2023") {
+			tagFileKey = strings.TrimSuffix(tagFileKey, constants.AL2023Suffix)
+			osFolder = "2023"
+			osKey = "al2023"
+		}
+		tagReleaseDate := eksDistroBaseTagYAMLMap[osKey].(map[string]interface{})[tagFileKey].(string)
+
+		if string(tagFileContents) != tagReleaseDate {
+			isUpdated = true
+			err = os.WriteFile(tagFile, []byte(fmt.Sprintf("%s\n", tagReleaseDate)), 0o644)
+			if err != nil {
+				return "", false, fmt.Errorf("writing tag file: %v", err)
+			}
+
+			updatedPackagesFilesContents, err := github.GetFileContents(client, constants.AWSOrgName, constants.EKSDistroBuildToolingRepoName, fmt.Sprintf(constants.EKSDistroBaseUpdatedPackagesFileFormat, osFolder, tagFileKey), "main")
+			if err != nil {
+				return "", false, fmt.Errorf("getting contents of EKS Distro Base image updated packages: %v", err)
+			}
+			updatedPackages = fmt.Sprintf("%s#### %s\nThe following yum packages were updated:\n```bash\n%s```\n\n", updatedPackages, imageName, string(updatedPackagesFilesContents))
+		}
+	}
+	return updatedPackages, isUpdated, nil
+}
+
 func getSupportedReleaseBranches(buildToolingRepoPath string) ([]string, error) {
 	supportedReleaseBranchesFilepath := filepath.Join(buildToolingRepoPath, constants.SupportedReleaseBranchesFile)
 
@@ -690,7 +777,7 @@ func applyPatchesToRepo(projectRootFilepath, projectRepo string, totalPatchCount
 		patchesApplied = totalPatchCount
 	} else {
 		failedFiles := []string{}
-		gitDescribeRegex := regexp.MustCompile(`v?\d+\.\d+\.\d+(-([0-9]+)-g.*)?`)
+		gitDescribeRegex := regexp.MustCompile(constants.GitDescribeRegex)
 		gitDescribeCmd := exec.Command("git", "-C", filepath.Join(projectRootFilepath, projectRepo), "describe", "--tag")
 		gitDescribeOutput, err := command.ExecCommand(gitDescribeCmd)
 		if err != nil {
@@ -890,16 +977,16 @@ func verifyBRImageExists(channel, format, bottlerocketVersion string) (bool, err
 	switch format {
 	case "ami":
 		variant = "aws"
-		imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.img.lz4", variant, kubeVersion, bottlerocketVersion)
+		imageTarget = fmt.Sprintf(constants.BottlerocketAMIImageTargetFormat, variant, kubeVersion, bottlerocketVersion)
 	case "ova":
 		variant = "vmware"
-		imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.ova", variant, kubeVersion, bottlerocketVersion)
+		imageTarget = fmt.Sprintf(constants.BottlerocketOVAImageTargetFormat, variant, kubeVersion, bottlerocketVersion)
 	case "raw":
 		variant = "metal"
-		imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.img.lz4", variant, kubeVersion, bottlerocketVersion)
+		imageTarget = fmt.Sprintf(constants.BottlerocketRawImageTargetFormat, variant, kubeVersion, bottlerocketVersion)
 	}
 
-	timestampURL := fmt.Sprintf("https://updates.bottlerocket.aws/2020-07-07/%s-k8s-%s/x86_64/timestamp.json", variant, kubeVersion)
+	timestampURL := fmt.Sprintf(constants.BottlerocketTimestampJSONURLFormat, variant, kubeVersion)
 	timestampManifest, err := file.ReadURL(timestampURL)
 	if err != nil {
 		return false, fmt.Errorf("reading Bottlerocket timestamp URL: %v", err)
@@ -914,7 +1001,7 @@ func verifyBRImageExists(channel, format, bottlerocketVersion string) (bool, err
 	version := timestampData.(map[string]interface{})["signed"].(map[string]interface{})["version"].(float64)
 	versionString := fmt.Sprintf("%.0f", version)
 
-	targetsURL := fmt.Sprintf("https://updates.bottlerocket.aws/2020-07-07/%s-k8s-%s/x86_64/%s.targets.json", variant, kubeVersion, versionString)
+	targetsURL := fmt.Sprintf(constants.BottlerocketTargetsJSONURLFormat, variant, kubeVersion, versionString)
 	targetsManifest, err := file.ReadURL(targetsURL)
 	if err != nil {
 		return false, fmt.Errorf("reading Bottlerocket targets URL: %v", err)
@@ -938,7 +1025,7 @@ func verifyBRImageExists(channel, format, bottlerocketVersion string) (bool, err
 
 func updateBottlerocketHostContainerMetadata(client *gogithub.Client, projectRootFilepath, projectPath, latestBottlerocketVersion string) ([]string, error) {
 	updatedHostContainerFiles := []string{}
-	hostContainersTOMLContents, err := github.GetFileContents(client, "bottlerocket-os", "bottlerocket", constants.BottlerocketHostContainersTOMLFile, latestBottlerocketVersion)
+	hostContainersTOMLContents, err := github.GetFileContents(client, constants.BottlerocketOrgName, constants.BottlerocketRepoName, constants.BottlerocketHostContainersTOMLFile, latestBottlerocketVersion)
 	if err != nil {
 		return nil, fmt.Errorf("getting contents of Bottlerocket host containers file: %v", err)
 	}
@@ -995,6 +1082,10 @@ func isEKSDistroUpgrade(projectName string) bool {
 	return projectName == "aws/eks-distro"
 }
 
+func isEKSDistroBuildToolingUpgrade(projectName string) bool {
+	return projectName == "aws/eks-distro-build-tooling"
+}
+
 func getDefaultReleaseBranch(buildToolingRepoPath string) (string, error) {
 	defaultReleaseBranchCommandSequence := fmt.Sprintf("make --no-print-directory -C %s get-default-release-branch", buildToolingRepoPath)
 	defaultReleaseBranchCmd := exec.Command("bash", "-c", defaultReleaseBranchCommandSequence)