Skip to content

Commit

Permalink
Add EKS Distro base tag files to version-tracker automation
Browse files Browse the repository at this point in the history
  • Loading branch information
@abhay-krishna
abhay-krishna committed Jan 8, 2025
1 parent 61c802d commit afa6776
Show file tree
Hide file tree
Showing 8 changed files with 177 additions and 38 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ generate-staging-buildspec: | ensure-locale
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "emissary-ingress_emissary" "$(BASE_DIRECTORY)/projects/emissary-ingress/emissary/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC"
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "goharbor_harbor" "$(BASE_DIRECTORY)/projects/goharbor/harbor/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC"
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_upgrader" "$(BASE_DIRECTORY)/projects/aws/upgrader/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true "DO_NOT_EXCLUDE_FROM_BUILDSPEC"
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/tools/version-tracker/buildspecs/upgrade.yml" "$(BASE_DIRECTORY)/buildspecs/upgrade-buildspec.yml" true EXCLUDE_FROM_UPGRADE_BUILDSPEC UPGRADE_BUILDSPECS false buildspecs/upgrade-eks-distro-buildspec.yml true
build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/tools/version-tracker/buildspecs/upgrade.yml" "$(BASE_DIRECTORY)/buildspecs/upgrade-buildspec.yml" true EXCLUDE_FROM_UPGRADE_BUILDSPEC UPGRADE_BUILDSPECS false "buildspecs/upgrade-eks-distro-buildspec.yml,buildspecs/upgrade-eks-distro-build-tooling-buildspec.yml" true

.PHONY: generate
generate: generate-project-list generate-staging-buildspec
Expand Down
61 changes: 34 additions & 27 deletions build/lib/generate_staging_buildspec.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ SKIP_DEPEND_ON="${5:-false}"
EXCLUDE_VAR="${6:-EXCLUDE_FROM_STAGING_BUILDSPEC}"
BUILDSPECS_VAR="${7:-BUILDSPECS}"
FAST_FAIL="${8:-true}"
ADD_FINAL_STAGE="${9:-}"
FINAL_STAGE_BUILDSPECS_CSV="${9:-}"
NO_DEPS_FOR_FINAL_STAGE="${10:-false}"

SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
Expand Down Expand Up @@ -85,7 +85,7 @@ for project in "${PROJECTS[@]}"; do
if [[ $(realpath $buildspec) == $(realpath $DEFAULT_BUILDSPEC_FILE) ]]; then
buildspec_field=""
fi

DEPEND_ON=""
# something other than empty string since some overrides are empty strings
PROJECT_DEPENDENCIES="false"
Expand All @@ -107,17 +107,17 @@ for project in "${PROJECTS[@]}"; do
break
fi
done

PREVIOUS_SPEC_IDENTIFIERS=""

BUILDSPEC_NAME=$(basename $buildspec .yml)
BUILDSPEC_IDENTIFIER_OVERRIDE="$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_IDENTIFIER_OVERRIDE)"
if [[ -n "$BUILDSPEC_IDENTIFIER_OVERRIDE" ]]; then
IDENTIFIER="$BUILDSPEC_IDENTIFIER_OVERRIDE"
elif [[ "${BUILDSPEC_NAME}" != *buildspec* ]]; then
IDENTIFIER+="_${BUILDSPEC_NAME//-/_}"
fi

echo "Adding: $IDENTIFIER"

if [ "$PROJECT_DEPENDENCIES" = "false" ]; then
Expand Down Expand Up @@ -146,7 +146,7 @@ for project in "${PROJECTS[@]}"; do
fi

DEP_IDENTIFIER=${DEP_ORG//-/_}_${DEP_REPO//-/_}

if [ -n "${DEP_RELEASE_BRANCH}" ]; then
DEP_IDENTIFIER=${DEP_ORG//-/_}_${DEP_REPO//-/_}_${DEP_RELEASE_BRANCH//[-\/]/_}
fi
Expand All @@ -170,7 +170,7 @@ for project in "${PROJECTS[@]}"; do
if [[ -z "$BUILDSPEC_VARS_KEYS" ]]; then
BUILDSPEC_VARS_KEYS=$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_VARS_KEYS)
fi

BUILDSPEC_PLATFORM=$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_PLATFORM)
if [[ -z "$BUILDSPEC_PLATFORM" ]]; then
BUILDSPEC_PLATFORM=$(make_var $PROJECT_PATH BUILDSPEC_PLATFORM)
Expand All @@ -192,13 +192,13 @@ for project in "${PROJECTS[@]}"; do
BUILDSPEC_VARS_KEYS="RELEASE_BRANCH"
BUILDSPEC_VARS_VALUES="SUPPORTED_K8S_VERSIONS"
fi
elif [[ "${BUILDSPEC_VARS_KEYS}" = "IMAGE_PLATFORMS" ]]; then
elif [[ "${BUILDSPEC_VARS_KEYS}" = "IMAGE_PLATFORMS" ]]; then
BUILDSPEC_VARS_KEYS=""
fi
fi

if [[ "$BUILDSPECS_VAR" == "UPGRADE_BUILDSPECS" ]]; then
if [[ "${IDENTIFIER}" = "kubernetes_sigs_image_builder" ]]; then
if [[ "${IDENTIFIER}" = "kubernetes_sigs_image_builder" ]]; then
BUILDSPEC_VARS_KEYS=""
elif [[ "true" == "$(make_var $PROJECT_PATH HAS_RELEASE_BRANCHES)" ]]; then
if [[ "false" == "$(make_var $PROJECT_PATH BINARIES_ARE_RELEASE_BRANCHED)" ]]; then
Expand All @@ -220,15 +220,15 @@ for project in "${PROJECTS[@]}"; do
fi
fi
VARS=(${BUILDSPEC_VARS_VALUES// / })
# Note: only support 1 or 2 vars for now since that is all we need for kind + image-builder

# Note: only support 1 or 2 vars for now since that is all we need for kind + image-builder
if [ ${#VARS[@]} -eq 1 ]; then
VALUES_1=$(make_var $PROJECT_PATH ${VARS[0]})
ARR_1=(${VALUES_1// / })
for val1 in "${ARR_1[@]}"; do

for val1 in "${ARR_1[@]}"; do
IDENTIFIER_WITH_VAL=${IDENTIFIER}_${val1//[-\/]/_}

# If building on one binary platform assume we want to run on a specific arch instance
ARCH_TYPE="\"type\":\"$BUILDSPEC_PLATFORM\",\"compute-type\":\"$BUILDSPEC_COMPUTE_TYPE\","
if [ "${KEYS[0]}" = "BINARY_PLATFORMS" ] || [ "${KEYS[0]}" = "IMAGE_PLATFORMS" ]; then
Expand All @@ -239,7 +239,7 @@ for project in "${PROJECTS[@]}"; do
fi
fi

EXTRA_VARS=""
EXTRA_VARS=""
if [ "${KEYS[0]}" = "IMAGE_PLATFORMS" ]; then
EXTRA_VARS+=",\"BINARY_PLATFORMS\":\"${val1}\",\"IMAGE_TAG_SUFFIX\":\"-${val1#linux/}\""
HAS_HELM_CHART=$(make_var $PROJECT_PATH HAS_HELM_CHART)
Expand Down Expand Up @@ -285,7 +285,7 @@ for project in "${PROJECTS[@]}"; do
PREVIOUS_SPEC_IDENTIFIERS+="$IDENTIFIER_WITH_VAL "
yq eval -i -P \
".batch.build-graph += [{\"identifier\":\"$IDENTIFIER_WITH_VAL\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\",\"${KEYS[1]}\":\"$val2\"$EXTRA_VARS}}}]" \
$STAGING_BUILDSPEC_FILE
$STAGING_BUILDSPEC_FILE
done
done
fi
Expand All @@ -294,21 +294,28 @@ for project in "${PROJECTS[@]}"; do
PREVIOUS_SPEC_IDENTIFIERS+="$IDENTIFIER "
yq eval -i -P \
".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL}}}]" \
$STAGING_BUILDSPEC_FILE
$STAGING_BUILDSPEC_FILE
fi

done
done

if [ -n "${ADD_FINAL_STAGE}" ]; then
ARCH_TYPE="\"type\":\"ARM_CONTAINER\",\"compute-type\":\"BUILD_GENERAL1_SMALL\""
DEPEND_ON=",\"depend-on\":[${ALL_PROJECT_IDS%?}]"
if [ "$NO_DEPS_FOR_FINAL_STAGE" = "true" ]; then
DEPEND_ON=""
fi
yq eval -i -P \
".batch.build-graph += [{\"identifier\":\"final_stage\",\"buildspec\":\"$ADD_FINAL_STAGE\",\"env\":{$ARCH_TYPE}$DEPEND_ON}]" \
$STAGING_BUILDSPEC_FILE
if [ -n "${FINAL_STAGE_BUILDSPECS_CSV}" ]; then
IFS=',' read -r -a FINAL_STAGE_BUILDSPECS <<< "$FINAL_STAGE_BUILDSPECS_CSV"
for i in "${!FINAL_STAGE_BUILDSPECS[@]}";
do
IDENTIFIER="final_stage"
if [[ "$i" -gt 0 ]]; then
IDENTIFIER="final_stage_$((i+1))"
fi
ARCH_TYPE="\"type\":\"ARM_CONTAINER\",\"compute-type\":\"BUILD_GENERAL1_SMALL\""
DEPEND_ON=",\"depend-on\":[${ALL_PROJECT_IDS%?}]"
if [ "$NO_DEPS_FOR_FINAL_STAGE" = "true" ]; then
DEPEND_ON=""
fi
yq eval -i -P \
".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",\"buildspec\":\"${FINAL_STAGE_BUILDSPECS[$i]}\",\"env\":{$ARCH_TYPE}$DEPEND_ON}]" \
$STAGING_BUILDSPEC_FILE
done
fi

HEAD_COMMENT=$(cat $BASE_DIRECTORY/hack/boilerplate.yq.txt)
Expand Down
14 changes: 14 additions & 0 deletions buildspecs/upgrade-eks-distro-build-tooling-buildspec.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
version: 0.2

env:
secrets-manager:
GITHUB_TOKEN: "github-eks-distro-pr-bot:github-token"

phases:
pre_build:
commands:
- ./build/lib/setup.sh

build:
commands:
- make upgrade -C tools/version-tracker PROJECT=aws/eks-distro-build-tooling VERBOSITY=6
5 changes: 5 additions & 0 deletions tools/version-tracker/buildspecs/upgrade.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,11 @@ batch:
env:
type: ARM_CONTAINER
compute-type: BUILD_GENERAL1_SMALL
- identifier: final_stage_2
buildspec: buildspecs/upgrade-eks-distro-build-tooling-buildspec.yml
env:
type: ARM_CONTAINER
compute-type: BUILD_GENERAL1_SMALL
version: 0.2
env:
secrets-manager:
Expand Down
2 changes: 1 addition & 1 deletion tools/version-tracker/pkg/commands/display/display.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func Run(displayOptions *types.DisplayOptions) error {
// Get base repository owner environment variable if set.
baseRepoOwner := os.Getenv(constants.BaseRepoOwnerEnvvar)
if baseRepoOwner == "" {
baseRepoOwner = constants.DefaultBaseRepoOwner
baseRepoOwner = constants.AWSOrgName
}

// Clone the eks-anywhere-build-tooling repository.
Expand Down
2 changes: 1 addition & 1 deletion tools/version-tracker/pkg/commands/listprojects/listprojects.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func Run() error {
// Get base repository owner environment variable if set.
baseRepoOwner := os.Getenv(constants.BaseRepoOwnerEnvvar)
if baseRepoOwner == "" {
baseRepoOwner = constants.DefaultBaseRepoOwner
baseRepoOwner = constants.AWSOrgName
}

// Clone the eks-anywhere-build-tooling repository.
Expand Down
105 changes: 98 additions & 7 deletions tools/version-tracker/pkg/commands/upgrade/upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,43 @@ func Run(upgradeOptions *types.UpgradeOptions) error {
if isUpdated {
updatedFiles = append(updatedFiles, constants.EKSDistroLatestReleasesFile)
}
} else if isEKSDistroBuildToolingUpgrade(projectName) {
headBranchName = fmt.Sprintf("update-eks-distro-base-image-tag-files-%s", branchName)
baseBranchName = branchName
commitMessage = "Bump EKS Distro base image tag files to latest"

// Checkout a new branch to keep track of version upgrade chaneges.
err = git.Checkout(worktree, headBranchName, true)
if err != nil {
return fmt.Errorf("checking out worktree at branch %s: %v", headBranchName, err)
}

// Reset current worktree to get a clean index.
err = git.ResetToHEAD(worktree, headCommit)
if err != nil {
return fmt.Errorf("resetting new branch to [origin/%s] HEAD: %v", branchName, err)
}

eksDistroBaseTagFilesGlobPattern := filepath.Join(buildToolingRepoPath, constants.EKSDistroBaseTagFilesPattern)
eksDistroBaseTagFilesGlob, err := filepath.Glob(eksDistroBaseTagFilesGlobPattern)
if err != nil {
return fmt.Errorf("finding filenames matching EKS Distro Base tag file pattern [%s]: %v", constants.EKSDistroBaseTagFilesPattern, err)
}

updatedPackages, isUpdated, err := updateEKSDistroBaseImageTagFiles(client, buildToolingRepoPath, eksDistroBaseTagFilesGlob)
if err != nil {
return fmt.Errorf("updating EKS Distro base tag files: %v", err)
}
if isUpdated {
pullRequestBody = fmt.Sprintf(constants.EKSDistroBuildToolingUpgradePullRequestBody, updatedPackages)
for _, tagFile := range eksDistroBaseTagFilesGlob {
tagFileRelativePath, err := filepath.Rel(buildToolingRepoPath, tagFile)
if err != nil {
return fmt.Errorf("getting relative path for tag file: %v", err)
}
updatedFiles = append(updatedFiles, tagFileRelativePath)
}
}
} else {
// Validate if the project name provided exists in the repository.
projectPath := filepath.Join("projects", projectName)
Expand Down Expand Up @@ -531,6 +568,56 @@ func updateEKSDistroReleasesFile(buildToolingRepoPath string) (bool, error) {
return isUpdated, nil
}

func updateEKSDistroBaseImageTagFiles(client *gogithub.Client, buildToolingRepoPath string, tagFileGlob []string) (string, bool, error) {
var updatedPackages string
var isUpdated bool

eksDistroBaseTagYAMLContents, err := github.GetFileContents(client, constants.AWSOrgName, constants.EKSDistroBuildToolingRepoName, constants.EKSDistroBaseTagsYAMLFile, "main")
if err != nil {
return "", false, fmt.Errorf("getting contents of EKS Distro Base tag file: %v", err)
}

var eksDistroBaseTagYAMLMap map[string]interface{}
err = yaml.Unmarshal(eksDistroBaseTagYAMLContents, &eksDistroBaseTagYAMLMap)
if err != nil {
return "", false, fmt.Errorf("unmarshalling EKS Distro Base tag file: %v", err)
}

for _, tagFile := range tagFileGlob {
tagFileContents, err := os.ReadFile(tagFile)
if err != nil {
return "", false, fmt.Errorf("reading tag file: %v", err)
}
tagFileName := filepath.Base(tagFile)
imageName := strings.TrimSuffix(tagFileName, constants.TagFileSuffix)

tagFileKey := strings.ReplaceAll(strings.ToLower(imageName), "_", "-")
osFolder := "2"
osKey := "al2"
if strings.HasSuffix(tagFileKey, "al2023") {
tagFileKey = strings.TrimSuffix(tagFileKey, constants.AL2023Suffix)
osFolder = "2023"
osKey = "al2023"
}
tagReleaseDate := eksDistroBaseTagYAMLMap[osKey].(map[string]interface{})[tagFileKey].(string)

if string(tagFileContents) != tagReleaseDate {
isUpdated = true
err = os.WriteFile(tagFile, []byte(fmt.Sprintf("%s\n", tagReleaseDate)), 0o644)
if err != nil {
return "", false, fmt.Errorf("writing tag file: %v", err)
}

updatedPackagesFilesContents, err := github.GetFileContents(client, constants.AWSOrgName, constants.EKSDistroBuildToolingRepoName, fmt.Sprintf(constants.EKSDistroBaseUpdatedPackagesFileFormat, osFolder, tagFileKey), "main")
if err != nil {
return "", false, fmt.Errorf("getting contents of EKS Distro Base image updated packages: %v", err)
}
updatedPackages = fmt.Sprintf("%s#### %s\nThe following yum packages were updated:\n```bash\n%s```\n\n", updatedPackages, imageName, string(updatedPackagesFilesContents))
}
}
return updatedPackages, isUpdated, nil
}

func getSupportedReleaseBranches(buildToolingRepoPath string) ([]string, error) {
supportedReleaseBranchesFilepath := filepath.Join(buildToolingRepoPath, constants.SupportedReleaseBranchesFile)

Expand Down Expand Up @@ -690,7 +777,7 @@ func applyPatchesToRepo(projectRootFilepath, projectRepo string, totalPatchCount
patchesApplied = totalPatchCount
} else {
failedFiles := []string{}
gitDescribeRegex := regexp.MustCompile(`v?\d+\.\d+\.\d+(-([0-9]+)-g.*)?`)
gitDescribeRegex := regexp.MustCompile(constants.GitDescribeRegex)
gitDescribeCmd := exec.Command("git", "-C", filepath.Join(projectRootFilepath, projectRepo), "describe", "--tag")
gitDescribeOutput, err := command.ExecCommand(gitDescribeCmd)
if err != nil {
Expand Down Expand Up @@ -890,16 +977,16 @@ func verifyBRImageExists(channel, format, bottlerocketVersion string) (bool, err
switch format {
case "ami":
variant = "aws"
imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.img.lz4", variant, kubeVersion, bottlerocketVersion)
imageTarget = fmt.Sprintf(constants.BottlerocketAMIImageTargetFormat, variant, kubeVersion, bottlerocketVersion)
case "ova":
variant = "vmware"
imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.ova", variant, kubeVersion, bottlerocketVersion)
imageTarget = fmt.Sprintf(constants.BottlerocketOVAImageTargetFormat, variant, kubeVersion, bottlerocketVersion)
case "raw":
variant = "metal"
imageTarget = fmt.Sprintf("bottlerocket-%s-k8s-%s-x86_64-%s.img.lz4", variant, kubeVersion, bottlerocketVersion)
imageTarget = fmt.Sprintf(constants.BottlerocketRawImageTargetFormat, variant, kubeVersion, bottlerocketVersion)
}

timestampURL := fmt.Sprintf("https://updates.bottlerocket.aws/2020-07-07/%s-k8s-%s/x86_64/timestamp.json", variant, kubeVersion)
timestampURL := fmt.Sprintf(constants.BottlerocketTimestampJSONURLFormat, variant, kubeVersion)
timestampManifest, err := file.ReadURL(timestampURL)
if err != nil {
return false, fmt.Errorf("reading Bottlerocket timestamp URL: %v", err)
Expand All @@ -914,7 +1001,7 @@ func verifyBRImageExists(channel, format, bottlerocketVersion string) (bool, err
version := timestampData.(map[string]interface{})["signed"].(map[string]interface{})["version"].(float64)
versionString := fmt.Sprintf("%.0f", version)

targetsURL := fmt.Sprintf("https://updates.bottlerocket.aws/2020-07-07/%s-k8s-%s/x86_64/%s.targets.json", variant, kubeVersion, versionString)
targetsURL := fmt.Sprintf(constants.BottlerocketTargetsJSONURLFormat, variant, kubeVersion, versionString)
targetsManifest, err := file.ReadURL(targetsURL)
if err != nil {
return false, fmt.Errorf("reading Bottlerocket targets URL: %v", err)
Expand All @@ -938,7 +1025,7 @@ func verifyBRImageExists(channel, format, bottlerocketVersion string) (bool, err

func updateBottlerocketHostContainerMetadata(client *gogithub.Client, projectRootFilepath, projectPath, latestBottlerocketVersion string) ([]string, error) {
updatedHostContainerFiles := []string{}
hostContainersTOMLContents, err := github.GetFileContents(client, "bottlerocket-os", "bottlerocket", constants.BottlerocketHostContainersTOMLFile, latestBottlerocketVersion)
hostContainersTOMLContents, err := github.GetFileContents(client, constants.BottlerocketOrgName, constants.BottlerocketRepoName, constants.BottlerocketHostContainersTOMLFile, latestBottlerocketVersion)
if err != nil {
return nil, fmt.Errorf("getting contents of Bottlerocket host containers file: %v", err)
}
Expand Down Expand Up @@ -995,6 +1082,10 @@ func isEKSDistroUpgrade(projectName string) bool {
return projectName == "aws/eks-distro"
}

func isEKSDistroBuildToolingUpgrade(projectName string) bool {
return projectName == "aws/eks-distro-build-tooling"
}

func getDefaultReleaseBranch(buildToolingRepoPath string) (string, error) {
defaultReleaseBranchCommandSequence := fmt.Sprintf("make --no-print-directory -C %s get-default-release-branch", buildToolingRepoPath)
defaultReleaseBranchCmd := exec.Command("bash", "-c", defaultReleaseBranchCommandSequence)
Expand Down
Loading

0 comments on commit afa6776

Please sign in to comment.