cleans up helm chart building and pushing

aws · Dec 1, 2023 · e8372b1 · e8372b1
1 parent 5bee9f9
commit e8372b1
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 59 deletions.
diff --git a/build/lib/buildkit.sh b/build/lib/buildkit.sh
@@ -105,7 +105,7 @@ else
     log_file=$(mktemp)
     trap "rm -f $log_file" EXIT
     if ! $CMD $ARGS 2>&1 | tee $log_file; then      
-        if grep -q "blobs/uploads/\": EOF" $log_file ; then
+        if grep -q "blobs/uploads/\": EOF" $log_file || grep -q "blobs/uploads.*404 Not Found" $log_file; then
             echo "******************************************************"
             echo "Ensure container registry and repository exists!!"
             echo "Try running make create-ecr-repos to create ecr repositories in your aws account."

diff --git a/build/lib/helm_push.sh b/build/lib/helm_push.sh
@@ -46,7 +46,7 @@ export HELM_REGISTRY_CONFIG="${DOCKER_CONFIG}/config.json"
 export HELM_EXPERIMENTAL_OCI=1
 TMPFILE=$(mktemp /tmp/helm-output.XXXXXX)
 function cleanup() {
-  if grep -q "blobs/uploads/\": EOF" $TMPFILE ; then
+  if grep -q "blobs/uploads/\": EOF" $TMPFILE || grep -q "blobs/uploads.*404 Not Found" $TMPFILE; then
     echo "******************************************************"
     echo "Ensure container registry and repository exists!!"
     echo "Try running make create-ecr-repos to create ecr repositories in your aws account."
@@ -69,11 +69,7 @@ helm push ${CHART_FILE} oci://${IMAGE_REGISTRY}/${HELM_DESTINATION_OWNER} 2>&1 |
 DIGEST=$(grep Digest $TMPFILE | $SED -e 's/Digest: //')
 
 # Adds a 2nd tag to the helm chart for the bundle-release jobs.
-if [[ "${IMAGE_REGISTRY}" != *"public.ecr.aws"* ]] && [[ "${IMAGE_REGISTRY}" == *"ecr"* ]]; then
-  MANIFEST=$(build::common::echo_and_run aws ecr batch-get-image --repository-name "$HELM_DESTINATION_REPOSITORY" --image-ids imageDigest=${DIGEST} --query "images[].imageManifest" --output text)
-  export AWS_PAGER=""
-  build::common::echo_and_run aws ecr put-image --repository-name ${HELM_DESTINATION_REPOSITORY} --image-tag ${SEMVER_GIT_TAG}-${LATEST_TAG}-helm --image-manifest "$MANIFEST" --image-manifest-media-type "application/vnd.oci.image.manifest.v1+json"
-fi
+build::common::echo_and_run skopeo copy docker://${IMAGE_REGISTRY}/${HELM_DESTINATION_REPOSITORY}@${DIGEST} docker://${IMAGE_REGISTRY}/${HELM_DESTINATION_REPOSITORY}:${SEMVER_GIT_TAG}-${LATEST_TAG}-helm
 
 {
     set +x

diff --git a/build/lib/helm_require.sh b/build/lib/helm_require.sh
@@ -35,11 +35,6 @@ DEST_DIR=${OUTPUT_DIR}/helm/${CHART_NAME}
 SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 source "${SCRIPT_ROOT}/common.sh"
 
-if [ "${HELM_USE_UPSTREAM_IMAGE}" != true ] && [[ "${IMAGE_REGISTRY}" == *"ecr"* ]] && ! aws sts get-caller-identity &> /dev/null; then
-  echo "The AWS cli is used to find the ECR registries and repos for the current AWS account please login!"
-  exit 1;
-fi
-
 #
 # Image tags
 #
@@ -56,11 +51,6 @@ spec:
 JSON_SCHEMA_FILE=$PROJECT_ROOT/helm/schema.json
 SEDFILE=${OUTPUT_DIR}/helm/sedfile
 
-export HELM_REGISTRY="$IMAGE_REGISTRY"
-if [ "${HELM_USE_UPSTREAM_IMAGE}" != true ] && [[ "${IMAGE_REGISTRY}" == *"ecr"* ]]; then
-  export HELM_REGISTRY=$(aws ecr-public describe-registries --region us-east-1 --output text --query 'registries[*].registryUri' 2> /dev/null)
-fi
-
 export IMAGE_TAG
 export HELM_TAG
 envsubst <$PROJECT_ROOT/helm/sedfile.template >${SEDFILE}
@@ -73,52 +63,34 @@ function get_image_shasum() {
   local -r tag=$2
 
   local image_shasum=
-  if [ "${HELM_USE_UPSTREAM_IMAGE}" = true ]; then
-    image_shasum=$(build::common::echo_and_run ${SCRIPT_ROOT}/image_shasum.sh ${IMAGE_REGISTRY} ${image} ${tag})
-  elif [ "${JOB_TYPE:-}" = "presubmit" ] || [[ "${IMAGE_REGISTRY}" != *"ecr"* ]]; then
-    image_shasum=${LATEST}
-  fi
-
-  if [[ -z ${image_shasum} ]] && aws --region us-east-1 ecr-public describe-repositories --repository-names ${image} &> /dev/null; then
-    image_shasum=$(build::common::echo_and_run ${SCRIPT_ROOT}/image_shasum.sh ${HELM_REGISTRY} ${image} ${tag})
+  if [ "${JOB_TYPE:-}" = "presubmit" ]; then
+    echo ${LATEST}
+  elif [[ -z ${image_shasum} ]]; then
+    build::common::echo_and_run ${SCRIPT_ROOT}/image_shasum.sh ${IMAGE_REGISTRY} ${image} ${tag}
   fi
-
-  if [[ -z ${image_shasum} ]] && aws ecr describe-repositories --repository-names ${image} &> /dev/null; then
-    image_shasum=$(build::common::echo_and_run ${SCRIPT_ROOT}/image_shasum.sh ${IMAGE_REGISTRY} ${image} ${tag})    
-  fi
-
-  if [[ -n ${image_shasum} ]]; then
-    echo ${image_shasum}
-  else
-    echo "${image} does not exist in ECR Public or Private"
-    exit 1
-  fi  
 }
 
   # query ecr for the image by latest tag and find the first non-latest tag the image is also tagged with
 function get_image_tag_not_latest() {
     local -r image=$1
-    local -r tag=$2
+    local -r shasum=$2
 
-    local use_tag=
+    # to find another tag associated with this image we have to use the aws cli
+    # the following only works for ecr repos
     if [ "${JOB_TYPE:-}" = "presubmit" ] ||  [[ "${IMAGE_REGISTRY}" != *"ecr"* ]]; then
-      use_tag=${tag}      
-    fi
-
-    if [[ -z ${use_tag} ]] && aws --region us-east-1 ecr-public describe-repositories --repository-names ${image} &> /dev/null; then
-      use_tag=$(build::common::echo_and_run aws --region us-east-1 ecr-public describe-images --repository-name ${image} --image-ids imageTag=${tag} --query 'imageDetails[0].imageTags' --output yaml 2> /dev/null | grep -v ${tag} | head -1| sed -e 's/- //')
-    fi
-
-    if [[ -z ${use_tag} ]] && aws ecr describe-repositories --repository-names ${image} &> /dev/null; then
-      use_tag=$(build::common::echo_and_run  aws ecr describe-images --repository-name ${image} --image-id imageTag=${tag} --query 'imageDetails[0].imageTags' --output yaml 2> /dev/null | grep -v ${tag} | head -1| sed -e 's/- //')
-    fi
-
-    if [[ -n ${use_tag} ]]; then
-      echo ${use_tag}
+      echo ${tag}      
     else
-      echo "${image}@${tag} does not exist in ECR Public or Private"
-      exit 1
-    fi  
+      if ! aws sts get-caller-identity &> /dev/null; then
+        echo "The AWS cli is used to find the ECR registries and repos for the current AWS account please login!"
+        exit 1;
+      fi
+
+      local service="ecr"
+      if [[ "${IMAGE_REGISTRY}" = *"public.ecr"* ]]; then
+        service="--region us-east-1 ecr-public"      
+      fi
+      build::common::echo_and_run aws ${service} describe-images --repository-name ${image} --image-id imageDigest=${shasum} --query 'imageDetails[0].imageTags' --output yaml | grep -v ${LATEST} | head -1| sed -e 's/- //'      
+    fi
 }
 
 for IMAGE in ${HELM_IMAGE_LIST:-}; do
@@ -132,9 +104,19 @@ for IMAGE in ${HELM_IMAGE_LIST:-}; do
 
   IMAGE_SHASUM=$(get_image_shasum ${IMAGE} ${TAG})
 
+  if [[ -z ${IMAGE_SHASUM} ]]; then
+    echo "${IMAGE} does not exist!"
+    exit 1
+  fi 
+
   echo "s,{{${IMAGE}}},${IMAGE_SHASUM},g" >>${SEDFILE}
-  if [ "${TAG}" = "${LATEST}" ];  then
-    USE_TAG=$(get_image_tag_not_latest ${IMAGE} ${LATEST})
+  if [ "${TAG}" = "${LATEST}" ]; then
+    # if finding an image from another project using the `latest` tag, find the image and a different tag associated with that image
+    USE_TAG=$(get_image_tag_not_latest ${IMAGE} ${IMAGE_SHASUM})
+    if [[ -z ${USE_TAG} ]]; then
+      echo "non-${LATEST} tag does not exist for ${IMAGE}@${IMAGE_SHASUM}!"
+      exit 1
+    fi 
   else
     USE_TAG=$TAG
   fi

diff --git a/build/lib/image_shasum.sh b/build/lib/image_shasum.sh
@@ -28,9 +28,8 @@ TMPFILE=$(mktemp)
 trap "rm -f $TMPFILE" exit
 TARGET=${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}
 
->&2 echo -n "Checking for the existence of ${TARGET}..."
-if skopeo inspect -n --raw docker://${TARGET} >${TMPFILE} 2>/dev/null; then
-  >&2 echo "Found!"
+if build::common::echo_and_run skopeo inspect -n --raw docker://${TARGET} >${TMPFILE}; then
+  >&2 echo "Found: $(skopeo manifest-digest ${TMPFILE})"
   skopeo manifest-digest ${TMPFILE}
 else
   >&2 echo "Not Found!"