From 5f446032d2c91b919bede988ef1025778d128b01 Mon Sep 17 00:00:00 2001 From: Michael Hoang Date: Tue, 1 Aug 2023 11:05:53 -0400 Subject: [PATCH] debugging starter project download Signed-off-by: Michael Hoang --- build-tools/dl_starter_projects.sh | 143 +++++++++++++++++++------ build-tools/update_devfiles_offline.sh | 143 +++++++++++++++++++++++++ 2 files changed, 252 insertions(+), 34 deletions(-) create mode 100755 build-tools/update_devfiles_offline.sh diff --git a/build-tools/dl_starter_projects.sh b/build-tools/dl_starter_projects.sh index a21301cc..1322fc5c 100755 --- a/build-tools/dl_starter_projects.sh +++ b/build-tools/dl_starter_projects.sh @@ -1,15 +1,17 @@ #!/bin/bash -if [[ -z "$@" ]] -then - echo "No starter projects specified." - exit 0 -fi - # Path of stacks directory in the registry STACKS_DIR=/registry/stacks # List of starter projects to use offline offline_starter_projects=( "$@" ) +# When no starter projects are specifed, +# all starter projects will be downloaded +download_all_starter_projects=false + +if [[ -z "$@" ]] +then + download_all_starter_projects=true +fi # Downloads a starter project from a remote git repository and packages it as a zip archive # to be used as an offline resource. @@ -34,14 +36,21 @@ download_git_starter_project() { if [ "${revision}" != "null" ] then - cd $local_path && git checkout $revision && cd - + cd $local_path + git checkout $revision + cd - fi if [ "${subDir}" != "null" ] then - cd $local_path/$subDir && zip -q ${local_path}.zip * .[^.]* && cd - + cd $local_path/$subDir + zip -q ../${name}-offline.zip * .[^.]* + cd - else - cd $local_path && rm -rf ./.git && zip -q ${local_path}.zip * .[^.]* && cd - + cd $local_path + rm -rf ./.git + zip -q ../${name}-offline.zip * .[^.]* + cd - fi rm -rf $local_path @@ -58,12 +67,62 @@ download_zip_starter_project() { curl -L $remote_url -o ${local_path}.zip } -# Read stacks list -read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')" +download_specific() { + for starter_project in ${offline_starter_projects[@]} + do + for stack in ${stacks[@]} + do + stack_root=$STACKS_DIR/$stack + stack_devfile=$stack_root/devfile.yaml + # Read version list for stack + read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')" + # If multi version stack + if [[ ${#versions[@]} -gt 0 ]] + then + for version in ${versions[@]} + do + stack_root=$STACKS_DIR/$stack/$version + stack_devfile=$stack_root/devfile.yaml + # If the specified starter project is found + if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] + then + # Starter project has a git remote + if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.." + download_git_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!" + # Starter project has a zip remote + elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.." + download_zip_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!" + fi + fi + done + # If not multi version stack & the specified starter project is found + elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] + then + # Starter project has a git remote + if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack}.." + download_git_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack}..done!" + # Starter project has a zip remote + elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack}.." + download_zip_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack}..done!" + fi + fi + done + done +} -echo "Downloading offline starter projects.." -for starter_project in ${offline_starter_projects[@]} -do +download_all() { for stack in ${stacks[@]} do stack_root=$STACKS_DIR/$stack @@ -77,9 +136,10 @@ do do stack_root=$STACKS_DIR/$stack/$version stack_devfile=$stack_root/devfile.yaml - # If the specified starter project is found - if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] - then + starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)" + + for starter_project in $starter_projects + do # Starter project has a git remote if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] then @@ -93,25 +153,40 @@ do download_zip_starter_project $stack_root $starter_project echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!" fi + done + done + # If not multi version stack + else + starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)" + for starter_project in $starter_projects + do + # Starter project has a git remote + if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack}.." + download_git_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack}..done!" + # Starter project has a zip remote + elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] + then + echo "Downloading ${starter_project} starter project in stack ${stack}.." + download_zip_starter_project $stack_root $starter_project + echo "Downloading ${starter_project} starter project in stack ${stack}..done!" fi done - # If not multi version stack & the specified starter project is found - elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ] - then - # Starter project has a git remote - if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ] - then - echo "Downloading ${starter_project} starter project in stack ${stack}.." - download_git_starter_project $stack_root $starter_project - echo "Downloading ${starter_project} starter project in stack ${stack}..done!" - # Starter project has a zip remote - elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ] - then - echo "Downloading ${starter_project} starter project in stack ${stack}.." - download_zip_starter_project $stack_root $starter_project - echo "Downloading ${starter_project} starter project in stack ${stack}..done!" - fi fi + echo done -done +} + +# Read stacks list +read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')" + +echo "Downloading offline starter projects.." +if [ "$download_all_starter_projects" = true ] +then + download_all +else + download_specific +fi echo "Downloading offline starter projects..done!" diff --git a/build-tools/update_devfiles_offline.sh b/build-tools/update_devfiles_offline.sh new file mode 100755 index 00000000..7dcf1a88 --- /dev/null +++ b/build-tools/update_devfiles_offline.sh @@ -0,0 +1,143 @@ +#!/bin/bash + +# This script will modify the devfiles in each stack to use offline resources. + +# Path of stacks directory in the registry +STACKS_DIR=/registry/stacks +# Automated comment used to check whether the devfile has already been modified +MODIFIED_MESSAGE="# AUTOMATED MODIFICATION -" + +cleanup() { + rm -f "$devfile.tmp" + rm -f "$offline_starter_projects" +} + +trap cleanup EXIT + +comment_out_starter_project() { + stack_root=$1 + name=$2 + devfile=$3 + + has_starterProjects=$(yq e '.starterProjects' $devfile 2> /dev/null) + if [[ $has_starterProjects != null ]] + then + # the first line of the diff needs to be removed; example of the diff: + # 44a45,52 + # > starterProjects: + # > - name: go-starter + # ... + # + # 's/^../#/' replaces the first 2 characters of each line with # + diff="$(diff -b <(yq e 'del(.starterProjects)' $devfile) $devfile | tail -n +2)" + starter_projects="$(printf '%s\n' "$diff" | sed 's/^../#/')" + + # comment out the starter projects + yq e 'del(.starterProjects)' "$devfile" > "$devfile.tmp" + echo "# Commented out original starter projects" >> "$devfile.tmp" + echo "$starter_projects" >> "$devfile.tmp" + mv "$devfile.tmp" "$devfile" + fi +} + +# Updates a devfile to use an offline starter project +new_starter_projects() { + stack_root=$1 + name=$2 + devfile=$3 + offline_starter_projects=$4 + + project=" + - name: ${name} + zip: + location: ${name}-offline.zip" + + echo -n "$project" >> $offline_starter_projects +} + +# Read stacks list +read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')" + +echo "Updating devfiles.." +for stack in ${stacks[@]} +do + stack_root=$STACKS_DIR/$stack + stack_devfile=$stack_root/devfile.yaml + # Exit early on failure to avoid bad overwriting of the devfile + offline_starter_projects=$(mktemp) || exit 1 + # Overwrite the temp file for each stack + echo -n "" > $offline_starter_projects + # Read version list for stack + read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')" + + # If multi version stack + if [[ ${#versions[@]} -gt 0 ]] + then + for version in ${versions[@]} + do + stack_root=$STACKS_DIR/$stack/$version + stack_devfile=$stack_root/devfile.yaml + starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)" + echo -n "starterProjects:" > $offline_starter_projects + + if [[ $starter_projects == "" ]] + then + echo "Skipping stack ${stack} version ${version}: no starter projects found." + continue + fi + + for starter_project in $starter_projects + do + if ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile"; + then + echo "Updating the ${starter_project} starter project in stack ${stack} version ${version}.." + new_starter_projects $stack_root $starter_project $stack_devfile $offline_starter_projects + comment_out_starter_project $stack_root $starter_project $stack_devfile + echo "Updating the ${starter_project} starter project in stack ${stack} version ${version}..done!" + else + echo "The ${starter_project} starter project in stack ${stack} version ${version} has already been modified." + fi + done + + # Only write to the devfile if starter projects have been commented out + has_starterProjects=$(yq e '.starterProjects' $stack_devfile 2> /dev/null) + if [[ $has_starterProjects == null ]] && ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile" + then + echo "${MODIFIED_MESSAGE} Updated starterProjects to use offline versions" >> $stack_devfile + cat $offline_starter_projects >> $stack_devfile + fi + done + # If not multi version stack + else + starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)" + echo -n "starterProjects:" > $offline_starter_projects + + if [[ $starter_projects == "" ]] + then + echo "Skipping stack ${stack}: no starter projects found." + continue + fi + + for starter_project in $starter_projects + do + if ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile"; + then + echo "Updating the ${starter_project} starter project in stack ${stack}.." + new_starter_projects $stack_root $starter_project $stack_devfile $offline_starter_projects + comment_out_starter_project $stack_root $starter_project $stack_devfile + echo "Updating the ${starter_project} starter project in stack ${stack}..done!" + else + echo "The ${starter_project} starter project in stack ${stack} has already been modified." + fi + done + + # Only write to the devfile if starter projects have been commented out + has_starterProjects=$(yq e '.starterProjects' $stack_devfile 2> /dev/null) + if [[ $has_starterProjects == null ]] && ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile" + then + echo "${MODIFIED_MESSAGE} Updated starterProjects to use offline versions" >> $stack_devfile + cat $offline_starter_projects >> $stack_devfile + fi + fi +done +echo "Updating devfiles....done!"