Skip to content

Commit

Permalink
debugging starter project download
Browse files Browse the repository at this point in the history
Signed-off-by: Michael Hoang <[email protected]>
  • Loading branch information
mike-hoang committed Aug 9, 2023
1 parent 605081d commit 5f44603
Show file tree
Hide file tree
Showing 2 changed files with 252 additions and 34 deletions.
143 changes: 109 additions & 34 deletions build-tools/dl_starter_projects.sh
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
#!/bin/bash

if [[ -z "$@" ]]
then
echo "No starter projects specified."
exit 0
fi

# Path of stacks directory in the registry
STACKS_DIR=/registry/stacks
# List of starter projects to use offline
offline_starter_projects=( "$@" )
# When no starter projects are specifed,
# all starter projects will be downloaded
download_all_starter_projects=false

if [[ -z "$@" ]]
then
download_all_starter_projects=true
fi

# Downloads a starter project from a remote git repository and packages it as a zip archive
# to be used as an offline resource.
Expand All @@ -34,14 +36,21 @@ download_git_starter_project() {

if [ "${revision}" != "null" ]
then
cd $local_path && git checkout $revision && cd -
cd $local_path
git checkout $revision
cd -
fi

if [ "${subDir}" != "null" ]
then
cd $local_path/$subDir && zip -q ${local_path}.zip * .[^.]* && cd -
cd $local_path/$subDir
zip -q ../${name}-offline.zip * .[^.]*
cd -
else
cd $local_path && rm -rf ./.git && zip -q ${local_path}.zip * .[^.]* && cd -
cd $local_path
rm -rf ./.git
zip -q ../${name}-offline.zip * .[^.]*
cd -
fi

rm -rf $local_path
Expand All @@ -58,12 +67,62 @@ download_zip_starter_project() {
curl -L $remote_url -o ${local_path}.zip
}

# Read stacks list
read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')"
download_specific() {
for starter_project in ${offline_starter_projects[@]}
do
for stack in ${stacks[@]}
do
stack_root=$STACKS_DIR/$stack
stack_devfile=$stack_root/devfile.yaml
# Read version list for stack
read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')"
# If multi version stack
if [[ ${#versions[@]} -gt 0 ]]
then
for version in ${versions[@]}
do
stack_root=$STACKS_DIR/$stack/$version
stack_devfile=$stack_root/devfile.yaml
# If the specified starter project is found
if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
fi
fi
done
# If not multi version stack & the specified starter project is found
elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
fi
fi
done
done
}

echo "Downloading offline starter projects.."
for starter_project in ${offline_starter_projects[@]}
do
download_all() {
for stack in ${stacks[@]}
do
stack_root=$STACKS_DIR/$stack
Expand All @@ -77,9 +136,10 @@ do
do
stack_root=$STACKS_DIR/$stack/$version
stack_devfile=$stack_root/devfile.yaml
# If the specified starter project is found
if [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"

for starter_project in $starter_projects
do
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
Expand All @@ -93,25 +153,40 @@ do
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack} version ${version}..done!"
fi
done
done
# If not multi version stack
else
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"
for starter_project in $starter_projects
do
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
fi
done
# If not multi version stack & the specified starter project is found
elif [ ! -z "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\")" $stack_devfile)" ]
then
# Starter project has a git remote
if [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").git" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_git_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
# Starter project has a zip remote
elif [ "$(yq e ".starterProjects[] | select(.name == \"${starter_project}\").zip" $stack_devfile)" != "null" ]
then
echo "Downloading ${starter_project} starter project in stack ${stack}.."
download_zip_starter_project $stack_root $starter_project
echo "Downloading ${starter_project} starter project in stack ${stack}..done!"
fi
fi
echo
done
done
}

# Read stacks list
read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')"

echo "Downloading offline starter projects.."
if [ "$download_all_starter_projects" = true ]
then
download_all
else
download_specific
fi
echo "Downloading offline starter projects..done!"
143 changes: 143 additions & 0 deletions build-tools/update_devfiles_offline.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
#!/bin/bash

# This script will modify the devfiles in each stack to use offline resources.

# Path of stacks directory in the registry
STACKS_DIR=/registry/stacks
# Automated comment used to check whether the devfile has already been modified
MODIFIED_MESSAGE="# AUTOMATED MODIFICATION -"

cleanup() {
rm -f "$devfile.tmp"
rm -f "$offline_starter_projects"
}

trap cleanup EXIT

comment_out_starter_project() {
stack_root=$1
name=$2
devfile=$3

has_starterProjects=$(yq e '.starterProjects' $devfile 2> /dev/null)
if [[ $has_starterProjects != null ]]
then
# the first line of the diff needs to be removed; example of the diff:
# 44a45,52
# > starterProjects:
# > - name: go-starter
# ...
#
# 's/^../#/' replaces the first 2 characters of each line with #
diff="$(diff -b <(yq e 'del(.starterProjects)' $devfile) $devfile | tail -n +2)"
starter_projects="$(printf '%s\n' "$diff" | sed 's/^../#/')"

# comment out the starter projects
yq e 'del(.starterProjects)' "$devfile" > "$devfile.tmp"
echo "# Commented out original starter projects" >> "$devfile.tmp"
echo "$starter_projects" >> "$devfile.tmp"
mv "$devfile.tmp" "$devfile"
fi
}

# Updates a devfile to use an offline starter project
new_starter_projects() {
stack_root=$1
name=$2
devfile=$3
offline_starter_projects=$4

project="
- name: ${name}
zip:
location: ${name}-offline.zip"

echo -n "$project" >> $offline_starter_projects
}

# Read stacks list
read -r -a stacks <<< "$(ls ${STACKS_DIR} | tr '\n' ' ')"

echo "Updating devfiles.."
for stack in ${stacks[@]}
do
stack_root=$STACKS_DIR/$stack
stack_devfile=$stack_root/devfile.yaml
# Exit early on failure to avoid bad overwriting of the devfile
offline_starter_projects=$(mktemp) || exit 1
# Overwrite the temp file for each stack
echo -n "" > $offline_starter_projects
# Read version list for stack
read -r -a versions <<< "$(ls ${STACKS_DIR}/${stack} | grep -e '[0-9].[0-9].[0-9]' | tr '\n' ' ')"

# If multi version stack
if [[ ${#versions[@]} -gt 0 ]]
then
for version in ${versions[@]}
do
stack_root=$STACKS_DIR/$stack/$version
stack_devfile=$stack_root/devfile.yaml
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"
echo -n "starterProjects:" > $offline_starter_projects

if [[ $starter_projects == "" ]]
then
echo "Skipping stack ${stack} version ${version}: no starter projects found."
continue
fi

for starter_project in $starter_projects
do
if ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile";
then
echo "Updating the ${starter_project} starter project in stack ${stack} version ${version}.."
new_starter_projects $stack_root $starter_project $stack_devfile $offline_starter_projects
comment_out_starter_project $stack_root $starter_project $stack_devfile
echo "Updating the ${starter_project} starter project in stack ${stack} version ${version}..done!"
else
echo "The ${starter_project} starter project in stack ${stack} version ${version} has already been modified."
fi
done

# Only write to the devfile if starter projects have been commented out
has_starterProjects=$(yq e '.starterProjects' $stack_devfile 2> /dev/null)
if [[ $has_starterProjects == null ]] && ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile"
then
echo "${MODIFIED_MESSAGE} Updated starterProjects to use offline versions" >> $stack_devfile
cat $offline_starter_projects >> $stack_devfile
fi
done
# If not multi version stack
else
starter_projects="$(yq e ".starterProjects[].name" $stack_devfile)"
echo -n "starterProjects:" > $offline_starter_projects

if [[ $starter_projects == "" ]]
then
echo "Skipping stack ${stack}: no starter projects found."
continue
fi

for starter_project in $starter_projects
do
if ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile";
then
echo "Updating the ${starter_project} starter project in stack ${stack}.."
new_starter_projects $stack_root $starter_project $stack_devfile $offline_starter_projects
comment_out_starter_project $stack_root $starter_project $stack_devfile
echo "Updating the ${starter_project} starter project in stack ${stack}..done!"
else
echo "The ${starter_project} starter project in stack ${stack} has already been modified."
fi
done

# Only write to the devfile if starter projects have been commented out
has_starterProjects=$(yq e '.starterProjects' $stack_devfile 2> /dev/null)
if [[ $has_starterProjects == null ]] && ! grep -q "$MODIFIED_MESSAGE" "$stack_devfile"
then
echo "${MODIFIED_MESSAGE} Updated starterProjects to use offline versions" >> $stack_devfile
cat $offline_starter_projects >> $stack_devfile
fi
fi
done
echo "Updating devfiles....done!"

0 comments on commit 5f44603

Please sign in to comment.