diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fa6ed72..6042f2a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: submodules: recursive - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static unzip wget xz-utils units + run: sudo apt-get update && sudo apt-get install -y binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static units unzip wget xxhash xz-utils shell: bash - name: Run pimod OpenWRT example diff --git a/Dockerfile b/Dockerfile index 769d183..e15af1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,10 +14,11 @@ RUN apt-get update && \ p7zip-full \ qemu \ qemu-user-static \ + units \ unzip \ wget \ - xz-utils \ - units + xxhash \ + xz-utils RUN mkdir /pimod COPY . /pimod/ diff --git a/README.md b/README.md index 671c1a0..1efd5f5 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Options: ### Debian ```bash -sudo apt-get install binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static unzip wget xz-utils units +sudo apt-get install binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static units unzip wget xxhash xz-utils sudo ./pimod.sh Pifile ``` diff --git a/action.yml b/action.yml index 99b72ad..33cdd87 100644 --- a/action.yml +++ b/action.yml @@ -14,7 +14,7 @@ runs: steps: - run: sudo apt-get update shell: bash - - run: sudo apt-get install -y binfmt-support fdisk file kpartx qemu qemu-user-static unzip p7zip-full wget xz-utils units + - run: sudo apt-get install -y binfmt-support fdisk file kpartx qemu qemu-user-static units unzip p7zip-full wget xxhash xz-utils shell: bash - run: sudo ${{ github.action_path }}/pimod.sh ${{ inputs.pifile }} shell: bash diff --git a/modules/from_remote.sh b/modules/cache.sh similarity index 60% rename from modules/from_remote.sh rename to modules/cache.sh index 2147289..f4111db 100644 --- a/modules/from_remote.sh +++ b/modules/cache.sh @@ -1,4 +1,4 @@ -if [ -z "${PIMOD_CACHE+x}" ]; then +if [ -z "${PIMOD_CACHE+x}" ]; then PIMOD_CACHE="/var/cache/pimod" fi @@ -29,7 +29,7 @@ from_remote_fetch() { local url local url_path local download_path - + url="${1}" url_path=$(echo "${url}" | sed 's/.*:\/\///') download_path="${PIMOD_CACHE}/${url_path}" @@ -83,3 +83,49 @@ from_remote_fetch() { export SOURCE_IMG="${tmpfile}" export SOURCE_IMG_TMP=1 } + +# cache_mk_hash calculates a hash for the tuple of an image and a command. +# Usage: cache_mk_hash IMG_FILE "RUN whoami" +cache_mk_hash() { + local img_hash + local cmd_hash + + img_hash="$(xxh128sum "${1}" | awk '{ print $1 }')" + cmd_hash="$(xxh128sum - <<< "${2}" | awk '{ print $1 }')" + + xxh128sum - <<< "${img_hash}${cmd_hash}" | awk '{ print $1 }' +} + +# cache_get_path returns the path for a snapshot. +# Usage: cache_get_path HASH +cache_get_path() { + echo "${PIMOD_CACHE}/__checkpoint__/${1}" +} + +# cache_img_checkpoint stores a checkpoint for the current state of an image. +# Usage: cache_img_checkpoint HASH IMG +cache_img_checkpoint() { + local cache_path + cache_path="$(cache_get_path "${1}")" + + mkdir -p "$(dirname "${cache_path}")" + if [[ ! -f "${cache_path}" ]]; then + cp --reflink=auto "${2}" "${cache_path}" + fi +} + +# cache_img_chk_load checks if a checkpoint does already exists and might loads +# it. The function's exit code says if the cache was used (0) or not (1). +# Usage: cache_img_chk_load HASH IMG +cache_img_chk_load() { + local cache_path + cache_path="$(cache_get_path "${1}")" + + echo ">>> cache_img_chk_load ${*}" + + if [[ ! -f "${cache_path}" ]]; then + return 1 + else + cp --reflink=auto "${cache_path}" "${2}" + fi +} diff --git a/pimod.sh b/pimod.sh index dcce2d8..46a6ff9 100755 --- a/pimod.sh +++ b/pimod.sh @@ -4,11 +4,11 @@ set -euE pushd "$(dirname "$0")" > /dev/null +. ./modules/cache.sh . ./modules/chroot.sh . ./modules/env.sh . ./modules/error.sh . ./modules/esceval.sh -. ./modules/from_remote.sh . ./modules/mount.sh . ./modules/path.sh . ./modules/pifile.sh @@ -24,13 +24,14 @@ Usage: ${0} [Options] Pifile Options: -c cache Define cache location. - -d Debug on failure; run an interactive shell before tear down + -d Debug on failure; run an interactive shell before tear down. + -s Save cache checkpoints for each command. -t Trace each executed command for debugging. -h Print this help message. EOF } -while getopts "c:dth" opt; do +while getopts "c:dsth" opt; do case "${opt}" in c) PIMOD_CACHE="${OPTARG}" @@ -38,6 +39,8 @@ while getopts "c:dth" opt; do d) PIMOD_DEBUG=1 ;; + s) + ;; t) set -x ;; diff --git a/stages/20-prepare.sh b/stages/20-prepare.sh index c28d79e..2678487 100644 --- a/stages/20-prepare.sh +++ b/stages/20-prepare.sh @@ -9,6 +9,12 @@ PUMP() { echo -e "\033[0;32m### PUMP ${1}\033[0m" + local input_hash="$(cache_mk_hash "${DEST_IMG}" "${*}")" + if cache_img_chk_load "${input_hash}" "${DEST_IMG}"; then + echo "Using cached image.."; + return + fi + BS="1M" # units does not print to stderr, thus test call before using output @@ -41,4 +47,6 @@ PUMP() { resize2fs "/dev/mapper/${loop}p${IMG_ROOT}" umount_image "${loop}" + + cache_img_checkpoint "${input_hash}" "${DEST_IMG}" } diff --git a/stages/30-chroot.sh b/stages/30-chroot.sh index bc1fa94..7106eb8 100644 --- a/stages/30-chroot.sh +++ b/stages/30-chroot.sh @@ -11,6 +11,12 @@ pre_stage() { # Usage: INSTALL [MODE] SOURCE DEST INSTALL() { echo -e "\033[0;32m### INSTALL $*\033[0m" + + local input_hash="$(cache_mk_hash "${DEST_IMG}" "${*}")" + if cache_img_chk_load "${input_hash}" "${DEST_IMG}"; then + echo "Using cached image.."; + return + fi local src="" local dst="" @@ -41,6 +47,8 @@ INSTALL() { if [[ "$#" -eq "3" ]]; then chmod "$1" "${CHROOT_MOUNT}/${dst}" fi + + cache_img_checkpoint "${input_hash}" "${DEST_IMG}" } # PATH adds the given path to an overlaying PATH variable, used within the RUN @@ -103,11 +111,19 @@ ENV() { RUN() { echo -e "\033[0;32m### RUN ${*}\033[0m" + local input_hash="$(cache_mk_hash "${DEST_IMG}" "${*}")" + if cache_img_chk_load "${input_hash}" "${DEST_IMG}"; then + echo "Using cached image.."; + return + fi + local cmd_esceval="$(esceval "$@")" local cmd_env_subst="$(env_vars_subst "$cmd_esceval")" PATH=${GUEST_PATH} chroot "${CHROOT_MOUNT}" \ /bin/sh -c "cd ${WORKDIR_PATH}; $(env_vars_export_cmd) ${cmd_env_subst}" + + cache_img_checkpoint "${input_hash}" "${DEST_IMG}" } # HOST executed a command on the local host and can be used to prepare files,