Experimental Checkpoints

Does not work yet; #48.

.github/workflows/tests.yml

@@ -17,7 +17,7 @@ jobs:
           submodules: recursive
 
       - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install -y binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static unzip wget xz-utils units
+        run: sudo apt-get update && sudo apt-get install -y binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static units unzip wget xxhash xz-utils
         shell: bash
 
       - name: Run pimod OpenWRT example

Dockerfile

@@ -14,10 +14,11 @@ RUN apt-get update && \
   p7zip-full \
   qemu \
   qemu-user-static \
+  units \
   unzip \
   wget \
-  xz-utils \
-  units
+  xxhash \
+  xz-utils
 
 RUN mkdir /pimod
 COPY . /pimod/

README.md

@@ -47,7 +47,7 @@ Options:
 
 ### Debian
 ```bash
-sudo apt-get install binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static unzip wget xz-utils units
+sudo apt-get install binfmt-support fdisk file kpartx lsof p7zip-full qemu qemu-user-static units unzip wget xxhash xz-utils
 
 sudo ./pimod.sh Pifile
 ```

action.yml

@@ -14,7 +14,7 @@ runs:
   steps: 
     - run: sudo apt-get update
       shell: bash
-    - run: sudo apt-get install -y binfmt-support fdisk file kpartx qemu qemu-user-static unzip p7zip-full wget xz-utils units
+    - run: sudo apt-get install -y binfmt-support fdisk file kpartx qemu qemu-user-static units unzip p7zip-full wget xxhash xz-utils
       shell: bash
     - run: sudo ${{ github.action_path }}/pimod.sh ${{ inputs.pifile }}
       shell: bash

modules/from_remote.sh → modules/cache.sh

@@ -1,4 +1,4 @@
-if [ -z "${PIMOD_CACHE+x}" ]; then 
+if [ -z "${PIMOD_CACHE+x}" ]; then
   PIMOD_CACHE="/var/cache/pimod"
 fi
 
@@ -29,7 +29,7 @@ from_remote_fetch() {
   local url
   local url_path
   local download_path
-  
+
   url="${1}"
   url_path=$(echo "${url}" | sed 's/.*:\/\///')
   download_path="${PIMOD_CACHE}/${url_path}"
@@ -83,3 +83,49 @@ from_remote_fetch() {
   export SOURCE_IMG="${tmpfile}"
   export SOURCE_IMG_TMP=1
 }
+
+# cache_mk_hash calculates a hash for the tuple of an image and a command.
+# Usage: cache_mk_hash IMG_FILE "RUN whoami"
+cache_mk_hash() {
+  local img_hash
+  local cmd_hash
+
+  img_hash="$(xxh128sum "${1}" | awk '{ print $1 }')"
+  cmd_hash="$(xxh128sum - <<< "${2}" | awk '{ print $1 }')"
+
+  xxh128sum - <<< "${img_hash}${cmd_hash}" | awk '{ print $1 }'
+}
+
+# cache_get_path returns the path for a snapshot.
+# Usage: cache_get_path HASH
+cache_get_path() {
+  echo "${PIMOD_CACHE}/__checkpoint__/${1}"
+}
+
+# cache_img_checkpoint stores a checkpoint for the current state of an image.
+# Usage: cache_img_checkpoint HASH IMG
+cache_img_checkpoint() {
+  local cache_path
+  cache_path="$(cache_get_path "${1}")"
+
+  mkdir -p "$(dirname "${cache_path}")"
+  if [[ ! -f "${cache_path}" ]]; then
+    cp --reflink=auto "${2}" "${cache_path}"
+  fi
+}
+
+# cache_img_chk_load checks if a checkpoint does already exists and might loads
+# it. The function's exit code says if the cache was used (0) or not (1).
+# Usage: cache_img_chk_load HASH IMG
+cache_img_chk_load() {
+  local cache_path
+  cache_path="$(cache_get_path "${1}")"
+
+  echo ">>> cache_img_chk_load ${*}"
+
+  if [[ ! -f "${cache_path}" ]]; then
+    return 1
+  else
+    cp --reflink=auto "${cache_path}" "${2}"
+  fi
+}

pimod.sh

@@ -4,11 +4,11 @@ set -euE
 
 pushd "$(dirname "$0")" > /dev/null
 
+. ./modules/cache.sh
 . ./modules/chroot.sh
 . ./modules/env.sh
 . ./modules/error.sh
 . ./modules/esceval.sh
-. ./modules/from_remote.sh
 . ./modules/mount.sh
 . ./modules/path.sh
 . ./modules/pifile.sh
@@ -24,20 +24,23 @@ Usage: ${0} [Options] Pifile
 
 Options:
   -c cache  Define cache location.
-  -d        Debug on failure; run an interactive shell before tear down
+  -d        Debug on failure; run an interactive shell before tear down.
+  -s        Save cache checkpoints for each command.
   -t        Trace each executed command for debugging.
   -h        Print this help message.
 EOF
 }
 
-while getopts "c:dth" opt; do
+while getopts "c:dsth" opt; do
   case "${opt}" in
   c)
     PIMOD_CACHE="${OPTARG}"
     ;;
   d)
     PIMOD_DEBUG=1
     ;;
+  s)
+    ;;
   t)
     set -x
     ;;

stages/20-prepare.sh

@@ -9,6 +9,12 @@ PUMP() {
 
   echo -e "\033[0;32m### PUMP ${1}\033[0m"
 
+  local input_hash="$(cache_mk_hash "${DEST_IMG}" "${*}")"
+  if cache_img_chk_load "${input_hash}" "${DEST_IMG}"; then
+    echo "Using cached image..";
+    return
+  fi
+
   BS="1M"
 
   # units does not print to stderr, thus test call before using output
@@ -41,4 +47,6 @@ PUMP() {
   resize2fs "/dev/mapper/${loop}p${IMG_ROOT}"
 
   umount_image "${loop}"
+
+  cache_img_checkpoint "${input_hash}" "${DEST_IMG}"
 }

stages/30-chroot.sh

@@ -11,6 +11,12 @@ pre_stage() {
 # Usage: INSTALL [MODE] SOURCE DEST
 INSTALL() {
   echo -e "\033[0;32m### INSTALL $*\033[0m"
+
+  local input_hash="$(cache_mk_hash "${DEST_IMG}" "${*}")"
+  if cache_img_chk_load "${input_hash}" "${DEST_IMG}"; then
+    echo "Using cached image..";
+    return
+  fi
 
   local src=""
   local dst=""
@@ -41,6 +47,8 @@ INSTALL() {
   if [[ "$#" -eq "3" ]]; then
     chmod "$1" "${CHROOT_MOUNT}/${dst}"
   fi
+
+  cache_img_checkpoint "${input_hash}" "${DEST_IMG}"
 }
 
 # PATH adds the given path to an overlaying PATH variable, used within the RUN
@@ -103,11 +111,19 @@ ENV() {
 RUN() {
   echo -e "\033[0;32m### RUN ${*}\033[0m"
 
+  local input_hash="$(cache_mk_hash "${DEST_IMG}" "${*}")"
+  if cache_img_chk_load "${input_hash}" "${DEST_IMG}"; then
+    echo "Using cached image..";
+    return
+  fi
+
   local cmd_esceval="$(esceval "$@")"
   local cmd_env_subst="$(env_vars_subst "$cmd_esceval")"
 
   PATH=${GUEST_PATH} chroot "${CHROOT_MOUNT}" \
     /bin/sh -c "cd ${WORKDIR_PATH}; $(env_vars_export_cmd) ${cmd_env_subst}"
+
+  cache_img_checkpoint "${input_hash}" "${DEST_IMG}"
 }
 
 # HOST executed a command on the local host and can be used to prepare files,