From 1d917123883b2c3d34c7030448d14059c070e3d7 Mon Sep 17 00:00:00 2001 From: Nathan Nguyen Date: Thu, 12 Oct 2023 15:08:37 +1000 Subject: [PATCH] fix: resolve compatibility issues with podman --- scripts/release_scripts/run_macaron.sh | 238 ++++++++++++------------- 1 file changed, 119 insertions(+), 119 deletions(-) diff --git a/scripts/release_scripts/run_macaron.sh b/scripts/release_scripts/run_macaron.sh index e5d44c6fd..a897a5b3a 100755 --- a/scripts/release_scripts/run_macaron.sh +++ b/scripts/release_scripts/run_macaron.sh @@ -5,7 +5,30 @@ # This script runs the Macaron Docker image. -if [[ -z ${MACARON_IMAGE_TAG} ]]; then +# Strict bash options. +# +# -e: exit immediately if a command fails (with non-zero return code), +# or if a function returns non-zero. +# +# -u: treat unset variables and parameters as error when performing +# parameter expansion. +# In case a variable ${VAR} is unset but we still need to expand, +# use the syntax ${VAR:-} to expand it to an empty string. +# +# -o pipefail: set the return value of a pipeline to the value of the last +# (rightmost) command to exit with a non-zero status, or zero +# if all commands in the pipeline exit successfully. +# +# Reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html. +set -euo pipefail + +# Determine the docker executable. +if [[ -z ${DOCKER_EXEC:-} ]]; then + DOCKER_EXEC="docker" +fi + +# Determine the Macaron image tag. +if [[ -z ${MACARON_IMAGE_TAG:-} ]]; then MACARON_IMAGE_TAG="latest" fi @@ -40,6 +63,11 @@ mounts=() # The proxy values obtained from the host environment. proxy_vars=() +# Write error (to stderr). +log_err() { + echo "[ERROR]: $*" >&2 +} + # Convert a path to absolute path if it is a relative path. # # Arguments: @@ -54,34 +82,33 @@ function ensure_absolute_path() { fi } -# Ensure a directory exists. -# This method is important since we want to ensure that all docker mounts works -# properly. If we mount a non-existing host directory into the container, docker -# creates an empty directory owned by root, which is not what we really want. +# Ensure a directory exists by creating it if it does not. +# +# This function is for use with volume mounts to ensure compatibility with +# both docker and podman, by always creating the directory on host if it does +# not exist before mounting. +# - docker's behavior: If we mount a non-existing host directory into a +# container, docker creates an empty directory owned by root. +# - podman's behavior: If we mount a non-existing host directory into a +# container, podman errors. +# See: https://github.com/containers/podman/issues/6234. # # Arguments: # $1: The path to the directory. -# Outputs: -# STDOUT: Error message if the directory does not exist; empty string string otherwise. -function check_dir_exists() { +function create_dir_if_not_exists() { if [[ ! -d "$1" ]]; then - echo "[ERROR] Directory $1 of argument $2 does not exist." - else - echo "" + mkdir --parents "$1" fi } -# Ensure a file exists. +# Check if a file exists. # # Arguments: # $1: The path to the file. -# Outputs: -# STDOUT: Error message if the directory does not exist; empty string string otherwise. function check_file_exists() { if [[ ! -f "$1" ]]; then - echo "[ERROR] File $1 of argument $2 does not exist." - else - echo "" + log_err "File $1 of argument $2 does not exist." + return 1 fi } @@ -93,12 +120,31 @@ function check_file_exists() { # STDOUT: Error message if the file or directory does not exist; empty string string otherwise. function check_path_exists() { if [[ ! -s "$1" ]]; then - echo "[ERROR] $1 of argument $2 is neither file nor directory." - else - echo "" + log_err "$1 of argument $2 is neither a file nor a directory." + return 1 fi } +# Add a directory to the list of volume mounts. +# +# Arguments: +# $1: The path to the host directory. +# $2: The path to the directory inside the container. +# $3: Mount options. +# Globals: +# mounts: the volume mount is added to this array +function mount_dir() { + host_dir=$1 + container_dir=$2 + mount_options=$3 + + # This ensures compatibility with podman, as podman does not create the + # host directory if it does not exist. + create_dir_if_not_exists "$host_dir" + host_dir=$(ensure_absolute_path "$host_dir") + mounts+=("-v" "${host_dir}:${container_dir}:${mount_options}") +} + # Parse main arguments. while [[ $# -gt 0 ]]; do case $1 in @@ -179,136 +225,93 @@ fi # MACARON entrypoint - Main argvs # Determine the output path to be mounted into ${MACARON_WORKSPACE}/output/ -if [[ -n "${arg_output}" ]]; then +if [[ -n "${arg_output:-}" ]]; then output="${arg_output}" - err=$(check_dir_exists "${output}" "-o/--output") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi argv_main+=("--output" "${MACARON_WORKSPACE}/output/") else output=$(pwd)/output echo "Setting default output directory to ${output}." fi -if [[ -n "${output}" ]]; then - output="$(ensure_absolute_path "${output}")" - # Mounting the necessary .m2 and .gradle directories. - m2_dir="${output}/.m2" - gradle_dir="${output}/.gradle" - mounts+=("-v" "${output}:${MACARON_WORKSPACE}/output:rw,Z") - mounts+=("-v" "${m2_dir}:${MACARON_WORKSPACE}/.m2:rw,Z") - mounts+=("-v" "${gradle_dir}:${MACARON_WORKSPACE}/.gradle:rw,Z") -fi + +output="$(ensure_absolute_path "${output}")" +# Mount the necessary .m2 and .gradle directories. +m2_dir="${output}/.m2" +gradle_dir="${output}/.gradle" + +mount_dir "$output" "${MACARON_WORKSPACE}/output" "rw,Z" +mount_dir "$m2_dir" "${MACARON_WORKSPACE}/.m2" "rw,Z" +mount_dir "$gradle_dir" "${MACARON_WORKSPACE}/.gradle" "rw,Z" # Determine the local repos path to be mounted into ${MACARON_WORKSPACE}/output/git_repos/local_repos/ -if [[ -n "${arg_local_repos_path}" ]]; then - local_repos_path="${arg_local_repos_path}" - err=$(check_dir_exists "${local_repos_path}" "-lr/--local-repos-path") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi - argv_main+=("--local-repos-path" "${MACARON_WORKSPACE}/output/git_repos/local_repos/") -fi -if [[ -n "${local_repos_path}" ]]; then - local_repos_path="$(ensure_absolute_path "${local_repos_path}")" - mounts+=("-v" "${local_repos_path}:${MACARON_WORKSPACE}/output/git_repos/local_repos/:rw,Z") +if [[ -n "${arg_local_repos_path:-}" ]]; then + container_local_repo_path="${MACARON_WORKSPACE}/output/git_repos/local_repos" + argv_main+=("--local-repos-path" "$container_local_repo_path") + mount_dir "$arg_local_repos_path" "$container_local_repo_path" "rw,Z" fi # Determine the defaults path to be mounted into ${MACARON_WORKSPACE}/defaults/${file_name} -if [[ -n "${arg_defaults_path}" ]]; then +if [[ -n "${arg_defaults_path:-}" ]]; then defaults_path="${arg_defaults_path}" - err=$(check_file_exists "${defaults_path}" "-dp/--defaults-path") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_file_exists "${defaults_path}" "-dp/--defaults-path" file_name="$(basename "${defaults_path}")" argv_main+=("--defaults-path" "${MACARON_WORKSPACE}/defaults/${file_name}") -fi -if [[ -n "${defaults_path}" ]]; then + defaults_path="$(ensure_absolute_path "${defaults_path}")" mounts+=("-v" "${defaults_path}:${MACARON_WORKSPACE}/defaults/${file_name}:ro") fi # Determine the policy path to be mounted into ${MACARON_WORKSPACE}/policy/${file_name} -if [[ -n "${arg_policy}" ]]; then +if [[ -n "${arg_policy:-}" ]]; then policy="${arg_policy}" - err=$(check_file_exists "${policy}" "-po/--policy") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_file_exists "${policy}" "-po/--policy" file_name="$(basename "${policy}")" argv_main+=("--policy" "${MACARON_WORKSPACE}/policy/${file_name}") -fi -if [[ -n "${policy}" ]]; then + policy="$(ensure_absolute_path "${policy}")" mounts+=("-v" "${policy}:${MACARON_WORKSPACE}/policy/${file_name}:ro") fi # MACARON entrypoint - Analyze action argvs # Determine the template path to be mounted into ${MACARON_WORKSPACE}/template/${file_name} -if [[ -n "${arg_template_path}" ]]; then +if [[ -n "${arg_template_path:-}" ]]; then template_path="${arg_template_path}" - err=$(check_file_exists "${template_path}" "-g/--template-path") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_file_exists "${template_path}" "-g/--template-path" file_name="$(basename "${template_path}")" argv_action+=("--template-path" "${MACARON_WORKSPACE}/template/${file_name}") -fi -if [[ -n "${template_path}" ]]; then + template_path="$(ensure_absolute_path "${template_path}")" mounts+=("-v" "${template_path}:${MACARON_WORKSPACE}/template/${file_name}:ro") fi # Determine the config path to be mounted into ${MACARON_WORKSPACE}/config/${file_name} -if [[ -n "${arg_config_path}" ]]; then +if [[ -n "${arg_config_path:-}" ]]; then config_path="${arg_config_path}" - err=$(check_file_exists "${config_path}" "-c/--config-path") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_file_exists "${config_path}" "-c/--config-path" file_name="$(basename "${config_path}")" argv_action+=("--config-path" "${MACARON_WORKSPACE}/config/${file_name}") -fi -if [[ -n "${config_path}" ]]; then + config_path="$(ensure_absolute_path "${config_path}")" mounts+=("-v" "${config_path}:${MACARON_WORKSPACE}/config/${file_name}:ro") fi # Determine the sbom path to be mounted into ${MACARON_WORKSPACE}/sbom/${file_name} -if [[ -n "${arg_sbom_path}" ]]; then +if [[ -n "${arg_sbom_path:-}" ]]; then sbom_path="${arg_sbom_path}" - err=$(check_file_exists "${sbom_path}" "-sbom/--sbom-path") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_file_exists "${sbom_path}" "-sbom/--sbom-path" file_name="$(basename "${sbom_path}")" argv_action+=("--sbom-path" "${MACARON_WORKSPACE}/sbom/${file_name}") -fi -if [[ -n "${sbom_path}" ]]; then + sbom_path="$(ensure_absolute_path "${sbom_path}")" mounts+=("-v" "${sbom_path}:${MACARON_WORKSPACE}/sbom/${file_name}:ro") fi # Determine the provenance expectation path to be mounted into ${MACARON_WORKSPACE}/prov_expectations/${file_name} -if [[ -n "${arg_prov_exp}" ]]; then +if [[ -n "${arg_prov_exp:-}" ]]; then prov_exp="${arg_prov_exp}" - err=$(check_path_exists "${prov_exp}" "-pe/--provenance-expectation") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_path_exists "${prov_exp}" "-pe/--provenance-expectation" pe_name="$(basename "${prov_exp}")" argv_action+=("--provenance-expectation" "${MACARON_WORKSPACE}/prov_expectations/${pe_name}") -fi -if [[ -n "${prov_exp}" ]]; then + prov_exp="$(ensure_absolute_path "${prov_exp}")" mounts+=("-v" "${prov_exp}:${MACARON_WORKSPACE}/prov_expectations/${pe_name}:ro") fi @@ -316,33 +319,23 @@ fi # MACARON entrypoint - verify-policy action argvs # This is for macaron verify-policy action. # Determine the database path to be mounted into ${MACARON_WORKSPACE}/database/macaron.db -if [[ -n "${arg_database}" ]]; then +if [[ -n "${arg_database:-}" ]]; then database="${arg_database}" - err=$(check_file_exists "${database}" "-d/--database") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_file_exists "${database}" "-d/--database" file_name="$(basename "${database}")" argv_action+=("--database" "${MACARON_WORKSPACE}/database/${file_name}") -fi -if [[ -n "${database}" ]]; then + database="$(ensure_absolute_path "${database}")" mounts+=("-v" "${database}:${MACARON_WORKSPACE}/database/${file_name}:rw,Z") fi # Determine the Datalog policy to be verified by verify-policy action. -if [[ -n "${arg_datalog_policy_file}" ]]; then +if [[ -n "${arg_datalog_policy_file:-}" ]]; then datalog_policy_file="${arg_datalog_policy_file}" - err=$(check_file_exists "${datalog_policy_file}" "-f/--file") - if [[ -n "${err}" ]]; then - echo "${err}" - exit 1 - fi + check_file_exists "${datalog_policy_file}" "-f/--file" file_name="$(basename "${datalog_policy_file}")" argv_action+=("--file" "${MACARON_WORKSPACE}/policy/${file_name}") -fi -if [[ -n "${datalog_policy_file}" ]]; then + datalog_policy_file="$(ensure_absolute_path "${datalog_policy_file}")" mounts+=("-v" "${datalog_policy_file}:${MACARON_WORKSPACE}/policy/${file_name}:ro") fi @@ -376,7 +369,7 @@ proxy_var_names=( ) for v in "${proxy_var_names[@]}"; do - [[ -n ${!v} ]] && proxy_vars+=("-e" "${v}=${!v}") + [[ -n ${!v:-} ]] && proxy_vars+=("-e" "${v}=${!v}") done prod_vars=( @@ -400,7 +393,7 @@ then entrypoint=("macaron") fi -if [[ -n "${DOCKER_PULL}" ]]; then +if [[ -n "${DOCKER_PULL:-}" ]]; then if [[ "${DOCKER_PULL}" != @(always|missing|never) ]]; then echo "DOCKER_PULL must be one of: always, missing, never (default: always)" exit 1 @@ -423,20 +416,27 @@ macaron_args=( # env var `MCN_DEBUG_ARGS=1`. # In this case, the script will just print the arguments to stderr without # running the Macaron container. -if [[ -n ${MCN_DEBUG_ARGS} ]]; then +if [[ -n ${MCN_DEBUG_ARGS:-} ]]; then >&2 echo "${macaron_args[@]}" exit 0 fi -docker run \ - --pull ${DOCKER_PULL} \ +# By default +# - docker maps the host user $UID to a user with the same $UID in the container. +# - podman maps the host user $UID to the root user in the container. +# To make podman behave similarly to docker, we need to set the following env var. +# Reference: https://docs.podman.io/en/v4.4/markdown/options/userns.container.html. +export PODMAN_USERNS=keep-id + +${DOCKER_EXEC} run \ + --pull "${DOCKER_PULL}" \ --network=host \ --rm -i "${tty[@]}" \ -e "USER_UID=${USER_UID}" \ -e "USER_GID=${USER_GID}" \ - -e "GITHUB_TOKEN=${GITHUB_TOKEN}" \ - -e "MCN_GITLAB_TOKEN=${MCN_GITLAB_TOKEN}" \ - -e "MCN_SELF_HOSTED_GITLAB_TOKEN=${MCN_SELF_HOSTED_GITLAB_TOKEN}" \ + -e "GITHUB_TOKEN=${GITHUB_TOKEN:-}" \ + -e "MCN_GITLAB_TOKEN=${MCN_GITLAB_TOKEN:-}" \ + -e "MCN_SELF_HOSTED_GITLAB_TOKEN=${MCN_SELF_HOSTED_GITLAB_TOKEN:-}" \ "${proxy_vars[@]}" \ "${prod_vars[@]}" \ "${mounts[@]}" \