_main

#!/usr/bin/env bash
# -*- mode: sh; indent-tabs-mode: nil; sh-basic-offset: 4 -*-
# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=bash

LOG_DB="${1}"
shift

if [ -z "${CRUCIBLE_HOME}" -o ! -e "${CRUCIBLE_HOME}" ]; then
    echo "ERROR: Could not find \${CRUCIBLE_HOME} [${CRUCIBLE_HOME}], exiting."
    exit 1
fi

if [ -z "${LOG_DB}" -o ! -e "${LOG_DB}" -a "$CRUCIBLE_USE_LOGGER" == "1" ]; then
    echo "ERROR: Could not find \${LOG_DB} [${LOG_DB}], exiting."
    exit 2
fi

. /etc/sysconfig/crucible
. $CRUCIBLE_HOME/bin/base

container_redis_args=()
container_redis_args+=("-d")
container_redis_args+=("--mount=type=bind,source=$CRUCIBLE_HOME/config/redis.conf,destination=/etc/redis.conf")

container_httpd_args=()
container_httpd_args+=("-d")
container_httpd_args+=("--mount=type=bind,source=$CRUCIBLE_HOME/config/httpd/,destination=/etc/httpd/")

container_es_args=()
container_es_args+=("-d")
#container_es_args+=("--mount=type=bind,source=$CRUCIBLE_HOME/config/elasticsearch/,destination=/etc/elasticsearch/")

container_rs_args=()
container_rs_args+=("-i")
container_rs_args+=("-e RS_REG_REPO=$CRUCIBLE_CLIENT_SERVER_REPO")
container_rs_args+=("-e RS_REG_AUTH=$CRUCIBLE_CLIENT_SERVER_AUTH")
container_rs_args+=("-e RS_REG_TLS_VERIFY=$CRUCIBLE_CLIENT_SERVER_TLS_VERIFY")
container_rs_args+=("-e RS_EMAIL=$CRUCIBLE_EMAIL")
container_rs_args+=("-e RS_NAME=\"$CRUCIBLE_NAME\"")

container_rs_run_args=()
container_rs_run_args+=("--mount=type=bind,source=/var/lib/containers,destination=/var/lib/containers")

function start_httpd() {
    local RC httpd_cmd
    httpd_cmd="/usr/sbin/httpd -DFOREGROUND"
    echo -n "Checking for httpd"
    if ! podman_running crucible-httpd; then
        firewall-cmd --zone=public --add-port=8080/tcp >/dev/null 2>&1
        $podman_run --name crucible-httpd "${container_common_args[@]}" "${container_httpd_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $httpd_cmd
        RC=$?
        if [ ${RC} != 0 ]; then
            echo "ERROR: Could not start httpd"
        fi
    fi
    return ${RC}
}

function start_redis() {
    local RC redis_cmd
    redis_cmd="redis-server /etc/redis.conf"
    echo -n "Checking for redis"
    if ! podman_running crucible-redis; then
        firewall-cmd --zone=public --add-port=6379/tcp >/dev/null 2>&1
        $podman_run --name crucible-redis "${container_common_args[@]}" "${container_redis_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $redis_cmd
        RC=$?
        if [ ${RC} != 0 ]; then
            echo "ERROR: Could not start redis"
        fi
    fi
    return ${RC}
}

function start_es() {
    local es_available count status_check es_timeout pod_name
    local es_start_cmd es_status_cmd common_container_args
    local RC
    es_start_cmd="${CRUCIBLE_HOME}/config/start-es.sh $var_crucible/es"
    es_status_cmd="curl --silent --show-error --stderr - -X GET localhost:9200/_cat/indices"
    echo -n "Checking for elasticsearch"
    if ! podman_running crucible-es; then
        firewall-cmd --zone=public --add-port=9200/tcp --add-port=9300/tcp >/dev/null 2>&1
        mkdir -p "$var_crucible/es"
        common_container_args=()
        for arg in "${container_common_args[@]}"; do
            # remove the hostfs mount from the ES container due to cgroup issues with ES/java
            if ! echo "${arg}" | grep -q "hostfs"; then
                common_container_args+=("${arg}")
            fi
        done
        $podman_run --name crucible-es "${common_container_args[@]}" "${container_es_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $es_start_cmd
        RC=$?
        if [ ${RC} != 0 ]; then
            echo "ERROR: Could not start elasticsearch"
        else
            es_available=0
            count=1
            es_timeout=60
            echo "Waiting for elasticsearch to be available (${es_timeout} seconds maximum)..."
            while [ ${es_available} != 1 -a ${count} -lt ${es_timeout} ]; do
                pod_name=crucible-es-status-check-${SESSION_ID}-$(uuidgen)
                status_check=$($podman_run --name ${pod_name} "${container_common_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $es_status_cmd)
                if echo "${status_check}" | grep -q "Connection refused"; then
                    sleep 1
                else
                    echo "Elasticsearch is online after ${count} seconds"
                    es_available=1
                fi
                (( count += 1 ))
            done
            if [ ${es_available} -eq 0 ]; then
                echo "ERROR: Elasticsearch failed to launch properly"
                RC=1
            else
                RC=0
            fi
        fi
    fi
    return ${RC}
}

function init_es() {
    local RC es_init_cmd
    es_init_cmd="${CRUCIBLE_HOME}/config/init-es.sh ${CRUCIBLE_HOME}/subprojects/core/CommonDataModel/templates"
    $podman_run --name crucible-init-es-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $es_init_cmd
    RC=$?
    if [ ${RC} != 0 ]; then
        echo "ERROR: Could not init elasticsearch"
    fi
    return ${RC}
}

function extract_run_id() {
    local RUN_DIR THIS_ID

    RUN_DIR=${1}
    shift

    if [ -e ${RUN_DIR} -a -d ${RUN_DIR} ]; then
        result_dumper=cat
        run_json="${RUN_DIR}/run/rickshaw-run.json"
        if [ ! -f $run_json ]; then
            result_dumper=xzcat
            run_json="${RUN_DIR}/run/rickshaw-run.json.xz"

            if [ ! -f $run_json ]; then
                echo "ERROR: Could not find a result json (ricksaw-run.json[.xz])"
                return 1
            fi
        fi

        THIS_ID=`$podman_run --name crucible-extract-run-id-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE /bin/bash -c "${result_dumper} ${run_json} | jq -r '. \"run-id\"'"`
        if [ -n "${THIS_ID}" -a "${THIS_ID}" != "null" ]; then
            echo "${THIS_ID}"
            return 0
        else
            THIS_ID=`$podman_run --name crucible-extract-run-id-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE /bin/bash -c "${result_dumper} ${run_json} | jq -r '. \"id\"'"`
            if [ -n "${THIS_ID}" -a "${THIS_ID}" != "null" ]; then
                echo "${THIS_ID}"
                return 0
            else
                echo "ERROR: Could not find run ID from rickshaw-result.json:, {.id} or {.run-id}"
                return 1
            fi
        fi
    else
        echo "ERROR: Could not find run directory at ${RUN_DIR}"
        return 1
    fi

    echo "ERROR: unknown error extracting run-id from ${RUN_DIR}"
    return 1
}

function post_process_run() {
    local rs_pp_b_cmd ps_pp_t_cmd rs_index_cmd cdm_query_cmd
    local run_json this_id RUN_DIR
    local post_process_fail RC result_summary

    RUN_DIR=${1}
    echo "Benchmark result is in ${RUN_DIR}"
    shift

    rs_pp_b_cmd="${CRUCIBLE_HOME}/subprojects/core/rickshaw/rickshaw-post-process-bench\
      --base-run-dir=${RUN_DIR}"
    rs_pp_t_cmd="${CRUCIBLE_HOME}/subprojects/core/rickshaw/rickshaw-post-process-tools\
      --base-run-dir=${RUN_DIR}"
    rs_index_cmd="${CRUCIBLE_HOME}/subprojects/core/rickshaw/rickshaw-index\
      --base-run-dir=${RUN_DIR}"

    start_httpd
    RC=$?
    if [ ${RC} != 0 ]; then
        return ${RC}
    fi

    start_es
    RC=$?
    if [ ${RC} != 0 ]; then
        return ${RC}
    fi

    post_process_fail=0
    $podman_run --name crucible-rickshaw-pp-bench-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $rs_pp_b_cmd
    RC=$?
    if [ ${RC} != 0 ]; then
        echo "ERROR: Failed to post-process the benchmark [rc=${RC}]"
        post_process_fail=1
    fi
    $podman_run --name crucible-rickshaw-pp-tools-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $rs_pp_t_cmd
    RC=$?
    if [ ${RC} != 0 ]; then
        echo "ERROR: Failed to post-process the tools [rc=${RC}]"
        post_process_fail=1
    fi
    if [ ${post_process_fail} == 1 ]; then
        return 1
    fi

    init_es
    RC=$?
    if [ ${RC} != 0 ]; then
        return ${RC}
    fi

    $podman_run --name crucible-rickshaw-index-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $rs_index_cmd
    RC=$?
    if [ ${RC} == 0 ]; then
        echo "Benchmark result now in elastic, localhost:9200"

        this_id=$(extract_run_id "${RUN_DIR}")
        RC=$?

        if [ ${RC} == 0 ]; then
            cdm_query_cmd="${CRUCIBLE_HOME}/subprojects/core/CommonDataModel/queries/cdmq/get-result-summary.sh"
            cdm_query_cmd+=" --run=${this_id}"
            echo "Generating benchmark summary report"
            result_summary="${RUN_DIR}/run/result-summary.txt"
            $podman_run --name crucible-get-result-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $cdm_query_cmd > ${result_summary}
            RC=$?
            cat ${result_summary}
            if [ ${RC} == 0 ]; then
                echo
                echo "Benchmark summary is complete and can be found in:"
                echo "${RUN_DIR}/run/result-summary.txt"
            else
                echo "ERROR: Could not generate benchmark summary"
                RC=1
            fi
        else
            echo "${this_id}"
        fi
    else
        echo "ERROR: Could not index result into elasticsearch [rc=${RC}]"
    fi

    return ${RC}
}

function unarchive_run() {
    local ARCHIVE ARCHIVE_FILE TMP_ARCHIVE_FILE LINK_ARCHIVE RUN_NAME UNARCHIVE_CMD
    ARCHIVE=${1}

    if ! echo "${ARCHIVE}" | grep -q "^${var_archive_crucible}"; then
        ARCHIVE_FILE="${var_archive_crucible}/${ARCHIVE}"
    else
        ARCHIVE_FILE="${ARCHIVE}"
    fi

    if [ ! -e "${ARCHIVE_FILE}" ]; then
        echo "ERROR: The specified archive does not exit (${ARCHIVE_FILE})"
        return 1
    fi

    if [ -d "${ARCHIVE_FILE}" ]; then
        echo "ERROR: The specified archive is a directory which does not make sense (${ARCHIVE_FILE})"
        return 1
    fi

    TMP_ARCHIVE_FILE=$(readlink -e ${ARCHIVE_FILE})
    LINK_CLEANUP=""
    if [ "${TMP_ARCHIVE_FILE}" != "${ARCHIVE_FILE}" ]; then
        echo "The specified archive is a symbolic link (${ARCHIVE_FILE} -> ${TMP_ARCHIVE_FILE})"
        LINK_CLEANUP=${ARCHIVE_FILE}
        ARCHIVE_FILE=${TMP_ARCHIVE_FILE}
    fi

    ARCHIVE_NAME=$(basename ${ARCHIVE_FILE})
    BASE_DIR=$(dirname ${ARCHIVE_FILE})
    RUN_NAME=$(echo "${ARCHIVE_NAME}" | sed -e "s/\.tar\.xz//")

    if [ -e "${var_run_crucible}/${RUN_NAME}" ]; then
        echo "ERROR: There is already a run result at ${var_run_crucible}/${RUN_NAME}"
        return 1
    fi

    UNARCHIVE_CMD="unxz --threads=0 --stdout ${ARCHIVE_FILE} | tar --extract --directory=${var_run_crucible}"

    echo "Unpacking archive:"
    $podman_run --name crucible-unarchive-${SESSION_ID} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} /bin/bash -c "${UNARCHIVE_CMD}"
    if [ $? == 0 ]; then
        echo "${ARCHIVE_FILE} successfully unpacked to ${var_run_crucible}/${RUN_NAME}"

        echo "Removing archived data:"
        echo -e "\t${ARCHIVE_NAME}"
        rm ${ARCHIVE_FILE}
        if [ $? != 0 ]; then
            return 1
        fi

        if [ -n "${LINK_CLEANUP}" ]; then
            echo -e "\t${LINK_CLEANUP} (symbolic link)"
            rm ${LINK_CLEANUP}
            if [ $? != 0 ]; then
                return 1
            fi
        fi
    else
        echo "ERROR: Failed to unpack ${ARCHIVE_FILE} to ${var_run_crucible}/${RUN_NAME}"
        return 1
    fi

    return 0
}

function archive_run() {
    local RUN RUN_DIR TMP_RUN_DIR LINK_CLEANUP RUN_NAME ARCHIVE_NAME ARCHIVE_CMD
    RUN=${1}

    if ! echo "${RUN}" | grep -q "^${var_run_crucible}"; then
        RUN_DIR="${var_run_crucible}/${RUN}"
    else
        RUN_DIR="${RUN}"
    fi

    if [ ! -e "${RUN_DIR}" ]; then
        echo "ERROR: The specified run does not exist (${RUN_DIR})"
        return 1
    fi

    if [ ! -d "${RUN_DIR}" ]; then
        echo "ERROR: The specified run is not a directory as expected (${RUN_DIR})"
        return 1
    fi

    TMP_RUN_DIR=$(readlink -e ${RUN_DIR})
    LINK_CLEANUP=""
    if [ "${TMP_RUN_DIR}" != "${RUN_DIR}" ]; then
        echo "The specified run directory is a symbolic link (${RUN_DIR} -> ${TMP_RUN_DIR})"
        LINK_CLEANUP=${RUN_DIR}
        RUN_DIR=${TMP_RUN_DIR}
    fi

    RUN_NAME=$(basename ${RUN_DIR})
    BASE_DIR=$(dirname ${RUN_DIR})
    ARCHIVE_NAME="${RUN_NAME}.tar.xz"

    if [ -e "${var_archive_crucible}/${ARCHIVE_NAME}" ]; then
        echo "ERROR: There is already an archive at ${var_archive_crucible}/${ARCHIVE_NAME}"
        return 1
    fi

    ARCHIVE_CMD="pushd ${BASE_DIR} > /dev/null && tar --create ${RUN_NAME} | xz --threads=0 --stdout > ${var_archive_crucible}/${ARCHIVE_NAME}"

    echo "Creating archive:"
    $podman_run --name crucible-archive-${SESSION_ID} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} /bin/bash -c "${ARCHIVE_CMD}"
    if [ $? == 0 ]; then
        echo "${RUN_DIR} successfully archived to ${var_archive_crucible}/${ARCHIVE_NAME}"

        echo "Removing non-archived data:"

        echo -e "\t${RUN_DIR}"
        rm -Rf "${RUN_DIR}"
        if [ $? != 0 ]; then
            return 1
        fi

        if [ -n "${LINK_CLEANUP}" ]; then
           echo -e "\t${LINK_CLEANUP} (symbolic link)"
           rm ${LINK_CLEANUP}
           if [ $? != 0 ]; then
               return 1
           fi
        fi
    else
        echo "ERROR: Failed to archive ${RUN_DIR} to ${var_archive_crucible}/${ARCHIVE_NAME}"
        return 1
    fi

    return 0
}

EXIT_VAL=0

if [ "${1}" == "log" ]; then
    shift
    crucible_log ${1} ${LOG_DB}
    EXIT_VAL=$?
elif [ "${1}" == "ls" -o "${1}" == "tags" -o "${1}" == "result-completion" ]; then
    result_process_cmd="${CRUCIBLE_HOME}/bin/result-processor.py"
    case "${1}" in
        ls|tags)
            $podman_run --name crucible-result-processor-${SESSION_ID} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} ${result_process_cmd} "$@"
            EXIT_VAL=$?
            ;;
        result-completion)
            shift
            $podman_run --name crucible-result-processor-${SESSION_ID} "${container_common_args[@]}" ${CRUCIBLE_CONTAINER_IMAGE} ${result_process_cmd} completion "$@"
            EXIT_VAL=$?
            ;;
    esac
elif [ "${1}" == "repo" ]; then
    shift
    if [ -z "${1}" ]; then
        ${CRUCIBLE_HOME}/bin/repo info
        EXIT_VAL=$?
    else
        ${CRUCIBLE_HOME}/bin/repo "$@"
        EXIT_VAL=$?
    fi
elif [ "${1}" == "update" ]; then
    shift
    if [ -z "${1}" ]; then
        ${CRUCIBLE_HOME}/bin/update all
        EXIT_VAL=$?
    else
        ${CRUCIBLE_HOME}/bin/update ${1}
        EXIT_VAL=$?
    fi
elif [ "${1}" == "start" ]; then
    shift
    if [ "${1}" == "httpd" ]; then
        shift
        start_httpd
        EXIT_VAL=$?
    elif [ "${1}" == "es" ]; then
        shift
        start_es
        EXIT_VAL=$?
    fi
elif [ "${1}" == "get" ]; then
    shift
    if start_httpd; then
        if ! start_es; then
            EXIT_VAL=$?
        fi
    else
        EXIT_VAL=$?
    fi
    if [ $EXIT_VAL == 0 ]; then
        if [ "${1}" == "result" ]; then
            shift
            get_result_cmd="${CRUCIBLE_HOME}/subprojects/core/CommonDataModel/queries/cdmq/get-result-summary.sh"
            $podman_run --name crucible-get-result-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $get_result_cmd "$@"
            EXIT_VAL=$?
        elif [ "${1}" == "metric" ]; then
            shift
            get_metric_cmd="${CRUCIBLE_HOME}/subprojects/core/CommonDataModel/queries/cdmq/get-metric-data.sh"
            $podman_run --name crucible-get-metric-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $get_metric_cmd "$@"
            EXIT_VAL=$?
        else
            echo "ERROR: unsupported 'get' argument [${1}]"
            EXIT_VAL=1
        fi
    fi
elif [ "${1}" == "rm" ]; then
    shift
    if start_es; then
       rm_result_cmd="${CRUCIBLE_HOME}/subprojects/core/CommonDataModel/queries/cdmq/delete-run.sh"
       $podman_run --name crucible-rm-result-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $rm_result_cmd "$@"
    fi
    EXIT_VAL=$?
elif [ "${1}" == "index" ]; then
    shift
    if start_es; then
        index_result_cmd="${CRUCIBLE_HOME}/subprojects/core/rickshaw/rickshaw-index"
        $podman_run --name crucible-rickshaw-index-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $index_result_cmd "$@"
    fi
    EXIT_VAL=$?
elif [ "${1}" == "extract" ]; then
    shift
    case "${1}" in
        run-id)
            shift
            this_id=$(extract_run_id "${1}")
            EXIT_VAL=$?
            echo "${this_id}"
            ;;
        *)
            echo "ERROR: Unknown extract command ${1}"
            EXIT_VAL=1
            ;;
    esac
elif [ "${1}" == "es" ]; then
    shift
    if start_es; then
        if [ "${1}" == "init" -o "${1}" == "rebuild" ]; then
            delete_es_cmd="${CRUCIBLE_HOME}/subprojects/core/CommonDataModel/templates/delete.sh"
            init_es_cmd="${CRUCIBLE_HOME}/subprojects/core/CommonDataModel/templates/init.sh"
            if $podman_run --name crucible-es-delete "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $delete_es_cmd; then
                if ! $podman_run --name crucible-es-init "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $init_es_cmd; then
                    EXIT_VAL=$?
                fi
            else
                EXIT_VAL=$?
            fi
        fi
        if [ ${EXIT_VAL} == 0 -a "${1}" == "rebuild" ]; then
            if pushd ${var_run_crucible} >/dev/null; then
                for base_run_dir in `/bin/ls | grep -v latest`; do
                    if [ -d "$base_run_dir" ]; then
                        echo "Going to post-process and index $base_run_dir"
                        post_process_run "${var_run_crucible}/${base_run_dir}"
                        RC=$?
                        if [ ${RC} != 0 ]; then
                            echo "ERROR: Failed to post-process ${base_run_dir} [rc=${RC}]"
                        fi
                    fi
                done
                popd >/dev/null
            else
                echo "ERROR: Could not chdir to ${var_run_crucible}"
                EXIT_VAL=1
            fi
        fi
    else
        EXIT_VAL=$?
    fi
elif [ "${1}" == "postprocess" ]; then
    shift
    base_run_dir=$(echo "$@" | sed -e "s/--base-run-dir\(\s\+\|=\)//")
    post_process_run "${base_run_dir}"
    EXIT_VAL=$?
elif [ "${1}" == "run" ]; then
    arg_list="$@"
    shift
    benchmark=${1}
    shift
    base_run_dir="${var_run_crucible}/${benchmark}--${datetime}--${SESSION_ID}"

    bench_params="bench-params.json"
    tool_params="tool-params.json"
    use_mv_params=0
    mv_params=""
    no_tools=0

    passthru_args=()
    while [ $# -gt 0 ]; do
        arg=${1}
        shift
        case "${arg}" in
            "--mv-params")
                val=${1}
                shift
                use_mv_params=1
                mv_params=${val}
                ;;
            "--tags")
                val=${1}
                shift
                passthru_args+=("${arg}")
                passthru_args+=("${val}")
                ;;
            "--bench-params")
                val=${1}
                shift
                bench_params=${val}
                ;;
            "--tool-params")
                val=${1}
                shift
                tool_params=${1}
                ;;
            "--no-tools")
                no_tools=1
                ;;
            *)
                passthru_args+=("${arg}")
                ;;
        esac
    done
    if [ -z "$CRUCIBLE_CLIENT_SERVER_REPO" ]; then
        exit_error "CRUCIBLE_CLIENT_SERVER_REPO is not defined"
    fi
    if [ -z "$CRUCIBLE_CONTAINER_IMAGE" ]; then
        exit_error "Exiting because CRUCIBLE_CONTAINER_IMAGE is not defined"
    fi
    mkdir -pv "$base_run_dir/config" >/dev/null
    if [ -e "${var_run_crucible}/latest" ]; then
        /bin/rm "${var_run_crucible}/latest"
    fi
    ln -sf "$base_run_dir" "${var_run_crucible}/latest"
    benchmark_subproj_dir="${CRUCIBLE_HOME}"/subprojects/benchmarks/$benchmark
    rs_dir="${CRUCIBLE_HOME}"/subprojects/core/rickshaw

    if [ ! -e "$benchmark_subproj_dir" ]; then
        echo "ERROR: Running benchmark ${benchmark} requires that the subproject be"
        echo "located in "${CRUCIBLE_HOME}"/subprojects/bench/$benchmark"
        echo "This directory could not be found.  Here are the benchmark"
        echo "subproject directories:"
        /bin/ls "${CRUCIBLE_HOME}"/subprojects/bench/$benchmark
        exit 1
    fi

    if [ ${use_mv_params} == 1 ]; then
        if [ ! -e "${mv_params}" ]; then
            exit_error "The multi-value params file you specified with --mv-params (${mv_params}) does not exist!"
        else
            echo "Generating --bench-params from --mv-params..."

            mv_params_run_dir=${base_run_dir}/config/mv-params.json
            cp ${mv_params} ${mv_params_run_dir}
            bench_params_run_file=${base_run_dir}/config/bench-params.json
            bench_params_run_output=${base_run_dir}/config/bench-params.txt

            multiplex_cmd="${CRUCIBLE_HOME}/subprojects/core/multiplex/multiplex.py --input ${mv_params_run_dir} --output ${bench_params_run_file}"
            if [ -e ${benchmark_subproj_dir}/multiplex.json ]; then
                multiplex_cmd+=" --requirements ${benchmark_subproj_dir}/multiplex.json"
            else
                echo "Skipping benchmark requirements validation because there is no ${benchmark_subproj_dir}/multiplex.json"
            fi

            multiplex_cmd="$podman_run -i --name crucible-multiplex-${SESSION_ID} "${container_common_args[@]}" $CRUCIBLE_CONTAINER_IMAGE ${multiplex_cmd}"
            echo "${multiplex_cmd} > ${bench_params_run_output} 2>&1"
            ${multiplex_cmd} > ${bench_params_run_output} 2>&1
            EXIT_VAL=$?
            if [ ${EXIT_VAL} != 0 ]; then
                echo "ERROR: multiplex failed with an error and returned rc=$rc"
                echo "multiplex output is:"
                cat ${bench_params_run_output}
                exit ${EXIT_VAL}
            fi

            bench_params=${bench_params_run_file}
        fi
    elif [ ! -e ${bench_params} ]; then
        echo "ERROR:"
        echo "Make sure you have defined the benchmark parameters and put them in a file \"./bench-params.json\""
        echo "or that you explicitly specify the benchmark parameters file with \"--bench-params=<file>\"."
        exit 1
    else
        bench_params_run_file=${base_run_dir}/config/bench-params.json
        cp ${bench_params} ${bench_params_run_file}
        bench_params=${bench_params_run_file}
    fi

    tool_params_file=${base_run_dir}/config/tool-params.json
    if [ "${no_tools}" == "0" ]; then
        if [ ! -e ${tool_params} ]; then
            cp $rs_dir/config/tool-params.json ${tool_params_file}
            echo "You do not have a \"tool-params.json\" in the current directory and have not explicitly"
            echo "specified where to find a tool parameters file with \"--tools-params=<file>\"."
            echo ""
            echo "Crucible will use the default tools found in $rs_dir/config/tool-params.json:"
            cat ${tool_params_file}
            echo ""
            echo "If you wish to use different tools, create a tool parameters file which adheres to"
            echo "the proper schema ($rs_dir/schema/tools/json) and either place that file in the"
            echo "current directory named \"tool-params.json\" or explicitly specify it with"
            echo "\"--tool-params=<file>\"."
            echo ""
            echo "You can also disable tools entirely using the \"--no-tools\" parameter."
        else
            cp ${tool_params} ${tool_params_file}
        fi
    else
        echo "Disabling all tools due to --no-tools"
        echo "[]" > ${tool_params_file}
    fi

    params_args=""
    params_args+=" --bench-params ${bench_params}"
    params_args+=" --tool-params ${tool_params_file}"

    start_redis
    EXIT_VAL=$?
    if [ ${EXIT_VAL} != 0 ]; then
        exit ${EXIT_VAL}
    fi

    ${CRUCIBLE_HOME}/bin/repo info > ${base_run_dir}/config/crucible.repo.info
    ${CRUCIBLE_HOME}/bin/repo details > ${base_run_dir}/config/crucible.repo.details

    rs_run_cmd="${CRUCIBLE_HOME}/subprojects/core/rickshaw/rickshaw-run\
      ${params_args}\
      --id ${SESSION_ID}\
      --bench-dir $benchmark_subproj_dir\
      --engine-dir=${CRUCIBLE_HOME}/subprojects/core/engine\
      --roadblock-dir=${CRUCIBLE_HOME}/subprojects/core/roadblock\
      --workshop-dir=${CRUCIBLE_HOME}/subprojects/core/workshop\
      --packrat-dir=${CRUCIBLE_HOME}/subprojects/core/packrat\
      --tools-dir=${CRUCIBLE_HOME}/subprojects/tools\
      --base-run-dir=$base_run_dir\
      ${passthru_args[@]}"
    $podman_run --name crucible-rickshaw-run-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" "${container_rs_run_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $rs_run_cmd
    RC=$?

    if [ ${RC} == 0 ]; then
        post_process_run ${base_run_dir}
        EXIT_VAL=$?
    else
        echo "Skipping run post-processing due to error(s) [rc=${RC}]"
        EXIT_VAL=${RC}
    fi

    if [ "$CRUCIBLE_USE_LOGGER" == "1" ]; then
        sleep 1
        logfile="${base_run_dir}/crucible.log.xz"
        echo "Archiving crucible log to ${logfile}"
        crucible_log view ${LOG_DB} sessionid ${SESSION_ID} | xz -9 -T0 > ${logfile}
        PIPES_RCS=$(echo ${PIPESTATUS[@]})
        RC=0
        for PIPE_RC in ${PIPES_RC}; do
            RC=$(( ${RC} + ${PIPE_RC} ))
        done
        if [ ${EXIT_VAL} == 0 ]; then
            EXIT_VAL=${RC}
        fi
    fi

    send_notification "Finished run command '${arg_list}' with return value '${EXIT_VAL}'.  Results are available in '${base_run_dir}'."
    RC=$?
    if [ ${EXIT_VAL} == 0 ]; then
        EXIT_VAL=${RC}
    fi
elif [ "${1}" == "notify" ]; then
    shift
    ntfy_doc_url="https://github.com/dschep/ntfy"
    if [ "${1}" == "config" ]; then
        shift
        echo "Currently notification configuration is a manual process where the user must create ${USER_STORE}/ntfy.yml with at least one backend.  See ${ntfy_doc_url} for details."
        EXIT_VAL=0
    elif [ "${1}" == "test" ]; then
        shift

        if [ ! -e ${USER_STORE}/ntfy.yml ]; then
            echo "ERROR: You must configure at least one notification backend in ${USER_STORE}/ntfy.yml.  See ${ntfy_doc_url} for details."
            EXIT_VAL=1
        else
            send_notification "notification test"
            EXIT_VAL=$?
        fi
    else
        echo "ERROR: unsupported 'notify' argument [${1}]"
        EXIT_VAL=1
    fi
elif [ "${1}" == "archive" ]; then
    shift
    while [ $# -gt 0 -a ${EXIT_VAL} == 0 ]; do
        archive_run "${1}"
        EXIT_VAL=$?
        shift
    done
elif [ "${1}" == "unarchive" ]; then
    shift
    while [ $# -gt 0 -a ${EXIT_VAL} == 0 ]; do
        unarchive_run "${1}"
        EXIT_VAL=$?
        shift
    done
elif [ "${1}" == "wrapper" ]; then
    shift
    $podman_run --name crucible-wrapper-${SESSION_ID} "${container_common_args[@]}" "${container_rs_args[@]}" $CRUCIBLE_CONTAINER_IMAGE $@
    EXIT_VAL=$?
else
    echo "ERROR: This function has not been implemented"
    EXIT_VAL=1
fi

exit ${EXIT_VAL}