diff --git a/bin/.helpers b/bin/.helpers index 4b67066..4c79638 100755 --- a/bin/.helpers +++ b/bin/.helpers @@ -161,18 +161,12 @@ function kill_benchbot() { # Kill specific processes within the container instead if it exists if [ -n "$(docker ps -q -f name=$HOSTNAME_ROBOT)" ]; then - kill_targets="$(docker exec benchbot_robot /bin/bash -c \ - 'ps aux | grep benchbot_robot_controller' | grep -v 'grep' | \ - awk '{ for (i=11; i<=NF; i++) { printf $i " " } print "(pid="$2")" }' \ - || true)" + # Use the supervisor to ask the simulator to stop printf "\n${colour_blue}%s${colour_nc}\n" \ - "Killing following targets in '$HOSTNAME_ROBOT' container:" - echo "$kill_targets" - if [ -n "$kill_targets" ]; then - echo "$kill_targets" | sed 's/.*pid=\(.*\)).*/\1/' | while read pid ; \ - do docker exec $HOSTNAME_ROBOT /bin/bash -c \ - 'kill -TERM '$pid'; while kill -0 '$pid' 2>/dev/null; do sleep 1; done'; done - fi + "Sending stop request to running controller:" + curl -sS -L "$HOSTNAME_ROBOT:$PORT_ROBOT/stop" + + # TODO some wait / success checking logic? fi fi if [ -n "$targets" ]; then diff --git a/bin/benchbot_run b/bin/benchbot_run index d56a0d6..3ff6fb7 100755 --- a/bin/benchbot_run +++ b/bin/benchbot_run @@ -14,13 +14,7 @@ source .helpers ########################### Script Specific Settings ########################### ################################################################################ -SIM_OMNI_PERSISTENT_CHK=$(printf "%s%s" \ - 'docker logs benchbot_robot 2>/dev/null | tail -n 100 | ' \ - 'grep -q "Omniverse System Monitor.*started"') - -SIM_OMNI_PERSISTENT_CMD=( - '/isaac-sim/start_nucleus.sh && sleep infinity' - ) +# None ################################################################################ ######################## Helper functions for commands ######################### @@ -71,6 +65,10 @@ OPTION DETAILS: novel work will focus on the latest software stack. You should only use this flag when it is inconvenient to update immediately. + -k, --kill-controller + Run a kill command that stops everything BenchBot currently + running, including the persistent robot controller. + --list-envs, --list-environments Search for & list all installed environments. The listed environment names are in the format needed for the '--env' option. @@ -172,40 +170,28 @@ log below for a dump of the crash output:" exit_code= kill_persist= function exit_gracefully() { - if [ -z "$simulator_required" ]; then + if [ "$simulator_required" -ne 0 ]; then printf "\n\n${colour_blue}%s${colour_nc}\n" \ "Re-closing network openings used for real robot:" close_network $network_forwarding $network_policy fi kill_benchbot "" $kill_persist xhost -local:root > /dev/null + trap '' SIGINT SIGQUIT SIGKILL SIGTERM EXIT exit ${exit_code:-0} } -function persist_chk() { - # $1 run type - if [ "$type" = "sim_omni" ]; then - echo "$SIM_OMNI_PERSISTENT_CHK" - fi -} - -function persist_cmd() { - # $1 run type - if [ "$type" = "sim_omni" ]; then - echo "$SIM_OMNI_PERSISTENT_CMD" - fi -} - ################################################################################ #################### Parse & handle command line arguments ##################### ################################################################################ # Safely parse options input -_args="help,env:,environment:,force-updateless,list-envs,list-environments,\ -list-formats,list-robots,list-tasks,robot:,show-env:,show-environment:,\ -show-format:,show-robot:,show-task:,task:,updates-check,version" -parse_out=$(getopt -o he:t:r:fuv --long $_args -n "$(basename "$abs_path")" \ +_args="help,env:,environment:,force-updateless,kill-controller,list-envs,\ +list-environments,list-formats,list-robots,list-tasks,robot:,show-env:,\ +show-environment:,show-format:,show-robot:,show-task:,task:,updates-check,\ +version" +parse_out=$(getopt -o he:t:r:fuvk --long $_args -n "$(basename "$abs_path")" \ -- "$@") if [ $? != 0 ]; then exit 1; fi eval set -- "$parse_out" @@ -222,6 +208,8 @@ while true; do environment="$2"; shift 2 ;; -f|--force-updateless) updates_skip=1 ; shift ;; + -k|--kill-controller) + kill_persist=0; simulator_required=0; exit_gracefully ;; --list-envs|--list-environments) list_environments "$_list_environments_pre" "an"; exit $? ;; --list-formats) @@ -263,10 +251,10 @@ if [ -z "$updates_exit" ]; then type="$(run_manager_cmd 'exists("robots", [("name", "'$robot'")]) and print(\ get_value_by_name("robots", "'$robot'", "type"))')" simulator_required=1 - if [[ "$type" != "sim_"* ]]; then simulator_required=0; fi + if [[ "$type" == "sim_"* ]]; then simulator_required=0; fi - # TODO this is logic is flaky as all hell... (see 'TODO DANGER' notice below) - [ -z "$(persist_cmd "$type")" ] && kill_persist=0 || kill_persist=1 + # TODO add an option for managing the persistent container + kill_persist=1 # Bail if any of the requested configurations are invalid validate_run_args "$robot" "$task" "$type" "$environment" "${environments[@]}" @@ -314,8 +302,8 @@ for i in "${!environments[@]}"; do "'${environments[$i]}'", "map_path"))')" done printf " %-22s" "Simulator required:" -printf "%s (%s)\n" $([ -z "$simulator_required" ] && echo "No" || echo "Yes") \ - "$type" +printf "%s (%s)\n" \ + $([ "$simulator_required" -ne 0 ] && echo "No" || echo "Yes") "$type" echo "" # Create the network for BenchBot software stack @@ -323,7 +311,7 @@ echo -e "${colour_blue}Creating shared network '$DOCKER_NETWORK':${colour_nc}" docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1 || \ docker network create "$DOCKER_NETWORK" --subnet="$URL_DOCKER_SUBNET" \ --ip-range="$URL_DOCKER_SUBNET" --gateway="$URL_DOCKER_GATEWAY" -if [ -z "$simulator_required" ]; then +if [ "$simulator_required" -ne 0 ]; then printf "\n${colour_blue}%s${colour_nc}\n" \ "Opening network to facilitate communications with real robot:" network_forwarding=$(cat /proc/sys/net/ipv4/conf/all/forwarding) @@ -352,7 +340,7 @@ cmd="${docker_run//'$name'/$ros_master_host}" ${cmd// /$'\t'} --ip "$URL_ROS" -d $DOCKER_TAG_BACKEND /bin/bash -c \ "$cmd_prefix"'roscore' -if [ -n "$simulator_required" ]; then +if [ "$simulator_required" -eq 0 ]; then # TODO would be nice to have a less stupid way to do this, but bash's lack of # nested arrays (and difficulties passing arrays in general from functions) # makes it hard... @@ -360,53 +348,20 @@ if [ -n "$simulator_required" ]; then if [ "$type" = "sim_omni" ]; then args=("${SIM_OMNI_ARGS[@]}") fi - persist_cmd="$(persist_cmd "$type")" - persist_chk="$(persist_chk "$type")" - - if [ -n "$persist_cmd" ]; then - # TODO DANGER: there are LOTS of ways this can go wrong. Need to do this - # more robustly if we ever expand outside Omniverse-only focus. Examples: - # - if next run uses a different simulator type, what do we do? - # - the persist variables are vary ad-hoc / hacky - # - it's really unclear to the user what is still running in background - # - our checks for the status of things are weak (how do we know if an exec - # has failed now???) - - # Run a persistent container and execute within it if our simulator has - # caching utilities (e.g. Cache in Omniverse) - printf "\n${colour_blue}%s${colour_nc}\n" \ - "Starting persistent container for BenchBot Robot Controller ($type):" - eval $persist_chk && loaded=0 || loaded=1 - if [ $loaded -eq 0 ]; then - printf "Skipping (already running)\n" - else - cmd="${docker_run//'$name'/benchbot_robot}" - ${cmd// /$'\t'} --ip "$URL_ROBOT" -d "${args[@]}" \ - -t $DOCKER_TAG_SIM_PREFIX$type /bin/bash -c $persist_cmd - fi - - printf "\n${colour_blue}%s${colour_nc}\n" \ - "Executing BenchBot Robot Controller in persistent container:" - while [ $loaded -eq 1 ]; do - eval $persist_chk && loaded=0 || loaded=1 - sleep 1 - done - - docker exec -d benchbot_robot /bin/bash -c \ - "$cmd_prefix"'rosrun benchbot_robot_controller benchbot_robot_controller' - controller_pid= - while [ -z "$controller_pid" ]; do - controller_pid="$(docker container top benchbot_robot | \ - grep 'benchbot_robot_controller' | grep -v 'grep' | awk '{print $2}' | \ - tr '\n' ',' | sed 's/,$//')" - sleep 1 - done - printf "Started in persistent container (pids = $controller_pid)\n" + # TODO DANGER: there are LOTS of ways this can go wrong. Need to do this + # more robustly if we ever expand outside Omniverse-only focus. Examples: + # - if next run uses a different simulator type, what do we do? + # - it's really unclear to the user what is still running in background + + # Run a persistent container and execute within it if our simulator has + # caching utilities (e.g. Cache in Omniverse) + printf "\n${colour_blue}%s${colour_nc}\n" \ + "Starting persistent container for BenchBot Robot Controller ($type):" + if [ "$(docker container inspect -f '{{.State.Running}}' \ + benchbot_robot 2> /dev/null)" == "true" ]; then + printf "Skipping (already running)\n" else - # Run a new container each time when there's no persistence requirement - printf "\n${colour_blue}%s${colour_nc}\n" \ - "Starting container for BenchBot Robot Controller ($type):" cmd="${docker_run//'$name'/benchbot_robot}" ${cmd// /$'\t'} --ip "$URL_ROBOT" -d "${args[@]}" \ -t $DOCKER_TAG_SIM_PREFIX$type /bin/bash -c \ @@ -431,13 +386,13 @@ header_block "BENCHBOT IS RUNNING (Ctrl^C to exit) ..." ${colour_green} docker logs --follow benchbot_supervisor & while [ -n "$(docker ps -q -f 'name=benchbot_supervisor')" ] && \ - ([ -z "$simulator_required" ] || \ + ([ "$simulator_required" -ne 0 ] || \ [ -n "$(docker ps -q -f 'name=benchbot_robot')" ]); do sleep 1 done # sleep infinity -if [ -n "$simulator_required" ] && \ +if [ "$simulator_required" -eq 0 ] && \ [ -z "$(docker ps -q -f 'name=benchbot_robot')" ]; then header_block "BENCHBOT ROBOT CONTROLLER ERROR" ${colour_red} echo -e "\n${colour_red}$_robot_err${colour_nc}\n"