Merge branch 'develop'

bin/.helpers

@@ -58,8 +58,18 @@ RETIRED_SIMULATORS=(
   "sim_unreal:Isaac Sim 2019.2, powered by Unreal engine"
   )
 
-SIZE_GB_FULL=64
-SIZE_GB_LITE=32
+SIZE_GB_FULL=32
+SIZE_GB_LITE=20
+
+SIM_OMNI_ARGS=(
+  -v $PATH_CACHE/isaac-sim/cache/ov:/root/.cache/ov:rw
+  -v $PATH_CACHE/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw
+  -v $PATH_CACHE/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw
+  -v $PATH_CACHE/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw
+  -v $PATH_CACHE/isaac-sim/config:/root/.nvidia-omniverse/config:rw
+  -v $PATH_CACHE/isaac-sim/data:/root/.local/share/ov/data:rw
+  -v $PATH_CACHE/isaac-sim/documents:/root/Documents:rw
+)
 
 SUPPORTED_SIMULATORS=(
   "sim_omni:The latest Isaac Sim, powered by Omniverse"
@@ -127,8 +137,10 @@ function eval_version() {
 }
 
 function kill_benchbot() {
+  # $1 set this to run without header block
+  # $2 binary flag on whether we should keep persistent containers
   # TODO make this quieter when I am confident it works as expected...
-  if [ -z "${1-}" ]; then
+  if [ -z "${1:-}" ]; then
     header_block "CLEANING UP ALL BENCHBOT REMNANTS" ${colour_blue}
   fi
 
@@ -140,6 +152,29 @@ function kill_benchbot() {
   fi
 
   targets=$(docker ps -q -f name='benchbot*' || true)
+  if [ "${2:-0}" -ne 0 ]; then
+    # Remove benchbot_robot container from our "stop list"
+    p="$(docker ps -q -f name=$HOSTNAME_ROBOT)"
+    if [ -n "$p" ]; then
+      targets="$(echo "$targets" | grep -v "$p" || true)"
+    fi
+
+    # Kill specific processes within the container instead if it exists
+    if [ -n "$(docker ps -q -f name=$HOSTNAME_ROBOT)" ]; then
+      kill_targets="$(docker exec benchbot_robot /bin/bash -c \
+        'ps aux | grep benchbot_robot_controller' | grep -v 'grep' | \
+        awk '{ for (i=11; i<=NF; i++) { printf $i " " } print "(pid="$2")" }' \
+        || true)"
+      printf "\n${colour_blue}%s${colour_nc}\n" \
+        "Killing following targets in '$HOSTNAME_ROBOT' container:"
+      echo "$kill_targets"
+      if [ -n "$kill_targets" ]; then
+        echo "$kill_targets" | sed 's/.*pid=\(.*\)).*/\1/' | while read pid ; \
+          do docker exec $HOSTNAME_ROBOT /bin/bash -c \
+          'kill -TERM '$pid'; while kill -0 '$pid' 2>/dev/null; do sleep 1; done'; done
+      fi
+    fi
+  fi
   if [ -n "$targets" ]; then
     echo -e "\n${colour_blue}Stopped the following containers:${colour_nc}"
     docker stop $targets

bin/benchbot_install

@@ -168,7 +168,8 @@ checks_list_pre=(
   "fsspace"  # Is /var/lib/docker on a filesystem with enough space?
   "Miscellaneous requirements:"
   "pip" # Is pip available? (required to install Python libraries)
-  "pythontk" # Is python-tk installed (required for matplotlib...)
+  "pythontk" # Is python3-tk installed (required for matplotlib...)
+  "pythonpil" # Is python3-pil & python3-pil.imagetk installed?
   )
 
 chk_ubuntu1804_name='Ubuntu version >= 18.04'
@@ -554,12 +555,24 @@ chk_pythontk_fail='Not found'
 chk_pythontk_check='python3 -c "import tkinter as tk; print(tk.__file__)" 2>/dev/null'
 chk_pythontk_eval='[ -n "$check_result" ]'
 chk_pythontk_issue="\
-  Python package pythontk was not found. It is required to use the
+  Python package python3-tk was not found. It is required to use the
   visualisation tools available with the BenchBot API. Please install it via
   package manager. "
 chk_pythontk_fix='sudo apt install -y python3-tk'
 chk_pythontk_reboot=1
 
+chk_pythonpil_name='PIL (with ImageTk) for Python installed'
+chk_pythonpil_pass='Found'
+chk_pythonpil_fail='Not found'
+chk_pythonpil_check='python3 -c "from PIL import ImageTk; print("1")" 2>/dev/null'
+chk_pythonpil_eval='[ -n "$check_result" ]'
+chk_pythonpil_issue="\
+  Python packages python3-pil & python3-pil.imagetk were not found. They are
+  required to use the visualisation tools available with the BenchBot API.
+  Please install it via package manager. "
+chk_pythonpil_fix='sudo apt install -y python3-pil python3-pil.imagetk'
+chk_pythonpil_reboot=1
+
 
 checks_list_sim_omni=(
   "Manual installation steps for Omniverse-powered Isaac Sim:"
@@ -1250,6 +1263,24 @@ for i in "${!simulators[@]}"; do
       "$build_ret"
     exit 1
   fi
+
+  # Pre-baking (TODO need a way to generalise this... at the moment it is very
+  # ad-hoc)
+  if [ "$s" = "sim_omni" ]; then
+    printf "\n${colour_blue}%s${colour_nc}\n" \
+      "BAKING SIMULATOR DATA INTO '$s' DOCKER IMAGE:"
+    docker container rm tmp >/dev/null 2>&1 || true
+    xhost +local:root > /dev/null
+    docker run "${SIM_OMNI_ARGS[@]}" --gpus all --name tmp \
+      --env DISPLAY --volume /tmp/.X11-unix:/tmp/.X11-unix \
+      -t $DOCKER_TAG_SIM_PREFIX$s /bin/bash -c "$(printf '%s %s' \
+      "./python.sh -c 'from omni.isaac.kit import SimulationApp; " \
+      "k = SimulationApp({\"headless\": False}); k.close()'"
+      )"
+    xhost -local:root > /dev/null
+    docker commit --change "ENV SIM_BAKED=1" tmp $DOCKER_TAG_SIM_PREFIX$s
+    docker container rm tmp
+  fi
 done
 
 # Build the BenchBot Submission Docker image

bin/benchbot_run

@@ -14,15 +14,13 @@ source .helpers
 ########################### Script Specific Settings ###########################
 ################################################################################
 
-SIM_OMNI_ARGS=(
-  -v $PATH_CACHE/isaac-sim/cache/ov:/root/.cache/ov:rw
-  -v $PATH_CACHE/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw
-  -v $PATH_CACHE/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw
-  -v $PATH_CACHE/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw
-  -v $PATH_CACHE/isaac-sim/config:/root/.nvidia-omniverse/config:rw
-  -v $PATH_CACHE/isaac-sim/data:/root/.local/share/ov/data:rw
-  -v $PATH_CACHE/isaac-sim/documents:/root/Documents:rw
-)
+SIM_OMNI_PERSISTENT_CHK=$(printf "%s%s" \
+  'docker logs benchbot_robot 2>/dev/null | tail -n 100 | ' \
+  'grep -q "Omniverse System Monitor.*started"')
+
+SIM_OMNI_PERSISTENT_CMD=(
+  '/isaac-sim/start_nucleus.sh && sleep infinity'
+  )
 
 ################################################################################
 ######################## Helper functions for commands #########################
@@ -172,17 +170,32 @@ should not happen under normal operating conditions. Please see the complete
 log below for a dump of the crash output:"
 
 exit_code=
+kill_persist=
 function exit_gracefully() {
   if [ -z "$simulator_required" ]; then
     printf "\n\n${colour_blue}%s${colour_nc}\n" \
       "Re-closing network openings used for real robot:"
     close_network $network_forwarding $network_policy
   fi
-  kill_benchbot
+  kill_benchbot "" $kill_persist
   xhost -local:root > /dev/null
   exit ${exit_code:-0}
 }
 
+function persist_chk() {
+  # $1 run type
+  if [ "$type" = "sim_omni" ]; then
+    echo "$SIM_OMNI_PERSISTENT_CHK"
+  fi
+}
+
+function persist_cmd() {
+  # $1 run type
+  if [ "$type" = "sim_omni" ]; then
+    echo "$SIM_OMNI_PERSISTENT_CMD"
+  fi
+}
+
 
 ################################################################################
 #################### Parse & handle command line arguments #####################
@@ -252,6 +265,9 @@ if [ -z "$updates_exit" ]; then
   simulator_required=1
   if [[ "$type" != "sim_"* ]]; then simulator_required=0; fi
 
+  # TODO this is logic is flaky as all hell... (see 'TODO DANGER' notice below)
+  [ -z "$(persist_cmd "$type")" ] && kill_persist=0 || kill_persist=1
+
   # Bail if any of the requested configurations are invalid
   validate_run_args "$robot" "$task" "$type" "$environment" "${environments[@]}"
 fi
@@ -272,7 +288,7 @@ fi
 if [ -n "$updates_exit" ]; then exit 0; fi
 
 # Run the BenchBot software stack (kill whenever they exit)
-kill_benchbot
+kill_benchbot "" $kill_persist
 trap exit_gracefully SIGINT SIGQUIT SIGKILL SIGTERM EXIT
 header_block "STARTING THE BENCHBOT SOFTWARE STACK" ${colour_blue}
 
@@ -304,7 +320,8 @@ echo ""
 
 # Create the network for BenchBot software stack
 echo -e "${colour_blue}Creating shared network '$DOCKER_NETWORK':${colour_nc}"
-docker network create "$DOCKER_NETWORK" --subnet="$URL_DOCKER_SUBNET" \
+docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1 || \
+  docker network create "$DOCKER_NETWORK" --subnet="$URL_DOCKER_SUBNET" \
   --ip-range="$URL_DOCKER_SUBNET" --gateway="$URL_DOCKER_GATEWAY"
 if [ -z "$simulator_required" ]; then
   printf "\n${colour_blue}%s${colour_nc}\n" \
@@ -337,18 +354,64 @@ ${cmd// /$'\t'} --ip "$URL_ROS" -d $DOCKER_TAG_BACKEND /bin/bash -c \
 
 if [ -n "$simulator_required" ]; then
   # TODO would be nice to have a less stupid way to do this, but bash's lack of
-  # nested arrays makes it hard...
+  # nested arrays (and difficulties passing arrays in general from functions)
+  # makes it hard...
   args=()
   if [ "$type" = "sim_omni" ]; then
       args=("${SIM_OMNI_ARGS[@]}")
   fi
-
-  printf "\n${colour_blue}%s${colour_nc}\n" \
-    "Starting container for BenchBot Robot Controller ($type):"
-  cmd="${docker_run//'$name'/benchbot_robot}"
-  ${cmd// /$'\t'} --ip "$URL_ROBOT" -d "${args[@]}" \
-    -it $DOCKER_TAG_SIM_PREFIX$type /bin/bash -c \
-    "$cmd_prefix"'rosrun benchbot_robot_controller benchbot_robot_controller'
+  persist_cmd="$(persist_cmd "$type")"
+  persist_chk="$(persist_chk "$type")"
+
+  if [ -n "$persist_cmd" ]; then 
+    # TODO DANGER: there are LOTS of ways this can go wrong. Need to do this
+    # more robustly if we ever expand outside Omniverse-only focus. Examples:
+    # - if next run uses a different simulator type, what do we do?
+    # - the persist variables are vary ad-hoc / hacky
+    # - it's really unclear to the user what is still running in background
+    # - our checks for the status of things are weak (how do we know if an exec
+    #   has failed now???)
+
+    # Run a persistent container and execute within it if our simulator has
+    # caching utilities (e.g. Cache in Omniverse)
+    printf "\n${colour_blue}%s${colour_nc}\n" \
+      "Starting persistent container for BenchBot Robot Controller ($type):"
+    eval $persist_chk && loaded=0 || loaded=1
+    if [ $loaded -eq 0 ]; then
+      printf "Skipping (already running)\n"
+    else
+      cmd="${docker_run//'$name'/benchbot_robot}"
+      ${cmd// /$'\t'} --ip "$URL_ROBOT" -d "${args[@]}" \
+        -t $DOCKER_TAG_SIM_PREFIX$type /bin/bash -c $persist_cmd
+    fi
+
+    printf "\n${colour_blue}%s${colour_nc}\n" \
+      "Executing BenchBot Robot Controller in persistent container:"
+    while [ $loaded -eq 1 ]; do 
+      eval $persist_chk && loaded=0 || loaded=1 
+      sleep 1
+    done
+
+    docker exec -d benchbot_robot /bin/bash -c \
+      "$cmd_prefix"'rosrun benchbot_robot_controller benchbot_robot_controller'
+
+    controller_pid=
+    while [ -z "$controller_pid" ]; do
+      controller_pid="$(docker container top benchbot_robot | \
+        grep 'benchbot_robot_controller' | grep -v 'grep' | awk '{print $2}' | \
+        tr '\n' ',' | sed 's/,$//')"
+      sleep 1
+    done
+    printf "Started in persistent container (pids = $controller_pid)\n"
+  else
+    # Run a new container each time when there's no persistence requirement
+    printf "\n${colour_blue}%s${colour_nc}\n" \
+      "Starting container for BenchBot Robot Controller ($type):"
+    cmd="${docker_run//'$name'/benchbot_robot}"
+    ${cmd// /$'\t'} --ip "$URL_ROBOT" -d "${args[@]}" \
+      -t $DOCKER_TAG_SIM_PREFIX$type /bin/bash -c \
+      "$cmd_prefix"'rosrun benchbot_robot_controller benchbot_robot_controller'
+  fi
 fi
 
 echo -e "\n${colour_blue}Starting container for BenchBot Supervisor:${colour_nc}"

docker/sim_omni.Dockerfile

@@ -2,6 +2,10 @@
 #   https://catalog.ngc.nvidia.com/orgs/nvidia/containers/isaac-sim
 FROM  nvcr.io/nvidia/isaac-sim:2021.2.1
 
+# Fix scripts provided with image
+RUN sed -i 's/$@/"\0"/' python.sh
+RUN sed -i 's/sleep/# \0/' start_nucleus.sh
+
 # Overrides to make things play nicely with the BenchBot ecosystem
 SHELL ["/bin/bash", "-c"]
 ENTRYPOINT []