Merge pull request #1 from 130s/update/commit_image

Rename the variables `COMMIT_IMAGE_*` to `DOCKER_COMMIT_IMAGE_*` and so on.

.travis.yml

@@ -58,6 +58,7 @@ env:
     - ROS_DISTRO=melodic OS_NAME=debian OS_CODE_NAME=stretch AFTER_SCRIPT='grep -q ID=debian /etc/os-release && grep -q VERSION_ID=\"9\" /etc/os-release'
     - ROS_DISTRO=melodic OS_NAME=debian EXPECT_EXIT_CODE=1
     - ROS_DISTRO=melodic OS_NAME=debian OS_CODE_NAME=bionic EXPECT_EXIT_CODE=1
+    - ROS_DISTRO=melodic DOCKER_COMMIT="img_temp"
 
     # external repositories
     - ROS_DISTRO=kinetic _EXTERNAL_REPO='ros-industrial/industrial_core#kinetic-devel'
@@ -77,3 +78,4 @@ install:
 script:
   - industrial_ci/_wrap_test.sh # this script is only for internal use
   - if ! [ -z "$POST_PROCESS" ]; then $POST_PROCESS; fi
+  - if ! [ -z "$DOCKER_COMMIT" ]; then  echo "Test if the Docker image is committed or not. See the size of the Docker image is non-zero or not."; if docker image inspect $DOCKER_COMMIT --format='{{.Size}}'; then exit 0; else exit 1; fi fi

doc/index.rst

@@ -185,12 +185,11 @@ Note that some of these currently tied only to a single option, but we still lea
 * **CATKIN_PARALLEL_JOBS** (default: -p4): Maximum number of packages to be built in parallel that is passed to underlining build tool. As of Jan 2016, this is only enabled with `catkin_tools`. See for more detail about `number of build jobs <http://catkin-tools.readthedocs.org/en/latest/verbs/catkin_build.html#controlling-the-number-of-build-jobs>`_ and `documentation of catkin_tools <https://catkin-tools.readthedocs.org/en/latest/verbs/catkin_build.html#full-command-line-interface>`_ that this env variable is passed to internally in `catkin-tools`.
 * **CATKIN_PARALLEL_TEST_JOBS** (default: -p4): Maximum number of packages which could be examined in parallel during the test run. If not set it's filled by `ROS_PARALLEL_JOBS`.
 * **CCACHE_DIR** (default: not set): If set, `ccache <https://en.wikipedia.org/wiki/Ccache>`_ gets enabled for your build to speed up the subsequent builds in the same job if anything. See `detail. <https://github.com/ros-industrial/industrial_ci/blob/master/doc/index.rst#cache-build-artifacts-to-speed-up-the-subsequent-builds-if-any>`_
-* **COMMIT_IMAGE** (default: false): If set to true, the docker images which contains the build and test will be saved in the outer-layer docker which runs the script.  The commited image is then be available to be pushed to a docker registry.  This behavior enables a building Docker images with CI/CD approach. If set to false, or unset the container will not be commited and is removed.
-* **COMMIT_IMAGE_NAME** (default: not set): used to specify an image name during the docker commit command which is triggered by setting COMMIT_IMAGE=true. If unset and if COMMIT_IMAGE=true then container id will be used as the image name.
-* **COMMIT_IMAGE_MSG** (default: not set): used to specify a commit during the docker commit command which is triggered by setting COMMIT_IMAGE=true. If unset and if COMMIT_IMAGE=true then the commit message will be empty.
 * **DEBUG_BASH** (default: not set): If set with any value (e.g. `true`), all executed commands that are not printed by default to reduce print space will be printed.
 * **DOCKER_BASE_IMAGE** (default: $OS_NAME:$OS_CODE_NAME): Base image used for building the CI image. Could be used to pre-bundle dependecies or to run tests for different architectures. See `this PR <https://github.com/ros-industrial/industrial_ci/pull/174>`_ for more info.
 * **DOCKER_BUILD_OPTS** (default: not set): Used do specify additional build options for Docker.
+* **DOCKER_COMMIT** (default: not set): If set, the docker images which contains the build and test will be saved in the outer-layer docker which runs the script.  The commited image is then be available to be pushed to a docker registry.  This behavior enables a building Docker images with CI/CD approach. If unset the container will not be commited and is removed. The value set is used to specify an image name during the docker commit command which is triggered by setting ``DOCKER_COMMIT_IMAGE=true``. If special value "use_containerid" is set, then container id will be used as the image name.
+* **DOCKER_COMMIT_MSG** (default: not set): used to specify a commit during the docker commit command which is triggered by setting DOCKER_COMMIT_IMAGE=true. If unset and if DOCKER_COMMIT_IMAGE=true then the commit message will be empty. See more ``DOCKER_COMMIT``.
 * **DOCKER_FILE** (default: not set): Instead of pulling an images from the Docker hub, build it from the given path or URL. Please note, this disables the handling of `ROS_REPOSITORY_PATH` and `ROS_DISTRO`, they have to be set in the build file instead.
 * **DOCKER_IMAGE** (default: not set): Selects a Docker images different from default one. Please note, this disables the handling of `ROS_REPOSITORY_PATH` and `ROS_DISTRO` as ROS needs already to be installed in the image.
 * **DOCKER_PULL** (default: true): set to false if custom docker image should not be pulled, e.g. if it was created locally
@@ -248,6 +247,24 @@ On CI platform usually some variables are available for the convenience. Since a
 Still, you may want to pass some other vars. `DOCKER_RUN_OPTS='-e MY_VARIABLE_VALUE'` should do the trick.
 You can even set it to a specific value: `DOCKER_RUN_OPTS='-e MY_VARIABLE_VALUE=42'` (format varies per CI platform. These are Gitlab CI example).
 
+Re-use the container image
+--------------------------
+
+``industrial_ci`` builds a ``Docker`` image using the associated repository on the specified operating system per every job. While the built Docker image is thrown away once the job finishes by default, there's a way to access the built image post job so that you can re-use it.
+
+To do so, simply set ``DOCKER_COMMIT`` the name of the image of your choice. Then you'll be able to access that image. For example in your CI config (e.g. ``.travis.yml``), add something like ::
+
+  variables:
+      DOCKER_COMMIT=registry.gitlab.com/your-org/your-repo:your_img
+  :
+  script:
+      - docker push $DOCKER_COMMIT
+
+Note for the built Docker image:
+
+- ``apt`` repositories (i.e. Result of ``apt-get update``) are removed.
+- Some utility packages are pre-installed. As of Sept 2018, those are: ``build-essential, python-catkin-tools, python-pip, python-rosdep, python-wstool, ros-$ROS_DISTRO-catkin, ssh-client``.
+
 (Gitlab CI) Access to private repositories
 ------------------------------------------
 
@@ -701,7 +718,6 @@ The cached images can be listed with
 
   $ rosrun industrial_ci rerun_ci --list
 
-
 For maintainers of industrial_ci repository
 ================================================
 

industrial_ci/scripts/rerun_ci

@@ -52,26 +52,26 @@ fi
 repo_dir=$(cd "$1" && pwd)
 shift
 
-COMMIT_IMAGE_MSG="$repo_dir $*"
-env_hash=($(sha256sum <<< "$COMMIT_IMAGE_MSG"))
-COMMIT_IMAGE="industrial-ci/rerun_ci/$(basename "$repo_dir"):${env_hash:0:12}"
+_COMMIT_IMAGE_MSG="$repo_dir $*"
+env_hash=($(sha256sum <<< "$_COMMIT_IMAGE_MSG"))
+_COMMIT_IMAGE="industrial-ci/rerun_ci/$(basename "$repo_dir"):${env_hash:0:12}"
 
 if [ "$remove" ]; then
-  exec docker rmi $COMMIT_IMAGE
+  exec docker rmi $_COMMIT_IMAGE
 elif [ "$clean" ]; then
-  docker rmi $COMMIT_IMAGE
+  docker rmi $_COMMIT_IMAGE
 fi
 
 
-force_env=(COMMIT_IMAGE=$COMMIT_IMAGE COMMIT_IMAGE_MSG=$COMMIT_IMAGE_MSG)
+force_env=(_COMMIT_IMAGE=$_COMMIT_IMAGE _COMMIT_IMAGE_MSG=$_COMMIT_IMAGE_MSG)
 keep_env=(DOCKER_PORT=$DOCKER_PORT SSH_AUTH_SOCK=$SSH_AUTH_SOCK TERM=$TERM)
 
-if docker image inspect $COMMIT_IMAGE &> /dev/null; then
-  force_env+=(DOCKER_IMAGE=$COMMIT_IMAGE  DOCKER_PULL=false)
+if docker image inspect $_COMMIT_IMAGE &> /dev/null; then
+  force_env+=(DOCKER_IMAGE=$_COMMIT_IMAGE  DOCKER_PULL=false)
 fi
 
 env -i "${keep_env[@]}" "$script_dir/run_ci" "$repo_dir" "$@" "${force_env[@]}" || ret=$?
 
-echo "Please do not forget to clean-up: docker rmi $COMMIT_IMAGE"
+echo "Please do not forget to clean-up: docker rmi $_COMMIT_IMAGE"
 
 exit $ret

industrial_ci/src/docker.sh

@@ -65,8 +65,8 @@ function ici_require_run_in_docker() {
 #######################################
 function ici_run_cmd_in_docker() {
   local run_opts=($DOCKER_RUN_OPTS)
-  local commit_image=$COMMIT_IMAGE
-  unset COMMIT_IMAGE
+  local commit_image=$DOCKER_COMMIT
+  unset DOCKER_COMMIT
 
   #forward ssh agent into docker container
  local ssh_docker_opts=()
@@ -111,7 +111,10 @@ function ici_run_cmd_in_docker() {
   wait %% || ret=$?
   trap - INT
   if [ -n "$commit_image" ]; then
-    docker commit -m "$COMMIT_IMAGE_MSG" "$cid" "$COMMIT_IMAGE_NAME"
+    if [ "use_containerid" != "$commit_image" ]; then
+        commit_image_name="$commit_image"
+    fi  # commit_image_name will be empty if 'use_containerid' is passed.
+    docker commit -m "$DOCKER_COMMIT_MSG" "$cid" "$commit_image_name"
   fi
   docker rm "$cid" > /dev/null
   return $ret

industrial_ci/src/env.sh

@@ -20,7 +20,6 @@ ici_mark_deprecated USE_DEB "Please migrate to UPSTREAM_WORKSPACE."
 ici_mark_deprecated UBUNTU_OS_CODE_NAME "Was renamed to OS_CODE_NAME."
 if [ ! "$CATKIN_PARALLEL_JOBS" ]; then export CATKIN_PARALLEL_JOBS="-p4"; fi
 if [ ! "$CATKIN_PARALLEL_TEST_JOBS" ]; then export CATKIN_PARALLEL_TEST_JOBS="$CATKIN_PARALLEL_JOBS"; fi
-if [ "$COMMIT_IMAGE" ]; then export COMMIT_IMAGE; else export COMMIT_IMAGE="false"; fi
 if [ ! "$ROS_PARALLEL_JOBS" ]; then export ROS_PARALLEL_JOBS="-j8"; fi
 if [ ! "$ROS_PARALLEL_TEST_JOBS" ]; then export ROS_PARALLEL_TEST_JOBS="$ROS_PARALLEL_JOBS"; fi
 # .rosintall file name
@@ -65,7 +64,7 @@ fi
 export OS_CODE_NAME
 export OS_NAME
 export DOCKER_BASE_IMAGE
-export COMMIT_IMAGE_NAME
+export DOCKER_COMMIT
 
 # exit with error if OS_NAME is set, but OS_CODE_NAME is not.
 # assume ubuntu as default