From bc1309f72f5b32f0f26719ba0ba76b725a46d6cc Mon Sep 17 00:00:00 2001 From: Lennart Reiher Date: Thu, 6 Jun 2024 13:19:41 +0000 Subject: [PATCH] add options to toggle to continue rosdep and catkin/colcon on failure --- .gitlab-ci/docker-ros.yml | 68 ++++++++++++++------------- README.md | 8 ++++ action.yml | 6 +++ docker/Dockerfile | 20 ++++++-- scripts/build.sh | 2 + scripts/ci.sh | 2 + templates/docker-compose.template.yml | 2 + 7 files changed, 72 insertions(+), 36 deletions(-) diff --git a/.gitlab-ci/docker-ros.yml b/.gitlab-ci/docker-ros.yml index 3f22462..a08d951 100644 --- a/.gitlab-ci/docker-ros.yml +++ b/.gitlab-ci/docker-ros.yml @@ -9,39 +9,41 @@ workflow: variables: - TARGET: run # Target stage of Dockerfile (comma-separated list) [dev|run] - PLATFORM: amd64 # Target platform architecture (comma-separated list) [amd64|arm64|...] - BASE_IMAGE: '' # Base image name:tag (required) - COMMAND: '' # Launch command of run image (required if target=run) - IMAGE_NAME: ${CI_REGISTRY_IMAGE} # Image name of run image - IMAGE_TAG: latest # Image tag of run image - DEV_IMAGE_NAME: ${IMAGE_NAME} # Image name of dev image - DEV_IMAGE_TAG: ${IMAGE_TAG}-dev # Image tag of dev image - BUILD_CONTEXT: . # Build context of Docker build process - REGISTRY: ${CI_REGISTRY} # Docker registry to push images to - REGISTRY_USER: ${CI_REGISTRY_USER} # Docker registry username - REGISTRY_PASSWORD: ${CI_REGISTRY_PASSWORD} # Docker registry password - ENABLE_INDUSTRIAL_CI: 'false' # Enable industrial_ci - ENABLE_SINGLEARCH_PUSH: 'false' # Enable push of single arch images with [-amd64|-arm64] postfix - ENABLE_PUSH_AS_LATEST: 'false' # Push images with tag `latest`/`latest-dev` in addition to the configured image names - RMW_IMPLEMENTATION: 'rmw_cyclonedds_cpp' # RMW implementation to use (only for ROS 2) - ROS_DISTRO: '' # ROS Distro (required if ROS is not installed in `base-image`) - GIT_HTTPS_SERVER: ${CI_SERVER_HOST} # Server URL (without protocol) for cloning private Git repositories via HTTPS - GIT_HTTPS_USER: gitlab-ci-token # Username for cloning private Git repositories via HTTPS - GIT_HTTPS_PASSWORD: ${CI_JOB_TOKEN} # Password for cloning private Git repositories via HTTPS - GIT_SSH_PRIVATE_KEY: '' # SSH private key for cloning private Git repositories via SSH - GIT_SSH_KNOWN_HOST_KEYS: '' # Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`) - ADDITIONAL_DEBS_FILE: docker/additional-debs.txt # Relative filepath to file containing additional apt deb packages to install - ENABLE_RECURSIVE_ADDITIONAL_DEBS: 'false' # Enable recursive discovery of files named `additional-debs-file` - ADDITIONAL_FILES_DIR: docker/additional-files # Relative path to directory containing additional files to copy into image" - ADDITIONAL_PIP_FILE: docker/additional-pip-requirements.txt # Relative filepath to file containing additional pip packages to install - ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file` - BLACKLISTED_PACKAGES_FILE: docker/blacklisted-packages.txt # Relative filepath to file containing the blacklisted packages - ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: 'false' # Enable recursive discovery of files named `blacklisted-packages-file` - CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands - ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file` - VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false) - ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `*.repos` + TARGET: run # Target stage of Dockerfile (comma-separated list) [dev|run] + PLATFORM: amd64 # Target platform architecture (comma-separated list) [amd64|arm64|...] + BASE_IMAGE: '' # Base image name:tag (required) + COMMAND: '' # Launch command of run image (required if target=run) + IMAGE_NAME: ${CI_REGISTRY_IMAGE} # Image name of run image + IMAGE_TAG: latest # Image tag of run image + DEV_IMAGE_NAME: ${IMAGE_NAME} # Image name of dev image + DEV_IMAGE_TAG: ${IMAGE_TAG}-dev # Image tag of dev image + BUILD_CONTEXT: . # Build context of Docker build process + REGISTRY: ${CI_REGISTRY} # Docker registry to push images to + REGISTRY_USER: ${CI_REGISTRY_USER} # Docker registry username + REGISTRY_PASSWORD: ${CI_REGISTRY_PASSWORD} # Docker registry password + ENABLE_INDUSTRIAL_CI: 'false' # Enable industrial_ci + ENABLE_SINGLEARCH_PUSH: 'false' # Enable push of single arch images with [-amd64|-arm64] postfix + ENABLE_PUSH_AS_LATEST: 'false' # Push images with tag `latest`/`latest-dev` in addition to the configured image names + RMW_IMPLEMENTATION: 'rmw_cyclonedds_cpp' # RMW implementation to use (only for ROS 2) + ROS_DISTRO: '' # ROS Distro (required if ROS is not installed in `base-image`) + GIT_HTTPS_SERVER: ${CI_SERVER_HOST} # Server URL (without protocol) for cloning private Git repositories via HTTPS + GIT_HTTPS_USER: gitlab-ci-token # Username for cloning private Git repositories via HTTPS + GIT_HTTPS_PASSWORD: ${CI_JOB_TOKEN} # Password for cloning private Git repositories via HTTPS + GIT_SSH_PRIVATE_KEY: '' # SSH private key for cloning private Git repositories via SSH + GIT_SSH_KNOWN_HOST_KEYS: '' # Known SSH host keys for cloning private Git repositories via SSH (may be obtained using `ssh-keyscan`) + ADDITIONAL_DEBS_FILE: docker/additional-debs.txt # Relative filepath to file containing additional apt deb packages to install + ENABLE_RECURSIVE_ADDITIONAL_DEBS: 'false' # Enable recursive discovery of files named `additional-debs-file` + ADDITIONAL_FILES_DIR: docker/additional-files # Relative path to directory containing additional files to copy into image" + ADDITIONAL_PIP_FILE: docker/additional-pip-requirements.txt # Relative filepath to file containing additional pip packages to install + ENABLE_RECURSIVE_ADDITIONAL_PIP: 'false' # Enable recursive discovery of files named `additional-pip-file` + BLACKLISTED_PACKAGES_FILE: docker/blacklisted-packages.txt # Relative filepath to file containing the blacklisted packages + ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: 'false' # Enable recursive discovery of files named `blacklisted-packages-file` + CUSTOM_SCRIPT_FILE: docker/custom.sh # Relative filepath to script containing custom installation commands + ENABLE_RECURSIVE_CUSTOM_SCRIPT: 'false' # Enable recursive discovery of files named `custom-script-file` + VCS_IMPORT_FILE: .repos # Relative filepath to file containing additional repos to install via vcstools (only relevant if ENABLE_RECURSIVE_VCS_IMPORT=false) + ENABLE_RECURSIVE_VCS_IMPORT: 'true' # Enable recursive discovery of files named `*.repos` + ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: 'false' # Enable `rosdep install -r` + ENABLE_CONTINUE_BUILD_DESPITE_ERRORS 'false' # Enable `catkin build --continue-on-failure` / `colcon build --continue-on-error` # ----- DOCKER_ROS_GIT_REF: main diff --git a/README.md b/README.md index f173e5e..961c2a2 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,8 @@ If your ROS-based repository (or any of your repository's upstream dependencies, Create a file `blacklisted-packages.txt` in your `docker` folder (or configure a different `BLACKLISTED_PACKAGES_FILE`) and list any ROS package name to blacklist. +If you are having problems with (upstream) packages that you don't need, also have a look at `ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS` and `ENABLE_CONTINUE_BUILD_DESPITE_ERRORS`. + ### Extra System Dependencies (*apt*) If your ROS-based repository requires system dependencies that cannot be installed by specifying their [rosdep](https://docs.ros.org/en/independent/api/rosdep/html/) keys in a `package.xml`, you can use a special `additional-debs.txt` file. @@ -395,6 +397,12 @@ Create a folder `additional-files` in your `docker` folder (or configure a diffe - **`git-https-user` | `GIT_HTTPS_USER`** Username for cloning private Git repositories via HTTPS *default:* `${{ github.actor }}` | `gitlab-ci-token` +- **`enable-continue-build-despite-errors` | `ENABLE_CONTINUE_BUILD_DESPITE_ERRORS`** + Enable `catkin build --continue-on-failure` / `colcon build --continue-on-error` + *default:* `false` +- **`enable-continue-rosdep-install-despite-errors` | `ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS`** + Enable `rosdep install -r` + *default:* `false` - **`enable-recursive-additional-debs` | `ENABLE_RECURSIVE_ADDITIONAL_DEBS`** Enable recursive discovery of files named `additional-debs-file` *default:* `false` diff --git a/action.yml b/action.yml index 6dbbd89..0eefdd7 100644 --- a/action.yml +++ b/action.yml @@ -216,6 +216,8 @@ runs: ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} + ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: ${{ inputs.enable-continue-rosdep-install-despite-errors }} + ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: ${{ inputs.enable-continue-build-despite-errors }} - name: Set up industrial_ci if: ${{ inputs.enable-industrial-ci == 'true' }} @@ -274,6 +276,8 @@ runs: ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} + ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: ${{ inputs.enable-continue-rosdep-install-despite-errors }} + ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: ${{ inputs.enable-continue-build-despite-errors }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} @@ -310,5 +314,7 @@ runs: ENABLE_RECURSIVE_CUSTOM_SCRIPT: ${{ inputs.enable-recursive-custom-script }} VCS_IMPORT_FILE: ${{ inputs.vcs-import-file }} ENABLE_RECURSIVE_VCS_IMPORT: ${{ inputs.enable-recursive-vcs-import }} + ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: ${{ inputs.enable-continue-rosdep-install-despite-errors }} + ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: ${{ inputs.enable-continue-build-despite-errors }} _ENABLE_IMAGE_PUSH: true _IMAGE_POSTFIX: ${{ github.ref != format('refs/heads/{0}', github.event.repository.default_branch) && format('_{0}_ci', steps.slugify-ref-name.outputs.slug) || '' }} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 8650d9d..02c1b96 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -95,6 +95,7 @@ RUN if [[ -x "$(command -v colcon)" ]]; then \ $WORKSPACE/.remove-packages.sh 2> /dev/null | xargs rm -rfv # create install script with list of rosdep dependencies +ARG ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS="false" RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \ source /opt/ros/$ROS_DISTRO/setup.bash && \ apt-get update && \ @@ -103,7 +104,11 @@ RUN echo "set -e" >> $WORKSPACE/.install-dependencies.sh && \ export OS="ubuntu:$(lsb_release -c | awk '{print $2}')" && \ if [[ "$ROS_DISTRO" = "rolling" && "$OS" = "ubuntu:focal" ]]; then export OS="ubuntu:jammy"; fi && \ set -o pipefail && \ - ROS_PACKAGE_PATH=$(pwd):$ROS_PACKAGE_PATH rosdep install --os $OS -y --simulate --from-paths src --ignore-src | tee -a $WORKSPACE/.install-dependencies.sh && \ + if [[ $ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS == 'true' ]]; then \ + ROS_PACKAGE_PATH=$(pwd):$ROS_PACKAGE_PATH rosdep install --os $OS -y --simulate --from-paths src --ignore-src -r | tee -a $WORKSPACE/.install-dependencies.sh ; \ + else \ + ROS_PACKAGE_PATH=$(pwd):$ROS_PACKAGE_PATH rosdep install --os $OS -y --simulate --from-paths src --ignore-src | tee -a $WORKSPACE/.install-dependencies.sh ; \ + fi && \ chmod +x $WORKSPACE/.install-dependencies.sh && \ rm -rf /var/lib/apt/lists/* @@ -250,12 +255,21 @@ CMD ["bash"] FROM dev as build # build ROS workspace +ARG ENABLE_CONTINUE_BUILD_DESPITE_ERRORS="false" RUN if [[ -x "$(command -v colcon)" ]]; then \ source /opt/ros/${ROS_DISTRO}/setup.bash && \ - colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release ; \ + if [[ $ENABLE_CONTINUE_BUILD_DESPITE_ERRORS == 'true' ]]; then \ + colcon build --continue-on-error --cmake-args -DCMAKE_BUILD_TYPE=Release ; \ + else \ + colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release ; \ + fi ; \ elif [[ -x "$(command -v catkin)" ]]; then \ catkin config --install --extend /opt/ros/${ROS_DISTRO} && \ - catkin build -DCMAKE_BUILD_TYPE=Release --force-color --no-status --summarize ; \ + if [[ $ENABLE_CONTINUE_BUILD_DESPITE_ERRORS == 'true' ]]; then \ + catkin build -DCMAKE_BUILD_TYPE=Release --force-color --no-status --summarize --continue-on-failure ; \ + else \ + catkin build -DCMAKE_BUILD_TYPE=Release --force-color --no-status --summarize ; \ + fi ; \ fi ############ run ############################################################### diff --git a/scripts/build.sh b/scripts/build.sh index 83b59a6..9a82b67 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -35,6 +35,8 @@ build_image() { $(if [[ -n "${ENABLE_RECURSIVE_CUSTOM_SCRIPT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_CUSTOM_SCRIPT=${ENABLE_RECURSIVE_CUSTOM_SCRIPT}"; fi) \ $(if [[ -n "${VCS_IMPORT_FILE}" ]]; then echo "--build-arg VCS_IMPORT_FILE=${VCS_IMPORT_FILE}"; fi) \ $(if [[ -n "${ENABLE_RECURSIVE_VCS_IMPORT}" ]]; then echo "--build-arg ENABLE_RECURSIVE_VCS_IMPORT=${ENABLE_RECURSIVE_VCS_IMPORT}"; fi) \ + $(if [[ -n "${ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS}" ]]; then echo "--build-arg ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS=${ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS}"; fi) \ + $(if [[ -n "${ENABLE_CONTINUE_BUILD_DESPITE_ERRORS}" ]]; then echo "--build-arg ENABLE_CONTINUE_BUILD_DESPITE_ERRORS=${ENABLE_CONTINUE_BUILD_DESPITE_ERRORS}"; fi) \ . echo "Successfully built stage '${TARGET}' for platform '${PLATFORM}' as '${IMAGE}'" } diff --git a/scripts/ci.sh b/scripts/ci.sh index 8f76fd4..6a1cad9 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -38,6 +38,8 @@ CUSTOM_SCRIPT_FILE="${CUSTOM_SCRIPT_FILE:-}" ENABLE_RECURSIVE_CUSTOM_SCRIPT="${ENABLE_RECURSIVE_CUSTOM_SCRIPT:-}" VCS_IMPORT_FILE="${VCS_IMPORT_FILE:-}" ENABLE_RECURSIVE_VCS_IMPORT="${ENABLE_RECURSIVE_VCS_IMPORT:-}" +ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS="${ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS:-}" +ENABLE_CONTINUE_BUILD_DESPITE_ERRORS="${ENABLE_CONTINUE_BUILD_DESPITE_ERRORS:-}" _ENABLE_IMAGE_PUSH="${_ENABLE_IMAGE_PUSH:-false}" _IMAGE_POSTFIX="${_IMAGE_POSTFIX:-""}" diff --git a/templates/docker-compose.template.yml b/templates/docker-compose.template.yml index 54280bf..686e7c4 100644 --- a/templates/docker-compose.template.yml +++ b/templates/docker-compose.template.yml @@ -22,6 +22,8 @@ x-build: &build ENABLE_RECURSIVE_BLACKLISTED_PACKAGES: $ENABLE_RECURSIVE_BLACKLISTED_PACKAGES ENABLE_RECURSIVE_CUSTOM_SCRIPT: $ENABLE_RECURSIVE_CUSTOM_SCRIPT ENABLE_RECURSIVE_VCS_IMPORT: $ENABLE_RECURSIVE_VCS_IMPORT + ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS: $ENABLE_CONTINUE_ROSDEP_INSTALL_DESPITE_ERRORS + ENABLE_CONTINUE_BUILD_DESPITE_ERRORS: $ENABLE_CONTINUE_BUILD_DESPITE_ERRORS GIT_HTTPS_PASSWORD: $GIT_HTTPS_PASSWORD GIT_HTTPS_SERVER: $GIT_HTTPS_SERVER GIT_HTTPS_USER: $GIT_HTTPS_USER