Docker branch tagging and scheduler modularization (#56)

Co-authored-by: lobi_th <[email protected]> New feature in the build system: docker images are now tagged by the branch name to enable separate images to exist while working on different branches (implemented by https://github.com/akoerner1) Structural changes: further modularization on dependencies ADORe schedular changed into a module (implemented by https://github.com/n-mat)
eclipse · Mar 29, 2023 · 781a025 · 781a025
1 parent 7e37327
commit 781a025
Show file tree

Hide file tree

Showing 126 changed files with 1,540 additions and 1,777 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -16,3 +16,5 @@ catkin_workspace/
 **.vagrant
 **.catkin_workspace
 **.docker_image_cache**
+.docker_image_cache
+.log
diff --git a/.gitmodules b/.gitmodules
@@ -61,14 +61,6 @@
 	ignore = untracked
 	shallow = true
     branch = adore
-[submodule "libadore/make_gadgets"]
-	path = libadore/make_gadgets
-	url = [email protected]:DLR-TS/make_gadgets.git
-	branch = master
-[submodule "adore_if_ros/make_gadgets"]
-	path = adore_if_ros/make_gadgets
-	url = [email protected]:DLR-TS/make_gadgets.git
-	branch = master
 [submodule "adore_if_ros/cppcheck_docker"]
 	path = adore_if_ros/cppcheck_docker
 	url = [email protected]:DLR-TS/cppcheck_docker.git
@@ -93,3 +85,10 @@
 	path = libadore/lizard_docker
 	url = [email protected]:DLR-TS/lizard_docker.git
 	branch = master
+[submodule "adore_if_ros/adore_scheduling"]
+	path = adore_if_ros/adore_scheduling
+	url = [email protected]:DLR-TS/adore_scheduling.git
+	shallow = true
+[submodule "libadore/make_gadgets"]
+	path = libadore/make_gadgets
+	url = [email protected]:DLR-TS/make_gadgets.git
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -17,7 +17,7 @@ pipeline {
                 sh '''#!/usr/bin/env bash
                     set -e
                     export DOCKER_CONFIG=$(realpath "${DOCKER_CONFIG}") make
-                    make
+                    make build
                 '''
             }
         }

diff --git a/Makefile b/Makefile
@@ -1,56 +1,71 @@
-include adore_if_ros/make_gadgets/Makefile
-include adore_if_ros/make_gadgets/docker/Makefile
-
 SHELL:=/bin/bash
 
 .DEFAULT_GOAL := all
 
 ROOT_DIR:=$(shell dirname "$(realpath $(firstword $(MAKEFILE_LIST)))")
-MAKEFLAGS += --no-print-directory
-
 
-.EXPORT_ALL_VARIABLES:
-CATKIN_WORKSPACE_DIRECTORY=catkin_workspace
+include adore_if_ros_msg/make_gadgets/make_gadgets.mk
+include adore_if_ros_msg/make_gadgets/docker/docker-tools.mk
+include adore_if_ros_msg/make_gadgets/docker/docker-image-cacher.mk
+include adore-cli.mk
+#include adore_if_ros/adore_if_ros.mk
+#include adore_if_ros_msg/adore_if_ros_msg.mk
+include v2x_if_ros_msg/v2x_if_ros_msg.mk
+#include plotlabserver/plotlabserver.mk
+#include catkin_base.mk
 
-DOCKER_IMAGE_EXCLUSION_LIST="adore_if_ros:latest adore_if_ros_msg:latest plotlablib:latest plotlabserver:latest plotlabserver_build:latest v2x_if_ros_msg:latest libzmq:latest csaps-cpp:latest adore-cli:latest carlasim/carla:0.9.12"
-DOCKER_IMAGE_INCLUSION_LIST="edrevo/dockerfile-plus:latest"
 
+.EXPORT_ALL_VARIABLES:
+DOCKER_IMAGE_EXCLUSION_LIST="adore_if_ros:latest adore_if_ros_msg:latest plotlablib:latest plotlabserver:latest plotlabserver_build:latest v2x_if_ros_msg:latest libzmq:latest csaps-cpp:latest adore-cli:latest carlasim/carla:0.9.13"
+DOCKER_IMAGE_INCLUSION_LIST="edrevo/dockerfile-plus:latest ubuntu:20.04 ubuntu:focal"
 DOCKER_IMAGE_CACHE_DIRECTORY="${ROOT_DIR}/.docker_image_cache"
 DOCKER_IMAGE_SEARCH_PATH=${ROOT_DIR}
 
+include apt_cacher_ng_docker/apt_cacher_ng_docker.mk
 
-DOCKER_BUILDKIT?=1
-COMPOSE_DOCKER_CLI_BUILD?=1 
-DOCKER_CONFIG:=$(shell realpath "${ROOT_DIR}")/apt_cacher_ng_docker
-
+.PHONY: all
+all: help 
 
-DOCKER_GID := $(shell getent group | grep docker | cut -d":" -f3)
-USER := $(shell whoami)
-UID := $(shell id -u)
-GID := $(shell id -g)
+.PHONY: build 
+build: _build ## Build ADORe 
 
-TEST_SCENARIOS?=baseline_test.launch baseline_test.launch
+.PHONY: build_adore_if_ros_msg
+build_adore_if_ros_msg:
+	cd adore_if_ros_msg && \
+	make build
 
+.PHONY: build_adore_v2x
+build_adore_if_v2x:
+	cd adore_if_v2x && \
+    APT_CACHER_NG_DOCKER_MAKEFILE_PATH= make build
+
+
+.PHONY: _build 
+_build: \
+        docker_load \
+        docker_host_context_check \
+        docker_storage_inventory_prebuild \
+        start_apt_cacher_ng \
+        build_adore_if_ros_msg \
+        build_v2x_if_ros_msg \
+        build_adore_if_ros \
+        build_adore_if_v2x \
+        build_adore_if_ros_msg \
+        docker_storage_inventory_postbuild \
+        clean_up 
+
+.PHONY: clean 
+clean: clean_adore-cli
+	cd adore_if_ros && make clean
+	cd libadore && make clean
+	cd sumo_if_ros && make clean
+	cd adore_if_ros_msg && make clean
+	cd v2x_if_ros_msg && make clean
+	cd adore_if_v2x && make clean
+	cd plotlabserver && make clean
 
-.PHONY: all
-all: \
-     docker_group_check \
-     root_check \
-     clean \
-     docker_conditional_load \
-     docker_storage_inventory_prebuild \
-     start_apt_cacher_ng \
-     build_adore_if_ros_msg\
-     build_adore_v2x_sim \
-     build_adore_if_v2x \
-     build_sumo_if_ros \
-     build_plotlabserver \
-     build_libadore \
-     build_adore_if_ros \
-     get_apt_cacher_ng_cache_statistics \
-     docker_storage_inventory_postbuild \
-     stop_apt_cacher_ng \
-     docker_save_images \
+.PHONY: clean_up
+clean_up: stop_apt_cacher_ng docker_save_images docker_delete_all_none_tags
 
 .PHONY: docker_storage_inventory_prebuild
 docker_storage_inventory_prebuild:
@@ -61,221 +76,61 @@ docker_storage_inventory_prebuild:
 docker_storage_inventory_postbuild:
 	bash tools/docker_storage_inventory.sh --log-directory .log
 
-.PHONY: build
-build: all
-
-.PHONY: clean
-clean: delete_all_none_tags 
-	cd plotlabserver && make clean
-	cd sumo_if_ros && make clean
-	cd adore_if_ros_msg && make clean
-	cd libadore && make clean
-	cd adore_if_ros && make clean
-	cd adore_if_v2x && make clean
-
-.PHONY: start_apt_cacher_ng 
-start_apt_cacher_ng: ## Start apt cacher ng service
-	cd apt_cacher_ng_docker && \
-    make up
-
-.PHONY: stop_apt_cacher_ng 
-stop_apt_cacher_ng: ## Stop apt cacher ng service
-	cd apt_cacher_ng_docker && make down
-
-.PHONY: get_apt_cacher_ng_cache_statistics 
-get_apt_cacher_ng_cache_statistics: ## returns the cache statistics for apt cahcer ng
-	@cd apt_cacher_ng_docker && \
-	make get_cache_statistics
-
-.PHONY: clean_apt_cacher_ng_cache
-clean_apt_cacher_ng_cache: ## Clean/delete apt cache
-	@cd apt_cacher_ng_docker && \
-	make clean
-
-.PHONY: submodules_update 
-submodules_update: # Updates git submodules
-	git submodule update --init --recursive
-
-.PHONY: build_adore_if_ros 
-build_adore_if_ros: ## build adore_if_ros
-	cd adore_if_ros && \
-    make
-.PHONY: build_adore_if_ros_msg
-build_adore_if_ros_msg: 
-	cd adore_if_ros_msg && \
-	make
-
-.PHONY: build_plotlabserver 
-build_plotlabserver: ## Build plotlabserver
-	cd plotlabserver && \
-    make build_fast
-
-.PHONY: build_adore_if_v2x 
-build_adore_if_v2x: ## Build adore_if_v2x
-	cd adore_if_v2x && \
-    make
-
-.PHONY: build_adore_v2x_sim 
-build_adore_v2x_sim: ## Build adore_v2x_sim
-	cd adore_v2x_sim && \
-    make
-
-.PHONY: build_libadore 
-build_libadore: start_apt_cacher_ng ## Build libadore
-	cd libadore && \
-    make
+.PHONY: docker_save_images
+docker_save_images:
+	@nohup sh -c 'make docker_save' > /dev/null 2>&1 &
 
-.PHONY: build_sumo_if_ros 
-build_sumo_if_ros: ## Build sumo_if_ros
-	cd sumo_if_ros && \
-    make
+.PHONY: docker_load_images
+docker_load_images:
+	@nohup make docker_load > /dev/null 2>&1 &
 
 .PHONY: test 
-test:
+test: ## Run ADORe unit tests
 	mkdir -p .log && \
     cd libadore && \
 	make test | tee "${ROOT_DIR}/.log/libadore_unit_test.log"; exit $$PIPESTATUS
 
-.PHONY: lint_sumo_if_ros 
-lint_sumo_if_ros:
-	cd sumo_if_ros && make lint
-
 .PHONY: lint 
-lint: start_apt_cacher_ng## Run linting for all modules
+lint: ## Run linting for all modules
 	mkdir -p .log
 	find . -name "**lint_report.log" -exec rm -rf {} \;
 	EXIT_STATUS=0; \
-        (cd adore_if_ros && make lint) || EXIT_STATUS=$$? && \
         (cd sumo_if_ros && make lint) || EXIT_STATUS=$$? && \
         (cd libadore && make lint) || EXIT_STATUS=$$? && \
-	    find . -name "**lint_report.log" -print0 | xargs -0 -I {} mv {} .log/ && \
+        (cd adore_if_ros && make lint) || EXIT_STATUS=$$? && \
+        find sumo_if_ros -type f -name 'lint_report.log' -exec mv {} .log/sumo_if_ros_lint_report.log \; && \
+        find libadore -type f -name 'lint_report.log' -exec mv {} .log/libadore_lint_report.log \; && \
+        find adore_if_ros -type f -name 'lint_report.log' -exec mv {} .log/adore_if_ros_lint_report.log \; && \
         exit $$EXIT_STATUS
 
 .PHONY: lizard 
-lizard: start_apt_cacher_ng## Run lizard static analysis tool for all modules
+lizard: ## Run lizard static analysis tool for all modules
 	mkdir -p .log
 	find . -name "**lizard_report.**" -exec rm -rf {} \;
-	EXIT_STATUS=0; \
+	unset -f DOCKER_CONFIG && \
+    EXIT_STATUS=0; \
+        (cd sumo_if_ros && make lizard) || EXIT_STATUS=$$? && \
         (cd libadore && make lizard) || EXIT_STATUS=$$? \ && \
         (cd adore_if_ros && make lizard) || EXIT_STATUS=$$? && \
-        (cd sumo_if_ros && make lizard) || EXIT_STATUS=$$? && \
-	    find . -name "**lizard_report.**" -print0 | xargs -0 -I {} mv {} .log/ && \
+        find sumo_if_ros -type f -name 'lizard_report.log' -exec mv {} .log/sumo_if_ros_lizard_report.log \; && \
+        find sumo_if_ros -type f -name 'lizard_report.xml' -exec mv {} .log/sumo_if_ros_lizard_report.xml \; && \
+        find libadore -type f -name 'lizard_report.log' -exec mv {} .log/libadore_lizard_report.log \; && \
+        find libadore -type f -name 'lizard_report.xml' -exec mv {} .log/libadore_lizard_report.xml \; && \
+        find adore_if_ros -type f -name 'lizard_report.log' -exec mv {} .log/adore_if_ros_lizard_report.log \; && \
+        find adore_if_ros -type f -name 'lizard_report.xml' -exec mv {} .log/adore_if_ros_if_ros_lizard_report.xml \; && \
         exit $$EXIT_STATUS
 
 .PHONY: cppcheck 
-cppcheck: start_apt_cacher_ng## Run cppcheck static checking tool for all modules.
+cppcheck: ## Run cppcheck static checking tool for all modules.
 	mkdir -p .log
 	find . -name "**cppcheck_report.log" -exec rm -rf {} \;
 	EXIT_STATUS=0; \
+        (cd sumo_if_ros && make cppcheck) || EXIT_STATUS=$$? && \
         (cd libadore && make cppcheck) || EXIT_STATUS=$$? && \
         (cd adore_if_ros && make cppcheck) || EXIT_STATUS=$$? && \
-        (cd sumo_if_ros && make cppcheck) || EXIT_STATUS=$$? && \
-	    find . -name "**cppcheck_report.log" -print0 | xargs -0 -I {} mv {} .log/ && \
+        find sumo_if_ros -type f -name 'cppcheck_report.log' -exec mv {} .log/sumo_if_ros_cppcheck_report.log \; && \
+        find libadore -type f -name 'cppcheck_report.log' -exec mv {} .log/libadore_cppcheck_report.log \; && \
+        find adore_if_ros -type f -name 'cppcheck_report.log' -exec mv {} .log/adore_if_ros_cppcheck_report.log \; && \
         exit $$EXIT_STATUS
 
-.PHONY: clean_catkin_workspace 
-clean_catkin_workspace:
-	rm -rf "${CATKIN_WORKSPACE_DIRECTORY}"
-
-.PHONY: build_catkin_base 
-build_catkin_base: ## Build a docker image with base catkin tools installed with tag catkin_base:latest
-	cd docker && \
-    docker build \
-                 --network host \
-                 --build-arg UID=${UID} \
-                 --build-arg GID=${GID} \
-                 --file Dockerfile.catkin_base \
-                 --tag catkin_base .
-
-.PHONY: create_catkin_workspace_docker
-create_catkin_workspace_docker: build_catkin_base
-	docker run -it \
-                   --user "${UID}:${GID}" \
-                   --mount type=bind,source="${ROOT_DIR}",target="${ROOT_DIR}" \
-                   catkin_base \
-				   /bin/bash -c 'cd "${ROOT_DIR}" && HOME="${ROOT_DIR}" CATKIN_WORKSPACE_DIRECTORY="${CATKIN_WORKSPACE_DIRECTORY}" bash tools/create_catkin_workspace.sh 2>&1 | tee -a .log/create_catkin_workspace.log'
-
-.PHONY: create_catkin_workspace
-create_catkin_workspace: clean_catkin_workspace## Creates a catkin workspace @ adore/catkin_workspace. Can be called within the adore-cli or on the host.
-	@if [ -f "/.dockerenv" ]; then\
-            bash tools/create_catkin_workspace.sh;\
-            exit 0;\
-        else\
-            make create_catkin_workspace_docker;\
-            exit 0;\
-        fi;
-
-.PHONY: build_adore-cli_fast
-build_adore-cli_fast: # build adore-cli if it does not already exist in the docker repository. If it does exist this is a noop.
-	@[ -n "$$(docker images -q adore-cli:latest)" ] || \
-    make build_adore-cli 
-
-.PHONY: build_adore-cli
-build_adore-cli: start_apt_cacher_ng build_catkin_base build_plotlabserver ## Builds the ADORe CLI docker context/image
-	docker compose build adore-cli \
-                         --build-arg UID=${UID} \
-                         --build-arg GID=${GID} \
-                         --build-arg DOCKER_GID=${DOCKER_GID}
-	docker compose build adore-cli-x11-display \
-                         --build-arg UID=${UID} \
-                         --build-arg GID=${GID} \
-                         --build-arg DOCKER_GID=${DOCKER_GID}
-	make stop_apt_cacher_ng
-
-.PHONY: run_ci_scenarios
-run_ci_scenarios:
-	bash tools/run_ci_scenarios.txt 
-
-
-.PHONY: adore-cli_setup
-adore-cli_setup: build_adore-cli_fast
-	@echo "Running adore-cli setup..."
-	@mkdir -p .log/.ros/bag_files
-	@mkdir -p plotlabserver/.log
-	@cd .log && ln -sf ../plotlabserver/.log plotlabserver
-	@touch .zsh_history
-	@touch .zsh_history.new
-	cd plotlabserver && \
-    make down
-
-.PHONY: adore-cli_teardown
-adore-cli_teardown:
-	@echo "Running adore-cli teardown..."
-	@docker compose down && xhost - 1> /dev/null
-	@docker compose rm -f
-
-.PHONY: adore-cli_start
-adore-cli_start:
-	@xhost + && \
-    docker compose up adore-cli-x11-display --force-recreate -V -d; \
-    xhost - 
-
-.PHONY: adore-cli_start_headless
-adore-cli_start_headless:
-	DISPLAY_MODE=headless make adore-cli_start
-
-.PHONY: adore-cli_attach
-adore-cli_attach:
-	docker exec -it --user adore-cli adore-cli /bin/zsh -c "bash tools/adore-cli.sh" || true
-
-.PHONY: adore-cli_scenarios_run
-adore-cli_scenarios_run:
-	docker exec -it --user adore-cli adore-cli /bin/zsh -c "bash tools/run_test_scenarios.sh" || true
-
-.PHONY: adore-cli
-adore-cli: adore-cli_setup adore-cli_start adore-cli_attach adore-cli_teardown ## Start an adore-cli context
-
-.PHONY: run_test_scenarios
-run_test_scenarios: adore-cli_setup adore-cli_start_headless adore-cli_scenarios_run adore-cli_teardown
-
-.PHONY: run_scenarios
-run_scenarios: adore-cli_setup adore-cli_start adore-cli_scenarios_run adore-cli_teardown
-
-.PHONY: docker_save_images
-docker_save_images:
-	@nohup make docker_save > /dev/null 2>&1 &
-
-.PHONY: clean_all_cache
-clean_all_cache:
-	echo todo
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,7 +17,7 @@ pipeline { @@
                     sh '''#!/usr/bin/env bash
                         set -e
                         export DOCKER_CONFIG=$(realpath "${DOCKER_CONFIG}") make
-                        make
+                        make build
                     '''
                 }
             }
@@ Expand Down @@