variables: GIT_STRATEGY: clone CAR_NAME: dr_drift KITCAR_REPO_PATH: /builds/kitcar/ INIT_INPUT: 1\nn # Do not install CUDA DISPLAY: ":1.0" default: image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci retry: max: 2 when: - runner_system_failure - stuck_or_timeout_failure - unknown_failure before_script: # Run the container start script to enable the pulseaudio & xserver - bash init/docker_start_script.sh # Print out python version for debugging - python3 -V # Install packages and requirements - printf $INIT_INPUT | ./init/init.sh || true # Source bashrc - source init/bashrc || true - source /home/kitcar/kitcar-rosbag/devel/setup.bash --extend stages: - lint - build - test - deploy lint-code: stage: lint variables: GIT_LFS_SKIP_SMUDGE: '1' # Don't pull Git LFS script: # Check if all files are correctly formatted and comply to our formatting rules. - pre-commit run --all-files lint-catkin: stage: lint only: changes: - "**/CMakeLists.txt" - "**/package.xml" script: - catkin_lint --resolve-env . --ignore unknown_package ######## BUILD STAGE ######## # Build simulation build-code: stage: build needs: [] variables: GIT_LFS_SKIP_SMUDGE: '1' # Don't pull Git LFS script: - cd simulation - catkin_make artifacts: paths: - simulation/devel - simulation/build expire_in: 2 day ######## TEST STAGE ######### # Run module test in utils packages test-python-packages: stage: test needs: ["build-code"] variables: GIT_LFS_SKIP_SMUDGE: '1' # Don't pull Git LFS script: - coverage run -m simulation.utils.geometry.test - coverage run -a -m simulation.utils.ros_base.test.parameter_object_test - coverage run -a -m simulation.utils.road.sections.test - coverage run -a -m simulation.src.simulation_evaluation.src.speaker.speakers.test - coverage run -a -m simulation.src.simulation_evaluation.src.state_machine.test # Test the urdf package - coverage run -a -m simulation.utils.urdf.test.test_core # Collect coverage - coverage report - cp .coverage .coverage_py artifacts: paths: - .coverage_py expire_in: 2 days test-machine-learning-package: image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci stage: test needs: ["build-code"] variables: GIT_LFS_SKIP_SMUDGE: '1' # Don't pull Git LFS CUDA_VISIBLE_DEVICES: "" # Don't use CUDA in CI script: - coverage run -m simulation.utils.machine_learning.models.test - coverage run -a -m simulation.utils.machine_learning.cycle_gan.models.test - coverage run -a -m simulation.utils.machine_learning.data.test # Test if the camera publishes images with the cycle gan turned on - dvc pull simulation/utils/machine_learning/cycle_gan/checkpoints/dr_drift/latest_net_g_b_to_a.pth - git lfs pull --include=simulation/utils/machine_learning/cycle_gan/resources/mask.png - rostest simulation_brain_link sensor_camera.test apply_gan:=true use_wasserstein_gan:=false - rostest simulation_brain_link sensor_camera.test apply_gan:=true use_wasserstein_gan:=true # Collect coverage - mv /root/.ros/.coverage* . - coverage combine -a - coverage report - cp .coverage .coverage_ml artifacts: paths: - .coverage_ml expire_in: 2 days test-ros-packages: stage: test needs: ["build-code"] script: - cd simulation - catkin_make run_tests -j1 - catkin_test_results # Ensure that pipeline fails when test fails - cd .. # Collect coverage - mv /root/.ros/.coverage* . - coverage combine - coverage report - cp .coverage .coverage_ros artifacts: paths: - .coverage_ros expire_in: 2 days # Run tests to ensure that the evaluation pipeline is working test-drive-evaluation: stage: test needs: ["build-code"] tags: # Always require to use the workstation to ensure enough computational power - workstation variables: PATH_FILES: $KITCAR_REPO_PATH/kitcar-gazebo-simulation/simulation/src/simulation_evaluation/param/drive/paths parallel: 3 script: # The drive.test is used to evaluate how the car drives. # With automatic_drive:=true the path of the car is predetermined and thus the correctness # of the evaluation can be tested. - mkdir rosbags - python3 -m simulation.utils.drive_test.run --config simulation/utils/drive_test/test/ci.yaml --runner_index $CI_NODE_INDEX --total_runners $CI_NODE_TOTAL artifacts: when: always paths: - rosbags expire_in: 1 week test-documentation: image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci stage: test needs: ["build-code"] script: # Get DVC files - dvc pull -R docs # Test if the onboarding documentation is correct - python3 docs/content/tutorials/road_examples/example.py - cd docs - make doctest - cd .. # Calculate coverage - docstr-coverage simulation/ -F 0 - echo "TOTAL $(docstr-coverage simulation/ -p -F 0 | cut -d . -f 1)%" ######### Documentation ######## # Build the sphinx documentation # If this is the master branch, the website deployer will also update doc.kitcar-team.de! build-documentation: image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci stage: deploy needs: ["test-documentation", "build-code"] script: # Get DVC files - dvc pull -R docs # Build the documentation - cd docs - make html artifacts: # Extract artifacts to view the documentation that has been generated paths: - docs/_build/html/* expire_in: 2 day collect-coverage: image: git.kitcar-team.de:4567/kitcar/kitcar-python-utils/ci/python:3.8.9-alpine stage: deploy needs: - build-code - test-python-packages - test-machine-learning-package - test-ros-packages before_script: # Install coverage pip package - pip install coverage script: - coverage combine .coverage_py .coverage_ml .coverage_ros - coverage xml - coverage html artifacts: paths: - htmlcov expire_in: 20 days reports: cobertura: coverage.xml cml-cyclegan: stage: deploy image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci:cml needs: [] rules: - if: '$CI_COMMIT_REF_NAME == "master"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never - when: manual allow_failure: true variables: INIT_INPUT: 1\ny # Install CUDA GIT_LFS_SKIP_SMUDGE: '1' # Don't pull Git LFS tags: - gpu script: - git lfs pull --include=simulation/utils/machine_learning/cycle_gan/resources/mask.png - dvc pull data/real_images/maschinen_halle_parking -j 8 - dvc pull data/simulated_images/test_images -j 8 - cd simulation/utils/machine_learning/cycle_gan # pull cache from server and checkout - dvc pull dvc.yaml -j 8 # Test current model, make videos and commit to dvc - dvc repro test_dr_drift -s - dvc repro test_discriminators_dr_drift -s - dvc repro make_video_dr_drift -s - cd ../../../../ # Create content for the report.md - echo "# Dr Drift" >> report.md - echo "## Test Generators" >> report.md - L="$(cml-publish simulation/utils/machine_learning/cycle_gan/results/dr_drift/stacked.mp4 --gitlab-uploads)" - echo "![]($L)" >> report.md - echo "[Link to mp4]($L)" >> report.md - echo "<br>" >> report.md - echo "## Test Discriminators" >> report.md - losses=`cat simulation/utils/machine_learning/cycle_gan/results/dr_drift/discriminator_losses.txt` - A="$(cut -d',' -f1 <<<"$losses")" - B="$(cut -d',' -f2 <<<"$losses")" - echo "**AVG-Loss Discriminator A =** $A" >> report.md - echo "<br>" >> report.md - echo "**AVG-Loss Discriminator B =** $B" >> report.md - echo "<br>" >> report.md - echo "## Loss values" >> report.md - cml-publish simulation/utils/machine_learning/cycle_gan/checkpoints/dr_drift/loss.png --gitlab-uploads --md --title 'Loss values' >> report.md # Send report.md as comment - cml-send-comment report.md # Build and deploy the CI docker image to the gitlab registry # This is done everytime the master branch is updated docker-images: stage: deploy image: docker:20.10.5 needs: ["build-code"] rules: - if: '$CI_COMMIT_REF_NAME == "master"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never - when: manual allow_failure: true tags: - docker - server services: - docker:20.10.5-dind variables: # Ensure that lfs files are available in resulting images! DOCKER_HOST: tcp://docker:2375 DOCKER_TLS_CERTDIR: '' before_script: - export IMAGE_URL=$CI_REGISTRY/kitcar/kitcar-gazebo-simulation - | if [ "$CI_COMMIT_REF_NAME" == "master" ]; then export CI_IMAGE_TAG=latest export CI_IMAGE_TAG_CML=cml export CI_IMAGE_TAG_ROS=latest else export CI_IMAGE_TAG=branch_${CI_COMMIT_REF_NAME} export CI_IMAGE_TAG_CML=branch_${CI_COMMIT_REF_NAME}_cml export CI_IMAGE_TAG_ROS=branch_${CI_COMMIT_REF_NAME} fi - apk add --no-cache docker-compose git - docker --version - docker info - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY script: # Clone kitcar-rosbag - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@git.kitcar-team.de/kitcar/kitcar-rosbag.git - cd docker # Build Docker images - docker-compose build # Push docker images - docker-compose push dvc-repro-images: image: git.kitcar-team.de:4567/kitcar/kitcar-gazebo-simulation/ci needs: ["build-code"] rules: - if: '$CI_COMMIT_REF_NAME == "master"' - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' when: never - when: manual allow_failure: true stage: deploy parallel: matrix: - DATA_FOLDER: [labeled_images, simulated_images] tags: - workstation script: - ./gitlab-ci/setup_cml_bot_git.sh # Run ssh-agent (inside the build environment) - eval $(ssh-agent -s) # Add the SSH key stored in CML_BOT_SSH_PRIVATE_KEY variable to the agent store - ssh-add <(echo "$CML_BOT_SSH_PRIVATE_KEY") - export NEW_BRANCH_NAME=ci/${CI_COMMIT_BRANCH}_${CI_JOB_ID} - echo ${NEW_BRANCH_NAME} - mv /home/kitcar/kitcar-rosbag $KITCAR_REPO_PATH - ls $KITCAR_REPO_PATH - source $KITCAR_REPO_PATH/kitcar-rosbag/devel/setup.bash --extend # Actual script! - dvc pull simulation/utils/machine_learning/cycle_gan/checkpoints/dr_drift/latest_net_g_b_to_a.pth - cd data/${DATA_FOLDER} - dvc pull dvc.yaml -j 8 || true - dvc repro -s - | # Check if lock file has changed if [ $(./../../gitlab-ci/has_file_changed.sh dvc.lock) -eq 1 ]; then git add dvc.lock # Push to DVC Remote dvc push -j 8 # Commit and Push to GitLab COMMIT_MSG="data: Reproduce ${DATA_FOLDER}" ./../../gitlab-ci/commit_and_push_to_branch.sh "${COMMIT_MSG}" "${NEW_BRANCH_NAME}" # Create Merge Request ENDPOINT_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests" BODY="{ \ \"id\": ${CI_PROJECT_ID}, \ \"source_branch\": \"${NEW_BRANCH_NAME}\", \ \"target_branch\": \"${CI_COMMIT_BRANCH}\",\ \"remove_source_branch\": true, \ \"title\": \"${COMMIT_MSG}\", \ \"labels\": \"topic::data\", \ \"assignee_id\": 600 \ }"; curl \ --request POST "${ENDPOINT_URL}" \ --header "PRIVATE-TOKEN: ${AUTO_MERGE_REQUEST}" \ --header "Content-Type: application/json" \ --data "${BODY}"; fi artifacts: paths: - data/labeled_images/dvc.lock - data/simulated_images/dvc.lock