.gitlab-ci.yml

stages:
  - test
  - doc
  - build
  - build_check
  - notif
  - schedule
  - schedule-test

before_script:
  - export COVERAGE_FILE=coverage-$CI_NODE_INDEX-$CI_PIPELINE_ID
  - echo 'sudo --preserve-env=PYTHONPATH,CI_NODE_TOTAL,CI_NODE_INDEX,COVERAGE_FILE -u www-data python3-coverage run -a --source se,sosse ./sosse/sosse_admin.py "$@"' > /tmp/sudo_sosse
  - chmod 755 /tmp/sudo_sosse
  - test -z "$APT_PROXY" || (echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf)

variables:
  GIT_SUBMODULE_STRATEGY: recursive

unit_tests:
  image: biolds/sosse:debian-test
  parallel: 3
  artifacts:
    when: always
    paths:
      - se/migrations/*.py
      - log/*
      - coverage-*
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  script:
    - export PYTHONPATH="$CI_PROJECT_DIR"
    - make _docker_unit_test_prepare
    - failed=0
    - /tmp/sudo_sosse test -v3 --failfast || failed=1
    - /tmp/sudo_sosse load_se tests/opensearch.xml || failed=1
    - /tmp/sudo_sosse update_se || failed=1
    - /tmp/sudo_sosse shell -c 'from django.contrib.auth.models import User; User.objects.all().delete()' || failed=1
    - /tmp/sudo_sosse default_admin || failed=1
    - /tmp/sudo_sosse generate_secret || failed=1
    - /tmp/sudo_sosse extract_doc cli > /dev/null || failed=1
    - /tmp/sudo_sosse extract_doc conf > /dev/null || failed=1
    - /tmp/sudo_sosse extract_doc se > /dev/null || failed=1
    - python3 ./sosse/sosse_admin.py makemigrations || failed=1
    - /tmp/sudo_sosse migrate || failed=1
    - /tmp/sudo_sosse migrate se 0001 || failed=1
    - mv /var/log/sosse log
    - exit $failed

coverage_report:
  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
  image: biolds/sosse:debian-test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  needs:
    - unit_tests
  artifacts:
    when: always
    paths:
      - htmlcov/*
    reports:
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml
  script:
    - export COVERAGE_FILE=coverage
    - python3-coverage combine --data-file=coverage coverage-*
    - python3-coverage report --omit "*/se/deps/*,*/dist-packages/*"
    - python3-coverage html --omit "*/se/deps/*,*/dist-packages/*"
    - python3-coverage xml --omit "*/se/deps/*,*/dist-packages/*"

.functional_tests: &functional_tests
  image: biolds/sosse:pip-test
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  needs:
    - doc_code_extract
  artifacts:
    when: always
    paths:
      - tests/robotframework/screenshots/*
      - tests/robotframework/*.html
      - log/*
  script:
    - make _pip_functional_tests
    - (make _rf_functional_$TEST_SUITE ; echo -n $? > /tmp/ret_code) || true
    - mv /var/log/sosse log
    - test "$(cat /tmp/ret_code)" -eq 0

functional_tests_chromium:
  <<: *functional_tests
  variables:
    TEST_SUITE: tests
    BROWSER: chromium

functional_tests_firefox:
  <<: *functional_tests
  variables:
    TEST_SUITE: tests
    BROWSER: firefox

functional_guides_chromium:
  <<: *functional_tests
  variables:
    TEST_SUITE: guides
    BROWSER: chromium

migrations:
  image: biolds/sosse:debian-test
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  script:
    - apt update
    - 'grep ^Depends: debian/control | sed -e "s/.*},//" -e "s/,//g" | xargs apt install -y'
    - /etc/init.d/postgresql start
    - export PYTHONPATH="$CI_PROJECT_DIR"
    - chown www-data:www-data -R /var/log/sosse
    - /tmp/sudo_sosse default_conf | sed -e "s/^#db_pass=.*/db_pass=sosse/" -e "s/^#debug=.*/debug=true/" > /etc/sosse/sosse.conf
    - MIGRATION_COUNT="$(ls se/migrations/*.py|grep '/[0-9]'|wc -l)"
    - python3 ./sosse/sosse_admin.py makemigrations
    - MIGRATION_COUNT2="$(ls se/migrations/*.py|grep '/[0-9]'|wc -l)"
    - test "$MIGRATION_COUNT" -eq "$MIGRATION_COUNT2"

static_checks:
  image: biolds/sosse:debian-test
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  script:
    - make static_checks

pre_commit:
  image: biolds/sosse:debian-test
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  script:
    - /pre-commit-venv/bin/pre-commit autoupdate
    - git diff --exit-code
    - /pre-commit-venv/bin/pre-commit run -a

doc_gen:
  image: biolds/sosse:debian-test
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  artifacts:
    paths:
      - doc/source/*_generated.rst
      - doc/source/*/*_generated.rst
      - doc/source/CHANGELOG.md
      - doc/source/introduction.md
  script:
    - make doc_gen

doc_code_extract:
  image: biolds/sosse:doc
  stage: test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  needs:
    - doc_gen
  artifacts:
    paths:
      - doc/code_blocks.json
  script:
    - sed -e 's/|[^|]*| //g' -e 's/^.. \(image\|figure\)::.*//' -i $(find doc -name \*.rst)
    - make _build_doc

doc:
  image: biolds/sosse:doc
  stage: doc
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  artifacts:
    paths:
      - doc/build/*
  needs:
    # if any artifact is need to bild the doc, then its build step must also be added to the `jobs_name` in doc/get_artifacts.py
    - doc_gen
    - functional_tests_chromium
    #- functional_tests_firefox -> commented because only the artifacts (screenshot) from Chromium are needed
    - functional_guides_chromium
  script:
    - make _build_doc
    - test -n "$RTD_TOKEN" && curl -X POST -d "branches=main" -d "token=$RTD_TOKEN" -d "default_branch=main" https://readthedocs.org/api/v2/webhook/sosse/236935/ || true

pip_pkg:
  image: biolds/sosse:pip-test
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  artifacts:
    paths:
      - dist/*
  stage: build
  needs:
    - doc
    - migrations
    - unit_tests
    - functional_tests_chromium
    - functional_tests_firefox
    - static_checks
  script:
    - sed -e "s/^SOSSE_VERSION_TAG = .*/SOSSE_VERSION_TAG = '${CI_COMMIT_TAG/v/}'/" -i sosse/settings.py
    - sed -e "s/^SOSSE_VERSION_COMMIT = .*/SOSSE_VERSION_COMMIT = '${CI_COMMIT_SHORT_SHA}'/" -i sosse/settings.py
    - make _pip_pkg

debian_pkg:
  image: biolds/sosse:debian-pkg
  #image: registry.gitlab.com/biolds1/sosse/debian-pkg:latest
  stage: build
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  needs:
    - doc
    - migrations
    - unit_tests
    - functional_tests_chromium
    - functional_tests_firefox
    - static_checks
  artifacts:
    paths:
      - deb/*.deb
  script:
    - cd "$CI_PROJECT_DIR"
    - rm -rf htmlcov
    - sed -e "s/^SOSSE_VERSION_TAG = .*/SOSSE_VERSION_TAG = '${CI_COMMIT_TAG/v/}'/" -i sosse/settings.py
    - sed -e "s/^SOSSE_VERSION_COMMIT = .*/SOSSE_VERSION_COMMIT = '${CI_COMMIT_SHORT_SHA}'/" -i sosse/settings.py
    - mkdir /deb
    - make _deb
    - mv /deb .

changelog:
  image: biolds/sosse:debian-test
  stage: build
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  needs: []
  artifacts:
    paths:
      - CHANGELOG.md
  variables:
    GIT_DEPTH: 0 # disable shallow cloning since the step requires all tags
  script:
    - 'last_ver="$(git tag -l | sort -V | tail -n 1 | sed -e "s/^v//")"'
    - 'new_ver="${last_ver%.*}.$((${last_ver/*./} + 1))"'
    - echo "$last_ver / $new_ver"
    - 'curl -X GET -H "PRIVATE-TOKEN: $GITLAB_REST_API_CHANGELOG" "https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/repository/changelog?version=$new_ver" | jq -r .notes | tee /tmp/new'
    - cat /tmp/new CHANGELOG.md > new.md && mv new.md CHANGELOG.md

doc_test_debian:
  image: debian:bookworm
  stage: build_check
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  needs:
    - doc_code_extract
  script:
    - apt update
    - apt install -y make jq
    - make _doc_test_debian

doc_test_pip:
  image: debian:bookworm
  stage: build_check
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  needs:
    - doc_code_extract
  script:
    - apt update
    - apt install -y make jq
    - make _doc_test_pip

.debian_pkg_check: &debian_pkg_check
  image: biolds/sosse:debian-base
  stage: build_check
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  artifacts:
    when: always
    paths:
      - tests/robotframework/screenshots/*
      - tests/robotframework/*.html
      - log/*
  needs:
    - doc_code_extract
    - debian_pkg
  script:
    - apt update
    - apt install -y jq make virtualenv
    - make _deb_pkg_functional_tests
    - (make _rf_functional_tests ; echo -n $? > /tmp/ret_code) || true
    - mv /var/log/sosse log
    - test "$(cat /tmp/ret_code)" -eq 0

debian_pkg_check_chromium:
  <<: *debian_pkg_check
  variables:
    TEST_SUITE: tests
    BROWSER: chromium

debian_pkg_check_firefox:
  <<: *debian_pkg_check
  variables:
    TEST_SUITE: tests
    BROWSER: firefox

.pip_pkg_check: &pip_pkg_check
  image: biolds/sosse:pip-test
  stage: build_check
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
  artifacts:
    when: always
    paths:
      - tests/robotframework/screenshots/*
      - tests/robotframework/*.html
      - log/*
  needs:
    - doc_code_extract
    - pip_pkg
  script:
    - make _pip_pkg_functional_tests
    - (make _rf_functional_tests ; echo -n $? > /tmp/ret_code) || true
    - mv /var/log/sosse log
    - test "$(cat /tmp/ret_code)" -eq 0

pip_pkg_check_chromium:
  <<: *pip_pkg_check
  variables:
    TEST_SUITE: tests
    BROWSER: chromium

pip_pkg_check_firefox:
  <<: *pip_pkg_check
  variables:
    TEST_SUITE: tests
    BROWSER: firefox

discord_tag_notif:
  image: debian:bookworm
  stage: notif
  allow_failure: true
  rules:
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
  needs:
    - doc_gen
    - pip_pkg
    - debian_pkg_check_chromium
    - debian_pkg_check_firefox
  script: |
    test -n "$DISCORD_TAG_NOTIF_URL"
    apt update
    apt install -y jq curl make
    changelog="$(grep -v '^# Changelog' < doc/source/CHANGELOG.md | sed '/^$/Q' | sed -e 's#(\(https\?://[^)]*\))#(<\1\>)#g')"
    msg="$(jq -n --arg tag "$CI_COMMIT_TAG" --arg changelog "$changelog" '{"content": ($tag + " was just released! 🎉\n\n" + $changelog)}')"
    curl -X POST -H 'Content-Type: application/json' -d "$msg" "$DISCORD_TAG_NOTIF_URL"

rebuild_upgrade_check:
  image: biolds/sosse:latest
  stage: schedule
  needs: []
  artifacts:
    when: always
    paths:
      - pkg_status/*
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "stable"
    - if: $CI_PIPELINE_SOURCE == "push"
  script:
    - test "$CI_PIPELINE_SOURCE" != push || exit 0
    - apt update
    - mkdir pkg_status
    - apt list --upgradable | tee pkg_status/apt_list_upgradable
    - /venv/bin/pip install -r requirements.txt --upgrade --dry-run | tee pkg_status/pip_upgrade

rebuild_docker:
  image: biolds/sosse:docker
  stage: schedule
  tags:
    - docker-socket
  needs:
    - job: rebuild_upgrade_check
      artifacts: true
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "stable"
    - if: $CI_PIPELINE_SOURCE == "push"
  script:
    - test "$CI_PIPELINE_SOURCE" = push || test -e pkg_status/apt_list_upgradable
    - test "$CI_PIPELINE_SOURCE" = push || test -e pkg_status/pip_upgrade
    - mkdir -p ~/.docker
    - 'echo "{\"auths\":{\"registry.gitlab.com\":{\"auth\":\"$GITLAB_DOCKER_REGISTRY_KEY\"}}}" > ~/.docker/config.json'
    - "if (cat pkg_status/apt_list_upgradable | grep -v ^Listing | grep -q .) || (grep -q ^Would pkg_status/pip_upgrade) ; then NO_BUILD=0 ; else NO_BUILD=1 ; fi"
    - "test $NO_BUILD -eq 0 || docker pull debian:bookworm"
    - "test $NO_BUILD -eq 0 || docker tag debian:bookworm registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"
    - "test $NO_BUILD -eq 0 || docker push registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"
    - "test $NO_BUILD -eq 0 || exit 0"
    - "docker rmi biolds/sosse:pip-base ||:"
    - "docker rmi biolds/sosse:pip-release ||:"
    - test "$CI_PIPELINE_SOURCE" != schedule || make docker_release_build
    - test "$CI_PIPELINE_SOURCE" != push || make docker_release_build_test
    - "docker tag biolds/sosse:pip-release registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"
    - "docker push registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"

rebuild_unit_tests:
  image: "registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"
  needs:
    - job: rebuild_docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "stable"
    - if: $CI_PIPELINE_SOURCE == "push"
  stage: schedule-test
  script:
    - "test -d /venv || exit 0" # No upgrade was built, we exit without error
    - "/etc/init.d/postgresql start ||:"
    - "until pg_isready; do sleep 1; done"
    - apt-get update
    - apt-get install -y rsync git
    - sed -e 's#python3#/venv/bin/python3#' -i ./tests/test_app.sh
    - ./tests/test_app.sh
    - bash -c "$(grep -E '(mkdir|touch|chown|test)' /run.sh)" # Create log dir and files
    - /venv/bin/sosse-admin default_conf | sed -e "s/^#db_pass=.*/db_pass=sosse/" -e "s/^#debug=.*/debug=true/" -e "s/#dl_check_time=.*/dl_check_time=1/" -e "s,#online_check_url.*,online_check_url=http://localhost:8000/," -e "s/^#chromium_options=\(.*\)/chromium_options=\1 --no-sandbox --disable-dev-shm-usage/" > /etc/sosse/sosse.conf
    - cat /etc/sosse/sosse.conf
    - su - postgres -c "psql --command 'ALTER USER sosse WITH SUPERUSER;'"
    - sudo -E -u www-data /venv/bin/python3 -m sosse.sosse_admin test -v3

rebuild_functional_tests_firefox:
  image: "registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"
  needs:
    - job: rebuild_docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "stable"
    - if: $CI_PIPELINE_SOURCE == "push"
  stage: schedule-test
  artifacts:
    when: always
    paths:
      - tests/robotframework/screenshots/*
      - tests/robotframework/*.html
      - log/*
  script:
    - "test -d /venv || exit 0" # No upgrade was built, we exit without error
    - /run.sh &
    - apt-get update
    - apt-get install -y rsync git curl
    - sed -e 's#python3#/venv/bin/python3#' -i ./tests/test_app.sh
    - ./tests/test_app.sh
    - echo -e "#!/bin/bash\nexec /venv/bin/python3 -m sosse.sosse_admin \"\$@\"" > sosse-admin.sh
    - chmod 755 sosse-admin.sh
    - 'echo "SOSSE_ADMIN: $PWD/sosse-admin.sh" > tests/robotframework/config.yaml'
    - until curl --silent --fail http://127.0.0.1/ > /dev/null; do sleep 1; done
    - mkdir -p /var/lib/sosse/screenshots && git clone --depth=1 https://github.com/GurvanKervern/dummy-static-website /var/lib/sosse/screenshots/website
    - make _rf_functional_tests

rebuild_functional_tests_chromium:
  image: "registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"
  needs:
    - job: rebuild_docker
  stage: schedule-test
  artifacts:
    when: always
    paths:
      - tests/robotframework/screenshots/*
      - tests/robotframework/*.html
      - log/*
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "stable"
    - if: $CI_PIPELINE_SOURCE == "push"
  script:
    - "test -d /venv || exit 0" # No upgrade was built, we exit without error
    - /run.sh &
    - apt-get update
    - apt-get install -y rsync git curl
    - sed -e 's#python3#/venv/bin/python3#' -i ./tests/test_app.sh
    - ./tests/test_app.sh
    - echo -e "#!/bin/bash\nexec /venv/bin/python3 -m sosse.sosse_admin \"\$@\"" > sosse-admin.sh
    - chmod 755 sosse-admin.sh
    - 'echo "SOSSE_ADMIN: $PWD/sosse-admin.sh" > tests/robotframework/config.yaml'
    - until curl --silent --fail http://127.0.0.1/ > /dev/null; do sleep 1; done
    - mkdir -p /var/lib/sosse/screenshots && git clone --depth=1 https://github.com/GurvanKervern/dummy-static-website /var/lib/sosse/screenshots/website
    - make _rf_functional_tests

rebuild_docker_push:
  image: biolds/sosse:docker
  tags:
    - docker-socket
  needs:
    - job: rebuild_unit_tests
    - job: rebuild_functional_tests_firefox
    - job: rebuild_functional_tests_chromium
    - job: rebuild_upgrade_check
      artifacts: true
  stage: schedule-test
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_BRANCH == "stable"
  script:
    - "if (cat pkg_status/apt_list_upgradable | grep -v ^Listing | grep -q .) || (grep -q ^Would pkg_status/pip_upgrade) ; then NO_BUILD=0 ; else NO_BUILD=1 ; fi"
    - "test $NO_BUILD -eq 0 || exit 0"
    - mkdir ~/.docker/
    - 'echo "{\"auths\":{\"registry.gitlab.com\":{\"auth\":\"$GITLAB_DOCKER_REGISTRY_KEY\"}}}" > ~/.docker/config.json'
    - docker pull "registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID"
    - docker tag "registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID" biolds/sosse:latest
    - docker tag "registry.gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_PIPELINE_ID" biolds/sosse:stable
    - 'echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$DOCKER_KEY\"}}}" > ~/.docker/config.json'
    - docker push biolds/sosse:latest
    - docker push biolds/sosse:stable