functional_tests.yaml

name: Functional tests
on:
  push:
    branches:
      - main
      - "releases/**"
  pull_request:
    branches:
      - main
      - "releases/**"

# Restrict tests to the most recent commit.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build_test_app:
    name: Build Test Client
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.testGen.outputs.tests }}
    steps:
      - name: Clone repository
        uses: actions/checkout@v3
        with:
          submodules: 'recursive'

      - name: Install build dependecies
        run: |
          sudo apt-get update
          sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control)

      - name: Cache grcov
        id: cache-grcov
        uses: actions/cache@v3
        with:
          path: grcov-build/
          key: ${{runner.os}}-grcov-v0.8.13

      - name: Install Grcov
        if: steps.cache-grcov.outputs.cache-hit != 'true'
        shell: bash
        run: |
          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.13

      - name: Compile test client
        shell: bash
        if: steps.cache-build.outputs.cache-hit != 'true'
        run: |
          pip3 install -r requirements.txt

          mkdir -p build/cmake
          mkdir -p build/profile
          cmake -S $(pwd) -B build/cmake \
              -DCMAKE_CXX_FLAGS="--coverage -fprofile-generate=$(pwd)/build/profile" \
              -DCMAKE_EXE_LINKER_FLAGS="--coverage -fprofile-generate=$(pwd)/build/profile"
          cmake --build build/cmake -j$(nproc) --target dummyvpn

          mkdir -p build/profile
          rsync -a --include '*/' --include '*.gcno' --exclude '*' \
              build/cmake/tests/dummyvpn/CMakeFiles/dummyvpn.dir/ build/profile/
          cp ./build/cmake/tests/dummyvpn/dummyvpn build/

      - name: Compile test addons
        shell: bash
        if: steps.cache-build.outputs.cache-hit != 'true'
        run: |
          mkdir -p build/addons
          cmake -S $(pwd)/tests/functional/addons -B build/addons
          cmake --build build/addons

      - uses: actions/upload-artifact@v3
        with:
          name: test-client-${{ github.sha }}
          path: |
            build/
            !build/cmake/

      - name: Generate tasklist
        id: testGen
        shell: bash
        run: |
          echo -n "tests=" >> $GITHUB_OUTPUT
          for test in $(find tests/functional -name 'test*.js' | sort); do
            printf '{"name": "%s", "path": "%s"}' $(basename ${test%.js} | sed -n 's/test//p') $test
          done | jq -s -c >> $GITHUB_OUTPUT

      - name: Check tests
        shell: bash
        env:
          TEST_LIST: ${{ steps.testGen.outputs.tests }}
        run: |
          echo $TEST_LIST | jq

  functionaltests:
    name: Functional tests
    needs:
      - build_test_app
    runs-on: ubuntu-22.04
    timeout-minutes: 45
    strategy:
      fail-fast: false # Don't cancel other jobs if a test fails
      matrix:
        test: ${{ fromJson(needs.build_test_app.outputs.matrix) }}
    steps:
      - name: Clone repository
        uses: actions/checkout@v3

      - uses: actions/download-artifact@v3
        with:
          name: test-client-${{ github.sha }}
          path: build/

      - name: Install test dependecies
        run: |
          sudo apt-get update
          sudo apt-get install -y $(./scripts/linux/getdeps.py -r linux/debian/control)
          sudo apt install --no-upgrade firefox xvfb -y
          pip3 install -r requirements.txt
          npm install

      - name: Cache grcov
        id: cache-grcov
        uses: actions/cache@v3
        with:
          path: grcov-build/
          key: ${{runner.os}}-grcov-v0.8.13

      - name: Check build
        shell: bash
        run: |
          chmod +x ./build/dummyvpn
          ./build/dummyvpn -v

          chmod +x ${{github.workspace}}/grcov-build/bin/grcov
          ${{github.workspace}}/grcov-build/bin/grcov --version

      - name: Running ${{matrix.test.name}} Tests
        id: runTests
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 15
          max_attempts: 3
          command: |
            export PATH=$GECKOWEBDRIVER:$(npm bin):$PATH
            export HEADLESS=yes
            export TZ=Europe/London
            mkdir -p $ARTIFACT_DIR
            xvfb-run -a npm run functionalTest --  --retries 3 ${{matrix.test.path}}
        env:
          ARTIFACT_DIR: ${{ runner.temp }}/artifacts
          MVPN_BIN: ./build/dummyvpn
          MVPN_ADDONS_PATH: ./build/addons

      - name: Generating grcov reports
        id: generateGrcov
        continue-on-error: true # Ignore GRCOV parsing errors, see github.com/mozilla/grcov/issues/570
        timeout-minutes: 1 # Give GRCOV a short timeout in case it hangs after a panic
        run: |
          export PATH=${{github.workspace}}/grcov-build/bin:$PATH
          grcov build/profile \
              -s src -t lcov --branch --ignore-not-existing \
              -o ${{runner.temp}}/artifacts/functional_lcov.info

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        if: steps.generateGrcov.outcome == 'success'
        with:
          directory: .
          flags: functional_tests
          name: codecov-poc
          files: ${{runner.temp}}/artifacts/functional_lcov.info
          verbose: true

      - name: Uploading artifacts
        uses: actions/upload-artifact@v3
        if: ${{ always() }}
        with:
          name: ${{matrix.test.name}} Logs
          path: ${{ runner.temp }}/artifacts