Functional test cleanup and speedups (#5080)

* Fix warnings about ::set-output deprecation * Update github runner to 22.04 - now with native Qt6 packages * Cache grcov for functional test runs * Minimize profiling build artifacts. * Fix profile generation flags * Remove upload of .deb dependencies. * Install all dependencies when running functional tests. * Build MacOS unit tests with ninja * Update Linux unit test runner to 22.04 and use native Qt packages
mozilla-mobile · Nov 30, 2022 · 3befd81 · 3befd81
1 parent 614b564
commit 3befd81
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 49 deletions.
diff --git a/.github/workflows/functional_tests.yaml b/.github/workflows/functional_tests.yaml
@@ -17,29 +17,32 @@ concurrency:
 jobs:
   build_test_app:
     name: Build Test Client
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
       matrix: ${{ steps.testGen.outputs.tests }}
     steps:
       - name: Clone repository
         uses: actions/checkout@v3
         with: 
           submodules: 'true'
-      - name: Install dependecies
-        if: steps.cache-build.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p build/archive
 
-          # Install grcov into the build directory, to reduce downstream compilation times.
-          cargo install --root $(pwd)/build grcov 
+      - name: Install build dependecies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control)
 
-          # Add external PPA for Qt6 support on Ubuntu 20.04
-          sudo add-apt-repository ppa:okirby/qt6-backports
+      - name: Cache grcov
+        id: cache-grcov
+        uses: actions/cache@v3
+        with:
+          path: grcov-build/
+          key: ${{runner.os}}-grcov-v0.8.13
 
-          # Download the build and runtime package dependencies.
-          sudo apt-get -o "Dir::Cache::archives=$(pwd)/build/archive" install -y \
-               $(./scripts/linux/getdeps.py -a linux/debian/control)
-          sudo chown -R $USER:$USER build/archive
+      - name: Install Grcov
+        if: steps.cache-grcov.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.13
 
       - name: Compile test client
         shell: bash
@@ -48,23 +51,33 @@ jobs:
           pip3 install -r requirements.txt
 
           mkdir -p build/cmake
+          mkdir -p build/profile
           cmake -S $(pwd) -B build/cmake -DBUILD_DUMMY=ON \
-              -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_EXE_LINKER_FLAGS=--coverage
+              -DCMAKE_CXX_FLAGS="--coverage -fprofile-generate=$(pwd)/build/profile" \
+              -DCMAKE_EXE_LINKER_FLAGS="--coverage -fprofile-generate=$(pwd)/build/profile"
           cmake --build build/cmake -j$(nproc)
 
+          mkdir -p build/profile
+          rsync -a --include '*/' --include '*.gcno' --exclude '*' \
+              build/cmake/src/CMakeFiles/mozillavpn.dir/ build/profile/
+          cp ./build/cmake/src/mozillavpn build/
+
       - uses: actions/upload-artifact@v3
         with:
           name: test-client-${{ github.sha }}
-          path: build/
+          path: |
+            build/
+            !build/cmake/
 
       - name: Generate tasklist
         id: testGen
         shell: bash
         run: |
-          echo -n "::set-output name=tests::"
+          echo -n "tests=" >> $GITHUB_OUTPUT
           for test in $(find tests/functional -name 'test*.js' | sort); do
             printf '{"name": "%s", "path": "%s"}' $(basename ${test%.js} | sed -n 's/test//p') $test
-          done | jq -s -c
+          done | jq -s -c >> $GITHUB_OUTPUT
+
       - name: Check tests
         shell: bash
         env:
@@ -76,7 +89,7 @@ jobs:
     name: Functional tests
     needs: 
       - build_test_app
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 45
     strategy:
       fail-fast: false # Don't cancel other jobs if a test fails
@@ -85,23 +98,35 @@ jobs:
     steps:
       - name: Clone repository
         uses: actions/checkout@v3
+
       - uses: actions/download-artifact@v3
         with:
           name: test-client-${{ github.sha }}
           path: build/
 
-      - name: Install dependecies
+      - name: Install test dependecies
         run: |
-          sudo apt install ./build/archive/*.deb
+          sudo apt-get update
+          sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control)
           sudo apt install --no-upgrade firefox xvfb -y
           pip3 install -r requirements.txt
           npm install
-      
+
+      - name: Cache grcov
+        id: cache-grcov
+        uses: actions/cache@v3
+        with:
+          path: grcov-build/
+          key: ${{runner.os}}-grcov-v0.8.13
+
       - name: Check build
         shell: bash
         run: |
-            chmod +x ./build/cmake/src/mozillavpn
-            ./build/cmake/src/mozillavpn -v
+            chmod +x ./build/mozillavpn
+            ./build/mozillavpn -v
+
+            chmod +x ${{github.workspace}}/grcov-build/bin/grcov
+            ${{github.workspace}}/grcov-build/bin/grcov --version
 
       - name: Build addons
         shell: bash
@@ -127,15 +152,15 @@ jobs:
             xvfb-run -a npm run functionalTest -- ${{matrix.test.path}}
         env:
           ARTIFACT_DIR: ${{ runner.temp }}/artifacts
-          MVPN_BIN: ./build/cmake/src/mozillavpn
+          MVPN_BIN: ./build//mozillavpn
 
       - name: Generating grcov reports
         id: generateGrcov
         continue-on-error: true # Ignore GRCOV parsing errors, see github.com/mozilla/grcov/issues/570
         timeout-minutes: 1 # Give GRCOV a short timeout in case it hangs after a panic
         run: |
-          chmod +x ./build/bin/grcov
-          ./build/bin/grcov build/cmake/src/CMakeFiles/mozillavpn.dir \
+          export PATH=${{github.workspace}}/grcov-build/bin:$PATH
+          grcov build/profile \
               -s src -t lcov --branch --ignore-not-existing \
               -o ${{runner.temp}}/artifacts/functional_lcov.info
 

diff --git a/.github/workflows/ppa-automation.yaml b/.github/workflows/ppa-automation.yaml
@@ -49,7 +49,7 @@ jobs:
 
           if [[ "$GITREF" == "refs/heads/main" ]]; then
             if [[ ! -z $(git rev-list --after="25 hours" ${{github.sha}}) ]]; then
-              echo "::set-output name=nightly-changes::true"
+              echo "nightly-changes=true" >> $GITHUB_OUTPUT
             fi
           fi
 

diff --git a/.github/workflows/test_lottie.yaml b/.github/workflows/test_lottie.yaml
@@ -36,13 +36,13 @@ jobs:
         uses: actions/cache@v3
         with:
           path: grcov-build/
-          key: ${{runner.os}}-grcov-v0.8.9
+          key: ${{runner.os}}-grcov-v0.8.13
 
       - name: Install Grcov
         if: steps.cache-grcov.outputs.cache-hit != 'true'
         shell: bash
         run: |
-          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.9
+          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.13
 
       - name: Building tests
         shell: bash
@@ -96,13 +96,13 @@ jobs:
         uses: actions/cache@v3
         with:
           path: grcov-build/
-          key: ${{runner.os}}-grcov-v0.8.9
+          key: ${{runner.os}}-grcov-v0.8.13
 
       - name: Install Grcov
         if: steps.cache-grcov.outputs.cache-hit != 'true'
         shell: bash
         run: |
-          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.9
+          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.13
 
       - name: Building tests
         shell: bash

diff --git a/.github/workflows/test_unit.yaml b/.github/workflows/test_unit.yaml
@@ -16,9 +16,7 @@ concurrency:
 
 jobs:
   linux-unit-tests:
-    runs-on: ubuntu-20.04
-    env:
-      QTVERSION: 6.2.4
+    runs-on: ubuntu-22.04
     name: Run Unit tests on Linux
     steps:
       - name: Clone repository
@@ -28,31 +26,28 @@ jobs:
 
       - name: Install dependences
         run: |
-          sudo apt update
-          sudo apt install git libgl-dev libegl-dev libpolkit-gobject-1-dev clang-10 liboath-dev python3 -y
-          python3 -m pip install -r requirements.txt
-          python3 -m pip install aqtinstall
-          aqt install-qt --outputdir /opt linux desktop $QTVERSION gcc_64 -m all
+          sudo apt-get update
+          sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control)
+          pip3 install -r requirements.txt
 
       - name: Cache grcov
         id: cache-grcov
         uses: actions/cache@v3
         with:
           path: grcov-build/
-          key: ${{runner.os}}-grcov-v0.8.9
+          key: ${{runner.os}}-grcov-v0.8.13
 
       - name: Install Grcov
         if: steps.cache-grcov.outputs.cache-hit != 'true'
         shell: bash
         run: |
-          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.9
+          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.13
 
       - name: Building tests
         shell: bash
         run: |
           mkdir -p build
-          cmake -S . -B $(pwd)/build -DBUILD_TESTING=ON \
-            -DCMAKE_PREFIX_PATH=/opt/$QTVERSION/gcc_64/lib/cmake/ \
+          cmake -S . -B $(pwd)/build \
             -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_EXE_LINKER_FLAGS=--coverage
           cmake --build $(pwd)/build --target build_tests -j$(nproc)
 
@@ -93,10 +88,12 @@ jobs:
         uses: actions/checkout@v3
         with: 
           submodules: 'true'
-      - name: Install python dependencies
+
+      - name: Install dependencies
         shell: bash
         run: |
           pip3 install -r requirements.txt
+          brew install ninja
 
       - name: Install Qt6
         shell: bash
@@ -114,20 +111,20 @@ jobs:
         uses: actions/cache@v3
         with:
           path: grcov-build/
-          key: ${{runner.os}}-grcov-v0.8.9
+          key: ${{runner.os}}-grcov-v0.8.13
 
       - name: Install Grcov
         if: steps.cache-grcov.outputs.cache-hit != 'true'
         shell: bash
         run: |
-          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.9
+          cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.13
 
       - name: Building tests
         shell: bash
         run: |
           export PATH=/opt/qt6/bin:${{github.workspace}}/grcov-build/bin:$PATH
           mkdir -p build
-          cmake -S . -B $(pwd)/build -DBUILD_TESTING=ON \
+          cmake -S . -B $(pwd)/build -GNinja \
             -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_EXE_LINKER_FLAGS=--coverage
           cmake --build $(pwd)/build --target build_tests
 

diff --git a/.github/workflows/wasm.yaml b/.github/workflows/wasm.yaml
@@ -63,10 +63,10 @@ jobs:
         id: testGen
         shell: bash
         run: |
-          echo -n "::set-output name=tests::"
+          echo -n "tests=" >> $GITHUB_OUTPUT
           for test in $(find tests/functional -name 'test*.js' | sort); do
             grep "$(basename ${test%.js})" tests/functional/wasm-test-ignore &>/dev/null || printf '{"name": "%s", "path": "%s"}' $(basename ${test%.js} | sed -n 's/test//p') $test
-          done | jq -s -c
+          done | jq -s -c >> $GITHUB_OUTPUT
 
       - name: Check tests
         shell: bash