Add Ctest support for functional testing (#10064)

* Add ctest support for functional testing * Use ctest for the Linux functional workflow * Download addons into build dir * Use QT_QPA_PLATFORM=offscreen instead of xvfb-run * Add socks proxy tests to the unit test group * Fix socks5connection tests on MacOS
mozilla-mobile · Nov 25, 2024 · 68c9c62 · 68c9c62
1 parent f4f7ecb
commit 68c9c62
Show file tree

Hide file tree

Showing 14 changed files with 105 additions and 78 deletions.
diff --git a/.github/workflows/auth_tests.yml b/.github/workflows/auth_tests.yml
@@ -36,9 +36,7 @@ jobs:
 
       - name: Running tests
         shell: bash
-        working-directory: ./build/tests/auth_tests
-        run: |
-          ctest --output-on-failure
+        run: ctest -L auth --output-on-failure
 
   macos-unit-tests:
     runs-on: macos-latest
@@ -78,9 +76,7 @@ jobs:
           cmake --build build/cmake --target app_auth_tests
 
       - name: Running tests
-        working-directory: ./build/cmake/tests/auth_tests
-        run: |
-          ctest --output-on-failure
+        run: ctest -L auth --output-on-failure
 
   windows-unit-tests:
     name: Run auth tests on Windows
@@ -118,6 +114,4 @@ jobs:
 
       - name: Running tests
         shell: bash
-        working-directory: ./build/tests/auth_tests
-        run: |
-          ctest --output-on-failure
+        run: ctest -L auth --output-on-failure
diff --git a/.github/workflows/linux_tests.yaml b/.github/workflows/linux_tests.yaml
@@ -25,7 +25,7 @@ jobs:
     name: Build Test Client
     runs-on: ubuntu-22.04
     outputs:
-      matrix: ${{ steps.testGen.outputs.tests }}
+      matrix: ${{ steps.enumerate.outputs.tests }}
     steps:
       - name: Clone repository
         uses: actions/checkout@v4
@@ -52,35 +52,29 @@ jobs:
       - name: Compile test client
         shell: bash
         run: |
-          mkdir -p build/cmake
-          cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          mkdir -p build-${{ runner.os }}
+          cmake -S $(pwd) -B build-${{ runner.os }} -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache
-          cmake --build build/cmake
-          cp ./build/cmake/src/mozillavpn build/
+          cmake --build build-${{ runner.os }} --target mozillavpn
 
       - uses: actions/upload-artifact@v4
         with:
           name: test-client-${{ github.sha }}
           path: |
-            build/
-            !build/cmake/
+            build-${{ runner.os }}/*.txt
+            build-${{ runner.os }}/*.cmake
+            build-${{ runner.os }}/src/mozillavpn
+            build-${{ runner.os }}/tests/functional
 
       - name: Generate tasklist
-        id: testGen
+        id: enumerate
         shell: bash
         run: |
           echo -n "tests=" >> $GITHUB_OUTPUT
           for test in $(find tests/functional -name 'test*.js' | sort); do
             printf '{"name": "%s", "path": "%s"}' $(basename ${test%.js} | sed -n 's/test//p') $test
           done | jq -s -c >> $GITHUB_OUTPUT
 
-      - name: Check tests
-        shell: bash
-        env:
-          TEST_LIST: ${{ steps.testGen.outputs.tests }}
-        run: |
-          echo $TEST_LIST | jq
-
   functionaltests:
     name: Functional tests
     needs:
@@ -99,18 +93,19 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: test-client-${{ github.sha }}
-          path: build/
+          path: build-${{ runner.os }}
 
       - uses: actions/download-artifact@v4
         with:
           name: test-addons-${{ github.sha }}
-          path: build/addons/
+          path: build-${{ runner.os }}/tests/functional/addons
 
       - name: Install test dependecies
         run: |
           sudo apt-get update
           sudo apt-get install -y $(./scripts/linux/getdeps.py -r linux/debian/control)
           sudo apt install --no-upgrade firefox xvfb -y
+          chmod +x build-${{ runner.os }}/src/mozillavpn
 
       - uses: actions/setup-python@v5
         with:
@@ -124,24 +119,14 @@ jobs:
           cache: "npm"
       - run: npm install
 
-      - name: Check build
-        shell: bash
-        run: |
-          chmod +x ./build/mozillavpn
-          ./build/mozillavpn -v
-
       - name: Running ${{ matrix.test.name }} Tests
         id: runTests
         shell: bash
         env:
           TZ: Europe/London
           HEADLESS: yes
           ARTIFACT_DIR: ${{ runner.temp }}/artifacts
-          MVPN_BIN: ./build/mozillavpn
-        run: |
-          export PATH=$GECKOWEBDRIVER:$(npm bin):$PATH
-          mkdir -p $ARTIFACT_DIR
-          xvfb-run -a npm run functionalTest --  --retries 3 ${{matrix.test.path}}
+        run: ctest -R test${{matrix.test.name}} --test-dir build-${{ runner.os }} --verbose
 
       - name: Uploading artifacts
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test_unit.yaml b/.github/workflows/test_unit.yaml
@@ -43,22 +43,15 @@ jobs:
 
       - name: Running native messaging tests
         shell: bash
-        working-directory: ./build/tests/nativemessaging
-        run: |
-          ctest --output-on-failure
+        run: ctest -L nativemessaging --test-dir $(pwd)/build --output-on-failure
 
       - name: Running QML tests
         shell: bash
-        working-directory: ./build/tests/qml
-        run: |
-          ctest --output-on-failure
+        run: ctest -L qml --test-dir $(pwd)/build --output-on-failure
 
       - name: Running unit tests
         shell: bash
-        working-directory: ./build/tests
-        run: |
-          ctest --test-dir unit --output-on-failure
-          ctest --test-dir unit_tests --output-on-failure
+        run: ctest -L unit --test-dir $(pwd)/build --output-on-failure
 
   macos-unit-tests:
     runs-on: macos-latest
@@ -100,20 +93,13 @@ jobs:
           cmake --build $(pwd)/build --target build_tests
 
       - name: Running native messaging tests
-        working-directory: ./build/tests/nativemessaging
-        run: |
-          ctest --output-on-failure
+        run: ctest -L nativemessaging --test-dir $(pwd)/build --output-on-failure
 
       - name: Running QML tests
-        working-directory: ./build/tests/qml
-        run: |
-          ctest --output-on-failure
+        run: ctest -L qml --test-dir $(pwd)/build --output-on-failure
 
       - name: Running unit tests
-        working-directory: ./build/tests
-        run: |
-          ctest --test-dir unit --output-on-failure
-          ctest --test-dir unit_tests --output-on-failure
+        run: ctest -L unit --test-dir $(pwd)/build --output-on-failure
 
   windows-unit-tests:
     name: Run Unit tests on Windows
@@ -167,22 +153,15 @@ jobs:
 
       - name: Running native messaging tests
         shell: bash
-        working-directory: ./build/tests/nativemessaging
-        run: |
-          ctest --output-on-failure
+        run: ctest -L nativemessaging --test-dir ./build --output-on-failure
 
       - name: Running QML tests
         shell: bash
-        working-directory: ./build/tests/qml
-        run: |
-          ctest --output-on-failure
+        run: ctest -L qml --test-dir ./build --output-on-failure
 
       - name: Running unit tests
         shell: bash
-        working-directory: ./build/tests
-        run: |
-          ctest --test-dir unit --output-on-failure
-          ctest --test-dir unit_tests --output-on-failure
+        run: ctest -L unit --test-dir ./build --output-on-failure
 
   rust-unit-tests:
     strategy:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -175,6 +175,9 @@ if(NOT CMAKE_CROSSCOMPILING)
 
         # E2E Tests
         add_subdirectory(tests/auth_tests EXCLUDE_FROM_ALL)
+
+        # Functional Tests
+        add_subdirectory(tests/functional)
     endif()
 endif()
 

diff --git a/extension/socks5proxy/tests/CMakeLists.txt b/extension/socks5proxy/tests/CMakeLists.txt
@@ -8,6 +8,8 @@ mz_add_test_target(testSocks5connection
         testsocks5connection.h
     DEPENDENCIES
         libSocks5proxy
+    LABELS
+        unit
 )
 
 mz_add_test_target(testSocks5
@@ -16,6 +18,8 @@ mz_add_test_target(testSocks5
         testsocks5.h
     DEPENDENCIES
         libSocks5proxy
+    LABELS
+        unit
 )
 
 # For testing named pipe support, add a little wrapper tool to run curl through

diff --git a/extension/socks5proxy/tests/testsocks5connection.cpp b/extension/socks5proxy/tests/testsocks5connection.cpp
@@ -122,8 +122,11 @@ void TestSocks5Connection::proxyFlowControl() {
 
   // On the receiving end of the socket we should be able to read one
   // buffer's worth of data.
-  QCOMPARE(recvSocket->waitForReadyRead(500), true);
-  QCOMPARE(recvSocket->bytesAvailable(), PROXY_MAX_BUFFER_SIZE);
+  QByteArray result;
+  while (recvSocket->waitForReadyRead(500)) {
+    result.append(recvSocket->readAll());
+  }
+  QCOMPARE(result.length(), PROXY_MAX_BUFFER_SIZE);
 
   // And we should now be able to send the last fragment of unwritten data.
   watermark = 0;

diff --git a/scripts/cmake/utilities.cmake b/scripts/cmake/utilities.cmake
@@ -232,7 +232,7 @@ function(mz_add_test_target TARGET_NAME)
         MZ_ADD_TEST # prefix
         "" # options
         "" # single-value args
-        "TEST_COMMAND;PARENT_TARGET;SOURCES;DEPENDENCIES" # multi-value args
+        "TEST_COMMAND;PARENT_TARGET;SOURCES;DEPENDENCIES;LABELS" # multi-value args
         ${ARGN})
 
     # Test targets are executable targets.
@@ -249,6 +249,10 @@ function(mz_add_test_target TARGET_NAME)
         COMMAND ${TARGET_NAME}
     )
 
+    if(DEFINED MZ_ADD_TEST_LABELS)
+        set_tests_properties(${TARGET_NAME} PROPERTIES LABELS ${MZ_ADD_TEST_LABELS})
+    endif()
+
     add_dependencies(build_tests ${TARGET_NAME})
     target_link_libraries(${TARGET_NAME} PRIVATE Qt6::Test)
     target_link_libraries(${TARGET_NAME} PUBLIC

diff --git a/tests/auth_tests/CMakeLists.txt b/tests/auth_tests/CMakeLists.txt
@@ -87,19 +87,20 @@ target_sources(app_auth_tests PRIVATE
 )
 
 ## Add the tests to be run, one for each test class.
-get_target_property(UTEST_SOURCES app_auth_tests SOURCES)
-list(FILTER UTEST_SOURCES INCLUDE REGEX "test.*.h$")
-foreach(filename ${UTEST_SOURCES})
+get_target_property(AUTH_TEST_SOURCES app_auth_tests SOURCES)
+list(FILTER AUTH_TEST_SOURCES INCLUDE REGEX "test.*.h$")
+foreach(filename ${AUTH_TEST_SOURCES})
     execute_process(
-        OUTPUT_VARIABLE UTEST_CLASS_LIST
+        OUTPUT_VARIABLE AUTH_TEST_CLASS_LIST
         OUTPUT_STRIP_TRAILING_WHITESPACE
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
         COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/scripts/tests/list_test_classes.py -p QObject ${filename}
     )
 
-    foreach(UTEST_CLASS ${UTEST_CLASS_LIST})
-        add_test(NAME ${UTEST_CLASS} COMMAND app_auth_tests ${UTEST_CLASS})
-        set_property(TEST ${UTEST_CLASS} PROPERTY ENVIRONMENT LANG="en" LANGUAGE="en")
+    foreach(AUTH_TEST_CLASS ${AUTH_TEST_CLASS_LIST})
+        add_test(NAME ${AUTH_TEST_CLASS} COMMAND app_auth_tests ${AUTH_TEST_CLASS})
+        set_property(TEST ${AUTH_TEST_CLASS} PROPERTY LABELS auth)
+        set_property(TEST ${AUTH_TEST_CLASS} PROPERTY ENVIRONMENT LANG="en" LANGUAGE="en")
     endforeach()
 endforeach()
 

diff --git a/tests/functional/CMakeLists.txt b/tests/functional/CMakeLists.txt
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+## Add ctests for each of the functional test scripts
+file(GLOB FTEST_SOURCES
+    LIST_DIRECTORIES FALSE
+    RELATIVE ${CMAKE_SOURCE_DIR}
+    "test*.js"
+)
+
+# We must build the test addons for functional testing.
+add_subdirectory(addons)
+
+foreach(FILENAME ${FTEST_SOURCES})
+    get_filename_component(FTEST_NAME ${FILENAME} NAME_WLE)
+    add_test(NAME ${FTEST_NAME}
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+        COMMAND npm run functionalTest -- ${FILENAME})
+
+    set_tests_properties(${FTEST_NAME} PROPERTIES
+        RUN_SERIAL TRUE
+        LABELS functional
+    )
+    set_property(TEST ${FTEST_NAME} PROPERTY ENVIRONMENT
+        MVPN_BIN=$<TARGET_FILE:mozillavpn>
+        MVPN_ADDONS_PATH=${CMAKE_CURRENT_BINARY_DIR}/addons
+    )
+    set_property(TEST ${FTEST_NAME} PROPERTY REQUIRED_FILES
+        ${CMAKE_CURRENT_BINARY_DIR}/addons/prod/manifest.json
+    )
+endforeach()
+
+file(GLOB ADDON_TARGETS
+    LIST_DIRECTORIES TRUE
+    RELATIVE ${CMAKE_SOURCE_DIR}/addons
+    "[0-9][0-9]_*")
+
+foreach(SCENARIO ${ADDON_TARGETS})
+    set_property(TEST testAddons APPEND PROPERTY REQUIRED_FILES
+        ${CMAKE_BINARY_DIR}/addons/${SCENARIO}/manifest.json
+    )
+endforeach()
diff --git a/tests/functional/setupVpn.js b/tests/functional/setupVpn.js
@@ -37,7 +37,14 @@ async function startAndConnect() {
     vpnProcessTerminatePromise = null;
   }
 
-  vpnProcess = spawn(app, ['ui', '--testing']);
+  // If we are on Linux and `HEADLESS` has been set, launch the client using
+  // an offscreen display buffer.
+  let vpnEnv = process.env
+  if (('HEADLESS' in process.env) && (process.platform == 'linux')) {
+    vpnEnv['QT_QPA_PLATFORM'] = 'offscreen'
+  }
+
+  vpnProcess = spawn(app, ['ui', '--testing'], env=vpnEnv);
   stdErr += 'VPN Process ID: ' + vpnProcess.pid;
   vpnProcess.stderr.on('data', (data) => {
     stdErr += data;

diff --git a/tests/nativemessaging/CMakeLists.txt b/tests/nativemessaging/CMakeLists.txt
@@ -32,3 +32,4 @@ target_sources(nativemessaging_tests PRIVATE
 
 add_dependencies(nativemessaging_tests mozillavpnnp)
 add_test(NAME nativemessaging_tests COMMAND nativemessaging_tests $<TARGET_FILE:mozillavpnnp>)
+set_property(TEST nativemessaging_tests PROPERTY LABELS nativemessaging)
diff --git a/tests/qml/CMakeLists.txt b/tests/qml/CMakeLists.txt
@@ -284,4 +284,5 @@ endif()
 file(GLOB QML_TEST_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "tst_*.qml")
 foreach(QML_SOURCE ${QML_TEST_SOURCES})
     add_test(NAME ${QML_SOURCE} COMMAND qml_tests ${QML_TEST_ARGS} -input ${CMAKE_CURRENT_SOURCE_DIR}/${QML_SOURCE})
+    set_property(TEST ${QML_SOURCE} PROPERTY LABELS qml)
 endforeach()
diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
@@ -365,6 +365,7 @@ foreach(filename ${UTEST_SOURCES})
 
     foreach(UTEST_CLASS ${UTEST_CLASS_LIST})
         add_test(NAME ${UTEST_CLASS} COMMAND unit_tests ${UTEST_CLASS})
+        set_property(TEST ${UTEST_CLASS} PROPERTY LABELS unit)
         set_property(TEST ${UTEST_CLASS} PROPERTY ENVIRONMENT LANG="en" LANGUAGE="en")
     endforeach()
 endforeach()
diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt
@@ -176,6 +176,7 @@ foreach(filename ${UTEST_SOURCES})
 
     foreach(UTEST_CLASS ${UTEST_CLASS_LIST})
         add_test(NAME ${UTEST_CLASS} COMMAND app_unit_tests ${UNIT_TEST_ARGS} ${UTEST_CLASS})
+        set_property(TEST ${UTEST_CLASS} PROPERTY LABELS unit)
         set_property(TEST ${UTEST_CLASS} PROPERTY ENVIRONMENT LANG="en" LANGUAGE="en")
     endforeach()
 endforeach()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -32,3 +32,4 @@ target_sources(nativemessaging_tests PRIVATE

		add_dependencies(nativemessaging_tests mozillavpnnp)
		add_test(NAME nativemessaging_tests COMMAND nativemessaging_tests $<TARGET_FILE:mozillavpnnp>)
		set_property(TEST nativemessaging_tests PROPERTY LABELS nativemessaging)