From 018a46635ba990e38b01e33984cece48040a4f0b Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Thu, 26 Oct 2023 14:42:27 -0700 Subject: [PATCH] VPN-5739: MacOS functional testing workflow (#7564) * Add MacOS functional testing in CI * Generate test addons in the binary dir * Deprecate generate_all_tests.py script * Fix cmake build path for wasm * Remove duplicate addon build during functional tests * A little bit of test renaming for consistency * Unify names and style of functional test YAML * Fix testSettings.js instability on MacOS * Fix testOnboarding.js instability on MacOS * Remove generate_all_tests.py script, it is no longer used * Fix testServers.js instability on MacOS * Try to get tests to fail faster with a condition timeout --- ...functional_tests.yaml => linux_tests.yaml} | 39 +++--- .github/workflows/macos_tests.yaml | 125 ++++++++++++++++++ .../workflows/{wasm.yaml => wasm_tests.yaml} | 86 ++++++------ README.md | 1 - docs/dev-setup/index.md | 1 - scripts/README.md | 1 - scripts/addon/generate_all_tests.py | 31 ----- src/ui/screens/home/servers/ServerCountry.qml | 1 + src/ui/screens/settings/ViewDNSSettings.qml | 1 + tests/dummyvpn/CMakeLists.txt | 11 +- tests/functional/helper.js | 10 +- tests/functional/queries.js | 2 + tests/functional/servers/addon.js | 10 +- tests/functional/testOnboarding.js | 9 +- tests/functional/testServers.js | 4 +- tests/functional/testSettings.js | 27 ++++ 16 files changed, 239 insertions(+), 120 deletions(-) rename .github/workflows/{functional_tests.yaml => linux_tests.yaml} (82%) create mode 100644 .github/workflows/macos_tests.yaml rename .github/workflows/{wasm.yaml => wasm_tests.yaml} (64%) delete mode 100755 scripts/addon/generate_all_tests.py diff --git a/.github/workflows/functional_tests.yaml b/.github/workflows/linux_tests.yaml similarity index 82% rename from .github/workflows/functional_tests.yaml rename to .github/workflows/linux_tests.yaml index 23c7103b90..494aa76c6e 100644 --- a/.github/workflows/functional_tests.yaml +++ b/.github/workflows/linux_tests.yaml @@ -1,4 +1,4 @@ -name: Functional tests +name: Linux Tests on: push: branches: @@ -26,8 +26,10 @@ jobs: with: submodules: 'recursive' - - name: Install build dependecies + - name: Install build dependencies + shell: bash run: | + pip3 install -r requirements.txt sudo apt-get update sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control) @@ -36,20 +38,17 @@ jobs: uses: actions/cache@v3 with: path: grcov-build/ - key: ${{runner.os}}-grcov-v0.8.13 + key: ${{ runner.os }}-grcov-v0.8.13 - name: Install Grcov if: steps.cache-grcov.outputs.cache-hit != 'true' shell: bash run: | - cargo install grcov --root ${{github.workspace}}/grcov-build --version 0.8.13 + cargo install grcov --root ${{ github.workspace }}/grcov-build --version 0.8.13 - name: Compile test client shell: bash - if: steps.cache-build.outputs.cache-hit != 'true' run: | - pip3 install -r requirements.txt - mkdir -p build/cmake mkdir -p build/profile cmake -S $(pwd) -B build/cmake \ @@ -61,14 +60,7 @@ jobs: rsync -a --include '*/' --include '*.gcno' --exclude '*' \ build/cmake/tests/dummyvpn/CMakeFiles/dummyvpn.dir/ build/profile/ cp ./build/cmake/tests/dummyvpn/dummyvpn build/ - - - name: Compile test addons - shell: bash - if: steps.cache-build.outputs.cache-hit != 'true' - run: | - mkdir -p build/addons - cmake -S $(pwd)/tests/functional/addons -B build/addons - cmake --build build/addons + cp -r ./build/cmake/tests/dummyvpn/addons build/addons - uses: actions/upload-artifact@v3 with: @@ -125,7 +117,7 @@ jobs: uses: actions/cache@v3 with: path: grcov-build/ - key: ${{runner.os}}-grcov-v0.8.13 + key: ${{ runner.os }}-grcov-v0.8.13 - name: Check build shell: bash @@ -133,10 +125,10 @@ jobs: chmod +x ./build/dummyvpn ./build/dummyvpn -v - chmod +x ${{github.workspace}}/grcov-build/bin/grcov - ${{github.workspace}}/grcov-build/bin/grcov --version + chmod +x ${{ github.workspace }}/grcov-build/bin/grcov + ${{ github.workspace }}/grcov-build/bin/grcov --version - - name: Running ${{matrix.test.name}} Tests + - name: Running ${{ matrix.test.name }} Tests id: runTests uses: nick-invision/retry@v2 with: @@ -151,17 +143,16 @@ jobs: env: ARTIFACT_DIR: ${{ runner.temp }}/artifacts MVPN_BIN: ./build/dummyvpn - MVPN_ADDONS_PATH: ./build/addons - name: Generating grcov reports id: generateGrcov continue-on-error: true # Ignore GRCOV parsing errors, see github.com/mozilla/grcov/issues/570 timeout-minutes: 1 # Give GRCOV a short timeout in case it hangs after a panic run: | - export PATH=${{github.workspace}}/grcov-build/bin:$PATH + export PATH=${{ github.workspace }}/grcov-build/bin:$PATH grcov build/profile \ -s src -t lcov --branch --ignore-not-existing \ - -o ${{runner.temp}}/artifacts/functional_lcov.info + -o ${{ runner.temp }}/artifacts/functional_lcov.info - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 @@ -170,12 +161,12 @@ jobs: directory: . flags: functional_tests name: codecov-poc - files: ${{runner.temp}}/artifacts/functional_lcov.info + files: ${{ runner.temp }}/artifacts/functional_lcov.info verbose: true - name: Uploading artifacts uses: actions/upload-artifact@v3 if: ${{ always() }} with: - name: ${{matrix.test.name}} Logs + name: ${{ matrix.test.name }} Logs path: ${{ runner.temp }}/artifacts diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml new file mode 100644 index 0000000000..74d4373111 --- /dev/null +++ b/.github/workflows/macos_tests.yaml @@ -0,0 +1,125 @@ +name: MacOS Tests +on: + push: + branches: + - main + - "releases/**" + pull_request: + branches: + - main + - "releases/**" + +# Restrict tests to the most recent commit. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build_test_app: + name: Build Test Client + runs-on: macos-latest + outputs: + matrix: ${{ steps.testGen.outputs.tests }} + steps: + - name: Clone repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Install build dependencies + shell: bash + run: | + pip3 install -r requirements.txt + brew install ninja + + - name: Install Qt6 + shell: bash + run: | + wget https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/mozillavpn.v2.mozillavpn.cache.level-3.toolchains.v3.qt-mac.latest/artifacts/public%2Fbuild%2Fqt6_mac.zip -O qt6_mac.zip + unzip -a -d ${{ github.workspace }} qt6_mac.zip + + - name: Compile test client + shell: bash + run: | + mkdir -p build/cmake + cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_PREFIX_PATH=${{ github.workspace }}/qt_dist/lib/cmake + cmake --build build/cmake -j$(nproc) --target dummyvpn + cp ./build/cmake/tests/dummyvpn/dummyvpn build/ + cp -r ./build/cmake/tests/dummyvpn/addons build/addons + + - uses: actions/upload-artifact@v3 + with: + name: test-client-${{ github.sha }} + path: | + build/ + !build/cmake/ + + - name: Generate tasklist + id: testGen + shell: bash + run: | + echo -n "tests=" >> $GITHUB_OUTPUT + for test in $(find tests/functional -name 'test*.js' | sort); do + printf '{"name": "%s", "path": "%s"}' $(basename ${test%.js} | sed -n 's/test//p') $test + done | jq -s -c >> $GITHUB_OUTPUT + + - name: Check tests + shell: bash + env: + TEST_LIST: ${{ steps.testGen.outputs.tests }} + run: | + echo $TEST_LIST | jq + + functionaltests: + name: Functional tests + needs: + - build_test_app + runs-on: macos-latest + timeout-minutes: 45 + strategy: + fail-fast: false # Don't cancel other jobs if a test fails + matrix: + test: ${{ fromJson(needs.build_test_app.outputs.matrix) }} + steps: + - name: Clone repository + uses: actions/checkout@v3 + + - uses: actions/download-artifact@v3 + with: + name: test-client-${{ github.sha }} + path: build/ + + - name: Install test dependecies + run: | + pip3 install -r requirements.txt + npm install + + - name: Check build + shell: bash + run: | + chmod +x ./build/dummyvpn + ./build/dummyvpn -v + + - name: Running ${{ matrix.test.name }} Tests + id: runTests + uses: nick-invision/retry@v2 + with: + timeout_minutes: 15 + max_attempts: 3 + command: | + export PATH=$GECKOWEBDRIVER:$(npm bin):$PATH + export HEADLESS=yes + export TZ=Europe/London + mkdir -p $ARTIFACT_DIR + npm run functionalTest -- --retries 3 ${{ matrix.test.path }} + env: + ARTIFACT_DIR: ${{ runner.temp }}/artifacts + MVPN_BIN: ./build/dummyvpn + + - name: Uploading artifacts + uses: actions/upload-artifact@v3 + if: ${{ always() }} + with: + name: ${{ matrix.test.name }} Logs + path: ${{ runner.temp }}/artifacts diff --git a/.github/workflows/wasm.yaml b/.github/workflows/wasm_tests.yaml similarity index 64% rename from .github/workflows/wasm.yaml rename to .github/workflows/wasm_tests.yaml index ddfcac5b76..c8def5fa22 100644 --- a/.github/workflows/wasm.yaml +++ b/.github/workflows/wasm_tests.yaml @@ -1,15 +1,15 @@ -name: WebAssembly - +name: WebAssembly Tests on: push: branches: - main - - 'releases/**' + - "releases/**" pull_request: branches: - main - - 'releases/**' + - "releases/**" +# Restrict tests to the most recent commit. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -27,9 +27,14 @@ jobs: - name: Clone repository uses: actions/checkout@v3 with: - submodules: 'true' + submodules: 'true' - - name: Install Qt + - name: Install build dependencies + shell: bash + run: | + pip3 install -r requirements.txt + + - name: Install Qt6 shell: bash run: | python3 -m pip install aqtinstall @@ -37,21 +42,34 @@ jobs: python3 -m aqt install-qt -O /opt linux desktop $QTVERSION python3 -m aqt install-qt -O /opt linux desktop $QTVERSION wasm_32 -m qtcharts qtwebsockets qt5compat - - name: Install python dependencies - shell: bash - run: | - pip install -r requirements.txt - - name: Setup emsdk uses: mymindstorm/setup-emsdk@v7 - - name: Compiling + - name: Compile test client shell: bash run: | export PATH=/opt/$QTVERSION/wasm_32/bin:/opt/$QTVERSION/gcc_64/bin:$PATH - mkdir build - /opt/$QTVERSION/wasm_32/bin/qt-cmake -S . -B build -DQT_HOST_PATH=/opt/$QTVERSION/gcc_64 -DQT_HOST_PATH_CMAKE_DIR=/opt/$QTVERSION/gcc_64/lib/cmake -DCMAKE_BUILD_TYPE=Release - cmake --build build -j4 + mkdir -p build/cmake + /opt/$QTVERSION/wasm_32/bin/qt-cmake -S $(pwd) -B build/cmake -DCMAKE_BUILD_TYPE=Release \ + -DQT_HOST_PATH_CMAKE_DIR=/opt/$QTVERSION/gcc_64/lib/cmake \ + -DQT_HOST_PATH=/opt/$QTVERSION/gcc_64 + cmake --build build/cmake -j4 + cp -r build/cmake/wasm_build build/wasm_build + + - name: Compile test addons + shell: bash + run: | + mkdir -p build/addons + cmake -S $(pwd)/tests/functional/addons -B build/addons \ + -DCMAKE_PREFIX_PATH=/opt/$QTVERSION/gcc_64/lib/cmake + cmake --build build/addons + + - uses: actions/upload-artifact@v3 + with: + name: WebAssembly Build Qt6 + path: | + build/ + !build/cmake/ - name: Generate tasklist id: testGen @@ -69,12 +87,6 @@ jobs: run: | echo $TEST_LIST | jq - - name: Uploading - uses: actions/upload-artifact@v3 - with: - name: WebAssembly Build Qt6 - path: build/wasm_build - functionaltests: name: Functional tests needs: @@ -92,37 +104,18 @@ jobs: - name: Clone repository uses: actions/checkout@v3 - - name: Cache build - id: cache-build - uses: actions/cache@v3 + - uses: actions/download-artifact@v3 with: + name: WebAssembly Build Qt6 path: build/ - key: ${{ github.sha }} - - name: Install Qt - shell: bash - run: | - python3 -m pip install aqtinstall - python3 -m aqt install-qt -O /opt linux desktop $QTVERSION - - - name: Install dependecies + - name: Install test dependecies run: | sudo apt install --no-upgrade firefox xvfb -y pip3 install -r requirements.txt npm install - - name: Download a Build Artifact - uses: actions/download-artifact@v3 - with: - name: WebAssembly Build Qt6 - # Destination path, WASM_BUILD_DIRECTORY - path: wasm_build - - - name: Build addons - shell: bash - run: ./scripts/addon/generate_all_tests.py -q /opt/$QTVERSION/gcc_64/bin - - - name: Running ${{matrix.test.name}} Tests + - name: Running ${{ matrix.test.name }} Tests id: runTests uses: nick-invision/retry@v2 with: @@ -131,5 +124,6 @@ jobs: command: | export PATH=$GECKOWEBDRIVER:$(npm bin):$PATH export HEADLESS=yes - export WASM_BUILD_DIRECTORY=$(pwd)/wasm_build - xvfb-run -a npm run functionalTestWasm -- --retries 3 ${{matrix.test.path}} + export WASM_BUILD_DIRECTORY=$(pwd)/build/wasm_build + export MVPN_ADDONS_PATH=$(pwd)/build/addons + xvfb-run -a npm run functionalTestWasm -- --retries 3 ${{ matrix.test.path }} diff --git a/README.md b/README.md index ab5a410a5c..8b9686ce3a 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,6 @@ export MVPN_BIN=$(pwd)/build/tests/dummyvpn/dummyvpn **Other dependencies**: * Install node (if needed) and then `npm install` to install the testing dependencies -* Compile the testing addons: `./scripts/addon/generate_all_tests.py` * Make a .env file and place it in the root folder for the repo. It should include: * `MVPN_BIN` (location of compiled mvpn binary. This must be a dummy binary, see note above.) * `ARTIFACT_DIR` - optional (directory to put screenshots from test failures) diff --git a/docs/dev-setup/index.md b/docs/dev-setup/index.md index ab374242be..c68d9573c2 100644 --- a/docs/dev-setup/index.md +++ b/docs/dev-setup/index.md @@ -86,7 +86,6 @@ team used for Xcode certificates. This defaults to `43AQ936H96` if not set. (Mac # Useful scripts * `./scripts/utils/bake_shaders.sh`: if you want to change the shaders -* `./scripts/addon/generate_all_tests.py`: compiles the testing addons, needed for functional tests # Troubleshooting diff --git a/scripts/README.md b/scripts/README.md index 7d82cf744a..10d5831402 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -40,7 +40,6 @@ can be used to run tests locally or via the CI. # Addons - ./addon/build.py - generate a single addon -- ./addon/generate_all_tests.py - generate addons for testing # Others diff --git a/scripts/addon/generate_all_tests.py b/scripts/addon/generate_all_tests.py deleted file mode 100755 index 7969d74c55..0000000000 --- a/scripts/addon/generate_all_tests.py +++ /dev/null @@ -1,31 +0,0 @@ -#! /usr/bin/env python3 -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import argparse -import os -import subprocess - -parser = argparse.ArgumentParser(description="Generate an addon package") -parser.add_argument( - "-q", - "--qt_path", - default=None, - dest="qtpath", - help="The QT binary path. If not set, we try to guess.", -) -args = parser.parse_args() - -test_addons_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "tests", "functional", "addons") -cmake_setup_args = ['cmake', '-S', test_addons_path, '-B', os.path.join(test_addons_path, 'generated')] - -if args.qtpath is not None: - qt_install_libs = subprocess.check_output([os.path.join(args.qtpath, 'qmake'), '-query', 'QT_INSTALL_LIBS']) - qt_cmake_prefix = os.path.join(qt_install_libs.decode().strip(), 'cmake') - cmake_setup_args.append(f'-DCMAKE_PREFIX_PATH={qt_cmake_prefix}') - -# Use CMake to build the test addons. -# TODO: At some point we should stop generating in the source directory. -subprocess.run(cmake_setup_args, check=True) -subprocess.run(['cmake', '--build', os.path.join(test_addons_path, 'generated')], check=True) diff --git a/src/ui/screens/home/servers/ServerCountry.qml b/src/ui/screens/home/servers/ServerCountry.qml index 12a95352cf..d0052944a2 100644 --- a/src/ui/screens/home/servers/ServerCountry.qml +++ b/src/ui/screens/home/servers/ServerCountry.qml @@ -22,6 +22,7 @@ MZClickableRow { property var currentCityIndex property alias serverCountryName: countryName.text property var cityList: cityListVisible ? cityLoader.item : cityLoader + property var busy: cityListVisible ? scrollAnimation.running : false Component.onCompleted:{ cityLoader.active = cityListVisible diff --git a/src/ui/screens/settings/ViewDNSSettings.qml b/src/ui/screens/settings/ViewDNSSettings.qml index 24aaa63198..9ed8c8c1fb 100644 --- a/src/ui/screens/settings/ViewDNSSettings.qml +++ b/src/ui/screens/settings/ViewDNSSettings.qml @@ -276,6 +276,7 @@ MZViewBase { id: dnsOverwriteLoader active: false + objectName: "dnsOverwriteLoader" sourceComponent: MZSimplePopup { id: dnsOverwritePopup diff --git a/tests/dummyvpn/CMakeLists.txt b/tests/dummyvpn/CMakeLists.txt index 4e15d4ef7c..741a43f199 100644 --- a/tests/dummyvpn/CMakeLists.txt +++ b/tests/dummyvpn/CMakeLists.txt @@ -59,15 +59,12 @@ target_compile_definitions(dummyvpn PRIVATE MZ_DEBUG) # Use #ifdef _WIN32 instead of MZ_WINDOWS in dummyvpn for conflicts in Windows-specific code. target_compile_definitions(dummyvpn PRIVATE MZ_DUMMY) -# We should not build into the source dir -# but that is how we are currently doing things, -# at least we no longer invoke python scripts to -# invoke cmake. -# ¯\_(ツ)_/¯ +# Build the addons for functional testing. +get_filename_component(QT6_PREFIX_PATH ${Qt6_DIR}/.. REALPATH) ExternalProject_Add(functional_test_addons SOURCE_DIR ${CMAKE_SOURCE_DIR}/tests/functional/addons - BINARY_DIR ${CMAKE_SOURCE_DIR}/tests/functional/addons/generated - CMAKE_CACHE_ARGS -DQt6_DIR:PATH=${Qt6_DIR} -DQt6QmlTools_DIR:PATH=${Qt6QmlTools_DIR} -DQt6CoreTools_DIR:PATH=${Qt6CoreTools_DIR} + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/addons + CMAKE_CACHE_ARGS -DCMAKE_PREFIX_PATH:PATH=${QT6_PREFIX_PATH} INSTALL_COMMAND "" ) add_dependencies(dummyvpn functional_test_addons) diff --git a/tests/functional/helper.js b/tests/functional/helper.js index 2db784db12..73bd51fc0f 100644 --- a/tests/functional/helper.js +++ b/tests/functional/helper.js @@ -317,8 +317,16 @@ module.exports = { }, async waitForCondition(condition, waitTimeInMilliSecs = 500) { + // If the condition takes longer than 15 seconds, give up. + let active = true; + let timeout = setTimeout(() => { active = false }, 15000); while (true) { - if (await condition()) return; + if (await condition()) { + clearTimeout(timeout) + return; + } + // Asserting here produces a more useful backtrace for diagnosing tests. + assert(active, "Condition timed out"); await new Promise(resolve => setTimeout(resolve, waitTimeInMilliSecs)); } }, diff --git a/tests/functional/queries.js b/tests/functional/queries.js index 0f5c9c7af9..03436234b9 100644 --- a/tests/functional/queries.js +++ b/tests/functional/queries.js @@ -279,6 +279,7 @@ const screenSettings = { INFORMATION_CARD: new QmlQueryComposer('//privacySettingsViewInformationCard'), + MODAL_LOADER: new QmlQueryComposer('//privacyOverwriteLoader'), MODAL_CLOSE_BUTTON: new QmlQueryComposer('//privacyOverwritePopupPopupCloseButton'), MODAL_PRIMARY_BUTTON: @@ -321,6 +322,7 @@ const screenSettings = { INFORMATION_CARD: new QmlQueryComposer('//DNSSettingsViewInformationCard'), + MODAL_LOADER: new QmlQueryComposer('//dnsOverwriteLoader'), MODAL_CLOSE_BUTTON: new QmlQueryComposer('//dnsOverwritePopupPopupCloseButton'), MODAL_PRIMARY_BUTTON: diff --git a/tests/functional/servers/addon.js b/tests/functional/servers/addon.js index c423381597..1e551117d1 100644 --- a/tests/functional/servers/addon.js +++ b/tests/functional/servers/addon.js @@ -6,16 +6,18 @@ const Server = require('./server.js'); const fs = require('fs'); const path = require('path'); +// If not specified, assume the test addons can be found in the 'addons' subdir +// under the MVPN_BIN binary const TEST_ADDONS_PATH = ('MVPN_ADDONS_PATH' in process.env) ? process.env.MVPN_ADDONS_PATH : - './tests/functional/addons/generated'; + path.join(path.dirname(process.env.MVPN_BIN), 'addons'); // This function exposes all the files for a particular addon scenario through // the addon server. function createScenario(scenario, addonPath) { const manifestPath = path.join(addonPath, 'manifest.json'); if (!fs.existsSync(manifestPath)) { - throw new Error(`No manifest file! ${manifestPath} should exist! Have you executed \`./scripts/addon/generate_all_tests.py'?`); + throw new Error(`No manifest file! ${manifestPath} should exist!?`); } const obj = {}; @@ -57,7 +59,7 @@ module.exports = { let scenarios = {}; if (!fs.existsSync(TEST_ADDONS_PATH)) { - throw new Error(`Addon path not found! Have you executed \`./scripts/addon/generate_all_tests.py'?`); + throw new Error(`Addon path not found!`); } // Generate test addon scenarios @@ -76,7 +78,7 @@ module.exports = { } if (Object.keys(scenarios).length == 0) { - throw new Error(`No addons found! Have you executed \`./scripts/addon/generate_all_tests.py'?`); + throw new Error(`No addons found!?`); } const endpoints = { diff --git a/tests/functional/testOnboarding.js b/tests/functional/testOnboarding.js index 633a1f07fc..097921e966 100644 --- a/tests/functional/testOnboarding.js +++ b/tests/functional/testOnboarding.js @@ -262,8 +262,9 @@ describe('Onboarding', function() { assert.equal(await vpn.getSetting('onboardingStep'), 0); //Proceed to second onboarding step - await vpn.waitForQueryAndClick(queries.screenOnboarding.DATA_NEXT_BUTTON.visible()); await vpn.waitForQuery(queries.screenOnboarding.STEP_NAV_STACK_VIEW.ready()); + await vpn.waitForQueryAndClick(queries.screenOnboarding.DATA_NEXT_BUTTON.visible()); + await vpn.waitForQuery(queries.screenOnboarding.ONBOARDING_VIEW.prop('currentIndex', 1)); //Quit and relaunch the app await vpn.quit(); @@ -275,8 +276,9 @@ describe('Onboarding', function() { assert.equal(await vpn.getSetting('onboardingStep'), 1); //Proceed to third onboarding step - await vpn.waitForQueryAndClick(queries.screenOnboarding.PRIVACY_NEXT_BUTTON.visible()); await vpn.waitForQuery(queries.screenOnboarding.STEP_NAV_STACK_VIEW.ready()); + await vpn.waitForQueryAndClick(queries.screenOnboarding.PRIVACY_NEXT_BUTTON.visible()); + await vpn.waitForQuery(queries.screenOnboarding.ONBOARDING_VIEW.prop('currentIndex', 2)); //Quit and relaunch the app await vpn.quit(); @@ -288,8 +290,9 @@ describe('Onboarding', function() { assert.equal(await vpn.getSetting('onboardingStep'), 2); //Proceed to fourth/final onboarding step - await vpn.waitForQueryAndClick(queries.screenOnboarding.DEVICES_NEXT_BUTTON.visible()); await vpn.waitForQuery(queries.screenOnboarding.STEP_NAV_STACK_VIEW.ready()); + await vpn.waitForQueryAndClick(queries.screenOnboarding.DEVICES_NEXT_BUTTON.visible()); + await vpn.waitForQuery(queries.screenOnboarding.ONBOARDING_VIEW.prop('currentIndex', 3)); //Quit and relaunch the app await vpn.quit(); diff --git a/tests/functional/testServers.js b/tests/functional/testServers.js index 6563dd23db..ab48d9997f 100644 --- a/tests/functional/testServers.js +++ b/tests/functional/testServers.js @@ -92,8 +92,9 @@ describe('Server list', function() { if (await vpn.getQueryProperty(countryId, 'cityListVisible') === 'false') { await vpn.clickOnQuery(countryId); + await vpn.waitForQuery(countryId.ready()); } - + await vpn.waitForQuery(countryId.prop('cityListVisible', true)); for (let city of server.cities) { @@ -129,6 +130,7 @@ describe('Server list', function() { await vpn.waitForQueryAndClick(queries.screenHome.serverListView.ALL_SERVERS_TAB.visible()); // One selected + await vpn.waitForQuery(queries.screenHome.STACKVIEW.ready()); await vpn.waitForQuery(cityId.checked()); } } diff --git a/tests/functional/testSettings.js b/tests/functional/testSettings.js index 06f376e6f0..e06baf9c65 100644 --- a/tests/functional/testSettings.js +++ b/tests/functional/testSettings.js @@ -228,18 +228,24 @@ describe('Settings', function() { assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_SECONDARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.BLOCK_ADS_CHECKBOX.visible()); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_CLOSE_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.BLOCK_ADS_CHECKBOX.visible()); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_PRIMARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 2); await vpn.setSetting('dnsProviderFlags', 1); @@ -248,18 +254,24 @@ describe('Settings', function() { assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_SECONDARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.BLOCK_TRACKERS_CHECKBOX.visible()); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_CLOSE_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.BLOCK_TRACKERS_CHECKBOX.visible()); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_PRIMARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 4); await vpn.setSetting('dnsProviderFlags', 1); @@ -268,18 +280,24 @@ describe('Settings', function() { assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_SECONDARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.BLOCK_MALWARE_CHECKBOX.visible()); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_CLOSE_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.BLOCK_MALWARE_CHECKBOX.visible()); assert.equal(await vpn.getSetting('dnsProviderFlags'), 1); await vpn.waitForQueryAndClick( queries.screenSettings.privacyView.MODAL_PRIMARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.privacyView.MODAL_LOADER.prop('active', false)); assert.equal(await vpn.getSetting('dnsProviderFlags'), 8); // Let's go back @@ -398,6 +416,9 @@ describe('Settings', function() { await vpn.waitForQueryAndClick( queries.screenSettings.appPreferencesView.dnsSettingsView .MODAL_SECONDARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.appPreferencesView.dnsSettingsView + .MODAL_LOADER.prop('active', false)); await vpn.waitForQueryAndClick(queries.screenSettings.appPreferencesView .dnsSettingsView.CUSTOM_DNS.visible()); @@ -405,6 +426,9 @@ describe('Settings', function() { await vpn.waitForQueryAndClick( queries.screenSettings.appPreferencesView.dnsSettingsView .MODAL_CLOSE_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.appPreferencesView.dnsSettingsView + .MODAL_LOADER.prop('active', false)); await vpn.waitForQueryAndClick(queries.screenSettings.appPreferencesView .dnsSettingsView.CUSTOM_DNS.visible()); @@ -412,6 +436,9 @@ describe('Settings', function() { await vpn.waitForQueryAndClick( queries.screenSettings.appPreferencesView.dnsSettingsView .MODAL_PRIMARY_BUTTON.visible()); + await vpn.waitForQuery( + queries.screenSettings.appPreferencesView.dnsSettingsView + .MODAL_LOADER.prop('active', false)); await vpn.setQueryProperty( queries.screenSettings.appPreferencesView.dnsSettingsView