Experiment: Building with CMake (#3350)

* Add cmake support for lottie, nebula and golang
* Build unit tests with cmake
* Build QML tests with cmake
* Build auth tests with cmake
* Build Ubuntu packages with CMake
* Move GI version mangling into Linux bundling script.
* Build functional tests with CMake
* Run CI unit and QML tests with CMake
* Add CMake support for building the web extension
* Update README.md with Linux build steps for CMake
* Add initial support for building MacOS
* Build the Windows MSI with CMake
* Patch wasm builds in CI to include lrelease tool
* Generate a dummy qmake project to aid in string updates.
* Fix assorted build pain with Xcode and balrog.
* Build Windows unit and QML tests with CMake
* Fix MergeModule import from other versions of MSVC
* Add cmake logic to download/install the split-tunnel driver.
* Combine sources and headers.
* Add nativemessaging tests, use CTest to run tests.
* Eliminate MVPN_OATHTOOL and just use the python module
* Add webextension to MacOS bundle

.clang-format-ignore

@@ -2,6 +2,7 @@
 ./android/tunnel/*
 ./android/src/com/wireguard/*
 ./src/hacl-star/*
+./src/platforms/windows/golang-msvc-types.h
 ./extension/app/json.hpp
 ./glean/org/mozilla/Glean/glean.js
 ./glean/org/mozilla/Glean/glean.lib.js

.github/l10n/check_l10n_issues.py

@@ -12,7 +12,7 @@
 vpn_root_folder = os.path.join(script_folder, os.pardir, os.pardir)
 
 # Extract all strings in the .ts file
-srcFile = os.path.join(vpn_root_folder, "src", "src.pro")
+srcFile = os.path.join(vpn_root_folder, "translations", "generated", "dummy.pro")
 os.system(f"lupdate {srcFile} -ts")
 
 # Load the exceptions file. String IDs can be added to each category to
@@ -21,7 +21,7 @@
     exceptions = json.load(f)
 
 # Load the .ts file
-ts_file = os.path.join(vpn_root_folder, "translations", "en", "mozillavpn_en.ts")
+ts_file = os.path.join(vpn_root_folder, "translations", "generated", "mozillavpn_en.ts")
 tree = etree.parse(ts_file)
 root = tree.getroot()
 

.github/workflows/functional_tests.yaml

@@ -60,15 +60,10 @@ jobs:
         run: |
           pip3 install -r requirements.txt
 
-          python3 scripts/utils/import_languages.py
-          python3 scripts/utils/generate_glean.py
-
-          # Delete unit tests, so we can get to testing faster
-          sed -i '/tests\/unit/d' mozillavpn.pro
-          qmake6 CONFIG+=DUMMY QMAKE_CXXFLAGS+=--coverage QMAKE_LFLAGS+=--coverage CONFIG+=debug CONFIG+=inspector QT+=svg
-          make -j$(nproc)
-          cp ./src/mozillavpn build/
-          cp -r ./src/.obj build/
+          mkdir -p build/cmake
+          cmake -S $(pwd) -B build/cmake -DBUILD_DUMMY=ON \
+              -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_EXE_LINKER_FLAGS=--coverage
+          cmake --build build/cmake -j$(nproc)
 
       - name: Generate tasklist
         id: testGen
@@ -116,7 +111,7 @@ jobs:
       
       - name: Check build
         shell: bash
-        run: ./build/mozillavpn -v
+        run: ./build/cmake/src/mozillavpn -v
 
       - name: Launching API proxy
         shell: bash
@@ -139,15 +134,15 @@ jobs:
           ACCOUNT_PASSWORD: ${{ secrets.ACCOUNT_PASSWORD }}
           ARTIFACT_DIR: ${{ runner.temp }}/artifacts
           MVPN_API_BASE_URL: http://localhost:5000
-          MVPN_BIN: ./build/mozillavpn
+          MVPN_BIN: ./build/cmake/src/mozillavpn
 
       - name: Generating grcov reports
         id: generateGrcov
         continue-on-error: true # Ignore GRCOV parsing errors, see github.com/mozilla/grcov/issues/570
         timeout-minutes: 1 # Give GRCOV a short timeout in case it hangs after a panic
         run: |
-          cp build/.obj/*.gcno src/.obj
-          ./build/bin/grcov src/.obj -s src -t lcov --branch --ignore-not-existing \
+          ./build/bin/grcov build/cmake/src/CMakeFiles/mozillavpn.dir \
+              -s src -t lcov --branch --ignore-not-existing \
               -o ${{runner.temp}}/artifacts/functional_lcov.info
 
       - name: Upload coverage to Codecov

.github/workflows/linux.yaml

@@ -21,25 +21,6 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v2
 
-      - name: Generate build version
-        shell: bash
-        env:
-          GITREF: ${{github.ref}}
-        run: |
-          VERSION=$(grep ':VERSION' version.pri | cut -d= -f2 | tr -d '[:space:]')
-          if [[ "$GITREF" =~ ^refs/pull/([0-9]+)/merge ]]; then
-            BUILDREV="~pr${BASH_REMATCH[1]}"
-          elif [[ "$GITREF" =~ ^refs/heads/releases/([0-9][^/]*) ]]; then
-            git fetch --unshallow
-            BUILDREV="~rc$(git rev-list --count --first-parent origin/main..HEAD)"
-            VERSION=${BASH_REMATCH[1]}
-          else
-            BUILDREV="~github${{github.run_number}}"
-          fi
-
-          # Force the upstream package version
-          sed -i "s/:VERSION.*/:VERSION = ${VERSION}${BUILDREV}/" version.pri
-
       - name: Install source dependencies
         shell: bash
         run: |
@@ -48,9 +29,11 @@ jobs:
 
       - name: Build source bundle
         shell: bash
-        run: ./scripts/linux/script.sh --source
+        env:
+          GITREF: ${{github.ref}}
+        run: ./scripts/linux/script.sh --source --gitref ${GITREF}
 
-      - name: Uploading
+      - name: Upload source bundle
         uses: actions/upload-artifact@v2
         with:
             name: Sources

.github/workflows/macos-build.yaml

@@ -34,7 +34,7 @@ jobs:
         shell: bash
         run: |
           auth_header="$(git config --local --get http.https://github.com/.extraheader)"
-          git clone https://github.com/mozilla-mobile/qt_static_macos
+          git clone https://github.com/mozilla-mobile/qt_static_macos --depth 1
           cd qt_static_macos
           cat qt6* > qt_static.tar.gz
           tar xf qt_static.tar.gz
@@ -192,7 +192,7 @@ jobs:
         shell: bash
         run: |
           auth_header="$(git config --local --get http.https://github.com/.extraheader)"
-          git clone https://github.com/mozilla-mobile/qt_static_macos
+          git clone https://github.com/mozilla-mobile/qt_static_macos --depth 1
           cd qt_static_macos
           cat qt6* > qt_static.tar.gz
           tar xf qt_static.tar.gz

.github/workflows/ppa-automation.yaml

@@ -20,39 +20,20 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Generate release version
-        id: gen-version
-        shell: bash
-        env:
-          GITREF: ${{github.ref}}
-        run: |
-          if [[ "$GITREF" =~ ^refs/heads/releases/([0-9][^/]*) ]]; then
-            BUILDREV="~rc$(git rev-list --count --first-parent origin/main..HEAD)"
-            VERSION=$(echo $GITREF | cut -d'/' -f4)
-          elif [[ "$GITREF" =~ ^refs/tags/v([0-9a-z.]+) ]]; then
-            BUILDREV=""
-            VERSION=${BASH_REMATCH[1]}
-          else
-            BUILDREV="~nightly$(date +%Y%m%d)-${{github.run_number}}"
-            VERSION=$(grep ':VERSION' version.pri | cut -d= -f2 | tr -d '[:space:]')
-            if [[ ! -z $(git rev-list --after="25 hours" ${{github.sha}}) ]]; then
-              echo "::set-output name=nightly-changes::true"
-            fi
-          fi
-          sed -i "s/:VERSION.*/:VERSION = ${VERSION}${BUILDREV}/" version.pri
-
       - name: Install dependencies
         shell: bash
         run: |
           sudo apt-get install golang debhelper devscripts dput-ng -y
           pip3 install -r requirements.txt
 
       - name: Build source bundle
+        id: gen-source
         shell: bash
         env:
           GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
           GPG_PASSWORD: ${{ secrets.GPG_PASSWORD }}
           GNUPGHOME: ${{ runner.temp }}/gnupg-data
+          GITREF: ${{github.ref}}
         run: |
           mkdir -m700 $GNUPGHOME
           echo "allow-preset-passphrase" > $GNUPGHOME/gpg-agent.conf
@@ -63,7 +44,13 @@ jobs:
           KEYGRIP=$(gpg --with-colons --with-keygrip --list-keys | grep -m1 '^grp:' | tr -d [grp:])
 
           echo "$GPG_PASSWORD" | /lib/gnupg2/gpg-preset-passphrase --preset $KEYGRIP
-          ./scripts/linux/script.sh --sign-key $KEYID --source
+          ./scripts/linux/script.sh --sign-key $KEYID --source --gitref ${GITREF}
+
+          if [[ "$GITREF" == "refs/heads/main" ]]; then
+            if [[ ! -z $(git rev-list --after="25 hours" ${{github.sha}}) ]]; then
+              echo "::set-output name=nightly-changes::true"
+            fi
+          fi
 
       - name: Uploading sources
         uses: actions/upload-artifact@v2
@@ -84,7 +71,7 @@ jobs:
 
       - name: Push to Launchpad Nightly PPA
         shell: bash
-        if: ${{ steps.gen-version.outputs.nightly-changes == 'true' }}
+        if: ${{ steps.gen-source.outputs.nightly-changes == 'true' }}
         env:
           PPA_URL: ppa:okirby/mozilla-vpn-nightly
           GNUPGHOME: ${{ runner.temp }}/gnupg-data

.github/workflows/test_lottie.yaml

@@ -49,7 +49,7 @@ jobs:
         shell: bash
         run: |
           auth_header="$(git config --local --get http.https://github.com/.extraheader)"
-          git clone https://github.com/mozilla-mobile/qt_static_macos
+          git clone https://github.com/mozilla-mobile/qt_static_macos --depth 1
           cd qt_static_macos
           cat qt6* > qt_static.tar.gz
           tar xf qt_static.tar.gz
@@ -88,7 +88,7 @@ jobs:
         shell: bash
         run: |
           auth_header="$(git config --local --get http.https://github.com/.extraheader)"
-          git clone https://github.com/mozilla-mobile/qt_static_windows
+          git clone https://github.com/mozilla-mobile/qt_static_windows --depth 1
           cd qt_static_windows
           cat qt6* > qt_static.tar.bz2
           tar xf qt_static.tar.bz2

.github/workflows/test_qml.yaml

@@ -21,45 +21,45 @@ jobs:
     env:
       QTVERSION: 6.2.4
     steps:
-      - name: Install Linux packages
+      - name: Clone repository
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install git libgl-dev libegl-dev libpolkit-gobject-1-dev clang-10 liboath-dev python3 -y
+          python3 -m pip install -r requirements.txt
           python3 -m pip install aqtinstall
           aqt install-qt --outputdir /opt linux desktop $QTVERSION gcc_64 -m all
 
-      - name: Clone repository
-        uses: actions/checkout@v2
-
       - name: Checkout submodules
         shell: bash
         run: |
           auth_header="$(git config --local --get http.https://github.com/.extraheader)"
           git submodule sync --recursive
           git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1
 
-      - name: Generating glean samples
-        shell: bash
-        run: |
-          pip3 install -r requirements.txt
-          python3 scripts/utils/generate_glean.py
-
-      - name: Importing translation files
-        shell: bash
-        run: |
-          export PATH=/opt/$QTVERSION/gcc_64/bin:$PATH
-          python3 scripts/utils/import_languages.py
-
       - name: Install Grcov
         shell: bash
         run: |
           cargo install grcov
+      
+      - name: Building tests
+        shell: bash
+        run: |
+          mkdir -p build
+          cmake -S . -B $(pwd)/build -DBUILD_TESTING=ON \
+            -DCMAKE_PREFIX_PATH=/opt/$QTVERSION/gcc_64/lib/cmake/ \
+            -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_EXE_LINKER_FLAGS=--coverage
+          cmake --build $(pwd)/build --target qml_tests
 
       - name: Running tests
         shell: bash
         run: |
-          export PATH=/opt/$QTVERSION/gcc_64/bin:$PATH
-          xvfb-run -a ./scripts/tests/qml_tests.sh --grcov qml_lcov.info
+          $(pwd)/build/tests/qml/qml_tests -platform offscreen
+          grcov $(pwd)/build/tests/qml/CMakeFiles/qml_tests.dir \
+              -s $(pwd)/build/tests/qml -t lcov --branch --ignore-not-existing \
+                  > ${{github.workspace}}/qml_lcov.info
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
@@ -82,7 +82,7 @@ jobs:
         shell: bash
         run: |
           auth_header="$(git config --local --get http.https://github.com/.extraheader)"
-          git clone https://github.com/mozilla-mobile/qt_static_macos
+          git clone https://github.com/mozilla-mobile/qt_static_macos --depth 1
           cd qt_static_macos
           cat qt6* > qt_static.tar.gz
           tar xf qt_static.tar.gz
@@ -147,7 +147,7 @@ jobs:
         shell: bash
         run: |
           auth_header="$(git config --local --get http.https://github.com/.extraheader)"
-          git clone https://github.com/mozilla-mobile/qt_static_windows
+          git clone https://github.com/mozilla-mobile/qt_static_windows --depth 1
           cd qt_static_windows
           cat qt6* > qt_static.tar.bz2
           tar xf qt_static.tar.bz2
@@ -165,20 +165,19 @@ jobs:
       - name: Add msbuild to PATH
         uses: microsoft/[email protected]
 
-      - name: Install glean depedencies
+      - name: Install depedencies
         shell: bash
         run: |
           pip3 install -r requirements.txt
-          python3 scripts/utils/generate_glean.py
 
-      - name: Importing translation files
+      - name: Building tests
         shell: bash
         run: |
           export PATH=/c/MozillaVPNBuild/bin:$PATH
-          python3 scripts/utils/import_languages.py -m
+          mkdir -p build
+          cmake -S . -B $(pwd)/build -DBUILD_TESTING=ON
+          cmake --build $(pwd)/build --config Release --target qml_tests
 
       - name: Running tests
         shell: bash
-        run: |
-          export PATH=/c/MozillaVPNBuild/bin:$PATH
-          ./scripts/tests/qml_tests.sh
+        run: build/tests/qml/Release/qml_tests.exe