From a2b2c0555a5e89c8a91045e385ebce511e1587b1 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 10:25:16 -0800 Subject: [PATCH 01/16] Try using ccache to speed up functional testing --- .github/workflows/macos_tests.yaml | 8 ++++++++ env.yml | 1 + 2 files changed, 9 insertions(+) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index cd27a85b1b..3681722e18 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -46,10 +46,18 @@ jobs: wget https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/mozillavpn.v2.mozillavpn.cache.level-3.toolchains.v3.qt-macos-6.6.latest/artifacts/public%2Fbuild%2Fqt6_mac.zip -O qt6_mac.zip unzip -a -d ${{ github.workspace }} qt6_mac.zip + - name: Setup compiler cache + if: github.event_name == 'pull_request' + uses: actions/cache@v4 + with: + path: .ccache + key: ${{ runner.os }}-ccache-${{ github.event.pull_request.base.sha }} + - name: Compile test client run: | mkdir -p build/cmake cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_PREFIX_PATH=${{ github.workspace }}/qt_dist/lib/cmake cmake --build build/cmake cp -r ./build/cmake/src/Mozilla\ VPN.app/ build/Mozilla\ VPN.app diff --git a/env.yml b/env.yml index 02b4e7086d..b3f6f436b5 100644 --- a/env.yml +++ b/env.yml @@ -5,6 +5,7 @@ dependencies: - clang=16.0.6 - clang-tools=16.0.6 - clangxx=16.0.6 + - ccache=4.10.1 - python=3.9 - nodejs=18.16.* - pip=22.3.1 From 9d2edb32bcd281d11d30028fdfe808c73a126844 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 10:55:25 -0800 Subject: [PATCH 02/16] Change some whitespace to see what needs rebuilding --- src/dnshelper.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dnshelper.h b/src/dnshelper.h index 42ee743893..44546b41a0 100644 --- a/src/dnshelper.h +++ b/src/dnshelper.h @@ -11,6 +11,7 @@ struct dnsData { QString ipAddress; QString dnsType; }; + class DNSHelper final { public: static QString getDNS(const QString& fallbackAddress); From 5613fd632eaa0cdaad208afb32f505c2ccaf084f Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 11:16:36 -0800 Subject: [PATCH 03/16] More caching! More debug! --- .github/workflows/macos_tests.yaml | 34 +++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 3681722e18..8c4b8566bb 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -29,11 +29,32 @@ jobs: with: submodules: "recursive" + - name: Get Datestamp + id: get-date + shell: bash + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" - environment-file: env.yml activate-environment: vpn + + - name: Setup conda cache + id: conda-cache + uses: actions/cache@v4 + with: + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('env.yml', 'requirements.txt') }} + + - name: Setup conda env + if: steps.conda-cache.outputs.cache-hit != 'true' + run: conda env update -n vpn -f env.yml + + - name: Setup compiler cache + uses: actions/cache@v4 + with: + path: .ccache + key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }} - name: Install build dependencies run: | @@ -46,15 +67,12 @@ jobs: wget https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/mozillavpn.v2.mozillavpn.cache.level-3.toolchains.v3.qt-macos-6.6.latest/artifacts/public%2Fbuild%2Fqt6_mac.zip -O qt6_mac.zip unzip -a -d ${{ github.workspace }} qt6_mac.zip - - name: Setup compiler cache - if: github.event_name == 'pull_request' - uses: actions/cache@v4 - with: - path: .ccache - key: ${{ runner.os }}-ccache-${{ github.event.pull_request.base.sha }} - - name: Compile test client run: | + echo "DEBUG: Where is the compiler cache?" + ccache -p + echo "" + mkdir -p build/cmake cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ From 52d6e8291f042bfc8b5d65160c2ff4f4f3de5fdf Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 11:51:47 -0800 Subject: [PATCH 04/16] Lets try again to get caching right --- .github/workflows/macos_tests.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 8c4b8566bb..cb4cd7a3e8 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -53,8 +53,14 @@ jobs: - name: Setup compiler cache uses: actions/cache@v4 with: - path: .ccache - key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }} + path: ${{ env.HOME }}/Library/Caches/ccache + key: | + github.event_name == 'pull_request' && + ccache-${{ runner.os }}-${{ runner.arch }}-pull-${{ github.event.pull_request.number || + ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.sha }} + restore-keys: | + github.event_name == 'pull_request' && + ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} - name: Install build dependencies run: | @@ -69,10 +75,6 @@ jobs: - name: Compile test client run: | - echo "DEBUG: Where is the compiler cache?" - ccache -p - echo "" - mkdir -p build/cmake cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ From a3858089a379f230754b5b33a391d446b93a2639 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 11:56:16 -0800 Subject: [PATCH 05/16] Once more with feeling --- .github/workflows/macos_tests.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index cb4cd7a3e8..ba09adfef9 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -54,12 +54,8 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.HOME }}/Library/Caches/ccache - key: | - github.event_name == 'pull_request' && - ccache-${{ runner.os }}-${{ runner.arch }}-pull-${{ github.event.pull_request.number || - ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.sha }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && 'pull' || 'sha' }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }} restore-keys: | - github.event_name == 'pull_request' && ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} - name: Install build dependencies From e25f54ae1f9a575aba76996e98bdd4b932feacc5 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 12:11:26 -0800 Subject: [PATCH 06/16] Remove conda cache for now and I guess HOME isn't set by default --- .github/workflows/macos_tests.yaml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index ba09adfef9..056479147b 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -37,23 +37,13 @@ jobs: - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" + environment-file: "env.yml" activate-environment: vpn - - - name: Setup conda cache - id: conda-cache - uses: actions/cache@v4 - with: - path: ${{ env.CONDA }}/envs - key: conda-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('env.yml', 'requirements.txt') }} - - - name: Setup conda env - if: steps.conda-cache.outputs.cache-hit != 'true' - run: conda env update -n vpn -f env.yml - name: Setup compiler cache uses: actions/cache@v4 with: - path: ${{ env.HOME }}/Library/Caches/ccache + path: /Users/runner/Library/Caches/ccache key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && 'pull' || 'sha' }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} From 83fecb95c7fa025f433337d8c14d9e4990d4e6c5 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 12:32:43 -0800 Subject: [PATCH 07/16] Cache just the conda packages --- .github/workflows/macos_tests.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 056479147b..2b39ad10fb 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -34,6 +34,12 @@ jobs: shell: bash run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + - name: Setup conda package cache + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: condapkg-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('env.yml') }} + - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" From 1ab449b62d9eaab8510036641f7568a6707d6515 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 12:47:37 -0800 Subject: [PATCH 08/16] Use ccache for Linux too --- .github/workflows/linux_tests.yaml | 13 +++++++++++-- .github/workflows/macos_tests.yaml | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux_tests.yaml b/.github/workflows/linux_tests.yaml index d3e01b23d1..2e2bd77ebb 100644 --- a/.github/workflows/linux_tests.yaml +++ b/.github/workflows/linux_tests.yaml @@ -27,7 +27,7 @@ jobs: submodules: "recursive" - run: | sudo apt-get update - sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control) + sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control) ccache - uses: actions/setup-python@v5 with: @@ -35,11 +35,20 @@ jobs: cache: "pip" - run: pip install -r requirements.txt + - name: Setup compiler cache + uses: actions/cache@v4 + with: + path: ~/.ccache + key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && 'pull' || 'sha' }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} + - name: Compile test client shell: bash run: | mkdir -p build/cmake - cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo + cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache cmake --build build/cmake cp ./build/cmake/src/mozillavpn build/ diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 2b39ad10fb..69a040a3dd 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -49,7 +49,7 @@ jobs: - name: Setup compiler cache uses: actions/cache@v4 with: - path: /Users/runner/Library/Caches/ccache + path: ~/Library/Caches/ccache key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && 'pull' || 'sha' }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} From d688c167ab4556c2cb37f064027e983bd65f76e4 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 13:02:36 -0800 Subject: [PATCH 09/16] Cache cargo packages too --- .github/workflows/macos_tests.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 69a040a3dd..b7ceaa7e8b 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -54,6 +54,18 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} + - name: Setup cargo package cache + id: cargo-vendor + uses: actions/cache@v4 + with: + path: 3rdparty/cargo-vendor + key: cargo-vendor-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + + - name: Download cargo packages + if: steps.cargo-vendor.outputs.cache-hit != 'true' + shell: bash + run: cargo vendor --manifest-path Cargo.toml 3rdparty/cargo-vendor > 3rdparty/cargo-vendor/config.toml + - name: Install build dependencies run: | ./scripts/macos/conda_install_extras.sh @@ -67,6 +79,9 @@ jobs: - name: Compile test client run: | + mkdir -p .cargo + cp 3rdparty/cargo-vendor/config.toml .cargo/config.toml + mkdir -p build/cmake cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ From 8861f445c85366f4e882dc066ccba436d1eac47a Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 13:06:45 -0800 Subject: [PATCH 10/16] Fix cargo vendor task --- .github/workflows/macos_tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index b7ceaa7e8b..a1baa3b4ac 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -64,7 +64,9 @@ jobs: - name: Download cargo packages if: steps.cargo-vendor.outputs.cache-hit != 'true' shell: bash - run: cargo vendor --manifest-path Cargo.toml 3rdparty/cargo-vendor > 3rdparty/cargo-vendor/config.toml + run: | + mkdir -p 3rdparty/cargo-vendor + cargo vendor --manifest-path Cargo.toml 3rdparty/cargo-vendor > 3rdparty/cargo-vendor/config.toml - name: Install build dependencies run: | From da95225c40cc5b134596525cdf2ae75dfed8e39c Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Tue, 19 Nov 2024 13:17:30 -0800 Subject: [PATCH 11/16] Remove cargo and conda caches - they don't really help --- .github/workflows/macos_tests.yaml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index a1baa3b4ac..2c83b2a3df 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -29,17 +29,6 @@ jobs: with: submodules: "recursive" - - name: Get Datestamp - id: get-date - shell: bash - run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT - - - name: Setup conda package cache - uses: actions/cache@v4 - with: - path: ~/conda_pkgs_dir - key: condapkg-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.today }}-${{ hashFiles('env.yml') }} - - uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: "latest" @@ -54,20 +43,6 @@ jobs: restore-keys: | ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} - - name: Setup cargo package cache - id: cargo-vendor - uses: actions/cache@v4 - with: - path: 3rdparty/cargo-vendor - key: cargo-vendor-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} - - - name: Download cargo packages - if: steps.cargo-vendor.outputs.cache-hit != 'true' - shell: bash - run: | - mkdir -p 3rdparty/cargo-vendor - cargo vendor --manifest-path Cargo.toml 3rdparty/cargo-vendor > 3rdparty/cargo-vendor/config.toml - - name: Install build dependencies run: | ./scripts/macos/conda_install_extras.sh @@ -81,9 +56,6 @@ jobs: - name: Compile test client run: | - mkdir -p .cargo - cp 3rdparty/cargo-vendor/config.toml .cargo/config.toml - mkdir -p build/cmake cmake -S $(pwd) -B build/cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \ From 92bc2c5879d97e2d0b9a6d1cf8c7510602bc1f88 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Wed, 20 Nov 2024 11:27:54 -0800 Subject: [PATCH 12/16] Try caching test addons too --- .github/workflows/linux_tests.yaml | 52 +++++++++++++++++++++++++----- .github/workflows/macos_tests.yaml | 48 +++++++++++++++++++++++---- 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/.github/workflows/linux_tests.yaml b/.github/workflows/linux_tests.yaml index 2e2bd77ebb..1e8421d2eb 100644 --- a/.github/workflows/linux_tests.yaml +++ b/.github/workflows/linux_tests.yaml @@ -15,6 +15,41 @@ concurrency: cancel-in-progress: true jobs: + build_test_adons: + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v4 + + - name: Setup addons cache + id: addons-cache + uses: actions/cache@v4 + with: + path: build-addons/ + key: test-addons-${{ hashFiles('addons/', 'test/functional/addons/') }} + + - name: Install build dependencies + if: steps.addons-cache.outputs.cache-hit != 'true' + shell: bash + run: | + git submodule init + git submodule update 3rdparty/i18n + sudo apt-get update + sudo apt-get install -y $(./scripts/linux/getdeps.py -b linux/debian/control) + + - name: Build test addons dependencies + if: steps.addons-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p build-addons/ + cmake -S $(pwd)/tests/functional/addons -B build-addons/ + cmake --build build-addons/ + + - uses: actions/upload-artifact@v4 + with: + name: test-addons-${{ github.sha }} + path: build-addons/ + build_test_app: name: Build Test Client runs-on: ubuntu-22.04 @@ -29,6 +64,9 @@ jobs: sudo apt-get update sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control) ccache + echo "DEBUG: ccache configuration" + ccache -p + - uses: actions/setup-python@v5 with: python-version: "3.9" @@ -52,14 +90,6 @@ jobs: cmake --build build/cmake cp ./build/cmake/src/mozillavpn build/ - - name: Compile test addons - shell: bash - run: | - mkdir -p build/addons - cmake -S $(pwd)/tests/functional/addons -B build/addons \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/qt_dist/lib/cmake - cmake --build build/addons - - uses: actions/upload-artifact@v4 with: name: test-client-${{ github.sha }} @@ -87,6 +117,7 @@ jobs: name: Functional tests needs: - build_test_app + - build_test_addons runs-on: ubuntu-22.04 timeout-minutes: 45 strategy: @@ -102,6 +133,11 @@ jobs: name: test-client-${{ github.sha }} path: build/ + - uses: actions/download-artifact@v4 + with: + name: test-addons-${{ github.sha }} + path: build/addons/ + - name: Install test dependecies run: | sudo apt-get update diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 2c83b2a3df..62ebcaba7b 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -15,6 +15,41 @@ concurrency: cancel-in-progress: true jobs: + build_test_adons: + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v4 + + - name: Setup addons cache + id: addons-cache + uses: actions/cache@v4 + with: + path: build-addons/ + key: test-addons-${{ hashFiles('addons/', 'test/functional/addons/') }} + + - name: Install build dependencies + if: steps.addons-cache.outputs.cache-hit != 'true' + shell: bash + run: | + git submodule init + git submodule update 3rdparty/i18n + sudo apt-get update + sudo apt-get install -y $(./scripts/linux/getdeps.py -b linux/debian/control) + + - name: Build test addons dependencies + if: steps.addons-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p build-addons/ + cmake -S $(pwd)/tests/functional/addons -B build-addons/ + cmake --build build-addons/ + + - uses: actions/upload-artifact@v4 + with: + name: test-addons-${{ github.sha }} + path: build-addons/ + build_test_app: name: Build Test Client runs-on: macos-latest @@ -63,13 +98,6 @@ jobs: cmake --build build/cmake cp -r ./build/cmake/src/Mozilla\ VPN.app/ build/Mozilla\ VPN.app - - name: Compile test addons - run: | - mkdir -p build/addons - cmake -S $(pwd)/tests/functional/addons -B build/addons \ - -DCMAKE_PREFIX_PATH=${{ github.workspace }}/qt_dist/lib/cmake - cmake --build build/addons - - uses: actions/upload-artifact@v4 with: name: test-client-${{ github.sha }} @@ -95,6 +123,7 @@ jobs: name: Functional tests needs: - build_test_app + - build_test_addons runs-on: macos-latest timeout-minutes: 45 strategy: @@ -110,6 +139,11 @@ jobs: name: test-client-${{ github.sha }} path: build/ + - uses: actions/download-artifact@v4 + with: + name: test-addons-${{ github.sha }} + path: build/addons/ + - uses: actions/setup-python@v5 env: PIP_DISABLE_PIP_VERSION_CHECK: 1 From f3aec4c672f62590e0f7e193af3f8412a36847d9 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Wed, 20 Nov 2024 11:29:08 -0800 Subject: [PATCH 13/16] Fix typo --- .github/workflows/linux_tests.yaml | 2 +- .github/workflows/macos_tests.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux_tests.yaml b/.github/workflows/linux_tests.yaml index 1e8421d2eb..59fc315dd5 100644 --- a/.github/workflows/linux_tests.yaml +++ b/.github/workflows/linux_tests.yaml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true jobs: - build_test_adons: + build_test_addons: runs-on: ubuntu-latest steps: - name: Clone repository diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 62ebcaba7b..6165a0ab54 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true jobs: - build_test_adons: + build_test_addons: runs-on: ubuntu-latest steps: - name: Clone repository From 2d00638606a14d8dd6608c5ea0e98e1cfc1a5f31 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Wed, 20 Nov 2024 11:40:56 -0800 Subject: [PATCH 14/16] Build addons with a reusable workflow --- .github/workflows/build_addons.yaml | 44 +++++++++++++++++++++++++++++ .github/workflows/linux_tests.yaml | 42 ++++----------------------- .github/workflows/macos_tests.yaml | 37 +++--------------------- 3 files changed, 53 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/build_addons.yaml diff --git a/.github/workflows/build_addons.yaml b/.github/workflows/build_addons.yaml new file mode 100644 index 0000000000..38da83ca92 --- /dev/null +++ b/.github/workflows/build_addons.yaml @@ -0,0 +1,44 @@ +# A reusable workflow to build addons +on: + workflow_call: + inputs: + test-addons-name: + required: true + type: string + +jobs: + build-test-addons: + name: Build Test Addons + runs-on: ubuntu-latest + steps: + - name: Clone repository + uses: actions/checkout@v4 + + - name: Setup addons cache + id: addons-cache + uses: actions/cache@v4 + with: + path: build-addons/ + key: test-addons-${{ hashFiles('addons/', 'test/functional/addons/') }} + + - name: Install build dependencies + if: steps.addons-cache.outputs.cache-hit != 'true' + shell: bash + run: | + git submodule init + git submodule update 3rdparty/i18n + sudo apt-get update + sudo apt-get install -y $(./scripts/linux/getdeps.py -b linux/debian/control) + + - name: Build test addons + if: steps.addons-cache.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p build-addons/ + cmake -S $(pwd)/tests/functional/addons -B build-addons/ + cmake --build build-addons/ + + - uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.test-addons-name }} + path: build-addons/ diff --git a/.github/workflows/linux_tests.yaml b/.github/workflows/linux_tests.yaml index 59fc315dd5..895776f3c8 100644 --- a/.github/workflows/linux_tests.yaml +++ b/.github/workflows/linux_tests.yaml @@ -16,39 +16,10 @@ concurrency: jobs: build_test_addons: - runs-on: ubuntu-latest - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Setup addons cache - id: addons-cache - uses: actions/cache@v4 - with: - path: build-addons/ - key: test-addons-${{ hashFiles('addons/', 'test/functional/addons/') }} - - - name: Install build dependencies - if: steps.addons-cache.outputs.cache-hit != 'true' - shell: bash - run: | - git submodule init - git submodule update 3rdparty/i18n - sudo apt-get update - sudo apt-get install -y $(./scripts/linux/getdeps.py -b linux/debian/control) - - - name: Build test addons dependencies - if: steps.addons-cache.outputs.cache-hit != 'true' - shell: bash - run: | - mkdir -p build-addons/ - cmake -S $(pwd)/tests/functional/addons -B build-addons/ - cmake --build build-addons/ - - - uses: actions/upload-artifact@v4 - with: - name: test-addons-${{ github.sha }} - path: build-addons/ + name: Build Test Addons + uses: ./.github/workflows/build_addons.yaml + with: + test-addons-name: test-addons-${{ github.sha }} build_test_app: name: Build Test Client @@ -64,9 +35,6 @@ jobs: sudo apt-get update sudo apt-get install -y $(./scripts/linux/getdeps.py -a linux/debian/control) ccache - echo "DEBUG: ccache configuration" - ccache -p - - uses: actions/setup-python@v5 with: python-version: "3.9" @@ -76,7 +44,7 @@ jobs: - name: Setup compiler cache uses: actions/cache@v4 with: - path: ~/.ccache + path: ~/.cache/ccache key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && 'pull' || 'sha' }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index 6165a0ab54..a08593103c 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -16,39 +16,10 @@ concurrency: jobs: build_test_addons: - runs-on: ubuntu-latest - steps: - - name: Clone repository - uses: actions/checkout@v4 - - - name: Setup addons cache - id: addons-cache - uses: actions/cache@v4 - with: - path: build-addons/ - key: test-addons-${{ hashFiles('addons/', 'test/functional/addons/') }} - - - name: Install build dependencies - if: steps.addons-cache.outputs.cache-hit != 'true' - shell: bash - run: | - git submodule init - git submodule update 3rdparty/i18n - sudo apt-get update - sudo apt-get install -y $(./scripts/linux/getdeps.py -b linux/debian/control) - - - name: Build test addons dependencies - if: steps.addons-cache.outputs.cache-hit != 'true' - shell: bash - run: | - mkdir -p build-addons/ - cmake -S $(pwd)/tests/functional/addons -B build-addons/ - cmake --build build-addons/ - - - uses: actions/upload-artifact@v4 - with: - name: test-addons-${{ github.sha }} - path: build-addons/ + name: Build Test Addons + uses: ./.github/workflows/build_addons.yaml + with: + test-addons-name: test-addons-${{ github.sha }} build_test_app: name: Build Test Client From 1bd9c6acafb852573635254cb25144be07e5289d Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Wed, 20 Nov 2024 12:21:20 -0800 Subject: [PATCH 15/16] Make cache read-only on PR --- .github/workflows/linux_tests.yaml | 6 +++--- .github/workflows/macos_tests.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux_tests.yaml b/.github/workflows/linux_tests.yaml index 895776f3c8..c13a71f2fa 100644 --- a/.github/workflows/linux_tests.yaml +++ b/.github/workflows/linux_tests.yaml @@ -45,9 +45,9 @@ jobs: uses: actions/cache@v4 with: path: ~/.cache/ccache - key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && 'pull' || 'sha' }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + lookup-only: github.event_name == 'pull_request' + restore-keys: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} - name: Compile test client shell: bash diff --git a/.github/workflows/macos_tests.yaml b/.github/workflows/macos_tests.yaml index a08593103c..7a8e17dac0 100644 --- a/.github/workflows/macos_tests.yaml +++ b/.github/workflows/macos_tests.yaml @@ -45,9 +45,9 @@ jobs: uses: actions/cache@v4 with: path: ~/Library/Caches/ccache - key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && 'pull' || 'sha' }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.sha }} - restore-keys: | - ccache-${{ runner.os }}-${{ runner.arch }}-sha-${{ github.event.pull_request.base.sha }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + lookup-only: github.event_name == 'pull_request' + restore-keys: ccache-${{ runner.os }}-${{ runner.arch }}-${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }} - name: Install build dependencies run: | From 282d0d5d16cc94ddcafeec690e0223558e97a246 Mon Sep 17 00:00:00 2001 From: Naomi Kirby Date: Wed, 20 Nov 2024 12:53:45 -0800 Subject: [PATCH 16/16] Reuse addons for Wasm too --- .github/workflows/wasm_tests.yaml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/wasm_tests.yaml b/.github/workflows/wasm_tests.yaml index 2d0694298e..f307e269f1 100644 --- a/.github/workflows/wasm_tests.yaml +++ b/.github/workflows/wasm_tests.yaml @@ -15,7 +15,13 @@ concurrency: cancel-in-progress: true jobs: - wasmQt6: + build_test_addons: + name: Build Test Addons + uses: ./.github/workflows/build_addons.yaml + with: + test-addons-name: test-addons-${{ github.sha }} + + build_test_app: name: Wasm Qt6 runs-on: ubuntu-20.04 outputs: @@ -62,17 +68,9 @@ jobs: cmake --build build/cmake -j4 cp -r build/cmake/wasm_build build/wasm_build - - name: Compile test addons - shell: bash - run: | - mkdir -p build/addons - cmake -S $(pwd)/tests/functional/addons -B build/addons \ - -DCMAKE_PREFIX_PATH=/opt/$QTVERSION/gcc_64/lib/cmake - cmake --build build/addons - - uses: actions/upload-artifact@v4 with: - name: WebAssembly Build Qt6 + name: test-wasm-${{ github.sha }} path: | build/ !build/cmake/ @@ -96,7 +94,8 @@ jobs: functionaltests: name: Functional tests needs: - - wasmQt6 + - build_test_app + - build_test_addons runs-on: ubuntu-20.04 timeout-minutes: 45 strategy: @@ -112,9 +111,14 @@ jobs: - uses: actions/download-artifact@v4 with: - name: WebAssembly Build Qt6 + name: test-wasm-${{ github.sha }} path: build/ + - uses: actions/download-artifact@v4 + with: + name: test-addons-${{ github.sha }} + path: build/addons/ + - uses: actions/setup-python@v5 with: python-version: "3.9"