diff --git a/CMakeLists.txt b/CMakeLists.txt index 9219dd1b3b..9f710faaa0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,9 @@ if(NOT (IS_MULTI_CONFIG OR DEFINED CMAKE_BUILD_TYPE)) set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "default build type" FORCE) message("Setting build type ${CMAKE_BUILD_TYPE}") endif() +if(WIN32 AND DEFINED ENV{CONDA_PREFIX}) + include(scripts/windows/conda-toolchain.cmake) +endif() file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/version.txt" APP_VERSION) message(APP_VERSION=${APP_VERSION}) @@ -56,6 +59,11 @@ if(WIN32) set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/unsigned" CACHE PATH "default install path" FORCE) endif() + + ## Generate a cargo/config.toml file to pass the Conda toolchain into rust too. + if(DEFINED ENV{CONDA_PREFIX}) + configure_file(scripts/windows/cargo-config.toml.in ${CMAKE_BINARY_DIR}/cargo_home/config.toml) + endif() endif() if(APPLE) enable_language(OBJC) diff --git a/docs/Building/windows.md b/docs/Building/windows.md index dd8d47283b..c3698efbbe 100644 --- a/docs/Building/windows.md +++ b/docs/Building/windows.md @@ -21,14 +21,12 @@ You may need to enable [powershell scripts.](https://learn.microsoft.com/en-us/p For the setup you need to use powershell. ```powershell -$ conda env create -f env.yml +$ conda env create -f env-windows.yml $ conda activate vpn -# Setup the conda env for clang-cl compilation -$ ./scripts/windows/conda_setup_win_sdk.ps1 +# Download the Windows SDK using Xwin +$ ./scripts/windows/conda-setup-xwin-sdk.ps1 # Setup the conda env to use a prebuild Qt from Moz-CI $ ./scripts/windows/conda_setup_win_qt.ps1 -# Install conda Packages only needed on Windows -$ ./scripts/windows/conda_install_extras.ps1 # Reactivate the env to apply the changes: $ conda deactivate $ conda activate vpn diff --git a/env-windows.yml b/env-windows.yml new file mode 100644 index 0000000000..3a7f4394fd --- /dev/null +++ b/env-windows.yml @@ -0,0 +1,24 @@ +name: vpn +channels: + - conda-forge +dependencies: + - clang=16.0.6 + - clang-tools=16.0.6 + - clangxx=16.0.6 + - llvm=16.0.6 + - llvmdev=16.0.6 + - lld=16.0.6 + - python=3.9 + - nodejs=18.16.* + - pip=22.3.1 + - rust=1.75 + - compiler-rt + - cmake=3.26.3 + - ninja=1.11.0 + - conda-pack=0.8.0 + - vswhere=3.1.7 + - pip: + - -r requirements.txt + - -r taskcluster/scripts/requirements.txt +variables: + QT_VERSION: 6.6.3 diff --git a/env.yml b/env.yml index 6eb6b71ebb..02b4e7086d 100644 --- a/env.yml +++ b/env.yml @@ -2,21 +2,21 @@ name: vpn channels: - conda-forge dependencies: - - clang=14.0.6 - - clang-tools=14.0.6 - - clangxx=14.0.6 + - clang=16.0.6 + - clang-tools=16.0.6 + - clangxx=16.0.6 - python=3.9 - nodejs=18.16.* - pip=22.3.1 - - rust=1.69 - - rust-std-aarch64-apple-darwin=1.69 - - rust-std-x86_64-apple-darwin=1.69 - - rust-std-aarch64-apple-ios=1.69 - - rust-std-x86_64-apple-ios=1.69 - - rust-std-armv7-linux-androideabi=1.69 - - rust-std-x86_64-linux-android=1.69 - - rust-std-i686-linux-android=1.69 - - rust-std-aarch64-linux-android=1.69 + - rust=1.75 + - rust-std-aarch64-apple-darwin=1.75 + - rust-std-x86_64-apple-darwin=1.75 + - rust-std-aarch64-apple-ios=1.75 + - rust-std-x86_64-apple-ios=1.75 + - rust-std-armv7-linux-androideabi=1.75 + - rust-std-x86_64-linux-android=1.75 + - rust-std-i686-linux-android=1.75 + - rust-std-aarch64-linux-android=1.75 - go=1.18 - compiler-rt - cmake=3.26.3 diff --git a/extension/CMakeLists.txt b/extension/CMakeLists.txt index 2988d8b057..a2183f9bad 100644 --- a/extension/CMakeLists.txt +++ b/extension/CMakeLists.txt @@ -12,7 +12,7 @@ if(APPLE AND CMAKE_OSX_ARCHITECTURES) string(REPLACE "arm64" "aarch64" OSXARCH ${OSXARCH}) add_custom_target(cargo_mozillavpnnp_${OSXARCH} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bridge - COMMAND ${CMAKE_COMMAND} -E env CARGO_HOME=${CMAKE_CURRENT_BINARY_DIR}/.cargo_home + COMMAND ${CMAKE_COMMAND} -E env CARGO_HOME=${CMAKE_BINARY_DIR}/cargo_home cargo build --release --target-dir "${CMAKE_CURRENT_BINARY_DIR}/bridge" --target "${OSXARCH}-apple-darwin" ) @@ -32,7 +32,7 @@ else() add_custom_target(cargo_mozillavpnnp ALL COMMENT "Building web extension crate" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/bridge - COMMAND ${CMAKE_COMMAND} -E env CARGO_HOME=${CMAKE_CURRENT_BINARY_DIR}/.cargo_home + COMMAND ${CMAKE_COMMAND} -E env CARGO_HOME=${CMAKE_BINARY_DIR}/cargo_home cargo build --release --target-dir ${CMAKE_CURRENT_BINARY_DIR}/bridge/target ) set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_CURRENT_BINARY_DIR}/bridge/target) diff --git a/linux/debian/control b/linux/debian/control index 95fa854ab1..b5212fed8d 100644 --- a/linux/debian/control +++ b/linux/debian/control @@ -9,7 +9,7 @@ Build-Depends: debhelper (>= 11.2), gcc (>=4:8.0.0~), g++ (>=4:8.0.0~), golang (>=2:1.18~) | golang-1.18, - cargo, + cargo (>=1.75), python3-yaml, python3-jinja2, python3-click, diff --git a/linux/mozillavpn.spec b/linux/mozillavpn.spec index 74176126a7..e033fbfe6b 100644 --- a/linux/mozillavpn.spec +++ b/linux/mozillavpn.spec @@ -15,7 +15,7 @@ Requires: qt6-qtsvg >= 6.0 Requires: qt6-qt5compat >= 6.0 Requires: wireguard-tools -BuildRequires: cargo +BuildRequires: cargo >= 1.75 BuildRequires: golang >= 1.18 BuildRequires: libcap-devel BuildRequires: libsecret-devel diff --git a/scripts/windows/cargo-config.toml.in b/scripts/windows/cargo-config.toml.in new file mode 100644 index 0000000000..e57e8ab38e --- /dev/null +++ b/scripts/windows/cargo-config.toml.in @@ -0,0 +1,10 @@ +# Auto generated by: ${CMAKE_CURRENT_LIST_FILE} +[target.x86_64-pc-windows-msvc] +linker="${CMAKE_LINKER}" + +[env] +CC_x86_64-pc-windows-msvc="${CMAKE_C_COMPILER}" +CXX_x86_64-pc-windows-msvc="${CMAKE_CXX_COMPILER}" +AR_x86_64-pc-windows-msvc="${CMAKE_AR}" +CFLAGS_x86_64-pc-windows-msvc="${CMAKE_C_FLAGS}" +CXXFLAGS_x86_64-pc-windows-msvc="${CMAKE_CXX_FLAGS}" diff --git a/scripts/windows/conda-setup-xwin-sdk.ps1 b/scripts/windows/conda-setup-xwin-sdk.ps1 new file mode 100644 index 0000000000..b67a04db47 --- /dev/null +++ b/scripts/windows/conda-setup-xwin-sdk.ps1 @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +$X_WIN_VERSION = "0.6.5" + +$conda_env = conda info --json | ConvertFrom-Json + +if( $conda_env.active_prefix_name -eq "base"){ + Write-Output("Not in an active conda env. abort") + return -1 +} +$conda_folder = $conda_env.active_prefix +Write-Output("Downloading x-win") +# Small X-Win appretiation comment. +# It's really great, +# it can process windows store manifests, request files from MS-servers and unpack it. +# we can use it to get the offical SDK files from microsoft. +# We can't cargo-install it because to use the clang-msvc compat mode, +# +Invoke-WebRequest -Uri "https://github.com/Jake-Shadle/xwin/releases/download/$X_WIN_VERSION/xwin-$X_WIN_VERSION-x86_64-pc-windows-msvc.tar.gz" -OutFile "$conda_folder\xwin.tar.gz" +$ProgressPreference = 'Continue' +Write-Output("Unpack x-win") +Start-Process -WorkingDirectory "$conda_folder" -Wait tar -ArgumentList @('-xf', "xwin.tar.gz") + +# Splat the CRT and SDK file to /xwin/crt and /xwin/sdk respectively +Write-Output("Downloading the windows SDK") +$env:PATH ="$conda_folder\xwin-$X_WIN_VERSION-x86_64-pc-windows-msvc;$env:PATH" +xwin --accept-license --manifest-version 16 splat --include-debug-symbols --include-debug-libs --use-winsysroot-style --preserve-ms-arch-notation --disable-symlinks --output "$conda_folder\xwin" + +Write-Output("Downloading Microsoft.VisualStudio.Component.VC.Redist.MSM") +python "$PSScriptRoot\fetch-vsix-package.py" --manifest-version 16 --output "$conda_folder\xwin" Microsoft.VisualStudio.Component.VC.Redist.MSM + +Write-Output("Cleaning Up") +Remove-Item "$conda_folder\xwin-$X_WIN_VERSION-x86_64-pc-windows-msvc" -ErrorAction SilentlyContinue -Force -Recurse +Remove-Item ".xwin-cache" -ErrorAction SilentlyContinue -Force -Recurse +Remove-Item "xwin.tar.gz" -ErrorAction SilentlyContinue -Force diff --git a/scripts/windows/conda-toolchain.cmake b/scripts/windows/conda-toolchain.cmake new file mode 100644 index 0000000000..f3ae972030 --- /dev/null +++ b/scripts/windows/conda-toolchain.cmake @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file is intended to be used as a CMake toolchain file, and selects the +# clang-cl (MSVC compatible) compiler from the current conda prefix for use in +# building the application. +# +# To use this, set the CMAKE_TOOLCHAIN_FILE argument when setting up the Cmake +# project. For example: +# cmake .. -DCMAKE_TOOLCHAIN_FILE=scripts/windows/conda-toolchain.cmake +if(NOT DEFINED ENV{CONDA_PREFIX}) + message(WARNING "Conda toolchain specified, but not in a Conda environment") + return() +endif() + +# Set the C++ compiler and tools. +if(NOT CMAKE_C_COMPILER) + find_program(CMAKE_C_COMPILER NAMES clang-cl REQUIRED DOC "Clang C Compiler (MSVC Compatible)") +endif() +if(NOT CMAKE_CXX_COMPILER) + find_program(CMAKE_CXX_COMPILER NAMES clang-cl REQUIRED DOC "Clang C++ Compiler (MSVC Compatible)") +endif() +if(NOT CMAKE_RC_COMPILER) + find_program(CMAKE_RC_COMPILER NAMES llvm-rc REQUIRED DOC "LLVM Resource Compiler") +endif() +if(NOT CMAKE_MT) + find_program(CMAKE_MT NAMES llvm-mt REQUIRED DOC "LLVM Manifest Tool") +endif() +if(NOT PYTHON_EXECUTABLE) + find_program(PYTHON_EXECUTABLE PATHS "$ENV{CONDA_PREFIX}" NAMES python REQUIRED DOC "Python Interpreter (Conda)") +endif() + +if(EXISTS $ENV{CONDA_PREFIX}/xwin) + cmake_path(CONVERT "$ENV{CONDA_PREFIX}\\xwin" TO_CMAKE_PATH_LIST XWIN_PREFIX) + + set(CMAKE_C_FLAGS_INIT "/winsysroot ${XWIN_PREFIX} -fuse-ld=lld-link") + set(CMAKE_CXX_FLAGS_INIT "/winsysroot ${XWIN_PREFIX} -fuse-ld=lld-link") + set(CMAKE_RC_FLAGS_INIT "/winsysroot ${XWIN_PREFIX} -fuse-ld=lld-link") + set(CMAKE_LINKER ${CMAKE_CURRENT_LIST_DIR}/xwin-link.bat CACHE FILEPATH "LLD Linker for the Xwin SDK") +elseif(NOT CMAKE_LINKER) + find_program(CMAKE_LINKER NAMES lld-link REQUIRED DOC "LLD Linker (MSVC Compatible)") +endif() diff --git a/scripts/windows/conda_install_extras.ps1 b/scripts/windows/conda_install_extras.ps1 deleted file mode 100644 index d715689f4e..0000000000 --- a/scripts/windows/conda_install_extras.ps1 +++ /dev/null @@ -1,23 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - - -# Installs Extra Conda Packages needed for windows only. -$conda_env = conda info --json | ConvertFrom-Json - -if( $conda_env.active_prefix_name -eq "base"){ - Write-Output("Not in an active conda env. abort") - return -1 -} - -$conda_folder = $conda_env.active_prefix - -conda install -y m2w64-gcc=5.3.0 -conda install -y llvm=14.0.6 llvmdev=14.0.6 lld=14.0.6 - -# Cmake refuses to auto-detect llvm-mt, even though it detects clang-cl. -# Given they are cli-compatible, just copy it :) -Copy-Item $conda_folder\Library\bin\llvm-mt.exe $conda_folder\Library\bin\mt.exe -Copy-Item $conda_folder\Library\bin\lld-link.exe $conda_folder\Library\bin\link.exe -Copy-Item $conda_folder\Library\bin\clang-cl.exe $conda_folder\Library\bin\cl.exe diff --git a/scripts/windows/conda_setup_win_sdk.ps1 b/scripts/windows/conda_setup_win_sdk.ps1 deleted file mode 100644 index d396b4218c..0000000000 --- a/scripts/windows/conda_setup_win_sdk.ps1 +++ /dev/null @@ -1,121 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - - -$X_WIN_VERSION = "0.2.10" - -$conda_env = conda info --json | ConvertFrom-Json -$OLD_PWD = $PWD # Backup that to go back once we done. - -if( $conda_env.active_prefix_name -eq "base"){ - Write-Output("Not in an active conda env. abort") - return -1 -} -$conda_folder = $conda_env.active_prefix -Set-Location $conda_folder -Write-Output("Downloading x-win") -# Small X-Win appretiation comment. -# It's really great, -# it can process windows store manifests, request files from MS-servers and unpack it. -# we can use it to get the offical SDK files from microsoft. -# We can't cargo-install it because to use the clang-msvc compat mode, -# -Invoke-WebRequest -Uri "https://github.com/Jake-Shadle/xwin/releases/download/$X_WIN_VERSION/xwin-$X_WIN_VERSION-x86_64-pc-windows-msvc.tar.gz" -OutFile "xwin.tar.gz" -$ProgressPreference = 'Continue' -Write-Output("Unpack x-win") -tar -xf xwin.tar.gz -Copy-Item -Path "xwin-$X_WIN_VERSION-x86_64-pc-windows-msvc/xwin.exe" -Destination "$conda_folder/bin" - -Remove-Item "xwin-0.2.10-x86_64-pc-windows-msvc" -Confirm -ErrorAction SilentlyContinue -Force -Recurse -Remove-Item "xwin.tar.gz" -Confirm -ErrorAction SilentlyContinue -Force -Recurse - -$env:PATH ="$conda_folder\bin;$env:PATH" - -# Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively -Write-Output("Downloading the windows SDK") -xwin --accept-license --manifest-version 16 splat --include-debug-symbols --include-debug-libs --output xwin - -## Generate the INCLUDE env -$XWIN_PATH="`$env:CONDA_PREFIX\xwin" -$INCLUDE_ADDS = ` - "$XWIN_PATH\sdk\include;",` - "$XWIN_PATH\crt\include;",` - "$XWIN_PATH\sdk\include\cppwinrt\winrt;",` - "$XWIN_PATH\sdk\include\shared;",` - "$XWIN_PATH\sdk\include\ucrt;" ,` - "$XWIN_PATH\sdk\include\um;",` - "$XWIN_PATH\sdk\include\winrt" -$INCLUDE_TARGET ="" -ForEach-Object -InputObject $INCLUDE_ADDS { - $INCLUDE_TARGET +=($XWIN_PATH+"\"+$_) -} -$INCLUDE_TARGET= $INCLUDE_TARGET-replace("; ",";") - -# Generate & write (de/)activation scripts - - -$activate = @" -`$env:CXX="clang-cl" -`$env:CC="clang-cl" -`$env:AR="llvm-lib" -`$env:LD="lld-link" - -`$env:CMAKE_CXX_COMPILER="clang-cl" -`$env:CMAKE_C_COMPILER="clang-cl" -`$env:CMAKE_C_COMPILER="clang-cl" -`$env:CMAKE_LINKER="lld-link" -`$env:CMAKE_MT="llvm-mt" -`$env:CMAKE_GENERATOR="Ninja" - -`$env:CC_x86_64_pc_windows_msvc="clang-cl" -`$env:CXX_x86_64_pc_windows_msvc="clang-cl" -`$env:AR_x86_64_pc_windows_msvc="llvm-lib" -`$env:LD_x86_64_pc_windows_msvc="lld-link" -`$env:CL_FLAGS="-Wno-unused-command-line-argument -fuse-ld=lld-link `$env:CONDA_PREFIX\\xwin\\crt\\include `$env:CONDA_PREFIX\\xwin\\sdk\\include\\ucrt `$env:CONDA_PREFIX\\xwin\\sdk\\include\\um `$env:CONDA_PREFIX\\xwin\\sdk\\include\\shared" -`$env:RUSTFLAGS="-Clinker=`$env:CONDA_PREFIX/Library/bin/lld-link.exe -Lnative=\\xwin\\crt\\lib\\x86_64 -Lnative=`$env:CONDA_PREFIX\\xwin\\sdk\\lib\\um\\x86_64 -Lnative=`$env:CONDA_PREFIX\\xwin\\sdk\\lib\\ucrt\\x86_64" - -# Conda/go does not ship an activate.ps1 -> therefore stuff is broken on powershell. -`$env:GOROOT="`$env:CONDA_PREFIX\go" - -`$env:INCLUDE="$INCLUDE_TARGET" -`$env:LIB="$XWIN_PATH\sdk\lib\ucrt\x86_64;$XWIN_PATH\sdk\lib\um\x86_64;$XWIN_PATH\crt\lib\x86_64;" -"@ -Out-File -Encoding utf8 ` - -FilePath "$conda_folder\etc\conda\activate.d\vpn_clang_cl.ps1"` - -InputObject $activate - - -$deactivate = @" -Remove-Item Env:\OPENSSL_ROOT_DIR -Remove-Item Env:\QT_HOST_PATH -Remove-Item Env:\CMAKE_PREFIX_PATH -Remove-Item Env:\CXX -Remove-Item Env:\CC -Remove-Item Env:\AR -Remove-Item Env:\LD -Remove-Item Env:\CMAKE_CXX_COMPILER -Remove-Item Env:\CMAKE_C_COMPILER -Remove-Item Env:\CMAKE_C_COMPILER -Remove-Item Env:\CMAKE_LINKER -Remove-Item Env:\CMAKE_MT -Remove-Item Env:\CMAKE_GENERATOR -Remove-Item Env:\CC_x86_64_pc_windows_msvc -Remove-Item Env:\CXX_x86_64_pc_windows_msvc -Remove-Item Env:\AR_x86_64_pc_windows_msvc -Remove-Item Env:\LD_x86_64_pc_windows_msvc -Remove-Item Env:\CL_FLAGS -Remove-Item Env:\RUSTFLAGS -Remove-Item Env:\GOROOT -Remove-Item Env:\INCLUDE -Remove-Item Env:\LIB - -"@ -Out-File -Encoding utf8 ` - -FilePath "$conda_folder\etc\conda\deactivate.d\vpn_clang_cl.ps1"` - -InputObject $deactivate - - -Remove-Item ".xwin-cache" -Confirm -ErrorAction SilentlyContinue -Force -Recurse -Write-Output("You are SET! - Please re-activate your conda env to have stuff applied.") -Set-Location $OLD_PWD diff --git a/scripts/windows/fetch-vsix-package.py b/scripts/windows/fetch-vsix-package.py new file mode 100755 index 0000000000..106b6bbb1b --- /dev/null +++ b/scripts/windows/fetch-vsix-package.py @@ -0,0 +1,129 @@ +#! /usr/bin/env python3 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# This file is largely inspired by the Jake-Shadle/xwin project, but this +# allows us to download additional Visual Studio components that aren't +# covered by the xwin tool. + +import argparse +import hashlib +import json +import os +import requests +import shutil +import sys +import tempfile +import zipfile + +def parse_vsman_packages(js): + # Task a JSON blob and parse it into a searchable dict. + return {pkg["id"]: pkg for pkg in js["packages"]} + +def download_pkg_manifest(url): + # Download the top-level manifest. + top_req = requests.get(url) + if top_req.status_code != 200: + top_req.raise_for_status() + + # Parse it for the Visual studio package manifest. + vsid = "Microsoft.VisualStudio.Manifests.VisualStudio" + for item in top_req.json()["channelItems"]: + if item["id"] != vsid: + continue + if len(item["payloads"]) != 1: + continue + + vsman_req = requests.get(item["payloads"][0]["url"]) + if vsman_req.status_code != 200: + vsman_req.raise_for_status() + return parse_vsman_packages(vsman_req.json()) + + raise LookupError(f"No manifest found for {vsid}") + +def unpack_vsix(fileobj, output): + # Load the VSIX file, it's really just a ZIP archive. + with zipfile.ZipFile(fileobj) as zf: + # Parse the manifest to see what to unpack. + with zf.open("manifest.json", 'r') as fp: + manifest = json.load(fp) + + # Unpack the desired files from the archive + for x in manifest["files"]: + # Select only the filenames in the Contents directory. + filename = x["fileName"].lstrip('/') + prefix, path = filename.split('/', maxsplit=1) + if prefix != 'Contents': + continue + + # The destination should be stripped of the Contents dir. + dst = os.path.join(output, path) + os.makedirs(os.path.split(dst)[0], exist_ok=True) + + # Extract the file and generate its hash at the same time. + h = hashlib.sha256() + with zf.open(filename, 'r') as infile: + with open(dst, 'wb') as outfile: + while True: + data = infile.read(65536) + if len(data) == 0: + break + h.update(data) + outfile.write(data) + + # Validate the checksum + sha256 = x["sha256"].lower() + if h.hexdigest().lower() != sha256: + raise Exception(f'Invalid sha256 for {filename}') + +def download_and_extract(vsman, name, output, done): + # Skip dependencies we have already downloaded. + if name in done: + return + + if name not in vsman: + raise LookupError(f'No package found for {name}') + + # Print out what we're about to download. + pkg = vsman[name] + done.append(name) + if "localizedResources" in pkg: + print(f'Downloading: {pkg["localizedResources"][0]["title"]}') + else: + print(f'Downloading: {pkg["id"]}') + + # Download and extract the payloads. + if "payloads" in pkg: + for file in pkg["payloads"]: + # Download the VSIX file + with tempfile.TemporaryFile(suffix="-vsix.zip") as archive: + r = requests.get(file["url"], stream=True) + if r.status_code != 200: + r.raise_for_status() + shutil.copyfileobj(r.raw, archive) + archive.seek(0) + + unpack_vsix(archive, output) + + # If there are dependencies, fetch them recursively. + if "dependencies" in pkg: + for dep in pkg["dependencies"].keys(): + download_and_extract(vsman, dep, output, done) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Fetch and unpack Visual Studio Extensions') + parser.add_argument('package', metavar='PKG', type=str, nargs='*', + help='VSIX Extension to fetch') + parser.add_argument('-o', '--output', metavar='DIR', type=str, action='store', default=os.getcwd(), + help='Extract VSIX extensions to DIR') + parser.add_argument('-r', '--manifest-version', metavar='VER', type=int, action='store', default=17, + help='Fetch manifest for Visual Studio version VER') + args = parser.parse_args() + + # Fetch the top-level manifest and locate the Visual studio packages + vsman = download_pkg_manifest(f"https://aka.ms/vs/{args.manifest_version}/release/channel") + done = [] + for name in args.package: + download_and_extract(vsman, name, args.output, done) diff --git a/scripts/windows/xwin-link.bat b/scripts/windows/xwin-link.bat new file mode 100644 index 0000000000..c59ad967ee --- /dev/null +++ b/scripts/windows/xwin-link.bat @@ -0,0 +1,9 @@ +@echo off +Rem Wrapper script to combine LLVM's lld-link.exe with Xwin +Rem This is mostly used to pass the /winsysroot argument to the Cargo/Rust +Rem linker, because I can't get it to respect "-Clink-args=..." when building +Rem for MSVC. I suspect this is a bug in Cargo somewhere. +Rem +Rem LLVM also has a bug that fails to find the sysroot if the target machine +Rem is not specified. See: https://github.com/llvm/llvm-project/issues/54409 +%CONDA_PREFIX%\Library\bin\lld-link.exe "/WINSYSROOT:%CONDA_PREFIX%\xwin" /MACHINE:x64 %* diff --git a/src/cmake/sentry.cmake b/src/cmake/sentry.cmake index 6618bbb119..873f956643 100644 --- a/src/cmake/sentry.cmake +++ b/src/cmake/sentry.cmake @@ -50,8 +50,6 @@ if( ${_SUPPORTED} GREATER -1 ) # We are using breakpad as a backend - in process stackwalking is never the best option ... however! # this is super easy to link against and we do not need another binary shipped with the client. SET(SENTRY_ARGS - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} -DSENTRY_BACKEND=breakpad ) @@ -71,7 +69,10 @@ if( ${_SUPPORTED} GREATER -1 ) target_link_libraries(shared-sources INTERFACE breakpad_client.lib) target_link_libraries(shared-sources INTERFACE dbghelp.lib) target_link_libraries(shared-sources INTERFACE version.lib) - SET(SENTRY_ARGS -DSENTRY_BACKEND=breakpad -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ) + SET(SENTRY_ARGS -DSENTRY_BACKEND=breakpad) + if(DEFINED ENV{CONDA_PREFIX}) + LIST(APPEND SENTRY_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_SOURCE_DIR}/scripts/windows/conda-toolchain.cmake) + endif() endif() if(ANDROID) @@ -94,7 +95,7 @@ if( ${_SUPPORTED} GREATER -1 ) target_link_directories( shared-sources INTERFACE ${EXTERNAL_INSTALL_LOCATION}/lib64) # We are using breakpad as a backend - in process stackwalking is never the best option ... however! # this is super easy to link against and we do not need another binary shipped with the client. - SET(SENTRY_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DSENTRY_BACKEND=breakpad) + SET(SENTRY_ARGS -DSENTRY_BACKEND=breakpad) endif() @@ -103,7 +104,16 @@ if( ${_SUPPORTED} GREATER -1 ) SOURCE_DIR ${CMAKE_SOURCE_DIR}/3rdparty/sentry GIT_SUBMODULES 3rdparty/sentry GIT_SUBMODULES_RECURSE true - CMAKE_ARGS -DSENTRY_TRANSPORT=none -DSENTRY_BUILD_TESTS=OFF -DSENTRY_BUILD_EXAMPLES=OFF -DSENTRY_BUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} ${SENTRY_ARGS} + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DSENTRY_TRANSPORT=none + -DSENTRY_BUILD_TESTS=OFF + -DSENTRY_BUILD_EXAMPLES=OFF + -DSENTRY_BUILD_SHARED_LIBS=OFF + -DCMAKE_INSTALL_PREFIX=${EXTERNAL_INSTALL_LOCATION} + ${SENTRY_ARGS} ) target_include_directories(shared-sources INTERFACE ${EXTERNAL_INSTALL_LOCATION}/include) diff --git a/src/cmake/windows.cmake b/src/cmake/windows.cmake index 7c3b320c68..02509f678d 100644 --- a/src/cmake/windows.cmake +++ b/src/cmake/windows.cmake @@ -72,8 +72,6 @@ else() target_sources(mozillavpn PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ui/qt6winhack.qrc) endif() -include(${CMAKE_SOURCE_DIR}/scripts/cmake/golang.cmake) - # Use Balrog for update support. target_compile_definitions(mozillavpn PRIVATE MVPN_BALROG) target_sources(mozillavpn PRIVATE @@ -95,7 +93,16 @@ set(WINDEPLOYQT_FLAGS "--verbose 1 --no-translations --compiler-runtime --dir . install(CODE "execute_process(COMMAND \"${QT_WINDEPLOY_EXECUTABLE}\" \"$\" ${WINDEPLOYQT_FLAGS} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX})") # Use the merge module that comes with our version of Visual Studio -cmake_path(CONVERT "$ENV{VCToolsRedistDir}" TO_CMAKE_PATH_LIST VC_TOOLS_REDIST_PATH) +if(DEFINED ENV{VCToolsRedistDir}) + cmake_path(CONVERT "$ENV{VCToolsRedistDir}" TO_CMAKE_PATH_LIST VC_TOOLS_REDIST_PATH) +elseif(DEFINED XWIN_PREFIX) + set(VC_TOOLS_REDIST_PATH ${XWIN_PREFIX}/VC/Redist/MSVC/v${MSVC_TOOLSET_VERSION}) +else() + # Last-ditch effort, let's try using vshwere. + execute_process(COMMAND vswhere -property installationPath OUTPUT_VARIABLE RAW_VS_INSTALL_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) + cmake_path(CONVERT "${RAW_VS_INSTALL_PATH}" TO_CMAKE_PATH_LIST VS_INSTALL_PATH) + set(VC_TOOLS_REDIST_PATH ${VS_INSTALL_PATH}/VC/Redist/MSVC/v${MSVC_TOOLSET_VERSION}) +endif() install(FILES ${VC_TOOLS_REDIST_PATH}/MergeModules/Microsoft_VC${MSVC_TOOLSET_VERSION}_CRT_x64.msm DESTINATION . RENAME Microsoft_CRT_x64.msm) diff --git a/taskcluster/kinds/build/windows.yml b/taskcluster/kinds/build/windows.yml index 23a9ba2744..f10f483507 100644 --- a/taskcluster/kinds/build/windows.yml +++ b/taskcluster/kinds/build/windows.yml @@ -12,13 +12,16 @@ task-defaults: worker-type: b-win2022 fetches: fetch: - - win-dev-env - win-sentry-cli - win-conda - wireguard-nt toolchain: - cargo-vendor - - conda-windows-x86_64 + conda-pack: + - artifact: conda-windows.tar.gz + dest: conda-windows + dependencies: + conda-pack: toolchain-windows-conda-pack worker: taskcluster-proxy: true chain-of-trust: true diff --git a/taskcluster/kinds/toolchain/conda_windows.yml b/taskcluster/kinds/toolchain/conda_windows.yml index 5717308207..31e5b88d3b 100644 --- a/taskcluster/kinds/toolchain/conda_windows.yml +++ b/taskcluster/kinds/toolchain/conda_windows.yml @@ -10,7 +10,7 @@ task-defaults: run: resources: - requirements.txt - - env.yml + - env-windows.yml windows-conda-pack: fetches: @@ -19,8 +19,6 @@ windows-conda-pack: run: script: conda_pack_windows.ps1 resources: - - scripts/windows/conda_install_extras.ps1 - - scripts/windows/conda_setup_win_qt.ps1 - - scripts/windows/conda_setup_win_sdk.ps1 + - scripts/windows/conda-setup-xwin-sdk.ps1 toolchain-alias: conda-windows-x86_64 toolchain-artifact: public/build/conda-windows.tar.gz diff --git a/taskcluster/scripts/build/linux_build_dpkg.sh b/taskcluster/scripts/build/linux_build_dpkg.sh index 3047521850..895e0d589b 100755 --- a/taskcluster/scripts/build/linux_build_dpkg.sh +++ b/taskcluster/scripts/build/linux_build_dpkg.sh @@ -91,6 +91,10 @@ dch -c $(pwd)/mozillavpn-source/debian/changelog -v ${DPKG_PACKAGE_DIST_VERSION} if [[ "$STATICQT" == "Y" ]]; then export PATH=${MOZ_FETCHES_DIR}/qt_dist/bin:${PATH} sed -rie '/\s+(qt6-|qml6-|libqt6|qmake)/d' $(pwd)/mozillavpn-source/debian/control + # Remove the cargo dependency if manually installed + if which cargo >/dev/null 2>&1; then + sed -rie '/\s+(cargo)/d' $(pwd)/mozillavpn-source/debian/control + fi fi # Install the package build dependencies. diff --git a/taskcluster/scripts/build/windows_clang_cl.ps1 b/taskcluster/scripts/build/windows_clang_cl.ps1 index dd519f8b08..92ecac0ca1 100644 --- a/taskcluster/scripts/build/windows_clang_cl.ps1 +++ b/taskcluster/scripts/build/windows_clang_cl.ps1 @@ -17,17 +17,6 @@ Set-Location -Path $TASK_WORKDIR # doesn't handle recursive submodules, so we'll need to do it ourselves. git -C "$REPO_ROOT_PATH" submodule update --init --recursive - -# We have not yet removed our VC_Redist strategy. Therefore we rely on the old vsstudio bundle to get us that :) -# TODO: We need to handle this at some point. -$env:VCToolsRedistDir=(resolve-path "$FETCHES_PATH/VisualStudio/VC/Redist/MSVC/14.30.30704/").ToString() -# TODO: Remove this and change all to Microsoft_VC143 once we know there is no cavecat building with msvcv143 -Copy-Item -Path $env:VCToolsRedistDir\\MergeModules\\Microsoft_VC143_CRT_x64.msm -Destination $REPO_ROOT_PATH\\Microsoft_VC142_CRT_x64.msm -Copy-Item -Path $env:VCToolsRedistDir\\MergeModules\\Microsoft_VC143_CRT_x86.msm -Destination $REPO_ROOT_PATH\\Microsoft_VC142_CRT_x86.msm -Copy-Item -Path $env:VCToolsRedistDir\\MergeModules\\Microsoft_VC143_CRT_x64.msm -Destination $env:VCToolsRedistDir\\MergeModules\\Microsoft_VC142_CRT_x64.msm -Copy-Item -Path $env:VCToolsRedistDir\\MergeModules\\Microsoft_VC143_CRT_x86.msm -Destination $env:VCToolsRedistDir\\MergeModules\\Microsoft_VC142_CRT_x86.msm - - # Setup Openssl Import $SSL_PATH = "$FETCHES_PATH/QT_OUT/SSL" if (Test-Path -Path $SSL_PATH) { @@ -49,28 +38,17 @@ replace-with = "vendored-sources" "@ | Out-File -Encoding utf8 $REPO_ROOT_PATH\.cargo\config.toml } +## Install MiniConda +New-Item -ItemType Directory -Force -Path "$TASK_WORKDIR/miniconda" +$MINICONDA_DIR =resolve-path "$TASK_WORKDIR/miniconda" +Start-Process -NoNewWindow "$FETCHES_PATH\miniconda_installer.exe" -Wait -ArgumentList @('/S',"/D=$MINICONDA_DIR") +. $MINICONDA_DIR\shell\condabin\conda-hook.ps1 -## Setup the conda environment -. $REPO_ROOT_PATH/scripts/utils/call_bat.ps1 $FETCHES_PATH/Scripts/activate.bat -conda-unpack - -# Conda Pack excpets to be run under cmd. therefore it will -# (unlike conda) ignore activate.d powershell scripts. -# So let's manually run the activation scripts. -# -$CONDA_PREFIX = $env:CONDA_PREFIX - -$ACTIVATION_SCRIPTS = Get-ChildItem -Path "$CONDA_PREFIX\etc\conda\activate.d" -Filter "*.ps1" -foreach ($script in $ACTIVATION_SCRIPTS) { - Write-Output "Activating: $CONDA_PREFIX\etc\conda\activate.d\$script" - . "$CONDA_PREFIX\etc\conda\activate.d\$script" -} -# This is a wierd bug `PREFIX/bin` does not seem to be on the PATH -# when we run the activate.bat :shrugs: -# This will cause go to be missing. -$env:PATH="$CONDA_PREFIX\bin;$env:Path" -gci env:* | sort-object name - +## Unpack and activate the conda environment. +$CONDA_DIR = resolve-path "$FETCHES_PATH\conda-windows" +Start-Process -NoNewWindow "$CONDA_DIR\Scripts\conda-unpack.exe" -Wait +conda activate $CONDA_DIR +conda info # Okay We are ready to build! mkdir $TASK_WORKDIR/cmake_build @@ -79,8 +57,6 @@ $BUILD_DIR =resolve-path "$TASK_WORKDIR/cmake_build" # Do the generic build cmake -S $REPO_ROOT_PATH -B $BUILD_DIR -GNinja ` -DCMAKE_BUILD_TYPE=Release ` - -DPYTHON_EXECUTABLE="$CONDA_PREFIX\python.exe" ` - -DGOLANG_BUILD_TOOL="$CONDA_PREFIX\bin\go.exe" ` -DWIREGUARD_FOLDER="$FETCHES_PATH\wireguard-nt" ` -DCMAKE_PREFIX_PATH="$QTPATH/lib/cmake" ` -DBUILD_TESTS=OFF diff --git a/taskcluster/scripts/toolchain/conda_pack_macos.sh b/taskcluster/scripts/toolchain/conda_pack_macos.sh index f9617b2674..07b101edef 100755 --- a/taskcluster/scripts/toolchain/conda_pack_macos.sh +++ b/taskcluster/scripts/toolchain/conda_pack_macos.sh @@ -29,7 +29,7 @@ pip config debug echo "Installing provided conda env..." conda env create -f ${VCS_PATH}/env.yml conda activate VPN -${VCS_PATH}/scripts/macos/conda_install_extras.sh +${VCS_PATH}/scripts/macos/conda_install_extras.sh conda info conda deactivate diff --git a/taskcluster/scripts/toolchain/conda_pack_windows.ps1 b/taskcluster/scripts/toolchain/conda_pack_windows.ps1 index 5ae0144ef1..4924117f1f 100644 --- a/taskcluster/scripts/toolchain/conda_pack_windows.ps1 +++ b/taskcluster/scripts/toolchain/conda_pack_windows.ps1 @@ -33,17 +33,9 @@ Import-Module "$Env:_CONDA_ROOT\shell\condabin\Conda.psm1" -ArgumentList $CondaM $env:PATH ="$CONDA_DIR;$FETCHES_PATH;$QTPATH;$env:PATH" ## Conda is now ready - let's enable the env -conda env create --force -f $REPO_ROOT_PATH/env.yml -n VPN - -conda info --envs - -conda activate VPN -. "$REPO_ROOT_PATH\scripts\windows\conda_setup_win_sdk.ps1" # <- This download's all sdk things we need :3 -. "$REPO_ROOT_PATH\scripts\windows\conda_install_extras.ps1" # <- Downloads gcc. -conda deactivate -conda activate VPN - -conda install conda-pack +conda env create --force -f $REPO_ROOT_PATH/env-windows.yml -n VPN +conda run -n VPN conda info +conda run -n VPN powershell -file "$REPO_ROOT_PATH\scripts\windows\conda-setup-xwin-sdk.ps1" New-Item -ItemType Directory $TASK_WORKDIR\public\build -conda-pack --name VPN -o $TASK_WORKDIR\public\build\conda-windows.tar.gz +conda run -n VPN conda-pack --name VPN -o $TASK_WORKDIR\public\build\conda-windows.tar.gz