diff --git a/CHANGES.txt b/CHANGES.txt index ecc82a4974..df70e61aa4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -57,6 +57,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER use an faster implementation under the assumption of ordered dictionaries. From Ryan Saunders: + - Added VCPkg() builder, integrating the vcpkg cross-platform package management tool for + 3rd-party C/C++ libraries (http://vcpkg.io). A project using SCons can use the VCPkg() + builder to to download and build any package known to vcpkg. Works on Windows, Linux and MacOS. - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. From Flaviu Tamas: diff --git a/RELEASE.txt b/RELEASE.txt old mode 100644 new mode 100755 index e9f2d02a8e..6045d27f29 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -26,6 +26,9 @@ NEW FUNCTIONALITY not be called until all AddOption() calls are completed. Resolves Issue #4187 - Added --experimental=tm_v2, which enables Andrew Morrow's NewParallel Job implementation. This should scale much better for highly parallel builds. You can also enable this via SetOption(). +- Added VCPkg() builder, integrating the vcpkg cross-platform package management tool for + 3rd-party C/C++ libraries (http://vcpkg.io). A project using SCons can use the VCPkg() + builder to to download and build any package known to vcpkg. Works on Windows, Linux and MacOS. DEPRECATED FUNCTIONALITY diff --git a/SCons/Tool/VCPkgTests.py b/SCons/Tool/VCPkgTests.py new file mode 100644 index 0000000000..78f7778112 --- /dev/null +++ b/SCons/Tool/VCPkgTests.py @@ -0,0 +1,696 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Unit tests for the VCPkg() builder +""" + +import os +import re +import tempfile +from contextlib import contextmanager +from pathlib import Path +import unittest + +import TestUnit + +import SCons.Errors +import SCons.Tool.vcpkg +from SCons.Environment import Environment + +# TODO: +# * Test upgrade/downgrade of vcpkg itself +# * Test parsing of real vcpkg.exe output +# * Test feature super-setting +# * Test "static" installs + +class MockPackage: + def __init__(self, name, version, dependencies, files): + self.name = name + self.version = version + self.dependencies = dependencies + self._installedFiles = [] + self._packageFiles = files + + def get_list_file(self, env, static): + version = self.version + hash_pos = version.find('#') + if hash_pos != -1: + version = version[0:hash_pos] + return env.File(f"$VCPKGROOT/installed/vcpkg/info/{self.name}_{version}_{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}.list") + + def install(self, env, static): + assert not self._installedFiles, f"Trying to install package '{self.name}' more than once!" + listfile = self.get_list_file(env, static) + Path(listfile.get_abspath()).touch() + self._installedFiles = [listfile.get_abspath()] + + def clean_up(self): + for file in self._installedFiles: + os.remove(file) + self._installedFiles = [] + + +class MockVCPkg: + """Singleton object that replaces low-level VCPkg builder functions with mocks""" + + # + # MockVCPkg lifecycle management + # + + __instance = None + + # Singleton accessor + def getInstance(): + if MockVCPkg.__instance is None: + MockVCPkg.__instance = MockVCPkg() + MockVCPkg.__instance.acquire() + return MockVCPkg.__instance + + def __init__(self): + self._availablePackages = {} + self._installedPackages = {} + self._expectations = [] + self._useCount = 0 + + def assert_empty(self): + """Asserts that all test configuration and expectations have been removed""" + assert not self._availablePackages, f"There is/are still {len(self._availablePackages)} AvailablePackage(s)" + assert not self._installedPackages, f"There is/are still {len(self._installedPackages)} InstalledPackage(s)" + assert not self._expectations, f"There is/are still {len(self._expectations)} Expectation(s)" + + def acquire(self): + """Called to acquire a ref-count on the singleton MockVCPkg object. This is needed because multiple objects can be using the MockVCPkg object simultaneously, and it needs to tear itself down when the last user releases it""" + self._useCount += 1 + if self._useCount == 1: + # There shouldn't be anything configured yet + self.assert_empty() + + # Save original functions to restore later + self._orig_bootstrap_vcpkg = SCons.Tool.vcpkg._bootstrap_vcpkg + self._orig_call_vcpkg = SCons.Tool.vcpkg._call_vcpkg + self._orig_install_packages = SCons.Tool.vcpkg._install_packages + self._orig_upgrade_packages = SCons.Tool.vcpkg._upgrade_packages + self._origis_mismatched_version_installed = SCons.Tool.vcpkg.is_mismatched_version_installed + self._orig_get_package_version = SCons.Tool.vcpkg._get_package_version + self._orig_get_package_deps = SCons.Tool.vcpkg._get_package_deps + self._orig_read_vcpkg_file_list = SCons.Tool.vcpkg._read_vcpkg_file_list + + # Replace the low-level vcpkg functions with our mocks + SCons.Tool.vcpkg._bootstrap_vcpkg = MockVCPkg._bootstrap_vcpkg + SCons.Tool.vcpkg._call_vcpkg = MockVCPkg._call_vcpkg + SCons.Tool.vcpkg._install_packages = MockVCPkg._install_packages + SCons.Tool.vcpkg._upgrade_packages = MockVCPkg._upgrade_packages + SCons.Tool.vcpkg.is_mismatched_version_installed = MockVCPkg.is_mismatched_version_installed + SCons.Tool.vcpkg._get_package_version = MockVCPkg._get_package_version + SCons.Tool.vcpkg._get_package_deps = MockVCPkg._get_package_deps + SCons.Tool.vcpkg._read_vcpkg_file_list = MockVCPkg._read_vcpkg_file_list + + def release(self): + """Called to release a ref-count on the singleton MockVCPkg object. When this hits zero, the MockVCPkg instance will tear itself down""" + assert(self._useCount > 0) + self._useCount -= 1 + if self._useCount == 0: + # There shouldn't be any configuration still remaining + self.assert_empty() + + # Restore original functions + SCons.Tool.vcpkg._bootstrap_vcpkg = self._orig_bootstrap_vcpkg + SCons.Tool.vcpkg._call_vcpkg = self._orig_call_vcpkg + SCons.Tool.vcpkg._install_packages = self._orig_install_packages + SCons.Tool.vcpkg._upgrade_packages = self._orig_upgrade_packages + SCons.Tool.vcpkg.is_mismatched_version_installed = self._origis_mismatched_version_installed + SCons.Tool.vcpkg._get_package_version = self._orig_get_package_version + SCons.Tool.vcpkg._get_package_deps = self._orig_get_package_deps + SCons.Tool.vcpkg._read_vcpkg_file_list = self._orig_read_vcpkg_file_list + + # Finally, free the singleton + MockVCPkg.__instance = None + + + # + # State modification functions used by contextmanager functions below + # + + def addAvailablePackage(self, name, version, dependencies, files): + assert name not in self._availablePackages, f"Already have an AvailablePackage with name '{name}' (version {self._availablePackages[name].version})" + pkg = MockPackage(name, version, dependencies, files) + self._availablePackages[name] = pkg + return pkg + + def removeAvailablePackage(self, pkg): + pkg.clean_up() + assert self._availablePackages.pop(pkg.name), f"Trying to remove AvailablePackage with name '{pkg.name}' that is not currently registered" + + def addInstalledPackage(self, env, name, version, dependencies, files, static): + assert name not in self._installedPackages, f"Already have an InstalledPackage with name '{name}' (version {self._availablePackages[name].version})" + pkg = MockPackage(name, version, dependencies, files) + pkg.install(env, static) + self._installedPackages[name] = pkg + return pkg + + def removeInstalledPackage(self, pkg): + pkg.clean_up() + assert self._installedPackages.pop(pkg.name), f"Trying to remove InstalledPackage with name '{pkg.name}' that is not currently registered" + + def addExpectation(self, exp): + assert exp not in self._expectations, "Trying to add an Expectation twice?" + self._expectations.append(exp) + return exp + + def removeExpectation(self, exp): + assert exp in self._expectations, "Trying to remove Expectation that is not currently registered" + self._expectations.remove(exp) + + + # + # Mock implementations of low-level VCPkg builder functions + # + + def _bootstrap_vcpkg(env): + pass + + def _call_vcpkg(env, params, check_output = False, check = True): + assert False, "_call_vcpkg() should never be called...did we forget to hook a function?" + + def _install_packages(env, packages): + instance = MockVCPkg.__instance + for exp in instance._expectations: + exp.onInstall(env, packages) + for p in packages: + name = p.get_name() + assert name not in instance._installedPackages, f"Trying to install package with name '{name}' that is reported as already-installed" + assert name in instance._availablePackages, f"Trying to install package with name '{name}' that is not among the available packages" + instance._availablePackages[name].install(env, p.get_static()) + + def _upgrade_packages(env, packages): + instance = MockVCPkg.__instance + for exp in MockVCPkg.__instance._expectations: + exp.onUpgrade(env, packages) + for p in packages: + name = p.get_name() + assert name in instance._installedPackages, f"Trying to upgrade package with name '{name}' that is not reported as already-installed" + assert name in instance._availablePackages, f"Trying to upgrade package with name '{name}' that is not among the available packages" + instance._installedPackages[name].clean_up() + instance._availablePackages[name].install(env, p.get_static()) + + def is_mismatched_version_installed(env, spec): + name = re.sub(r':.*$', '', spec) + instance = MockVCPkg.__instance + return name in instance._installedPackages and (name not in instance._availablePackages or instance._installedPackages[name].version != instance._availablePackages[name].version) + + def _get_package_version(env, spec): + name = re.sub(r':.*$', '', spec) + pkg = MockVCPkg.__instance._availablePackages[name] + assert pkg is not None, f"_get_package_version() for not-registered package '{spec}'" + return pkg.version + + def _get_package_deps(env, spec, static): + name = re.sub(r':.*$', '', spec) + return MockVCPkg.__instance._availablePackages[name].dependencies + + def _read_vcpkg_file_list(env, list_file): + # Find the correct package. It could be in either 'available' or 'installed' + instance = MockVCPkg.__instance + package = None + static = False + for name in instance._installedPackages: + p = instance._installedPackages[name] + for s in [False, True]: + if p.get_list_file(env, s).get_abspath() == list_file.get_abspath(): + package = p + static = s + break + if package is not None: + break + if package is None: + for name in instance._availablePackages: + p = instance._availablePackages[name] + for s in [False, True]: + if p.get_list_file(env, s).get_abspath() == list_file.get_abspath(): + package = p + static = s + break + if package is not None: + break + + if package is not None: + prefix = f"$VCPKGROOT/installed/{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}/" + files = [] + for f in package._packageFiles: + files.append(env.File(prefix + f)) + return files + + assert False, f"Did not find a package matching list_file '{list_file}'" + + +class InstallExpectation: + def __init__(self, packages): + self._packageInstalled = {} + for p in packages: + self._packageInstalled[p] = False + + def onInstall(self, env, packages): + for p in packages: + name = p.get_name() + assert name in self._packageInstalled, f"Installing unexpected package '{name}'" + assert self._packageInstalled[name] is False, f"Installing package '{name}' more than once!" + self._packageInstalled[name] = True + + def onUpgrade(self, env, packages): + for p in packages: + assert p.get_name() not in self._packageInstalled, f"Expected package '{p.get_name()}' to be installed, but it was upgraded instead." + + def finalize(self): + for p in self._packageInstalled: + assert self._packageInstalled[p], f"Expected package '{p}' to be installed, but it was not." + + +class UpgradeExpectation: + def __init__(self, packages, static = False): + self._packageUpgraded = {} + for p in packages: + self._packageUpgraded[p] = False + + def onInstall(self, env, packages): + for p in packages: + assert p.get_name() not in self._packageUpgraded, f"Expected package '{p.get_name()}' to be upgraded, but it was installed instead." + + def onUpgrade(self, env, packages): + for p in packages: + name = p.get_name() + assert name in self._packageUpgraded, f"Upgrading unexpected package '{name}'" + assert self._packageUpgraded[name] is False, f"Upgrading package '{name}' more than once!" + self._packageUpgraded[name] = True + + def finalize(self): + for p in self._packageUpgraded: + assert self._packageUpgraded[p], f"Expected package '{p}' to be upgraded, but it was not." + + +class NoChangeExpectation: + def onInstall(self, env, packages): + assert False, "Expected no package changes, but this/these were installed: " + ' '.join(map(lambda p: str(p), packages)) + + def onUpgrade(self, env, packages): + assert False, "Expected no package changes, but this/these were upgraded: " + ' '.join(map(lambda p: str(p), packages)) + + def finalize(self): + pass + + +@contextmanager +def MockVCPkgUser(): + """ContextManager providing scoped usage of the MockVCPkg singleton""" + instance = MockVCPkg.getInstance() + try: + yield instance + finally: + instance.release() + + +@contextmanager +def AvailablePackage(name, version, dependencies = [], files = []): + """ContextManager temporarily adding an 'available' package to the MockVCPkg during its 'with' scope""" + with MockVCPkgUser() as vcpkg: + pkg = vcpkg.addAvailablePackage(name, version, dependencies, files) + try: + yield pkg + finally: + vcpkg.removeAvailablePackage(pkg) + + +@contextmanager +def InstalledPackage(env, name, version, dependencies = [], files = [], static = False): + """ContextManager temporarily adding an 'installed' package to the MockVCPkg during its 'with' scope""" + with MockVCPkgUser() as vcpkg: + pkg = vcpkg.addInstalledPackage(env, name, version, dependencies, files, static) + try: + yield pkg + finally: + vcpkg.removeInstalledPackage(pkg) + + +@contextmanager +def Expect(exp): + """ContextManager temporarily adding an expectation to the MockVCPkg that must be fulfilled within its 'with' scope""" + with MockVCPkgUser() as vcpkg: + exp = vcpkg.addExpectation(exp) + try: + yield exp + exp.finalize() + finally: + vcpkg.removeExpectation(exp) + + +@contextmanager +def ExpectInstall(packages): + """ContextManager adding an expectation that the specified list of packages will be installed within its 'with' scope""" + exp = InstallExpectation(packages) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectNoInstall(): + """ContextManager adding an expectation that no packages will be installed within its 'with' scope""" + exp = InstallExpectation(packages = []) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectUpgrade(packages): + """ContextManager adding an expectation that the specified list of packages will be upgraded within its 'with' scope""" + exp = UpgradeExpectation(packages) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectNoUpgrade(): + """ContextManager adding an expectation that no packages will be upgraded within its 'with' scope""" + exp = UpgradeExpectation(packages = []) + with Expect(exp): + yield exp + + +@contextmanager +def ExpectNoChange(): + """ContextManager temporarily adding an expectation that no package installation changes will occur within its 'with' scope""" + exp = NoChangeExpectation() + with Expect(exp): + yield exp + + +@contextmanager +def MakeVCPkgEnv(debug = False): + """Returns an Environment suitable for testing VCPkg""" + with tempfile.TemporaryDirectory() as vcpkg_root: + # Ensure that the .vcpkg-root sentinel file and directory structure exists + Path(f"{vcpkg_root}/.vcpkg-root").touch() + os.makedirs(f"{vcpkg_root}/installed/vcpkg/info") + + env = Environment(tools=['default', 'vcpkg']) + env['VCPKGROOT'] = vcpkg_root + if debug: + env['VCPKGDEBUG'] = True + yield env + + +def assert_package_files(env, static, actual_files, expected_subpaths): + """Verify that 'actual_files' contains exactly the list in 'expected_subpaths' (after prepending the path to the + vcpkg triplet subdirectory containing installed files)""" + prefix = env.subst(f"$VCPKGROOT/installed/{SCons.Tool.vcpkg._get_vcpkg_triplet(env, static)}/") + subpath_used = {} + for s in expected_subpaths: + subpath_used[s] = False + for f in actual_files: + path = f.get_abspath() + matched_subpath = None + for s in expected_subpaths: + if path == env.File(prefix + s).get_abspath(): + assert matched_subpath is None, f"File '{path}' matched more than one subpath ('{s}' and '{matched_subpath}')" + assert subpath_used[s] is False, f"Subpath '{s}' matched more than one file" + matched_subpath = s + subpath_used[s] = True + assert matched_subpath is not None, f"File '{path}' does not match any expected subpath" + for s in expected_subpaths: + assert subpath_used[s] is True, f"Suffix '{s}' did not match any file" + + +class VCPkgTestCase(unittest.TestCase): + def test_VCPKGROOT(self): + """Test that VCPkg() fails with an exception if the VCPKGROOT environment variable is unset or invalid""" + + env = Environment(tools=['vcpkg']) + + # VCPKGROOT unset (should fail) + exc_caught = None + try: + if 'VCPKGROOT' in env: + del env['VCPKGROOT'] + env.VCPkg('pretend_package') + except SCons.Errors.UserError as e: + exc_caught = 1 + assert "$VCPKGROOT must be set" in str(e), e + assert exc_caught, "did not catch expected UserError" + + # VCPKGROOT pointing to a bogus path (should fail) + exc_caught = None + try: + env['VCPKGROOT'] = '/usr/bin/phony/path' + env.VCPkg('pretend_package') + except SCons.Errors.UserError as e: + exc_caught = 1 + assert "$VCPKGROOT must point to" in str(e), e + assert exc_caught, "did not catch expected UserError" + + # VCPKGROOT pointing to a valid path that is not a vcpkg instance (should fail) + exc_caught = None + try: + env['VCPKGROOT'] = '#/' + env.VCPkg('pretend_package') + except SCons.Errors.UserError as e: + exc_caught = 1 + assert "$VCPKGROOT must point to" in str(e), e + assert exc_caught, "did not catch expected UserError" + + def test_install_existing_with_no_dependency(self): + """Test that the VCPkg builder installs missing packages""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3"), \ + InstalledPackage(env, "frobotz", "1.2.3"), \ + ExpectNoInstall(): + env.VCPkg("frobotz") + + def test_install_with_no_dependency(self): + """Test that the VCPkg builder installs missing packages""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3"), \ + ExpectInstall(["frobotz"]): + env.VCPkg("frobotz") + + def test_install_multiple(self): + """Test that the VCPkg builder installs multiple missing packages specified in a single VCPkg() call""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("abcd", "0.1"), \ + AvailablePackage("efgh", "1.2.3"), \ + ExpectInstall(["abcd", "efgh"]): + env.VCPkg(["abcd", "efgh"]) + + def test_duplicate_install(self): + """Test that duplicate invocations of the VCPkg builder installs a package only once""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3"), \ + ExpectInstall(["frobotz"]): + env.VCPkg("frobotz") + env.VCPkg("frobotz") + + def test_install_with_satisfied_dependency(self): + """Test that installing a package depending on an installed package does not attempt to reinstall that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy"]), \ + InstalledPackage(env, "xyzzy", "0.1"), \ + ExpectInstall(["frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_install_with_missing_dependency(self): + """Test that installing a package depending on a not-installed package also installs that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy"]), \ + ExpectInstall(["xyzzy", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_install_with_missing_dependencies(self): + """Test that installing a package depending on multiple not-installed packages also installs those packages""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("battered_lantern", "0.2"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy", "battered_lantern"]), \ + ExpectInstall(["xyzzy", "battered_lantern", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_install_with_mixed_dependencies(self): + """Test that installing a package depending on a not-installed package also installs that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("battered_lantern", "0.2"), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["xyzzy"]), \ + InstalledPackage(env, "battered_lantern", "0.2"), \ + ExpectInstall(["xyzzy", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_install_with_dependency_reached_by_multiple_paths(self): + """Test that installing a package depending on a not-installed package also installs that package""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1"), \ + AvailablePackage("glowing_sword", "0.1", dependencies = ["xyzzy"]), \ + AvailablePackage("battered_lantern", "0.2", dependencies = ["xyzzy"]), \ + AvailablePackage("frobotz", "1.2.1", dependencies = ["glowing_sword", "battered_lantern"]), \ + ExpectInstall(["xyzzy", "glowing_sword", "battered_lantern", "frobotz"]), \ + ExpectNoUpgrade(): + env.VCPkg("frobotz") + + def test_upgrade(self): + """Test that the VCPkg builder correctly upgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.1"), \ + InstalledPackage(env, "frobotz", "1.2.2"), \ + ExpectNoInstall(), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") + + def test_downgrade(self): + """Test that the VCPkg builder correctly downgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3"), \ + InstalledPackage(env, "frobotz", "1.2.2"), \ + ExpectNoInstall(), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") + + def test_upgrade_with_satisfied_dependency(self): + """Test that the VCPkg builder correctly upgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("glowing_sword", "0.5"), \ + AvailablePackage("frobotz", "1.2.2", dependencies = ["glowing_sword"]), \ + InstalledPackage(env, "glowing_sword", "0.5"), \ + InstalledPackage(env, "frobotz", "1.2.3"), \ + ExpectNoInstall(), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") + + def test_upgrade_with_missing_dependency(self): + """Test that the VCPkg builder correctly upgrades existing packages to match the vcpkg configuration""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("glowing_sword", "0.5"), \ + AvailablePackage("frobotz", "1.2.2", dependencies = ["glowing_sword"]), \ + InstalledPackage(env, "frobotz", "1.2.3"), \ + ExpectInstall(["glowing_sword"]), \ + ExpectUpgrade(["frobotz"]): + env.VCPkg("frobotz") + + def test_empty_package_spec_is_rejected(self): + """Test that the VCPkg builder rejects package names that are the empty string""" + with MakeVCPkgEnv() as env, \ + MockVCPkgUser(), \ + self.assertRaises(ValueError): + env.VCPkg('') + + def test_package_version_with_hash_suffix_is_trimmed(self): + """Ensure that package versions with #n suffix get trimmed off""" + with MakeVCPkgEnv() as env, \ + AvailablePackage("frobotz", "1.2.3#4"): + assert str(env.VCPkg("frobotz")).__contains__("1.2.3_"), "#4 suffix should've been trimmed" + + def test_enumerating_package_contents(self): + """Ensure that we can enumerate the contents of a package (and, optionally, its transitive dependency set)""" + xyzzyFiles = ['bin/xyzzy.dll', + 'bin/xyzzy.pdb', + 'debug/bin/xyzzy.dll', + 'debug/bin/xyzzy.pdb', + 'debug/lib/xyzzy.lib', + 'debug/lib/pkgconfig/xyzzy.pc', + 'include/xyzzy.h', + 'lib/xyzzy.lib', + 'lib/pkgconfig/xyzzy.pc', + 'shared/xyzzy/copyright'] + swordFiles = ['bin/sword.dll', + 'bin/sword.pdb', + 'debug/bin/sword.dll', + 'debug/bin/sword.pdb', + 'debug/lib/sword.lib', + 'debug/lib/pkgconfig/sword.pc', + 'include/sword.h', + 'lib/sword.lib', + 'lib/pkgconfig/sword.pc', + 'shared/sword/copyright'] + frobotzFiles = ['bin/frobotz.dll', + 'bin/frobotz.pdb', + 'debug/bin/frobotz.dll', + 'debug/bin/frobotz.pdb', + 'debug/lib/frobotz.lib', + 'debug/lib/pkgconfig/frobotz.pc', + 'include/frobotz.h', + 'lib/frobotz.lib', + 'lib/pkgconfig/frobotz.pc', + 'shared/frobotz/copyright'] + # By default, we should get libraries under bin/ and lib/ + with MakeVCPkgEnv() as env, \ + AvailablePackage("xyzzy", "0.1", files = xyzzyFiles), \ + AvailablePackage("glowing_sword", "0.1", files = swordFiles, dependencies = ["xyzzy"]), \ + AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy", "glowing_sword"]): + # For simplicity, override the platform-specific naming for static and shared libraries so that we don't + # have to modify the file lists to make the test pass on all platforms + env['SHLIBPREFIX'] = '' + env['LIBPREFIX'] = '' + env['SHLIBSUFFIX'] = '.dll' + env['LIBSUFFIX'] = '.lib' + for pkg in env.VCPkg("frobotz"): + assert_package_files(env, False, pkg.FilesUnderSubPath(''), frobotzFiles) + assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) + assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/sword.h', 'include/xyzzy.h']) + assert_package_files(env, False, pkg.SharedLibraries(), ['bin/frobotz.dll']) + assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['bin/frobotz.dll', 'bin/sword.dll', 'bin/xyzzy.dll']) + assert_package_files(env, False, pkg.StaticLibraries(), ['lib/frobotz.lib']) + assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['lib/frobotz.lib', 'lib/sword.lib', 'lib/xyzzy.lib']) + + # with $VCPKGDEBUG = True, we should get libraries under debug/bin/ and debug/lib/ + with MakeVCPkgEnv(debug = True) as env, \ + AvailablePackage("xyzzy", "0.1", files = xyzzyFiles), \ + AvailablePackage("glowing_sword", "0.1", files = swordFiles, dependencies = ["xyzzy"]), \ + AvailablePackage("frobotz", "1.2.3", files = frobotzFiles, dependencies = ["xyzzy", "glowing_sword"]): + # For simplicity, override the platform-specific naming for static and shared libraries so that we don't + # have to modify the file lists to make the test pass on all platforms + env['SHLIBPREFIX'] = '' + env['LIBPREFIX'] = '' + env['SHLIBSUFFIX'] = '.dll' + env['LIBSUFFIX'] = '.lib' + for pkg in env.VCPkg("frobotz"): + assert_package_files(env, False, pkg.FilesUnderSubPath(''), frobotzFiles) + assert_package_files(env, False, pkg.Headers(), ['include/frobotz.h']) + assert_package_files(env, False, pkg.Headers(transitive = True), ['include/frobotz.h', 'include/sword.h', 'include/xyzzy.h']) + assert_package_files(env, False, pkg.SharedLibraries(), ['debug/bin/frobotz.dll']) + assert_package_files(env, False, pkg.SharedLibraries(transitive = True), ['debug/bin/frobotz.dll', 'debug/bin/sword.dll', 'debug/bin/xyzzy.dll']) + assert_package_files(env, False, pkg.StaticLibraries(), ['debug/lib/frobotz.lib']) + assert_package_files(env, False, pkg.StaticLibraries(transitive = True), ['debug/lib/frobotz.lib', 'debug/lib/sword.lib', 'debug/lib/xyzzy.lib']) + + +if __name__ == "__main__": + suite = unittest.makeSuite(VCPkgTestCase, 'test_') + TestUnit.run(suite) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/__init__.py b/SCons/Tool/__init__.py index 33c1d33abf..20118eaf68 100644 --- a/SCons/Tool/__init__.py +++ b/SCons/Tool/__init__.py @@ -696,7 +696,7 @@ def tool_list(platform, env): assemblers = ['masm', 'nasm', 'gas', '386asm'] fortran_compilers = ['gfortran', 'g77', 'ifl', 'cvf', 'f95', 'f90', 'fortran'] ars = ['mslib', 'ar', 'tlib'] - other_plat_tools = ['msvs', 'midl', 'wix'] + other_plat_tools = ['msvs', 'midl', 'vcpkg', 'wix'] elif str(platform) == 'os2': "prefer IBM tools on OS/2" linkers = ['ilink', 'gnulink', ] # 'mslink'] @@ -746,6 +746,7 @@ def tool_list(platform, env): assemblers = ['as'] fortran_compilers = ['gfortran', 'f95', 'f90', 'g77'] ars = ['ar'] + other_plat_tools += ['vcpkg'] elif str(platform) == 'cygwin': "prefer GNU tools on Cygwin, except for a platform-specific linker" linkers = ['cyglink', 'mslink', 'ilink'] @@ -762,6 +763,9 @@ def tool_list(platform, env): assemblers = ['gas', 'nasm', 'masm'] fortran_compilers = ['gfortran', 'g77', 'ifort', 'ifl', 'f95', 'f90', 'f77'] ars = ['ar', ] + # VCPkg is supported on Linux; no official support for other *nix variants + if str(platform) == 'posix': + other_plat_tools += ['vcpkg'] if not str(platform) == 'win32': other_plat_tools += ['m4', 'rpm'] diff --git a/SCons/Tool/default.xml b/SCons/Tool/default.xml index 15dc2f7fea..d0d0124425 100644 --- a/SCons/Tool/default.xml +++ b/SCons/Tool/default.xml @@ -55,7 +55,7 @@ are selected if their respective conditions are met: &t-link-jar;, &t-link-javac;, &t-link-javah;, &t-link-rmic;, &t-link-dvipdf;, &t-link-dvips;, &t-link-gs;, &t-link-tex;, &t-link-latex;, &t-link-pdflatex;, &t-link-pdftex;, -&t-link-tar;, &t-link-zip;, &t-link-textfile;. +&t-link-tar;, &t-link-zip;, &t-link-textfile; &t-link-vcpkg;. diff --git a/SCons/Tool/docbook/__init__.py b/SCons/Tool/docbook/__init__.py index 5cf5e61980..0b2ecdd7a3 100644 --- a/SCons/Tool/docbook/__init__.py +++ b/SCons/Tool/docbook/__init__.py @@ -69,7 +69,7 @@ # lxml etree XSLT global max traversal depth # -lmxl_xslt_global_max_depth = 3100 +lmxl_xslt_global_max_depth = 4000 if has_lxml and lmxl_xslt_global_max_depth: def __lxml_xslt_set_global_max_depth(max_depth): diff --git a/SCons/Tool/vcpkg.py b/SCons/Tool/vcpkg.py new file mode 100644 index 0000000000..c8f71d2ef5 --- /dev/null +++ b/SCons/Tool/vcpkg.py @@ -0,0 +1,658 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Tool-specific initialization for vcpkg. + +There normally shouldn't be any need to import this module directly. +It will usually be imported through the generic SCons.Tool.Tool() +selection method. + + +TODO: + * find a way to push package build into SCons's normal build phase + * ensure Linux works + * handle complex feature supersetting scenarios + * parallel builds? + * can we ensure granular detection, and fail on undetected dependencies? + * batch depend-info calls to vcpkg for better perf? + * Make "vcpkg search" faster by supporting a strict match option + * Is there a way to make vcpkg build only dgb/rel libs? + +""" + +# +# Copyright (c) 2001 - 2019 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import re +import subprocess +import SCons.Builder +import SCons.Node.Python +import SCons.Script +from SCons.Errors import UserError, InternalError, BuildError + + +# Named constants for verbosity levels supported by _vcpkg_print +Silent = 0 +Normal = 1 +Debug = 2 + +_max_verbosity = Normal # Can be changed with --silent or --vcpkg-debug + +# Add --vcpkg-debug command-line option +SCons.Script.AddOption('--vcpkg-debug', dest = 'vcpkg-debug', default = False, action = 'store_true', + help = 'Emit verbose debugging spew from the vcpkg builder') + +def _vcpkg_print(verbosity, *args): + """If the user wants to see messages of 'verbosity', prints *args with a 'vcpkg' prefix""" + + if verbosity <= _max_verbosity: + print('vcpkg: ', end='') + print(*args) + + +def _get_built_vcpkg_full_version(vcpkg_exe): + """Runs 'vcpkg version' and parses the version string from the output""" + + if not vcpkg_exe.exists(): + raise InternalError(vcpkg_exe.get_path() + ' does not exist') + + line = subprocess.check_output([vcpkg_exe.get_abspath(), 'version'], text = True) + match_version = re.search(r' version (\S+)', line) + if (match_version): + version = match_version.group(1) + _vcpkg_print(Debug, 'vcpkg full version is "' + version + '"') + return version + raise InternalError(vcpkg_exe.get_path() + ': failed to parse version string') + + +def _get_built_vcpkg_base_version(vcpkg_exe): + """Returns just the base version from 'vcpkg version' (i.e., with any '-whatever' suffix stripped off""" + full_version = _get_built_vcpkg_full_version(vcpkg_exe) + + # Allow either 2022-02-02 or 2022.02.02 date syntax + match = re.match(r'^(\d{4}[.-]\d{2}[.-]\d{2})', full_version) + if match: + base_version = match.group(1) + _vcpkg_print(Debug, 'vcpkg base version is "' + base_version + '"') + return base_version + + _vcpkg_print(Debug, 'vcpkg base version is identical to full version "' + full_version + '"') + return full_version + + +def _get_source_vcpkg_version(vcpkg_root): + """Parses the vcpkg source code version out of VERSION.txt""" + + if not vcpkg_root.exists(): + raise InternalError(vcpkg_root.get_path() + ' does not exist') + + # Older vcpkg versions had the source version in VERSION.txt + version_txt = vcpkg_root.File('toolsrc/VERSION.txt') + if version_txt.exists(): + _vcpkg_print(Debug, "Looking for source version in " + version_txt.get_path()) + for line in open(version_txt.get_abspath()): + match_version = re.match(r'^"(.*)"$', line) + if (match_version): + version = match_version.group(1) + + # Newer versions of vcpkg have a hard-coded invalid date in the VERSION.txt for local builds, + # and the actual version comes from the bootstrap.ps1 script + if version != '9999.99.99': + _vcpkg_print(Debug, 'Found valid vcpkg source version "' + version + '" in VERSION.txt') + return version + else: + _vcpkg_print(Debug, 'VERSION.txt contains invalid vcpkg source version "' + version + '"') + break + + # Newer versions of bootstrap-vcpkg simply download a pre-built executable from GitHub, and the version to download + # is hard-coded in a deployment script. + bootstrap_ps1 = vcpkg_root.File('scripts/bootstrap.ps1') + if bootstrap_ps1.exists(): + for line in open(bootstrap_ps1.get_abspath()): + match_version = re.match(r"\$versionDate\s*=\s*'(.*)'", line) + if match_version: + version = match_version.group(1) + _vcpkg_print(Debug, 'Found valid vcpkg source version "' + version + '" in bootstrap.ps1') + return version + + raise InternalError("Failed to determine vcpkg source version") + + +def _bootstrap_vcpkg(env): + """Ensures that VCPKGROOT is set, and that the vcpkg executable has been built.""" + + # If we've already done these checks, return the previous result + if '_VCPKG_EXE' in env: + return env['_VCPKG_EXE'] + + if 'VCPKGROOT' not in env: + raise UserError('$VCPKGROOT must be set in order to use the VCPkg builder') + + vcpkgroot_dir = env.Dir(env.subst('$VCPKGROOT')) + sentinel_file = vcpkgroot_dir.File('.vcpkg-root') + if not sentinel_file.exists(): + raise UserError(sentinel_file.get_path() + ' does not exist...$VCPKGROOT must point to the root directory of a VCPkg repository containing this file') + + if env['PLATFORM'] == 'win32': + vcpkg_exe = vcpkgroot_dir.File('vcpkg.exe') + bootstrap_vcpkg_script = vcpkgroot_dir.File('bootstrap-vcpkg.bat') + elif env['PLATFORM'] == 'darwin' or env['PLATFORM'] == 'posix': + vcpkg_exe = vcpkgroot_dir.File('vcpkg') + bootstrap_vcpkg_script = vcpkgroot_dir.File('bootstrap-vcpkg.sh') + else: + raise UserError('This architecture/platform (%s/%s) is currently unsupported with VCPkg' % (env['TARGET_ARCH'], env['PLATFORM'])) + + # We need to build vcpkg.exe if it doesn't exist, or if it has a different "base" version than the source code + build_vcpkg = not vcpkg_exe.exists() + if not build_vcpkg: + built_version = _get_built_vcpkg_base_version(vcpkg_exe) + source_version = _get_source_vcpkg_version(vcpkgroot_dir) + if built_version != source_version and not built_version.startswith(source_version + '-'): + _vcpkg_print(Normal, 'vcpkg executable (version ' + built_version + ') is out of date (source version: ' + source_version + '); rebuilding') + build_vcpkg = True + else: + _vcpkg_print(Debug, 'vcpkg executable (version ' + built_version + ') is up-to-date') + + # If we need to build, do it now, and ensure that it built + if build_vcpkg: + if not bootstrap_vcpkg_script.exists(): + raise InternalError(bootstrap_vcpkg_script.get_path() + ' does not exist...what gives?') + _vcpkg_print(Normal, 'Building vcpkg binary') + if subprocess.call(bootstrap_vcpkg_script.get_abspath()) != 0: + raise BuildError(bootstrap_vcpkg_script.get_path() + ' failed') + vcpkg_exe.clear_memoized_values() + if not vcpkg_exe.exists(): + raise BuildError(bootstrap_vcpkg_script.get_path() + ' failed to create ' + vcpkg_exe.get_path()) + + # Remember this, so we don't run these checks again + env['_VCPKG_EXE'] = vcpkg_exe + return vcpkg_exe + + +def _call_vcpkg(env, params, check_output = False, check = True): + """Run the vcpkg executable wth the given set of parameters, optionally returning its standard output as a string. If the vcpkg executable is not yet built, or out of date, it will be rebuilt.""" + + vcpkg_exe = _bootstrap_vcpkg(env) + command_line = [vcpkg_exe.get_abspath()] + params + _vcpkg_print(Debug, "Running " + str(command_line)) + try: + result = subprocess.run(command_line, text = True, capture_output = check_output or _max_verbosity == Silent, check = check) + if check_output: + _vcpkg_print(Debug, result.stdout) + return result.stdout + else: + return result.returncode + except subprocess.CalledProcessError as ex: + if check_output: + _vcpkg_print(Silent, ex.stdout) + _vcpkg_print(Silent, ex.stderr) + return ex.stdout + else: + return ex.returncode + + +def _install_packages(env, packages): + packages_args = list(map(lambda p: str(p), packages)) + _vcpkg_print(Silent, ' '.join(packages_args) + ' (install)') + result = _call_vcpkg(env, ['install'] + packages_args) + if result != 0: + _vcpkg_print(Silent, "Failed to install package(s) " + ' '.join(packages_args)) + return result + + +def _upgrade_packages(env, packages): + packages_args = list(map(lambda p: str(p), packages)) + _vcpkg_print(Silent, ' '.join(packages_args) + ' (upgrade)') + result = _call_vcpkg(env, ['upgrade', '--no-dry-run'] + packages_args) + if result != 0: + _vcpkg_print(Silent, "Failed to upgrade package(s) " + ' '.join(packages_args)) + return result + + +def _get_vcpkg_triplet(env, static): + """Computes the appropriate VCPkg 'triplet' for the current build environment""" + + platform = env['PLATFORM'] + + # TODO: this relies on having a C++ compiler tool enabled. Is there a better way to compute this? + if 'TARGET_ARCH' in env and env['TARGET_ARCH'] is not None: + arch = env['TARGET_ARCH'] + else: + arch = env['HOST_ARCH'] + + if platform == 'win32': + if arch == 'x86' or arch == 'i386': + return 'x86-windows-static' if static else 'x86-windows' + elif arch == 'x86_64' or arch == 'x64' or arch == 'amd64': + return 'x64-windows-static' if static else 'x64-windows' + elif arch == 'arm': + return 'arm-windows' + elif arch == 'arm64': + return 'arm64-windows' + elif platform == 'darwin': + if arch == 'x86_64': + return 'x64-osx' + elif platform == 'posix': + if arch == 'x86_64': + return 'x64-linux' + + raise UserError('This architecture/platform (%s/%s) is currently unsupported with VCPkg' % (arch, platform)) + + +def _read_vcpkg_file_list(env, list_file): + """Read a .list file for a built package and return a list of File nodes for the files it contains (ignoring directories)""" + + files = [] + for line in open(list_file.get_abspath()): + if not line.rstrip().endswith('/'): + files.append(env.File('$VCPKGROOT/installed/' + line)) + return files + + +def is_mismatched_version_installed(env, spec): + _vcpkg_print(Debug, 'Checking for mismatched version of "' + spec + '"') + output = _call_vcpkg(env, ['update'], check_output = True) + for line in output.split('\n'): + match = re.match(r'^\s*(\S+)\s*(\S+) -> (\S+)', line) + if match and match.group(1) == spec: + _vcpkg_print(Debug, 'Package "' + spec + '" can be updated (' + match.group(2) + ' -> ' + match.group(3) + ')') + return True + return False + + +def _get_package_version(env, spec): + """Read the available version of a package (i.e., what would be installed)""" + + name = spec.split('[')[0] + output = _call_vcpkg(env, ['search', name], check_output = True) + for line in output.split('\n'): + match = re.match(r'^(\S+)\s*(\S+)', line) + if match and match.group(1) == name: + version = match.group(2) + _vcpkg_print(Debug, 'Available version of package "' + name + '" is ' + version) + return version + raise UserError('No package "' + name + '" found via vcpkg search') + + +def _get_package_deps(env, spec, static): + """Call 'vcpkg depend-info' to query for the dependencies of a package""" + + # TODO: compute these from the vcpkg base version + _vcpkg_supports_triplet_param = True + _vcpkg_supports_max_recurse_param = True + _vcpkg_supports_no_recurse_param = False + params = ['depend-info'] + + # Try to filter to only first-level dependencies + if _vcpkg_supports_max_recurse_param: + params.append('--max-recurse=0') + elif _vcpkg_supports_no_recurse_param: + params.append('--no-recurse') + + if _vcpkg_supports_triplet_param: + # Append the package spec + triplet + params += ['--triplet', _get_vcpkg_triplet(env, static), spec] + else: + # OK, VCPkg doesn't know about the --triplet param, which means that it also doesn't understnd package specs + # containing feature specifications. So, we'll strip these out (but possibly miss some dependencies). + params.append(spec.split('[')[0]) + + name = spec.split('[')[0] + output = _call_vcpkg(env, params, check_output = True) + for line in output.split('\n'): + match = re.match(r'^([^:[]+)(?:\[[^]]+\])?:\s*(.+)?', line) + if match and match.group(1) == name: + deps_list = [] + if match.group(2): + deps_list = list(filter(lambda s: s != "", map(lambda s: s.strip(), match.group(2).split(',')))) + _vcpkg_print(Debug, 'Package ' + spec + ' has dependencies [' + ','.join(deps_list) + ']') + return deps_list + raise InternalError('Failed to parse output from vcpkg ' + ' '.join(params) + '\n' + output) + + +def get_package_descriptors_map(env): + """Returns an Environment-global mapping of previously-seen package name -> PackageDescriptor, ensuring a 1:1 + correspondence between a package and its PackageDescriptor. This global mapping is needed because a package + may need to be built due to being a dependency of a user-requested package, and a given package may be a + dependency of multiple user-requested packages, or may be a dependency of a user-requested package via multiple + paths in the package dependency graph.""" + if not hasattr(env, '_vcpkg_package_descriptors_map'): + env._vcpkg_package_descriptors_map = {} + return env._vcpkg_package_descriptors_map + +def get_package_targets_map(env): + """Returns an Environment-global mapping of previously-seen package name -> PackageContents. This global mapping + is needed because the contents-enumeration methods of PackageContents need to be able to access the contents + of their dependencies when transitive = True is specified.""" + if not hasattr(env, '_vcpkg_package_targets_map'): + env._vcpkg_package_targets_map = {} + return env._vcpkg_package_targets_map + +class PackageDescriptor(SCons.Node.Python.Value): + """PackageDescriptor is the 'source' node for the VCPkg builder. A PackageDescriptor instance includes the package + name, version and linkage (static or dynamic), and a list of other packages that it depends on.""" + + def __init__(self, env, spec, static = False): + _bootstrap_vcpkg(env) + + triplet = _get_vcpkg_triplet(env, static) + env.AppendUnique(CPPPATH = ['$VCPKGROOT/installed/' + triplet + '/include/']) + if env.subst('$VCPKGDEBUG') == 'True': + env.AppendUnique(LIBPATH = ['$VCPKGROOT/installed/' + triplet + '/debug/lib/']) + else: + env.AppendUnique(LIBPATH = ['$VCPKGROOT/installed/' + triplet + '/lib/']) + + if spec is None or spec == '': + raise ValueError('VCPkg: Package spec must not be empty') + + matches = re.match(r'^([^[]+)(?:\[([^[]+)\])?$', spec) + if not matches: + raise ValueError('VCPkg: Malformed package spec "' + spec + '"') + + name = matches[1] + features = matches[2] + version = _get_package_version(env, name) + depends = _get_package_deps(env, spec, static) + value = { + 'name': name, + 'features': features, + 'static': static, + 'version': version, + 'triplet': triplet + } + + super(PackageDescriptor, self).__init__(value) + self.env = env + self.package_deps = list(map(lambda p: get_package_descriptor(env, p), depends)) + + def get_name(self): + return self.value['name'] + + def get_static(self): + return self.value['static'] + + def get_triplet(self): + return self.value['triplet'] + + def get_package_string(self): + s = self.value['name'] + if self.value['features'] is not None: + s += '[' + self.value['features'] + ']' + s += ':' + self.value['triplet'] + return s + + def get_listfile_basename(self): + # Trim off any suffix like '#3' from the version, as this doesn't appear in the listfile name + version = self.value['version'] + hash_pos = version.find('#') + if hash_pos != -1: + version = version[0:hash_pos] + return self.value['name'] + '_' + version + '_' + self.value['triplet'] + + def __str__(self): + return self.get_package_string() + + def target_from_source(self, pre, suf, splitext): + _bootstrap_vcpkg(self.env) + + target = PackageContents(self.env, self) + target.state = SCons.Node.up_to_date + + package_targets_map = get_package_targets_map(self.env) + + for pkg in self.package_deps: + if pkg in package_targets_map: + _vcpkg_print(Debug, 'Reused dep: ' + str(package_targets_map[pkg])) + self.env.Depends(target, package_targets_map[pkg]) + else: + _vcpkg_print(Debug, "New dep: " + str(pkg)) + dep = self.env.VCPkg(pkg) + self.env.Depends(target, dep[0]) + + if not SCons.Script.GetOption('help'): + if not target.exists(): + if is_mismatched_version_installed(self.env, str(self)): + if _upgrade_packages(self.env, [self]) != 0: + target.state = SCons.Node.failed + else: + if _install_packages(self.env, [self]) != 0: + target.state = SCons.Node.failed + target.clear_memoized_values() + if not target.exists(): + _vcpkg_print(Silent, "What gives? vcpkg install failed to create '" + target.get_abspath() + "'") + target.state = SCons.Node.failed + + target.precious = True + target.noclean = True + + _vcpkg_print(Debug, "Caching target for package: " + self.value['name']) + package_targets_map[self] = target + + return target + + +class PackageContents(SCons.Node.FS.File): + """PackageContents is a File node (referring to the installed package's .list file) and is the 'target' node of + the VCPkg builder (though currently, it doesn't actually get built during SCons's normal build phase, since + vcpkg currently can't tell us what files will be installed without actually doing the work). + + It includes functionality for enumerating the different kinds of files (headers, libraries, etc.) produced by + installing the package.""" + + def __init__(self, env, descriptor): + super().__init__(descriptor.get_listfile_basename() + ".list", env.Dir('$VCPKGROOT/installed/vcpkg/info/'), env.fs) + self.descriptor = descriptor + self.loaded = False + + def Headers(self, transitive = False): + """Returns the list of C/C++ header files belonging to the package. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + return self.FilesUnderSubPath('include/', transitive) + + def StaticLibraries(self, transitive = False): + """Returns the list of static libraries belonging to the package. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + if self.env.subst('$VCPKGDEBUG') == 'True': + return self.FilesUnderSubPath('debug/lib/', transitive, self.env['LIBSUFFIX']) + else: + return self.FilesUnderSubPath('lib/', transitive, self.env['LIBSUFFIX']) + + def SharedLibraries(self, transitive = False): + """Returns the list of shared libraries belonging to the package. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included.""" + if self.env.subst('$VCPKGDEBUG') == 'True': + return self.FilesUnderSubPath('debug/bin/', transitive, self.env['SHLIBSUFFIX']) + else: + return self.FilesUnderSubPath('bin/', transitive, self.env['SHLIBSUFFIX']) + + def FilesUnderSubPath(self, subpath, transitive = False, suffix_filters = None, packages_visited = None): + """Returns a (possibly empty) list of File nodes belonging to this package that are located under the + relative path `subpath` underneath the triplet install directory. + If `transitive` is True, then files belonging to upstream dependencies of this package are also included. + If 'suffix_filters is not None, but instead a string or a list, then only files ending in the substring(s) + listed therein will be included.""" + + # Load the listfile contents, if we haven't already. This returns a list of File nodes. + if not self.loaded: + if not self.exists(): + raise InternalError(self.get_path() + ' does not exist') + _vcpkg_print(Debug, 'Loading ' + str(self.descriptor) + ' listfile: ' + self.get_path()) + self.files = _read_vcpkg_file_list(self.env, self) + self.loaded = True + + # Compute the appropriate filter check based on 'suffix_filters' + if suffix_filters is None: + matches_filters = lambda f: True + elif type(suffix_filters) is str: + matches_filters = lambda f: f.endswith(suffix_filters) + elif type(suffix_filters) is list: + matches_filters = lambda f: len(filter(lambda s: f.endswith(s), suffix_filters)) > 0 + + prefix = self.env.Dir(self.env.subst('$VCPKGROOT/installed/' + self.descriptor.get_triplet() + "/" + subpath)) + _vcpkg_print(Debug, 'Looking for files under ' + str(prefix)) + matching_files = [] + for file in self.files: + if file.is_under(prefix) and matches_filters(file.get_abspath()): + _vcpkg_print(Debug, 'Matching file ' + file.get_abspath()) + matching_files += [file] + + # If the caller requested us to also recurse into dependencies, do that now. However, don't visit the same + # package more than once (i.e., if it's reachable via multiple paths in the dependency graph) + if transitive: + if packages_visited is None: + packages_visited = [] + package_targets_map = get_package_targets_map(self.env) + for dep in self.descriptor.package_deps: + dep_pkg = package_targets_map[dep] + if dep_pkg not in packages_visited: + matching_files += dep_pkg.FilesUnderSubPath(subpath, transitive, suffix_filters, packages_visited) + packages_visited.append(self) + + return SCons.Util.NodeList(matching_files) + + def __str__(self): + return "Package: " + super(PackageContents, self).__str__() + + +def get_package_descriptor(env, spec): + package_descriptors_map = get_package_descriptors_map(env) + if spec in package_descriptors_map: + return package_descriptors_map[spec] + desc = PackageDescriptor(env, spec) + package_descriptors_map[spec] = desc + return desc + + +# TODO: at the moment, we can't execute vcpkg install at the "normal" point in time, because we need to know what +# files are produced by running this, and we can't do that without actually running the command. Thus, we have to +# shoe-horn the building of packages into the target_from_source function. If vcpkg supported some kind of "outputs" +# mode where it could spit out the contents of the .list file without actually doing the build, then we could defer +# the build until vcpkg_action. +def vcpkg_action(target, source, env): + pass +# packages = list(map(str, source)) +# return _call_vcpkg(env, ['install'] + packages) + + +def get_vcpkg_deps(node, env, path, arg): + deps = [] + if node.package_deps is not None: + for pkg in node.package_deps: + target = env.VCPkg(pkg) + deps += target[0] + _vcpkg_print(Debug, 'Found dependency: "' + str(node) + '" -> "' + str(target[0])) + return deps + + +vcpkg_source_scanner = SCons.Script.Scanner(function = get_vcpkg_deps, + argument = None) + + +# TODO: do we need the emitter at all? +def vcpkg_emitter(target, source, env): + _bootstrap_vcpkg(env) + + for t in target: + if not t.exists(): + vcpkg_action(target, source, env) + break + + built = [] +# for t in target: +# built += _read_vcpkg_file_list(env, t) + + for f in built: + f.precious = True + f.noclean = True + f.state = SCons.Node.up_to_date + + target += built + + return target, source + + +class VCPkgBuilder(SCons.Builder.BuilderBase): + def __init__(self, env): + # single_source = True shouldn't be required, as VCPkgBuilder is capable of handling lists of inputs. + # However, there are appears to be a bug in how Builder._createNodes processes lists of nodes, and + # the result is that VCPkgBuilder only gets the first item in the list. + super().__init__(action = SCons.Action.Action(vcpkg_action), + source_factory = lambda spec: get_package_descriptor(env, spec), + target_factory = lambda desc: PackageContents(env, desc), + source_scanner = vcpkg_source_scanner, + single_source = True, + suffix = '.list', + emitter = vcpkg_emitter) + + # Need to override operator ==, since a VCPkgBuilder is bound to a specific instance of Environment, but + # Environment unit tests expect to be able to compare the BUILDERS dictionaries between two Environments. + def __eq__(self, other): + return type(self) == type(other) + + +# TODO: static? +def generate(env): + """Add Builders and construction variables for vcpkg to an Environment.""" + + # Set verbosity to the appropriate level + global _max_verbosity + if SCons.Script.GetOption('vcpkg-debug'): + _max_verbosity = Debug + elif SCons.Script.GetOption('silent'): + _max_verbosity = Silent + else: + _max_verbosity = Normal + + env['BUILDERS']['VCPkg'] = VCPkgBuilder(env) + +def exists(env): + return 1 + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/vcpkg.xml b/SCons/Tool/vcpkg.xml new file mode 100644 index 0000000000..3e11c7a301 --- /dev/null +++ b/SCons/Tool/vcpkg.xml @@ -0,0 +1,152 @@ + + + + + %scons; + + %builders-mod; + + %functions-mod; + + %tools-mod; + + %variables-mod; + ]> + + + + + + + Integrates the vcpkg package manager into SCons. + If the is given on the command line, lots of extra information will be + emitted, indicating what the vcpkg tool is doing and why. + + + + + + + + Downloads and builds one or more software packages (plus any dependencies of those packages) via the + vcpkg package manager, making the built artifacts available + for other builders. + + + +# Download and build FreeImage, plus all of its dependencies +env.VCPkg('freeimage') + + + + vcpkg is distributed as a Git repository, containing the vcpkg executable (or a script to build it) + and a "snapshot" of the current versions of all available packages. A typical usage pattern is for your + project to incorporate vcpkg as a Git submodule underneath your project (run 'git submodule --help'), + though system-wide installation is also supported. + + + + Packages built with vcpkg may produce header files, static libraries and shared libraries + (.dll/.so files), organized in directories underneath + &cv-link-VCPKGROOT;. The VCPkg builder makes these artifacts available to the SCons build as + straightforwardly as possible: + + + The directory containing header files is automatically added to &cv-link-CPPPATH;. + + + The directory containing static libraries is automatically added to &cv-link-LIBPATH;. + + + + The object returned by invoking the VCPkg builder provides methods for enumerating the + files produced by the package (and optionally, any of its upstream dependencies), allowing + your SConstruct file to do arbitrary, further processing on them. + + + + + + + Since the operating system needs to be able to find any shared libraries that your program depends on, + you will need to ensure that these libraries end up somewhere in the library search path. One way to do + this is to manually add the path to where these shared libraries to the search path, but this has the + downsides of being "manual", and also of potentially breaking if vcpkg ever alters its directory + structure. A better way is to enumerate the shared libraries and &t-install; them into the same + directory as your program builds into: + + + +# Ensure that the 'freeimage' and 'fftw3' packages are built, and then install all shared libraries produced by them +# or any of their dependencies into myVariantDir +for pkg in env.VCPkg(['freeimage', 'fftw3']): + env.Install(myVariantDir, pkg.SharedLibraries(transitive = True)) + + # Of course, packages contain more than shared libraries. While a typical project likely won't need to do this, + # you can enumerate other artifacts from the package by calling other functions: + print("Header files: " + ' '.join(pkg.Headers())) + print("Static libs (incl. dependencies): " + ' '.join(pkg.StaticLibraries(transitive = True))) + print("Everything: " + ' '.join(pkg.FilesUnderSubPath(''))) + print(".txt files under share: " + ' '.join(pkg.FilesUnderSubPath('share/', suffix_filters = '.txt') + + + + An additional benefit of this approach is that it works better with multiple variants: let's say + that you have "debug" and "release" build configurations, building into different variant directories. + By installing the shared libraries into these directories, you can use the corresponding "debug" and + "release" builds of the VCPkg-built libraries without conflict. + + + + Note that the return value from invoking the VCPkg builder is always a list, even if you only specify + a single package to build, as SCons always returns a list from invoking a Builder. Thus, the "for" + loop iterating over the packages is still necessary, even in the single-package case. + + + + + + VCPKGROOT + VCPKGDEBUG + + + + + + + Specifies the path to the root directory of the vcpkg installation. This must be set in the + SConstruct/SConscript file, and must point to an existing vcpkg installation. Often, this directory + will be a Git sub-module of your project, in which case VCPKGROOT will be specified relative to the + project root. + + + +# vcpkg is a submodule located in the 'vcpkg' directory underneath the project root +env['VCPKGROOT'] = '#/vcpkg' + + + + + + + + Specifies whether vcpkg should build debug or optimized versions of packages. If True, then "debug" + packages will be built and used, with full debugging information and most optimizations disabled. If + False (or unset), then packages will be built using optimized settings. + + + Note that, while you may choose to set this to match the optimization settings of your project's build, + this is not required: it's perfectly fine to use optimized packages with a "debug" build of your project. + + + + + diff --git a/bin/files b/bin/files index 08b1caa3bd..0207edc6c9 100644 --- a/bin/files +++ b/bin/files @@ -99,6 +99,7 @@ ./SCons/Tool/tar.py ./SCons/Tool/tex.py ./SCons/Tool/tlib.py +./SCons/Tool/vcpkg.py ./SCons/Tool/yacc.py ./SCons/Util.py ./SCons/Warnings.py diff --git a/doc/generated/tools.mod b/doc/generated/tools.mod index 35eea5e2b4..22143cf148 100644 --- a/doc/generated/tools.mod +++ b/doc/generated/tools.mod @@ -104,6 +104,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. tex"> textfile"> tlib"> +vcpkg"> xgettext"> yacc"> zip"> @@ -210,6 +211,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. tex"> textfile"> tlib"> +vcpkg"> xgettext"> yacc"> zip">