From 8adb93a01503f7b80f32a0ddde838d657e161d6b Mon Sep 17 00:00:00 2001 From: dmn-star <52277783+dmn-star@users.noreply.github.com> Date: Sun, 25 Apr 2021 20:08:47 +0200 Subject: [PATCH] (#4286) [msys2] Bump version and support two profiles approach. * Bump version and support two profiles approach. https://docs.conan.io/en/latest/reference/profiles.html#build-profiles-and-host-profiles * PR 4286: invalid syntax * PR 4286: add new version (20210105) * PR 4286: CR avoid any check which are related to the running machine In general, we should avoid any check using platform, os_info,... which are related to the running machine, not to the package being generated. * PR 4286: try to move the msys2 download into source(.) https://docs.conan.io/en/latest/reference/conanfile/methods.html#source Please read the Note * PR 4286: 7zip -> tools.unzip(.) * PR 4286: install the new keyring # https://www.msys2.org/news/#2020-06-29-new-packagers * PR 4286: skip msys2 v.20161025 * PR 4286: fix invalid syntax * PR 4286: try to kill pacman first * PR 4286: try to kill pacman first on windows only * Revert "PR 4286: try to kill pacman first on windows only" This reverts commit e774845fcc7a138cafdc9d1ad3c64253c1c4f5b5. * Revert "PR 4286: try to kill pacman first" This reverts commit 0048efaae6ea2954756ceaca98273ba3aec3a77d. * PR 4286: pac-man workarounds * PR 4286: Always kill all running msys2 processes * PR 4286: remove configure * PR 4286: more kills * PR 4286: add --ask 20 * PR 4286: less kills * PR 4286: skip hook warning * - more logs * - even more logs Signed-off-by: SSE4 * - wait for the process Signed-off-by: SSE4 * - it's possible we have several CI jobs which may incorrectly terminate each other with kill pacman Signed-off-by: SSE4 * - guard to pass export on Linux Signed-off-by: SSE4 * - typo * - ValueError * - syntax * - don't raise * - move downloads to build - drop version 20161025 - use new conandata.yml format - avoid unnecessary copying Co-authored-by: SSE4 --- recipes/msys2/all/conandata.yml | 62 +++--- recipes/msys2/all/conanfile.py | 214 +++++++++++++++++--- recipes/msys2/all/test_package/conanfile.py | 5 +- recipes/msys2/config.yml | 5 +- 4 files changed, 224 insertions(+), 62 deletions(-) diff --git a/recipes/msys2/all/conandata.yml b/recipes/msys2/all/conandata.yml index 974510895df7a..0faed3525af92 100644 --- a/recipes/msys2/all/conandata.yml +++ b/recipes/msys2/all/conandata.yml @@ -1,34 +1,34 @@ sources: - "20161025": - - url: [ - "http://repo.msys2.org/distrib/i686/msys2-base-i686-20161025.tar.xz", - "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20161025.tar.xz", - ] - sha256: "8bafd3d52f5a51528a8671c1cae5591b36086d6ea5b1e76e17e390965cf6768f" - - url: [ - "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz", - "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20161025.tar.xz" - ] - sha256: "bb1f1a0b35b3d96bf9c15092da8ce969a84a134f7b08811292fbc9d84d48c65d" "20190524": - - url: [ - "http://repo.msys2.org/distrib/i686/msys2-base-i686-20190524.tar.xz", - "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20190524.tar.xz", - ] - sha256: "7a463afae8bf6ce8262f010a9a1648d056ad5cefc66d6eb69fe948c57d4ccb53" - - url: [ - "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20190524.tar.xz", - "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20190524.tar.xz", - ] - sha256: "168e156fa9f00d90a8445676c023c63be6e82f71487f4e2688ab5cb13b345383" + "x86_64": + url: [ + "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20190524.tar.xz", + "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20190524.tar.xz", + ] + sha256: "168e156fa9f00d90a8445676c023c63be6e82f71487f4e2688ab5cb13b345383" + "x86": + url: [ + "http://repo.msys2.org/distrib/i686/msys2-base-i686-20190524.tar.xz", + "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20190524.tar.xz", + ] + sha256: "7a463afae8bf6ce8262f010a9a1648d056ad5cefc66d6eb69fe948c57d4ccb53" "20200517": - - url: [ - "http://repo.msys2.org/distrib/i686/msys2-base-i686-20200517.tar.xz", - "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20200517.tar.xz", - ] - sha256: "4dbdde708a4bcf3c056e8f4f28b1b0e7a3210ffacc6296a7d11fdaf076fd431e" - - url: [ - "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20200517.tar.xz", - "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20200517.tar.xz", - ] - sha256: "c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60" + "x86_64": + url: [ + "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20200517.tar.xz", + "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20200517.tar.xz", + ] + sha256: "c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60" + "x86": + url: [ + "http://repo.msys2.org/distrib/i686/msys2-base-i686-20200517.tar.xz", + "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20200517.tar.xz", + ] + sha256: "4dbdde708a4bcf3c056e8f4f28b1b0e7a3210ffacc6296a7d11fdaf076fd431e" + "20210105": + "x86_64": + url: [ + "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20210105.tar.xz", + "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20210105.tar.xz", + ] + sha256: "982e54de087d53adfc6a8caf7614d4a7add36dd02dcb0b7838060dd893e9f596" diff --git a/recipes/msys2/all/conanfile.py b/recipes/msys2/all/conanfile.py index affd28ac50de0..d100d8b401039 100644 --- a/recipes/msys2/all/conanfile.py +++ b/recipes/msys2/all/conanfile.py @@ -1,7 +1,38 @@ from conans import ConanFile, tools -from conans.errors import ConanInvalidConfiguration +from conans.errors import ConanInvalidConfiguration, ConanException +import fnmatch import os import shutil +import subprocess + +try: + import ctypes + from ctypes import wintypes +except ImportError: + pass +except ValueError: + pass + +class lock: + def __init__(self): + self.handle = ctypes.windll.kernel32.CreateMutexA(None, 0, "Global\\ConanMSYS2".encode()) + if not self.handle: + raise ctypes.WinError() + + def __enter__(self): + status = ctypes.windll.kernel32.WaitForSingleObject(self.handle, 0xFFFFFFFF) + if status not in [0, 0x80]: + raise ctypes.WinError() + + def __exit__(self, exc_type, exc_val, exc_tb): + status = ctypes.windll.kernel32.ReleaseMutex(self.handle) + if not status: + raise ctypes.WinError() + + def close(self): + ctypes.windll.kernel32.CloseHandle(self.handle) + + __del__ = close class MSYS2Conan(ConanFile): @@ -11,24 +42,28 @@ class MSYS2Conan(ConanFile): homepage = "http://www.msys2.org" license = "MSYS license" topics = ("conan", "msys", "unix", "subsystem") - build_requires = "7zip/19.00" short_paths = True - options = {"exclude_files": "ANY", # Comma separated list of file patterns to exclude from the package - "packages": "ANY", # Comma separated - "additional_packages": "ANY"} # Comma separated - default_options = {"exclude_files": "*/link.exe", - "packages": "base-devel,binutils,gcc", - "additional_packages": None} - settings = "os_build", "arch_build" - - def configure(self): - if self.settings.os_build != "Windows": + # "exclude_files" "packages" "additional_packages" values are a comma separated list + options = { + "exclude_files": "ANY", + "packages": "ANY", + "additional_packages": "ANY" + } + default_options = { + "exclude_files": "*/link.exe", + "packages": "base-devel,binutils,gcc", + "additional_packages": None + } + settings = "os", "arch" + + + def validate(self): + if self.settings.os != "Windows": raise ConanInvalidConfiguration("Only Windows supported") - - def source(self): - # build tools have to download files in build method when the - # source files downloaded will be different based on architecture or OS - pass + if tools.Version(self.version) >= "20210105" and self.settings.arch != "x86_64": + raise ConanInvalidConfiguration("Only Windows x64 supported") + if tools.Version(self.version) <= "20161025": + raise ConanInvalidConfiguration("msys2 v.20161025 is no longer supported") def _download(self, url, sha256): from six.moves.urllib.parse import urlparse @@ -36,29 +71,146 @@ def _download(self, url, sha256): tools.download(url=url, filename=filename, sha256=sha256) return filename + @property + def _keyring_subfolder(self): + return os.path.join(self.package_folder, "bin", "keyring") + + @property + def _keyring_file(self): + return os.path.join(self._keyring_subfolder, "msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz") + + @property + def _keyring_sig(self): + return os.path.join(self._keyring_subfolder, "msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig") + + # download the new keyring + # https://www.msys2.org/news/#2020-06-29-new-packagers + def _download_keyring(self): + tools.download( "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz", + self._keyring_file, + sha256="f1cc152902fd6018868b64d015cab9bf547ff9789d8bd7c0d798fb2b22367b2b" ) + tools.download( "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig", + self._keyring_sig, + sha256="bbd22e88f33c81c40b145c34d8027d60f714d4fd1d0dccd456895f398cc56297" ) + + def _install_pacman_keyring(self): + with tools.chdir(os.path.join(self._msys_dir, "usr", "bin")): + self.run('bash -l -c "pacman-key --init;pacman-key --populate"') + verify_command = 'bash -l -c "pacman-key --verify %s"' % self._keyring_sig.replace("\\", "/") + self.run(verify_command) + + install_command = 'bash -l -c "pacman -U %s --noconfirm"' % self._keyring_file.replace("\\", "/") + self.run(install_command) + self._kill_pacman() + + def _update_very_old_pacman(self): + if (tools.Version(self.version) < "20200517"): + # Fix (https://github.com/msys2/MSYS2-packages/issues/1967) + # install zstd and pacman manually + pacman_packages_x64=["http://repo.msys2.org/msys/x86_64/libzstd-1.4.4-2-x86_64.pkg.tar.xz", + "http://repo.msys2.org/msys/x86_64/zstd-1.4.4-2-x86_64.pkg.tar.xz", + "http://repo.msys2.org/msys/x86_64/pacman-5.2.1-6-x86_64.pkg.tar.xz"] + + if (self.settings.arch == "x86_64"): + for i in pacman_packages_x64: + self.run('bash -l -c "pacman --noconfirm -U %s"' % i) + + pacman_packages_x32=["http://repo.msys2.org/msys/i686/libzstd-1.4.4-2-i686.pkg.tar.xz", + "http://repo.msys2.org/msys/i686/zstd-1.4.4-2-i686.pkg.tar.xz", + "http://repo.msys2.org/msys/i686/pacman-5.2.1-6-i686.pkg.tar.xz"] + + if (self.settings.arch == "x86"): + for i in pacman_packages_x32: + self.run('bash -l -c "pacman --noconfirm -U %s"' % i) + + self._kill_pacman() + + + def _update_pacman(self): + with tools.chdir(os.path.join(self._msys_dir, "usr", "bin")): + try: + self._update_very_old_pacman() + + self._kill_pacman() + # https://www.msys2.org/news/ see 2020-05-31 - Update may fail with "could not open file" + # update pacman separately first + self.run('bash -l -c "pacman --debug --noconfirm -Sydd pacman"') + + # https://www.msys2.org/docs/ci/ + self.run('bash -l -c "pacman --debug --noconfirm --ask 20 -Syuu"') # Core update (in case any core packages are outdated) + self._kill_pacman() + self.run('bash -l -c "pacman --debug --noconfirm --ask 20 -Syuu"') # Normal update + self._kill_pacman() + self.run('bash -l -c "pacman --debug -Rc dash --noconfirm"') + except ConanException: + self.run('bash -l -c "cat /var/log/pacman.log || echo nolog"') + raise + + # https://github.com/msys2/MSYS2-packages/issues/1966 + def _kill_pacman(self): + if (self.settings.os == "Windows"): + taskkill_exe = os.path.join(os.environ.get('SystemRoot'), 'system32', 'taskkill.exe') + + log_out = True + if log_out: + out = subprocess.PIPE + err = subprocess.STDOUT + else: + out = file(os.devnull, 'w') + err = subprocess.PIPE + + if os.path.exists(taskkill_exe): + taskkill_cmds = [taskkill_exe + " /f /t /im pacman.exe", + taskkill_exe + " /f /im gpg-agent.exe", + taskkill_exe + " /f /im dirmngr.exe", + taskkill_exe + ' /fi "MODULES eq msys-2.0.dll"'] + for taskkill_cmd in taskkill_cmds: + try: + proc = subprocess.Popen(taskkill_cmd, stdout=out, stderr=err, bufsize=1) + proc.wait() + except OSError as e: + if e.errno == errno.ENOENT: + raise ConanException("Cannot kill pacman") + @property def _msys_dir(self): - return "msys64" if self.settings.arch_build == "x86_64" else "msys32" + subdir = "msys64" if (tools.Version(self.version) >= "20210105" or self.settings.arch == "x86_64") else "msys32" + return os.path.join(self.package_folder, "bin", subdir) + + def source(self): + # sources are different per configuration - do download in build + pass + + def _do_source(self): + filename = self._download(**self.conan_data["sources"][self.version][str(self.settings.arch)]) + tools.unzip(filename) + self._download_keyring() def build(self): - arch = 0 if self.settings.arch_build == "x86" else 1 # index in the sources list - filename = self._download(**self.conan_data["sources"][self.version][arch]) - tar_name = filename.replace(".xz", "") - self.run("7z.exe x {0}".format(filename)) - self.run("7z.exe x {0}".format(tar_name)) - os.unlink(filename) - os.unlink(tar_name) + os.makedirs(os.path.join(self.package_folder, "bin")) + with tools.chdir(os.path.join(self.package_folder, "bin")): + self._do_source() + with lock(): + self._do_build() + def _do_build(self): packages = [] if self.options.packages: packages.extend(str(self.options.packages).split(",")) if self.options.additional_packages: packages.extend(str(self.options.additional_packages).split(",")) + if (tools.Version(self.version) < "20210105"): + self._install_pacman_keyring() + + self._update_pacman() + with tools.chdir(os.path.join(self._msys_dir, "usr", "bin")): for package in packages: self.run('bash -l -c "pacman -S %s --noconfirm"' % package) + self._kill_pacman() + # create /tmp dir in order to avoid # bash.exe: warning: could not find /tmp, please create! tmp_dir = os.path.join(self._msys_dir, 'tmp') @@ -76,12 +228,18 @@ def package(self): excludes = None if self.options.exclude_files: excludes = tuple(str(self.options.exclude_files).split(",")) - self.copy("*", dst="bin", src=self._msys_dir, excludes=excludes) - shutil.copytree(os.path.join(self.package_folder, "bin", "usr", "share", "licenses"), + #self.copy("*", dst="bin", src=self._msys_dir, excludes=excludes) + for exclude in excludes: + for root, _, filenames in os.walk(self._msys_dir): + for filename in filenames: + fullname = os.path.join(root, filename) + if fnmatch.fnmatch(fullname, exclude): + os.unlink(fullname) + shutil.copytree(os.path.join(self._msys_dir, "usr", "share", "licenses"), os.path.join(self.package_folder, "licenses")) def package_info(self): - msys_root = os.path.join(self.package_folder, "bin") + msys_root = self._msys_dir msys_bin = os.path.join(msys_root, "usr", "bin") self.output.info("Creating MSYS_ROOT env var : %s" % msys_root) diff --git a/recipes/msys2/all/test_package/conanfile.py b/recipes/msys2/all/test_package/conanfile.py index 6217a01294882..eb3a4a0152e23 100644 --- a/recipes/msys2/all/test_package/conanfile.py +++ b/recipes/msys2/all/test_package/conanfile.py @@ -3,7 +3,10 @@ from io import StringIO class TestPackage(ConanFile): - + + def build(self): + pass # nothing to do, skip hook warning + def test(self): bash = tools.which("bash.exe") diff --git a/recipes/msys2/config.yml b/recipes/msys2/config.yml index ae025bf247598..5689dd93abc18 100644 --- a/recipes/msys2/config.yml +++ b/recipes/msys2/config.yml @@ -1,7 +1,8 @@ versions: - "20161025": - folder: "all" "20190524": folder: "all" "20200517": folder: "all" + "20210105": + folder: "all" +