(conan-io#4286) [msys2] Bump version and support two profiles approach.

* Bump version and support two profiles approach. https://docs.conan.io/en/latest/reference/profiles.html#build-profiles-and-host-profiles * PR 4286: invalid syntax * PR 4286: add new version (20210105) * PR 4286: CR avoid any check which are related to the running machine In general, we should avoid any check using platform, os_info,... which are related to the running machine, not to the package being generated. * PR 4286: try to move the msys2 download into source(.) https://docs.conan.io/en/latest/reference/conanfile/methods.html#source Please read the Note * PR 4286: 7zip -> tools.unzip(.) * PR 4286: install the new keyring # https://www.msys2.org/news/#2020-06-29-new-packagers * PR 4286: skip msys2 v.20161025 * PR 4286: fix invalid syntax * PR 4286: try to kill pacman first * PR 4286: try to kill pacman first on windows only * Revert "PR 4286: try to kill pacman first on windows only" This reverts commit e774845. * Revert "PR 4286: try to kill pacman first" This reverts commit 0048efa. * PR 4286: pac-man workarounds * PR 4286: Always kill all running msys2 processes * PR 4286: remove configure * PR 4286: more kills * PR 4286: add --ask 20 * PR 4286: less kills * PR 4286: skip hook warning * - more logs * - even more logs Signed-off-by: SSE4 <[email protected]> * - wait for the process Signed-off-by: SSE4 <[email protected]> * - it's possible we have several CI jobs which may incorrectly terminate each other with kill pacman Signed-off-by: SSE4 <[email protected]> * - guard to pass export on Linux Signed-off-by: SSE4 <[email protected]> * - typo * - ValueError * - syntax * - don't raise * - move downloads to build - drop version 20161025 - use new conandata.yml format - avoid unnecessary copying Co-authored-by: SSE4 <[email protected]>
AndreyMlashkin · Apr 26, 2021 · 1f79a9d · 1f79a9d
1 parent 1b0f712
commit 1f79a9d
Show file tree

Hide file tree

Showing 4 changed files with 224 additions and 62 deletions.
diff --git a/recipes/msys2/all/conandata.yml b/recipes/msys2/all/conandata.yml
@@ -1,34 +1,34 @@
 sources:
-  "20161025":
-    - url: [
-             "http://repo.msys2.org/distrib/i686/msys2-base-i686-20161025.tar.xz",
-             "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20161025.tar.xz",
-      ]
-      sha256: "8bafd3d52f5a51528a8671c1cae5591b36086d6ea5b1e76e17e390965cf6768f"
-    - url: [
-             "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20161025.tar.xz",
-             "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20161025.tar.xz"
-    ]
-      sha256: "bb1f1a0b35b3d96bf9c15092da8ce969a84a134f7b08811292fbc9d84d48c65d"
   "20190524":
-    - url: [
-             "http://repo.msys2.org/distrib/i686/msys2-base-i686-20190524.tar.xz",
-             "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20190524.tar.xz",
-    ]
-      sha256: "7a463afae8bf6ce8262f010a9a1648d056ad5cefc66d6eb69fe948c57d4ccb53"
-    - url: [
-             "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20190524.tar.xz",
-             "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20190524.tar.xz",
-      ]
-      sha256: "168e156fa9f00d90a8445676c023c63be6e82f71487f4e2688ab5cb13b345383"
+    "x86_64":
+          url: [
+                 "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20190524.tar.xz",
+                 "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20190524.tar.xz",
+          ]
+          sha256: "168e156fa9f00d90a8445676c023c63be6e82f71487f4e2688ab5cb13b345383"
+    "x86":
+          url: [
+                 "http://repo.msys2.org/distrib/i686/msys2-base-i686-20190524.tar.xz",
+                 "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20190524.tar.xz",
+          ]
+          sha256: "7a463afae8bf6ce8262f010a9a1648d056ad5cefc66d6eb69fe948c57d4ccb53"
   "20200517":
-    - url: [
-             "http://repo.msys2.org/distrib/i686/msys2-base-i686-20200517.tar.xz",
-             "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20200517.tar.xz",
-    ]
-      sha256: "4dbdde708a4bcf3c056e8f4f28b1b0e7a3210ffacc6296a7d11fdaf076fd431e"
-    - url: [
-             "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20200517.tar.xz",
-             "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20200517.tar.xz",
-      ]
-      sha256: "c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60"
+    "x86_64":
+          url: [
+                 "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20200517.tar.xz",
+                 "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20200517.tar.xz",
+          ]
+          sha256: "c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60"
+    "x86":
+          url: [
+                 "http://repo.msys2.org/distrib/i686/msys2-base-i686-20200517.tar.xz",
+                 "https://sourceforge.net/projects/msys2/files/Base/i686/msys2-base-i686-20200517.tar.xz",
+          ]
+          sha256: "4dbdde708a4bcf3c056e8f4f28b1b0e7a3210ffacc6296a7d11fdaf076fd431e"
+  "20210105":
+    "x86_64":
+          url: [
+                 "http://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20210105.tar.xz",
+                 "https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20210105.tar.xz",
+          ]
+          sha256: "982e54de087d53adfc6a8caf7614d4a7add36dd02dcb0b7838060dd893e9f596"
diff --git a/recipes/msys2/all/conanfile.py b/recipes/msys2/all/conanfile.py
@@ -1,7 +1,38 @@
 from conans import ConanFile, tools
-from conans.errors import ConanInvalidConfiguration
+from conans.errors import ConanInvalidConfiguration, ConanException
+import fnmatch
 import os
 import shutil
+import subprocess
+
+try:
+    import ctypes
+    from ctypes import wintypes
+except ImportError:
+    pass
+except ValueError:
+    pass
+
+class lock:
+    def __init__(self):
+        self.handle = ctypes.windll.kernel32.CreateMutexA(None, 0, "Global\\ConanMSYS2".encode())
+        if not self.handle:
+            raise ctypes.WinError()
+
+    def __enter__(self):
+        status = ctypes.windll.kernel32.WaitForSingleObject(self.handle, 0xFFFFFFFF)
+        if status not in [0, 0x80]:
+            raise ctypes.WinError()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        status = ctypes.windll.kernel32.ReleaseMutex(self.handle)
+        if not status:
+            raise ctypes.WinError()
+
+    def close(self):
+        ctypes.windll.kernel32.CloseHandle(self.handle)
+
+    __del__ = close
 
 
 class MSYS2Conan(ConanFile):
@@ -11,54 +42,175 @@ class MSYS2Conan(ConanFile):
     homepage = "http://www.msys2.org"
     license = "MSYS license"
     topics = ("conan", "msys", "unix", "subsystem")
-    build_requires = "7zip/19.00"
     short_paths = True
-    options = {"exclude_files": "ANY",  # Comma separated list of file patterns to exclude from the package
-               "packages": "ANY",  # Comma separated
-               "additional_packages": "ANY"}    # Comma separated
-    default_options = {"exclude_files": "*/link.exe",
-                       "packages": "base-devel,binutils,gcc",
-                       "additional_packages": None}
-    settings = "os_build", "arch_build"
-
-    def configure(self):
-        if self.settings.os_build != "Windows":
+    # "exclude_files" "packages" "additional_packages" values are a comma separated list
+    options = {
+        "exclude_files": "ANY",
+        "packages": "ANY",
+        "additional_packages": "ANY"
+    }
+    default_options = {
+        "exclude_files": "*/link.exe",
+        "packages": "base-devel,binutils,gcc",
+        "additional_packages": None
+    }
+    settings = "os", "arch"
+
+
+    def validate(self):
+        if self.settings.os != "Windows":
             raise ConanInvalidConfiguration("Only Windows supported")
-
-    def source(self):
-        # build tools have to download files in build method when the
-        # source files downloaded will be different based on architecture or OS
-        pass
+        if tools.Version(self.version) >= "20210105" and self.settings.arch != "x86_64":
+            raise ConanInvalidConfiguration("Only Windows x64 supported")
+        if tools.Version(self.version) <= "20161025":
+            raise ConanInvalidConfiguration("msys2 v.20161025 is no longer supported")    
 
     def _download(self, url, sha256):
         from six.moves.urllib.parse import urlparse
         filename = os.path.basename(urlparse(url[0]).path)
         tools.download(url=url, filename=filename, sha256=sha256)
         return filename
 
+    @property
+    def _keyring_subfolder(self):
+        return os.path.join(self.package_folder, "bin", "keyring")
+
+    @property
+    def _keyring_file(self):
+        return os.path.join(self._keyring_subfolder, "msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz")
+
+    @property
+    def _keyring_sig(self):
+        return os.path.join(self._keyring_subfolder, "msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig")    
+
+    # download the new keyring
+    # https://www.msys2.org/news/#2020-06-29-new-packagers
+    def _download_keyring(self):
+        tools.download( "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz",
+                        self._keyring_file,
+                        sha256="f1cc152902fd6018868b64d015cab9bf547ff9789d8bd7c0d798fb2b22367b2b" )
+        tools.download( "http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig",
+                        self._keyring_sig,
+                        sha256="bbd22e88f33c81c40b145c34d8027d60f714d4fd1d0dccd456895f398cc56297" )
+
+    def _install_pacman_keyring(self):
+        with tools.chdir(os.path.join(self._msys_dir, "usr", "bin")):
+            self.run('bash -l -c  "pacman-key --init;pacman-key --populate"')
+            verify_command = 'bash -l -c "pacman-key --verify %s"' % self._keyring_sig.replace("\\", "/")
+            self.run(verify_command)
+
+            install_command = 'bash -l -c "pacman -U %s --noconfirm"' % self._keyring_file.replace("\\", "/")
+            self.run(install_command)
+            self._kill_pacman()
+
+    def _update_very_old_pacman(self):
+        if (tools.Version(self.version) < "20200517"):
+            # Fix (https://github.com/msys2/MSYS2-packages/issues/1967)
+            # install zstd and pacman manually
+            pacman_packages_x64=["http://repo.msys2.org/msys/x86_64/libzstd-1.4.4-2-x86_64.pkg.tar.xz",
+                                 "http://repo.msys2.org/msys/x86_64/zstd-1.4.4-2-x86_64.pkg.tar.xz",
+                                 "http://repo.msys2.org/msys/x86_64/pacman-5.2.1-6-x86_64.pkg.tar.xz"]
+
+            if (self.settings.arch == "x86_64"):
+                for i in pacman_packages_x64:
+                    self.run('bash -l -c "pacman --noconfirm -U %s"' % i)
+
+            pacman_packages_x32=["http://repo.msys2.org/msys/i686/libzstd-1.4.4-2-i686.pkg.tar.xz",
+                                 "http://repo.msys2.org/msys/i686/zstd-1.4.4-2-i686.pkg.tar.xz",
+                                 "http://repo.msys2.org/msys/i686/pacman-5.2.1-6-i686.pkg.tar.xz"]
+
+            if (self.settings.arch == "x86"):                     
+                for i in pacman_packages_x32:
+                    self.run('bash -l -c "pacman --noconfirm -U %s"' % i)
+
+            self._kill_pacman()
+
+
+    def _update_pacman(self):
+        with tools.chdir(os.path.join(self._msys_dir, "usr", "bin")):
+            try:
+                self._update_very_old_pacman()
+
+                self._kill_pacman()
+                # https://www.msys2.org/news/   see  2020-05-31 - Update may fail with "could not open file"
+                # update pacman separately first
+                self.run('bash -l -c "pacman --debug --noconfirm -Sydd pacman"')
+
+                # https://www.msys2.org/docs/ci/
+                self.run('bash -l -c "pacman --debug --noconfirm --ask 20 -Syuu"')  # Core update (in case any core packages are outdated)
+                self._kill_pacman()
+                self.run('bash -l -c "pacman --debug --noconfirm --ask 20 -Syuu"')  # Normal update
+                self._kill_pacman()
+                self.run('bash -l -c "pacman --debug -Rc dash --noconfirm"')
+            except ConanException:
+                self.run('bash -l -c "cat /var/log/pacman.log || echo nolog"')
+                raise
+
+    # https://github.com/msys2/MSYS2-packages/issues/1966
+    def _kill_pacman(self):
+        if (self.settings.os == "Windows"):
+            taskkill_exe = os.path.join(os.environ.get('SystemRoot'), 'system32', 'taskkill.exe')
+
+            log_out = True
+            if log_out:
+                out = subprocess.PIPE
+                err = subprocess.STDOUT
+            else:
+                out = file(os.devnull, 'w')
+                err = subprocess.PIPE
+
+            if os.path.exists(taskkill_exe):
+                taskkill_cmds = [taskkill_exe + " /f /t /im pacman.exe",
+                                 taskkill_exe + " /f /im gpg-agent.exe",
+                                 taskkill_exe + " /f /im dirmngr.exe",
+                                 taskkill_exe + ' /fi "MODULES eq msys-2.0.dll"']
+                for taskkill_cmd in taskkill_cmds:
+                    try:
+                        proc = subprocess.Popen(taskkill_cmd, stdout=out, stderr=err, bufsize=1)
+                        proc.wait()
+                    except OSError as e:
+                        if e.errno == errno.ENOENT:
+                            raise ConanException("Cannot kill pacman")
+
     @property
     def _msys_dir(self):
-        return "msys64" if self.settings.arch_build == "x86_64" else "msys32"
+        subdir = "msys64" if (tools.Version(self.version) >= "20210105" or self.settings.arch == "x86_64") else "msys32"
+        return os.path.join(self.package_folder, "bin", subdir)
+
+    def source(self):
+        # sources are different per configuration - do download in build
+        pass
+
+    def _do_source(self):
+        filename = self._download(**self.conan_data["sources"][self.version][str(self.settings.arch)])
+        tools.unzip(filename)
+        self._download_keyring()
 
     def build(self):
-        arch = 0 if self.settings.arch_build == "x86" else 1  # index in the sources list
-        filename = self._download(**self.conan_data["sources"][self.version][arch])
-        tar_name = filename.replace(".xz", "")
-        self.run("7z.exe x {0}".format(filename))
-        self.run("7z.exe x {0}".format(tar_name))
-        os.unlink(filename)
-        os.unlink(tar_name)
+        os.makedirs(os.path.join(self.package_folder, "bin"))
+        with tools.chdir(os.path.join(self.package_folder, "bin")):
+            self._do_source()
+        with lock():
+            self._do_build()
 
+    def _do_build(self):
         packages = []
         if self.options.packages:
             packages.extend(str(self.options.packages).split(","))
         if self.options.additional_packages:
             packages.extend(str(self.options.additional_packages).split(","))
 
+        if (tools.Version(self.version) < "20210105"):
+            self._install_pacman_keyring()
+
+        self._update_pacman()
+
         with tools.chdir(os.path.join(self._msys_dir, "usr", "bin")):
             for package in packages:
                 self.run('bash -l -c "pacman -S %s --noconfirm"' % package)
 
+        self._kill_pacman()
+
         # create /tmp dir in order to avoid
         # bash.exe: warning: could not find /tmp, please create!
         tmp_dir = os.path.join(self._msys_dir, 'tmp')
@@ -76,12 +228,18 @@ def package(self):
         excludes = None
         if self.options.exclude_files:
             excludes = tuple(str(self.options.exclude_files).split(","))
-        self.copy("*", dst="bin", src=self._msys_dir, excludes=excludes)
-        shutil.copytree(os.path.join(self.package_folder, "bin", "usr", "share", "licenses"),
+        #self.copy("*", dst="bin", src=self._msys_dir, excludes=excludes)
+        for exclude in excludes:
+            for root, _, filenames in os.walk(self._msys_dir):
+                for filename in filenames:
+                    fullname = os.path.join(root, filename)
+                    if fnmatch.fnmatch(fullname, exclude):
+                        os.unlink(fullname)
+        shutil.copytree(os.path.join(self._msys_dir, "usr", "share", "licenses"),
                         os.path.join(self.package_folder, "licenses"))
 
     def package_info(self):
-        msys_root = os.path.join(self.package_folder, "bin")
+        msys_root = self._msys_dir
         msys_bin = os.path.join(msys_root, "usr", "bin")
 
         self.output.info("Creating MSYS_ROOT env var : %s" % msys_root)

diff --git a/recipes/msys2/all/test_package/conanfile.py b/recipes/msys2/all/test_package/conanfile.py
@@ -3,7 +3,10 @@
 from io import StringIO
 
 class TestPackage(ConanFile):
-
+
+    def build(self):
+        pass # nothing to do, skip hook warning
+
     def test(self):
         bash = tools.which("bash.exe")
 

diff --git a/recipes/msys2/config.yml b/recipes/msys2/config.yml
@@ -1,7 +1,8 @@
 versions:
-    "20161025":
-      folder: "all"
     "20190524":
       folder: "all"
     "20200517":
       folder: "all"
+    "20210105":
+      folder: "all"
+