From 419d841be4258e44d855496325ac52e8bddd98b8 Mon Sep 17 00:00:00 2001 From: Andrey Kulikov Date: Tue, 3 Jan 2023 20:05:40 +0300 Subject: [PATCH] Add support for converting pyproject.toml-based Python3 packages. Fixes #1780 Fixes #1873 Fixes #1860 Fixes magma/magma#13847 --- lib/fpm/package/pyfpm_toml/__init__.py | 1 + .../package/pyfpm_toml/get_metadata_toml.py | 67 +++++++ lib/fpm/package/python.rb | 173 ++++++++++++++++-- 3 files changed, 230 insertions(+), 11 deletions(-) create mode 100644 lib/fpm/package/pyfpm_toml/__init__.py create mode 100644 lib/fpm/package/pyfpm_toml/get_metadata_toml.py diff --git a/lib/fpm/package/pyfpm_toml/__init__.py b/lib/fpm/package/pyfpm_toml/__init__.py new file mode 100644 index 0000000000..583cfbf289 --- /dev/null +++ b/lib/fpm/package/pyfpm_toml/__init__.py @@ -0,0 +1 @@ +__all__ = [ "get_metadata_toml" ] diff --git a/lib/fpm/package/pyfpm_toml/get_metadata_toml.py b/lib/fpm/package/pyfpm_toml/get_metadata_toml.py new file mode 100644 index 0000000000..84e91ebc34 --- /dev/null +++ b/lib/fpm/package/pyfpm_toml/get_metadata_toml.py @@ -0,0 +1,67 @@ +from importlib_metadata import Distribution +import os +import sys +import pkg_resources +try: + import json +except ImportError: + import simplejson as json + +PY3 = sys.version_info[0] == 3 + +if PY3: + def u(s): + return s +else: + def u(s): + if isinstance(u, unicode): + return u + return s.decode('utf-8') + + +# Note, the last time I coded python daily was at Google, so it's entirely +# possible some of my techniques below are outdated or bad. +# If you have fixes, let me know. + + +class get_metadata_toml(): + + @staticmethod + def run(output_path): + dist = Distribution.at('.') + data = { + "name": dist.name, + "version": dist.version, + "author": ' '.join(dist.metadata.get_all('Author-email')), + "description": dist.metadata['Summary'], + "license": "FIXME!!!!", + "url": ' '.join(dist.metadata.get_all('Project-URL')), + } +# @todo FIXME!!! +# if dist.metadata.has_ext_modules(): +# data["architecture"] = "native" +# else: +# data["architecture"] = "all" + + data["architecture"] = "all" + + final_deps = [] + +# @todo FIXME!!! + data["dependencies"] = final_deps + + output = open(output_path, "w") + if hasattr(json, 'dumps'): + def default_to_str(obj): + """ Fall back to using __str__ if possible """ + # This checks if the class of obj defines __str__ itself, + # so we don't fall back to an inherited __str__ method. + if "__str__" in type(obj).__dict__: + return str(obj) + return json.JSONEncoder.default(self, obj) + + output.write(json.dumps(data, indent=2, default=default_to_str)) + else: + # For Python 2.5 and Debian's python-json + output.write(json.write(data)) + output.close() diff --git a/lib/fpm/package/python.rb b/lib/fpm/package/python.rb index 5754a053d4..5377b0a3a8 100644 --- a/lib/fpm/package/python.rb +++ b/lib/fpm/package/python.rb @@ -99,17 +99,25 @@ def input(package) if File.directory?(path_to_package) setup_py = File.join(path_to_package, "setup.py") + pyproject_toml = File.join(path_to_package, "pyproject.toml") else setup_py = path_to_package + pyproject_toml = path_to_package end - if !File.exist?(setup_py) - logger.error("Could not find 'setup.py'", :path => setup_py) - raise "Unable to find python package; tried #{setup_py}" + if File.exist?(setup_py) + logger.debug("Do job with setup.py") + load_package_info(setup_py) + install_to_staging(setup_py) + elsif File.exist?(pyproject_toml) + logger.debug("Do job with pyproject.toml") + load_package_info_toml(setup_py) + install_to_staging_toml(setup_py) + else + logger.error("Could not find neither 'setup.py' nor 'pyproject.toml'", :path => path_to_package) + raise "Unable to find python package; tried #{setup_py} and #{pyproject_toml}" end - load_package_info(setup_py) - install_to_staging(setup_py) end # def input # Download the given package if necessary. If version is given, that version @@ -195,8 +203,131 @@ def download_if_necessary(package, version=nil) return dirs.first end # def download - # Load the package information like name, version, dependencies. - def load_package_info(setup_py) + # Load the package information like name, version, dependencies via pyproject.toml. + def load_package_info_toml(setup_data) + if !attributes[:python_package_prefix].nil? + attributes[:python_package_name_prefix] = attributes[:python_package_prefix] + end + + begin + json_test_code = [ + "try:", + " import json", + "except ImportError:", + " import simplejson as json" + ].join("\n") + safesystem("#{attributes[:python_bin]} -c '#{json_test_code}'") + rescue FPM::Util::ProcessFailed => e + logger.error("Your python environment is missing json support (either json or simplejson python module). I cannot continue without this.", :python => attributes[:python_bin], :error => e) + raise FPM::Util::ProcessFailed, "Python (#{attributes[:python_bin]}) is missing simplejson or json modules." + end + + begin + safesystem("#{attributes[:python_bin]} -c 'import importlib_metadata'") + rescue FPM::Util::ProcessFailed => e + logger.error("Your python environment is missing a working importlib_metadata module. I tried to find the 'importlib_metadata' module but failed.", :python => attributes[:python_bin], :error => e) + raise FPM::Util::ProcessFailed, "Python (#{attributes[:python_bin]}) is missing importlib_metadata module." + end + + # Add ./pyfpm/ to the python library path + pylib = File.expand_path(File.dirname(__FILE__)) + + # chdir to the directory holding setup.py because some python setup.py's assume that you are + # in the same directory. + setup_dir = File.dirname(setup_data) + + output = ::Dir.chdir(setup_dir) do + tmp = build_path("metadata.json") + + toml_metadata_code = [ + "from pyfpm_toml import get_metadata_toml", + "gmt = get_metadata_toml.get_metadata_toml()", + "gmt.run('#{tmp}')" + ].join("; ") + + get_metadata_cmd = "env PYTHONPATH=#{pylib}:$PYTHONPATH #{attributes[:python_bin]} " \ + " -c " \ + "#{Shellwords.escape(toml_metadata_code)}" + + # @todo FIXME! + # if attributes[:python_obey_requirements_txt?] + # setup_cmd += " --load-requirements-txt" + # end + + # Capture the output, which will be JSON metadata describing this python + # package. See fpm/lib/fpm/package/pyfpm_toml/get_metadata.py for more + # details. + logger.info("fetching package metadata", :get_metadata_cmd => get_metadata_cmd) + + success = safesystem(get_metadata_cmd) + #%x{#{get_metadata_cmd}} + if !success + logger.error("pyfpm_toml get_metadata failed", :command => get_metadata_cmd, + :exitcode => $?.exitstatus) + raise "An unexpected error occurred while processing the pyproject.toml file" + end + File.read(tmp) + end + + logger.debug("result from `pyfpm_toml get_metadata`", :data => output) + metadata = JSON.parse(output) + logger.info("object output of pyfpm_toml get_metadata", :json => metadata) + + self.architecture = metadata["architecture"] + self.description = metadata["description"] + # Sometimes the license field is multiple lines; do best-effort and just + # use the first line. + if metadata["license"] + self.license = metadata["license"].split(/[\r\n]+/).first + end + self.version = metadata["version"] + self.url = metadata["url"] + + # name prefixing is optional, if enabled, a name 'foo' will become + # 'python-foo' (depending on what the python_package_name_prefix is) + if attributes[:python_fix_name?] + self.name = fix_name(metadata["name"]) + else + self.name = metadata["name"] + end + + # convert python-Foo to python-foo if flag is set + self.name = self.name.downcase if attributes[:python_downcase_name?] + + if !attributes[:no_auto_depends?] and attributes[:python_dependencies?] + metadata["dependencies"].each do |dep| + dep_re = /^([^<>!= ]+)\s*(?:([~<>!=]{1,2})\s*(.*))?$/ + match = dep_re.match(dep) + if match.nil? + logger.error("Unable to parse dependency", :dependency => dep) + raise FPM::InvalidPackageConfiguration, "Invalid dependency '#{dep}'" + end + name, cmp, version = match.captures + + next if attributes[:python_disable_dependency].include?(name) + + # convert == to = + if cmp == "==" or cmp == "~=" + logger.info("Converting == dependency requirement to =", :dependency => dep ) + cmp = "=" + end + + # dependency name prefixing is optional, if enabled, a name 'foo' will + # become 'python-foo' (depending on what the python_package_name_prefix + # is) + name = fix_name(name) if attributes[:python_fix_dependencies?] + + # convert dependencies from python-Foo to python-foo + name = name.downcase if attributes[:python_downcase_dependencies?] + + self.dependencies << "#{name} #{cmp} #{version}" + end + end # if attributes[:python_dependencies?] + end # def load_package_info_toml + + + # Load the package information like name, version, dependencies via setup.py. + def load_package_info(setup_data) if !attributes[:python_package_prefix].nil? attributes[:python_package_name_prefix] = attributes[:python_package_prefix] end @@ -226,7 +357,7 @@ def load_package_info(setup_py) # chdir to the directory holding setup.py because some python setup.py's assume that you are # in the same directory. - setup_dir = File.dirname(setup_py) + setup_dir = File.dirname(setup_data) output = ::Dir.chdir(setup_dir) do tmp = build_path("metadata.json") @@ -322,9 +453,29 @@ def fix_name(name) end end # def fix_name - # Install this package to the staging directory - def install_to_staging(setup_py) - project_dir = File.dirname(setup_py) + # Install this package to the staging directory via pyproject.toml + def install_to_staging_toml(setup_data) + project_dir = File.dirname(setup_data) + + # Some setup's assume $PWD == current directory of pyproject.toml, so let's + # chdir first. + ::Dir.chdir(project_dir) do + flags = [ "--root", staging_path ] + if !attributes[:prefix].nil? + flags += [ "--prefix", attributes[:prefix] ] + else + flags += ["--prefix", "/usr/local/"] + end + + # "--check-build-dependencies", + safesystem(*attributes[:python_pip], "install", ".", "--use-pep517", *flags) + end + end # def install_to_staging_toml + + + # Install this package to the staging directory via setup.py + def install_to_staging(setup_data) + project_dir = File.dirname(setup_data) prefix = "/" prefix = attributes[:prefix] unless attributes[:prefix].nil?