diff --git a/setuptools/command/_requirestxt.py b/setuptools/command/_requirestxt.py new file mode 100644 index 0000000000..435d71dca5 --- /dev/null +++ b/setuptools/command/_requirestxt.py @@ -0,0 +1,127 @@ +"""Helper code used to generate ``requires.txt`` files in the egg-info directory. + +The ``requires.txt`` file has an specific format: + - Environment markers need to be part of the section headers and + should not be part of the requirement spec itself. + +See https://setuptools.pypa.io/en/latest/deprecated/python_eggs.html#requires-txt +""" +import io +from collections import defaultdict +from itertools import filterfalse +from typing import Dict, List, Tuple, Mapping, TypeVar + +from .. import _reqs +from ..extern.jaraco.text import yield_lines +from ..extern.packaging.requirements import Requirement + + +# dict can work as an ordered set +_T = TypeVar("_T") +_Ordered = Dict[_T, None] +_ordered = dict +_StrOrIter = _reqs._StrOrIter + + +def _prepare( + install_requires: _StrOrIter, extras_require: Mapping[str, _StrOrIter] +) -> Tuple[List[str], Dict[str, List[str]]]: + """Given values for ``install_requires`` and ``extras_require`` + create modified versions in a way that can be written in ``requires.txt`` + """ + extras = _convert_extras_requirements(extras_require) + return _move_install_requirements_markers(install_requires, extras) + + +def _convert_extras_requirements( + extras_require: _StrOrIter, +) -> Mapping[str, _Ordered[Requirement]]: + """ + Convert requirements in `extras_require` of the form + `"extra": ["barbazquux; {marker}"]` to + `"extra:{marker}": ["barbazquux"]`. + """ + output: Mapping[str, _Ordered[Requirement]] = defaultdict(dict) + for section, v in extras_require.items(): + # Do not strip empty sections. + output[section] + for r in _reqs.parse(v): + output[section + _suffix_for(r)].setdefault(r) + + return output + + +def _move_install_requirements_markers( + install_requires: _StrOrIter, extras_require: Mapping[str, _Ordered[Requirement]] +) -> Tuple[List[str], Dict[str, List[str]]]: + """ + The ``requires.txt`` file has an specific format: + - Environment markers need to be part of the section headers and + should not be part of the requirement spec itself. + + Move requirements in ``install_requires`` that are using environment + markers ``extras_require``. + """ + + # divide the install_requires into two sets, simple ones still + # handled by install_requires and more complex ones handled by extras_require. + + inst_reqs = list(_reqs.parse(install_requires)) + simple_reqs = filter(_no_marker, inst_reqs) + complex_reqs = filterfalse(_no_marker, inst_reqs) + simple_install_requires = list(map(str, simple_reqs)) + + for r in complex_reqs: + extras_require[':' + str(r.marker)].setdefault(r) + + expanded_extras = dict( + # list(dict.fromkeys(...)) ensures a list of unique strings + (k, list(dict.fromkeys(str(r) for r in map(_clean_req, v)))) + for k, v in extras_require.items() + ) + + return simple_install_requires, expanded_extras + + +def _suffix_for(req): + """Return the 'extras_require' suffix for a given requirement.""" + return ':' + str(req.marker) if req.marker else '' + + +def _clean_req(req): + """Given a Requirement, remove environment markers and return it""" + req.marker = None + return req + + +def _no_marker(req): + return not req.marker + + +def _write_requirements(stream, reqs): + lines = yield_lines(reqs or ()) + + def append_cr(line): + return line + '\n' + + lines = map(append_cr, lines) + stream.writelines(lines) + + +def write_requirements(cmd, basename, filename): + dist = cmd.distribution + data = io.StringIO() + install_requires, extras_require = _prepare( + dist.install_requires or (), dist.extras_require or {} + ) + _write_requirements(data, install_requires) + for extra in sorted(extras_require): + data.write('\n[{extra}]\n'.format(**vars())) + _write_requirements(data, extras_require[extra]) + cmd.write_or_delete_file("requirements", filename, data.getvalue()) + + +def write_setup_requirements(cmd, basename, filename): + data = io.StringIO() + _write_requirements(data, cmd.distribution.setup_requires) + cmd.write_or_delete_file("setup-requirements", filename, data.getvalue()) diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index afc3265ef8..a5199deb33 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -12,12 +12,12 @@ import os import re import sys -import io import time import collections from .._importlib import metadata from .. import _entry_points, _normalization +from . import _requirestxt from setuptools import Command from setuptools.command.sdist import sdist @@ -28,7 +28,6 @@ from setuptools.glob import glob from setuptools.extern import packaging -from setuptools.extern.jaraco.text import yield_lines from ..warnings import SetuptoolsDeprecationWarning @@ -692,31 +691,9 @@ def warn_depends_obsolete(cmd, basename, filename): """ -def _write_requirements(stream, reqs): - lines = yield_lines(reqs or ()) - - def append_cr(line): - return line + '\n' - - lines = map(append_cr, lines) - stream.writelines(lines) - - -def write_requirements(cmd, basename, filename): - dist = cmd.distribution - data = io.StringIO() - _write_requirements(data, dist.install_requires) - extras_require = dist.extras_require or {} - for extra in sorted(extras_require): - data.write('\n[{extra}]\n'.format(**vars())) - _write_requirements(data, extras_require[extra]) - cmd.write_or_delete_file("requirements", filename, data.getvalue()) - - -def write_setup_requirements(cmd, basename, filename): - data = io.StringIO() - _write_requirements(data, cmd.distribution.setup_requires) - cmd.write_or_delete_file("setup-requirements", filename, data.getvalue()) +# Export API used in entry_points +write_requirements = _requirestxt.write_requirements +write_setup_requirements = _requirestxt.write_setup_requirements def write_toplevel_names(cmd, basename, filename): diff --git a/setuptools/config/_apply_pyprojecttoml.py b/setuptools/config/_apply_pyprojecttoml.py index 2d64860b04..4b8f803c1b 100644 --- a/setuptools/config/_apply_pyprojecttoml.py +++ b/setuptools/config/_apply_pyprojecttoml.py @@ -216,7 +216,7 @@ def _dependencies(dist: "Distribution", val: list, _root_dir): def _optional_dependencies(dist: "Distribution", val: dict, _root_dir): - existing = getattr(dist, "extras_require", {}) + existing = getattr(dist, "extras_require", None) or {} _set_config(dist, "extras_require", {**existing, **val}) @@ -383,8 +383,8 @@ def _acessor(obj): "entry-points": _get_previous_entrypoints, "scripts": _get_previous_scripts, "gui-scripts": _get_previous_gui_scripts, - "dependencies": _some_attrgetter("_orig_install_requires", "install_requires"), - "optional-dependencies": _some_attrgetter("_orig_extras_require", "extras_require"), + "dependencies": _attrgetter("install_requires"), + "optional-dependencies": _attrgetter("extras_require"), } diff --git a/setuptools/dist.py b/setuptools/dist.py index 5ae2061274..5e05920356 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -8,7 +8,6 @@ import re import sys import textwrap -from collections import defaultdict from contextlib import suppress from email import message_from_file from glob import iglob @@ -490,11 +489,6 @@ def __init__(self, attrs=None): # sdist (e.g. `version = file: VERSION.txt`) self._referenced_files: Set[str] = set() - # Save the original dependencies before they are processed into the egg format - self._orig_extras_require = {} - self._orig_install_requires = [] - self._tmp_extras_require = defaultdict(OrderedSet) - self.set_defaults = ConfigDiscovery(self) self._set_metadata_defaults(attrs) @@ -575,81 +569,23 @@ def _finalize_requires(self): if getattr(self, 'python_requires', None): self.metadata.python_requires = self.python_requires - if getattr(self, 'extras_require', None): - # Save original before it is messed by _convert_extras_requirements - self._orig_extras_require = self._orig_extras_require or self.extras_require + self._normalize_requires() + + if self.extras_require: for extra in self.extras_require.keys(): - # Since this gets called multiple times at points where the - # keys have become 'converted' extras, ensure that we are only - # truly adding extras we haven't seen before here. + # Setuptools allows a weird ": syntax for extras extra = extra.split(':')[0] if extra: self.metadata.provides_extras.add(extra) - if getattr(self, 'install_requires', None) and not self._orig_install_requires: - # Save original before it is messed by _move_install_requirements_markers - self._orig_install_requires = self.install_requires - - self._convert_extras_requirements() - self._move_install_requirements_markers() - - def _convert_extras_requirements(self): - """ - Convert requirements in `extras_require` of the form - `"extra": ["barbazquux; {marker}"]` to - `"extra:{marker}": ["barbazquux"]`. - """ - spec_ext_reqs = getattr(self, 'extras_require', None) or {} - tmp = defaultdict(OrderedSet) - self._tmp_extras_require = getattr(self, '_tmp_extras_require', tmp) - for section, v in spec_ext_reqs.items(): - # Do not strip empty sections. - self._tmp_extras_require[section] - for r in _reqs.parse(v): - suffix = self._suffix_for(r) - self._tmp_extras_require[section + suffix].append(r) - - @staticmethod - def _suffix_for(req): - """ - For a requirement, return the 'extras_require' suffix for - that requirement. - """ - return ':' + str(req.marker) if req.marker else '' - - def _move_install_requirements_markers(self): - """ - Move requirements in `install_requires` that are using environment - markers `extras_require`. - """ - - # divide the install_requires into two sets, simple ones still - # handled by install_requires and more complex ones handled - # by extras_require. - - def is_simple_req(req): - return not req.marker - - spec_inst_reqs = getattr(self, 'install_requires', None) or () - inst_reqs = list(_reqs.parse(spec_inst_reqs)) - simple_reqs = filter(is_simple_req, inst_reqs) - complex_reqs = itertools.filterfalse(is_simple_req, inst_reqs) - self.install_requires = list(map(str, simple_reqs)) - - for r in complex_reqs: - self._tmp_extras_require[':' + str(r.marker)].append(r) - self.extras_require = dict( - # list(dict.fromkeys(...)) ensures a list of unique strings - (k, list(dict.fromkeys(str(r) for r in map(self._clean_req, v)))) - for k, v in self._tmp_extras_require.items() - ) - - def _clean_req(self, req): - """ - Given a Requirement, remove environment markers and return it. - """ - req.marker = None - return req + def _normalize_requires(self): + """Make sure requirement-related attributes exist and are normalized""" + install_requires = getattr(self, "install_requires", None) or [] + extras_require = getattr(self, "extras_require", None) or {} + self.install_requires = list(map(str, _reqs.parse(install_requires))) + self.extras_require = { + k: list(map(str, _reqs.parse(v or []))) for k, v in extras_require.items() + } def _finalize_license_files(self): """Compute names of all license files which should be included.""" diff --git a/setuptools/tests/config/test_apply_pyprojecttoml.py b/setuptools/tests/config/test_apply_pyprojecttoml.py index a6a827094d..8a654ff94a 100644 --- a/setuptools/tests/config/test_apply_pyprojecttoml.py +++ b/setuptools/tests/config/test_apply_pyprojecttoml.py @@ -388,12 +388,12 @@ def test_optional_dependencies_dont_remove_env_markers(self, tmp_path): dist = makedist(tmp_path, install_requires=install_req) dist = pyprojecttoml.apply_configuration(dist, pyproject) assert "foo" in dist.extras_require - assert ':python_version < "3.7"' in dist.extras_require egg_info = dist.get_command_obj("egg_info") write_requirements(egg_info, tmp_path, tmp_path / "requires.txt") reqs = (tmp_path / "requires.txt").read_text(encoding="utf-8") assert "importlib-resources" in reqs assert "bar" in reqs + assert ':python_version < "3.7"' in reqs @pytest.mark.parametrize( "field,group", [("scripts", "console_scripts"), ("gui-scripts", "gui_scripts")]