diff --git a/.flake8 b/.flake8 index b61e46d48..684ff8df7 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,10 @@ [flake8] +# NOTE: Can ruff replace flake8? See `tool.isort`, these two configs should be +# kept in sync until we pick one. ignore = E203, E266, E501, W503, F403, F401 +per-file-ignores = + conda_lock/src_parser/meta_yaml.py:E122 + max-line-length = 89 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 \ No newline at end of file +max-complexity = 19 +select = B,C,E,F,W,T4,B9 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 927d61db7..b8ba74834 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -37,7 +37,7 @@ jobs: - name: Publish a Python distribution to PyPI if: ${{ github.event_name == 'release' }} - uses: pypa/gh-action-pypi-publish@v1.8.8 + uses: pypa/gh-action-pypi-publish@v1.8.10 with: user: __token__ password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/update-lockfile.yaml b/.github/workflows/update-lockfile.yaml index a54f14b2c..ed6df98e6 100644 --- a/.github/workflows/update-lockfile.yaml +++ b/.github/workflows/update-lockfile.yaml @@ -11,6 +11,8 @@ on: jobs: conda-lock: + # Don't run scheduled job on forks. Ref: + if: (github.event_name == 'schedule' && github.repository == 'conda/conda-lock') || (github.event_name != 'schedule') defaults: run: # Ensure the environment is activated diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcfcef7dd..8c45525aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,21 +8,28 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - - id: trailing-whitespace - exclude: "^.*\\.patch$" - - id: check-ast + - id: trailing-whitespace + exclude: "^.*\\.patch$" + - id: check-ast - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black language_version: python3 -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.286 hooks: - - id: flake8 + - id: ruff + args: ["--fix"] +# Ruff should catch (and mostly fix) everything that flake8 and isort do; if +# either of these checks fails, can Ruff's config be updated to catch the same? +- repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: @@ -30,7 +37,7 @@ repos: args: ["--profile", "black", "--filter-files"] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 + rev: v1.5.1 hooks: - id: mypy additional_dependencies: diff --git a/conda_lock/__init__.py b/conda_lock/__init__.py index 07b31b259..c9048190d 100644 --- a/conda_lock/__init__.py +++ b/conda_lock/__init__.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import distribution from conda_lock.conda_lock import main @@ -7,6 +7,6 @@ try: - __version__ = pkg_resources.get_distribution("conda_lock").version + __version__ = distribution("conda_lock").version except Exception: __version__ = "unknown" diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 669dc5202..0c96d4c84 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -15,6 +15,7 @@ from contextlib import contextmanager from functools import partial +from importlib.metadata import distribution from types import TracebackType from typing import ( AbstractSet, @@ -32,7 +33,6 @@ from urllib.parse import urlsplit import click -import pkg_resources import yaml from ensureconda.api import ensureconda @@ -241,7 +241,7 @@ def fn_to_dist_name(fn: str) -> str: return fn -def make_lock_files( # noqa: C901 +def make_lock_files( *, conda: PathLike, src_files: List[pathlib.Path], @@ -484,7 +484,7 @@ def do_render( "platform": plat, "dev-dependencies": str(include_dev_dependencies).lower(), "input-hash": lockfile.metadata.content_hash, - "version": pkg_resources.get_distribution("conda_lock").version, + "version": distribution("conda_lock").version, "timestamp": datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ"), } @@ -768,7 +768,7 @@ def create_lockfile_from_spec( *, conda: PathLike, spec: LockSpecification, - platforms: List[str] = [], + platforms: Optional[List[str]] = None, lockfile_path: pathlib.Path, update_spec: Optional[UpdateSpecification] = None, metadata_choices: AbstractSet[MetadataOption] = frozenset(), @@ -778,6 +778,8 @@ def create_lockfile_from_spec( """ Solve or update specification """ + if platforms is None: + platforms = [] assert spec.virtual_package_repo is not None virtual_package_channel = spec.virtual_package_repo.channel @@ -943,10 +945,10 @@ def _render_lockfile_for_install( if platform not in lock_content.metadata.platforms: suggested_platforms_section = "platforms:\n- " suggested_platforms_section += "\n- ".join( - [platform] + lock_content.metadata.platforms + [platform, *lock_content.metadata.platforms] ) suggested_platform_args = "--platform=" + " --platform=".join( - [platform] + lock_content.metadata.platforms + [platform, *lock_content.metadata.platforms] ) raise PlatformValidationError( f"The lockfile {filename} does not contain a solution for the current " diff --git a/conda_lock/conda_solver.py b/conda_lock/conda_solver.py index eb9fb6ef6..9c414e640 100644 --- a/conda_lock/conda_solver.py +++ b/conda_lock/conda_solver.py @@ -476,7 +476,7 @@ def update_specs_for_arch( proc = subprocess.run( [ str(arg) - for arg in args + ["-p", prefix, "--json", "--dry-run", *to_update] + for arg in [*args, "-p", prefix, "--json", "--dry-run", *to_update] ], env=conda_env_override(platform), stdout=subprocess.PIPE, diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index 3d69db1d2..4cc93dea4 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -70,7 +70,7 @@ def dep_name(manager: str, dep: str) -> str: # If we operate on lists of pip names and this is a conda dependency, we # convert the name to a pip name. if convert_to_pip_names and manager == "conda": - return conda_name_to_pypi_name(dep).lower() + return conda_name_to_pypi_name(dep) return dep for name, request in requested.items(): diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index 038a7e14e..71ad4352f 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -1,3 +1,6 @@ +# isort: skip_file +# TODO: Remove the isort skip comment above if/when isort is no longer used. This skip +# exists because isort and ruff disagree about how to sort the imports in this file. from collections import defaultdict from typing import ClassVar, Dict, List, Optional diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index d13910ed2..a56e1be35 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -4,6 +4,7 @@ import requests import yaml +from packaging.utils import NormalizedName, canonicalize_name from typing_extensions import TypedDict @@ -11,7 +12,7 @@ class MappingEntry(TypedDict): conda_name: str # legacy field, generally not used by anything anymore conda_forge: str - pypi_name: str + pypi_name: NormalizedName class _LookupLoader: @@ -28,15 +29,15 @@ def mapping_url(self, value: str) -> None: self._mapping_url = value @cached_property - def pypi_lookup(self) -> Dict[str, MappingEntry]: + def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: res = requests.get(self._mapping_url) res.raise_for_status() lookup = yaml.safe_load(res.content) # lowercase and kebabcase the pypi names assert lookup is not None - lookup = {k.lower().replace("_", "-"): v for k, v in lookup.items()} + lookup = {canonicalize_name(k): v for k, v in lookup.items()} for v in lookup.values(): - v["pypi_name"] = v["pypi_name"].lower().replace("_", "-") + v["pypi_name"] = canonicalize_name(v["pypi_name"]) return lookup @cached_property @@ -47,7 +48,7 @@ def conda_lookup(self) -> Dict[str, MappingEntry]: LOOKUP_OBJECT = _LookupLoader() -def get_forward_lookup() -> Dict[str, MappingEntry]: +def get_forward_lookup() -> Dict[NormalizedName, MappingEntry]: global LOOKUP_OBJECT return LOOKUP_OBJECT.pypi_lookup @@ -65,12 +66,14 @@ def set_lookup_location(lookup_url: str) -> None: LOOKUP_OBJECT.mapping_url = lookup_url -def conda_name_to_pypi_name(name: str) -> str: +def conda_name_to_pypi_name(name: str) -> NormalizedName: """return the pypi name for a conda package""" lookup = get_lookup() - return lookup.get(name, {"pypi_name": name})["pypi_name"] + cname = canonicalize_name(name) + return lookup.get(cname, {"pypi_name": cname})["pypi_name"] def pypi_name_to_conda_name(name: str) -> str: """return the conda name for a pypi package""" - return get_forward_lookup().get(name, {"conda_name": name})["conda_name"] + cname = canonicalize_name(name) + return get_forward_lookup().get(cname, {"conda_name": cname})["conda_name"] diff --git a/conda_lock/models/__init__.py b/conda_lock/models/__init__.py index 5d96b5ade..2b74db890 100644 --- a/conda_lock/models/__init__.py +++ b/conda_lock/models/__init__.py @@ -1,11 +1,5 @@ from pydantic import BaseModel -class StrictModel(BaseModel): - """A Pydantic BaseModel forbidding extra fields and encoding frozensets as lists""" - - class Config: - extra = "forbid" - json_encoders = { - frozenset: list, - } +class StrictModel(BaseModel, extra="forbid"): + """A Pydantic BaseModel forbidding extra fields""" diff --git a/conda_lock/models/channel.py b/conda_lock/models/channel.py index 5645c8184..60874f3f4 100644 --- a/conda_lock/models/channel.py +++ b/conda_lock/models/channel.py @@ -42,7 +42,7 @@ from typing import FrozenSet, List, Optional, cast from urllib.parse import unquote, urlparse, urlunparse -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field if typing.TYPE_CHECKING: @@ -90,15 +90,14 @@ def __repr_args__(self: BaseModel) -> "ReprArgs": class Channel(ZeroValRepr, BaseModel): + model_config = ConfigDict(frozen=True) # type: ignore + url: str used_env_vars: FrozenSet[str] = Field(default=frozenset()) def __lt__(self, other: "Channel") -> bool: return tuple(self.dict().values()) < tuple(other.dict().values()) - class Config: - frozen = True - @classmethod def from_string(cls, value: str) -> "Channel": if "://" in value: @@ -144,7 +143,7 @@ def _detect_used_env_var( if value.startswith("$"): return value.lstrip("$").strip("{}") - for suffix in preferred_env_var_suffix + [""]: + for suffix in [*preferred_env_var_suffix, ""]: candidates = {v: k for k, v in os.environ.items() if k.upper().endswith(suffix)} # try first with a simple match key = candidates.get(value) diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index e29b723f1..56b85d819 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -350,7 +350,7 @@ def solve_pypi( # is essentially a dictionary of: # - pip package name -> list of LockedDependency that are needed for this package for conda_name, locked_dep in conda_locked.items(): - pypi_name = conda_name_to_pypi_name(conda_name).lower() + pypi_name = conda_name_to_pypi_name(conda_name) if pypi_name in planned: planned[pypi_name].append(locked_dep) else: diff --git a/conda_lock/src_parser/meta_yaml.py b/conda_lock/src_parser/meta_yaml.py index 79470027a..5ebd8768f 100644 --- a/conda_lock/src_parser/meta_yaml.py +++ b/conda_lock/src_parser/meta_yaml.py @@ -47,7 +47,7 @@ def __init__( # type: ignore __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \ __complex__ = __pow__ = __rpow__ = \ - lambda self, *args, **kwargs: self._return_undefined(self._undefined_name) # noqa: E122 + lambda self, *args, **kwargs: self._return_undefined(self._undefined_name) # fmt: on # Accessing an attribute of an Undefined variable @@ -60,7 +60,7 @@ def __getattr__(self, k: str) -> "UndefinedNeverFail": # Unlike the methods above, Python requires that these # few methods must always return the correct type - __str__ = __repr__ = lambda self: self._return_value(str()) # type: ignore # noqa: E731 + __str__ = __repr__ = lambda self: self._return_value(str()) # type: ignore __unicode__ = lambda self: self._return_value("") # noqa: E731 __int__ = lambda self: self._return_value(0) # type: ignore # noqa: E731 __float__ = lambda self: self._return_value(0.0) # type: ignore # noqa: E731 diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 4401cf58b..fe440f8b7 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -25,7 +25,8 @@ else: from tomli import load as toml_load -from pkg_resources import Requirement +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name as canonicalize_pypi_name from typing_extensions import Literal from conda_lock.common import get_in @@ -73,17 +74,19 @@ def join_version_components(pieces: Sequence[Union[str, int]]) -> str: def normalize_pypi_name(name: str) -> str: - name = name.replace("_", "-").lower() - if name in get_lookup(): - lookup = get_lookup()[name] + cname = canonicalize_pypi_name(name) + if cname in get_lookup(): + lookup = get_lookup()[cname] res = lookup.get("conda_name") or lookup.get("conda_forge") if res is not None: return res else: - logging.warning(f"Could not find conda name for {name}. Assuming identity.") - return name + logging.warning( + f"Could not find conda name for {cname}. Assuming identity." + ) + return cname else: - return name + return cname def poetry_version_to_conda_version(version_string: Optional[str]) -> Optional[str]: @@ -368,19 +371,20 @@ def parse_requirement_specifier( requirement: str, ) -> Requirement: """Parse a url requirement to a conda spec""" - requirement_specifier = requirement.split(";")[0].strip() - if ( - requirement_specifier.startswith("git+") - or requirement_specifier.startswith("https://") - or requirement_specifier.startswith("ssh://") + requirement.startswith("git+") + or requirement.startswith("https://") + or requirement.startswith("ssh://") ): - parsed_req = Requirement.parse( - requirement_specifier.split("/")[-1].replace("@", "==") - ) - parsed_req.url = requirement_specifier - return parsed_req - return Requirement.parse(requirement_specifier) + # Handle the case where only the URL is specified without a package name + repo_name_and_maybe_tag = requirement.split("/")[-1] + repo_name = repo_name_and_maybe_tag.split("@")[0] + if repo_name.endswith(".git"): + repo_name = repo_name[:-4] + # Use the repo name as a placeholder for the package name + return Requirement(f"{repo_name} @ {requirement}") + else: + return Requirement(requirement) def unpack_git_url(url: str) -> Tuple[str, Optional[str]]: @@ -407,8 +411,8 @@ def parse_python_requirement( ) -> Dependency: """Parse a requirements.txt like requirement to a conda spec""" parsed_req = parse_requirement_specifier(requirement) - name = parsed_req.unsafe_name.lower() - collapsed_version = ",".join("".join(spec) for spec in parsed_req.specs) + name = canonicalize_pypi_name(parsed_req.name) + collapsed_version = str(parsed_req.specifier) conda_version = poetry_version_to_conda_version(collapsed_version) if conda_version: conda_version = ",".join(sorted(conda_version.split(","))) diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py index c0f615e9b..aa505efaa 100644 --- a/conda_lock/virtual_package.py +++ b/conda_lock/virtual_package.py @@ -24,7 +24,6 @@ class FakePackage(BaseModel): """A minimal representation of the required metadata for a conda package""" class Config: - allow_mutation = False frozen = True name: str diff --git a/environments/dev-environment.yaml b/environments/dev-environment.yaml index e9fdd6045..bee5b5a24 100644 --- a/environments/dev-environment.yaml +++ b/environments/dev-environment.yaml @@ -7,19 +7,14 @@ dependencies: - black - check-manifest - doctr -- flake8 -- flake8-builtins -- flake8-comprehensions -- flake8-mutable +- filelock - python-build - freezegun - isort - mypy - pre-commit -- pylint - pytest - pytest-cov -- pytest-flake8 - pytest-xdist - pytest-timeout - tomli diff --git a/pyproject.toml b/pyproject.toml index a7c712602..9fb2c0b64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,18 +26,19 @@ classifiers = [ dynamic = ["version"] license-files = { paths = ["LICENSE"] } dependencies = [ + # conda-lock dependencies "click >=8.0", "click-default-group", "ensureconda >=1.3", "gitpython >=3.1.30", "jinja2", - "pydantic >=1.8.1", + "pydantic >=1.10", "pyyaml >= 5.1", - "ruamel.yaml", 'tomli; python_version<"3.11"', "typing-extensions", + # conda dependencies + "ruamel.yaml", "toolz >=0.12.0,<1.0.0", - "filelock >=3.8.0", # The following dependencies were added in the process of vendoring Poetry 1.1.15. # poetry: "cachecontrol[filecache] >=0.12.9", @@ -49,8 +50,6 @@ dependencies = [ "crashtest >=0.3.0", # poetry: "html5lib >=1.0", - # poetry, poetry-core: - 'importlib-metadata >=1.7.0; python_version <= "3.7"', # poetry: "keyring >=21.2.0", # poetry: @@ -110,6 +109,8 @@ exclude = [ ] [tool.isort] +# NOTE: Can ruff replace isort? See `tool.ruff.isort`, these two configs should +# be kept in sync until we pick one. atomic = true force_grid_wrap = 0 include_trailing_comma = true @@ -158,3 +159,48 @@ platforms = ["linux-64", "osx-64", "osx-arm64", "win-64", "osx-arm64", "linux-aa # This is necessary to pull in the lockfile/filelock dependency # since we don't handle the optional dependency. cachecontrol-with-filecache = ">=0.12.9" + + +[tool.ruff] +target-version = "py38" +ignore = [ + "E501", + "F401", + "F403", + # Disabled during migration to Ruff: + "A001", + "A002", + "A003", + "C401", + "C405", + "C408", + "C409", + "C413", + "C414", + "C416", + "RUF012", + "RUF015", +] +line-length = 89 +select = [ + "A", # flake8-builtins + # "B", # flake8-bugbear + "B006", # Do not use mutable data structures for argument defaults + "C4", # flake8-comprehensions + "C9", # mccabe + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "RUF", # ruff rules + "W", # pycodestyle warnings +] + +[tool.ruff.mccabe] +max-complexity = 18 + +[tool.ruff.isort] +# NOTE: Can ruff replace isort? See `tool.isort`, these two configs should be +# kept in sync until we pick one. +lines-after-imports = 2 +lines-between-types = 1 +known-first-party = ["attr"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 62bcf44c4..b4feceec3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,19 +1,12 @@ black check-manifest doctr -flake8 -flake8-builtins -flake8-comprehensions -flake8-mutable build freezegun -isort mypy pre_commit -pylint pytest pytest-cov -pytest-flake8 pytest-xdist pytest-timeout tomli; python_version<"3.11"