From b249119e78d5c68625cc656e29480ad58ed3ad2d Mon Sep 17 00:00:00 2001 From: Marius van Niekerk Date: Thu, 13 Jul 2023 16:47:37 -0500 Subject: [PATCH 01/20] Update to pydantic 2.0 --- conda_lock/models/__init__.py | 8 +------- conda_lock/models/channel.py | 7 +++---- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/conda_lock/models/__init__.py b/conda_lock/models/__init__.py index 5d96b5ade..f018fd811 100644 --- a/conda_lock/models/__init__.py +++ b/conda_lock/models/__init__.py @@ -1,11 +1,5 @@ from pydantic import BaseModel -class StrictModel(BaseModel): +class StrictModel(BaseModel, extra="forbid"): """A Pydantic BaseModel forbidding extra fields and encoding frozensets as lists""" - - class Config: - extra = "forbid" - json_encoders = { - frozenset: list, - } diff --git a/conda_lock/models/channel.py b/conda_lock/models/channel.py index 5645c8184..f679fd69c 100644 --- a/conda_lock/models/channel.py +++ b/conda_lock/models/channel.py @@ -42,7 +42,7 @@ from typing import FrozenSet, List, Optional, cast from urllib.parse import unquote, urlparse, urlunparse -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field if typing.TYPE_CHECKING: @@ -90,15 +90,14 @@ def __repr_args__(self: BaseModel) -> "ReprArgs": class Channel(ZeroValRepr, BaseModel): + model_config = ConfigDict(frozen=True) + url: str used_env_vars: FrozenSet[str] = Field(default=frozenset()) def __lt__(self, other: "Channel") -> bool: return tuple(self.dict().values()) < tuple(other.dict().values()) - class Config: - frozen = True - @classmethod def from_string(cls, value: str) -> "Channel": if "://" in value: diff --git a/pyproject.toml b/pyproject.toml index a7c712602..432fb18eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "ensureconda >=1.3", "gitpython >=3.1.30", "jinja2", - "pydantic >=1.8.1", + "pydantic >=2", "pyyaml >= 5.1", "ruamel.yaml", 'tomli; python_version<"3.11"', From ffac6b27b1d001c8751c89f670014eebaa474307 Mon Sep 17 00:00:00 2001 From: Marius van Niekerk Date: Thu, 13 Jul 2023 17:36:11 -0500 Subject: [PATCH 02/20] make mypy happy --- .pre-commit-config.yaml | 2 +- conda_lock/models/channel.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fcfcef7dd..bde7db27c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: check-ast - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black language_version: python3 diff --git a/conda_lock/models/channel.py b/conda_lock/models/channel.py index f679fd69c..5cb3b6143 100644 --- a/conda_lock/models/channel.py +++ b/conda_lock/models/channel.py @@ -90,7 +90,7 @@ def __repr_args__(self: BaseModel) -> "ReprArgs": class Channel(ZeroValRepr, BaseModel): - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(frozen=True) # type: ignore url: str used_env_vars: FrozenSet[str] = Field(default=frozenset()) From c83478020cf7a61590bd65e3c8c53f347584aafe Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sat, 15 Jul 2023 18:11:01 +0200 Subject: [PATCH 03/20] Relax Pydantic pin The stuff we actually need was introduced in Pydantic 1.10 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 432fb18eb..e02aa54da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "ensureconda >=1.3", "gitpython >=3.1.30", "jinja2", - "pydantic >=2", + "pydantic >=1.10", "pyyaml >= 5.1", "ruamel.yaml", 'tomli; python_version<"3.11"', From 34b95c44754b2aa30b610432d48c632d9e88cc15 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sat, 15 Jul 2023 23:53:02 +0200 Subject: [PATCH 04/20] Drop implicit filelock dependency It's implied by cachecontrol[filecache]. It is however an explicit dev dependency since it's imported in the test suite. --- environments/dev-environment.yaml | 1 + pyproject.toml | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/environments/dev-environment.yaml b/environments/dev-environment.yaml index e9fdd6045..14a05a878 100644 --- a/environments/dev-environment.yaml +++ b/environments/dev-environment.yaml @@ -7,6 +7,7 @@ dependencies: - black - check-manifest - doctr +- filelock - flake8 - flake8-builtins - flake8-comprehensions diff --git a/pyproject.toml b/pyproject.toml index a7c712602..8dce4752e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ classifiers = [ dynamic = ["version"] license-files = { paths = ["LICENSE"] } dependencies = [ + # conda-lock dependencies "click >=8.0", "click-default-group", "ensureconda >=1.3", @@ -33,11 +34,11 @@ dependencies = [ "jinja2", "pydantic >=1.8.1", "pyyaml >= 5.1", - "ruamel.yaml", 'tomli; python_version<"3.11"', "typing-extensions", + # conda dependencies + "ruamel.yaml", "toolz >=0.12.0,<1.0.0", - "filelock >=3.8.0", # The following dependencies were added in the process of vendoring Poetry 1.1.15. # poetry: "cachecontrol[filecache] >=0.12.9", From 22de1727a63c8e0ca78812651d950eb682dd90e7 Mon Sep 17 00:00:00 2001 From: Marius van Niekerk Date: Fri, 21 Jul 2023 08:51:45 -0400 Subject: [PATCH 05/20] Update conda_lock/models/__init__.py --- conda_lock/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_lock/models/__init__.py b/conda_lock/models/__init__.py index f018fd811..2b74db890 100644 --- a/conda_lock/models/__init__.py +++ b/conda_lock/models/__init__.py @@ -2,4 +2,4 @@ class StrictModel(BaseModel, extra="forbid"): - """A Pydantic BaseModel forbidding extra fields and encoding frozensets as lists""" + """A Pydantic BaseModel forbidding extra fields""" From 42e1e281a26d56f5de68d5e6bdaa43ff08274af5 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 12 Jul 2023 12:51:35 +0200 Subject: [PATCH 06/20] Remove deprecated pkg_resources --- conda_lock/__init__.py | 4 ++-- conda_lock/conda_lock.py | 4 ++-- conda_lock/src_parser/pyproject_toml.py | 28 ++++++++++++------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/conda_lock/__init__.py b/conda_lock/__init__.py index 07b31b259..c9048190d 100644 --- a/conda_lock/__init__.py +++ b/conda_lock/__init__.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import distribution from conda_lock.conda_lock import main @@ -7,6 +7,6 @@ try: - __version__ = pkg_resources.get_distribution("conda_lock").version + __version__ = distribution("conda_lock").version except Exception: __version__ = "unknown" diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index 669dc5202..ff398783d 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -15,6 +15,7 @@ from contextlib import contextmanager from functools import partial +from importlib.metadata import distribution from types import TracebackType from typing import ( AbstractSet, @@ -32,7 +33,6 @@ from urllib.parse import urlsplit import click -import pkg_resources import yaml from ensureconda.api import ensureconda @@ -484,7 +484,7 @@ def do_render( "platform": plat, "dev-dependencies": str(include_dev_dependencies).lower(), "input-hash": lockfile.metadata.content_hash, - "version": pkg_resources.get_distribution("conda_lock").version, + "version": distribution("conda_lock").version, "timestamp": datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ"), } diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index 4401cf58b..eb6b596e9 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -25,7 +25,8 @@ else: from tomli import load as toml_load -from pkg_resources import Requirement +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name as canonicalize_pypi_name from typing_extensions import Literal from conda_lock.common import get_in @@ -368,19 +369,18 @@ def parse_requirement_specifier( requirement: str, ) -> Requirement: """Parse a url requirement to a conda spec""" - requirement_specifier = requirement.split(";")[0].strip() - if ( - requirement_specifier.startswith("git+") - or requirement_specifier.startswith("https://") - or requirement_specifier.startswith("ssh://") + requirement.startswith("git+") + or requirement.startswith("https://") + or requirement.startswith("ssh://") ): - parsed_req = Requirement.parse( - requirement_specifier.split("/")[-1].replace("@", "==") - ) - parsed_req.url = requirement_specifier - return parsed_req - return Requirement.parse(requirement_specifier) + # Handle the case where only the URL is specified without a package name + repo_name_and_maybe_tag = requirement.split("/")[-1] + repo_name = repo_name_and_maybe_tag.split("@")[0] + # Use the repo name as a placeholder for the package name + return Requirement(f"{repo_name} @ {requirement}") + else: + return Requirement(requirement) def unpack_git_url(url: str) -> Tuple[str, Optional[str]]: @@ -407,8 +407,8 @@ def parse_python_requirement( ) -> Dependency: """Parse a requirements.txt like requirement to a conda spec""" parsed_req = parse_requirement_specifier(requirement) - name = parsed_req.unsafe_name.lower() - collapsed_version = ",".join("".join(spec) for spec in parsed_req.specs) + name = canonicalize_pypi_name(parsed_req.name) + collapsed_version = str(parsed_req.specifier) conda_version = poetry_version_to_conda_version(collapsed_version) if conda_version: conda_version = ",".join(sorted(conda_version.split(","))) From cad88fc03f0c1431e8403edd731edcec08c4c1b9 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 12 Jul 2023 13:16:27 +0200 Subject: [PATCH 07/20] Standardize canonicalization of PyPI names --- conda_lock/lockfile/__init__.py | 2 +- conda_lock/lookup.py | 19 +++++++++++-------- conda_lock/pypi_solver.py | 2 +- conda_lock/src_parser/pyproject_toml.py | 16 ++++++++++------ 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index 3d69db1d2..4cc93dea4 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -70,7 +70,7 @@ def dep_name(manager: str, dep: str) -> str: # If we operate on lists of pip names and this is a conda dependency, we # convert the name to a pip name. if convert_to_pip_names and manager == "conda": - return conda_name_to_pypi_name(dep).lower() + return conda_name_to_pypi_name(dep) return dep for name, request in requested.items(): diff --git a/conda_lock/lookup.py b/conda_lock/lookup.py index d13910ed2..a56e1be35 100644 --- a/conda_lock/lookup.py +++ b/conda_lock/lookup.py @@ -4,6 +4,7 @@ import requests import yaml +from packaging.utils import NormalizedName, canonicalize_name from typing_extensions import TypedDict @@ -11,7 +12,7 @@ class MappingEntry(TypedDict): conda_name: str # legacy field, generally not used by anything anymore conda_forge: str - pypi_name: str + pypi_name: NormalizedName class _LookupLoader: @@ -28,15 +29,15 @@ def mapping_url(self, value: str) -> None: self._mapping_url = value @cached_property - def pypi_lookup(self) -> Dict[str, MappingEntry]: + def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]: res = requests.get(self._mapping_url) res.raise_for_status() lookup = yaml.safe_load(res.content) # lowercase and kebabcase the pypi names assert lookup is not None - lookup = {k.lower().replace("_", "-"): v for k, v in lookup.items()} + lookup = {canonicalize_name(k): v for k, v in lookup.items()} for v in lookup.values(): - v["pypi_name"] = v["pypi_name"].lower().replace("_", "-") + v["pypi_name"] = canonicalize_name(v["pypi_name"]) return lookup @cached_property @@ -47,7 +48,7 @@ def conda_lookup(self) -> Dict[str, MappingEntry]: LOOKUP_OBJECT = _LookupLoader() -def get_forward_lookup() -> Dict[str, MappingEntry]: +def get_forward_lookup() -> Dict[NormalizedName, MappingEntry]: global LOOKUP_OBJECT return LOOKUP_OBJECT.pypi_lookup @@ -65,12 +66,14 @@ def set_lookup_location(lookup_url: str) -> None: LOOKUP_OBJECT.mapping_url = lookup_url -def conda_name_to_pypi_name(name: str) -> str: +def conda_name_to_pypi_name(name: str) -> NormalizedName: """return the pypi name for a conda package""" lookup = get_lookup() - return lookup.get(name, {"pypi_name": name})["pypi_name"] + cname = canonicalize_name(name) + return lookup.get(cname, {"pypi_name": cname})["pypi_name"] def pypi_name_to_conda_name(name: str) -> str: """return the conda name for a pypi package""" - return get_forward_lookup().get(name, {"conda_name": name})["conda_name"] + cname = canonicalize_name(name) + return get_forward_lookup().get(cname, {"conda_name": cname})["conda_name"] diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index e29b723f1..56b85d819 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -350,7 +350,7 @@ def solve_pypi( # is essentially a dictionary of: # - pip package name -> list of LockedDependency that are needed for this package for conda_name, locked_dep in conda_locked.items(): - pypi_name = conda_name_to_pypi_name(conda_name).lower() + pypi_name = conda_name_to_pypi_name(conda_name) if pypi_name in planned: planned[pypi_name].append(locked_dep) else: diff --git a/conda_lock/src_parser/pyproject_toml.py b/conda_lock/src_parser/pyproject_toml.py index eb6b596e9..fe440f8b7 100644 --- a/conda_lock/src_parser/pyproject_toml.py +++ b/conda_lock/src_parser/pyproject_toml.py @@ -74,17 +74,19 @@ def join_version_components(pieces: Sequence[Union[str, int]]) -> str: def normalize_pypi_name(name: str) -> str: - name = name.replace("_", "-").lower() - if name in get_lookup(): - lookup = get_lookup()[name] + cname = canonicalize_pypi_name(name) + if cname in get_lookup(): + lookup = get_lookup()[cname] res = lookup.get("conda_name") or lookup.get("conda_forge") if res is not None: return res else: - logging.warning(f"Could not find conda name for {name}. Assuming identity.") - return name + logging.warning( + f"Could not find conda name for {cname}. Assuming identity." + ) + return cname else: - return name + return cname def poetry_version_to_conda_version(version_string: Optional[str]) -> Optional[str]: @@ -377,6 +379,8 @@ def parse_requirement_specifier( # Handle the case where only the URL is specified without a package name repo_name_and_maybe_tag = requirement.split("/")[-1] repo_name = repo_name_and_maybe_tag.split("@")[0] + if repo_name.endswith(".git"): + repo_name = repo_name[:-4] # Use the repo name as a placeholder for the package name return Requirement(f"{repo_name} @ {requirement}") else: From 90bc903149da2724153045b9c6d74626bc640e9e Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Aug 2023 09:08:57 +0200 Subject: [PATCH 08/20] Don't run scheduled relock on forks This should prevent forks from updating their own dependencies twice a week. --- .github/workflows/update-lockfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update-lockfile.yaml b/.github/workflows/update-lockfile.yaml index a54f14b2c..f32574971 100644 --- a/.github/workflows/update-lockfile.yaml +++ b/.github/workflows/update-lockfile.yaml @@ -8,9 +8,12 @@ on: schedule: # At 5:28am UTC Monday and Thursday - cron: 28 5 * * MON,THU + jobs: conda-lock: + # Don't run scheduled job on forks. Ref: + if: (github.event_name == 'schedule' && github.repository == 'conda/conda-lock') || (github.event_name != 'schedule') defaults: run: # Ensure the environment is activated From 6a20fc3f7634c968da77fd81570cca00d6895e58 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 3 Aug 2023 09:10:56 +0200 Subject: [PATCH 09/20] Lint --- .github/workflows/update-lockfile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update-lockfile.yaml b/.github/workflows/update-lockfile.yaml index f32574971..ed6df98e6 100644 --- a/.github/workflows/update-lockfile.yaml +++ b/.github/workflows/update-lockfile.yaml @@ -8,7 +8,6 @@ on: schedule: # At 5:28am UTC Monday and Thursday - cron: 28 5 * * MON,THU - jobs: conda-lock: From d153283edc2942d8b671ae36aa38a44417994157 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Mon, 31 Jul 2023 09:27:14 +0200 Subject: [PATCH 10/20] Remove inapplicable importlib-metadata dependency --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 94e0be225..390b7dd7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,6 @@ dependencies = [ "crashtest >=0.3.0", # poetry: "html5lib >=1.0", - # poetry, poetry-core: - 'importlib-metadata >=1.7.0; python_version <= "3.7"', # poetry: "keyring >=21.2.0", # poetry: From 60cdf7b26e1abf34330b8c443d16f9a4ea9b8ea2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 10 Aug 2023 22:07:13 +0000 Subject: [PATCH 11/20] Update pypa/gh-action-pypi-publish action to v1.8.10 --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 927d61db7..b8ba74834 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -37,7 +37,7 @@ jobs: - name: Publish a Python distribution to PyPI if: ${{ github.event_name == 'release' }} - uses: pypa/gh-action-pypi-publish@v1.8.8 + uses: pypa/gh-action-pypi-publish@v1.8.10 with: user: __token__ password: ${{ secrets.PYPI_PASSWORD }} From 41f5927360a1a05a1616978dd421112f638c2d66 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 16:54:16 +0000 Subject: [PATCH 12/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/flake8: 6.0.0 → 6.1.0](https://github.com/pycqa/flake8/compare/6.0.0...6.1.0) - [github.com/pre-commit/mirrors-mypy: v1.4.1 → v1.5.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.4.1...v1.5.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bde7db27c..a11c4d3e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: language_version: python3 - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 @@ -30,7 +30,7 @@ repos: args: ["--profile", "black", "--filter-files"] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 + rev: v1.5.0 hooks: - id: mypy additional_dependencies: From 1758dc7616829fb74921c0e31922e8e5907371cb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 16:54:50 +0000 Subject: [PATCH 13/20] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v1.5.0 → v1.5.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.5.0...v1.5.1) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a11c4d3e4..8f6c693af 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,7 +30,7 @@ repos: args: ["--profile", "black", "--filter-files"] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.0 + rev: v1.5.1 hooks: - id: mypy additional_dependencies: From e21db500a0a19a9ba4ce5ddc6a28c25d1857922f Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 28 Aug 2023 08:40:00 -0600 Subject: [PATCH 14/20] Switch from flake8 (and isort) to ruff Ruff is fast, and this commit only aims to take advange of that, not to add new rules. --- .flake8 | 5 --- .pre-commit-config.yaml | 19 +++++------- environments/dev-environment.yaml | 6 ---- pyproject.toml | 51 +++++++++++++++++++++++++------ requirements-dev.txt | 7 ----- 5 files changed, 48 insertions(+), 40 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index b61e46d48..000000000 --- a/.flake8 +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -ignore = E203, E266, E501, W503, F403, F401 -max-line-length = 89 -max-complexity = 18 -select = B,C,E,F,W,T4,B9 \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f6c693af..7d51e8ebe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,9 +8,9 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: - - id: trailing-whitespace - exclude: "^.*\\.patch$" - - id: check-ast + - id: trailing-whitespace + exclude: "^.*\\.patch$" + - id: check-ast - repo: https://github.com/psf/black rev: 23.7.0 @@ -18,16 +18,11 @@ repos: - id: black language_version: python3 -- repo: https://github.com/pycqa/flake8 - rev: 6.1.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.286 hooks: - - id: flake8 - -- repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - args: ["--profile", "black", "--filter-files"] + - id: ruff + args: ["--fix"] - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.5.1 diff --git a/environments/dev-environment.yaml b/environments/dev-environment.yaml index 14a05a878..bee5b5a24 100644 --- a/environments/dev-environment.yaml +++ b/environments/dev-environment.yaml @@ -8,19 +8,13 @@ dependencies: - check-manifest - doctr - filelock -- flake8 -- flake8-builtins -- flake8-comprehensions -- flake8-mutable - python-build - freezegun - isort - mypy - pre-commit -- pylint - pytest - pytest-cov -- pytest-flake8 - pytest-xdist - pytest-timeout - tomli diff --git a/pyproject.toml b/pyproject.toml index 390b7dd7c..7de4748f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,16 +108,6 @@ exclude = [ "tests", ] -[tool.isort] -atomic = true -force_grid_wrap = 0 -include_trailing_comma = true -lines_after_imports = 2 -lines_between_types = 1 -multi_line_output = 3 -use_parentheses = true -known_first_party = "attr" - [tool.pytest.ini_options] addopts = "-vrsx -n auto" flake8-max-line-length = 105 @@ -157,3 +147,44 @@ platforms = ["linux-64", "osx-64", "osx-arm64", "win-64", "osx-arm64", "linux-aa # This is necessary to pull in the lockfile/filelock dependency # since we don't handle the optional dependency. cachecontrol-with-filecache = ">=0.12.9" + + +[tool.ruff] +ignore = [ + "E501", + "F401", + "F403", + # Disabled during migration to Ruff: + "A001", + "A002", + "A003", + "C401", + "C405", + "C408", + "C409", + "C413", + "C414", + "C416", + "RUF012", + "RUF015", +] +line-length = 89 +select = [ + "A", # flake8-builtins + # "B", # flake8-bugbear + "C4", # flake8-comprehensions + "C9", # mccabe + "E", # pycodestyle errors + "F", # pyflakes + "I", # isort + "RUF", # ruff rules + "W", # pycodestyle warnings +] + +[tool.ruff.mccabe] +max-complexity = 18 + +[tool.ruff.isort] +lines-after-imports = 2 +lines-between-types = 1 +known-first-party = ["attr"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 62bcf44c4..b4feceec3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,19 +1,12 @@ black check-manifest doctr -flake8 -flake8-builtins -flake8-comprehensions -flake8-mutable build freezegun -isort mypy pre_commit -pylint pytest pytest-cov -pytest-flake8 pytest-xdist pytest-timeout tomli; python_version<"3.11" From f015fdd3f82c821364d4eca27c94ff47ba5eb08a Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Sun, 27 Aug 2023 18:25:53 -0600 Subject: [PATCH 15/20] Autofix a manageable set of new rules These all felt likely to be uncontroversial choices for letting Ruff autofix some new rules. --- conda_lock/conda_lock.py | 6 +++--- conda_lock/conda_solver.py | 2 +- conda_lock/lockfile/v2prelim/models.py | 4 +++- conda_lock/models/channel.py | 2 +- conda_lock/src_parser/meta_yaml.py | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index ff398783d..d8c4e44b7 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -241,7 +241,7 @@ def fn_to_dist_name(fn: str) -> str: return fn -def make_lock_files( # noqa: C901 +def make_lock_files( *, conda: PathLike, src_files: List[pathlib.Path], @@ -943,10 +943,10 @@ def _render_lockfile_for_install( if platform not in lock_content.metadata.platforms: suggested_platforms_section = "platforms:\n- " suggested_platforms_section += "\n- ".join( - [platform] + lock_content.metadata.platforms + [platform, *lock_content.metadata.platforms] ) suggested_platform_args = "--platform=" + " --platform=".join( - [platform] + lock_content.metadata.platforms + [platform, *lock_content.metadata.platforms] ) raise PlatformValidationError( f"The lockfile {filename} does not contain a solution for the current " diff --git a/conda_lock/conda_solver.py b/conda_lock/conda_solver.py index eb9fb6ef6..9c414e640 100644 --- a/conda_lock/conda_solver.py +++ b/conda_lock/conda_solver.py @@ -476,7 +476,7 @@ def update_specs_for_arch( proc = subprocess.run( [ str(arg) - for arg in args + ["-p", prefix, "--json", "--dry-run", *to_update] + for arg in [*args, "-p", prefix, "--json", "--dry-run", *to_update] ], env=conda_env_override(platform), stdout=subprocess.PIPE, diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index 038a7e14e..2cc35c26a 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -7,10 +7,12 @@ GitMeta, HashModel, InputMeta, + LockMeta, + MetadataOption, + TimeMeta, ) from conda_lock.lockfile.v1.models import LockedDependency as LockedDependencyV1 from conda_lock.lockfile.v1.models import Lockfile as LockfileV1 -from conda_lock.lockfile.v1.models import LockMeta, MetadataOption, TimeMeta from conda_lock.models import StrictModel diff --git a/conda_lock/models/channel.py b/conda_lock/models/channel.py index 5cb3b6143..60874f3f4 100644 --- a/conda_lock/models/channel.py +++ b/conda_lock/models/channel.py @@ -143,7 +143,7 @@ def _detect_used_env_var( if value.startswith("$"): return value.lstrip("$").strip("{}") - for suffix in preferred_env_var_suffix + [""]: + for suffix in [*preferred_env_var_suffix, ""]: candidates = {v: k for k, v in os.environ.items() if k.upper().endswith(suffix)} # try first with a simple match key = candidates.get(value) diff --git a/conda_lock/src_parser/meta_yaml.py b/conda_lock/src_parser/meta_yaml.py index 79470027a..5ebd8768f 100644 --- a/conda_lock/src_parser/meta_yaml.py +++ b/conda_lock/src_parser/meta_yaml.py @@ -47,7 +47,7 @@ def __init__( # type: ignore __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \ __complex__ = __pow__ = __rpow__ = \ - lambda self, *args, **kwargs: self._return_undefined(self._undefined_name) # noqa: E122 + lambda self, *args, **kwargs: self._return_undefined(self._undefined_name) # fmt: on # Accessing an attribute of an Undefined variable @@ -60,7 +60,7 @@ def __getattr__(self, k: str) -> "UndefinedNeverFail": # Unlike the methods above, Python requires that these # few methods must always return the correct type - __str__ = __repr__ = lambda self: self._return_value(str()) # type: ignore # noqa: E731 + __str__ = __repr__ = lambda self: self._return_value(str()) # type: ignore __unicode__ = lambda self: self._return_value("") # noqa: E731 __int__ = lambda self: self._return_value(0) # type: ignore # noqa: E731 __float__ = lambda self: self._return_value(0.0) # type: ignore # noqa: E731 From 3aca73e99e92e0cbd5d9784a69ca8e504821b378 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Sun, 27 Aug 2023 18:43:31 -0600 Subject: [PATCH 16/20] Specify target version --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7de4748f4..cce0cc543 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,6 +150,7 @@ cachecontrol-with-filecache = ">=0.12.9" [tool.ruff] +target-version = "py38" ignore = [ "E501", "F401", From dbd131ebae4dad837cef787c63da030f31e6b21a Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Sun, 27 Aug 2023 18:52:30 -0600 Subject: [PATCH 17/20] Enable and autofix B006 --- conda_lock/conda_lock.py | 4 +++- pyproject.toml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index d8c4e44b7..0c96d4c84 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -768,7 +768,7 @@ def create_lockfile_from_spec( *, conda: PathLike, spec: LockSpecification, - platforms: List[str] = [], + platforms: Optional[List[str]] = None, lockfile_path: pathlib.Path, update_spec: Optional[UpdateSpecification] = None, metadata_choices: AbstractSet[MetadataOption] = frozenset(), @@ -778,6 +778,8 @@ def create_lockfile_from_spec( """ Solve or update specification """ + if platforms is None: + platforms = [] assert spec.virtual_package_repo is not None virtual_package_channel = spec.virtual_package_repo.channel diff --git a/pyproject.toml b/pyproject.toml index cce0cc543..b6523232f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -173,6 +173,7 @@ line-length = 89 select = [ "A", # flake8-builtins # "B", # flake8-bugbear + "B006", # Do not use mutable data structures for argument defaults "C4", # flake8-comprehensions "C9", # mccabe "E", # pycodestyle errors From 95f64eafa6724ad5745ac1a7ab0e4f148b8d01ad Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 28 Aug 2023 19:17:02 -0600 Subject: [PATCH 18/20] Restore flake8 and isort in a way that's compatible with ruff I'm sure there are better ways to do this than `# isort: skip_file` and flake8 `per-file-ignores`, but this is what I had time for today :) --- .flake8 | 10 ++++++++++ .pre-commit-config.yaml | 12 ++++++++++++ conda_lock/lockfile/v2prelim/models.py | 5 ++--- pyproject.toml | 14 ++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..684ff8df7 --- /dev/null +++ b/.flake8 @@ -0,0 +1,10 @@ +[flake8] +# NOTE: Can ruff replace flake8? See `tool.isort`, these two configs should be +# kept in sync until we pick one. +ignore = E203, E266, E501, W503, F403, F401 +per-file-ignores = + conda_lock/src_parser/meta_yaml.py:E122 + +max-line-length = 89 +max-complexity = 19 +select = B,C,E,F,W,T4,B9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d51e8ebe..8c45525aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,18 @@ repos: - id: ruff args: ["--fix"] +# Ruff should catch (and mostly fix) everything that flake8 and isort do; if +# either of these checks fails, can Ruff's config be updated to catch the same? +- repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] + - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.5.1 hooks: diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index 2cc35c26a..fd36dbade 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -1,3 +1,4 @@ +# isort: skip_file from collections import defaultdict from typing import ClassVar, Dict, List, Optional @@ -7,12 +8,10 @@ GitMeta, HashModel, InputMeta, - LockMeta, - MetadataOption, - TimeMeta, ) from conda_lock.lockfile.v1.models import LockedDependency as LockedDependencyV1 from conda_lock.lockfile.v1.models import Lockfile as LockfileV1 +from conda_lock.lockfile.v1.models import LockMeta, MetadataOption, TimeMeta from conda_lock.models import StrictModel diff --git a/pyproject.toml b/pyproject.toml index b6523232f..9fb2c0b64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,18 @@ exclude = [ "tests", ] +[tool.isort] +# NOTE: Can ruff replace isort? See `tool.ruff.isort`, these two configs should +# be kept in sync until we pick one. +atomic = true +force_grid_wrap = 0 +include_trailing_comma = true +lines_after_imports = 2 +lines_between_types = 1 +multi_line_output = 3 +use_parentheses = true +known_first_party = "attr" + [tool.pytest.ini_options] addopts = "-vrsx -n auto" flake8-max-line-length = 105 @@ -187,6 +199,8 @@ select = [ max-complexity = 18 [tool.ruff.isort] +# NOTE: Can ruff replace isort? See `tool.isort`, these two configs should be +# kept in sync until we pick one. lines-after-imports = 2 lines-between-types = 1 known-first-party = ["attr"] From 52cb31b69a87ca879c5c3e8228e2125c16932bef Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 29 Aug 2023 18:22:48 -0600 Subject: [PATCH 19/20] Add context to isort skip comment --- conda_lock/lockfile/v2prelim/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index fd36dbade..71ad4352f 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -1,4 +1,6 @@ # isort: skip_file +# TODO: Remove the isort skip comment above if/when isort is no longer used. This skip +# exists because isort and ruff disagree about how to sort the imports in this file. from collections import defaultdict from typing import ClassVar, Dict, List, Optional From dfe68e59dcfa5a28dfbcac89035ffea5a20cb769 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sat, 5 Aug 2023 12:55:13 +0200 Subject: [PATCH 20/20] Remove deprecated allow_mutation flag --- conda_lock/virtual_package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_lock/virtual_package.py b/conda_lock/virtual_package.py index c0f615e9b..aa505efaa 100644 --- a/conda_lock/virtual_package.py +++ b/conda_lock/virtual_package.py @@ -24,7 +24,6 @@ class FakePackage(BaseModel): """A minimal representation of the required metadata for a conda package""" class Config: - allow_mutation = False frozen = True name: str