diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 888253cbae..e80ad31cfd 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -141,6 +141,8 @@ jobs: run: tox -e package-version-checks - name: Check package dependencies run: tox -e package-dependencies-checks + - name: Check dependencies + run: tox -e check-dependencies - name: Check generate protocols run: tox -e check-generate-all-protocols - name: Generate Documentation diff --git a/.pylintrc b/.pylintrc index 6dba6f0946..1946fac307 100644 --- a/.pylintrc +++ b/.pylintrc @@ -36,7 +36,7 @@ disable=C0103,C0201,C0301,C0302,W0105,W0707,W1202,W1203,R0801,E1136,E0611,C0209, # C0206: consider-using-dict-items [IMPORTS] -ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,web3,aioprometheus,pyaes,Crypto,asn1crypto,cosmpy,google,coverage,pylint,pytest,gitpython,protobuf,docker,signal,anchorpy,cryptography.fernet,solana,solders,flashbots,hexbytes +ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,web3,aioprometheus,pyaes,Crypto,asn1crypto,cosmpy,google,coverage,pylint,pytest,gitpython,protobuf,docker,signal,anchorpy,cryptography.fernet,solana,solders,flashbots,hexbytes,toml [DESIGN] min-public-methods=1 diff --git a/Pipfile b/Pipfile index 8145cc2fe3..2423afd6c0 100644 --- a/Pipfile +++ b/Pipfile @@ -8,30 +8,33 @@ url = "https://test.pypi.org/simple" verify_ssl = true name = "test-pypi" +[packages] +# we don't specify dependencies for the library here for intallation as per: https://pipenv-fork.readthedocs.io/en/latest/advanced.html#pipfile-vs-setuppy +# aea and plugin dependencies are specified in setup.py + [dev-packages] # we fix exact versions as it's sufficient to have at least one set of compatible dependencies for development setuptools = "==59.6.0" -aiohttp = ">=3.8.5,<4.0.0" -asn1crypto = "==1.4.0" +aiohttp = "<4.0.0,>=3.8.5" +asn1crypto = "<1.5.0,>=1.4.0" bech32 = "==1.2.0" defusedxml = "==0.6.0" # ^ still used? docker = "==4.2.0" ecdsa = ">=0.15" -eth-account = ">=0.8.0,<0.9.0" +eth-account = "<0.9.0,>=0.8.0" gym = "==0.15.6" hypothesis = "==6.21.6" ipfshttpclient = "==0.8.0a2" liccheck = "==0.6.0" -matplotlib = ">=3.3.0,<3.4" memory-profiler = "==0.57.0" # ^ still used? numpy = ">=1.18.1" openapi-core = "==0.13.2" openapi-spec-validator = "==0.2.8" -packaging = ">=23.1,<24.0" +packaging = "<24.0,>=23.1" pexpect = "==4.8.0" -protobuf = ">=4.21.6,<5.0.0" +protobuf = "<5.0.0,>=4.21.6" psutil = "==5.7.0" pycryptodome = ">=3.10.1" pytest-custom-exit-code = "==0.3.0" @@ -39,7 +42,7 @@ GitPython = "<4.0.0,>=3.1.37" requests = "==2.28.1" idna = "<=3.3" open-aea-cosmpy = "==0.6.7" -web3 = ">=6.0.0,<7" +web3 = "<7,>=6.0.0" semver = "<3.0.0,>=2.9.1" py-multibase = ">=1.0.0" py-multicodec = ">=0.2.0" @@ -52,11 +55,8 @@ docspec-python = "==2.2.1" hexbytes = "==0.3.0" ledgerwallet = "==0.1.3" construct = "<=2.10.61" - -[packages] -# we don't specify dependencies for the library here for intallation as per: https://pipenv-fork.readthedocs.io/en/latest/advanced.html#pipfile-vs-setuppy -# aea and plugin dependencies are specified in setup.py - -# pending upstream releases. -# solana = "==0.29.2" -# anchorpy = {git = "https://github.com/kevinheavey/anchorpy.git@a3cc292574679bae1610e01ab69161b6614bca9"} +werkzeug = "*" +pytest-asyncio = "*" +multidict = "*" +toml = "==0.10.2" +matplotlib = "<3.4,>=3.3.0" diff --git a/aea/configurations/data_types.py b/aea/configurations/data_types.py index 78d4da52e1..91ff0ab406 100644 --- a/aea/configurations/data_types.py +++ b/aea/configurations/data_types.py @@ -19,6 +19,7 @@ # ------------------------------------------------------------------------------ """Base config data types.""" import functools +import json import re from abc import ABC, abstractmethod from enum import Enum @@ -62,8 +63,31 @@ T = TypeVar("T") PackageVersionLike = Union[str, semver.VersionInfo] -PYPI_RE = r"(?P[a-zA-Z0-9_-]+)(?P(>|<|=|~).+)?" -GIT_RE = r"git\+(?Phttps://github.com/([a-z-_0-9A-Z]+\/[a-z-_0-9A-Z]+)\.git)@(?P.+)#egg=(?P.+)" +PACKAGE_NAME_RE = r"[a-zA-Z0-9_-]+" +VERSION_SPECIFIER_RE = r"(>|<|=|~|\*)+.*" +PIP_RE = ( + r"(?P" + + PACKAGE_NAME_RE + + r")(?P\[[,a-zA-Z0-9_-]+\]+)?(?P" + + VERSION_SPECIFIER_RE + + r")?" +) +GIT_RE = ( + r"git\+(?Phttps://github.com/(" + + PACKAGE_NAME_RE + + r"\/" + + PACKAGE_NAME_RE + + r")\.git)@(?P.+)#egg=(?P.+)" +) +PIPFILE_VERSION_ONLY_RE = ( + r"(?P[a-zA-Z0-9_-]+) = \"(?P" + VERSION_SPECIFIER_RE + r")\"" +) +PIPFILE_EXTRA_RE = ( + r"(?P[a-zA-Z0-9_-]+) = \{(version = \"(?P" + + VERSION_SPECIFIER_RE + + r")\"), extras = (?P\[[, a-zA-Z\"]+\])\}" +) +PIPFILE_GIT_RE = r"(?P[a-zA-Z0-9_-]+) = \{(ref = \"(?P[a-zA-Z0-9]+)\"), git = \"git\+(?Phttps:\/\/.*\.git)\"\}" class JSONSerializable(ABC): @@ -781,7 +805,7 @@ class Dependency: These fields will be forwarded to the 'pip' command. """ - __slots__ = ("_name", "_version", "_index", "_git", "_ref") + __slots__ = ("_name", "_version", "_index", "_git", "_ref", "_extras") def __init__( self, @@ -790,6 +814,7 @@ def __init__( index: Optional[str] = None, git: Optional[str] = None, ref: Optional[Union[GitRef, str]] = None, + extras: Optional[List[str]] = None, ) -> None: """ Initialize a PyPI dependency. @@ -799,12 +824,14 @@ def __init__( :param index: the URL to the PyPI server. :param git: the URL to a git repository. :param ref: the Git reference (branch/commit/tag). + :param extras: Include extras section for the dependency. """ self._name: PyPIPackageName = PyPIPackageName(name) self._version: SpecifierSet = self._parse_version(version) self._index: Optional[str] = index self._git: Optional[str] = git self._ref: Optional[GitRef] = GitRef(ref) if ref is not None else None + self._extras = extras or [] @property def name(self) -> str: @@ -831,6 +858,11 @@ def ref(self) -> Optional[str]: """Get the ref.""" return str(self._ref) if self._ref else None + @property + def extras(self) -> List[str]: + """Get the ref.""" + return self._extras + @staticmethod def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet: """ @@ -841,6 +873,37 @@ def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet: """ return version if isinstance(version, SpecifierSet) else SpecifierSet(version) + @classmethod + def from_pipfile_string(cls, string: str) -> "Dependency": + """Parse from Pipfile version specifier.""" + match = re.match(PIPFILE_VERSION_ONLY_RE, string=string) + if match is not None: + data = match.groupdict() + return Dependency( + name=data["name"], + version=data["version"] if data["version"] != "*" else "", + ) + + match = re.match(PIPFILE_EXTRA_RE, string=string) + if match is not None: + data = match.groupdict() + return Dependency( + name=data["name"], + version=data["version"] if data["version"] != "*" else "", + extras=json.loads(data["extras"]), + ) + + match = re.match(PIPFILE_GIT_RE, string=string) + if match is not None: + data = match.groupdict() + return Dependency( + name=data["name"], + git=data["git"], + ref=data["ref"], + ) + + raise ValueError(f"Invalid string provided for Pipfile specifier: {string}") + @classmethod def from_string(cls, string: str) -> "Dependency": """Parse from string.""" @@ -849,11 +912,14 @@ def from_string(cls, string: str) -> "Dependency": data = match.groupdict() return cls(name=data["name"], git=data["git"], ref=data["ref"]) - match = re.match(PYPI_RE, string) + match = re.match(PIP_RE, string) if match is None: raise ValueError(f"Cannot parse the dependency string '{string}'") data = match.groupdict() + extras = [] + if data["extras"] is not None: + extras = re.findall("[a-zA-Z0-9_-]+", data["extras"]) return Dependency( name=data["name"], version=( @@ -861,6 +927,7 @@ def from_string(cls, string: str) -> "Dependency": if data["version"] is not None else SpecifierSet("") ), + extras=extras, ) @classmethod @@ -894,25 +961,51 @@ def to_json(self) -> Dict[str, Dict[str, str]]: result["ref"] = cast(str, self.ref) return {self.name: result} + def to_pip_string(self) -> str: + """To pip specifier.""" + if self.git is not None: + revision = self.ref if self.ref is not None else DEFAULT_GIT_REF + return "git+" + self.git + "@" + revision + "#egg=" + self.name + if len(self.extras) > 0: + return self.name + "[" + ",".join(self.extras) + "]" + str(self.version) + return self.name + str(self.version) + + def to_pipfile_string(self) -> str: + """To Pipfile specifier.""" + if self.git is not None: + string = self.name + " = {" + revision = self.ref if self.ref is not None else DEFAULT_GIT_REF + string += f'ref = "{revision}", ' + string += f'git = "git+{self.git}"' + string += "}" + return string + + if len(self.extras) > 0: + version = self.version if self.version != "" else "*" + string = self.name + " = {" + string += f'version = "{version}", ' + string += "extras = [" + string += ", ".join(map(lambda x: f'"{x}"', self.extras)) + string += "]}" + return string + + version = self.version if self.version != "" else "*" + return f'{self.name} = "{version}"' + def get_pip_install_args(self) -> List[str]: """Get 'pip install' arguments.""" - name = self.name - index = self.index - git_url = self.git - revision = self.ref if self.ref is not None else DEFAULT_GIT_REF - version_constraint = str(self.version) command: List[str] = [] - if index is not None: - command += ["-i", index] - if git_url is not None: - command += ["git+" + git_url + "@" + revision + "#egg=" + name] - else: - command += [name + version_constraint] + if self.index is not None: + command += ["-i", self.index] + command += [self.to_pip_string()] return command def __str__(self) -> str: """Get the string representation.""" - return f"{self.__class__.__name__}(name='{self.name}', version='{self.version}', index='{self.index}', git='{self.git}', ref='{self.ref}')" + return ( + f"{self.__class__.__name__}(name='{self.name}', version='{self.version}', " + f"index='{self.index}', git='{self.git}', ref='{self.ref}', extras='{self.extras}')" + ) def __eq__(self, other: Any) -> bool: """Check equality.""" diff --git a/docs/api/configurations/data_types.md b/docs/api/configurations/data_types.md index 5dc85a362e..c75c754cdb 100644 --- a/docs/api/configurations/data_types.md +++ b/docs/api/configurations/data_types.md @@ -999,7 +999,8 @@ def __init__(name: Union[PyPIPackageName, str], version: Union[str, SpecifierSet] = "", index: Optional[str] = None, git: Optional[str] = None, - ref: Optional[Union[GitRef, str]] = None) -> None + ref: Optional[Union[GitRef, str]] = None, + extras: Optional[List[str]] = None) -> None ``` Initialize a PyPI dependency. @@ -1011,6 +1012,7 @@ Initialize a PyPI dependency. - `index`: the URL to the PyPI server. - `git`: the URL to a git repository. - `ref`: the Git reference (branch/commit/tag). +- `extras`: Include extras section for the dependency. @@ -1067,6 +1069,28 @@ def ref() -> Optional[str] Get the ref. + + +#### extras + +```python +@property +def extras() -> List[str] +``` + +Get the ref. + + + +#### from`_`pipfile`_`string + +```python +@classmethod +def from_pipfile_string(cls, string: str) -> "Dependency" +``` + +Parse from Pipfile version specifier. + #### from`_`string @@ -1099,6 +1123,26 @@ def to_json() -> Dict[str, Dict[str, str]] Transform the object to JSON. + + +#### to`_`pip`_`string + +```python +def to_pip_string() -> str +``` + +To pip specifier. + + + +#### to`_`pipfile`_`string + +```python +def to_pipfile_string() -> str +``` + +To Pipfile specifier. + #### get`_`pip`_`install`_`args diff --git a/scripts/check_dependencies.py b/scripts/check_dependencies.py new file mode 100755 index 0000000000..4daf76908c --- /dev/null +++ b/scripts/check_dependencies.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------ +# +# Copyright 2023 Valory AG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ +""" +This script checks that the pipfile of the repository meets the requirements. + +In particular: +- Avoid the usage of "*" + +It is assumed the script is run from the repository root. +""" + +import itertools +import logging +import re +import sys +from collections import OrderedDict +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional +from typing import OrderedDict as OrderedDictType +from typing import Tuple, cast + +import click +import toml + +from aea.configurations.data_types import Dependency +from aea.package_manager.base import load_configuration +from aea.package_manager.v1 import PackageManagerV1 + + +ANY_SPECIFIER = "*" + + +class PathArgument(click.Path): + """Path parameter for CLI.""" + + def convert( + self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context] + ) -> Optional[Path]: + """Convert path string to `pathlib.Path`""" + path_string = super().convert(value, param, ctx) + return None if path_string is None else Path(path_string) + + +class Pipfile: + """Class to represent Pipfile config.""" + + ignore = [ + "open-aea-ledger-cosmos", + "open-aea-ledger-ethereum", + "open-aea-ledger-fetchai", + "open-aea-flashbots", + "open-aea-flashbots", + "tomte", + ] + + def __init__( + self, + sources: List[str], + packages: OrderedDictType[str, Dependency], + dev_packages: OrderedDictType[str, Dependency], + file: Path, + ) -> None: + """Initialize object.""" + self.sources = sources + self.packages = packages + self.dev_packages = dev_packages + self.file = file + + def __iter__(self) -> Iterator[Dependency]: + """Iterate dependencies as from aea.configurations.data_types.Dependency object.""" + for name, dependency in itertools.chain( + self.packages.items(), self.dev_packages.items() + ): + if name.startswith("comment_") or name in self.ignore: + continue + yield dependency + + def update(self, dependency: Dependency) -> None: + """Update dependency specifier""" + if dependency.name in self.ignore: + return + if dependency.name in self.packages: + if dependency.version == "": + return + self.packages[dependency.name] = dependency + else: + self.dev_packages[dependency.name] = dependency + + def check(self, dependency: Dependency) -> Tuple[Optional[str], int]: + """Check dependency specifier""" + if dependency.name in self.ignore: + return None, 0 + + if dependency.name in self.packages: + expected = self.packages[dependency.name] + if expected != dependency: + return ( + f"in Pipfile {expected.get_pip_install_args()[0]}; " + f"got {dependency.get_pip_install_args()[0]}" + ), logging.WARNING + return None, 0 + + if dependency.name not in self.dev_packages: + return f"{dependency.name} not found in Pipfile", logging.ERROR + + expected = self.dev_packages[dependency.name] + if expected != dependency: + return ( + f"in Pipfile {expected.get_pip_install_args()[0]}; " + f"got {dependency.get_pip_install_args()[0]}" + ), logging.WARNING + + return None, 0 + + @classmethod + def parse( + cls, content: str + ) -> Tuple[List[str], OrderedDictType[str, OrderedDictType[str, Dependency]]]: + """Parse from string.""" + sources = [] + sections: OrderedDictType = OrderedDict() + lines = content.split("\n") + comments = 0 + while len(lines) > 0: + line = lines.pop(0) + if "[[source]]" in line: + source = line + "\n" + while True: + line = lines.pop(0) + if line == "": + break + source += line + "\n" + sources.append(source) + if "[dev-packages]" in line or "[packages]" in line: + section = line + sections[section] = OrderedDict() + while len(lines) > 0: + line = lines.pop(0).strip() + if line == "": + break + if line.startswith("#"): + sections[section][f"comment_{comments}"] = line + comments += 1 + else: + dep = Dependency.from_pipfile_string(line) + sections[section][dep.name] = dep + return sources, sections + + def compile(self) -> str: + """Compile to Pipfile string.""" + content = "" + for source in self.sources: + content += source + "\n" + + content += "[packages]\n" + for package, dep in self.packages.items(): + if package.startswith("comment"): + content += str(dep) + "\n" + else: + content += dep.to_pipfile_string() + "\n" + + content += "\n[dev-packages]\n" + for package, dep in self.dev_packages.items(): + if package.startswith("comment"): + content += str(dep) + "\n" + else: + content += dep.to_pipfile_string() + "\n" + return content + + @classmethod + def load(cls, file: Path) -> "Pipfile": + """Load from file.""" + sources, sections = cls.parse( + content=file.read_text(encoding="utf-8"), + ) + return cls( + sources=sources, + packages=sections.get("[packages]", OrderedDict()), + dev_packages=sections.get("[dev-packages]", OrderedDict()), + file=file, + ) + + def dump(self) -> None: + """Write to Pipfile.""" + self.file.write_text(self.compile(), encoding="utf-8") + + +class ToxFile: + """Class to represent tox.ini file.""" + + skip = [ + "open-aea-ledger-cosmos", + "open-aea-ledger-ethereum", + "open-aea-ledger-fetchai", + ] + + def __init__( + self, + dependencies: Dict[str, Dict[str, Any]], + file: Path, + ) -> None: + """Initialize object.""" + self.dependencies = dependencies + self.file = file + self.extra: Dict[str, Dependency] = {} + + def __iter__(self) -> Iterator[Dependency]: + """Iter dependencies.""" + for obj in self.dependencies.values(): + yield obj["dep"] + + def update(self, dependency: Dependency) -> None: + """Update dependency specifier""" + if dependency.name in self.skip: + return + if dependency.name in self.dependencies: + if dependency.version == "": + return + self.dependencies[dependency.name]["dep"] = dependency + return + self.extra[dependency.name] = dependency + + def check(self, dependency: Dependency) -> Tuple[Optional[str], int]: + """Check dependency specifier""" + if dependency.name in self.skip: + return None, 0 + + if dependency.name in self.dependencies: + expected = self.dependencies[dependency.name]["dep"] + if expected != dependency: + return ( + f"in tox.ini {expected.get_pip_install_args()[0]}; " + f"got {dependency.get_pip_install_args()[0]}" + ), logging.WARNING + return None, 0 + return f"{dependency.name} not found in tox.ini", logging.ERROR + + @classmethod + def parse(cls, content: str) -> Dict[str, Dict[str, Any]]: + """Parse file content.""" + deps = {} + lines = content.split("\n") + while len(lines) > 0: + line = lines.pop(0) + if line.startswith("deps"): + while True: + line = lines.pop(0) + if not line.startswith(" "): + break + if ( + line.startswith(" {") + or line.startswith(" ;") + or line.strip() == "" + or "tomte" in line + ): + continue + dep = Dependency.from_string(line.lstrip()) + deps[dep.name] = { + "original": line, + "dep": dep, + } + return deps + + @classmethod + def load(cls, file: Path) -> "ToxFile": + """Load tox.ini file.""" + content = file.read_text(encoding="utf-8") + dependencies = cls.parse(content=content) + return cls( + dependencies=dependencies, + file=file, + ) + + def _include_extra(self, content: str) -> str: + """Include extra dependencies.""" + lines = content.split("\n") + extra = [] + for dep in self.extra.values(): + extra.append(f" {dep.get_pip_install_args()[0]}") + + if "[extra-deps]" in lines: + start_idx = lines.index("[extra-deps]") + 2 + end_idx = lines.index("; end-extra") + extra = list(sorted(set(extra + lines[start_idx:end_idx]))) + lines = lines[:start_idx] + extra + lines[end_idx:] + else: + idx = lines.index("[testenv]") + lines = [ + *lines[:idx], + "[extra-deps]", + "deps = ", + *list(sorted(extra)), + "; end-extra\n", + *lines[idx:], + ] + + return "\n".join(lines) + + def write(self) -> None: + """Dump config.""" + content = self.file.read_text(encoding="utf-8") + for obj in self.dependencies.values(): + replace = " " + cast(Dependency, obj["dep"]).get_pip_install_args()[0] + content = re.sub(obj["original"], replace, content) + + if len(self.extra) > 0: + content = self._include_extra(content=content) + + self.file.write_text(content, encoding="utf-8") + + +class PyProjectToml: + """Class to represent pyproject.toml file.""" + + ignore = [ + "python", + ] + + def __init__( + self, + dependencies: OrderedDictType[str, Dependency], + config: Dict[str, Dict], + file: Path, + ) -> None: + """Initialize object.""" + self.dependencies = dependencies + self.config = config + self.file = file + + def __iter__(self) -> Iterator[Dependency]: + """Iterate dependencies as from aea.configurations.data_types.Dependency object.""" + for dependency in self.dependencies.values(): + if dependency.name not in self.ignore: + yield dependency + + def update(self, dependency: Dependency) -> None: + """Update dependency specifier""" + if dependency.name in self.ignore: + return + if dependency.name in self.dependencies and dependency.version == "": + return + self.dependencies[dependency.name] = dependency + + def check(self, dependency: Dependency) -> Tuple[Optional[str], int]: + """Check dependency specifier""" + if dependency.name in self.ignore: + return None, 0 + + if dependency.name not in self.dependencies: + return f"{dependency.name} not found in pyproject.toml", logging.ERROR + + expected = self.dependencies[dependency.name] + if expected != dependency: + return ( + f"in pyproject.toml {expected.get_pip_install_args()[0]}; " + f"got {dependency.get_pip_install_args()[0]}" + ), logging.WARNING + + return None, 0 + + @classmethod + def load(cls, pyproject_path: Path) -> Optional["PyProjectToml"]: + """Load pyproject.yaml dependencies""" + config = toml.load(pyproject_path) + dependencies = OrderedDict() + try: + config["tool"]["poetry"]["dependencies"] + except KeyError: + return None + for name, version in config["tool"]["poetry"]["dependencies"].items(): + if isinstance(version, str): + dependencies[name] = Dependency( + name=name, + version=version.replace("^", "==") if version != "*" else "", + ) + continue + data = cast(Dict, version) + if "extras" in data: + version = data["version"] + if re.match(r"^\d", version): + version = f"=={version}" + dependencies[name] = Dependency( + name=name, + version=version, + extras=data["extras"], + ) + continue + + return cls( + dependencies=dependencies, + config=config, + file=pyproject_path, + ) + + def dump(self) -> None: + """Dump to file.""" + self.config["tool"]["poetry"]["dependencies"] = { + package.name: package.version if package.version != "" else "*" + for package in self.dependencies.values() + } + with self.file.open("w") as fp: + toml.dump(self.config, fp) + + +def load_packages_dependencies(packages_dir: Path) -> List[Dependency]: + """Returns a list of package dependencies.""" + package_manager = PackageManagerV1.from_dir(packages_dir=packages_dir) + dependencies: Dict[str, Dependency] = {} + for package in package_manager.iter_dependency_tree(): + if package.package_type.value == "service": + continue + _dependencies = load_configuration( # type: ignore + package_type=package.package_type, + package_path=package_manager.package_path_from_package_id( + package_id=package + ), + ).dependencies + for key, value in _dependencies.items(): + if key not in dependencies: + dependencies[key] = value + else: + if value.version == "": + continue + if dependencies[key].version == "": + dependencies[key] = value + if value == dependencies[key]: + continue + print( + f"Non-matching dependency versions for {key}: {value} vs {dependencies[key]}" + ) + + return list(dependencies.values()) + + +def _update( + packages_dependencies: List[Dependency], + tox: ToxFile, + pipfile: Optional[Pipfile] = None, + pyproject: Optional[PyProjectToml] = None, +) -> None: + """Update dependencies.""" + + if pipfile is not None: + for dependency in packages_dependencies: + pipfile.update(dependency=dependency) + + for dependency in pipfile: + tox.update(dependency=dependency) + + for dependency in tox: + pipfile.update(dependency=dependency) + + pipfile.dump() + + if pyproject is not None: + for dependency in packages_dependencies: + pyproject.update(dependency=dependency) + + for dependency in pyproject: + tox.update(dependency=dependency) + + for dependency in tox: + pyproject.update(dependency=dependency) + + pyproject.dump() + + tox.write() + + +def _check( + packages_dependencies: List[Dependency], + tox: ToxFile, + pipfile: Optional[Pipfile] = None, + pyproject: Optional[PyProjectToml] = None, +) -> None: + """Update dependencies.""" + + fail_check = 0 + + if pipfile is not None: + print("Comparing dependencies from Pipfile and packages") + for dependency in packages_dependencies: + error, level = pipfile.check(dependency=dependency) + if error is not None: + logging.log(level=level, msg=error) + fail_check = level or fail_check + + print("Comparing dependencies from tox and Pipfile") + for dependency in pipfile: + error, level = tox.check(dependency=dependency) + if error is not None: + logging.log(level=level, msg=error) + fail_check = level or fail_check + + print("Comparing dependencies from Pipfile and tox") + for dependency in tox: + error, level = pipfile.check(dependency=dependency) + if error is not None: + logging.log(level=level, msg=error) + fail_check = level or fail_check + + if pyproject is not None: + print("Comparing dependencies from pyproject.toml and packages") + for dependency in packages_dependencies: + error, level = pyproject.check(dependency=dependency) + if error is not None: + logging.log(level=level, msg=error) + fail_check = level or fail_check + + print("Comparing dependencies from pyproject.toml and tox") + for dependency in pyproject: + error, level = tox.check(dependency=dependency) + if error is not None: + logging.log(level=level, msg=error) + fail_check = level or fail_check + + print("Comparing dependencies from tox and pyproject.toml") + for dependency in tox: + error, level = pyproject.check(dependency=dependency) + if error is not None: + logging.log(level=level, msg=error) + fail_check = level or fail_check + + print("Comparing dependencies from tox and packages") + for dependency in packages_dependencies: + error, level = tox.check(dependency=dependency) + if error is not None: + logging.log(level=level, msg=error) + fail_check = level or fail_check + + if fail_check == logging.ERROR: + print("Dependencies check failed") + sys.exit(1) + + if fail_check == logging.WARNING: + print("Please address warnings to avoid errors") + sys.exit(0) + + print("No issues found") + + +@click.command(name="dm") +@click.option( + "--check", + is_flag=True, + help="Perform dependency checks.", +) +@click.option( + "--packages", + "packages_dir", + type=PathArgument( + exists=True, + file_okay=False, + dir_okay=True, + ), + help="Path of the packages directory.", +) +@click.option( + "--tox", + "tox_path", + type=PathArgument( + exists=True, + file_okay=True, + dir_okay=False, + ), + help="Tox config path.", +) +@click.option( + "--pipfile", + "pipfile_path", + type=PathArgument( + exists=True, + file_okay=True, + dir_okay=False, + ), + help="Pipfile path.", +) +@click.option( + "--pyproject", + "pyproject_path", + type=PathArgument( + exists=True, + file_okay=True, + dir_okay=False, + ), + help="Pipfile path.", +) +def main( + check: bool = False, + packages_dir: Optional[Path] = None, + tox_path: Optional[Path] = None, + pipfile_path: Optional[Path] = None, + pyproject_path: Optional[Path] = None, +) -> None: + """Check dependencies across packages, tox.ini, pyproject.toml and setup.py""" + + logging.basicConfig(format="- %(levelname)s: %(message)s") + + tox_path = tox_path or Path.cwd() / "tox.ini" + tox = ToxFile.load(tox_path) + + pipfile_path = pipfile_path or Path.cwd() / "Pipfile" + pipfile = Pipfile.load(pipfile_path) if pipfile_path.exists() else None + + pyproject_path = pyproject_path or Path.cwd() / "pyproject.toml" + pyproject = PyProjectToml.load(pyproject_path) if pyproject_path.exists() else None + + packages_dir = packages_dir or Path.cwd() / "packages" + packages_dependencies = load_packages_dependencies(packages_dir=packages_dir) + + if check: + return _check( + tox=tox, + pipfile=pipfile, + pyproject=pyproject, + packages_dependencies=packages_dependencies, + ) + + return _update( + tox=tox, + pipfile=pipfile, + pyproject=pyproject, + packages_dependencies=packages_dependencies, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/whitelist.py b/scripts/whitelist.py index 5ab033a0f8..7a62f1b4ad 100644 --- a/scripts/whitelist.py +++ b/scripts/whitelist.py @@ -339,3 +339,5 @@ bundle_and_send # unused function (aea/crypto/base.py:432) raw_signed_transactions # unused variable (aea/crypto/base.py:434) target_blocks # unused variable (aea/crypto/base.py:435) +from_pipfile_string # unused method (aea/configurations/data_types.py:876) +to_pipfile_string # unused method (aea/configurations/data_types.py:973) diff --git a/setup.cfg b/setup.cfg index ab2836a709..fd543d66d4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -321,6 +321,9 @@ ignore_missing_imports=True [mypy-asyncio.*] ignore_missing_imports=True +[mypy-toml.*] +ignore_missing_imports=True + [darglint] docstring_style=sphinx strictness=short diff --git a/tests/test_configurations/test_base.py b/tests/test_configurations/test_base.py index c58381e4e7..c2e3336956 100644 --- a/tests/test_configurations/test_base.py +++ b/tests/test_configurations/test_base.py @@ -1100,7 +1100,7 @@ def test_dependency_to_string(): ) assert ( str(dependency) - == "Dependency(name='package_1', version='==0.1.0', index='https://index.com', git='https://some-repo.git', ref='branch')" + == "Dependency(name='package_1', version='==0.1.0', index='https://index.com', git='https://some-repo.git', ref='branch', extras='[]')" ) diff --git a/tests/test_configurations/test_data_types.py b/tests/test_configurations/test_data_types.py index 1ed11b5395..7db4396a13 100644 --- a/tests/test_configurations/test_data_types.py +++ b/tests/test_configurations/test_data_types.py @@ -157,3 +157,65 @@ def test_any_latest_and_numeric_unequal(version_like_pair: Tuple[str]): for f in funcs: with pytest.raises(TypeError, match="not supported between"): assert f(self, other) + + +class TestDependency: + """Test Dependency class.""" + + def test_parse_from_string(self) -> None: + """Test from_string method.""" + + string = "tomte==0.2.13" + dep = Dependency.from_string(string=string) + assert dep.name == "tomte" + assert str(dep.version) == "==0.2.13" + assert dep.to_pip_string() == string + + def test_parse_from_string_wth_extras(self) -> None: + """Test from_string method.""" + + string = "tomte[tox,tests]==0.2.13" + dep = Dependency.from_string(string=string) + assert dep.name == "tomte" + assert str(dep.version) == "==0.2.13" + assert dep.extras == ["tox", "tests"] + assert dep.to_pip_string() == string + + def test_parse_from_git_string(self) -> None: + """Test from_string method.""" + + string = "git+https://github.com/valory-xyz/tomte.git@03adee2a0a5681c6d8e77160be27edffaa8e3bee#egg=tomte" + dep = Dependency.from_string(string=string) + assert dep.name == "tomte" + assert str(dep.ref) == "03adee2a0a5681c6d8e77160be27edffaa8e3bee" + assert dep.git == "https://github.com/valory-xyz/tomte.git" + assert dep.to_pip_string() == string + + def test_parse_from_pipfile_specifier(self) -> None: + """Test from_pipfile_specifier method""" + + string = 'tomte = "==0.2.13"' + dep = Dependency.from_pipfile_string(string) + assert dep.name == "tomte" + assert str(dep.version) == "==0.2.13" + assert dep.to_pipfile_string() == string + + def test_parse_from_pipfile_specifier_with_extras(self) -> None: + """Test from_pipfile_specifier method""" + + string = 'tomte = {version = "==0.2.13", extras = ["tox", "tests"]}' + dep = Dependency.from_pipfile_string(string) + assert dep.name == "tomte" + assert str(dep.version) == "==0.2.13" + assert dep.extras == ["tox", "tests"] + assert dep.to_pipfile_string() == string + + def test_parse_from_pipfile_specifier_with_git_ref(self) -> None: + """Test from_pipfile_specifier method""" + + string = 'tomte = {ref = "03adee2a0a5681c6d8e77160be27edffaa8e3bee", git = "git+https://github.com/valory-xyz/tomte.git"}' + dep = Dependency.from_pipfile_string(string) + assert dep.name == "tomte" + assert str(dep.ref) == "03adee2a0a5681c6d8e77160be27edffaa8e3bee" + assert dep.git == "https://github.com/valory-xyz/tomte.git" + assert dep.to_pipfile_string() == string diff --git a/tox.ini b/tox.ini index 4e562ed054..e53e5627d4 100644 --- a/tox.ini +++ b/tox.ini @@ -13,13 +13,13 @@ skip_missing_interpreters = true [packages-deps] deps = gym==0.15.6 - aiohttp>=3.8.5,<4.0.0 + aiohttp<4.0.0,>=3.8.5 gym==0.15.6 hypothesis==6.21.6 numpy>=1.18.1 openapi-core==0.13.2 openapi-spec-validator==0.2.8 - asn1crypto==1.4.0 + asn1crypto<1.5.0,>=1.4.0 tomte[tests]==0.2.13 [tests-common] @@ -28,10 +28,10 @@ deps = docker==4.2.0 pexpect==4.8.0 GitPython<4.0.0,>=3.1.37 - packaging>=23.1,<24.0 + packaging<24.0,>=23.1 py-multibase>=1.0.0 py-multicodec>=0.2.0 - protobuf>=4.21.6,<5.0.0 + protobuf<5.0.0,>=4.21.6 requests==2.28.1 mistune==2.0.3 tomte[isort]==0.2.13 @@ -41,12 +41,12 @@ deps = ; because we use --no-deps to install the plugins. ; aea_ledger_cosmos/aea_ledger_fetchai ecdsa>=0.15 - asn1crypto==1.4.0 + asn1crypto<1.5.0,>=1.4.0 bech32==1.2.0 ; aea_ledger_ethereum - web3>=6.0.0,<7 + web3<7,>=6.0.0 ipfshttpclient==0.8.0a2 - eth-account>=0.8.0,<0.9.0 + eth-account<0.9.0,>=0.8.0 ; for password encryption in cosmos pycryptodome>=3.10.1 open-aea-cosmpy==0.6.7 @@ -59,8 +59,17 @@ deps = ledgerwallet==0.1.3 construct<=2.10.61 defusedxml==0.6.0 - semver>=2.9.1,<3.0.0 + semver<3.0.0,>=2.9.1 +[extra-deps] +deps = + matplotlib<3.4,>=3.3.0 + memory-profiler==0.57.0 + multidict + pytest-asyncio + werkzeug +; end-extra + [testenv] basepython = python3.10 whitelist_externals = /bin/sh @@ -400,8 +409,7 @@ commands = skipsdist = True skip_install = True passenv = * -deps = - .[all] +deps = .[all] whitelist_externals = /bin/sh commands = - /bin/sh -c "rm -fr ./*private_key.txt" @@ -469,3 +477,13 @@ deps = docspec==2.2.1 docspec-python==2.2.1 commands = {toxinidir}/scripts/generate_api_docs.py + +[testenv:update-dependencies] +deps = + toml==0.10.2 +commands = {toxinidir}/scripts/check_dependencies.py + +[testenv:check-dependencies] +deps = + toml==0.10.2 +commands = {toxinidir}/scripts/check_dependencies.py --check