Merge pull request #687 from valory-xyz/feat/dependency-management

Dependency management

.github/workflows/workflow.yml

@@ -141,6 +141,8 @@ jobs:
         run: tox -e package-version-checks
       - name: Check package dependencies
         run: tox -e package-dependencies-checks
+      - name: Check dependencies
+        run: tox -e check-dependencies
       - name: Check generate protocols
         run: tox -e check-generate-all-protocols
       - name: Generate Documentation

.pylintrc

@@ -36,7 +36,7 @@ disable=C0103,C0201,C0301,C0302,W0105,W0707,W1202,W1203,R0801,E1136,E0611,C0209,
 # C0206: consider-using-dict-items
 
 [IMPORTS]
-ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,web3,aioprometheus,pyaes,Crypto,asn1crypto,cosmpy,google,coverage,pylint,pytest,gitpython,protobuf,docker,signal,anchorpy,cryptography.fernet,solana,solders,flashbots,hexbytes
+ignored-modules=bech32,ecdsa,lru,eth_typing,eth_keys,eth_account,ipfshttpclient,werkzeug,openapi_spec_validator,aiohttp,multidict,yoti_python_sdk,defusedxml,gym,fetch,matplotlib,memory_profiler,numpy,oef,openapi_core,psutil,tensorflow,temper,skimage,web3,aioprometheus,pyaes,Crypto,asn1crypto,cosmpy,google,coverage,pylint,pytest,gitpython,protobuf,docker,signal,anchorpy,cryptography.fernet,solana,solders,flashbots,hexbytes,toml
 
 [DESIGN]
 min-public-methods=1

Pipfile

@@ -8,38 +8,41 @@ url = "https://test.pypi.org/simple"
 verify_ssl = true
 name = "test-pypi"
 
+[packages]
+# we don't specify dependencies for the library here for intallation as per: https://pipenv-fork.readthedocs.io/en/latest/advanced.html#pipfile-vs-setuppy
+# aea and plugin dependencies are specified in setup.py
+
 [dev-packages]
 # we fix exact versions as it's sufficient to have at least one set of compatible dependencies for development
 setuptools = "==59.6.0"
-aiohttp = ">=3.8.5,<4.0.0"
-asn1crypto = "==1.4.0"
+aiohttp = "<4.0.0,>=3.8.5"
+asn1crypto = "<1.5.0,>=1.4.0"
 bech32 = "==1.2.0"
 defusedxml = "==0.6.0"
 # ^ still used?
 docker = "==4.2.0"
 ecdsa = ">=0.15"
-eth-account = ">=0.8.0,<0.9.0"
+eth-account = "<0.9.0,>=0.8.0"
 gym = "==0.15.6"
 hypothesis = "==6.21.6"
 ipfshttpclient = "==0.8.0a2"
 liccheck = "==0.6.0"
-matplotlib = ">=3.3.0,<3.4"
 memory-profiler = "==0.57.0"
 # ^ still used?
 numpy = ">=1.18.1"
 openapi-core = "==0.13.2"
 openapi-spec-validator = "==0.2.8"
-packaging = ">=23.1,<24.0"
+packaging = "<24.0,>=23.1"
 pexpect = "==4.8.0"
-protobuf = ">=4.21.6,<5.0.0"
+protobuf = "<5.0.0,>=4.21.6"
 psutil = "==5.7.0"
 pycryptodome = ">=3.10.1"
 pytest-custom-exit-code = "==0.3.0"
 GitPython = "<4.0.0,>=3.1.37"
 requests = "==2.28.1"
 idna = "<=3.3"
 open-aea-cosmpy = "==0.6.7"
-web3 = ">=6.0.0,<7"
+web3 = "<7,>=6.0.0"
 semver = "<3.0.0,>=2.9.1"
 py-multibase = ">=1.0.0"
 py-multicodec = ">=0.2.0"
@@ -52,11 +55,8 @@ docspec-python = "==2.2.1"
 hexbytes = "==0.3.0"
 ledgerwallet = "==0.1.3"
 construct = "<=2.10.61"
-
-[packages]
-# we don't specify dependencies for the library here for intallation as per: https://pipenv-fork.readthedocs.io/en/latest/advanced.html#pipfile-vs-setuppy
-# aea and plugin dependencies are specified in setup.py
-
-# pending upstream releases.
-# solana = "==0.29.2"
-# anchorpy = {git = "https://github.com/kevinheavey/anchorpy.git@a3cc292574679bae1610e01ab69161b6614bca9"}
+werkzeug = "*"
+pytest-asyncio = "*"
+multidict = "*"
+toml = "==0.10.2"
+matplotlib = "<3.4,>=3.3.0"

aea/configurations/data_types.py

@@ -19,6 +19,7 @@
 # ------------------------------------------------------------------------------
 """Base config data types."""
 import functools
+import json
 import re
 from abc import ABC, abstractmethod
 from enum import Enum
@@ -62,8 +63,31 @@
 T = TypeVar("T")
 PackageVersionLike = Union[str, semver.VersionInfo]
 
-PYPI_RE = r"(?P<name>[a-zA-Z0-9_-]+)(?P<version>(>|<|=|~).+)?"
-GIT_RE = r"git\+(?P<git>https://github.com/([a-z-_0-9A-Z]+\/[a-z-_0-9A-Z]+)\.git)@(?P<ref>.+)#egg=(?P<name>.+)"
+PACKAGE_NAME_RE = r"[a-zA-Z0-9_-]+"
+VERSION_SPECIFIER_RE = r"(>|<|=|~|\*)+.*"
+PIP_RE = (
+    r"(?P<name>"
+    + PACKAGE_NAME_RE
+    + r")(?P<extras>\[[,a-zA-Z0-9_-]+\]+)?(?P<version>"
+    + VERSION_SPECIFIER_RE
+    + r")?"
+)
+GIT_RE = (
+    r"git\+(?P<git>https://github.com/("
+    + PACKAGE_NAME_RE
+    + r"\/"
+    + PACKAGE_NAME_RE
+    + r")\.git)@(?P<ref>.+)#egg=(?P<name>.+)"
+)
+PIPFILE_VERSION_ONLY_RE = (
+    r"(?P<name>[a-zA-Z0-9_-]+) = \"(?P<version>" + VERSION_SPECIFIER_RE + r")\""
+)
+PIPFILE_EXTRA_RE = (
+    r"(?P<name>[a-zA-Z0-9_-]+) = \{(version = \"(?P<version>"
+    + VERSION_SPECIFIER_RE
+    + r")\"), extras = (?P<extras>\[[, a-zA-Z\"]+\])\}"
+)
+PIPFILE_GIT_RE = r"(?P<name>[a-zA-Z0-9_-]+) = \{(ref = \"(?P<ref>[a-zA-Z0-9]+)\"), git = \"git\+(?P<git>https:\/\/.*\.git)\"\}"
 
 
 class JSONSerializable(ABC):
@@ -781,7 +805,7 @@ class Dependency:
     These fields will be forwarded to the 'pip' command.
     """
 
-    __slots__ = ("_name", "_version", "_index", "_git", "_ref")
+    __slots__ = ("_name", "_version", "_index", "_git", "_ref", "_extras")
 
     def __init__(
         self,
@@ -790,6 +814,7 @@ def __init__(
         index: Optional[str] = None,
         git: Optional[str] = None,
         ref: Optional[Union[GitRef, str]] = None,
+        extras: Optional[List[str]] = None,
     ) -> None:
         """
         Initialize a PyPI dependency.
@@ -799,12 +824,14 @@ def __init__(
         :param index: the URL to the PyPI server.
         :param git: the URL to a git repository.
         :param ref: the Git reference (branch/commit/tag).
+        :param extras: Include extras section for the dependency.
         """
         self._name: PyPIPackageName = PyPIPackageName(name)
         self._version: SpecifierSet = self._parse_version(version)
         self._index: Optional[str] = index
         self._git: Optional[str] = git
         self._ref: Optional[GitRef] = GitRef(ref) if ref is not None else None
+        self._extras = extras or []
 
     @property
     def name(self) -> str:
@@ -831,6 +858,11 @@ def ref(self) -> Optional[str]:
         """Get the ref."""
         return str(self._ref) if self._ref else None
 
+    @property
+    def extras(self) -> List[str]:
+        """Get the ref."""
+        return self._extras
+
     @staticmethod
     def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet:
         """
@@ -841,6 +873,37 @@ def _parse_version(version: Union[str, SpecifierSet]) -> SpecifierSet:
         """
         return version if isinstance(version, SpecifierSet) else SpecifierSet(version)
 
+    @classmethod
+    def from_pipfile_string(cls, string: str) -> "Dependency":
+        """Parse from Pipfile version specifier."""
+        match = re.match(PIPFILE_VERSION_ONLY_RE, string=string)
+        if match is not None:
+            data = match.groupdict()
+            return Dependency(
+                name=data["name"],
+                version=data["version"] if data["version"] != "*" else "",
+            )
+
+        match = re.match(PIPFILE_EXTRA_RE, string=string)
+        if match is not None:
+            data = match.groupdict()
+            return Dependency(
+                name=data["name"],
+                version=data["version"] if data["version"] != "*" else "",
+                extras=json.loads(data["extras"]),
+            )
+
+        match = re.match(PIPFILE_GIT_RE, string=string)
+        if match is not None:
+            data = match.groupdict()
+            return Dependency(
+                name=data["name"],
+                git=data["git"],
+                ref=data["ref"],
+            )
+
+        raise ValueError(f"Invalid string provided for Pipfile specifier: {string}")
+
     @classmethod
     def from_string(cls, string: str) -> "Dependency":
         """Parse from string."""
@@ -849,18 +912,22 @@ def from_string(cls, string: str) -> "Dependency":
             data = match.groupdict()
             return cls(name=data["name"], git=data["git"], ref=data["ref"])
 
-        match = re.match(PYPI_RE, string)
+        match = re.match(PIP_RE, string)
         if match is None:
             raise ValueError(f"Cannot parse the dependency string '{string}'")
 
         data = match.groupdict()
+        extras = []
+        if data["extras"] is not None:
+            extras = re.findall("[a-zA-Z0-9_-]+", data["extras"])
         return Dependency(
             name=data["name"],
             version=(
                 SpecifierSet(data["version"])
                 if data["version"] is not None
                 else SpecifierSet("")
             ),
+            extras=extras,
         )
 
     @classmethod
@@ -894,25 +961,51 @@ def to_json(self) -> Dict[str, Dict[str, str]]:
             result["ref"] = cast(str, self.ref)
         return {self.name: result}
 
+    def to_pip_string(self) -> str:
+        """To pip specifier."""
+        if self.git is not None:
+            revision = self.ref if self.ref is not None else DEFAULT_GIT_REF
+            return "git+" + self.git + "@" + revision + "#egg=" + self.name
+        if len(self.extras) > 0:
+            return self.name + "[" + ",".join(self.extras) + "]" + str(self.version)
+        return self.name + str(self.version)
+
+    def to_pipfile_string(self) -> str:
+        """To Pipfile specifier."""
+        if self.git is not None:
+            string = self.name + " = {"
+            revision = self.ref if self.ref is not None else DEFAULT_GIT_REF
+            string += f'ref = "{revision}", '
+            string += f'git = "git+{self.git}"'
+            string += "}"
+            return string
+
+        if len(self.extras) > 0:
+            version = self.version if self.version != "" else "*"
+            string = self.name + " = {"
+            string += f'version = "{version}", '
+            string += "extras = ["
+            string += ", ".join(map(lambda x: f'"{x}"', self.extras))
+            string += "]}"
+            return string
+
+        version = self.version if self.version != "" else "*"
+        return f'{self.name} = "{version}"'
+
     def get_pip_install_args(self) -> List[str]:
         """Get 'pip install' arguments."""
-        name = self.name
-        index = self.index
-        git_url = self.git
-        revision = self.ref if self.ref is not None else DEFAULT_GIT_REF
-        version_constraint = str(self.version)
         command: List[str] = []
-        if index is not None:
-            command += ["-i", index]
-        if git_url is not None:
-            command += ["git+" + git_url + "@" + revision + "#egg=" + name]
-        else:
-            command += [name + version_constraint]
+        if self.index is not None:
+            command += ["-i", self.index]
+        command += [self.to_pip_string()]
         return command
 
     def __str__(self) -> str:
         """Get the string representation."""
-        return f"{self.__class__.__name__}(name='{self.name}', version='{self.version}', index='{self.index}', git='{self.git}', ref='{self.ref}')"
+        return (
+            f"{self.__class__.__name__}(name='{self.name}', version='{self.version}', "
+            f"index='{self.index}', git='{self.git}', ref='{self.ref}', extras='{self.extras}')"
+        )
 
     def __eq__(self, other: Any) -> bool:
         """Check equality."""

docs/api/configurations/data_types.md

@@ -999,7 +999,8 @@ def __init__(name: Union[PyPIPackageName, str],
              version: Union[str, SpecifierSet] = "",
              index: Optional[str] = None,
              git: Optional[str] = None,
-             ref: Optional[Union[GitRef, str]] = None) -> None
+             ref: Optional[Union[GitRef, str]] = None,
+             extras: Optional[List[str]] = None) -> None
 ```
 
 Initialize a PyPI dependency.
@@ -1011,6 +1012,7 @@ Initialize a PyPI dependency.
 - `index`: the URL to the PyPI server.
 - `git`: the URL to a git repository.
 - `ref`: the Git reference (branch/commit/tag).
+- `extras`: Include extras section for the dependency.
 
 <a id="aea.configurations.data_types.Dependency.name"></a>
 
@@ -1067,6 +1069,28 @@ def ref() -> Optional[str]
 
 Get the ref.
 
+<a id="aea.configurations.data_types.Dependency.extras"></a>
+
+#### extras
+
+```python
+@property
+def extras() -> List[str]
+```
+
+Get the ref.
+
+<a id="aea.configurations.data_types.Dependency.from_pipfile_string"></a>
+
+#### from`_`pipfile`_`string
+
+```python
+@classmethod
+def from_pipfile_string(cls, string: str) -> "Dependency"
+```
+
+Parse from Pipfile version specifier.
+
 <a id="aea.configurations.data_types.Dependency.from_string"></a>
 
 #### from`_`string
@@ -1099,6 +1123,26 @@ def to_json() -> Dict[str, Dict[str, str]]
 
 Transform the object to JSON.
 
+<a id="aea.configurations.data_types.Dependency.to_pip_string"></a>
+
+#### to`_`pip`_`string
+
+```python
+def to_pip_string() -> str
+```
+
+To pip specifier.
+
+<a id="aea.configurations.data_types.Dependency.to_pipfile_string"></a>
+
+#### to`_`pipfile`_`string
+
+```python
+def to_pipfile_string() -> str
+```
+
+To Pipfile specifier.
+
 <a id="aea.configurations.data_types.Dependency.get_pip_install_args"></a>
 
 #### get`_`pip`_`install`_`args