From f4d1d659df64cb505cbe3442e38b44013d1bbf9f Mon Sep 17 00:00:00 2001 From: Ulrich Petri Date: Tue, 14 May 2019 18:55:19 +0200 Subject: [PATCH 1/2] Fix sdist package Setuptools_scm forcibly includes all files under version control into the sdist package, ignoring `MANIFEST.in` and `find_packages`. This fixes this by replacing the `find_files` function with a dummy one (`see setup.py::EggInfo.__init__()`). See pypa/setuptools_scm#190 --- MANIFEST.in | 1 + Makefile | 2 +- raiden/network/resolver/__init__.py | 0 raiden/storage/migrations/__init__.py | 0 setup.py | 86 ++++++++++++++++----------- 5 files changed, 52 insertions(+), 37 deletions(-) create mode 100644 raiden/network/resolver/__init__.py create mode 100644 raiden/storage/migrations/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in index 5e28dc925f..d332d2ed64 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include README.rst include requirements.txt +include constraints.txt diff --git a/Makefile b/Makefile index 15800255cc..9290a77dd1 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ clean-test: LINT_PATHS = raiden/ tools/ ISORT_PARAMS = --ignore-whitespace --settings-path ./ --skip-glob '*/node_modules/*' --recursive $(LINT_PATHS) -BLACK_PATHS = raiden/ tools/ +BLACK_PATHS = raiden/ tools/ setup.py lint: mypy mypy-all flake8 raiden/ tools/ diff --git a/raiden/network/resolver/__init__.py b/raiden/network/resolver/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/raiden/storage/migrations/__init__.py b/raiden/storage/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/setup.py b/setup.py index 8a88dd1759..337efb3d4f 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python from setuptools import find_packages, setup +from setuptools.command.egg_info import egg_info from setuptools.command.test import test as TestCommand class PyTest(TestCommand): - def finalize_options(self): TestCommand.finalize_options(self) self.test_args = [] @@ -13,61 +13,75 @@ def finalize_options(self): def run_tests(self): # import here, cause outside the eggs aren't loaded import pytest + errno = pytest.main(self.test_args) raise SystemExit(errno) -with open('README.rst') as readme_file: +class EggInfo(egg_info): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + # FFS, setuptools_scm forcibly includes all files under version control into the sdist + # package, ignoring `MANIFEST.in` and `find_packages`. + # See https://github.com/pypa/setuptools_scm/issues/190 + # We 'fix' this by replacing the `find_files` function with a dummy one. + # The reason this is done here and not on the top level is that setuptools_scm is a + # setup time requirement so it may not have been installed when this setup.py is initially + # executed. + try: + import setuptools_scm.integration + + setuptools_scm.integration.find_files = lambda _: [] + except ImportError: + pass + + +with open("README.rst") as readme_file: readme = readme_file.read() -history = '' +history = "" -with open('constraints.txt') as req_file: - install_requires = list({ - requirement - for requirement in req_file - if requirement.strip() and not requirement.lstrip().startswith('#') - }) +with open("constraints.txt") as req_file: + install_requires = list( + { + requirement + for requirement in req_file + if requirement.strip() and not requirement.lstrip().startswith("#") + } + ) test_requirements = [] # Do not edit: this is maintained by bumpversion (see .bumpversion_client.cfg) -version = '0.100.3-rc5' +version = "0.100.3-rc5" setup( - name='raiden', - description='', - long_description=readme + '\n\n' + history, - author='Brainbot Labs Est.', - author_email='contact@brainbot.li', - url='https://github.com/raiden-network/raiden', - packages=find_packages(exclude=('tools',)), + name="raiden", + description="", + long_description=readme + "\n\n" + history, + author="Brainbot Labs Est.", + author_email="contact@brainbot.li", + url="https://github.com/raiden-network/raiden", + packages=find_packages(include=("raiden", "raiden.*")), package_data={"raiden": ["py.typed"]}, - include_package_data=True, - license='MIT', + license="MIT", zip_safe=False, - keywords='raiden', + keywords="raiden", classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", ], - cmdclass={ - 'test': PyTest, - }, + cmdclass={"test": PyTest, "egg_info": EggInfo}, use_scm_version=True, - setup_requires=['setuptools_scm'], + setup_requires=["setuptools_scm"], install_requires=install_requires, tests_require=test_requirements, - python_requires='>=3.7', - entry_points={ - 'console_scripts': [ - 'raiden = raiden.__main__:main', - ], - }, + python_requires=">=3.7", + entry_points={"console_scripts": ["raiden = raiden.__main__:main"]}, ) From e6afb137156505e9d6d023523fe509ff65efedc4 Mon Sep 17 00:00:00 2001 From: Ulrich Petri Date: Wed, 15 May 2019 13:57:12 +0200 Subject: [PATCH 2/2] Fix mypy issues in missed packages --- raiden/network/resolver/client.py | 2 ++ raiden/storage/migrations/v17_to_v18.py | 8 +++-- raiden/storage/migrations/v18_to_v19.py | 12 +++---- raiden/storage/migrations/v19_to_v20.py | 42 ++++++++++++++++--------- raiden/storage/migrations/v21_to_v22.py | 20 ++++++------ 5 files changed, 52 insertions(+), 32 deletions(-) diff --git a/raiden/network/resolver/client.py b/raiden/network/resolver/client.py index b2da7c3d47..649953ab78 100644 --- a/raiden/network/resolver/client.py +++ b/raiden/network/resolver/client.py @@ -4,6 +4,7 @@ from eth_utils import to_bytes, to_hex from raiden.raiden_service import RaidenService +from raiden.storage.wal import WriteAheadLog from raiden.transfer.mediated_transfer.events import SendSecretRequest from raiden.transfer.mediated_transfer.state_change import ReceiveSecretReveal @@ -15,6 +16,7 @@ def reveal_secret_with_resolver( if "resolver_endpoint" not in raiden.config: return False + assert isinstance(raiden.wal, WriteAheadLog), "RaidenService has not been started" current_state = raiden.wal.state_manager.current_state task = current_state.payment_mapping.secrethashes_to_task[secret_request_event.secrethash] token = task.target_state.transfer.token diff --git a/raiden/storage/migrations/v17_to_v18.py b/raiden/storage/migrations/v17_to_v18.py index f3b5b23f53..692d70d9b4 100644 --- a/raiden/storage/migrations/v17_to_v18.py +++ b/raiden/storage/migrations/v17_to_v18.py @@ -20,7 +20,7 @@ def get_token_network_by_identifier( return None -def _transform_snapshot(raw_snapshot: Dict[Any, Any]) -> str: +def _transform_snapshot(raw_snapshot: str) -> str: """ This migration upgrades the object: - `MediatorTransferState` such that a list of routes is added @@ -48,7 +48,11 @@ def _transform_snapshot(raw_snapshot: Dict[Any, Any]) -> str: token_network_identifier = transfer["balance_proof"]["token_network_identifier"] token_network = get_token_network_by_identifier(snapshot, token_network_identifier) channel_identifier = transfer["balance_proof"]["channel_identifier"] - channel = token_network.get("channelidentifiers_to_channels").get(channel_identifier) + channel = None + if token_network is not None: + channel = token_network.get("channelidentifiers_to_channels", {}).get( + channel_identifier + ) if not channel: raise ChannelNotFound( f"Upgrading to v18 failed. " diff --git a/raiden/storage/migrations/v18_to_v19.py b/raiden/storage/migrations/v18_to_v19.py index cd9e89aeb0..449585d278 100644 --- a/raiden/storage/migrations/v18_to_v19.py +++ b/raiden/storage/migrations/v18_to_v19.py @@ -22,7 +22,7 @@ class BlockHashCache: def __init__(self, web3: Web3): self.web3 = web3 - self.mapping = {} + self.mapping: Dict[BlockNumber, str] = {} def get(self, block_number: BlockNumber) -> str: """Given a block number returns the hex representation of the blockhash""" @@ -47,7 +47,7 @@ def _query_blocknumber_and_update_statechange_data( ) -> Tuple[str, int]: data = record.data data["block_hash"] = record.cache.get(record.block_number) - return (json.dumps(data), record.state_change_identifier) + return json.dumps(data), record.state_change_identifier def _add_blockhash_to_state_changes(storage: SQLiteStorage, cache: BlockHashCache) -> None: @@ -69,7 +69,7 @@ def _add_blockhash_to_state_changes(storage: SQLiteStorage, cache: BlockHashCach data = json.loads(state_change.data) assert "block_hash" not in data, "v18 state changes cant contain blockhash" record = BlockQueryAndUpdateRecord( - block_number=int(data["block_number"]), + block_number=BlockNumber(int(data["block_number"])), data=data, state_change_identifier=state_change.state_change_identifier, cache=cache, @@ -112,7 +112,7 @@ def _add_blockhash_to_events(storage: SQLiteStorage, cache: BlockHashCache) -> N if "block_hash" in statechange_data: data["triggered_by_block_hash"] = statechange_data["block_hash"] elif "block_number" in statechange_data: - block_number = int(statechange_data["block_number"]) + block_number = BlockNumber(int(statechange_data["block_number"])) data["triggered_by_block_hash"] = cache.get(block_number) updated_events.append((json.dumps(data), event.event_identifier)) @@ -123,7 +123,7 @@ def _add_blockhash_to_events(storage: SQLiteStorage, cache: BlockHashCache) -> N def _transform_snapshot(raw_snapshot: str, storage: SQLiteStorage, cache: BlockHashCache) -> str: """Upgrades a single snapshot by adding the blockhash to it and to any pending transactions""" snapshot = json.loads(raw_snapshot) - block_number = int(snapshot["block_number"]) + block_number = BlockNumber(int(snapshot["block_number"])) snapshot["block_hash"] = cache.get(block_number) pending_transactions = snapshot["pending_transactions"] @@ -158,7 +158,7 @@ class TransformSnapshotRecord(NamedTuple): cache: BlockHashCache -def _do_transform_snapshot(record: TransformSnapshotRecord) -> Tuple[Dict[str, Any], int]: +def _do_transform_snapshot(record: TransformSnapshotRecord) -> Tuple[str, int]: new_snapshot = _transform_snapshot( raw_snapshot=record.data, storage=record.storage, cache=record.cache ) diff --git a/raiden/storage/migrations/v19_to_v20.py b/raiden/storage/migrations/v19_to_v20.py index d7081517fa..1366cb808d 100644 --- a/raiden/storage/migrations/v19_to_v20.py +++ b/raiden/storage/migrations/v19_to_v20.py @@ -1,5 +1,6 @@ import json from functools import partial +from typing import TYPE_CHECKING from eth_utils import to_canonical_address from gevent.pool import Pool @@ -7,12 +8,23 @@ from raiden.constants import EMPTY_MERKLE_ROOT from raiden.exceptions import RaidenUnrecoverableError from raiden.network.proxies.utils import get_onchain_locksroots -from raiden.storage.sqlite import SQLiteStorage, StateChangeRecord +from raiden.storage.sqlite import SnapshotRecord, SQLiteStorage, StateChangeRecord from raiden.transfer.identifiers import CanonicalIdentifier from raiden.utils.serialization import serialize_bytes -from raiden.utils.typing import Any, Dict, Locksroot, Tuple +from raiden.utils.typing import ( + Any, + ChainID, + ChannelID, + Dict, + Locksroot, + TokenNetworkAddress, + Tuple, +) + +if TYPE_CHECKING: + # pylint: disable=unused-import + from raiden.raiden_service import RaidenService # noqa: F401 -RaidenService = "RaidenService" SOURCE_VERSION = 19 TARGET_VERSION = 20 @@ -31,7 +43,7 @@ def _find_channel_new_state_change( def _get_onchain_locksroots( - raiden: RaidenService, + raiden: "RaidenService", storage: SQLiteStorage, token_network: Dict[str, Any], channel: Dict[str, Any], @@ -49,9 +61,9 @@ def _get_onchain_locksroots( ) canonical_identifier = CanonicalIdentifier( - chain_identifier=-1, - token_network_address=to_canonical_address(token_network["address"]), - channel_identifier=int(channel["identifier"]), + chain_identifier=ChainID(-1), + token_network_address=TokenNetworkAddress(to_canonical_address(token_network["address"])), + channel_identifier=ChannelID(int(channel["identifier"])), ) our_locksroot, partner_locksroot = get_onchain_locksroots( @@ -96,7 +108,7 @@ def _add_onchain_locksroot_to_channel_new_state_changes(storage: SQLiteStorage,) def _add_onchain_locksroot_to_channel_settled_state_changes( - raiden: RaidenService, storage: SQLiteStorage + raiden: "RaidenService", storage: SQLiteStorage ) -> None: """ Adds `our_onchain_locksroot` and `partner_onchain_locksroot` to ContractReceiveChannelSettled. """ @@ -134,9 +146,11 @@ def _add_onchain_locksroot_to_channel_settled_state_changes( new_channel_state = channel_state_data["channel_state"] canonical_identifier = CanonicalIdentifier( - chain_identifier=-1, - token_network_address=to_canonical_address(token_network_identifier), - channel_identifier=int(channel_identifier), + chain_identifier=ChainID(-1), + token_network_address=TokenNetworkAddress( + to_canonical_address(token_network_identifier) + ), + channel_identifier=ChannelID(int(channel_identifier)), ) our_locksroot, partner_locksroot = get_onchain_locksroots( chain=raiden.chain, @@ -156,8 +170,8 @@ def _add_onchain_locksroot_to_channel_settled_state_changes( def _add_onchain_locksroot_to_snapshot( - raiden: RaidenService, storage: SQLiteStorage, snapshot_record: StateChangeRecord -) -> str: + raiden: "RaidenService", storage: SQLiteStorage, snapshot_record: SnapshotRecord +) -> Tuple[str, int]: """ Add `onchain_locksroot` to each NettingChannelEndState """ @@ -178,7 +192,7 @@ def _add_onchain_locksroot_to_snapshot( return json.dumps(snapshot, indent=4), snapshot_record.identifier -def _add_onchain_locksroot_to_snapshots(raiden: RaidenService, storage: SQLiteStorage) -> None: +def _add_onchain_locksroot_to_snapshots(raiden: "RaidenService", storage: SQLiteStorage) -> None: snapshots = storage.get_snapshots() transform_func = partial(_add_onchain_locksroot_to_snapshot, raiden, storage) diff --git a/raiden/storage/migrations/v21_to_v22.py b/raiden/storage/migrations/v21_to_v22.py index ed93428d74..12b1d6cf73 100644 --- a/raiden/storage/migrations/v21_to_v22.py +++ b/raiden/storage/migrations/v21_to_v22.py @@ -1,5 +1,5 @@ import json -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, Tuple, TypeVar from eth_utils import to_checksum_address @@ -16,15 +16,15 @@ BATCH_UNLOCK = "raiden.transfer.state_change.ContractReceiveChannelBatchUnlock" -SPELLING_VARS_TOKEN_NETWORK = ( +SPELLING_VARS_TOKEN_NETWORK = [ "token_network_address", "token_network_id", "token_network_identifier", -) +] -SPELLING_VARS_CHANNEL = ("channel_identifier", "channel_id", "identifier") +SPELLING_VARS_CHANNEL = ["channel_identifier", "channel_id", "identifier"] -SPELLING_VARS_CHAIN = ("chain_id", "chain_identifier") +SPELLING_VARS_CHAIN = ["chain_id", "chain_identifier"] # these are missing the chain-id @@ -224,8 +224,8 @@ def _add_canonical_identifier_to_statechanges( our_address = str(to_checksum_address(raiden.address)).lower() for state_change_batch in storage.batch_query_state_changes(batch_size=500): - updated_state_changes = list() - delete_state_changes = list() + updated_state_changes: List[Tuple[str, int]] = list() + delete_state_changes: List[int] = list() for state_change_record in state_change_batch: state_change_obj = json.loads(state_change_record.data) @@ -236,16 +236,16 @@ def _add_canonical_identifier_to_statechanges( ) if should_delete: - delete_state_changes.append(state_change_record.identifier) + delete_state_changes.append(state_change_record.state_change_identifier) else: - channel_id = None + channel_id: Optional[int] = None if is_unlock: channel_id = resolve_channel_id_for_unlock( storage, state_change_obj, our_address ) walk_dicts( state_change_obj, - lambda obj, channel_id=channel_id: upgrade_object(obj, chain_id, channel_id), + lambda obj, channel_id_=channel_id: upgrade_object(obj, chain_id, channel_id_), ) walk_dicts(state_change_obj, constraint_has_canonical_identifier_or_values_removed)