Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add "remove" BigMapAction, rename others #79

Merged
merged 3 commits into from
Jul 2, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/demo_hic_et_nunc/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
spec_version: 1.0
spec_version: 1.1
package: demo_hic_et_nunc

database:
kind: sqlite
path: hic_et_nunc.sqlite3

contracts:
HEN_objkts:
HEN_objkts:
address: ${HEN_OBJKTS:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton}
typename: hen_objkts
HEN_minter:
HEN_minter:
address: ${HEN_MINTER:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9}
typename: hen_minter

2 changes: 1 addition & 1 deletion src/demo_quipuswap/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_quipuswap

database:
2 changes: 1 addition & 1 deletion src/demo_registrydao/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_registrydao

database:
2 changes: 1 addition & 1 deletion src/demo_tezos_domains/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_tezos_domains

database:
2 changes: 1 addition & 1 deletion src/demo_tezos_domains_big_map/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_tezos_domains_big_map

database:
Original file line number Diff line number Diff line change
@@ -2,15 +2,16 @@
from demo_tezos_domains_big_map.types.name_registry.big_map.store_expiry_map_key import StoreExpiryMapKey
from demo_tezos_domains_big_map.types.name_registry.big_map.store_expiry_map_value import StoreExpiryMapValue
from dipdup.context import HandlerContext
from dipdup.models import BigMapAction, BigMapDiff
from dipdup.models import BigMapDiff


async def on_update_expiry_map(
ctx: HandlerContext,
store_expiry_map: BigMapDiff[StoreExpiryMapKey, StoreExpiryMapValue],
) -> None:
if store_expiry_map.action == BigMapAction.REMOVE:
if not store_expiry_map.action.has_value:
return
assert store_expiry_map.key
assert store_expiry_map.value

expiry = store_expiry_map.value.__root__
5 changes: 3 additions & 2 deletions src/demo_tezos_domains_big_map/handlers/on_update_records.py
Original file line number Diff line number Diff line change
@@ -2,15 +2,16 @@
from demo_tezos_domains_big_map.types.name_registry.big_map.store_records_key import StoreRecordsKey
from demo_tezos_domains_big_map.types.name_registry.big_map.store_records_value import StoreRecordsValue
from dipdup.context import HandlerContext
from dipdup.models import BigMapAction, BigMapDiff
from dipdup.models import BigMapDiff


async def on_update_records(
ctx: HandlerContext,
store_records: BigMapDiff[StoreRecordsKey, StoreRecordsValue],
) -> None:
if store_records.action == BigMapAction.REMOVE:
if not store_records.action.has_value:
return
assert store_records.key
assert store_records.value

record_name = bytes.fromhex(store_records.key.__root__).decode()
4 changes: 2 additions & 2 deletions src/demo_tzbtc/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_tzbtc

database:
@@ -29,4 +29,4 @@ indexes:
- callback: on_mint
pattern:
- destination: tzbtc_mainnet
entrypoint: mint
entrypoint: mint
4 changes: 2 additions & 2 deletions src/demo_tzcolors/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_tzcolors

database:
@@ -49,4 +49,4 @@ indexes:
values:
datasource: tzkt
minter: tzcolors_minter
auction: tzcolors_auction
auction: tzcolors_auction
2 changes: 1 addition & 1 deletion src/dipdup/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
__version__ = '1.1.2'
__spec_version__ = '1.0'
__spec_version__ = '1.1'
35 changes: 25 additions & 10 deletions src/dipdup/cli.py
Original file line number Diff line number Diff line change
@@ -15,12 +15,14 @@
from dipdup import __spec_version__, __version__
from dipdup.config import DipDupConfig, LoggingConfig
from dipdup.dipdup import DipDup
from dipdup.exceptions import ConfigurationError

_logger = logging.getLogger(__name__)

spec_version_to_version = {
'0.1': 'dipdup <0.4.3',
'1.0': 'dipdup ^1.0.0',
'0.1': 'dipdup v0.4.3 and below',
'1.0': 'dipdup v1.0.0 - v1.1.2',
'1.1': 'dipdup v1.2.0 and above',
}

migration_required_message = """
@@ -79,6 +81,8 @@ async def cli(ctx, config: List[str], logging_config: str):
_logging_config.apply()

_config = DipDupConfig.load(config)
if _config.spec_version not in spec_version_to_version:
raise ConfigurationError('Unknown `spec_version`')
if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate':
migration_required(_config.spec_version, __spec_version__)

@@ -121,16 +125,27 @@ async def init(ctx):
@click.pass_context
@click_async
async def migrate(ctx):
def _bump_spec_version(spec_version: str):
for config_path in ctx.obj.config_paths:
for line in fileinput.input(config_path, inplace=True):
if 'spec_version' in line:
print(f'spec_version: {spec_version}')
else:
print(line.rstrip())

config: DipDupConfig = ctx.obj.config
config.pre_initialize()
await DipDup(config).migrate()

for config_path in ctx.obj.config_paths:
for line in fileinput.input(config_path, inplace=True):
if 'spec_version' in line:
print(f'spec_version: {__spec_version__}')
else:
print(line.rstrip())

if config.spec_version == __spec_version__:
_logger.error('Project is already at latest version')
elif config.spec_version == '0.1':
await DipDup(config).migrate_to_v10()
_bump_spec_version('1.0')
elif config.spec_version == '1.0':
await DipDup(config).migrate_to_v11()
_bump_spec_version('1.1')
else:
raise ConfigurationError('Unknown `spec_version`')


@cli.command(help='Clear development request cache')
25 changes: 24 additions & 1 deletion src/dipdup/codegen.py
Original file line number Diff line number Diff line change
@@ -371,7 +371,7 @@ async def _get_schema(
self._schemas[datasource_config][address] = address_schemas_json
return self._schemas[datasource_config][address]

async def migrate_user_handlers_to_v1(self) -> None:
async def migrate_user_handlers_to_v10(self) -> None:
remove_lines = [
'from dipdup.models import',
'from dipdup.context import',
@@ -407,3 +407,26 @@ async def migrate_user_handlers_to_v1(self) -> None:
newfile.append(line)
with open(path, 'w') as file:
file.write('\n'.join(newfile))

async def migrate_user_handlers_to_v11(self) -> None:
replace_table = {
'BigMapAction.ADD': 'BigMapAction.ADD_KEY',
'BigMapAction.UPDATE': 'BigMapAction.UPDATE_KEY',
'BigMapAction.REMOVE': 'BigMapAction.REMOVE_KEY',
}
handlers_path = join(self._config.package_path, 'handlers')

for root, _, files in os.walk(handlers_path):
for filename in files:
if filename == '__init__.py' or not filename.endswith('.py'):
continue
path = join(root, filename)
newfile = []
with open(path) as file:
for line in file.read().split('\n'):
# Replace by table
for from_, to in replace_table.items():
line = line.replace(from_, to)
newfile.append(line)
with open(path, 'w') as file:
file.write('\n'.join(newfile))
28 changes: 16 additions & 12 deletions src/dipdup/dipdup.py
Original file line number Diff line number Diff line change
@@ -32,7 +32,7 @@
from dipdup.datasources.bcd.datasource import BcdDatasource
from dipdup.datasources.datasource import IndexDatasource
from dipdup.datasources.tzkt.datasource import TzktDatasource
from dipdup.exceptions import ConfigurationError, HandlerImportError
from dipdup.exceptions import ConfigurationError
from dipdup.hasura import configure_hasura
from dipdup.index import BigMapIndex, Index, OperationIndex
from dipdup.models import BigMapData, IndexType, OperationData, State
@@ -203,22 +203,20 @@ async def run(self, reindex: bool, oneshot: bool) -> None:
with suppress(AttributeError, SchedulerNotRunningError):
await self._scheduler.shutdown(wait=True)

async def migrate(self) -> None:
async def migrate_to_v10(self) -> None:
codegen = DipDupCodeGenerator(self._config, self._datasources_by_config)
await codegen.generate_default_handlers(recreate=True)
await codegen.migrate_user_handlers_to_v1()
self._logger.warning('==================== WARNING =====================')
self._logger.warning('Your handlers have just been migrated to v1.0.0 format.')
self._logger.warning('Review and commit changes before proceeding.')
self._logger.warning('==================== WARNING =====================')
quit()
await codegen.migrate_user_handlers_to_v10()
self._finish_migration('1.0')

async def migrate_to_v11(self) -> None:
codegen = DipDupCodeGenerator(self._config, self._datasources_by_config)
await codegen.migrate_user_handlers_to_v11()
self._finish_migration('1.1')

async def _configure(self) -> None:
"""Run user-defined initial configuration handler"""
try:
configure_fn = self._config.get_configure_fn()
except HandlerImportError:
await self.migrate()
configure_fn = self._config.get_configure_fn()
await configure_fn(self._ctx)
self._config.initialize()

@@ -309,3 +307,9 @@ async def _execute_sql_scripts(self, reindex: bool) -> None:

self._logger.info('Executing `%s`', filename)
await get_connection(None).execute_script(sql)

def _finish_migration(self, version: str) -> None:
self._logger.warning('==================== WARNING =====================')
self._logger.warning('Your project has been migrated to spec version %s.', version)
self._logger.warning('Review and commit changes before proceeding.')
self._logger.warning('==================== WARNING =====================')
17 changes: 9 additions & 8 deletions src/dipdup/index.py
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@
)
from dipdup.context import DipDupContext, HandlerContext
from dipdup.datasources.tzkt.datasource import BigMapFetcher, OperationFetcher, TzktDatasource
from dipdup.models import BigMapAction, BigMapData, BigMapDiff, OperationData, Origination, State, TemporaryState, Transaction
from dipdup.models import BigMapData, BigMapDiff, OperationData, Origination, State, TemporaryState, Transaction
from dipdup.utils import FormattedLogger, in_global_transaction

OperationGroup = namedtuple('OperationGroup', ('hash', 'counter'))
@@ -400,17 +400,18 @@ async def _on_match(
matched_big_map: BigMapData,
) -> None:
"""Prepare handler arguments, parse key and value. Schedule callback in executor."""
if matched_big_map.action == BigMapAction.ALLOCATE:
return

key_type = handler_config.key_type_cls
key = key_type.parse_obj(matched_big_map.key)

if matched_big_map.action == BigMapAction.REMOVE:
value = None
if matched_big_map.action.has_key:
key_type = handler_config.key_type_cls
key = key_type.parse_obj(matched_big_map.key)
else:
key = None

if matched_big_map.action.has_value:
value_type = handler_config.value_type_cls
value = value_type.parse_obj(matched_big_map.value)
else:
value = None

big_map_context = BigMapDiff( # type: ignore
data=matched_big_map,
17 changes: 13 additions & 4 deletions src/dipdup/models.py
Original file line number Diff line number Diff line change
@@ -174,9 +174,18 @@ class BigMapAction(Enum):
"""Mapping for action in TzKT response"""

ALLOCATE = 'allocate'
ADD = 'add_key'
UPDATE = 'update_key'
REMOVE = 'remove_key'
ADD_KEY = 'add_key'
UPDATE_KEY = 'update_key'
REMOVE_KEY = 'remove_key'
REMOVE = 'remove'

@property
def has_key(self) -> bool:
return self in (BigMapAction.ADD_KEY, BigMapAction.UPDATE_KEY, BigMapAction.REMOVE_KEY)

@property
def has_value(self) -> bool:
return self in (BigMapAction.ADD_KEY, BigMapAction.UPDATE_KEY)


@dataclass
@@ -201,5 +210,5 @@ class BigMapDiff(Generic[KeyType, ValueType]):

action: BigMapAction
data: BigMapData
key: KeyType
key: Optional[KeyType]
value: Optional[ValueType]
2 changes: 1 addition & 1 deletion tests/integration_tests/hic_et_nunc.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_hic_et_nunc

database:
2 changes: 1 addition & 1 deletion tests/integration_tests/hic_et_nunc_job.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_hic_et_nunc

database:
2 changes: 1 addition & 1 deletion tests/integration_tests/quipuswap.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_quipuswap

database:
2 changes: 1 addition & 1 deletion tests/integration_tests/registrydao.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_registrydao

database:
2 changes: 1 addition & 1 deletion tests/integration_tests/tezos_domains.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_tezos_domains

database:
2 changes: 1 addition & 1 deletion tests/integration_tests/tezos_domains_big_map.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_tezos_domains_big_map

database:
2 changes: 1 addition & 1 deletion tests/integration_tests/tzcolors.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_tzcolors

database:
2 changes: 1 addition & 1 deletion tests/test_dipdup/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 1.0
spec_version: 1.1
package: demo_hic_et_nunc

database: