diff --git a/src/demo_hic_et_nunc/dipdup.yml b/src/demo_hic_et_nunc/dipdup.yml index d9f85ad3a..2214df0fb 100644 --- a/src/demo_hic_et_nunc/dipdup.yml +++ b/src/demo_hic_et_nunc/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_hic_et_nunc database: diff --git a/src/demo_quipuswap/dipdup.yml b/src/demo_quipuswap/dipdup.yml index 257520277..23a123909 100644 --- a/src/demo_quipuswap/dipdup.yml +++ b/src/demo_quipuswap/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_quipuswap database: diff --git a/src/demo_registrydao/dipdup.yml b/src/demo_registrydao/dipdup.yml index 9531038d5..f9121d344 100644 --- a/src/demo_registrydao/dipdup.yml +++ b/src/demo_registrydao/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_registrydao database: diff --git a/src/demo_tezos_domains/dipdup.yml b/src/demo_tezos_domains/dipdup.yml index 667621542..bae781d16 100644 --- a/src/demo_tezos_domains/dipdup.yml +++ b/src/demo_tezos_domains/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_tezos_domains database: diff --git a/src/demo_tezos_domains_big_map/dipdup.yml b/src/demo_tezos_domains_big_map/dipdup.yml index 3e61e833f..cbb24adb7 100644 --- a/src/demo_tezos_domains_big_map/dipdup.yml +++ b/src/demo_tezos_domains_big_map/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_tezos_domains_big_map database: diff --git a/src/demo_tzbtc/dipdup.yml b/src/demo_tzbtc/dipdup.yml index 23d4a8b09..71011557b 100644 --- a/src/demo_tzbtc/dipdup.yml +++ b/src/demo_tzbtc/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_tzbtc database: diff --git a/src/demo_tzcolors/dipdup.yml b/src/demo_tzcolors/dipdup.yml index 8c8c62afd..70ffec7c1 100644 --- a/src/demo_tzcolors/dipdup.yml +++ b/src/demo_tzcolors/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_tzcolors database: diff --git a/src/dipdup/__init__.py b/src/dipdup/__init__.py index 908c0bb70..78d3afc72 100644 --- a/src/dipdup/__init__.py +++ b/src/dipdup/__init__.py @@ -1 +1,2 @@ __version__ = '0.4.3' +__spec_version__ = '1.0' diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 661841354..a868cae6b 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -1,21 +1,50 @@ import asyncio +import fileinput import logging import os from dataclasses import dataclass from functools import wraps from os.path import dirname, join -from typing import List +from typing import List, NoReturn import click from fcache.cache import FileCache # type: ignore -from dipdup import __version__ +from dipdup import __spec_version__, __version__ from dipdup.config import DipDupConfig, LoggingConfig from dipdup.dipdup import DipDup -from dipdup.exceptions import HandlerImportError _logger = logging.getLogger(__name__) +spec_version_to_version = { + '0.1': 'dipdup <0.4.3', + '1.0': 'dipdup ^1.0.0', +} + +migration_required_message = """ + +Migration required! + +project spec version: %s (%s) +current spec version: %s (%s) + + 1. Run `dipdup migrate` + 2. Review and commit changes + +See https://baking-bad.org/blog/ for additional release information. +""" + + +def migration_required(from_: str, to: str) -> NoReturn: + _logger.warning( + migration_required_message, + from_, + spec_version_to_version[from_], + to, + spec_version_to_version[to], + ) + quit() + def click_async(fn): @wraps(fn) @@ -27,6 +56,7 @@ def wrapper(*args, **kwargs): @dataclass class CLIContext: + config_paths: List[str] config: DipDupConfig logging_config: LoggingConfig @@ -45,8 +75,13 @@ async def cli(ctx, config: List[str], logging_config: str): path = join(dirname(__file__), 'configs', logging_config) _logging_config = LoggingConfig.load(path) _logging_config.apply() + _config = DipDupConfig.load(config) + if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate': + migration_required(_config.spec_version, __spec_version__) + ctx.obj = CLIContext( + config_paths=config, config=_config, logging_config=_logging_config, ) @@ -59,12 +94,7 @@ async def cli(ctx, config: List[str], logging_config: str): @click_async async def run(ctx, reindex: bool, oneshot: bool) -> None: config: DipDupConfig = ctx.obj.config - - try: - config.initialize() - except HandlerImportError: - await DipDup(config).migrate() - + config.initialize() dipdup = DipDup(config) await dipdup.run(reindex, oneshot) @@ -79,6 +109,22 @@ async def init(ctx): await dipdup.init() +@cli.command(help='Migrate project to the new spec version') +@click.pass_context +@click_async +async def migrate(ctx): + config: DipDupConfig = ctx.obj.config + config.pre_initialize() + await DipDup(config).migrate() + + for config_path in ctx.obj.config_paths: + for line in fileinput.input(config_path, inplace=True): + if 'spec_version' in line: + print(f'spec_version: {__spec_version__}') + else: + print(line.rstrip()) + + @cli.command(help='Clear development request cache') @click.pass_context @click_async diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index f47fc5106..7a6e1abf9 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -249,20 +249,14 @@ async def generate_default_handlers(self, recreate=False) -> None: self._logger.info('Generating handler `%s`', CONFIGURE_HANDLER) handler_code = configure_template.render() handler_path = join(handlers_path, f'{CONFIGURE_HANDLER}.py') - if exists(handler_path) and recreate: - old_handler_path = join(handlers_path, f'{CONFIGURE_HANDLER}.py.bak') - os.rename(handler_path, old_handler_path) - if not exists(handler_path): + if not exists(handler_path) or recreate: with open(handler_path, 'w') as file: file.write(handler_code) self._logger.info('Generating handler `%s`', ROLLBACK_HANDLER) handler_code = rollback_template.render() handler_path = join(handlers_path, f'{ROLLBACK_HANDLER}.py') - if exists(handler_path) and recreate: - old_handler_path = join(handlers_path, f'{ROLLBACK_HANDLER}.py.bak') - os.rename(handler_path, old_handler_path) - if not exists(handler_path): + if not exists(handler_path) or recreate: with open(handler_path, 'w') as file: file.write(handler_code) @@ -342,7 +336,6 @@ async def _get_schema( return self._schemas[datasource_config][address] async def migrate_user_handlers_to_v1(self) -> None: - # TODO: Save backups remove_lines = [ 'from dipdup.models import', 'from dipdup.context import', @@ -356,8 +349,8 @@ async def migrate_user_handlers_to_v1(self) -> None: 'TransactionContext': 'Transaction', 'OriginationContext': 'Origination', 'BigMapContext': 'BigMapDiff', - 'HandlerContext': 'HandlerContext', - 'HandlerContext': 'HandlerContext', + 'OperationHandlerContext': 'HandlerContext', + 'BigMapHandlerContext': 'HandlerContext', } handlers_path = join(self._config.package_path, 'handlers') @@ -366,10 +359,8 @@ async def migrate_user_handlers_to_v1(self) -> None: if filename == '__init__.py' or not filename.endswith('.py'): continue path = join(root, filename) - bak_path = path + '.bak' - os.rename(path, bak_path) newfile = copy(add_lines) - with open(bak_path) as file: + with open(path) as file: for line in file.read().split('\n'): # Skip existing models imports if any(map(lambda l: l in line, remove_lines)): diff --git a/src/dipdup/config.py b/src/dipdup/config.py index 774bba6c2..430615015 100644 --- a/src/dipdup/config.py +++ b/src/dipdup/config.py @@ -623,12 +623,6 @@ def valid_url(cls, v): return v -@dataclass -class ConfigurationConfig: - interval: int = 60 - args: Dict[str, Any] = Field(default_factory=dict) - - @dataclass class DipDupConfig: """Main dapp config @@ -652,7 +646,6 @@ class DipDupConfig: templates: Optional[Dict[str, IndexConfigTemplateT]] = None database: Union[SqliteDatabaseConfig, MySQLDatabaseConfig, PostgresDatabaseConfig] = SqliteDatabaseConfig(kind='sqlite') hasura: Optional[HasuraConfig] = None - configuration: Optional[ConfigurationConfig] = None def __post_init_post_parse__(self): self._callback_patterns: Dict[str, List[Sequence[HandlerPatternConfigT]]] = defaultdict(list) @@ -843,15 +836,9 @@ def load( def _initialize_handler_callback(self, handler_config: HandlerConfig) -> None: _logger.info('Registering handler callback `%s`', handler_config.callback) - try: - handler_module = importlib.import_module(f'{self.package}.handlers.{handler_config.callback}') - callback_fn = getattr(handler_module, handler_config.callback) - handler_config.callback_fn = callback_fn - except ImportError as e: - if 'Context' in str(e): - _logger.warning('Found broken imports, attemping to fix them') - raise HandlerImportError from e - raise + handler_module = importlib.import_module(f'{self.package}.handlers.{handler_config.callback}') + callback_fn = getattr(handler_module, handler_config.callback) + handler_config.callback_fn = callback_fn def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None: if index_name in self._initialized: diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index 9c545fcbe..3454060cd 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -46,7 +46,7 @@ def __init__(self, ctx: HandlerContext) -> None: async def add_index(self, index_config: IndexConfigTemplateT) -> None: if index_config.name in self._indexes: return - self._logger.info('Adding index `%s` to dispatcher') + self._logger.info('Adding index `%s` to dispatcher', index_config.name) if isinstance(index_config, OperationIndexConfig): datasource_name = cast(TzktDatasourceConfig, index_config.datasource).name datasource = self._ctx.datasources[datasource_name] diff --git a/tests/integration_tests/hic_et_nunc.yml b/tests/integration_tests/hic_et_nunc.yml index 8ff08acda..b2944f75b 100644 --- a/tests/integration_tests/hic_et_nunc.yml +++ b/tests/integration_tests/hic_et_nunc.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_hic_et_nunc database: diff --git a/tests/integration_tests/quipuswap.yml b/tests/integration_tests/quipuswap.yml index 4e7528a81..543083145 100644 --- a/tests/integration_tests/quipuswap.yml +++ b/tests/integration_tests/quipuswap.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_quipuswap database: diff --git a/tests/integration_tests/registrydao.yml b/tests/integration_tests/registrydao.yml index 9e774ca98..23ba55045 100644 --- a/tests/integration_tests/registrydao.yml +++ b/tests/integration_tests/registrydao.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_registrydao database: diff --git a/tests/integration_tests/tezos_domains.yml b/tests/integration_tests/tezos_domains.yml index 2b535cd7b..bde47370d 100644 --- a/tests/integration_tests/tezos_domains.yml +++ b/tests/integration_tests/tezos_domains.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_tezos_domains database: diff --git a/tests/integration_tests/tezos_domains_big_map.yml b/tests/integration_tests/tezos_domains_big_map.yml index a7b3a10d6..c3175507f 100644 --- a/tests/integration_tests/tezos_domains_big_map.yml +++ b/tests/integration_tests/tezos_domains_big_map.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_tezos_domains_big_map database: diff --git a/tests/integration_tests/tzcolors.yml b/tests/integration_tests/tzcolors.yml index e8e779dde..d26ce1b7b 100644 --- a/tests/integration_tests/tzcolors.yml +++ b/tests/integration_tests/tzcolors.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_tzcolors database: diff --git a/tests/test_dipdup/dipdup.yml b/tests/test_dipdup/dipdup.yml index e8ca1c390..561d2dfd9 100644 --- a/tests/test_dipdup/dipdup.yml +++ b/tests/test_dipdup/dipdup.yml @@ -1,4 +1,4 @@ -spec_version: 0.1 +spec_version: 1.0 package: demo_hic_et_nunc database: