Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

migrate command #61

Merged
merged 1 commit into from
Jun 18, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/demo_hic_et_nunc/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_hic_et_nunc

database:
Expand Down
2 changes: 1 addition & 1 deletion src/demo_quipuswap/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_quipuswap

database:
Expand Down
2 changes: 1 addition & 1 deletion src/demo_registrydao/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_registrydao

database:
Expand Down
2 changes: 1 addition & 1 deletion src/demo_tezos_domains/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_tezos_domains

database:
Expand Down
2 changes: 1 addition & 1 deletion src/demo_tezos_domains_big_map/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_tezos_domains_big_map

database:
Expand Down
2 changes: 1 addition & 1 deletion src/demo_tzbtc/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_tzbtc

database:
Expand Down
2 changes: 1 addition & 1 deletion src/demo_tzcolors/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_tzcolors

database:
Expand Down
1 change: 1 addition & 0 deletions src/dipdup/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
__version__ = '0.4.3'
__spec_version__ = '1.0'
64 changes: 55 additions & 9 deletions src/dipdup/cli.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,50 @@
import asyncio
import fileinput
import logging
import os
from dataclasses import dataclass
from functools import wraps
from os.path import dirname, join
from typing import List
from typing import List, NoReturn

import click
from fcache.cache import FileCache # type: ignore

from dipdup import __version__
from dipdup import __spec_version__, __version__
from dipdup.config import DipDupConfig, LoggingConfig
from dipdup.dipdup import DipDup
from dipdup.exceptions import HandlerImportError

_logger = logging.getLogger(__name__)

spec_version_to_version = {
'0.1': 'dipdup <0.4.3',
'1.0': 'dipdup ^1.0.0',
}

migration_required_message = """

Migration required!

project spec version: %s (%s)
current spec version: %s (%s)

1. Run `dipdup migrate`
2. Review and commit changes

See https://baking-bad.org/blog/ for additional release information.
"""


def migration_required(from_: str, to: str) -> NoReturn:
_logger.warning(
migration_required_message,
from_,
spec_version_to_version[from_],
to,
spec_version_to_version[to],
)
quit()


def click_async(fn):
@wraps(fn)
Expand All @@ -27,6 +56,7 @@ def wrapper(*args, **kwargs):

@dataclass
class CLIContext:
config_paths: List[str]
config: DipDupConfig
logging_config: LoggingConfig

Expand All @@ -45,8 +75,13 @@ async def cli(ctx, config: List[str], logging_config: str):
path = join(dirname(__file__), 'configs', logging_config)
_logging_config = LoggingConfig.load(path)
_logging_config.apply()

_config = DipDupConfig.load(config)
if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate':
migration_required(_config.spec_version, __spec_version__)

ctx.obj = CLIContext(
config_paths=config,
config=_config,
logging_config=_logging_config,
)
Expand All @@ -59,12 +94,7 @@ async def cli(ctx, config: List[str], logging_config: str):
@click_async
async def run(ctx, reindex: bool, oneshot: bool) -> None:
config: DipDupConfig = ctx.obj.config

try:
config.initialize()
except HandlerImportError:
await DipDup(config).migrate()

config.initialize()
dipdup = DipDup(config)
await dipdup.run(reindex, oneshot)

Expand All @@ -79,6 +109,22 @@ async def init(ctx):
await dipdup.init()


@cli.command(help='Migrate project to the new spec version')
@click.pass_context
@click_async
async def migrate(ctx):
config: DipDupConfig = ctx.obj.config
config.pre_initialize()
await DipDup(config).migrate()

for config_path in ctx.obj.config_paths:
for line in fileinput.input(config_path, inplace=True):
if 'spec_version' in line:
print(f'spec_version: {__spec_version__}')
else:
print(line.rstrip())


@cli.command(help='Clear development request cache')
@click.pass_context
@click_async
Expand Down
19 changes: 5 additions & 14 deletions src/dipdup/codegen.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,20 +249,14 @@ async def generate_default_handlers(self, recreate=False) -> None:
self._logger.info('Generating handler `%s`', CONFIGURE_HANDLER)
handler_code = configure_template.render()
handler_path = join(handlers_path, f'{CONFIGURE_HANDLER}.py')
if exists(handler_path) and recreate:
old_handler_path = join(handlers_path, f'{CONFIGURE_HANDLER}.py.bak')
os.rename(handler_path, old_handler_path)
if not exists(handler_path):
if not exists(handler_path) or recreate:
with open(handler_path, 'w') as file:
file.write(handler_code)

self._logger.info('Generating handler `%s`', ROLLBACK_HANDLER)
handler_code = rollback_template.render()
handler_path = join(handlers_path, f'{ROLLBACK_HANDLER}.py')
if exists(handler_path) and recreate:
old_handler_path = join(handlers_path, f'{ROLLBACK_HANDLER}.py.bak')
os.rename(handler_path, old_handler_path)
if not exists(handler_path):
if not exists(handler_path) or recreate:
with open(handler_path, 'w') as file:
file.write(handler_code)

Expand Down Expand Up @@ -342,7 +336,6 @@ async def _get_schema(
return self._schemas[datasource_config][address]

async def migrate_user_handlers_to_v1(self) -> None:
# TODO: Save backups
remove_lines = [
'from dipdup.models import',
'from dipdup.context import',
Expand All @@ -356,8 +349,8 @@ async def migrate_user_handlers_to_v1(self) -> None:
'TransactionContext': 'Transaction',
'OriginationContext': 'Origination',
'BigMapContext': 'BigMapDiff',
'HandlerContext': 'HandlerContext',
'HandlerContext': 'HandlerContext',
'OperationHandlerContext': 'HandlerContext',
'BigMapHandlerContext': 'HandlerContext',
}
handlers_path = join(self._config.package_path, 'handlers')

Expand All @@ -366,10 +359,8 @@ async def migrate_user_handlers_to_v1(self) -> None:
if filename == '__init__.py' or not filename.endswith('.py'):
continue
path = join(root, filename)
bak_path = path + '.bak'
os.rename(path, bak_path)
newfile = copy(add_lines)
with open(bak_path) as file:
with open(path) as file:
for line in file.read().split('\n'):
# Skip existing models imports
if any(map(lambda l: l in line, remove_lines)):
Expand Down
19 changes: 3 additions & 16 deletions src/dipdup/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -623,12 +623,6 @@ def valid_url(cls, v):
return v


@dataclass
class ConfigurationConfig:
interval: int = 60
args: Dict[str, Any] = Field(default_factory=dict)


@dataclass
class DipDupConfig:
"""Main dapp config
Expand All @@ -652,7 +646,6 @@ class DipDupConfig:
templates: Optional[Dict[str, IndexConfigTemplateT]] = None
database: Union[SqliteDatabaseConfig, MySQLDatabaseConfig, PostgresDatabaseConfig] = SqliteDatabaseConfig(kind='sqlite')
hasura: Optional[HasuraConfig] = None
configuration: Optional[ConfigurationConfig] = None

def __post_init_post_parse__(self):
self._callback_patterns: Dict[str, List[Sequence[HandlerPatternConfigT]]] = defaultdict(list)
Expand Down Expand Up @@ -843,15 +836,9 @@ def load(

def _initialize_handler_callback(self, handler_config: HandlerConfig) -> None:
_logger.info('Registering handler callback `%s`', handler_config.callback)
try:
handler_module = importlib.import_module(f'{self.package}.handlers.{handler_config.callback}')
callback_fn = getattr(handler_module, handler_config.callback)
handler_config.callback_fn = callback_fn
except ImportError as e:
if 'Context' in str(e):
_logger.warning('Found broken imports, attemping to fix them')
raise HandlerImportError from e
raise
handler_module = importlib.import_module(f'{self.package}.handlers.{handler_config.callback}')
callback_fn = getattr(handler_module, handler_config.callback)
handler_config.callback_fn = callback_fn

def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None:
if index_name in self._initialized:
Expand Down
2 changes: 1 addition & 1 deletion src/dipdup/dipdup.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def __init__(self, ctx: HandlerContext) -> None:
async def add_index(self, index_config: IndexConfigTemplateT) -> None:
if index_config.name in self._indexes:
return
self._logger.info('Adding index `%s` to dispatcher')
self._logger.info('Adding index `%s` to dispatcher', index_config.name)
if isinstance(index_config, OperationIndexConfig):
datasource_name = cast(TzktDatasourceConfig, index_config.datasource).name
datasource = self._ctx.datasources[datasource_name]
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/hic_et_nunc.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_hic_et_nunc

database:
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/quipuswap.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_quipuswap

database:
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/registrydao.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_registrydao

database:
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/tezos_domains.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_tezos_domains

database:
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/tezos_domains_big_map.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_tezos_domains_big_map

database:
Expand Down
2 changes: 1 addition & 1 deletion tests/integration_tests/tzcolors.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_tzcolors

database:
Expand Down
2 changes: 1 addition & 1 deletion tests/test_dipdup/dipdup.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
spec_version: 0.1
spec_version: 1.0
package: demo_hic_et_nunc

database:
Expand Down