diff --git a/src/dipdup/__init__.py b/src/dipdup/__init__.py index 6084375a1..0b6718bfb 100644 --- a/src/dipdup/__init__.py +++ b/src/dipdup/__init__.py @@ -5,3 +5,8 @@ '1.0': '>=1.0.0, <=1.1.2', '1.1': '>=2.0.0', } +spec_reindex_mapping = { + '0.1': False, + '1.0': False, + '1.1': True, +} \ No newline at end of file diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 49d1f062b..6945e2328 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -13,7 +13,7 @@ from fcache.cache import FileCache # type: ignore from sentry_sdk.integrations.aiohttp import AioHttpIntegration -from dipdup import __spec_version__, __version__, spec_version_mapping +from dipdup import __spec_version__, __version__, spec_version_mapping, spec_reindex_mapping from dipdup.codegen import DEFAULT_DOCKER_ENV_FILE, DEFAULT_DOCKER_IMAGE, DEFAULT_DOCKER_TAG, DipDupCodeGenerator from dipdup.config import DipDupConfig, LoggingConfig, PostgresDatabaseConfig from dipdup.dipdup import DipDup @@ -71,7 +71,8 @@ async def cli(ctx, config: List[str], env_file: List[str], logging_config: str): if _config.spec_version not in spec_version_mapping: raise ConfigurationError('Unknown `spec_version`, correct ones: {}') if _config.spec_version != __spec_version__ and ctx.invoked_subcommand != 'migrate': - raise MigrationRequiredError(None, _config.spec_version, __spec_version__) + reindex = spec_reindex_mapping[__spec_version__] + raise MigrationRequiredError(None, _config.spec_version, __spec_version__, reindex) if _config.sentry: sentry_sdk.init( diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index 7ae5d8cef..c123d981e 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -32,7 +32,7 @@ from dipdup.datasources.coinbase.datasource import CoinbaseDatasource from dipdup.datasources.datasource import IndexDatasource from dipdup.datasources.tzkt.datasource import TzktDatasource -from dipdup.exceptions import ConfigurationError +from dipdup.exceptions import ConfigurationError, ReindexingRequiredError from dipdup.hasura import HasuraGateway from dipdup.index import BigMapIndex, Index, OperationIndex from dipdup.models import BigMapData, HeadBlockData, IndexType, OperationData, State @@ -287,16 +287,20 @@ async def _initialize_database(self, reindex: bool = False) -> None: self._logger.info('Checking database schema') connection_name, connection = next(iter(Tortoise._connections.items())) - schema_sql = get_schema_sql(connection, False) - - # NOTE: Column order could differ in two generated schemas for the same models, drop commas and sort strings to eliminate this - processed_schema_sql = '\n'.join(sorted(schema_sql.replace(',', '').split('\n'))).encode() - schema_hash = hashlib.sha256(processed_schema_sql).hexdigest() try: schema_state = await State.get_or_none(index_type=IndexType.schema, index_name=connection_name) except OperationalError: schema_state = None + # TODO: Process exception in Tortoise + except KeyError as e: + raise ReindexingRequiredError(None) from e + + schema_sql = get_schema_sql(connection, False) + + # NOTE: Column order could differ in two generated schemas for the same models, drop commas and sort strings to eliminate this + processed_schema_sql = '\n'.join(sorted(schema_sql.replace(',', '').split('\n'))).encode() + schema_hash = hashlib.sha256(processed_schema_sql).hexdigest() # NOTE: `State.index_hash` field contains schema hash when `index_type` is `IndexType.schema` if schema_state is None: diff --git a/src/dipdup/exceptions.py b/src/dipdup/exceptions.py index ec1e9f407..52948d7ec 100644 --- a/src/dipdup/exceptions.py +++ b/src/dipdup/exceptions.py @@ -7,6 +7,8 @@ from dipdup import spec_version_mapping +_tab = '\n\n' + ('_' * 80) + '\n\n' + _migration_required_message = """Project migration required! {version_table} @@ -17,6 +19,14 @@ See https://baking-bad.org/blog/ for additional release information. """ +_reindexing_required_message = """Reindexing required! + +Recent changes in the framework have made it necessary to reindex the project. + + 1. Optionally backup a database + 2. Run `dipdup run --reindex` +""" + _handler_import_message = """Failed to import `{obj}` from `{module}`. Reasons in order of possibility: @@ -52,8 +62,6 @@ {error_context} """ -_tab = '\n\n' + ('_' * 80) + '\n\n' - class DipDupError(ABC, Exception): exit_code = 1 @@ -86,12 +94,13 @@ def format_help(self) -> str: class MigrationRequiredError(DipDupError): - """Project and DipDup spec versions don't match """ + """Project and DipDup spec versions don't match""" - def __init__(self, ctx, from_: str, to: str) -> None: + def __init__(self, ctx, from_: str, to: str, reindex: bool = False) -> None: super().__init__(ctx) self.from_ = from_ self.to = to + self.reindex = reindex def format_help(self) -> str: version_table = tabulate( @@ -101,7 +110,16 @@ def format_help(self) -> str: ], headers=['', 'spec_version', 'DipDup version'], ) - return _migration_required_message.format(version_table=version_table) + message = _migration_required_message.format(version_table=version_table) + if self.reindex: + message += _tab +_reindexing_required_message + return message + + +class ReindexingRequiredError(DipDupError): + """Performed migration requires reindexing""" + def format_help(self) -> str: + return _reindexing_required_message class HandlerImportError(DipDupError):