From 849cd5b9135b0e5915dd7e21331a469b81d9a900 Mon Sep 17 00:00:00 2001 From: MikhailBurdukov <102754618+MikhailBurdukov@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:42:20 +0300 Subject: [PATCH] MDB-30734 Metadata cleanup with system drop replica (#185) * MDB-30734 Metadata cleanup with system drop replica * test fix * Add cleanup for database * Test fix * fix * again * Fix for 22.8 * Review fix * Missed --- ch_backup/backup/metadata/backup_metadata.py | 10 +- ch_backup/ch_backup.py | 18 ++- ch_backup/clickhouse/control.py | 67 ++++++++- ch_backup/clickhouse/metadata_cleaner.py | 127 ++++++++++++++++++ ch_backup/clickhouse/models.py | 9 +- ch_backup/config.py | 1 + ch_backup/logic/database.py | 16 ++- ch_backup/logic/table.py | 24 +--- ch_backup/util.py | 12 +- ch_backup/zookeeper/zookeeper.py | 67 +-------- images/clickhouse/config/shared_zookeeper.xml | 18 +++ images/clickhouse/entrypoint.py | 6 + tests/integration/configuration.py | 1 + .../features/backup_replicated.feature | 72 ++++++++++ tests/integration/modules/clickhouse.py | 14 ++ tests/integration/modules/zookeeper.py | 12 ++ tests/integration/steps/clickhouse.py | 33 +++++ tests/integration/steps/zookeeper.py | 8 ++ tests/unit/test_backup_tables.py | 4 +- tests/unit/test_upload_part_observer.py | 4 +- 20 files changed, 422 insertions(+), 101 deletions(-) create mode 100644 ch_backup/clickhouse/metadata_cleaner.py create mode 100644 images/clickhouse/config/shared_zookeeper.xml diff --git a/ch_backup/backup/metadata/backup_metadata.py b/ch_backup/backup/metadata/backup_metadata.py index 31dd2981..bdb9f746 100644 --- a/ch_backup/backup/metadata/backup_metadata.py +++ b/ch_backup/backup/metadata/backup_metadata.py @@ -228,7 +228,13 @@ def get_database(self, db_name: str) -> Database: Get database. """ db_meta = self._databases[db_name] - return Database(db_name, db_meta.get("engine"), db_meta.get("metadata_path")) + return Database( + db_name, + db_meta.get("engine"), + db_meta.get("metadata_path"), + db_meta.get("uuid"), + db_meta.get("engine_full"), + ) def add_database(self, db: Database) -> None: """ @@ -239,6 +245,8 @@ def add_database(self, db: Database) -> None: self._databases[db.name] = { "engine": db.engine, "metadata_path": db.metadata_path, + "uuid": db.uuid, + "engine_full": db.engine_full, "tables": {}, } diff --git a/ch_backup/ch_backup.py b/ch_backup/ch_backup.py index b425db47..82a8497f 100644 --- a/ch_backup/ch_backup.py +++ b/ch_backup/ch_backup.py @@ -16,6 +16,7 @@ from ch_backup.backup.metadata import BackupMetadata, BackupState, TableMetadata from ch_backup.backup.sources import BackupSources from ch_backup.backup_context import BackupContext +from ch_backup.clickhouse.metadata_cleaner import MetadataCleaner, select_replica_drop from ch_backup.clickhouse.models import Database from ch_backup.config import Config from ch_backup.exceptions import ( @@ -514,8 +515,20 @@ def _restore( db.set_engine_from_sql(db_sql) databases[db_name] = db + metadata_cleaner: Optional[MetadataCleaner] = None + + if clean_zookeeper and len(self._context.zk_config.get("hosts")) > 0: + metadata_cleaner = MetadataCleaner( + self._context.ch_ctl, + select_replica_drop( + replica_name, self._context.ch_ctl.get_macros() + ), + ) + # Restore databases. - self._database_backup_manager.restore(self._context, databases, keep_going) + self._database_backup_manager.restore( + self._context, databases, keep_going, metadata_cleaner + ) # Restore tables and data stored on local disks. self._table_backup_manager.restore( @@ -524,12 +537,11 @@ def _restore( schema_only=sources.schema_only, tables=tables, exclude_tables=exclude_tables, - replica_name=replica_name, + metadata_cleaner=metadata_cleaner, cloud_storage_source_bucket=cloud_storage_source_bucket, cloud_storage_source_path=cloud_storage_source_path, cloud_storage_source_endpoint=cloud_storage_source_endpoint, skip_cloud_storage=skip_cloud_storage, - clean_zookeeper=clean_zookeeper, keep_going=keep_going, ) diff --git a/ch_backup/clickhouse/control.py b/ch_backup/clickhouse/control.py index c01c88b1..7cf8eb1e 100644 --- a/ch_backup/clickhouse/control.py +++ b/ch_backup/clickhouse/control.py @@ -155,12 +155,39 @@ """ ) +DROP_REPLICA_BY_ZK_PATH_SQL = strip_query( + """ + SYSTEM DROP REPLICA '{replica_name}' FROM ZKPATH '{zk_path}' +""" +) + +DROP_DATABASE_REPLICA_BY_ZK_PATH_SQL = strip_query( + """ + SYSTEM DROP DATABASE REPLICA '{replica_name}' FROM ZKPATH '{zk_path}' +""" +) + GET_DATABASES_SQL = strip_query( """ SELECT name, engine, - metadata_path + metadata_path, + uuid, + engine_full + FROM system.databases + WHERE name NOT IN ('system', '_temporary_and_external_tables', 'information_schema', 'INFORMATION_SCHEMA', '{system_db}') + FORMAT JSON +""" +) + +GET_DATABASES_SQL_22_8 = strip_query( + """ + SELECT + name, + engine, + metadata_path, + uuid FROM system.databases WHERE name NOT IN ('system', '_temporary_and_external_tables', 'information_schema', 'INFORMATION_SCHEMA', '{system_db}') FORMAT JSON @@ -361,6 +388,7 @@ def __init__( self._freeze_timeout = self._ch_ctl_config["freeze_timeout"] self._unfreeze_timeout = self._ch_ctl_config["unfreeze_timeout"] self._restore_replica_timeout = self._ch_ctl_config["restore_replica_timeout"] + self._drop_replica_timeout = self._ch_ctl_config["drop_replica_timeout"] self._ch_client = ClickhouseClient(self._ch_ctl_config) self._ch_version = self._ch_client.query(GET_VERSION_SQL) self._disks = self.get_disks() @@ -503,12 +531,23 @@ def get_databases( result: List[Database] = [] system_database = self._backup_config["system_database"] - ch_resp = self._ch_client.query( + + query = ( GET_DATABASES_SQL.format(system_db=system_database) + if self.ch_version_ge("23.3") + else GET_DATABASES_SQL_22_8.format(system_db=system_database) ) + + ch_resp = self._ch_client.query(query) if "data" in ch_resp: result = [ - Database(row["name"], row["engine"], row["metadata_path"]) + Database( + row["name"], + row["engine"], + row["metadata_path"], + row["uuid"], + row.get("engine_full"), + ) for row in ch_resp["data"] if row["name"] not in exclude_dbs ] @@ -666,6 +705,28 @@ def drop_named_collection(self, nc_name: str) -> None: """ self._ch_client.query(DROP_NAMED_COLLECTION_SQL.format(nc_name=escape(nc_name))) + def system_drop_replica(self, replica: str, zookeeper_path: str) -> None: + """ + System drop replica query. + """ + self._ch_client.query( + DROP_REPLICA_BY_ZK_PATH_SQL.format( + replica_name=replica, zk_path=zookeeper_path + ), + timeout=self._drop_replica_timeout, + ) + + def system_drop_database_replica(self, replica: str, zookeeper_path: str) -> None: + """ + System drop database replica query. + """ + self._ch_client.query( + DROP_DATABASE_REPLICA_BY_ZK_PATH_SQL.format( + replica_name=replica, zk_path=zookeeper_path + ), + timeout=self._drop_replica_timeout, + ) + def get_database_metadata_path(self, database: str) -> str: """ Get filesystem absolute path to database metadata. diff --git a/ch_backup/clickhouse/metadata_cleaner.py b/ch_backup/clickhouse/metadata_cleaner.py new file mode 100644 index 00000000..c47004ca --- /dev/null +++ b/ch_backup/clickhouse/metadata_cleaner.py @@ -0,0 +1,127 @@ +""" +Zookeeper metadata cleaner for clickhouse. +""" + +import copy +import os +from typing import Dict, List, Optional + +from ch_backup import logging +from ch_backup.clickhouse.client import ClickhouseError +from ch_backup.clickhouse.control import ClickhouseCTL +from ch_backup.clickhouse.models import Database, Table +from ch_backup.exceptions import ConfigurationError +from ch_backup.util import ( + get_database_zookeeper_paths, + get_table_zookeeper_paths, + replace_macros, +) + + +def select_replica_drop(replica_name: Optional[str], macros: Dict) -> str: + """ + Select replica to drop from zookeeper. + """ + selected_replica = replica_name + if not selected_replica: + selected_replica = macros.get("replica", None) + + if not selected_replica: + raise ConfigurationError( + "Can't get the replica name. Please, specify it through macros or replica_name knob." + ) + return selected_replica + + +class MetadataCleaner: + """ + Class for cleaning up replica metadata from zookeeper. + """ + + def __init__(self, ch_ctl: ClickhouseCTL, replica_to_drop: str) -> None: + self._ch_ctl = ch_ctl + self._macros = self._ch_ctl.get_macros() + self._replica_to_drop = replica_to_drop or self._macros.get("replica") + + if not self._replica_to_drop: + raise ConfigurationError( + "Can't get the replica name. Please, specify it through macros or replica_name knob." + ) + + def clean_tables_metadata(self, replicated_tables: List[Table]) -> None: + """ + Remove replica tables metadata from zookeeper. + """ + replicated_table_paths = get_table_zookeeper_paths(replicated_tables) + + for table, table_path in replicated_table_paths: + table_macros = copy.copy(self._macros) + macros_to_override = dict( + database=table.database, table=table.name, uuid=table.uuid + ) + table_macros.update(macros_to_override) + + path_resolved = os.path.abspath(replace_macros(table_path, table_macros)) + + logging.debug( + "Removing replica {} from table {} metadata from zookeeper {}.", + self._replica_to_drop, + f"{table.database}.{table.database}", + path_resolved, + ) + try: + self._ch_ctl.system_drop_replica( + replica=self._replica_to_drop, zookeeper_path=path_resolved # type: ignore + ) + except ClickhouseError as ch_error: + if "does not look like a table path" in str(ch_error): + logging.warning( + "System drop replica failed with: {}\n Will ignore it, probably different configuration for zookeeper or tables schema.", + repr(ch_error), + ) + else: + raise + + def clean_database_metadata(self, replicated_databases: List[Database]) -> None: + """ + Remove replica database metadata from zookeeper. + """ + if not self._ch_ctl.ch_version_ge("23.3"): + logging.warning( + "Ch version is too old, will skip replicated database cleanup." + ) + return + + replicated_databases_paths = get_database_zookeeper_paths(replicated_databases) + + for database, database_path, shard in replicated_databases_paths: + db_macros = copy.copy(self._macros) + + macros_to_override = dict(database=database.name, uuid=database.uuid) + db_macros.update(macros_to_override) + + path_resolved = os.path.abspath(replace_macros(database_path, db_macros)) + full_replica_name = ( + f"{replace_macros(shard, db_macros)}|{self._replica_to_drop}" + ) + + logging.debug( + "Removing replica {} from database {} metadata from zookeeper {}.", + full_replica_name, + database.name, + path_resolved, + ) + try: + self._ch_ctl.system_drop_database_replica( + replica=full_replica_name, zookeeper_path=path_resolved # type: ignore + ) + except ClickhouseError as ch_error: + if "does not look like a path of Replicated database" in str( + ch_error + ) or "node doesn't exist" in str(ch_error): + logging.warning( + "System drop database replica failed with: {}\n Will ignore it, probably different configuration for zookeeper or database schema.", + repr(ch_error), + ) + else: + raise diff --git a/ch_backup/clickhouse/models.py b/ch_backup/clickhouse/models.py index 10679373..20746695 100644 --- a/ch_backup/clickhouse/models.py +++ b/ch_backup/clickhouse/models.py @@ -167,12 +167,19 @@ class Database(SimpleNamespace): """ def __init__( - self, name: str, engine: Optional[str], metadata_path: Optional[str] + self, + name: str, + engine: Optional[str], + metadata_path: Optional[str], + uuid: Optional[str], + engine_full: Optional[str], ) -> None: super().__init__() self.name = name self.engine = engine self.metadata_path = metadata_path + self.uuid = uuid + self.engine_full = engine_full def is_atomic(self) -> bool: """ diff --git a/ch_backup/config.py b/ch_backup/config.py index 1658096f..c6dc9076 100644 --- a/ch_backup/config.py +++ b/ch_backup/config.py @@ -34,6 +34,7 @@ def _as_seconds(t: str) -> int: "freeze_timeout": _as_seconds("45 min"), "unfreeze_timeout": _as_seconds("1 hour"), "restore_replica_timeout": _as_seconds("30 min"), + "drop_replica_timeout": _as_seconds("1 hour"), "user": "clickhouse", "group": "clickhouse", "clickhouse_user": None, diff --git a/ch_backup/logic/database.py b/ch_backup/logic/database.py index d90089bf..c187e590 100644 --- a/ch_backup/logic/database.py +++ b/ch_backup/logic/database.py @@ -2,10 +2,11 @@ Clickhouse backup logic for databases """ -from typing import Dict, Sequence +from typing import Dict, Optional, Sequence from ch_backup import logging from ch_backup.backup_context import BackupContext +from ch_backup.clickhouse.metadata_cleaner import MetadataCleaner from ch_backup.clickhouse.models import Database from ch_backup.clickhouse.schema import ( embedded_schema_db_sql, @@ -32,7 +33,10 @@ def backup(self, context: BackupContext, databases: Sequence[Database]) -> None: @staticmethod def restore( - context: BackupContext, databases: Dict[str, Database], keep_going: bool + context: BackupContext, + databases: Dict[str, Database], + keep_going: bool, + metadata_cleaner: Optional[MetadataCleaner], ) -> None: """ Restore database objects. @@ -56,6 +60,14 @@ def restore( databases_to_restore[name] = db continue + if metadata_cleaner: + replicated_databases = [ + database + for database in databases_to_restore.values() + if database.is_replicated_db_engine() + ] + metadata_cleaner.clean_database_metadata(replicated_databases) + logging.info("Restoring databases: {}", ", ".join(databases_to_restore.keys())) for db in databases_to_restore.values(): if db.has_embedded_metadata(): diff --git a/ch_backup/logic/table.py b/ch_backup/logic/table.py index c57f1f12..d8b0c372 100644 --- a/ch_backup/logic/table.py +++ b/ch_backup/logic/table.py @@ -18,6 +18,7 @@ from ch_backup.backup_context import BackupContext from ch_backup.clickhouse.client import ClickhouseError from ch_backup.clickhouse.disks import ClickHouseTemporaryDisks +from ch_backup.clickhouse.metadata_cleaner import MetadataCleaner from ch_backup.clickhouse.models import Database, FrozenPart, Table from ch_backup.clickhouse.schema import ( rewrite_table_schema, @@ -27,7 +28,7 @@ from ch_backup.exceptions import ClickhouseBackupError from ch_backup.logic.backup_manager import BackupManager from ch_backup.logic.upload_part_observer import UploadPartObserver -from ch_backup.util import compare_schema, get_table_zookeeper_paths +from ch_backup.util import compare_schema @dataclass @@ -271,12 +272,11 @@ def restore( schema_only: bool, tables: List[TableMetadata], exclude_tables: List[TableMetadata], - replica_name: Optional[str], + metadata_cleaner: Optional[MetadataCleaner], cloud_storage_source_bucket: Optional[str], cloud_storage_source_path: Optional[str], cloud_storage_source_endpoint: Optional[str], skip_cloud_storage: bool, - clean_zookeeper: bool, keep_going: bool, ) -> None: """ @@ -341,8 +341,7 @@ def restore( context, databases, tables_to_restore, - clean_zookeeper, - replica_name, + metadata_cleaner, keep_going, ) @@ -587,8 +586,7 @@ def _restore_tables( context: BackupContext, databases: Dict[str, Database], tables: Iterable[Table], - clean_zookeeper: bool = False, - replica_name: Optional[str] = None, + metadata_cleaner: Optional[MetadataCleaner], keep_going: bool = False, ) -> List[Table]: logging.info("Preparing tables for restoring") @@ -614,19 +612,11 @@ def _restore_tables( else: other_tables.append(table) - if clean_zookeeper and len(context.zk_config.get("hosts")) > 0: # type: ignore - macros = context.ch_ctl.get_macros() + if metadata_cleaner: # type: ignore replicated_tables = [ table for table in merge_tree_tables if table.is_replicated() ] - - logging.debug( - "Deleting replica metadata for replicated tables: {}", - ", ".join([f"{t.database}.{t.name}" for t in replicated_tables]), - ) - context.zk_ctl.delete_replica_metadata( - get_table_zookeeper_paths(replicated_tables), replica_name, macros - ) + metadata_cleaner.clean_tables_metadata(replicated_tables) return self._restore_table_objects( context, diff --git a/ch_backup/util.py b/ch_backup/util.py index 2d1a3913..def4f5d6 100644 --- a/ch_backup/util.py +++ b/ch_backup/util.py @@ -301,21 +301,19 @@ def get_table_zookeeper_paths(tables: Iterable) -> Iterable[Tuple]: return result -def get_database_zookeeper_paths(databases: Iterable[str]) -> Iterable[str]: +def get_database_zookeeper_paths(databases: Iterable) -> Iterable[Tuple]: """ - Parse ZooKeeper path from create statement. + Parse ZooKeeper path from database create statement and return path to database and shard from schema. """ result = [] - for db_sql in databases: + for database in databases: match = re.search( R"""Replicated\(\'(?P[^']+)\', '(?P[^']+)', '(?P[^']+)'""", - db_sql, + database.engine_full, ) if not match: continue - result.append( - f'{match.group("zk_path")}/replicas/{match.group("shard")}|{match.group("replica")}' - ) + result.append((database, match.group("zk_path"), match.group("shard"))) return result diff --git a/ch_backup/zookeeper/zookeeper.py b/ch_backup/zookeeper/zookeeper.py index 729173cd..054dd07e 100644 --- a/ch_backup/zookeeper/zookeeper.py +++ b/ch_backup/zookeeper/zookeeper.py @@ -3,18 +3,12 @@ """ import logging as py_logging -import os -from typing import Dict, Iterable, Tuple from kazoo.client import KazooClient -from kazoo.exceptions import KazooException, NoNodeError +from kazoo.exceptions import KazooException from kazoo.handlers.threading import KazooTimeoutError -from ch_backup import logging -from ch_backup.exceptions import ConfigurationError - -from ..clickhouse.models import Table -from ..util import replace_macros, retry +from ..util import retry KAZOO_RETRIES = retry( (KazooException, KazooTimeoutError), max_attempts=5, max_interval=60 @@ -80,60 +74,3 @@ def zk_root_path(self) -> str: Getter zk_root_path """ return self._zk_root_path - - @KAZOO_RETRIES - def delete_replica_metadata( - self, tables: Iterable[Tuple[Table, str]], replica: str, macros: Dict = None - ) -> None: - """ - Remove replica metadata from zookeeper for all tables from args. - """ - if macros is None: - macros = {} - if replica is None: - if macros.get("replica") is not None: - replica = macros.get("replica") - else: - raise ConfigurationError( - "Can't get the replica name. Please, specify it through macros or replica_name knob." - ) - - with self._zk_client as client: - for table, table_path in tables: - table_macros = dict( - database=table.database, table=table.name, uuid=table.uuid - ) - table_macros.update(macros) - path = os.path.join( - self._zk_root_path, - replace_macros(table_path[1:], table_macros), - "replicas", - replica, - ) # remove leading '/' - logging.debug(f'Deleting zk node: "{path}"') - try: - client.delete(path, recursive=True) - except NoNodeError: - pass - - @KAZOO_RETRIES - def delete_replicated_database_metadata( - self, databases: Iterable[str], replica: str, macros: Dict = None - ) -> None: - """ - Remove replica metadata from zookeeper for all replicated databases from args. - """ - if macros is None: - macros = {} - macros["replica"] = replica - - with self._zk_client as client: - for zk_path in databases: - path = os.path.join( - self._zk_root_path, replace_macros(zk_path[1:], macros) - ) # remove leading '/' - logging.debug(f'Deleting zk node: "{path}"') - try: - client.delete(path, recursive=True) - except NoNodeError: - pass diff --git a/images/clickhouse/config/shared_zookeeper.xml b/images/clickhouse/config/shared_zookeeper.xml new file mode 100644 index 00000000..268eeb15 --- /dev/null +++ b/images/clickhouse/config/shared_zookeeper.xml @@ -0,0 +1,18 @@ + +{% if feature_enabled('zookeeper') %} + + + {{ conf.zk.uri }} +{% if conf.zk.secure %} + {{ conf.zk.secure_port }} + 1 +{% else %} + {{ conf.zk.port }} +{% endif %} + + 3000 + /{{ conf.zk.shared_node }} + {{ conf.zk.user }}:{{ conf.zk.password }} + +{% endif %} + diff --git a/images/clickhouse/entrypoint.py b/images/clickhouse/entrypoint.py index 4781254c..7f1b7d97 100644 --- a/images/clickhouse/entrypoint.py +++ b/images/clickhouse/entrypoint.py @@ -9,7 +9,13 @@ if __name__ == "__main__": client = KazooClient("{{ conf.zk.uri }}:{{ conf.zk.port }}") client.start() + + # In our test by default ch nodes have different root paths in the zookeeper(`/clickhouse01/` and `/clickhouse02/`). + # So they are not connected to each other. + # If you need the same path for nodes, use `step_enable_shared_zookeeper_for_clickhouse` step to override configs. client.ensure_path("/{{ instance_name }}") + client.ensure_path("/{{ conf.zk.shared_node }}") + client.stop() subprocess.Popen(["supervisord", "-c", "/etc/supervisor/supervisord.conf"]).wait() diff --git a/tests/integration/configuration.py b/tests/integration/configuration.py index f0d5733a..4fb6e6a8 100644 --- a/tests/integration/configuration.py +++ b/tests/integration/configuration.py @@ -42,6 +42,7 @@ def create(): "secure": True, "user": "clickhouse", "password": "password.password.password", + "shared_node": "shared", } return { diff --git a/tests/integration/features/backup_replicated.feature b/tests/integration/features/backup_replicated.feature index 21ae3d1f..60343255 100644 --- a/tests/integration/features/backup_replicated.feature +++ b/tests/integration/features/backup_replicated.feature @@ -786,3 +786,75 @@ Feature: Backup replicated merge tree table When we restore clickhouse backup #0 to clickhouse02 Then clickhouse02 has same schema as clickhouse01 And we got same clickhouse data at clickhouse01 clickhouse02 + + + Scenario: Host resetup with zookeeper table cleanup + Given we have enabled shared zookeeper for clickhouse01 + And we have enabled shared zookeeper for clickhouse02 + And we have executed queries on clickhouse01 + """ + DROP DATABASE IF EXISTS test_db SYNC; + CREATE DATABASE test_db; + CREATE TABLE test_db.table_01 ( + EventDate DateTime, + CounterID UInt32, + UserID UInt32 + ) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/shard01/test_db.table_01', '{replica}') + PARTITION BY CounterID % 10 + ORDER BY (CounterID, EventDate, intHash32(UserID)) + SAMPLE BY intHash32(UserID); + INSERT INTO test_db.table_01 SELECT now(), number, rand() FROM system.numbers LIMIT 10 + """ + When we create clickhouse01 clickhouse backup + Then we got the following backups on clickhouse01 + | num | state | data_count | link_count | + | 0 | created | 10 | 0 | + + When we stop clickhouse at clickhouse01 + When we restore clickhouse backup #0 to clickhouse02 + """ + replica_name: clickhouse01 + schema_only: true + """ + When we start clickhouse at clickhouse01 + Then there are no zk node on zookeeper01 + """ + zookeeper_path: /{{ conf.zk.shared_node }}/clickhouse/tables/shard01/test_db.table_01/replicas/clickhouse01 + """ + + Scenario Outline: Host resetup with database table cleanup + Given we have enabled shared zookeeper for clickhouse01 + And we have enabled shared zookeeper for clickhouse02 + Given ClickHouse settings + """ + allow_experimental_database_replicated: 1 + """ + And we have executed queries on clickhouse01 + """ + DROP DATABASE IF EXISTS db_repl SYNC; + CREATE DATABASE db_repl ENGINE = Replicated('', 'shard_01', '{replica}') + """ + When we create clickhouse01 clickhouse backup + Then we got the following backups on clickhouse01 + | num | state | data_count | link_count | + | 0 | created | 0 | 0 | + + When we stop clickhouse at clickhouse01 + When we restore clickhouse backup #0 to clickhouse02 + """ + replica_name: clickhouse01 + schema_only: true + """ + When we start clickhouse at clickhouse01 + Then database replica db_repl on clickhouse01 does not exists + + @require_version_24.8 + Examples: + | zookeeper_path | + |/databases/{uuid}/db_repl| + + @require_version_22.8 + Examples: + | zookeeper_path | + |/databases/replicated/db_repl| diff --git a/tests/integration/modules/clickhouse.py b/tests/integration/modules/clickhouse.py index a37dd000..f2ddb4bd 100644 --- a/tests/integration/modules/clickhouse.py +++ b/tests/integration/modules/clickhouse.py @@ -257,6 +257,20 @@ def drop_database(self, db_name: str) -> None: """ self._query("POST", f"DROP DATABASE `{db_name}`") + def is_replica_ro(self, database: str, table: str) -> int: + resp = self._query( + "GET", + f"SELECT is_readonly FROM system.replicas WHERE database='{database}' and table='{table}' FORMAT JSON", + )["data"][0] + return resp["is_readonly"] + + def is_database_replica_exists(self, database: str) -> bool: + resp = self._query( + "GET", + f"SELECT count() as cnt FROM system.clusters WHERE cluster='{database}' and host_name like '{{replica}}%' FORMAT JSON", + )["data"][0] + return int(resp["cnt"]) > 0 + def drop_test_table(self, db_num: int, table_num: int) -> None: """ Drop test table. diff --git a/tests/integration/modules/zookeeper.py b/tests/integration/modules/zookeeper.py index d9c2a19f..4fe3f584 100644 --- a/tests/integration/modules/zookeeper.py +++ b/tests/integration/modules/zookeeper.py @@ -54,6 +54,18 @@ def delete_znode(context: ContextT, node: str, znode: str) -> None: zk.stop() +def znode_exists(context: ContextT, node: str, zk_path: str) -> bool: + """ + Check if the znode is exists. + """ + zk = _get_zookeeper_client(context, node) + zk.start() + + result = zk.exists(path=zk_path) + zk.stop() + return result + + def get_children_list(context: ContextT, node: str, zk_path: str) -> Optional[List]: zk = _get_zookeeper_client(context, node) zk_config = context.conf.get("zk", {}) diff --git a/tests/integration/steps/clickhouse.py b/tests/integration/steps/clickhouse.py index 91fffe64..4ad840c0 100644 --- a/tests/integration/steps/clickhouse.py +++ b/tests/integration/steps/clickhouse.py @@ -21,6 +21,23 @@ def step_wait_for_clickhouse_alive(context, node): ClickhouseClient(context, node).ping() +@given("we have enabled shared zookeeper for {node:w}") +def step_enable_shared_zookeeper_for_clickhouse(context, node): + """ + Replace a part of CH config on the fly to enable shared zookeeper for clickhouse nodes. + """ + container = get_container(context, node) + + override_config = "/config/shared_zookeeper.xml" + assert ( + container.exec_run( + f"ln -s {override_config} /etc/clickhouse-server/conf.d/" + ).exit_code + == 0 + ) + assert container.exec_run("supervisorctl restart clickhouse").exit_code == 0 + + @given("clickhouse on {node:w} has test schema") @when("clickhouse on {node:w} has test schema") def step_init_test_schema(context, node): @@ -244,6 +261,16 @@ def step_stop_clickhouse(context, node): context.exit_code = result.exit_code +@when("we start clickhouse at {node:w}") +def step_start_clickhouse(context, node): + container = get_container(context, node) + result = container.exec_run( + ["bash", "-c", "supervisorctl start clickhouse"], user="root" + ) + context.response = result.output.decode().strip() + context.exit_code = result.exit_code + + @when("we save all user's data in context on {node:w}") def step_save_user_data(context, node): ch_client = ClickhouseClient(context, node) @@ -255,3 +282,9 @@ def step_check_data_equal(context, node): ch_client = ClickhouseClient(context, node) new_user_data = ch_client.get_all_user_data() assert new_user_data == context.user_data + + +@then("database replica {database} on {node:w} does not exists") +def step_check_no_database_replica(context, database, node): + ch_client = ClickhouseClient(context, node) + assert not ch_client.is_database_replica_exists(database) diff --git a/tests/integration/steps/zookeeper.py b/tests/integration/steps/zookeeper.py index 08530b37..bbe84b7b 100644 --- a/tests/integration/steps/zookeeper.py +++ b/tests/integration/steps/zookeeper.py @@ -4,11 +4,13 @@ from behave import given, then, when from hamcrest import assert_that, has_length +from tests.integration.modules.steps import get_step_data from tests.integration.modules.zookeeper import ( delete_znode, get_children_list, initialize_zookeeper_roots, write_znode, + znode_exists, ) @@ -56,3 +58,9 @@ def step_acquire_zookeeper_lock(context, node, zk_lock_path): @when("we release zookeeper lock on {node:w} with path {zk_lock_path}") def step_release_zookeeper_lock(context, node, zk_lock_path): delete_znode(context, node, zk_lock_path + "/__lock__-0000000000") + + +@then("there are no zk node on {node:w}") +def step_zk_node_not_exists(context, node): + data = get_step_data(context) + assert not znode_exists(context, node, data["zookeeper_path"]) diff --git a/tests/unit/test_backup_tables.py b/tests/unit/test_backup_tables.py index ca4c641f..2e7eb790 100644 --- a/tests/unit/test_backup_tables.py +++ b/tests/unit/test_backup_tables.py @@ -27,7 +27,9 @@ def test_backup_table_skipping_if_metadata_updated_during_backup( # Prepare involved data objects context = BackupContext(DEFAULT_CONFIG) # type: ignore[arg-type] - db = Database(db_name, "MergeTree", "/var/lib/clickhouse/metadata/db1.sql") + db = Database( + db_name, "MergeTree", "/var/lib/clickhouse/metadata/db1.sql", None, None + ) table_backup = TableBackup() backup_meta = BackupMetadata( name="20181017T210300", diff --git a/tests/unit/test_upload_part_observer.py b/tests/unit/test_upload_part_observer.py index f5f928a1..68c4b34d 100644 --- a/tests/unit/test_upload_part_observer.py +++ b/tests/unit/test_upload_part_observer.py @@ -24,7 +24,9 @@ time_format="%Y-%m-%dT%H:%M:%S%Z", hostname="clickhouse01.test_net_711", ) -DB = Database(DB_NAME, ENGINE, f"/var/lib/clickhouse/metadata/{DB_NAME}.sql") +DB = Database( + DB_NAME, ENGINE, f"/var/lib/clickhouse/metadata/{DB_NAME}.sql", None, None +) @parametrize(