ClickHouse 23.9 support (#75)

* ClickHouse 23.9 support [MDB-24031] * handling of backward-incompatible changes in clickhouse-disks
yandex · Oct 1, 2023 · aeb2227 · aeb2227
1 parent 9c10f49
commit aeb2227
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 23 deletions.
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -56,6 +56,8 @@ jobs:
           clickhouse: "22.8.21.38"
         - python: "3.10"
           clickhouse: "23.3.13.6"
+        - python: "3.10"
+          clickhouse: "23.8.3.48"
         - python: "3.10"
           clickhouse: "latest"
         - python: "3.11"

diff --git a/ch_backup/clickhouse/control.py b/ch_backup/clickhouse/control.py
@@ -277,25 +277,14 @@ def __init__(self, ch_ctl_config: dict, main_config: dict) -> None:
         self._disks = self.get_disks()
         settings = {
             "allow_experimental_database_materialized_postgresql": 1,
+            "allow_experimental_database_materialized_mysql": 1,
             "allow_experimental_database_replicated": 1,
             "allow_experimental_funnel_functions": 1,
             "allow_experimental_live_view": 1,
+            "allow_experimental_window_view": 1,
             "allow_suspicious_codecs": 1,
             "allow_suspicious_low_cardinality_types": 1,
         }
-        if self.ch_version_ge("21.9"):
-            settings.update(
-                {
-                    "allow_experimental_database_materialized_mysql": 1,
-                    "allow_experimental_nlp_functions": 1,
-                }
-            )
-        if self.ch_version_ge("21.12"):
-            settings.update(
-                {
-                    "allow_experimental_window_view": 1,
-                }
-            )
         if self.ch_version_ge("22.3"):
             settings.update(
                 {

diff --git a/ch_backup/clickhouse/disks.py b/ch_backup/clickhouse/disks.py
@@ -157,12 +157,44 @@ def copy_part(
                     part.name,
                     "",
                 )
-                _copy_dir(source_disk.name, source_path, target_disk.name, target_path)
+                self._copy_dir(
+                    source_disk.name, source_path, target_disk.name, target_path
+                )
                 return
         raise RuntimeError(
             f'Disk "{target_disk.name}" path not found for table `{table.database}`.`{table.name}`'
         )
 
+    def _copy_dir(
+        self, from_disk: str, from_path: str, to_disk: str, to_path: str
+    ) -> None:
+        if self._ch_ctl.ch_version_ge("23.9"):
+            command_args = [
+                "--disk-from",
+                from_disk,
+                "--disk-to",
+                to_disk,
+                from_path,
+                to_path,
+            ]
+        else:
+            command_args = [
+                "--diskFrom",
+                from_disk,
+                "--diskTo",
+                to_disk,
+                from_path,
+                to_path,
+            ]
+
+        result = _exec(
+            "copy",
+            common_args=[],
+            command_args=command_args,
+        )
+
+        logging.info(f"clickhouse-disks copy result: {os.linesep.join(result)}")
+
 
 def _get_config_path(config_dir: str, disk_name: str) -> str:
     return os.path.join(config_dir, f"cloud_storage_tmp_disk_{disk_name}.xml")
@@ -172,15 +204,6 @@ def _get_tmp_disk_name(disk_name: str) -> str:
     return f"{disk_name}_source"
 
 
-def _copy_dir(from_disk: str, from_path: str, to_disk: str, to_path: str) -> None:
-    result = _exec(
-        "copy",
-        common_args=[],
-        command_args=["--diskFrom", from_disk, "--diskTo", to_disk, from_path, to_path],
-    )
-    logging.warning(f"clickhouse-disks copy result: {os.linesep.join(result)}")
-
-
 def _exec(command: str, common_args: List[str], command_args: List[str]) -> List[str]:
     ch_disks_logger = logging.getLogger("clickhouse-disks")
     command_args = [