diff --git a/ch_backup/clickhouse/control.py b/ch_backup/clickhouse/control.py index 6f4f6816..10714154 100644 --- a/ch_backup/clickhouse/control.py +++ b/ch_backup/clickhouse/control.py @@ -24,6 +24,7 @@ from ch_backup.exceptions import ClickhouseBackupError from ch_backup.util import ( chown_dir_contents, + chown_file, escape, list_dir_files, retry, @@ -788,6 +789,26 @@ def chown_dir(self, dir_path: str) -> None: need_recursion, ) + def create_shadow_increment(self) -> None: + """ + Create shadow/increment.txt to fix race condition with parallel freeze. + Must be used before freezing more than one table at once. + """ + default_shadow_path = Path(self._root_data_path) / "shadow" + increment_path = default_shadow_path / "increment.txt" + if os.path.exists(increment_path): + return + if not os.path.exists(default_shadow_path): + os.mkdir(default_shadow_path) + self.chown_dir(str(default_shadow_path)) + with open(increment_path, "w", encoding="utf-8") as file: + file.write("0") + chown_file( + self._ch_ctl_config["user"], + self._ch_ctl_config["group"], + str(increment_path), + ) + @retry(OSError) def _remove_shadow_data(self, path: str) -> None: if path.find("/shadow") == -1: diff --git a/ch_backup/logic/table.py b/ch_backup/logic/table.py index 03c04b5f..0e6f07e2 100644 --- a/ch_backup/logic/table.py +++ b/ch_backup/logic/table.py @@ -120,6 +120,10 @@ def _backup( # See https://en.wikipedia.org/wiki/Optimistic_concurrency_control mtimes = self._collect_local_metadata_mtime(context, db, tables) + # Create shadow/increment.txt if not exists manually to avoid + # race condition with parallel freeze + context.ch_ctl.create_shadow_increment() + with ThreadPoolExecutor( max_workers=self._freeze_workers ) as freeze_executor: diff --git a/ch_backup/util.py b/ch_backup/util.py index 8070781a..1b23bb71 100644 --- a/ch_backup/util.py +++ b/ch_backup/util.py @@ -78,6 +78,13 @@ def chown_dir_contents( shutil.chown(os.path.join(dir_path, path), user, group) +def chown_file(user: str, group: str, file_path: str) -> None: + """ + Change directory user/group + """ + shutil.chown(file_path, user, group) + + def list_dir_files(dir_path: str) -> List[str]: """ Returns paths of all files of directory (recursively), relative to its path