Fix race condition with shadow/increment.txt

ch_backup/clickhouse/control.py

@@ -24,6 +24,7 @@
 from ch_backup.exceptions import ClickhouseBackupError
 from ch_backup.util import (
     chown_dir_contents,
+    chown_file,
     escape,
     list_dir_files,
     retry,
@@ -788,6 +789,26 @@ def chown_dir(self, dir_path: str) -> None:
             need_recursion,
         )
 
+    def create_shadow_increment(self) -> None:
+        """
+        Create shadow/increment.txt to fix race condition with parallel freeze.
+        Must be used before freezing more than one table at once.
+        """
+        default_shadow_path = Path(self._root_data_path) / "shadow"
+        increment_path = default_shadow_path / "increment.txt"
+        if os.path.exists(increment_path):
+            return
+        if not os.path.exists(default_shadow_path):
+            os.mkdir(default_shadow_path)
+            self.chown_dir(str(default_shadow_path))
+        with open(increment_path, "w", encoding="utf-8") as file:
+            file.write("0")
+            chown_file(
+                self._ch_ctl_config["user"],
+                self._ch_ctl_config["group"],
+                str(increment_path),
+            )
+
     @retry(OSError)
     def _remove_shadow_data(self, path: str) -> None:
         if path.find("/shadow") == -1:

ch_backup/logic/table.py

@@ -120,6 +120,10 @@ def _backup(
             # See https://en.wikipedia.org/wiki/Optimistic_concurrency_control
             mtimes = self._collect_local_metadata_mtime(context, db, tables)
 
+            # Create shadow/increment.txt if not exists manually to avoid
+            # race condition with parallel freeze
+            context.ch_ctl.create_shadow_increment()
+
             with ThreadPoolExecutor(
                 max_workers=self._freeze_workers
             ) as freeze_executor:

ch_backup/util.py

@@ -78,6 +78,13 @@ def chown_dir_contents(
             shutil.chown(os.path.join(dir_path, path), user, group)
 
 
+def chown_file(user: str, group: str, file_path: str) -> None:
+    """
+    Change directory user/group
+    """
+    shutil.chown(file_path, user, group)
+
+
 def list_dir_files(dir_path: str) -> List[str]:
     """
     Returns paths of all files of directory (recursively), relative to its path