Fix handling of compressed timeline files on startup

Removed hardcoded WAL file type from upload event.
Aiven-Open · Aug 18, 2022 · c794f30 · c794f30
1 parent f0d3221
commit c794f30
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 12 deletions.
diff --git a/pghoard/pghoard.py b/pghoard/pghoard.py
@@ -601,11 +601,13 @@ def startup_walk_for_missed_files(self):
                 with open(metadata_path, "r") as fp:
                     metadata = json.load(fp)
 
+                file_type = FileType.Wal if is_xlog else FileType.Timeline
+
                 transfer_event = UploadEvent(
-                    file_type=FileType.Wal,
+                    file_type=file_type,
                     backup_site_name=site,
                     file_size=os.path.getsize(full_path),
-                    file_path=FileTypePrefixes[FileType.Wal] / filename,
+                    file_path=FileTypePrefixes[file_type] / filename,
                     source_data=Path(full_path),
                     callback_queue=None,
                     metadata=metadata

diff --git a/test/test_pghoard.py b/test/test_pghoard.py
@@ -529,27 +529,34 @@ def test_startup_walk_for_missed_uncompressed_files(self):
         assert self.pghoard.compression_queue.qsize() == 2
         assert self.pghoard.transfer_queue.qsize() == 0
 
-    def test_startup_walk_for_missed_uncompressed_files_timeline(self):
+    @pytest.mark.parametrize(
+        "file_type, file_name", [(FileType.Wal, "000000010000000000000004"), (FileType.Timeline, "00000002.history")]
+    )
+    def test_startup_walk_for_missed_uncompressed_files(self, file_type: FileType, file_name: str):
         compressed_wal_path, _ = self.pghoard.create_backup_site_paths(self.test_site)
         uncompressed_wal_path = compressed_wal_path + "_incoming"
-        with open(os.path.join(uncompressed_wal_path, "00000002.history"), "wb") as fp:
+        with open(os.path.join(uncompressed_wal_path, file_name), "wb") as fp:
             fp.write(b"foo")
         self.pghoard.startup_walk_for_missed_files()
         assert self.pghoard.compression_queue.qsize() == 1
         assert self.pghoard.transfer_queue.qsize() == 0
         compress_event = self.pghoard.compression_queue.get(timeout=1.0)
-        assert compress_event.file_type == FileType.Timeline
+        assert compress_event.file_type == file_type
 
-    def test_startup_walk_for_missed_uncompressed_files_wal(self):
+    @pytest.mark.parametrize(
+        "file_type, file_name", [(FileType.Wal, "000000010000000000000005"), (FileType.Timeline, "00000003.history")]
+    )
+    def test_startup_walk_for_missed_compressed_files(self, file_type: FileType, file_name: str):
         compressed_wal_path, _ = self.pghoard.create_backup_site_paths(self.test_site)
-        uncompressed_wal_path = compressed_wal_path + "_incoming"
-        with open(os.path.join(uncompressed_wal_path, "000000010000000000000004"), "wb") as fp:
+        with open(os.path.join(compressed_wal_path, file_name), "wb") as fp:
             fp.write(b"foo")
+        with open(os.path.join(compressed_wal_path, f"{file_name}.metadata"), "wb") as fp:
+            fp.write(b"{}")
         self.pghoard.startup_walk_for_missed_files()
-        assert self.pghoard.compression_queue.qsize() == 1
-        assert self.pghoard.transfer_queue.qsize() == 0
-        compress_event = self.pghoard.compression_queue.get(timeout=1.0)
-        assert compress_event.file_type == FileType.Wal
+        assert self.pghoard.compression_queue.qsize() == 0
+        assert self.pghoard.transfer_queue.qsize() == 1
+        upload_event = self.pghoard.transfer_queue.get(timeout=1.0)
+        assert upload_event.file_type == file_type
 
 
 class TestPGHoardWithPG: