Write parquet data in row groups (#1278)

nautechsystems · Oct 18, 2023 · 783740e · 783740e
1 parent 8adadb0
commit 783740e
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/nautilus_trader/persistence/catalog/parquet.py b/nautilus_trader/persistence/catalog/parquet.py
@@ -213,6 +213,7 @@ def write_chunk(
                 base_dir=path,
                 format="parquet",
                 filesystem=self.fs,
+                max_rows_per_group=5000,
                 **self.dataset_kwargs,
                 **kwargs,
             )
@@ -224,7 +225,7 @@ def _fast_write(
         fs: fsspec.AbstractFileSystem,
     ) -> None:
         fs.mkdirs(path, exist_ok=True)
-        pq.write_table(table, where=f"{path}/part-0.parquet", filesystem=fs)
+        pq.write_table(table, where=f"{path}/part-0.parquet", filesystem=fs, row_group_size=5000)
 
     def write_data(self, data: list[Data | Event], **kwargs: Any) -> None:
         def key(obj: Any) -> tuple[str, str | None]: