Skip to content

Commit

Permalink
Fix ParquetDataCatalog path globbing
Browse files Browse the repository at this point in the history
  • Loading branch information
cjdsellers committed Apr 19, 2024
1 parent e9e6143 commit 54489e4
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 3 deletions.
1 change: 1 addition & 0 deletions RELEASES.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Released on TBD (UTC).
- Fixed Databento bars decoding (was incorrectly applying display factor)
- Fixed `Binance` bar (kline) to use `close_time` for `ts_event` was `opentime` (#1591), thanks for reporting @OnlyC
- Fixed `AccountMarginExceeded` error condition (margin must actually be exceeded now, and can be zero)
- Fixed `ParquetDataCatalog` path globbing which was including all paths with substrings of specified instrument IDs

---

Expand Down
13 changes: 10 additions & 3 deletions nautilus_trader/persistence/catalog/parquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -412,15 +412,22 @@ def backend_session(

file_prefix = class_to_filename(data_cls)
glob_path = f"{self.path}/data/{file_prefix}/**/*"
dirs = self.fs.glob(glob_path)
dirs: list[str] = self.fs.glob(glob_path)
if self.show_query_paths:
print(dirs)

for idx, path in enumerate(dirs):
assert self.fs.exists(path)
if instrument_ids and not any(urisafe_instrument_id(x) in path for x in instrument_ids):
# Parse the parent directory which *should* be the instrument ID,
# this prevents us matching all instrument ID substrings.
parent_dir = path.split("/")[-2]
if instrument_ids and not any(
parent_dir.startswith(urisafe_instrument_id(x)) for x in instrument_ids
):
continue
if bar_types and not any(urisafe_instrument_id(x) in path for x in bar_types):
if bar_types and not any(
parent_dir.startswith(urisafe_instrument_id(x)) for x in bar_types
):
continue
table = f"{file_prefix}_{idx}"
query = self._build_query(
Expand Down

0 comments on commit 54489e4

Please sign in to comment.