Skip to content

Commit

Permalink
Remove redundant data catalog config options
Browse files Browse the repository at this point in the history
  • Loading branch information
cjdsellers committed Oct 15, 2023
1 parent dc43169 commit 420b22a
Show file tree
Hide file tree
Showing 8 changed files with 12 additions and 36 deletions.
2 changes: 0 additions & 2 deletions nautilus_trader/config/backtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ class BacktestDataConfig(NautilusConfig, frozen=True):
client_id: Optional[str] = None
metadata: Optional[dict] = None
bar_spec: Optional[str] = None
use_rust: Optional[bool] = False
batch_size: Optional[int] = 10_000

@property
Expand All @@ -104,7 +103,6 @@ def query(self) -> dict[str, Any]:
"end": self.end_time,
"filter_expr": parse_filters_expr(filter_expr),
"metadata": self.metadata,
"use_rust": self.use_rust,
}

@property
Expand Down
3 changes: 0 additions & 3 deletions nautilus_trader/config/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,15 +349,12 @@ class DataCatalogConfig(NautilusConfig, frozen=True):
The fsspec file system protocol for the data catalog.
fs_storage_options : dict, optional
The fsspec storage options for the data catalog.
use_rust : bool, default False
If queries will be for Rust schema versions (when implemented).
"""

path: str
fs_protocol: Optional[str] = None
fs_storage_options: Optional[dict] = None
use_rust: bool = False


class ActorConfig(NautilusConfig, kw_only=True, frozen=True):
Expand Down
1 change: 0 additions & 1 deletion nautilus_trader/data/engine.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ cdef class DataEngine(Component):
cdef readonly Cache _cache
cdef readonly DataClient _default_client
cdef readonly object _catalog
cdef readonly bint _use_rust

cdef readonly dict _clients
cdef readonly dict _routing_map
Expand Down
18 changes: 2 additions & 16 deletions nautilus_trader/data/engine.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,6 @@ cdef class DataEngine(Component):
self._routing_map: dict[Venue, DataClient] = {}
self._default_client: Optional[DataClient] = None
self._catalog: Optional[ParquetDataCatalog] = None
self._use_rust: bool = False
self._order_book_intervals: dict[(InstrumentId, int), list[Callable[[Bar], None]]] = {}
self._bar_aggregators: dict[BarType, BarAggregator] = {}
self._synthetic_quote_feeds: dict[InstrumentId, list[SyntheticInstrument]] = {}
Expand Down Expand Up @@ -229,7 +228,7 @@ cdef class DataEngine(Component):

# --REGISTRATION ----------------------------------------------------------------------------------

def register_catalog(self, catalog: ParquetDataCatalog, bint use_rust=False) -> None:
def register_catalog(self, catalog: ParquetDataCatalog) -> None:
"""
Register the given data catalog with the engine.

Expand All @@ -242,7 +241,6 @@ cdef class DataEngine(Component):
Condition.not_none(catalog, "catalog")

self._catalog = catalog
self._use_rust = use_rust

cpdef void register_client(self, DataClient client):
"""
Expand Down Expand Up @@ -1302,25 +1300,18 @@ cdef class DataEngine(Component):
if instrument_id is None:
data = self._catalog.instruments(as_nautilus=True)
else:
data = self._catalog.instruments(
instrument_ids=[str(instrument_id)],
as_nautilus=True,
)
data = self._catalog.instruments(instrument_ids=[str(instrument_id)])
elif request.data_type.type == QuoteTick:
data = self._catalog.quote_ticks(
instrument_ids=[str(request.data_type.metadata.get("instrument_id"))],
start=ts_start,
end=ts_end,
as_nautilus=True,
use_rust=self._use_rust,
)
elif request.data_type.type == TradeTick:
data = self._catalog.trade_ticks(
instrument_ids=[str(request.data_type.metadata.get("instrument_id"))],
start=ts_start,
end=ts_end,
as_nautilus=True,
use_rust=self._use_rust,
)
elif request.data_type.type == Bar:
bar_type = request.data_type.metadata.get("bar_type")
Expand All @@ -1332,24 +1323,19 @@ cdef class DataEngine(Component):
bar_type=str(bar_type),
start=ts_start,
end=ts_end,
as_nautilus=True,
use_rust=False, # Until implemented
)
elif request.data_type.type == InstrumentClose:
data = self._catalog.instrument_closes(
instrument_ids=[str(request.data_type.metadata.get("instrument_id"))],
start=ts_start,
end=ts_end,
as_nautilus=True,
use_rust=False, # Until implemented
)
else:
data = self._catalog.generic_data(
cls=request.data_type.type,
metadata=request.data_type.metadata,
start=ts_start,
end=ts_end,
as_nautilus=True,
)

# Validation data is not from the future
Expand Down
5 changes: 1 addition & 4 deletions nautilus_trader/system/kernel.py
Original file line number Diff line number Diff line change
Expand Up @@ -401,10 +401,7 @@ def __init__( # noqa (too complex)
fs_protocol=config.catalog.fs_protocol,
fs_storage_options=config.catalog.fs_storage_options,
)
self._data_engine.register_catalog(
catalog=self._catalog,
use_rust=config.catalog.use_rust,
)
self._data_engine.register_catalog(catalog=self._catalog)

# Create importable actors
for actor_config in config.actors:
Expand Down
6 changes: 3 additions & 3 deletions tests/performance_tests/test_perf_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def setup():
return (cls.catalog,), {}

def run(catalog):
quotes = catalog.quote_ticks(as_nautilus=True)
quotes = catalog.quote_ticks()
assert len(quotes) == 9500

benchmark.pedantic(run, setup=setup, rounds=1, iterations=1, warmup_rounds=1)
Expand All @@ -66,13 +66,13 @@ def setup():

cls.catalog = data_catalog_setup(protocol="file", path=tempdir)

cls._load_quote_ticks_into_catalog(use_rust=True)
cls._load_quote_ticks_into_catalog()

# Act
return (cls.catalog,), {}

def run(catalog):
quotes = catalog.quote_ticks(as_nautilus=True, use_rust=True)
quotes = catalog.quote_ticks()
assert len(quotes) == 9500

benchmark.pedantic(run, setup=setup, rounds=1, iterations=1, warmup_rounds=1)
Expand Down
9 changes: 4 additions & 5 deletions tests/unit_tests/backtest/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ def test_backtest_data_config_load(self):
"filter_expr": None,
"start": 1580398089820000000,
"end": 1580504394501000000,
"use_rust": False,
"metadata": None,
}

Expand Down Expand Up @@ -198,7 +197,7 @@ def test_run_config_to_json(self) -> None:
)
json = msgspec.json.encode(run_config)
result = len(msgspec.json.encode(json))
assert result == 1010 # UNIX
assert result == 986 # UNIX

@pytest.mark.skipif(sys.platform == "win32", reason="redundant to also test Windows")
def test_run_config_parse_obj(self) -> None:
Expand All @@ -219,7 +218,7 @@ def test_run_config_parse_obj(self) -> None:
assert isinstance(config, BacktestRunConfig)
node = BacktestNode(configs=[config])
assert isinstance(node, BacktestNode)
assert len(raw) == 754 # UNIX
assert len(raw) == 737 # UNIX

@pytest.mark.skipif(sys.platform == "win32", reason="redundant to also test Windows")
def test_backtest_data_config_to_dict(self) -> None:
Expand All @@ -240,15 +239,15 @@ def test_backtest_data_config_to_dict(self) -> None:
)
json = msgspec.json.encode(run_config)
result = len(msgspec.json.encode(json))
assert result == 1866
assert result == 1798

@pytest.mark.skipif(sys.platform == "win32", reason="redundant to also test Windows")
def test_backtest_run_config_id(self) -> None:
token = self.backtest_config.id
print("token:", token)
value: bytes = msgspec.json.encode(self.backtest_config.dict(), enc_hook=json_encoder)
print("token_value:", value.decode())
assert token == "e85939d3f49c300d8d12b22a702ad9ea1dccf942b23016b66c00101c0de6f3c6" # UNIX
assert token == "d1add7c871b0bdd762b495345e394276431eda714a00d839037df33e8a427fd1" # UNIX

@pytest.mark.skip(reason="fix after merge")
@pytest.mark.parametrize(
Expand Down
4 changes: 2 additions & 2 deletions tests/unit_tests/data/test_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -2163,7 +2163,7 @@ def test_request_instruments_for_venue_when_catalog_registered(self):
# data: bytes = writer.flush_bytes()
# f.write(data)
#
# self.data_engine.register_catalog(catalog, use_rust=True)
# self.data_engine.register_catalog(catalog)
#
# # Act
# handler: list[DataResponse] = []
Expand Down Expand Up @@ -2246,7 +2246,7 @@ def test_request_instruments_for_venue_when_catalog_registered(self):
# data: bytes = writer.flush_bytes()
# f.write(data)
#
# self.data_engine.register_catalog(catalog, use_rust=True)
# self.data_engine.register_catalog(catalog)
#
# # Act
# handler: list[DataResponse] = []
Expand Down

0 comments on commit 420b22a

Please sign in to comment.