Integrate CacheDatabaseAdapter with Rust core

RELEASES.md

@@ -3,11 +3,14 @@
 Released on TBD (UTC).
 
 ### Enhancements
-None
+- Added `CacheDatabaseFacade` and `CacheDatabaseAdapter` to abstract backing technology from Python codebase
+- Added `RedisCacheDatabase` implemented in Rust with separate MPSC channel thread for insert, update and delete operations
+- Removed `redis` and `hiredis` dependencies from Python codebase
 
 ### Breaking Changes
 - Changed `RedisCacheDatabase` data structure for currencies from hashset to simpler key-value (you'll need to clear cache or delete all curreny keys)
-- `Actor` state loading now uses the standard `Serializer`
+- Changed `Actor` state loading to now use the standard `Serializer`
+- Removed `infrastructure` subpackage (now redundant with new Rust implementation)
 
 ### Fixes
 None

docs/getting_started/installation.md

@@ -24,11 +24,10 @@ Install optional dependencies as 'extras' for specific integrations:
 - `betfair`: Betfair adapter
 - `docker`: Needed for Docker when using the IB gateway
 - `ib`: Interactive Brokers adapter
-- `redis`: Use Redis as a cache database
 
 To install with specific extras using _pip_:
 
-    pip install -U "nautilus_trader[docker,ib,redis]"
+    pip install -U "nautilus_trader[docker,ib]"
 
 ## From Source
 Installation from source requires the `Python.h` header file, which is included in development releases such as `python-dev`. 
@@ -75,4 +74,4 @@ To install a binary wheel from GitHub, first navigate to the [latest release](ht
 Download the appropriate `.whl` for your operating system and Python version, then run:
 
     pip install <file-name>.whl
-    
+

examples/live/binance_futures_market_maker.py

@@ -21,13 +21,13 @@
 from nautilus_trader.adapters.binance.config import BinanceExecClientConfig
 from nautilus_trader.adapters.binance.factories import BinanceLiveDataClientFactory
 from nautilus_trader.adapters.binance.factories import BinanceLiveExecClientFactory
+from nautilus_trader.config import CacheDatabaseConfig
 from nautilus_trader.config import InstrumentProviderConfig
 from nautilus_trader.config import LiveExecEngineConfig
 from nautilus_trader.config import LoggingConfig
 from nautilus_trader.config import TradingNodeConfig
 from nautilus_trader.examples.strategies.volatility_market_maker import VolatilityMarketMaker
 from nautilus_trader.examples.strategies.volatility_market_maker import VolatilityMarketMakerConfig
-from nautilus_trader.infrastructure.cache import CacheDatabaseConfig
 from nautilus_trader.live.node import TradingNode
 
 

examples/live/binance_spot_market_maker.py

@@ -52,7 +52,7 @@
     #     stream="quoters",
     #     use_instance_id=False,
     #     timestamps_as_iso8601=True,
-    #     types_filter=[QuoteTick],
+    #     # types_filter=[QuoteTick],
     #     autotrim_mins=30,
     # ),
     # heartbeat_interval=1.0,

nautilus_trader/cache/cache.pxd

@@ -19,7 +19,7 @@ from libc.stdint cimport uint64_t
 from nautilus_trader.accounting.accounts.base cimport Account
 from nautilus_trader.accounting.calculators cimport ExchangeRateCalculator
 from nautilus_trader.cache.base cimport CacheFacade
-from nautilus_trader.cache.database cimport CacheDatabase
+from nautilus_trader.cache.facade cimport CacheDatabaseFacade
 from nautilus_trader.common.actor cimport Actor
 from nautilus_trader.common.logging cimport LoggerAdapter
 from nautilus_trader.core.rust.model cimport OmsType
@@ -53,7 +53,7 @@ from nautilus_trader.trading.strategy cimport Strategy
 
 cdef class Cache(CacheFacade):
     cdef LoggerAdapter _log
-    cdef CacheDatabase _database
+    cdef CacheDatabaseFacade _database
     cdef ExchangeRateCalculator _xrate_calculator
 
     cdef dict _general

nautilus_trader/cache/cache.pyx

@@ -29,7 +29,7 @@ from libc.stdint cimport uint64_t
 
 from nautilus_trader.accounting.accounts.base cimport Account
 from nautilus_trader.accounting.calculators cimport ExchangeRateCalculator
-from nautilus_trader.cache.base cimport CacheFacade
+from nautilus_trader.cache.facade cimport CacheDatabaseFacade
 from nautilus_trader.common.logging cimport LogColor
 from nautilus_trader.common.logging cimport Logger
 from nautilus_trader.common.logging cimport LoggerAdapter
@@ -77,8 +77,8 @@ cdef class Cache(CacheFacade):
     ----------
     logger : Logger
         The logger for the cache.
-    database : CacheDatabase, optional
-        The database for the cache. If ``None`` then will bypass persistence.
+    database : CacheDatabaseFacade, optional
+        The database adapter for the cache. If ``None`` then will bypass persistence.
     config : CacheConfig, optional
         The cache configuration.
     snapshot_orders : bool, default False
@@ -95,7 +95,7 @@ cdef class Cache(CacheFacade):
     def __init__(
         self,
         Logger logger not None,
-        CacheDatabase database: Optional[CacheDatabase] = None,
+        CacheDatabaseFacade database: Optional[CacheDatabaseFacade] = None,
         bint snapshot_orders: bool = False,
         bint snapshot_positions: bool = False,
         config: Optional[CacheConfig] = None,

nautilus_trader/cache/database.pxd

@@ -13,74 +13,37 @@
 #  limitations under the License.
 # -------------------------------------------------------------------------------------------------
 
-from cpython.datetime cimport datetime
-from libc.stdint cimport uint64_t
-
-from nautilus_trader.accounting.accounts.base cimport Account
-from nautilus_trader.common.actor cimport Actor
-from nautilus_trader.common.logging cimport LoggerAdapter
-from nautilus_trader.execution.messages cimport SubmitOrder
-from nautilus_trader.execution.messages cimport SubmitOrderList
-from nautilus_trader.model.identifiers cimport AccountId
-from nautilus_trader.model.identifiers cimport ClientId
-from nautilus_trader.model.identifiers cimport ClientOrderId
-from nautilus_trader.model.identifiers cimport ComponentId
-from nautilus_trader.model.identifiers cimport InstrumentId
-from nautilus_trader.model.identifiers cimport OrderListId
-from nautilus_trader.model.identifiers cimport PositionId
-from nautilus_trader.model.identifiers cimport StrategyId
-from nautilus_trader.model.identifiers cimport VenueOrderId
-from nautilus_trader.model.instruments.base cimport Instrument
-from nautilus_trader.model.instruments.synthetic cimport SyntheticInstrument
-from nautilus_trader.model.objects cimport Currency
-from nautilus_trader.model.objects cimport Money
-from nautilus_trader.model.orders.base cimport Order
-from nautilus_trader.model.position cimport Position
-from nautilus_trader.trading.strategy cimport Strategy
-
-
-cdef class CacheDatabase:
-    cdef LoggerAdapter _log
-
-    cpdef void flush(self)
-    cpdef dict load(self)
-    cpdef dict load_currencies(self)
-    cpdef dict load_instruments(self)
-    cpdef dict load_synthetics(self)
-    cpdef dict load_accounts(self)
-    cpdef dict load_orders(self)
-    cpdef dict load_positions(self)
-    cpdef dict load_index_order_position(self)
-    cpdef dict load_index_order_client(self)
-    cpdef Currency load_currency(self, str code)
-    cpdef Instrument load_instrument(self, InstrumentId instrument_id)
-    cpdef SyntheticInstrument load_synthetic(self, InstrumentId instrument_id)
-    cpdef Account load_account(self, AccountId account_id)
-    cpdef Order load_order(self, ClientOrderId order_id)
-    cpdef Position load_position(self, PositionId position_id)
-    cpdef dict load_actor(self, ComponentId component_id)
-    cpdef void delete_actor(self, ComponentId component_id)
-    cpdef dict load_strategy(self, StrategyId strategy_id)
-    cpdef void delete_strategy(self, StrategyId strategy_id)
-
-    cpdef void add(self, str key, bytes value)
-    cpdef void add_currency(self, Currency currency)
-    cpdef void add_instrument(self, Instrument instrument)
-    cpdef void add_synthetic(self, SyntheticInstrument instrument)
-    cpdef void add_account(self, Account account)
-    cpdef void add_order(self, Order order, PositionId position_id=*, ClientId client_id=*)
-    cpdef void add_position(self, Position position)
-
-    cpdef void index_venue_order_id(self, ClientOrderId client_order_id, VenueOrderId venue_order_id)
-    cpdef void index_order_position(self, ClientOrderId client_order_id, PositionId position_id)
-
-    cpdef void update_account(self, Account account)
-    cpdef void update_order(self, Order order)
-    cpdef void update_position(self, Position position)
-    cpdef void update_actor(self, Actor actor)
-    cpdef void update_strategy(self, Strategy strategy)
-
-    cpdef void snapshot_order_state(self, Order order)
-    cpdef void snapshot_position_state(self, Position position, uint64_t ts_snapshot, Money unrealized_pnl=*)
-
-    cpdef void heartbeat(self, datetime timestamp)
+from nautilus_trader.cache.facade cimport CacheDatabaseFacade
+from nautilus_trader.serialization.base cimport Serializer
+
+
+cdef class CacheDatabaseAdapter(CacheDatabaseFacade):
+    cdef str _key_trader
+    cdef str _key_general
+    cdef str _key_currencies
+    cdef str _key_instruments
+    cdef str _key_synthetics
+    cdef str _key_accounts
+    cdef str _key_orders
+    cdef str _key_positions
+    cdef str _key_actors
+    cdef str _key_strategies
+
+    cdef str _key_index_order_ids
+    cdef str _key_index_order_position
+    cdef str _key_index_order_client
+    cdef str _key_index_orders
+    cdef str _key_index_orders_open
+    cdef str _key_index_orders_closed
+    cdef str _key_index_orders_emulated
+    cdef str _key_index_orders_inflight
+    cdef str _key_index_positions
+    cdef str _key_index_positions_open
+    cdef str _key_index_positions_closed
+
+    cdef str _key_snapshots_orders
+    cdef str _key_snapshots_positions
+    cdef str _key_heartbeat
+
+    cdef Serializer _serializer
+    cdef object _backing