Improve core OrderBook updating

nautilus_core/model/src/orderbook/book.rs

@@ -36,8 +36,8 @@ pub enum InvalidBookOperation {
 
 #[derive(Error, Debug)]
 pub enum BookIntegrityError {
-    #[error("Invalid book operation: order ID {0} not found")]
-    OrderNotFound(u64),
+    #[error("Integrity error: order not found: order_id={0}, ts_event={1}, sequence={2}")]
+    OrderNotFound(u64, u64, u64),
     #[error("Integrity error: invalid `NoOrderSide` in book")]
     NoOrderSide,
     #[error("Integrity error: orders in cross [{0} @ {1}]")]
@@ -131,8 +131,8 @@ impl OrderBook {
         };
 
         match order.side {
-            OrderSide::Buy => self.bids.delete(order),
-            OrderSide::Sell => self.asks.delete(order),
+            OrderSide::Buy => self.bids.delete(order, ts_event, sequence),
+            OrderSide::Sell => self.asks.delete(order, ts_event, sequence),
             _ => panic!("{}", BookIntegrityError::NoOrderSide),
         }
 
@@ -278,13 +278,29 @@ impl OrderBook {
     }
 
     pub fn update_quote_tick(&mut self, tick: &QuoteTick) {
-        self.update_bid(BookOrder::from_quote_tick(tick, OrderSide::Buy));
-        self.update_ask(BookOrder::from_quote_tick(tick, OrderSide::Sell));
+        self.update_bid(
+            BookOrder::from_quote_tick(tick, OrderSide::Buy),
+            tick.ts_event,
+            0,
+        );
+        self.update_ask(
+            BookOrder::from_quote_tick(tick, OrderSide::Sell),
+            tick.ts_event,
+            0,
+        );
     }
 
     pub fn update_trade_tick(&mut self, tick: &TradeTick) {
-        self.update_bid(BookOrder::from_trade_tick(tick, OrderSide::Buy));
-        self.update_ask(BookOrder::from_trade_tick(tick, OrderSide::Sell));
+        self.update_bid(
+            BookOrder::from_trade_tick(tick, OrderSide::Buy),
+            tick.ts_event,
+            0,
+        );
+        self.update_ask(
+            BookOrder::from_trade_tick(tick, OrderSide::Sell),
+            tick.ts_event,
+            0,
+        );
     }
 
     pub fn simulate_fills(&self, order: &BookOrder) -> Vec<(Price, Quantity)> {
@@ -441,12 +457,12 @@ impl OrderBook {
         }
     }
 
-    fn update_bid(&mut self, order: BookOrder) {
+    fn update_bid(&mut self, order: BookOrder, ts_event: u64, sequence: u64) {
         match self.bids.top() {
             Some(top_bids) => match top_bids.first() {
                 Some(top_bid) => {
                     let order_id = top_bid.order_id;
-                    self.bids.remove(order_id);
+                    self.bids.remove(order_id, ts_event, sequence);
                     self.bids.add(order);
                 }
                 None => {
@@ -459,12 +475,12 @@ impl OrderBook {
         }
     }
 
-    fn update_ask(&mut self, order: BookOrder) {
+    fn update_ask(&mut self, order: BookOrder, ts_event: u64, sequence: u64) {
         match self.asks.top() {
             Some(top_asks) => match top_asks.first() {
                 Some(top_ask) => {
                     let order_id = top_ask.order_id;
-                    self.asks.remove(order_id);
+                    self.asks.remove(order_id, ts_event, sequence);
                     self.asks.add(order);
                 }
                 None => {

nautilus_core/model/src/orderbook/ladder.rs

@@ -108,48 +108,53 @@ impl Ladder {
     }
 
     pub fn add(&mut self, order: BookOrder) {
+        let order_id = order.order_id;
         let book_price = order.to_book_price();
+
+        self.cache.insert(order_id, book_price);
+
         match self.levels.get_mut(&book_price) {
             Some(level) => {
                 level.add(order);
             }
             None => {
-                let order_id = order.order_id;
                 let level = Level::from_order(order);
-                self.cache.insert(order_id, book_price);
                 self.levels.insert(book_price, level);
             }
         }
     }
 
     pub fn update(&mut self, order: BookOrder) {
-        if let Some(price) = self.cache.get(&order.order_id) {
-            if let Some(level) = self.levels.get_mut(price) {
+        let price_opt = self.cache.get(&order.order_id).copied();
+
+        if let Some(price) = price_opt {
+            if let Some(level) = self.levels.get_mut(&price) {
                 if order.price == level.price.value {
                     // Update at current price level
                     level.update(order);
                     return;
                 }
 
                 // Price update: delete and insert at new level
+                self.cache.remove(&order.order_id);
                 level.delete(&order);
                 if level.is_empty() {
-                    self.levels.remove(price);
+                    self.levels.remove(&price);
                 }
             }
         }
 
         self.add(order);
     }
 
-    pub fn delete(&mut self, order: BookOrder) {
-        self.remove(order.order_id);
+    pub fn delete(&mut self, order: BookOrder, ts_event: u64, sequence: u64) {
+        self.remove(order.order_id, ts_event, sequence);
     }
 
-    pub fn remove(&mut self, order_id: OrderId) {
+    pub fn remove(&mut self, order_id: OrderId, ts_event: u64, sequence: u64) {
         if let Some(price) = self.cache.remove(&order_id) {
             if let Some(level) = self.levels.get_mut(&price) {
-                level.remove_by_id(order_id);
+                level.remove_by_id(order_id, ts_event, sequence);
                 if level.is_empty() {
                     self.levels.remove(&price);
                 }
@@ -402,7 +407,7 @@ mod tests {
         let mut ladder = Ladder::new(OrderSide::Buy);
         let order = BookOrder::new(OrderSide::Buy, Price::from("10.00"), Quantity::from(20), 1);
 
-        ladder.delete(order);
+        ladder.delete(order, 0, 0);
 
         assert_eq!(ladder.len(), 0);
     }
@@ -416,7 +421,7 @@ mod tests {
 
         let order = BookOrder::new(OrderSide::Buy, Price::from("11.00"), Quantity::from(10), 1);
 
-        ladder.delete(order);
+        ladder.delete(order, 0, 0);
         assert_eq!(ladder.len(), 0);
         assert_eq!(ladder.sizes(), 0.0);
         assert_eq!(ladder.exposures(), 0.0);
@@ -432,7 +437,7 @@ mod tests {
 
         let order = BookOrder::new(OrderSide::Sell, Price::from("10.00"), Quantity::from(10), 1);
 
-        ladder.delete(order);
+        ladder.delete(order, 0, 0);
         assert_eq!(ladder.len(), 0);
         assert_eq!(ladder.sizes(), 0.0);
         assert_eq!(ladder.exposures(), 0.0);

nautilus_core/model/src/orderbook/level.rs

@@ -99,9 +99,12 @@ impl Level {
         self.update_insertion_order();
     }
 
-    pub fn remove_by_id(&mut self, order_id: OrderId) {
+    pub fn remove_by_id(&mut self, order_id: OrderId, ts_event: u64, sequence: u64) {
         if self.orders.remove(&order_id).is_none() {
-            panic!("{}", &BookIntegrityError::OrderNotFound(order_id));
+            panic!(
+                "{}",
+                &BookIntegrityError::OrderNotFound(order_id, ts_event, sequence)
+            );
         }
         self.update_insertion_order();
     }
@@ -311,7 +314,7 @@ mod tests {
 
         level.add(order1);
         level.add(order2);
-        level.remove_by_id(order2_id);
+        level.remove_by_id(order2_id, 0, 0);
         assert_eq!(level.len(), 1);
         assert!(level.orders.contains_key(&order1_id));
         assert_eq!(level.size(), 10.0);
@@ -344,10 +347,12 @@ mod tests {
     }
 
     #[rstest]
-    #[should_panic(expected = "Invalid book operation: order ID 1 not found")]
+    #[should_panic(
+        expected = "Integrity error: order not found: order_id=1, ts_event=2, sequence=3"
+    )]
     fn test_remove_nonexistent_order() {
         let mut level = Level::new(BookPrice::new(Price::from("1.00"), OrderSide::Buy));
-        level.remove_by_id(1);
+        level.remove_by_id(1, 2, 3);
     }
 
     #[rstest]

tests/unit_tests/model/test_orderbook.py

@@ -20,6 +20,7 @@
 import pandas as pd
 import pytest
 
+from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader
 from nautilus_trader.model.book import OrderBook
 from nautilus_trader.model.data import BookOrder
 from nautilus_trader.model.data import OrderBookDelta
@@ -33,6 +34,7 @@
 from nautilus_trader.test_kit.providers import TestInstrumentProvider
 from nautilus_trader.test_kit.stubs.data import TestDataStubs
 from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs
+from tests import TEST_DATA_DIR
 
 
 class TestOrderBook:
@@ -699,3 +701,64 @@ def make_delta(side: OrderSide, price: float, size: float, ts):
         # Assert
         assert book.ts_last == new.ts_last
         assert book.sequence == new.sequence
+
+    @pytest.mark.skip(reason="Used for development")
+    def test_orderbook_spy_xnas_itch_mbo_l3(self) -> None:
+        loader = DatabentoDataLoader()
+        path = TEST_DATA_DIR / "databento" / "temp" / "spy-xnas-itch-20231127.mbo.dbn.zst"
+        instrument = TestInstrumentProvider.equity(symbol="SPY", venue="XNAS")
+
+        # Act
+        data = loader.from_dbn(path, instrument_id=instrument.id)
+
+        book = TestDataStubs.make_book(
+            instrument=instrument,
+            book_type=BookType.L3_MBO,
+        )
+
+        for delta in data:
+            if not isinstance(delta, OrderBookDelta):
+                continue
+            book.apply_delta(delta)
+
+        # Assert
+        assert book.ts_last == 1701129555644234540
+        assert book.sequence == 429411899
+        assert book.count == 6197580
+        assert len(book.bids()) == 52
+        assert len(book.asks()) == 38
+        assert book.best_bid_price() == Price.from_str("454.84")
+        assert book.best_ask_price() == Price.from_str("454.90")
+
+    def test_orderbook_esh4_glbx_20231224_mbo_l3(self) -> None:
+        loader = DatabentoDataLoader()
+        instrument = TestInstrumentProvider.es_future(expiry_year=2024, expiry_month=3)
+
+        path_20231224 = TEST_DATA_DIR / "databento" / "esh4-glbx-mdp3-20231224.mbo.dbn.zst"
+        path_20231225 = TEST_DATA_DIR / "databento" / "esh4-glbx-mdp3-20231225.mbo.dbn.zst"
+        # path_20231226 = TEST_DATA_DIR / "temp" / "databento" / "esh4-glbx-mdp3-20231226.mbo.dbn.zst"
+
+        # Act
+        data = loader.from_dbn(path_20231224, instrument_id=instrument.id)
+        data.extend(loader.from_dbn(path_20231225, instrument_id=instrument.id))
+        # data.extend(loader.from_dbn(path_20231226, instrument_id=instrument.id))
+
+        book = TestDataStubs.make_book(
+            instrument=instrument,
+            book_type=BookType.L3_MBO,
+        )
+
+        for delta in data:
+            if not isinstance(delta, OrderBookDelta):
+                continue
+            book.apply_delta(delta)
+
+        # Assert
+        assert len(data) == 77517
+        assert book.ts_last == 1703548799446821072
+        assert book.sequence == 59585
+        assert book.count == 74509
+        assert len(book.bids()) == 922
+        assert len(book.asks()) == 565
+        assert book.best_bid_price() == Price.from_str("4810.00")
+        assert book.best_ask_price() == Price.from_str("4810.25")