From 0009f3a12dc125354ee396d689689a7b101f5845 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 13 Feb 2024 20:05:57 +1100 Subject: [PATCH] Implement OrderBookDeltas FFI and pyo3 interfaces --- nautilus_core/model/src/data/deltas.rs | 27 ++-- nautilus_core/model/src/data/mod.rs | 3 +- nautilus_core/model/src/ffi/data/deltas.rs | 118 ++++++++++++++ nautilus_core/model/src/ffi/data/mod.rs | 1 + nautilus_core/model/src/python/data/deltas.rs | 126 +++++++++++++++ nautilus_core/model/src/python/data/mod.rs | 1 + nautilus_trader/core/includes/model.h | 50 ++++++ nautilus_trader/core/rust/model.pxd | 44 ++++++ nautilus_trader/model/data.pxd | 14 +- nautilus_trader/model/data.pyx | 148 +++++++++++++++++- .../tracemalloc_orderbook_delta.py | 37 +++++ .../tracemalloc_orderbook_deltas.py | 19 +-- 12 files changed, 543 insertions(+), 45 deletions(-) create mode 100644 nautilus_core/model/src/ffi/data/deltas.rs create mode 100644 nautilus_core/model/src/python/data/deltas.rs create mode 100644 tests/mem_leak_tests/tracemalloc_orderbook_delta.py diff --git a/nautilus_core/model/src/data/deltas.rs b/nautilus_core/model/src/data/deltas.rs index c6a66745be22..1d8bdd0e4aee 100644 --- a/nautilus_core/model/src/data/deltas.rs +++ b/nautilus_core/model/src/data/deltas.rs @@ -22,7 +22,8 @@ use super::delta::OrderBookDelta; use crate::identifiers::instrument_id::InstrumentId; /// Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. -#[repr(C)] +/// +/// This type cannot be `repr(C)` due to the `deltas` vec. #[derive(Clone, Debug)] #[cfg_attr( feature = "python", @@ -46,14 +47,14 @@ pub struct OrderBookDeltas { impl OrderBookDeltas { #[allow(clippy::too_many_arguments)] #[must_use] - pub fn new( - instrument_id: InstrumentId, - deltas: Vec, - flags: u8, - sequence: u64, - ts_event: UnixNanos, - ts_init: UnixNanos, - ) -> Self { + pub fn new(instrument_id: InstrumentId, deltas: Vec) -> Self { + assert!(!deltas.is_empty(), "`deltas` cannot be empty"); + // SAFETY: We asserted `deltas` is not empty + let last = deltas.last().unwrap(); + let flags = last.flags; + let sequence = last.sequence; + let ts_event = last.ts_event; + let ts_init = last.ts_init; Self { instrument_id, deltas, @@ -65,7 +66,7 @@ impl OrderBookDeltas { } } -// TODO: Potentially implement later +// TODO: Implement // impl Serializable for OrderBookDeltas {} // TODO: Exact format for Debug and Display TBD @@ -195,7 +196,7 @@ pub mod stubs { let deltas = vec![delta0, delta1, delta2, delta3, delta4, delta5, delta6]; - OrderBookDeltas::new(instrument_id, deltas, flags, sequence, ts_event, ts_init) + OrderBookDeltas::new(instrument_id, deltas) } } @@ -310,10 +311,6 @@ mod tests { let deltas = OrderBookDeltas::new( instrument_id, vec![delta0, delta1, delta2, delta3, delta4, delta5, delta6], - flags, - sequence, - ts_event, - ts_init, ); assert_eq!(deltas.instrument_id, instrument_id); diff --git a/nautilus_core/model/src/data/mod.rs b/nautilus_core/model/src/data/mod.rs index 9ac16901fc1a..e354fb78e182 100644 --- a/nautilus_core/model/src/data/mod.rs +++ b/nautilus_core/model/src/data/mod.rs @@ -30,7 +30,6 @@ use self::{ #[repr(C)] #[derive(Clone, Debug)] -#[cfg_attr(feature = "trivial_copy", derive(Copy))] #[allow(clippy::large_enum_variant)] // TODO: Optimize this (largest variant 1008 vs 136 bytes) pub enum Data { Delta(OrderBookDelta), @@ -129,5 +128,5 @@ impl From for Data { #[no_mangle] pub extern "C" fn data_clone(data: &Data) -> Data { - *data // Actually a copy + data.clone() } diff --git a/nautilus_core/model/src/ffi/data/deltas.rs b/nautilus_core/model/src/ffi/data/deltas.rs new file mode 100644 index 000000000000..f52632de71e2 --- /dev/null +++ b/nautilus_core/model/src/ffi/data/deltas.rs @@ -0,0 +1,118 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::ops::{Deref, DerefMut}; + +use nautilus_core::{ffi::cvec::CVec, time::UnixNanos}; + +use crate::{ + data::{delta::OrderBookDelta, deltas::OrderBookDeltas}, + enums::BookAction, + identifiers::instrument_id::InstrumentId, +}; + +/// Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. +/// +/// This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function +/// calls, enabling interaction with `OrderBookDeltas` in a C environment. +/// +/// It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be +/// dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without +/// having to manually access the underlying `OrderBookDeltas` instance. +#[repr(C)] +#[allow(non_camel_case_types)] +pub struct OrderBookDeltas_API(Box); + +impl Deref for OrderBookDeltas_API { + type Target = OrderBookDeltas; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for OrderBookDeltas_API { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// Creates a new `OrderBookDeltas` object from a CVec of `OrderBookDelta`. +/// +/// # Safety +/// - The `deltas` must be a valid pointer to a `CVec` containing `OrderBookDelta` objects +/// - This function clones the data pointed to by `deltas` into Rust-managed memory, then forgets the original `Vec` to prevent Rust from auto-deallocating it +/// - The caller is responsible for managing the memory of `deltas` (including its deallocation) to avoid memory leaks +#[no_mangle] +pub extern "C" fn orderbook_deltas_new( + instrument_id: InstrumentId, + deltas: &CVec, +) -> OrderBookDeltas_API { + let CVec { ptr, len, cap } = *deltas; + let deltas: Vec = + unsafe { Vec::from_raw_parts(ptr as *mut OrderBookDelta, len, cap) }; + let cloned_deltas = deltas.clone(); + std::mem::forget(deltas); // Prevents Rust from dropping `deltas` + OrderBookDeltas_API(Box::new(OrderBookDeltas::new(instrument_id, cloned_deltas))) +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_drop(deltas: OrderBookDeltas_API) { + drop(deltas); // Memory freed here +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_instrument_id(deltas: &OrderBookDeltas_API) -> InstrumentId { + deltas.instrument_id +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_vec_deltas(deltas: &OrderBookDeltas_API) -> CVec { + deltas.deltas.clone().into() +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_is_snapshot(deltas: &OrderBookDeltas_API) -> u8 { + u8::from(deltas.deltas[0].action == BookAction::Clear) +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_flags(deltas: &OrderBookDeltas_API) -> u8 { + deltas.flags +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_sequence(deltas: &OrderBookDeltas_API) -> u64 { + deltas.sequence +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_ts_event(deltas: &OrderBookDeltas_API) -> UnixNanos { + deltas.ts_event +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_ts_init(deltas: &OrderBookDeltas_API) -> UnixNanos { + deltas.ts_init +} + +#[allow(clippy::drop_non_drop)] +#[no_mangle] +pub extern "C" fn orderbook_deltas_vec_drop(v: CVec) { + let CVec { ptr, len, cap } = v; + let deltas: Vec = + unsafe { Vec::from_raw_parts(ptr as *mut OrderBookDelta, len, cap) }; + drop(deltas); // Memory freed here +} diff --git a/nautilus_core/model/src/ffi/data/mod.rs b/nautilus_core/model/src/ffi/data/mod.rs index e1a81c7edec2..ce93bb068149 100644 --- a/nautilus_core/model/src/ffi/data/mod.rs +++ b/nautilus_core/model/src/ffi/data/mod.rs @@ -15,6 +15,7 @@ pub mod bar; pub mod delta; +pub mod deltas; pub mod depth; pub mod order; pub mod quote; diff --git a/nautilus_core/model/src/python/data/deltas.rs b/nautilus_core/model/src/python/data/deltas.rs new file mode 100644 index 000000000000..fcd7b8acfb81 --- /dev/null +++ b/nautilus_core/model/src/python/data/deltas.rs @@ -0,0 +1,126 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +// use std::{ +// collections::{hash_map::DefaultHasher, HashMap}, +// hash::{Hash, Hasher}, +// }; + +use nautilus_core::time::UnixNanos; +use pyo3::prelude::*; + +use crate::{ + data::{delta::OrderBookDelta, deltas::OrderBookDeltas}, + identifiers::instrument_id::InstrumentId, + python::PY_MODULE_MODEL, +}; + +#[pymethods] +impl OrderBookDeltas { + #[new] + fn py_new(instrument_id: InstrumentId, deltas: Vec) -> Self { + Self::new(instrument_id, deltas) + } + + // TODO: Implement + // fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { + // match op { + // CompareOp::Eq => self.eq(other).into_py(py), + // CompareOp::Ne => self.ne(other).into_py(py), + // _ => py.NotImplemented(), + // } + // } + + // TODO: Implement + // fn __hash__(&self) -> isize { + // let mut h = DefaultHasher::new(); + // self.hash(&mut h); + // h.finish() as isize + // } + + fn __str__(&self) -> String { + self.to_string() + } + + fn __repr__(&self) -> String { + format!("{self:?}") + } + + #[getter] + #[pyo3(name = "instrument_id")] + fn py_instrument_id(&self) -> InstrumentId { + self.instrument_id + } + + #[getter] + #[pyo3(name = "deltas")] + fn py_deltas(&self) -> Vec { + // `OrderBookDelta` is `Copy` + self.deltas.clone() + } + + #[getter] + #[pyo3(name = "flags")] + fn py_flags(&self) -> u8 { + self.flags + } + + #[getter] + #[pyo3(name = "sequence")] + fn py_sequence(&self) -> u64 { + self.sequence + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + + #[staticmethod] + #[pyo3(name = "fully_qualified_name")] + fn py_fully_qualified_name() -> String { + format!("{}:{}", PY_MODULE_MODEL, stringify!(OrderBookDeltas)) + } + + // /// Creates a `PyCapsule` containing a raw pointer to a `Data::Delta` object. + // /// + // /// This function takes the current object (assumed to be of a type that can be represented as + // /// `Data::Delta`), and encapsulates a raw pointer to it within a `PyCapsule`. + // /// + // /// # Safety + // /// + // /// This function is safe as long as the following conditions are met: + // /// - The `Data::Delta` object pointed to by the capsule must remain valid for the lifetime of the capsule. + // /// - The consumer of the capsule must ensure proper handling to avoid dereferencing a dangling pointer. + // /// + // /// # Panics + // /// + // /// The function will panic if the `PyCapsule` creation fails, which can occur if the + // /// `Data::Delta` object cannot be converted into a raw pointer. + // /// + // #[pyo3(name = "as_pycapsule")] + // fn py_as_pycapsule(&self, py: Python<'_>) -> PyObject { + // data_to_pycapsule(py, Data::Delta(*self)) + // } + + // TODO: Implement `Serializable` and the other methods can be added +} diff --git a/nautilus_core/model/src/python/data/mod.rs b/nautilus_core/model/src/python/data/mod.rs index 48c110cbbad8..f5aca4393bf4 100644 --- a/nautilus_core/model/src/python/data/mod.rs +++ b/nautilus_core/model/src/python/data/mod.rs @@ -15,6 +15,7 @@ pub mod bar; pub mod delta; +pub mod deltas; pub mod depth; pub mod order; pub mod quote; diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index 068797f8db9f..3b9172b2f8a5 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -665,6 +665,13 @@ typedef struct Level Level; */ typedef struct OrderBook OrderBook; +/** + * Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. + * + * This type cannot be `repr(C)` due to the `deltas` vec. + */ +typedef struct OrderBookDeltas_t OrderBookDeltas_t; + /** * Represents a synthetic instrument with prices derived from component instruments using a * formula. @@ -1011,6 +1018,20 @@ typedef struct Data_t { }; } Data_t; +/** + * Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. + * + * This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function + * calls, enabling interaction with `OrderBookDeltas` in a C environment. + * + * It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be + * dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without + * having to manually access the underlying `OrderBookDeltas` instance. + */ +typedef struct OrderBookDeltas_API { + struct OrderBookDeltas_t *_0; +} OrderBookDeltas_API; + /** * Represents a valid trader ID. * @@ -1376,6 +1397,35 @@ uint8_t orderbook_delta_eq(const struct OrderBookDelta_t *lhs, const struct Orde uint64_t orderbook_delta_hash(const struct OrderBookDelta_t *delta); +/** + * Creates a new `OrderBookDeltas` object from a CVec of `OrderBookDelta`. + * + * # Safety + * - The `deltas` must be a valid pointer to a `CVec` containing `OrderBookDelta` objects + * - This function clones the data pointed to by `deltas` into Rust-managed memory, then forgets the original `Vec` to prevent Rust from auto-deallocating it + * - The caller is responsible for managing the memory of `deltas` (including its deallocation) to avoid memory leaks + */ +struct OrderBookDeltas_API orderbook_deltas_new(struct InstrumentId_t instrument_id, + const CVec *deltas); + +void orderbook_deltas_drop(struct OrderBookDeltas_API deltas); + +struct InstrumentId_t orderbook_deltas_instrument_id(const struct OrderBookDeltas_API *deltas); + +CVec orderbook_deltas_vec_deltas(const struct OrderBookDeltas_API *deltas); + +uint8_t orderbook_deltas_is_snapshot(const struct OrderBookDeltas_API *deltas); + +uint8_t orderbook_deltas_flags(const struct OrderBookDeltas_API *deltas); + +uint64_t orderbook_deltas_sequence(const struct OrderBookDeltas_API *deltas); + +uint64_t orderbook_deltas_ts_event(const struct OrderBookDeltas_API *deltas); + +uint64_t orderbook_deltas_ts_init(const struct OrderBookDeltas_API *deltas); + +void orderbook_deltas_vec_drop(CVec v); + /** * # Safety * diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index ee07a4950812..165a60ed703a 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -359,6 +359,12 @@ cdef extern from "../includes/model.h": cdef struct OrderBook: pass + # Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. + # + # This type cannot be `repr(C)` due to the `deltas` vec. + cdef struct OrderBookDeltas_t: + pass + # Represents a synthetic instrument with prices derived from component instruments using a # formula. cdef struct SyntheticInstrument: @@ -546,6 +552,17 @@ cdef extern from "../includes/model.h": TradeTick_t trade; Bar_t bar; + # Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. + # + # This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function + # calls, enabling interaction with `OrderBookDeltas` in a C environment. + # + # It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be + # dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without + # having to manually access the underlying `OrderBookDeltas` instance. + cdef struct OrderBookDeltas_API: + OrderBookDeltas_t *_0; + # Represents a valid trader ID. # # Must be correctly formatted with two valid strings either side of a hyphen. @@ -827,6 +844,33 @@ cdef extern from "../includes/model.h": uint64_t orderbook_delta_hash(const OrderBookDelta_t *delta); + # Creates a new `OrderBookDeltas` object from a CVec of `OrderBookDelta`. + # + # # Safety + # - The `deltas` must be a valid pointer to a `CVec` containing `OrderBookDelta` objects + # - This function clones the data pointed to by `deltas` into Rust-managed memory, then forgets the original `Vec` to prevent Rust from auto-deallocating it + # - The caller is responsible for managing the memory of `deltas` (including its deallocation) to avoid memory leaks + OrderBookDeltas_API orderbook_deltas_new(InstrumentId_t instrument_id, + const CVec *deltas); + + void orderbook_deltas_drop(OrderBookDeltas_API deltas); + + InstrumentId_t orderbook_deltas_instrument_id(const OrderBookDeltas_API *deltas); + + CVec orderbook_deltas_vec_deltas(const OrderBookDeltas_API *deltas); + + uint8_t orderbook_deltas_is_snapshot(const OrderBookDeltas_API *deltas); + + uint8_t orderbook_deltas_flags(const OrderBookDeltas_API *deltas); + + uint64_t orderbook_deltas_sequence(const OrderBookDeltas_API *deltas); + + uint64_t orderbook_deltas_ts_event(const OrderBookDeltas_API *deltas); + + uint64_t orderbook_deltas_ts_init(const OrderBookDeltas_API *deltas); + + void orderbook_deltas_vec_drop(CVec v); + # # Safety # # - Assumes `bids` and `asks` are valid pointers to arrays of `BookOrder` of length 10. diff --git a/nautilus_trader/model/data.pxd b/nautilus_trader/model/data.pxd index 78912429c2a3..21eaf3945cdc 100644 --- a/nautilus_trader/model/data.pxd +++ b/nautilus_trader/model/data.pxd @@ -32,6 +32,7 @@ from nautilus_trader.core.rust.model cimport HaltReason from nautilus_trader.core.rust.model cimport InstrumentCloseType from nautilus_trader.core.rust.model cimport MarketStatus from nautilus_trader.core.rust.model cimport OrderBookDelta_t +from nautilus_trader.core.rust.model cimport OrderBookDeltas_API from nautilus_trader.core.rust.model cimport OrderBookDepth10_t from nautilus_trader.core.rust.model cimport OrderSide from nautilus_trader.core.rust.model cimport PriceType @@ -235,18 +236,7 @@ cdef class OrderBookDelta(Data): cdef class OrderBookDeltas(Data): - cdef readonly InstrumentId instrument_id - """The instrument ID for the order book.\n\n:returns: `InstrumentId`""" - cdef readonly list deltas - """The order book deltas.\n\n:returns: `list[OrderBookDelta]`""" - cdef readonly bint is_snapshot - """If the deltas represent a snapshot (an initial CLEAR then deltas).\n\n:returns: `bool`""" - cdef readonly uint64_t sequence - """If the sequence number for the last delta.\n\n:returns: `bool`""" - cdef readonly uint64_t ts_event - """The UNIX timestamp (nanoseconds) when the last delta event occurred.\n\n:returns: `uint64_t`""" - cdef readonly uint64_t ts_init - """The UNIX timestamp (nanoseconds) when the last delta event was initialized.\n\n:returns: `uint64_t`""" + cdef OrderBookDeltas_API _mem @staticmethod cdef OrderBookDeltas from_dict_c(dict values) diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index 36efef53a3a5..084a37cafe75 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -78,6 +78,16 @@ from nautilus_trader.core.rust.model cimport instrument_id_from_cstr from nautilus_trader.core.rust.model cimport orderbook_delta_eq from nautilus_trader.core.rust.model cimport orderbook_delta_hash from nautilus_trader.core.rust.model cimport orderbook_delta_new +from nautilus_trader.core.rust.model cimport orderbook_deltas_drop +from nautilus_trader.core.rust.model cimport orderbook_deltas_flags +from nautilus_trader.core.rust.model cimport orderbook_deltas_instrument_id +from nautilus_trader.core.rust.model cimport orderbook_deltas_is_snapshot +from nautilus_trader.core.rust.model cimport orderbook_deltas_new +from nautilus_trader.core.rust.model cimport orderbook_deltas_sequence +from nautilus_trader.core.rust.model cimport orderbook_deltas_ts_event +from nautilus_trader.core.rust.model cimport orderbook_deltas_ts_init +from nautilus_trader.core.rust.model cimport orderbook_deltas_vec_deltas +from nautilus_trader.core.rust.model cimport orderbook_deltas_vec_drop from nautilus_trader.core.rust.model cimport orderbook_depth10_ask_counts_array from nautilus_trader.core.rust.model cimport orderbook_depth10_asks_array from nautilus_trader.core.rust.model cimport orderbook_depth10_bid_counts_array @@ -2130,12 +2140,40 @@ cdef class OrderBookDeltas(Data): ) -> None: Condition.not_empty(deltas, "deltas") - self.instrument_id = instrument_id - self.deltas = deltas - self.is_snapshot = deltas[0].is_clear - self.sequence = deltas[-1].sequence - self.ts_event = deltas[-1].ts_event - self.ts_init = deltas[-1].ts_init + cdef uint64_t len_ = len(deltas) + + # Create a C OrderBookDeltas_t buffer + cdef OrderBookDelta_t* data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) + if not data: + raise MemoryError() + + cdef uint64_t i + cdef OrderBookDelta delta + for i in range(len_): + delta = deltas[i] + data[i] = delta._mem + + # Create CVec + cdef CVec* cvec = PyMem_Malloc(1 * sizeof(CVec)) + if not cvec: + raise MemoryError() + + cvec.ptr = data + cvec.len = len_ + cvec.cap = len_ + + # Transfer data to Rust + self._mem = orderbook_deltas_new( + instrument_id._mem, + cvec, + ) + + PyMem_Free(cvec.ptr) # De-allocate buffer + PyMem_Free(cvec) # De-allocate cvec + + def __del__(self) -> None: + if self._mem._0 != NULL: + orderbook_deltas_drop(self._mem) def __eq__(self, OrderBookDeltas other) -> bool: return OrderBookDeltas.to_dict_c(self) == OrderBookDeltas.to_dict_c(other) @@ -2154,6 +2192,102 @@ cdef class OrderBookDeltas(Data): f"ts_init={self.ts_init})" ) + @property + def instrument_id(self) -> InstrumentId: + """ + Return the deltas book instrument ID. + + Returns + ------- + InstrumentId + + """ + return InstrumentId.from_mem_c(orderbook_deltas_instrument_id(&self._mem)) + + @property + def deltas(self) -> list[OrderBookDelta]: + """ + Return the contained deltas. + + Returns + ------- + list[OrderBookDeltas] + + """ + cdef CVec raw_deltas_vec = orderbook_deltas_vec_deltas(&self._mem) + cdef OrderBookDelta_t* raw_deltas = raw_deltas_vec.ptr + + cdef list[OrderBookDelta] deltas = [] + + cdef: + uint64_t i + for i in range(raw_deltas_vec.len): + deltas.append(delta_from_mem_c(raw_deltas[i])) + + orderbook_deltas_vec_drop(raw_deltas_vec) + + return deltas + + @property + def is_snapshot(self) -> bool: + """ + If the deltas is a snapshot. + + Returns + ------- + bool + + """ + return orderbook_deltas_is_snapshot(&self._mem) + + @property + def flags(self) -> uint8_t: + """ + Return the flags for the delta. + + Returns + ------- + uint8_t + + """ + return orderbook_deltas_flags(&self._mem) + + @property + def sequence(self) -> uint64_t: + """ + Return the sequence number for the delta. + + Returns + ------- + uint64_t + + """ + return orderbook_deltas_sequence(&self._mem) + + @property + def ts_event(self) -> int: + """ + The UNIX timestamp (nanoseconds) when the data event occurred. + + Returns + ------- + int + + """ + return orderbook_deltas_ts_event(&self._mem) + + @property + def ts_init(self) -> int: + """ + The UNIX timestamp (nanoseconds) when the object was initialized. + + Returns + ------- + int + + """ + return orderbook_deltas_ts_init(&self._mem) + @staticmethod cdef OrderBookDeltas from_dict_c(dict values): Condition.not_none(values, "values") @@ -2167,7 +2301,7 @@ cdef class OrderBookDeltas(Data): Condition.not_none(obj, "obj") return { "type": obj.__class__.__name__, - "instrument_id": obj.instrument_id.to_str(), + "instrument_id": obj.instrument_id.value, "deltas": [OrderBookDelta.to_dict_c(d) for d in obj.deltas], } diff --git a/tests/mem_leak_tests/tracemalloc_orderbook_delta.py b/tests/mem_leak_tests/tracemalloc_orderbook_delta.py new file mode 100644 index 000000000000..c36bce772ee6 --- /dev/null +++ b/tests/mem_leak_tests/tracemalloc_orderbook_delta.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +from nautilus_trader.model.data import OrderBookDelta +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 +from nautilus_trader.test_kit.stubs.data import TestDataStubs +from tests.mem_leak_tests.conftest import snapshot_memory + + +@snapshot_memory(4000) +def run_repr(*args, **kwargs): + delta = TestDataStubs.order_book_delta() + repr(delta) # Copies bids and asks book order data from Rust on every iteration + + +@snapshot_memory(4000) +def run_from_pyo3(*args, **kwargs): + pyo3_delta = TestDataProviderPyo3.order_book_delta() + OrderBookDelta.from_pyo3(pyo3_delta) + + +if __name__ == "__main__": + run_repr() + run_from_pyo3() diff --git a/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py b/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py index 79c6f0ecfd88..b12f2cb3d78e 100644 --- a/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py +++ b/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py @@ -14,24 +14,25 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -from nautilus_trader.model.data import OrderBookDelta -from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 +from nautilus_trader.model.data import OrderBookDeltas from nautilus_trader.test_kit.stubs.data import TestDataStubs from tests.mem_leak_tests.conftest import snapshot_memory @snapshot_memory(4000) def run_repr(*args, **kwargs): - depth = TestDataStubs.order_book_delta() - repr(depth) # Copies bids and asks book order data from Rust on every iteration + delta = TestDataStubs.order_book_delta() + deltas = OrderBookDeltas(delta.instrument_id, deltas=[delta] * 1024) + repr(deltas.deltas) + repr(deltas) -@snapshot_memory(4000) -def run_from_pyo3(*args, **kwargs): - pyo3_delta = TestDataProviderPyo3.order_book_delta() - OrderBookDelta.from_pyo3(pyo3_delta) +# @snapshot_memory(4000) +# def run_from_pyo3(*args, **kwargs): +# pyo3_delta = TestDataProviderPyo3.order_book_delta() +# OrderBookDelta.from_pyo3(pyo3_delta) if __name__ == "__main__": run_repr() - run_from_pyo3() + # run_from_pyo3()