From 029a53089f7991f98f49167151a8d12fb8878f22 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 10 Feb 2024 07:43:53 +1100 Subject: [PATCH 001/104] Bump version --- RELEASES.md | 15 +++++++++++++++ nautilus_core/Cargo.lock | 28 ++++++++++++++-------------- nautilus_core/Cargo.toml | 2 +- poetry.lock | 6 +++--- pyproject.toml | 2 +- version.json | 2 +- 6 files changed, 35 insertions(+), 20 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 65b7c6cef449..fd69da689bcf 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,18 @@ +# NautilusTrader 1.188.0 Beta + +Released on TBD (UTC). + +### Enhancements +None + +### Breaking Changes +None + +### Fixes +None + +--- + # NautilusTrader 1.187.0 Beta Released on 9th February 2024 (UTC). diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 682cfd2ed0a1..08c0e7031751 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2118,12 +2118,12 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "fe8f25ce1159c7740ff0b9b2f5cdf4a8428742ba7c112b9f20f22cd5219c7dab" dependencies = [ "hermit-abi 0.3.5", - "rustix", + "libc", "windows-sys 0.52.0", ] @@ -2413,7 +2413,7 @@ dependencies = [ [[package]] name = "nautilus-accounting" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "cbindgen", @@ -2429,7 +2429,7 @@ dependencies = [ [[package]] name = "nautilus-adapters" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "chrono", @@ -2460,7 +2460,7 @@ dependencies = [ [[package]] name = "nautilus-backtest" -version = "0.18.0" +version = "0.19.0" dependencies = [ "cbindgen", "nautilus-common", @@ -2474,7 +2474,7 @@ dependencies = [ [[package]] name = "nautilus-common" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "cbindgen", @@ -2498,7 +2498,7 @@ dependencies = [ [[package]] name = "nautilus-core" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "cbindgen", @@ -2517,7 +2517,7 @@ dependencies = [ [[package]] name = "nautilus-indicators" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "nautilus-core", @@ -2529,7 +2529,7 @@ dependencies = [ [[package]] name = "nautilus-infrastructure" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "nautilus-common", @@ -2544,7 +2544,7 @@ dependencies = [ [[package]] name = "nautilus-model" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "cbindgen", @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "nautilus-network" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "axum", @@ -2597,7 +2597,7 @@ dependencies = [ [[package]] name = "nautilus-persistence" -version = "0.18.0" +version = "0.19.0" dependencies = [ "anyhow", "binary-heap-plus", @@ -2621,7 +2621,7 @@ dependencies = [ [[package]] name = "nautilus-pyo3" -version = "0.18.0" +version = "0.19.0" dependencies = [ "nautilus-accounting", "nautilus-adapters", diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index e1c218d63437..8d897493da42 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -17,7 +17,7 @@ members = [ [workspace.package] rust-version = "1.76.0" -version = "0.18.0" +version = "0.19.0" edition = "2021" authors = ["Nautech Systems "] description = "A high-performance algorithmic trading platform and event-driven backtester" diff --git a/poetry.lock b/poetry.lock index fd77ee67e791..e0b64054ce80 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2356,13 +2356,13 @@ files = [ [[package]] name = "uc-micro-py" -version = "1.0.2" +version = "1.0.3" description = "Micro subset of unicode data files for linkify-it-py projects." optional = false python-versions = ">=3.7" files = [ - {file = "uc-micro-py-1.0.2.tar.gz", hash = "sha256:30ae2ac9c49f39ac6dce743bd187fcd2b574b16ca095fa74cd9396795c954c54"}, - {file = "uc_micro_py-1.0.2-py3-none-any.whl", hash = "sha256:8c9110c309db9d9e87302e2f4ad2c3152770930d88ab385cd544e7a7e75f3de0"}, + {file = "uc-micro-py-1.0.3.tar.gz", hash = "sha256:d321b92cff673ec58027c04015fcaa8bb1e005478643ff4a500882eaab88c48a"}, + {file = "uc_micro_py-1.0.3-py3-none-any.whl", hash = "sha256:db1dffff340817673d7b466ec86114a9dc0e9d4d9b5ba229d9d60e5c12600cd5"}, ] [package.extras] diff --git a/pyproject.toml b/pyproject.toml index 0432bcfc6c33..736fc62981df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nautilus_trader" -version = "1.187.0" +version = "1.188.0" description = "A high-performance algorithmic trading platform and event-driven backtester" authors = ["Nautech Systems "] license = "LGPL-3.0-or-later" diff --git a/version.json b/version.json index 8c5225749776..b64e7ef6788b 100644 --- a/version.json +++ b/version.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, "label": "", - "message": "v1.187.0", + "message": "v1.188.0", "color": "orange" } From 912b55d7651765c52a690927fa50f3e2d88442f1 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 10 Feb 2024 09:10:26 +1100 Subject: [PATCH 002/104] Fix typo --- nautilus_core/model/src/identifiers/account_id.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautilus_core/model/src/identifiers/account_id.rs b/nautilus_core/model/src/identifiers/account_id.rs index 9b586a4153ae..cb6091dd0c69 100644 --- a/nautilus_core/model/src/identifiers/account_id.rs +++ b/nautilus_core/model/src/identifiers/account_id.rs @@ -43,7 +43,7 @@ pub struct AccountId { impl AccountId { pub fn new(s: &str) -> Result { check_valid_string(s, "`accountid` value")?; - check_string_contains(s, "-", "`traderid` value")?; + check_string_contains(s, "-", "`AccountId` value")?; Ok(Self { value: Ustr::from(s), From 68862f8e397c51677c3cfccd342ead40b4c20519 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 10 Feb 2024 09:11:05 +1100 Subject: [PATCH 003/104] Refine Databento InstrumentId parsing --- .../adapters/src/databento/common.rs | 17 ---- .../adapters/src/databento/loader.rs | 81 ++++++++++++------- .../adapters/src/databento/parsing.rs | 12 +-- .../src/databento/python/historical.rs | 43 ++++++---- .../adapters/src/databento/python/live.rs | 47 ++++++----- .../adapters/src/databento/symbology.rs | 21 ++--- 6 files changed, 117 insertions(+), 104 deletions(-) diff --git a/nautilus_core/adapters/src/databento/common.rs b/nautilus_core/adapters/src/databento/common.rs index 49cd2cdbe703..f4fb7f8ac0c6 100644 --- a/nautilus_core/adapters/src/databento/common.rs +++ b/nautilus_core/adapters/src/databento/common.rs @@ -16,28 +16,11 @@ use anyhow::Result; use databento::historical::DateTimeRange; use nautilus_core::time::UnixNanos; -use nautilus_model::identifiers::{instrument_id::InstrumentId, symbol::Symbol, venue::Venue}; use time::OffsetDateTime; -use ustr::Ustr; - -use super::types::DatabentoPublisher; pub const DATABENTO: &str = "DATABENTO"; pub const ALL_SYMBOLS: &str = "ALL_SYMBOLS"; -#[must_use] -pub fn nautilus_instrument_id_from_databento( - raw_symbol: Ustr, - publisher: &DatabentoPublisher, -) -> InstrumentId { - let symbol = Symbol { value: raw_symbol }; - let venue = Venue { - value: Ustr::from(publisher.venue.as_str()), - }; // TODO: Optimize - - InstrumentId::new(symbol, venue) -} - pub fn get_date_time_range(start: UnixNanos, end: Option) -> Result { match end { Some(end) => Ok(DateTimeRange::from(( diff --git a/nautilus_core/adapters/src/databento/loader.rs b/nautilus_core/adapters/src/databento/loader.rs index 2b33f68d1043..8d280b49c3ad 100644 --- a/nautilus_core/adapters/src/databento/loader.rs +++ b/nautilus_core/adapters/src/databento/loader.rs @@ -35,10 +35,8 @@ use time; use ustr::Ustr; use super::{ - parsing::{parse_instrument_def_msg_v1, parse_record}, - types::DatabentoPublisher, - types::Dataset, - types::PublisherId, + parsing::{parse_instrument_def_msg_v1, parse_raw_ptr_to_ustr, parse_record}, + types::{DatabentoPublisher, Dataset, PublisherId}, }; /// Provides a Nautilus data loader for Databento Binary Encoding (DBN) format data. @@ -75,15 +73,17 @@ use super::{ pyclass(module = "nautilus_trader.core.nautilus_pyo3.databento") )] pub struct DatabentoDataLoader { - publishers: IndexMap, - venue_dataset: IndexMap, + publishers_map: IndexMap, + venue_dataset_map: IndexMap, + publisher_venue_map: IndexMap, } impl DatabentoDataLoader { pub fn new(path: Option) -> Result { let mut loader = Self { - publishers: IndexMap::new(), - venue_dataset: IndexMap::new(), + publishers_map: IndexMap::new(), + venue_dataset_map: IndexMap::new(), + publisher_venue_map: IndexMap::new(), }; // Load publishers @@ -108,13 +108,13 @@ impl DatabentoDataLoader { let file_content = fs::read_to_string(path)?; let publishers: Vec = serde_json::from_str(&file_content)?; - self.publishers = publishers + self.publishers_map = publishers .clone() .into_iter() .map(|p| (p.publisher_id, p)) .collect::>(); - self.venue_dataset = publishers + self.venue_dataset_map = publishers .iter() .map(|p| { ( @@ -124,42 +124,55 @@ impl DatabentoDataLoader { }) .collect::>(); + self.publisher_venue_map = publishers + .clone() + .into_iter() + .map(|p| (p.publisher_id, Venue::from(p.venue.as_str()))) + .collect::>(); + Ok(()) } /// Return the internal Databento publishers currently held by the loader. #[must_use] pub fn get_publishers(&self) -> &IndexMap { - &self.publishers + &self.publishers_map } // Return the dataset which matches the given `venue` (if found). #[must_use] pub fn get_dataset_for_venue(&self, venue: &Venue) -> Option<&Dataset> { - self.venue_dataset.get(venue) + self.venue_dataset_map.get(venue) + } + + // Return the venue which matches the given `publisher_id` (if found). + #[must_use] + pub fn get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option<&Venue> { + self.publisher_venue_map.get(&publisher_id) } pub fn get_nautilus_instrument_id_for_record( &self, record: &dbn::RecordRef, metadata: &dbn::Metadata, + venue: Venue, ) -> Result { - let (publisher_id, instrument_id, nanoseconds) = match record.rtype()? { + let (instrument_id, nanoseconds) = match record.rtype()? { dbn::RType::Mbo => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Mbp0 => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Mbp1 => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Mbp10 => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Ohlcv1S | dbn::RType::Ohlcv1M @@ -167,7 +180,7 @@ impl DatabentoDataLoader { | dbn::RType::Ohlcv1D | dbn::RType::OhlcvEod => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.hd.ts_event) + (msg.hd.instrument_id, msg.hd.ts_event) } _ => bail!("RType is currently unsupported by NautilusTrader"), }; @@ -175,7 +188,7 @@ impl DatabentoDataLoader { let duration = time::Duration::nanoseconds(nanoseconds as i64); let datetime = time::OffsetDateTime::UNIX_EPOCH .checked_add(duration) - .unwrap(); + .unwrap(); // SAFETY: Relying on correctness of record timestamps let date = datetime.date(); let symbol_map = metadata.symbol_map_for_date(date)?; let raw_symbol = symbol_map @@ -185,10 +198,6 @@ impl DatabentoDataLoader { let symbol = Symbol { value: Ustr::from(raw_symbol), }; - let venue_str = self.publishers.get(&publisher_id).unwrap().venue.as_str(); - let venue = Venue { - value: Ustr::from(venue_str), - }; Ok(InstrumentId::new(symbol, venue)) } @@ -221,9 +230,16 @@ impl DatabentoDataLoader { let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); let instrument_id = match &instrument_id { Some(id) => *id, // Copy - None => self - .get_nautilus_instrument_id_for_record(&rec_ref, &metadata) - .expect("Error resolving symbology mapping for {rec_ref}"), + None => { + let publisher_id = rec_ref.publisher().expect("No publisher for record") + as PublisherId; + let venue = self + .publisher_venue_map + .get(&publisher_id) + .expect("`Venue` not found for `publisher_id`"); + self.get_nautilus_instrument_id_for_record(&rec_ref, &metadata, *venue) + .expect("Error resolving symbology mapping for {rec_ref}") + } }; match parse_record(&rec_ref, rtype, instrument_id, price_precision, None) { @@ -251,9 +267,18 @@ impl DatabentoDataLoader { let rec_ref = dbn::RecordRef::from(record); let msg = rec_ref.get::().unwrap(); - let publisher = self.publishers.get(&msg.hd.publisher_id).unwrap(); + let raw_symbol = unsafe { + parse_raw_ptr_to_ustr(record.raw_symbol.as_ptr()) + .expect("Error parsing `raw_symbol`") + }; + let symbol = Symbol { value: raw_symbol }; + let venue = self + .publisher_venue_map + .get(&msg.hd.publisher_id) + .expect("`Venue` not found `publisher_id`"); + let instrument_id = InstrumentId::new(symbol, *venue); - match parse_instrument_def_msg_v1(record, publisher, msg.ts_recv) { + match parse_instrument_def_msg_v1(record, instrument_id, msg.ts_recv) { Ok(data) => Some(Ok(data)), Err(e) => Some(Err(e)), } diff --git a/nautilus_core/adapters/src/databento/parsing.rs b/nautilus_core/adapters/src/databento/parsing.rs index 7f17f7e838ef..fe99146a8fea 100644 --- a/nautilus_core/adapters/src/databento/parsing.rs +++ b/nautilus_core/adapters/src/databento/parsing.rs @@ -47,8 +47,6 @@ use nautilus_model::{ }; use ustr::Ustr; -use super::{common::nautilus_instrument_id_from_databento, types::DatabentoPublisher}; - const BAR_SPEC_1S: BarSpecification = BarSpecification { step: 1, aggregation: BarAggregation::Second, @@ -569,12 +567,9 @@ pub fn parse_record( pub fn parse_instrument_def_msg_v1( record: &dbn::compat::InstrumentDefMsgV1, - publisher: &DatabentoPublisher, + instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result> { - let raw_symbol = unsafe { parse_raw_ptr_to_ustr(record.raw_symbol.as_ptr())? }; - let instrument_id = nautilus_instrument_id_from_databento(raw_symbol, publisher); - match record.instrument_class as u8 as char { 'K' => Ok(Box::new(parse_equity_v1(record, instrument_id, ts_init)?)), 'F' => Ok(Box::new(parse_futures_contract_v1( @@ -601,12 +596,9 @@ pub fn parse_instrument_def_msg_v1( pub fn parse_instrument_def_msg( record: &dbn::InstrumentDefMsg, - publisher: &DatabentoPublisher, + instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result> { - let raw_symbol = unsafe { parse_raw_ptr_to_ustr(record.raw_symbol.as_ptr())? }; - let instrument_id = nautilus_instrument_id_from_databento(raw_symbol, publisher); - match record.instrument_class as u8 as char { 'K' => Ok(Box::new(parse_equity(record, instrument_id, ts_init)?)), 'F' => Ok(Box::new(parse_futures_contract( diff --git a/nautilus_core/adapters/src/databento/python/historical.rs b/nautilus_core/adapters/src/databento/python/historical.rs index f0a5a5415601..e45b1ad0da9c 100644 --- a/nautilus_core/adapters/src/databento/python/historical.rs +++ b/nautilus_core/adapters/src/databento/python/historical.rs @@ -25,6 +25,7 @@ use nautilus_core::{ use nautilus_model::{ data::{bar::Bar, quote::QuoteTick, trade::TradeTick, Data}, enums::BarAggregation, + identifiers::{instrument_id::InstrumentId, symbol::Symbol, venue::Venue}, }; use pyo3::{ exceptions::PyException, @@ -35,7 +36,7 @@ use tokio::sync::Mutex; use crate::databento::{ common::get_date_time_range, - parsing::{parse_instrument_def_msg, parse_record}, + parsing::{parse_instrument_def_msg, parse_raw_ptr_to_ustr, parse_record}, symbology::parse_nautilus_instrument_id, types::{DatabentoPublisher, PublisherId}, }; @@ -49,7 +50,7 @@ use super::loader::convert_instrument_to_pyobject; pub struct DatabentoHistoricalClient { clock: &'static AtomicTime, inner: Arc>, - publishers: Arc>, + publisher_venue_map: Arc>, #[pyo3(get)] pub key: String, } @@ -67,15 +68,16 @@ impl DatabentoHistoricalClient { let file_content = fs::read_to_string(publishers_path)?; let publishers_vec: Vec = serde_json::from_str(&file_content).map_err(to_pyvalue_err)?; - let publishers = publishers_vec + + let publisher_venue_map = publishers_vec .into_iter() - .map(|p| (p.publisher_id, p)) - .collect::>(); + .map(|p| (p.publisher_id, Venue::from(p.venue.as_str()))) + .collect::>(); Ok(Self { clock: get_atomic_clock_realtime(), inner: Arc::new(Mutex::new(client)), - publishers: Arc::new(publishers), + publisher_venue_map: Arc::new(publisher_venue_map), key, }) } @@ -122,7 +124,7 @@ impl DatabentoHistoricalClient { .limit(limit.and_then(NonZeroU64::new)) .build(); - let publishers = self.publishers.clone(); + let publisher_venue_map = self.publisher_venue_map.clone(); let ts_init = self.clock.get_time_ns(); pyo3_asyncio::tokio::future_into_py(py, async move { @@ -138,9 +140,12 @@ impl DatabentoHistoricalClient { let mut instruments = Vec::new(); while let Ok(Some(rec)) = decoder.decode_record::().await { - let publisher_id = rec.publisher().unwrap() as PublisherId; - let publisher = publishers.get(&publisher_id).unwrap(); - let result = parse_instrument_def_msg(rec, publisher, ts_init); + let raw_symbol = unsafe { parse_raw_ptr_to_ustr(rec.raw_symbol.as_ptr()).unwrap() }; + let symbol = Symbol { value: raw_symbol }; + let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); + let instrument_id = InstrumentId::new(symbol, *venue); + + let result = parse_instrument_def_msg(rec, instrument_id, ts_init); match result { Ok(instrument) => instruments.push(instrument), Err(e) => eprintln!("{e:?}"), @@ -180,7 +185,7 @@ impl DatabentoHistoricalClient { .build(); let price_precision = 2; // TODO: Hard coded for now - let publishers = self.publishers.clone(); + let publisher_venue_map = self.publisher_venue_map.clone(); let ts_init = self.clock.get_time_ns(); pyo3_asyncio::tokio::future_into_py(py, async move { @@ -197,7 +202,9 @@ impl DatabentoHistoricalClient { while let Ok(Some(rec)) = decoder.decode_record::().await { let rec_ref = dbn::RecordRef::from(rec); let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); - let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, &publishers) + + let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); + let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, *venue) .map_err(to_pyvalue_err)?; let (data, _) = parse_record( @@ -243,7 +250,7 @@ impl DatabentoHistoricalClient { .build(); let price_precision = 2; // TODO: Hard coded for now - let publishers = self.publishers.clone(); + let publisher_venue_map = self.publisher_venue_map.clone(); let ts_init = self.clock.get_time_ns(); pyo3_asyncio::tokio::future_into_py(py, async move { @@ -260,7 +267,9 @@ impl DatabentoHistoricalClient { while let Ok(Some(rec)) = decoder.decode_record::().await { let rec_ref = dbn::RecordRef::from(rec); let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); - let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, &publishers) + + let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); + let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, *venue) .map_err(to_pyvalue_err)?; let (data, _) = parse_record( @@ -315,7 +324,7 @@ impl DatabentoHistoricalClient { .build(); let price_precision = 2; // TODO: Hard coded for now - let publishers = self.publishers.clone(); + let publisher_venue_map = self.publisher_venue_map.clone(); let ts_init = self.clock.get_time_ns(); pyo3_asyncio::tokio::future_into_py(py, async move { @@ -332,7 +341,9 @@ impl DatabentoHistoricalClient { while let Ok(Some(rec)) = decoder.decode_record::().await { let rec_ref = dbn::RecordRef::from(rec); let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); - let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, &publishers) + + let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); + let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, *venue) .map_err(to_pyvalue_err)?; let (data, _) = parse_record( diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index a70ec9ad79df..9e572cdac6c9 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -38,7 +38,7 @@ use time::OffsetDateTime; use tokio::sync::Mutex; use tokio::time::{timeout, Duration}; -use crate::databento::parsing::{parse_instrument_def_msg, parse_record}; +use crate::databento::parsing::{parse_instrument_def_msg, parse_raw_ptr_to_ustr, parse_record}; use crate::databento::types::{DatabentoPublisher, PublisherId}; use super::loader::convert_instrument_to_pyobject; @@ -54,7 +54,7 @@ pub struct DatabentoLiveClient { pub dataset: String, inner: Option>>, runtime: tokio::runtime::Runtime, - publishers: Arc>, + publisher_venue_map: Arc>, } impl DatabentoLiveClient { @@ -86,17 +86,18 @@ impl DatabentoLiveClient { let file_content = fs::read_to_string(publishers_path)?; let publishers_vec: Vec = serde_json::from_str(&file_content).map_err(to_pyvalue_err)?; - let publishers = publishers_vec + + let publisher_venue_map = publishers_vec .into_iter() - .map(|p| (p.publisher_id, p)) - .collect::>(); + .map(|p| (p.publisher_id, Venue::from(p.venue.as_str()))) + .collect::>(); Ok(Self { key, dataset, inner: None, runtime: tokio::runtime::Runtime::new()?, - publishers: Arc::new(publishers), + publisher_venue_map: Arc::new(publisher_venue_map), }) } @@ -145,7 +146,7 @@ impl DatabentoLiveClient { #[pyo3(name = "start")] fn py_start<'py>(&mut self, py: Python<'py>, callback: PyObject) -> PyResult<&'py PyAny> { let arc_client = self.get_inner_client().map_err(to_pyruntime_err)?; - let publishers = self.publishers.clone(); + let publisher_venue_map = self.publisher_venue_map.clone(); pyo3_asyncio::tokio::future_into_py(py, async move { let clock = get_atomic_clock_realtime(); @@ -207,10 +208,14 @@ impl DatabentoLiveClient { let msg = record .get::() .expect("Error converting record to `InstrumentDefMsg`"); - let publisher_id = record.publisher().unwrap() as PublisherId; - let publisher = publishers.get(&publisher_id).unwrap(); + let raw_symbol = + unsafe { parse_raw_ptr_to_ustr(msg.raw_symbol.as_ptr()).unwrap() }; + let symbol = Symbol { value: raw_symbol }; + let venue = publisher_venue_map.get(&msg.hd.publisher_id).unwrap(); + let instrument_id = InstrumentId::new(symbol, *venue); + let ts_init = clock.get_time_ns(); - let result = parse_instrument_def_msg(msg, publisher, ts_init); + let result = parse_instrument_def_msg(msg, instrument_id, ts_init); match result { Ok(instrument) => { @@ -230,19 +235,23 @@ impl DatabentoLiveClient { _ => { let raw_symbol = symbol_map .get_for_rec(&record) - .expect("Cannot resolve raw_symbol from `symbol_map`"); - - let symbol = Symbol::from_str_unchecked(raw_symbol); + .expect("Cannot resolve `raw_symbol` from `symbol_map`"); let publisher_id = record.publisher().unwrap() as PublisherId; - let venue_str = publishers.get(&publisher_id).unwrap().venue.as_str(); - let venue = Venue::from_str_unchecked(venue_str); + let symbol = Symbol::from_str_unchecked(raw_symbol); + let venue = publisher_venue_map.get(&publisher_id).unwrap(); + let instrument_id = InstrumentId::new(symbol, *venue); - let instrument_id = InstrumentId::new(symbol, venue); + let price_precision = 2; // Hard coded for now let ts_init = clock.get_time_ns(); - let (data, maybe_data) = - parse_record(&record, rtype, instrument_id, 2, Some(ts_init)) - .map_err(to_pyvalue_err)?; + let (data, maybe_data) = parse_record( + &record, + rtype, + instrument_id, + price_precision, + Some(ts_init), + ) + .map_err(to_pyvalue_err)?; Python::with_gil(|py| { call_python_with_data(py, &callback, data); diff --git a/nautilus_core/adapters/src/databento/symbology.rs b/nautilus_core/adapters/src/databento/symbology.rs index 04358cb1113b..7b7cfb605134 100644 --- a/nautilus_core/adapters/src/databento/symbology.rs +++ b/nautilus_core/adapters/src/databento/symbology.rs @@ -16,33 +16,30 @@ use anyhow::{bail, Result}; use databento::dbn; use dbn::Record; -use indexmap::IndexMap; use nautilus_model::identifiers::{instrument_id::InstrumentId, symbol::Symbol, venue::Venue}; use ustr::Ustr; -use super::{types::DatabentoPublisher, types::PublisherId}; - pub fn parse_nautilus_instrument_id( record: &dbn::RecordRef, metadata: &dbn::Metadata, - publishers: &IndexMap, + venue: Venue, ) -> Result { - let (publisher_id, instrument_id, nanoseconds) = match record.rtype()? { + let (instrument_id, nanoseconds) = match record.rtype()? { dbn::RType::Mbo => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Mbp0 => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Mbp1 => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Mbp10 => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.ts_recv) + (msg.hd.instrument_id, msg.ts_recv) } dbn::RType::Ohlcv1S | dbn::RType::Ohlcv1M @@ -50,7 +47,7 @@ pub fn parse_nautilus_instrument_id( | dbn::RType::Ohlcv1D | dbn::RType::OhlcvEod => { let msg = record.get::().unwrap(); // SAFETY: RType known - (msg.hd.publisher_id, msg.hd.instrument_id, msg.hd.ts_event) + (msg.hd.instrument_id, msg.hd.ts_event) } _ => bail!("RType is currently unsupported by NautilusTrader"), }; @@ -68,10 +65,6 @@ pub fn parse_nautilus_instrument_id( let symbol = Symbol { value: Ustr::from(raw_symbol), }; - let venue_str = publishers.get(&publisher_id).unwrap().venue.as_str(); - let venue = Venue { - value: Ustr::from(venue_str), - }; Ok(InstrumentId::new(symbol, venue)) } From 4f06291a784b30799c2a03051065de51903f7b13 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 10 Feb 2024 12:05:47 +1100 Subject: [PATCH 004/104] Optimize DatabentoDataLoader --- .../adapters/src/databento/python/loader.rs | 143 ++++++++++++++++-- nautilus_core/model/src/python/data/mod.rs | 26 ++++ nautilus_core/model/src/python/mod.rs | 1 + nautilus_trader/adapters/databento/loaders.py | 74 ++++++--- nautilus_trader/core/nautilus_pyo3.pyi | 15 +- .../adapters/databento/test_loaders.py | 14 ++ .../mem_leak_tests/memray_databento_loader.py | 34 +++++ 7 files changed, 274 insertions(+), 33 deletions(-) create mode 100644 tests/mem_leak_tests/memray_databento_loader.py diff --git a/nautilus_core/adapters/src/databento/python/loader.rs b/nautilus_core/adapters/src/databento/python/loader.rs index d31671a703b7..3058cb12dac0 100644 --- a/nautilus_core/adapters/src/databento/python/loader.rs +++ b/nautilus_core/adapters/src/databento/python/loader.rs @@ -15,7 +15,10 @@ use std::{any::Any, collections::HashMap, path::PathBuf}; -use nautilus_core::python::to_pyvalue_err; +use nautilus_core::{ + ffi::cvec::CVec, + python::{to_pyruntime_err, to_pyvalue_err}, +}; use nautilus_model::{ data::{ bar::Bar, delta::OrderBookDelta, depth::OrderBookDepth10, quote::QuoteTick, @@ -27,9 +30,15 @@ use nautilus_model::{ Instrument, }, }; -use pyo3::{prelude::*, types::PyList}; +use pyo3::{ + prelude::*, + types::{PyCapsule, PyList}, +}; -use crate::databento::{loader::DatabentoDataLoader, types::DatabentoPublisher}; +use crate::databento::{ + loader::DatabentoDataLoader, + types::{DatabentoPublisher, PublisherId}, +}; #[pymethods] impl DatabentoDataLoader { @@ -51,6 +60,12 @@ impl DatabentoDataLoader { self.get_dataset_for_venue(venue).map(|d| d.to_string()) } + #[pyo3(name = "get_venue_for_publisher")] + pub fn py_get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option { + self.get_venue_for_publisher(publisher_id) + .map(|d| d.to_string()) + } + #[pyo3(name = "schema_for_file")] pub fn py_schema_for_file(&self, path: String) -> PyResult> { self.schema_from_file(PathBuf::from(path)) @@ -112,6 +127,21 @@ impl DatabentoDataLoader { Ok(data) } + #[pyo3(name = "load_order_book_deltas_as_pycapsule")] + pub fn py_load_order_book_deltas_as_pycapsule( + &self, + py: Python, + path: String, + instrument_id: Option, + ) -> PyResult { + let path_buf = PathBuf::from(path); + let iter = self + .read_records::(path_buf, instrument_id) + .map_err(to_pyvalue_err)?; + + exhaust_data_iter_to_pycapsule(py, iter) + } + #[pyo3(name = "load_order_book_depth10")] pub fn py_load_order_book_depth10( &self, @@ -138,8 +168,23 @@ impl DatabentoDataLoader { Ok(data) } - #[pyo3(name = "load_quote_ticks")] - pub fn py_load_quote_ticks( + #[pyo3(name = "load_order_book_depth10_as_pycapsule")] + pub fn py_load_order_book_depth10_as_pycapsule( + &self, + py: Python, + path: String, + instrument_id: Option, + ) -> PyResult { + let path_buf = PathBuf::from(path); + let iter = self + .read_records::(path_buf, instrument_id) + .map_err(to_pyvalue_err)?; + + exhaust_data_iter_to_pycapsule(py, iter) + } + + #[pyo3(name = "load_quotes")] + pub fn py_load_quotes( &self, path: String, instrument_id: Option, @@ -164,17 +209,32 @@ impl DatabentoDataLoader { Ok(data) } - #[pyo3(name = "load_tbbo_trade_ticks")] - pub fn py_load_tbbo_trade_ticks( + #[pyo3(name = "load_quotes_as_pycapsule")] + pub fn py_load_quotes_as_pycapsule( &self, + py: Python, path: String, instrument_id: Option, - ) -> PyResult> { + ) -> PyResult { let path_buf = PathBuf::from(path); let iter = self .read_records::(path_buf, instrument_id) .map_err(to_pyvalue_err)?; + exhaust_data_iter_to_pycapsule(py, iter) + } + + #[pyo3(name = "load_tbbo_trades")] + pub fn py_load_tbbo_trades( + &self, + path: String, + instrument_id: Option, + ) -> PyResult> { + let path_buf = PathBuf::from(path); + let iter = self + .read_records::(path_buf, instrument_id) + .map_err(to_pyvalue_err)?; + let mut data = Vec::new(); for result in iter { match result { @@ -190,8 +250,23 @@ impl DatabentoDataLoader { Ok(data) } - #[pyo3(name = "load_trade_ticks")] - pub fn py_load_trade_ticks( + #[pyo3(name = "load_tbbo_trades_as_pycapsule")] + pub fn py_load_tbbo_trades_as_pycapsule( + &self, + py: Python, + path: String, + instrument_id: Option, + ) -> PyResult { + let path_buf = PathBuf::from(path); + let iter = self + .read_records::(path_buf, instrument_id) + .map_err(to_pyvalue_err)?; + + exhaust_data_iter_to_pycapsule(py, iter) + } + + #[pyo3(name = "load_trades")] + pub fn py_load_trades( &self, path: String, instrument_id: Option, @@ -216,6 +291,21 @@ impl DatabentoDataLoader { Ok(data) } + #[pyo3(name = "load_trades_as_pycapsule")] + pub fn py_load_trades_as_pycapsule( + &self, + py: Python, + path: String, + instrument_id: Option, + ) -> PyResult { + let path_buf = PathBuf::from(path); + let iter = self + .read_records::(path_buf, instrument_id) + .map_err(to_pyvalue_err)?; + + exhaust_data_iter_to_pycapsule(py, iter) + } + #[pyo3(name = "load_bars")] pub fn py_load_bars( &self, @@ -241,6 +331,21 @@ impl DatabentoDataLoader { Ok(data) } + + #[pyo3(name = "load_bars_as_pycapsule")] + pub fn py_load_bars_as_pycapsule( + &self, + py: Python, + path: String, + instrument_id: Option, + ) -> PyResult { + let path_buf = PathBuf::from(path); + let iter = self + .read_records::(path_buf, instrument_id) + .map_err(to_pyvalue_err)?; + + exhaust_data_iter_to_pycapsule(py, iter) + } } pub fn convert_instrument_to_pyobject( @@ -262,3 +367,21 @@ pub fn convert_instrument_to_pyobject( "Unknown instrument type", )) } + +fn exhaust_data_iter_to_pycapsule( + py: Python, + iter: impl Iterator)>>, +) -> PyResult { + let mut data = Vec::new(); + for result in iter { + match result { + Ok((item1, _)) => data.push(item1), + Err(e) => return Err(to_pyvalue_err(e)), + } + } + + let cvec: CVec = data.into(); + let capsule = PyCapsule::new::(py, cvec, None).map_err(to_pyruntime_err)?; + + Ok(capsule.into_py(py)) +} diff --git a/nautilus_core/model/src/python/data/mod.rs b/nautilus_core/model/src/python/data/mod.rs index bf43fe842a7e..48c110cbbad8 100644 --- a/nautilus_core/model/src/python/data/mod.rs +++ b/nautilus_core/model/src/python/data/mod.rs @@ -20,6 +20,7 @@ pub mod order; pub mod quote; pub mod trade; +use nautilus_core::ffi::cvec::CVec; use pyo3::{prelude::*, types::PyCapsule}; use crate::data::Data; @@ -46,3 +47,28 @@ pub fn data_to_pycapsule(py: Python, data: Data) -> PyObject { let capsule = PyCapsule::new(py, data, None).expect("Error creating `PyCapsule`"); capsule.into_py(py) } + +/// Drops a `PyCapsule` containing a `CVec` structure. +/// +/// This function safely extracts and drops the `CVec` instance encapsulated within +/// a `PyCapsule` object. It is intended for cleaning up after the `Data` instances +/// have been transferred into Python and are no longer needed. +/// +/// # Panics +/// +/// Panics if the capsule cannot be downcast to a `PyCapsule`, indicating a type mismatch +/// or improper capsule handling. +/// +/// # Safety +/// +/// This function is unsafe as it involves raw pointer dereferencing and manual memory +/// management. The caller must ensure the `PyCapsule` contains a valid `CVec` pointer. +/// Incorrect usage can lead to memory corruption or undefined behavior. +#[pyfunction] +pub fn drop_cvec_pycapsule(capsule: &PyAny) { + let capsule: &PyCapsule = capsule.downcast().expect("Error on downcast to capsule"); + let cvec: &CVec = unsafe { &*(capsule.pointer() as *const CVec) }; + let data: Vec = + unsafe { Vec::from_raw_parts(cvec.ptr.cast::(), cvec.len, cvec.cap) }; + drop(data); +} diff --git a/nautilus_core/model/src/python/mod.rs b/nautilus_core/model/src/python/mod.rs index cfcb5905af20..bd9b4a66f1c4 100644 --- a/nautilus_core/model/src/python/mod.rs +++ b/nautilus_core/model/src/python/mod.rs @@ -230,6 +230,7 @@ mod tests { #[pymodule] pub fn model(_: Python<'_>, m: &PyModule) -> PyResult<()> { // Data + m.add_function(wrap_pyfunction!(data::drop_cvec_pycapsule, m)?)?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/nautilus_trader/adapters/databento/loaders.py b/nautilus_trader/adapters/databento/loaders.py index a7f07c9d8d8e..5ec430486134 100644 --- a/nautilus_trader/adapters/databento/loaders.py +++ b/nautilus_trader/adapters/databento/loaders.py @@ -20,11 +20,8 @@ from nautilus_trader.adapters.databento.enums import DatabentoSchema from nautilus_trader.core import nautilus_pyo3 from nautilus_trader.core.data import Data -from nautilus_trader.model.data import Bar -from nautilus_trader.model.data import OrderBookDelta -from nautilus_trader.model.data import OrderBookDepth10 -from nautilus_trader.model.data import QuoteTick -from nautilus_trader.model.data import TradeTick +from nautilus_trader.core.nautilus_pyo3 import drop_cvec_pycapsule +from nautilus_trader.model.data import capsule_to_list from nautilus_trader.model.identifiers import InstrumentId from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.instruments import instruments_from_pyo3 @@ -169,30 +166,58 @@ def from_dbn_file( match schema: case DatabentoSchema.DEFINITION.value: - data = self._pyo3_loader.load_instruments(path) # type: ignore + data = self._pyo3_loader.load_instruments(str(path)) if as_legacy_cython: data = instruments_from_pyo3(data) return data case DatabentoSchema.MBO.value: - data = self._pyo3_loader.load_order_book_deltas(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: - data = OrderBookDelta.from_pyo3_list(data) - return data + capsule = self._pyo3_loader.load_order_book_deltas_as_pycapsule( + path=str(path), + instrument_id=pyo3_instrument_id, + ) + data = capsule_to_list(capsule) + # Drop encapsulated `CVec` as data is now transferred + drop_cvec_pycapsule(capsule) + return data + else: + return self._pyo3_loader.load_order_book_deltas(str(path), pyo3_instrument_id) case DatabentoSchema.MBP_1.value | DatabentoSchema.TBBO.value: - data = self._pyo3_loader.load_quote_ticks(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: - data = QuoteTick.from_pyo3_list(data) - return data + capsule = self._pyo3_loader.load_quotes_as_pycapsule( + path=str(path), + instrument_id=pyo3_instrument_id, + ) + data = capsule_to_list(capsule) + # Drop encapsulated `CVec` as data is now transferred + drop_cvec_pycapsule(capsule) + return data + else: + return self._pyo3_loader.load_quotes(str(path), pyo3_instrument_id) case DatabentoSchema.MBP_10.value: - data = self._pyo3_loader.load_order_book_depth10(path) # type: ignore if as_legacy_cython: - data = OrderBookDepth10.from_pyo3_list(data) - return data + capsule = self._pyo3_loader.load_order_book_depth10_as_pycapsule( + path=str(path), + instrument_id=pyo3_instrument_id, + ) + data = capsule_to_list(capsule) + # Drop encapsulated `CVec` as data is now transferred + drop_cvec_pycapsule(capsule) + return data + else: + return self._pyo3_loader.load_order_book_depth10(str(path), pyo3_instrument_id) case DatabentoSchema.TRADES.value: - data = self._pyo3_loader.load_trade_ticks(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: - data = TradeTick.from_pyo3_list(data) - return data + capsule = self._pyo3_loader.load_trades_as_pycapsule( + path=str(path), + instrument_id=pyo3_instrument_id, + ) + data = capsule_to_list(capsule) + # Drop encapsulated `CVec` as data is now transferred + drop_cvec_pycapsule(capsule) + return data + else: + return self._pyo3_loader.load_trades(str(path), pyo3_instrument_id) case ( DatabentoSchema.OHLCV_1S.value | DatabentoSchema.OHLCV_1M.value @@ -200,9 +225,16 @@ def from_dbn_file( | DatabentoSchema.OHLCV_1D.value | DatabentoSchema.OHLCV_EOD ): - data = self._pyo3_loader.load_bars(path, pyo3_instrument_id) # type: ignore if as_legacy_cython: - data = Bar.from_pyo3_list(data) - return data + capsule = self._pyo3_loader.load_bars_as_pycapsule( + path=str(path), + instrument_id=pyo3_instrument_id, + ) + data = capsule_to_list(capsule) + # Drop encapsulated `CVec` as data is now transferred + drop_cvec_pycapsule(capsule) + return data + else: + return self._pyo3_loader.load_bars(str(path), pyo3_instrument_id) case _: raise RuntimeError(f"Loading schema {schema} not currently supported") diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index bde8efcbf9df..61ff099fac8c 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -293,8 +293,6 @@ class Position: def notional_value(self, price: Price) -> Money: ... - - class MarginAccount: def __init__( self, @@ -386,6 +384,8 @@ class CashAccount: ### Data types +def drop_cvec_pycapsule(capsule: object) -> None: ... + class BarSpecification: def __init__( self, @@ -1966,6 +1966,17 @@ class DatabentoDataLoader: def get_dataset_for_venue(self, venue: Venue) -> str: ... def load_publishers(self, path: PathLike[str] | str) -> None: ... def schema_for_file(self, path: str) -> str: ... + def load_instruments(self, path: str) -> list[Instrument]: ... + def load_order_book_deltas(self, path: str, instrument_id: InstrumentId | None) -> list[OrderBookDelta]: ... + def load_order_book_deltas_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... + def load_order_book_depth10(self, path: str, instrument_id: InstrumentId | None) -> list[OrderBookDepth10]: ... + def load_order_book_depth10_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... + def load_quotes(self, path: str, instrument_id: InstrumentId | None) -> list[QuoteTick]: ... + def load_quotes_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... + def load_trades(self, path: str, instrument_id: InstrumentId | None) -> list[TradeTick]: ... + def load_trades_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... + def load_bars(self, path: str, instrument_id: InstrumentId | None) -> list[Bar]: ... + def load_bars_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... class DatabentoHistoricalClient: def __init__( diff --git a/tests/integration_tests/adapters/databento/test_loaders.py b/tests/integration_tests/adapters/databento/test_loaders.py index 120b1f86b955..330a40667890 100644 --- a/tests/integration_tests/adapters/databento/test_loaders.py +++ b/tests/integration_tests/adapters/databento/test_loaders.py @@ -344,6 +344,20 @@ def test_loader_with_trades() -> None: assert trade.ts_init == 1609160400099150057 +@pytest.mark.skip("development_only") +def test_loader_with_trades_large() -> None: + # Arrange + loader = DatabentoDataLoader() + path = DATABENTO_TEST_DATA_DIR / "temp" / "tsla-xnas-20240107-20240206.trades.dbn.zst" + instrument_id = InstrumentId.from_str("TSLA.XNAS") + + # Act + data = loader.from_dbn_file(path, instrument_id=instrument_id, as_legacy_cython=True) + + # Assert + assert len(data) == 6_885_435 + + def test_loader_with_ohlcv_1s() -> None: # Arrange loader = DatabentoDataLoader() diff --git a/tests/mem_leak_tests/memray_databento_loader.py b/tests/mem_leak_tests/memray_databento_loader.py new file mode 100644 index 000000000000..38d26219ceae --- /dev/null +++ b/tests/mem_leak_tests/memray_databento_loader.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader +from nautilus_trader.model.identifiers import InstrumentId +from tests.integration_tests.adapters.databento.test_loaders import DATABENTO_TEST_DATA_DIR + + +if __name__ == "__main__": + loader = DatabentoDataLoader() + path = DATABENTO_TEST_DATA_DIR / "temp" / "tsla-xnas-20240107-20240206.trades.dbn.zst" + instrument_id = InstrumentId.from_str("TSLA.XNAS") + + count = 0 + total_runs = 128 + while count < total_runs: + count += 1 + print(f"Run: {count}/{total_runs}") + + data = loader.from_dbn_file(path, instrument_id=instrument_id, as_legacy_cython=True) + assert len(data) == 6_885_435 From d8cc92a6d84144c94fe70872a1c23689158d3c88 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 10 Feb 2024 13:16:33 +1100 Subject: [PATCH 005/104] Refine Databento include_trades filtering --- .../adapters/src/databento/loader.rs | 12 +++- .../adapters/src/databento/parsing.rs | 61 +++++++++++-------- .../src/databento/python/historical.rs | 9 ++- .../adapters/src/databento/python/live.rs | 5 +- .../adapters/src/databento/python/loader.rs | 55 +++++++++++------ .../adapters/src/databento/python/parsing.rs | 14 +++-- nautilus_trader/adapters/databento/loaders.py | 18 +++++- nautilus_trader/core/nautilus_pyo3.pyi | 8 +-- tests/unit_tests/model/test_orderbook.py | 2 - 9 files changed, 122 insertions(+), 62 deletions(-) diff --git a/nautilus_core/adapters/src/databento/loader.rs b/nautilus_core/adapters/src/databento/loader.rs index 8d280b49c3ad..589ac6e7f311 100644 --- a/nautilus_core/adapters/src/databento/loader.rs +++ b/nautilus_core/adapters/src/databento/loader.rs @@ -212,7 +212,8 @@ impl DatabentoDataLoader { &self, path: PathBuf, instrument_id: Option, - ) -> Result)>> + '_> + include_trades: bool, + ) -> Result, Option)>> + '_> where T: dbn::Record + dbn::HasRType + 'static, { @@ -242,7 +243,14 @@ impl DatabentoDataLoader { } }; - match parse_record(&rec_ref, rtype, instrument_id, price_precision, None) { + match parse_record( + &rec_ref, + rtype, + instrument_id, + price_precision, + None, + include_trades, + ) { Ok(data) => Some(Ok(data)), Err(e) => Some(Err(e)), } diff --git a/nautilus_core/adapters/src/databento/parsing.rs b/nautilus_core/adapters/src/databento/parsing.rs index fe99146a8fea..5824926a5131 100644 --- a/nautilus_core/adapters/src/databento/parsing.rs +++ b/nautilus_core/adapters/src/databento/parsing.rs @@ -275,19 +275,24 @@ pub fn parse_mbo_msg( instrument_id: InstrumentId, price_precision: u8, ts_init: UnixNanos, + include_trades: bool, ) -> Result<(Option, Option)> { let side = parse_order_side(record.side); if is_trade_msg(side, record.action) { - let trade = TradeTick::new( - instrument_id, - Price::from_raw(record.price, price_precision)?, - Quantity::from_raw(u64::from(record.size) * FIXED_SCALAR as u64, 0)?, - parse_aggressor_side(record.side), - TradeId::new(itoa::Buffer::new().format(record.sequence))?, - record.ts_recv, - ts_init, - ); - return Ok((None, Some(trade))); + if include_trades { + let trade = TradeTick::new( + instrument_id, + Price::from_raw(record.price, price_precision)?, + Quantity::from_raw(u64::from(record.size) * FIXED_SCALAR as u64, 0)?, + parse_aggressor_side(record.side), + TradeId::new(itoa::Buffer::new().format(record.sequence))?, + record.ts_recv, + ts_init, + ); + return Ok((None, Some(trade))); + } else { + return Ok((None, None)); + } }; let order = BookOrder::new( @@ -334,6 +339,7 @@ pub fn parse_mbp1_msg( instrument_id: InstrumentId, price_precision: u8, ts_init: UnixNanos, + include_trades: bool, ) -> Result<(QuoteTick, Option)> { let top_level = &record.levels[0]; let quote = QuoteTick::new( @@ -346,8 +352,8 @@ pub fn parse_mbp1_msg( ts_init, )?; - let trade = match record.action as u8 as char { - 'T' => Some(TradeTick::new( + let maybe_trade = if include_trades && record.action as u8 as char == 'T' { + Some(TradeTick::new( instrument_id, Price::from_raw(record.price, price_precision)?, Quantity::from_raw(u64::from(record.size) * FIXED_SCALAR as u64, 0)?, @@ -355,11 +361,12 @@ pub fn parse_mbp1_msg( TradeId::new(itoa::Buffer::new().format(record.sequence))?, record.ts_recv, ts_init, - )), - _ => None, + )) + } else { + None }; - Ok((quote, trade)) + Ok((quote, maybe_trade)) } pub fn parse_mbp10_msg( @@ -501,7 +508,8 @@ pub fn parse_record( instrument_id: InstrumentId, price_precision: u8, ts_init: Option, -) -> Result<(Data, Option)> { + include_trades: bool, +) -> Result<(Option, Option)> { let result = match rtype { dbn::RType::Mbo => { let msg = record.get::().unwrap(); // SAFETY: RType known @@ -509,10 +517,12 @@ pub fn parse_record( Some(ts_init) => ts_init, None => msg.ts_recv, }; - let result = parse_mbo_msg(msg, instrument_id, price_precision, ts_init)?; + let result = + parse_mbo_msg(msg, instrument_id, price_precision, ts_init, include_trades)?; match result { - (Some(delta), None) => (Data::Delta(delta), None), - (None, Some(trade)) => (Data::Trade(trade), None), + (Some(delta), None) => (Some(Data::Delta(delta)), None), + (None, Some(trade)) => (Some(Data::Trade(trade)), None), + (None, None) => (None, None), _ => bail!("Invalid `MboMsg` parsing combination"), } } @@ -523,7 +533,7 @@ pub fn parse_record( None => msg.ts_recv, }; let trade = parse_trade_msg(msg, instrument_id, price_precision, ts_init)?; - (Data::Trade(trade), None) + (Some(Data::Trade(trade)), None) } dbn::RType::Mbp1 => { let msg = record.get::().unwrap(); // SAFETY: RType known @@ -531,10 +541,11 @@ pub fn parse_record( Some(ts_init) => ts_init, None => msg.ts_recv, }; - let result = parse_mbp1_msg(msg, instrument_id, price_precision, ts_init)?; + let result = + parse_mbp1_msg(msg, instrument_id, price_precision, ts_init, include_trades)?; match result { - (quote, None) => (Data::Quote(quote), None), - (quote, Some(trade)) => (Data::Quote(quote), Some(Data::Trade(trade))), + (quote, None) => (Some(Data::Quote(quote)), None), + (quote, Some(trade)) => (Some(Data::Quote(quote)), Some(Data::Trade(trade))), } } dbn::RType::Mbp10 => { @@ -544,7 +555,7 @@ pub fn parse_record( None => msg.ts_recv, }; let depth = parse_mbp10_msg(msg, instrument_id, price_precision, ts_init)?; - (Data::Depth10(depth), None) + (Some(Data::Depth10(depth)), None) } dbn::RType::Ohlcv1S | dbn::RType::Ohlcv1M @@ -557,7 +568,7 @@ pub fn parse_record( None => msg.hd.ts_event, }; let bar = parse_ohlcv_msg(msg, instrument_id, price_precision, ts_init)?; - (Data::Bar(bar), None) + (Some(Data::Bar(bar)), None) } _ => bail!("RType {:?} is not currently supported", rtype), }; diff --git a/nautilus_core/adapters/src/databento/python/historical.rs b/nautilus_core/adapters/src/databento/python/historical.rs index e45b1ad0da9c..e3c2605f1c2b 100644 --- a/nautilus_core/adapters/src/databento/python/historical.rs +++ b/nautilus_core/adapters/src/databento/python/historical.rs @@ -213,11 +213,12 @@ impl DatabentoHistoricalClient { instrument_id, price_precision, Some(ts_init), + false, // Don't include trades ) .map_err(to_pyvalue_err)?; match data { - Data::Quote(quote) => { + Some(Data::Quote(quote)) => { result.push(quote); } _ => panic!("Invalid data element not `QuoteTick`, was {data:?}"), @@ -278,11 +279,12 @@ impl DatabentoHistoricalClient { instrument_id, price_precision, Some(ts_init), + false, // Not applicable (trade will be decoded regardless) ) .map_err(to_pyvalue_err)?; match data { - Data::Trade(trade) => { + Some(Data::Trade(trade)) => { result.push(trade); } _ => panic!("Invalid data element not `TradeTick`, was {data:?}"), @@ -352,11 +354,12 @@ impl DatabentoHistoricalClient { instrument_id, price_precision, Some(ts_init), + false, // Not applicable ) .map_err(to_pyvalue_err)?; match data { - Data::Bar(bar) => { + Some(Data::Bar(bar)) => { result.push(bar); } _ => panic!("Invalid data element not `Bar`, was {data:?}"), diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 9e572cdac6c9..3038105f3c66 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -250,11 +250,14 @@ impl DatabentoLiveClient { instrument_id, price_precision, Some(ts_init), + true, // Always include trades ) .map_err(to_pyvalue_err)?; Python::with_gil(|py| { - call_python_with_data(py, &callback, data); + if let Some(data) = data { + call_python_with_data(py, &callback, data); + } if let Some(data) = maybe_data { call_python_with_data(py, &callback, data); diff --git a/nautilus_core/adapters/src/databento/python/loader.rs b/nautilus_core/adapters/src/databento/python/loader.rs index 3058cb12dac0..ede2ad6ae2ef 100644 --- a/nautilus_core/adapters/src/databento/python/loader.rs +++ b/nautilus_core/adapters/src/databento/python/loader.rs @@ -101,6 +101,7 @@ impl DatabentoDataLoader { Ok(PyList::new(py, &data).into()) } + /// Cannot include trades #[pyo3(name = "load_order_book_deltas")] pub fn py_load_order_book_deltas( &self, @@ -109,17 +110,18 @@ impl DatabentoDataLoader { ) -> PyResult> { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; let mut data = Vec::new(); for result in iter { match result { - Ok((item1, _)) => { + Ok((Some(item1), _)) => { if let Data::Delta(delta) = item1 { data.push(delta); } } + Ok((None, _)) => continue, Err(e) => return Err(to_pyvalue_err(e)), } } @@ -133,10 +135,11 @@ impl DatabentoDataLoader { py: Python, path: String, instrument_id: Option, + include_trades: Option, ) -> PyResult { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, include_trades.unwrap_or(false)) .map_err(to_pyvalue_err)?; exhaust_data_iter_to_pycapsule(py, iter) @@ -150,17 +153,18 @@ impl DatabentoDataLoader { ) -> PyResult> { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; let mut data = Vec::new(); for result in iter { match result { - Ok((item1, _)) => { + Ok((Some(item1), _)) => { if let Data::Depth10(depth) = item1 { data.push(depth); } } + Ok((None, _)) => continue, Err(e) => return Err(to_pyvalue_err(e)), } } @@ -177,7 +181,7 @@ impl DatabentoDataLoader { ) -> PyResult { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; exhaust_data_iter_to_pycapsule(py, iter) @@ -188,20 +192,22 @@ impl DatabentoDataLoader { &self, path: String, instrument_id: Option, + include_trades: Option, ) -> PyResult> { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, include_trades.unwrap_or(false)) .map_err(to_pyvalue_err)?; let mut data = Vec::new(); for result in iter { match result { - Ok((item1, _)) => { + Ok((Some(item1), _)) => { if let Data::Quote(quote) = item1 { data.push(quote); } } + Ok((None, _)) => continue, Err(e) => return Err(to_pyvalue_err(e)), } } @@ -215,10 +221,11 @@ impl DatabentoDataLoader { py: Python, path: String, instrument_id: Option, + include_trades: Option, ) -> PyResult { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, include_trades.unwrap_or(false)) .map_err(to_pyvalue_err)?; exhaust_data_iter_to_pycapsule(py, iter) @@ -232,7 +239,7 @@ impl DatabentoDataLoader { ) -> PyResult> { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; let mut data = Vec::new(); @@ -259,7 +266,7 @@ impl DatabentoDataLoader { ) -> PyResult { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; exhaust_data_iter_to_pycapsule(py, iter) @@ -273,17 +280,18 @@ impl DatabentoDataLoader { ) -> PyResult> { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; let mut data = Vec::new(); for result in iter { match result { - Ok((item1, _)) => { + Ok((Some(item1), _)) => { if let Data::Trade(trade) = item1 { data.push(trade); } } + Ok((None, _)) => continue, Err(e) => return Err(to_pyvalue_err(e)), } } @@ -300,7 +308,7 @@ impl DatabentoDataLoader { ) -> PyResult { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; exhaust_data_iter_to_pycapsule(py, iter) @@ -314,17 +322,18 @@ impl DatabentoDataLoader { ) -> PyResult> { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; let mut data = Vec::new(); for result in iter { match result { - Ok((item1, _)) => { + Ok((Some(item1), _)) => { if let Data::Bar(bar) = item1 { data.push(bar); } } + Ok((None, _)) => continue, Err(e) => return Err(to_pyvalue_err(e)), } } @@ -341,7 +350,7 @@ impl DatabentoDataLoader { ) -> PyResult { let path_buf = PathBuf::from(path); let iter = self - .read_records::(path_buf, instrument_id) + .read_records::(path_buf, instrument_id, false) .map_err(to_pyvalue_err)?; exhaust_data_iter_to_pycapsule(py, iter) @@ -370,12 +379,20 @@ pub fn convert_instrument_to_pyobject( fn exhaust_data_iter_to_pycapsule( py: Python, - iter: impl Iterator)>>, + iter: impl Iterator, Option)>>, ) -> PyResult { let mut data = Vec::new(); for result in iter { match result { - Ok((item1, _)) => data.push(item1), + Ok((Some(item1), None)) => data.push(item1), + Ok((None, Some(item2))) => data.push(item2), + Ok((Some(item1), Some(item2))) => { + data.push(item1); + data.push(item2) + } + Ok((None, None)) => { + continue; + } Err(e) => return Err(to_pyvalue_err(e)), } } diff --git a/nautilus_core/adapters/src/databento/python/parsing.rs b/nautilus_core/adapters/src/databento/python/parsing.rs index c087261fe6f9..2f833c1a0b19 100644 --- a/nautilus_core/adapters/src/databento/python/parsing.rs +++ b/nautilus_core/adapters/src/databento/python/parsing.rs @@ -68,11 +68,10 @@ pub fn py_parse_mbo_msg( price_precision: u8, ts_init: UnixNanos, ) -> PyResult { - let result = parse_mbo_msg(record, instrument_id, price_precision, ts_init); + let result = parse_mbo_msg(record, instrument_id, price_precision, ts_init, false); match result { - Ok((Some(delta), None)) => Ok(delta.into_py(py)), - Ok((None, Some(trade))) => Ok(trade.into_py(py)), + Ok((Some(data), None)) => Ok(data.into_py(py)), Err(e) => Err(to_pyvalue_err(e)), _ => Err(PyRuntimeError::new_err("Error parsing MBO message")), } @@ -97,8 +96,15 @@ pub fn py_parse_mbp1_msg( instrument_id: InstrumentId, price_precision: u8, ts_init: UnixNanos, + include_trades: bool, ) -> PyResult { - let result = parse_mbp1_msg(record, instrument_id, price_precision, ts_init); + let result = parse_mbp1_msg( + record, + instrument_id, + price_precision, + ts_init, + include_trades, + ); match result { Ok((quote, Some(trade))) => { diff --git a/nautilus_trader/adapters/databento/loaders.py b/nautilus_trader/adapters/databento/loaders.py index 5ec430486134..b00b9a67d2fd 100644 --- a/nautilus_trader/adapters/databento/loaders.py +++ b/nautilus_trader/adapters/databento/loaders.py @@ -121,6 +121,7 @@ def from_dbn_file( path: PathLike[str] | str, instrument_id: InstrumentId | None = None, as_legacy_cython: bool = True, + include_trades: bool = False, ) -> list[Data]: """ Return a list of data objects decoded from the DBN file at the given `path`. @@ -138,6 +139,9 @@ def from_dbn_file( If data should be converted to 'legacy Cython' objects. You would typically only set this False if passing the objects directly to a data catalog for the data to then be written in Nautilus Parquet format. + include_trades : bool, False + If separate `TradeTick` elements will be included in the data for MBO and MBO-1 schemas + (your code will have to handle these two types in the returned list). Returns ------- @@ -175,25 +179,35 @@ def from_dbn_file( capsule = self._pyo3_loader.load_order_book_deltas_as_pycapsule( path=str(path), instrument_id=pyo3_instrument_id, + include_trades=include_trades, ) data = capsule_to_list(capsule) # Drop encapsulated `CVec` as data is now transferred drop_cvec_pycapsule(capsule) return data else: - return self._pyo3_loader.load_order_book_deltas(str(path), pyo3_instrument_id) + return self._pyo3_loader.load_order_book_deltas( + path=str(path), + instrument_id=pyo3_instrument_id, + include_trades=include_trades, + ) case DatabentoSchema.MBP_1.value | DatabentoSchema.TBBO.value: if as_legacy_cython: capsule = self._pyo3_loader.load_quotes_as_pycapsule( path=str(path), instrument_id=pyo3_instrument_id, + include_trades=include_trades, ) data = capsule_to_list(capsule) # Drop encapsulated `CVec` as data is now transferred drop_cvec_pycapsule(capsule) return data else: - return self._pyo3_loader.load_quotes(str(path), pyo3_instrument_id) + return self._pyo3_loader.load_quotes( + path=str(path), + instrument_id=pyo3_instrument_id, + include_trades=include_trades, + ) case DatabentoSchema.MBP_10.value: if as_legacy_cython: capsule = self._pyo3_loader.load_order_book_depth10_as_pycapsule( diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 61ff099fac8c..7d2b737513dd 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1967,12 +1967,12 @@ class DatabentoDataLoader: def load_publishers(self, path: PathLike[str] | str) -> None: ... def schema_for_file(self, path: str) -> str: ... def load_instruments(self, path: str) -> list[Instrument]: ... - def load_order_book_deltas(self, path: str, instrument_id: InstrumentId | None) -> list[OrderBookDelta]: ... - def load_order_book_deltas_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... + def load_order_book_deltas(self, path: str, instrument_id: InstrumentId | None, include_trades: bool | None) -> list[OrderBookDelta]: ... + def load_order_book_deltas_as_pycapsule(self, path: str, instrument_id: InstrumentId | None, include_trades: bool | None) -> object: ... def load_order_book_depth10(self, path: str, instrument_id: InstrumentId | None) -> list[OrderBookDepth10]: ... def load_order_book_depth10_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... - def load_quotes(self, path: str, instrument_id: InstrumentId | None) -> list[QuoteTick]: ... - def load_quotes_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... + def load_quotes(self, path: str, instrument_id: InstrumentId | None, include_trades: bool | None) -> list[QuoteTick]: ... + def load_quotes_as_pycapsule(self, path: str, instrument_id: InstrumentId | None, include_trades: bool | None) -> object: ... def load_trades(self, path: str, instrument_id: InstrumentId | None) -> list[TradeTick]: ... def load_trades_as_pycapsule(self, path: str, instrument_id: InstrumentId | None) -> object: ... def load_bars(self, path: str, instrument_id: InstrumentId | None) -> list[Bar]: ... diff --git a/tests/unit_tests/model/test_orderbook.py b/tests/unit_tests/model/test_orderbook.py index 01eb3afd8e31..52138d4946eb 100644 --- a/tests/unit_tests/model/test_orderbook.py +++ b/tests/unit_tests/model/test_orderbook.py @@ -699,8 +699,6 @@ def test_orderbook_esh4_glbx_20231224_mbo_l3(self) -> None: ) for delta in data: - if not isinstance(delta, OrderBookDelta): - continue book.apply_delta(delta) # Assert From 800cad54088d7e7a932e9cbc550e86b1ba53b6b5 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 10 Feb 2024 18:20:28 +1100 Subject: [PATCH 006/104] Improve clarity of param description --- nautilus_trader/adapters/databento/loaders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nautilus_trader/adapters/databento/loaders.py b/nautilus_trader/adapters/databento/loaders.py index b00b9a67d2fd..63cbbc38bae1 100644 --- a/nautilus_trader/adapters/databento/loaders.py +++ b/nautilus_trader/adapters/databento/loaders.py @@ -140,8 +140,8 @@ def from_dbn_file( You would typically only set this False if passing the objects directly to a data catalog for the data to then be written in Nautilus Parquet format. include_trades : bool, False - If separate `TradeTick` elements will be included in the data for MBO and MBO-1 schemas - (your code will have to handle these two types in the returned list). + If separate `TradeTick` elements will be included in the data for MBO and MBP-1 schemas + when applicable (your code will have to handle these two types in the returned list). Returns ------- From 34aefe8ba7ffd491cdb6950a6835770754e66557 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 10 Feb 2024 18:52:11 +1100 Subject: [PATCH 007/104] Standardize instruments pyo3 interface --- .../model/src/instruments/crypto_future.rs | 20 --- .../model/src/instruments/crypto_perpetual.rs | 27 +--- .../model/src/instruments/currency_pair.rs | 31 ++-- nautilus_core/model/src/instruments/equity.rs | 14 +- .../model/src/instruments/futures_contract.rs | 16 --- .../model/src/instruments/options_contract.rs | 18 --- nautilus_core/model/src/instruments/stubs.rs | 6 + .../src/python/instruments/crypto_future.rs | 132 +++++++++++++++++- .../python/instruments/crypto_perpetual.rs | 125 ++++++++++++++++- .../src/python/instruments/currency_pair.rs | 127 ++++++++++++++++- .../model/src/python/instruments/equity.rs | 83 +++++++++-- .../python/instruments/futures_contract.rs | 105 +++++++++++++- .../python/instruments/options_contract.rs | 122 +++++++++++++++- nautilus_trader/core/nautilus_pyo3.pyi | 83 ++++++----- nautilus_trader/model/instruments/base.pxd | 2 +- nautilus_trader/model/instruments/base.pyx | 2 +- .../model/instruments/crypto_future.pyx | 2 +- .../model/instruments/crypto_perpetual.pyx | 2 +- .../model/instruments/currency_pair.pyx | 2 +- nautilus_trader/model/instruments/equity.pyx | 4 +- .../model/instruments/futures_contract.pyx | 2 +- .../model/instruments/options_contract.pyx | 2 +- .../instruments/test_currency_pair_pyo3.py | 2 + 23 files changed, 734 insertions(+), 195 deletions(-) diff --git a/nautilus_core/model/src/instruments/crypto_future.rs b/nautilus_core/model/src/instruments/crypto_future.rs index 4f3758de29a1..6fd1bec66382 100644 --- a/nautilus_core/model/src/instruments/crypto_future.rs +++ b/nautilus_core/model/src/instruments/crypto_future.rs @@ -38,45 +38,25 @@ use crate::{ )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct CryptoFuture { - #[pyo3(get)] pub id: InstrumentId, - #[pyo3(get)] pub raw_symbol: Symbol, - #[pyo3(get)] pub underlying: Currency, - #[pyo3(get)] pub quote_currency: Currency, - #[pyo3(get)] pub settlement_currency: Currency, - #[pyo3(get)] pub activation_ns: UnixNanos, - #[pyo3(get)] pub expiration_ns: UnixNanos, - #[pyo3(get)] pub price_precision: u8, - #[pyo3(get)] pub size_precision: u8, - #[pyo3(get)] pub price_increment: Price, - #[pyo3(get)] pub size_increment: Quantity, - #[pyo3(get)] pub lot_size: Option, - #[pyo3(get)] pub max_quantity: Option, - #[pyo3(get)] pub min_quantity: Option, - #[pyo3(get)] pub max_notional: Option, - #[pyo3(get)] pub min_notional: Option, - #[pyo3(get)] pub max_price: Option, - #[pyo3(get)] pub min_price: Option, - #[pyo3(get)] pub ts_event: UnixNanos, - #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/instruments/crypto_perpetual.rs b/nautilus_core/model/src/instruments/crypto_perpetual.rs index 0421d648a182..3089cf4c9552 100644 --- a/nautilus_core/model/src/instruments/crypto_perpetual.rs +++ b/nautilus_core/model/src/instruments/crypto_perpetual.rs @@ -39,51 +39,28 @@ use crate::{ )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct CryptoPerpetual { - #[pyo3(get)] pub id: InstrumentId, - #[pyo3(get)] pub raw_symbol: Symbol, - #[pyo3(get)] pub base_currency: Currency, - #[pyo3(get)] pub quote_currency: Currency, - #[pyo3(get)] pub settlement_currency: Currency, - #[pyo3(get)] pub is_inverse: bool, - #[pyo3(get)] pub price_precision: u8, - #[pyo3(get)] pub size_precision: u8, - #[pyo3(get)] pub price_increment: Price, - #[pyo3(get)] pub size_increment: Quantity, - #[pyo3(get)] pub maker_fee: Decimal, - #[pyo3(get)] pub taker_fee: Decimal, - #[pyo3(get)] pub margin_init: Decimal, - #[pyo3(get)] pub margin_maint: Decimal, - #[pyo3(get)] pub lot_size: Option, - #[pyo3(get)] pub max_quantity: Option, - #[pyo3(get)] pub min_quantity: Option, - #[pyo3(get)] pub max_notional: Option, - #[pyo3(get)] pub min_notional: Option, - #[pyo3(get)] pub max_price: Option, - #[pyo3(get)] pub min_price: Option, - #[pyo3(get)] pub ts_event: UnixNanos, - #[pyo3(get)] pub ts_init: UnixNanos, } @@ -240,15 +217,19 @@ impl Instrument for CryptoPerpetual { fn as_any(&self) -> &dyn Any { self } + fn taker_fee(&self) -> Decimal { self.taker_fee } + fn maker_fee(&self) -> Decimal { self.maker_fee } + fn margin_init(&self) -> Decimal { self.margin_init } + fn margin_maint(&self) -> Decimal { self.margin_maint } diff --git a/nautilus_core/model/src/instruments/currency_pair.rs b/nautilus_core/model/src/instruments/currency_pair.rs index a6e07e8c001d..66383b5550e8 100644 --- a/nautilus_core/model/src/instruments/currency_pair.rs +++ b/nautilus_core/model/src/instruments/currency_pair.rs @@ -28,7 +28,7 @@ use super::Instrument; use crate::{ enums::{AssetClass, InstrumentClass}, identifiers::{instrument_id::InstrumentId, symbol::Symbol}, - types::{currency::Currency, price::Price, quantity::Quantity}, + types::{currency::Currency, money::Money, price::Price, quantity::Quantity}, }; #[repr(C)] @@ -39,43 +39,26 @@ use crate::{ )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct CurrencyPair { - #[pyo3(get)] pub id: InstrumentId, - #[pyo3(get)] pub raw_symbol: Symbol, - #[pyo3(get)] pub base_currency: Currency, - #[pyo3(get)] pub quote_currency: Currency, - #[pyo3(get)] pub price_precision: u8, - #[pyo3(get)] pub size_precision: u8, - #[pyo3(get)] pub price_increment: Price, - #[pyo3(get)] pub size_increment: Quantity, - #[pyo3(get)] pub maker_fee: Decimal, - #[pyo3(get)] pub taker_fee: Decimal, - #[pyo3(get)] pub margin_init: Decimal, - #[pyo3(get)] pub margin_maint: Decimal, - #[pyo3(get)] pub lot_size: Option, - #[pyo3(get)] pub max_quantity: Option, - #[pyo3(get)] pub min_quantity: Option, - #[pyo3(get)] + pub max_notional: Option, + pub min_notional: Option, pub max_price: Option, - #[pyo3(get)] pub min_price: Option, - #[pyo3(get)] pub ts_event: UnixNanos, - #[pyo3(get)] pub ts_init: UnixNanos, } @@ -97,6 +80,8 @@ impl CurrencyPair { lot_size: Option, max_quantity: Option, min_quantity: Option, + max_notional: Option, + min_notional: Option, max_price: Option, min_price: Option, ts_event: UnixNanos, @@ -118,6 +103,8 @@ impl CurrencyPair { lot_size, max_quantity, min_quantity, + max_notional, + min_notional, max_price, min_price, ts_event, @@ -225,15 +212,19 @@ impl Instrument for CurrencyPair { fn as_any(&self) -> &dyn Any { self } + fn margin_init(&self) -> Decimal { self.margin_init } + fn margin_maint(&self) -> Decimal { self.margin_maint } + fn taker_fee(&self) -> Decimal { self.taker_fee } + fn maker_fee(&self) -> Decimal { self.maker_fee } diff --git a/nautilus_core/model/src/instruments/equity.rs b/nautilus_core/model/src/instruments/equity.rs index aa5c223e25d2..86258b53fed0 100644 --- a/nautilus_core/model/src/instruments/equity.rs +++ b/nautilus_core/model/src/instruments/equity.rs @@ -39,31 +39,19 @@ use crate::{ )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct Equity { - #[pyo3(get)] pub id: InstrumentId, - #[pyo3(get)] pub raw_symbol: Symbol, - /// The instruments ISIN (International Securities Identification Number). + /// The ISIN (International Securities Identification Number). pub isin: Option, - #[pyo3(get)] pub currency: Currency, - #[pyo3(get)] pub price_precision: u8, - #[pyo3(get)] pub price_increment: Price, - #[pyo3(get)] pub lot_size: Option, - #[pyo3(get)] pub max_quantity: Option, - #[pyo3(get)] pub min_quantity: Option, - #[pyo3(get)] pub max_price: Option, - #[pyo3(get)] pub min_price: Option, - #[pyo3(get)] pub ts_event: UnixNanos, - #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/instruments/futures_contract.rs b/nautilus_core/model/src/instruments/futures_contract.rs index 670890ac0455..753c4d68e2c3 100644 --- a/nautilus_core/model/src/instruments/futures_contract.rs +++ b/nautilus_core/model/src/instruments/futures_contract.rs @@ -39,38 +39,22 @@ use crate::{ )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct FuturesContract { - #[pyo3(get)] pub id: InstrumentId, - #[pyo3(get)] pub raw_symbol: Symbol, - #[pyo3(get)] pub asset_class: AssetClass, pub underlying: Ustr, - #[pyo3(get)] pub activation_ns: UnixNanos, - #[pyo3(get)] pub expiration_ns: UnixNanos, - #[pyo3(get)] pub currency: Currency, - #[pyo3(get)] pub price_precision: u8, - #[pyo3(get)] pub price_increment: Price, - #[pyo3(get)] pub multiplier: Quantity, - #[pyo3(get)] pub lot_size: Quantity, - #[pyo3(get)] pub max_quantity: Option, - #[pyo3(get)] pub min_quantity: Option, - #[pyo3(get)] pub max_price: Option, - #[pyo3(get)] pub min_price: Option, - #[pyo3(get)] pub ts_event: UnixNanos, - #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/instruments/options_contract.rs b/nautilus_core/model/src/instruments/options_contract.rs index 6edd39b245a1..3330d369ab6a 100644 --- a/nautilus_core/model/src/instruments/options_contract.rs +++ b/nautilus_core/model/src/instruments/options_contract.rs @@ -39,42 +39,24 @@ use crate::{ )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct OptionsContract { - #[pyo3(get)] pub id: InstrumentId, - #[pyo3(get)] pub raw_symbol: Symbol, - #[pyo3(get)] pub asset_class: AssetClass, pub underlying: Ustr, - #[pyo3(get)] pub option_kind: OptionKind, - #[pyo3(get)] pub activation_ns: UnixNanos, - #[pyo3(get)] pub expiration_ns: UnixNanos, - #[pyo3(get)] pub strike_price: Price, - #[pyo3(get)] pub currency: Currency, - #[pyo3(get)] pub price_precision: u8, - #[pyo3(get)] pub price_increment: Price, - #[pyo3(get)] pub multiplier: Quantity, - #[pyo3(get)] pub lot_size: Quantity, - #[pyo3(get)] pub max_quantity: Option, - #[pyo3(get)] pub min_quantity: Option, - #[pyo3(get)] pub max_price: Option, - #[pyo3(get)] pub min_price: Option, - #[pyo3(get)] pub ts_event: UnixNanos, - #[pyo3(get)] pub ts_init: UnixNanos, } diff --git a/nautilus_core/model/src/instruments/stubs.rs b/nautilus_core/model/src/instruments/stubs.rs index eb1617121a61..2673b1feb297 100644 --- a/nautilus_core/model/src/instruments/stubs.rs +++ b/nautilus_core/model/src/instruments/stubs.rs @@ -179,6 +179,8 @@ pub fn currency_pair_btcusdt() -> CurrencyPair { None, Some(Quantity::from("9000")), Some(Quantity::from("0.000001")), + None, + None, Some(Price::from("1000000")), Some(Price::from("0.01")), 0, @@ -205,6 +207,8 @@ pub fn currency_pair_ethusdt() -> CurrencyPair { None, Some(Quantity::from("9000")), Some(Quantity::from("0.00001")), + None, + None, Some(Price::from("1000000")), Some(Price::from("0.01")), 0, @@ -238,6 +242,8 @@ pub fn default_fx_ccy(symbol: Symbol, venue: Option) -> CurrencyPair { Some(Quantity::from("100")), None, None, + None, + None, 0, 0, ) diff --git a/nautilus_core/model/src/python/instruments/crypto_future.rs b/nautilus_core/model/src/python/instruments/crypto_future.rs index 5ff1ae9dce16..6f90a16bb201 100644 --- a/nautilus_core/model/src/python/instruments/crypto_future.rs +++ b/nautilus_core/model/src/python/instruments/crypto_future.rs @@ -82,11 +82,6 @@ impl CryptoFuture { .map_err(to_pyvalue_err) } - #[getter] - fn instrument_type(&self) -> &str { - "CryptoFuture" - } - fn __hash__(&self) -> isize { let mut hasher = DefaultHasher::new(); self.hash(&mut hasher); @@ -100,11 +95,138 @@ impl CryptoFuture { } } + #[getter] + #[pyo3(name = "instrument_type")] + fn py_instrument_type(&self) -> &str { + stringify!(CryptoFuture) + } + + #[getter] + #[pyo3(name = "id")] + fn py_id(&self) -> InstrumentId { + self.id + } + + #[getter] + #[pyo3(name = "raw_symbol")] + fn py_raw_symbol(&self) -> Symbol { + self.raw_symbol + } + + #[getter] + #[pyo3(name = "underlying")] + fn py_underlying(&self) -> Currency { + self.underlying + } + + #[getter] + #[pyo3(name = "quote_currency")] + fn py_quote_currency(&self) -> Currency { + self.quote_currency + } + + #[getter] + #[pyo3(name = "settlement_currency")] + fn py_settlement_currency(&self) -> Currency { + self.settlement_currency + } + + #[getter] + #[pyo3(name = "activation_ns")] + fn py_activation_ns(&self) -> UnixNanos { + self.activation_ns + } + + #[getter] + #[pyo3(name = "expiration_ns")] + fn py_expiration_ns(&self) -> UnixNanos { + self.expiration_ns + } + + #[getter] + #[pyo3(name = "price_precision")] + fn py_price_precision(&self) -> u8 { + self.price_precision + } + + #[getter] + #[pyo3(name = "size_precision")] + fn py_size_precision(&self) -> u8 { + self.size_precision + } + + #[getter] + #[pyo3(name = "price_increment")] + fn py_price_increment(&self) -> Price { + self.price_increment + } + + #[getter] + #[pyo3(name = "size_increment")] + fn py_size_increment(&self) -> Quantity { + self.size_increment + } + + #[getter] + #[pyo3(name = "lot_size")] + fn py_lot_size(&self) -> Option { + self.lot_size + } + + #[getter] + #[pyo3(name = "max_quantity")] + fn py_max_quantity(&self) -> Option { + self.max_quantity + } + + #[getter] + #[pyo3(name = "min_quantity")] + fn py_min_quantity(&self) -> Option { + self.min_quantity + } + + #[getter] + #[pyo3(name = "max_notional")] + fn py_max_notional(&self) -> Option { + self.max_notional + } + + #[getter] + #[pyo3(name = "min_notional")] + fn py_min_notional(&self) -> Option { + self.min_notional + } + + #[getter] + #[pyo3(name = "max_price")] + fn py_max_price(&self) -> Option { + self.max_price + } + + #[getter] + #[pyo3(name = "min_price")] + fn py_min_price(&self) -> Option { + self.min_price + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + #[staticmethod] #[pyo3(name = "from_dict")] fn py_from_dict(py: Python<'_>, values: Py) -> PyResult { from_dict_pyo3(py, values) } + #[pyo3(name = "to_dict")] fn py_to_dict(&self, py: Python<'_>) -> PyResult { let dict = PyDict::new(py); diff --git a/nautilus_core/model/src/python/instruments/crypto_perpetual.rs b/nautilus_core/model/src/python/instruments/crypto_perpetual.rs index 69867e99aad1..2b91fd8a29c7 100644 --- a/nautilus_core/model/src/python/instruments/crypto_perpetual.rs +++ b/nautilus_core/model/src/python/instruments/crypto_perpetual.rs @@ -88,11 +88,6 @@ impl CryptoPerpetual { .map_err(to_pyvalue_err) } - #[getter] - fn instrument_type(&self) -> &str { - "CryptoPerpetual" - } - fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { match op { CompareOp::Eq => self.eq(other).into_py(py), @@ -106,6 +101,126 @@ impl CryptoPerpetual { hasher.finish() as isize } + #[getter] + #[pyo3(name = "instrument_type")] + fn py_instrument_type(&self) -> &str { + stringify!(CryptoPerpetual) + } + + #[getter] + #[pyo3(name = "id")] + fn py_id(&self) -> InstrumentId { + self.id + } + + #[getter] + #[pyo3(name = "raw_symbol")] + fn py_raw_symbol(&self) -> Symbol { + self.raw_symbol + } + + #[getter] + #[pyo3(name = "base_currency")] + fn py_base_currency(&self) -> Currency { + self.base_currency + } + + #[getter] + #[pyo3(name = "quote_currency")] + fn py_quote_currency(&self) -> Currency { + self.quote_currency + } + + #[getter] + #[pyo3(name = "settlement_currency")] + fn py_settlement_currency(&self) -> Currency { + self.settlement_currency + } + + #[getter] + #[pyo3(name = "is_inverse")] + fn py_is_inverse(&self) -> bool { + self.is_inverse + } + + #[getter] + #[pyo3(name = "price_precision")] + fn py_price_precision(&self) -> u8 { + self.price_precision + } + + #[getter] + #[pyo3(name = "size_precision")] + fn py_size_precision(&self) -> u8 { + self.size_precision + } + + #[getter] + #[pyo3(name = "price_increment")] + fn py_price_increment(&self) -> Price { + self.price_increment + } + + #[getter] + #[pyo3(name = "size_increment")] + fn py_size_increment(&self) -> Quantity { + self.size_increment + } + + #[getter] + #[pyo3(name = "lot_size")] + fn py_lot_size(&self) -> Option { + self.lot_size + } + + #[getter] + #[pyo3(name = "max_quantity")] + fn py_max_quantity(&self) -> Option { + self.max_quantity + } + + #[getter] + #[pyo3(name = "min_quantity")] + fn py_min_quantity(&self) -> Option { + self.min_quantity + } + + #[getter] + #[pyo3(name = "max_notional")] + fn py_max_notional(&self) -> Option { + self.max_notional + } + + #[getter] + #[pyo3(name = "min_notional")] + fn py_min_notional(&self) -> Option { + self.min_notional + } + + #[getter] + #[pyo3(name = "max_price")] + fn py_max_price(&self) -> Option { + self.max_price + } + + #[getter] + #[pyo3(name = "min_price")] + fn py_min_price(&self) -> Option { + self.min_price + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + #[staticmethod] #[pyo3(name = "from_dict")] fn py_from_dict(py: Python<'_>, values: Py) -> PyResult { diff --git a/nautilus_core/model/src/python/instruments/currency_pair.rs b/nautilus_core/model/src/python/instruments/currency_pair.rs index 38386e2093c6..5d33352bc421 100644 --- a/nautilus_core/model/src/python/instruments/currency_pair.rs +++ b/nautilus_core/model/src/python/instruments/currency_pair.rs @@ -28,7 +28,7 @@ use rust_decimal::{prelude::ToPrimitive, Decimal}; use crate::{ identifiers::{instrument_id::InstrumentId, symbol::Symbol}, instruments::currency_pair::CurrencyPair, - types::{currency::Currency, price::Price, quantity::Quantity}, + types::{currency::Currency, money::Money, price::Price, quantity::Quantity}, }; #[pymethods] @@ -53,6 +53,8 @@ impl CurrencyPair { lot_size: Option, max_quantity: Option, min_quantity: Option, + max_notional: Option, + min_notional: Option, max_price: Option, min_price: Option, ) -> PyResult { @@ -72,6 +74,8 @@ impl CurrencyPair { lot_size, max_quantity, min_quantity, + max_notional, + min_notional, max_price, min_price, ts_event, @@ -80,11 +84,6 @@ impl CurrencyPair { .map_err(to_pyvalue_err) } - #[getter] - fn instrument_type(&self) -> &str { - "CurrencyPair" - } - fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { match op { CompareOp::Eq => self.eq(other).into_py(py), @@ -98,6 +97,114 @@ impl CurrencyPair { hasher.finish() as isize } + #[getter] + #[pyo3(name = "instrument_type")] + fn py_instrument_type(&self) -> &str { + stringify!(CurrencyPair) + } + + #[getter] + #[pyo3(name = "id")] + fn py_id(&self) -> InstrumentId { + self.id + } + + #[getter] + #[pyo3(name = "raw_symbol")] + fn py_raw_symbol(&self) -> Symbol { + self.raw_symbol + } + + #[getter] + #[pyo3(name = "base_currency")] + fn py_base_currency(&self) -> Currency { + self.base_currency + } + + #[getter] + #[pyo3(name = "quote_currency")] + fn py_quote_currency(&self) -> Currency { + self.quote_currency + } + + #[getter] + #[pyo3(name = "price_precision")] + fn py_price_precision(&self) -> u8 { + self.price_precision + } + + #[getter] + #[pyo3(name = "size_precision")] + fn py_size_precision(&self) -> u8 { + self.size_precision + } + + #[getter] + #[pyo3(name = "price_increment")] + fn py_price_increment(&self) -> Price { + self.price_increment + } + + #[getter] + #[pyo3(name = "size_increment")] + fn py_size_increment(&self) -> Quantity { + self.size_increment + } + + #[getter] + #[pyo3(name = "lot_size")] + fn py_lot_size(&self) -> Option { + self.lot_size + } + + #[getter] + #[pyo3(name = "max_quantity")] + fn py_max_quantity(&self) -> Option { + self.max_quantity + } + + #[getter] + #[pyo3(name = "min_quantity")] + fn py_min_quantity(&self) -> Option { + self.min_quantity + } + + #[getter] + #[pyo3(name = "max_notional")] + fn py_max_notional(&self) -> Option { + self.max_notional + } + + #[getter] + #[pyo3(name = "min_notional")] + fn py_min_notional(&self) -> Option { + self.min_notional + } + + #[getter] + #[pyo3(name = "max_price")] + fn py_max_price(&self) -> Option { + self.max_price + } + + #[getter] + #[pyo3(name = "min_price")] + fn py_min_price(&self) -> Option { + self.min_price + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + #[staticmethod] #[pyo3(name = "from_dict")] fn py_from_dict(py: Python<'_>, values: Py) -> PyResult { @@ -134,6 +241,14 @@ impl CurrencyPair { Some(value) => dict.set_item("min_quantity", value.to_string())?, None => dict.set_item("min_quantity", py.None())?, } + match self.max_notional { + Some(value) => dict.set_item("max_notional", value.to_string())?, + None => dict.set_item("max_notional", py.None())?, + } + match self.min_notional { + Some(value) => dict.set_item("min_notional", value.to_string())?, + None => dict.set_item("min_notional", py.None())?, + } match self.max_price { Some(value) => dict.set_item("max_price", value.to_string())?, None => dict.set_item("max_price", py.None())?, diff --git a/nautilus_core/model/src/python/instruments/equity.rs b/nautilus_core/model/src/python/instruments/equity.rs index da4c1cb24bfe..604a740e4317 100644 --- a/nautilus_core/model/src/python/instruments/equity.rs +++ b/nautilus_core/model/src/python/instruments/equity.rs @@ -25,7 +25,6 @@ use nautilus_core::{ use pyo3::{basic::CompareOp, prelude::*, types::PyDict}; use ustr::Ustr; -use crate::instruments::Instrument; use crate::{ identifiers::{instrument_id::InstrumentId, symbol::Symbol}, instruments::equity::Equity, @@ -69,11 +68,6 @@ impl Equity { .map_err(to_pyvalue_err) } - #[getter] - fn instrument_type(&self) -> &str { - "Equity" - } - fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { match op { CompareOp::Eq => self.eq(other).into_py(py), @@ -88,7 +82,26 @@ impl Equity { } #[getter] - fn isin(&self) -> Option<&str> { + #[pyo3(name = "instrument_type")] + fn py_instrument_type(&self) -> &str { + stringify!(Equity) + } + + #[getter] + #[pyo3(name = "id")] + fn py_id(&self) -> InstrumentId { + self.id + } + + #[getter] + #[pyo3(name = "raw_symbol")] + fn py_raw_symbol(&self) -> Symbol { + self.raw_symbol + } + + #[getter] + #[pyo3(name = "isin")] + fn py_isin(&self) -> Option<&str> { match self.isin { Some(isin) => Some(isin.as_str()), None => None, @@ -96,17 +109,65 @@ impl Equity { } #[getter] - #[pyo3(name = "quote_currency")] + #[pyo3(name = "quote_currency")] // TODO: Currency property standardization fn py_quote_currency(&self) -> Currency { - self.quote_currency() + self.currency } #[getter] - #[pyo3(name = "base_currency")] - fn py_base_currenct(&self) -> u8 { + #[pyo3(name = "price_precision")] + fn py_price_precision(&self) -> u8 { self.price_precision } + #[getter] + #[pyo3(name = "price_increment")] + fn py_price_increment(&self) -> Price { + self.price_increment + } + + #[getter] + #[pyo3(name = "lot_size")] + fn py_lot_size(&self) -> Option { + self.lot_size + } + + #[getter] + #[pyo3(name = "max_quantity")] + fn py_max_quantity(&self) -> Option { + self.max_quantity + } + + #[getter] + #[pyo3(name = "min_quantity")] + fn py_min_quantity(&self) -> Option { + self.min_quantity + } + + #[getter] + #[pyo3(name = "max_price")] + fn py_max_price(&self) -> Option { + self.max_price + } + + #[getter] + #[pyo3(name = "min_price")] + fn py_min_price(&self) -> Option { + self.min_price + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + #[staticmethod] #[pyo3(name = "from_dict")] fn py_from_dict(py: Python<'_>, values: Py) -> PyResult { diff --git a/nautilus_core/model/src/python/instruments/futures_contract.rs b/nautilus_core/model/src/python/instruments/futures_contract.rs index a07f26209331..49b8f347c30e 100644 --- a/nautilus_core/model/src/python/instruments/futures_contract.rs +++ b/nautilus_core/model/src/python/instruments/futures_contract.rs @@ -96,10 +96,113 @@ impl FuturesContract { } #[getter] - fn underlying(&self) -> &str { + #[pyo3(name = "instrument_type")] + fn py_instrument_type(&self) -> &str { + stringify!(FuturesContract) + } + + #[getter] + #[pyo3(name = "id")] + fn py_id(&self) -> InstrumentId { + self.id + } + + #[getter] + #[pyo3(name = "raw_symbol")] + fn py_raw_symbol(&self) -> Symbol { + self.raw_symbol + } + + #[getter] + #[pyo3(name = "asset_class")] + fn py_asset_class(&self) -> AssetClass { + self.asset_class + } + + #[getter] + #[pyo3(name = "underlying")] + fn py_underlying(&self) -> &str { self.underlying.as_str() } + #[getter] + #[pyo3(name = "activation_ns")] + fn py_activation_ns(&self) -> UnixNanos { + self.activation_ns + } + + #[getter] + #[pyo3(name = "expiration_ns")] + fn py_expiration_ns(&self) -> UnixNanos { + self.expiration_ns + } + + #[getter] + #[pyo3(name = "currency")] + fn py_currency(&self) -> Currency { + self.currency + } + + #[getter] + #[pyo3(name = "price_precision")] + fn py_price_precision(&self) -> u8 { + self.price_precision + } + + #[getter] + #[pyo3(name = "price_increment")] + fn py_price_increment(&self) -> Price { + self.price_increment + } + + #[getter] + #[pyo3(name = "multiplier")] + fn py_multiplier(&self) -> Quantity { + self.multiplier + } + + #[getter] + #[pyo3(name = "lot_size")] + fn py_lot_size(&self) -> Quantity { + self.lot_size + } + + #[getter] + #[pyo3(name = "max_quantity")] + fn py_max_quantity(&self) -> Option { + self.max_quantity + } + + #[getter] + #[pyo3(name = "min_quantity")] + fn py_min_quantity(&self) -> Option { + self.min_quantity + } + + #[getter] + #[pyo3(name = "max_price")] + fn py_max_price(&self) -> Option { + self.max_price + } + + #[getter] + #[pyo3(name = "min_price")] + fn py_min_price(&self) -> Option { + self.min_price + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + #[staticmethod] #[pyo3(name = "from_dict")] fn py_from_dict(py: Python<'_>, values: Py) -> PyResult { diff --git a/nautilus_core/model/src/python/instruments/options_contract.rs b/nautilus_core/model/src/python/instruments/options_contract.rs index 3db0b892bf56..12c8f0bdb7e7 100644 --- a/nautilus_core/model/src/python/instruments/options_contract.rs +++ b/nautilus_core/model/src/python/instruments/options_contract.rs @@ -81,11 +81,6 @@ impl OptionsContract { .map_err(to_pyvalue_err) } - #[getter] - fn instrument_type(&self) -> &str { - "OptionsContract" - } - fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { match op { CompareOp::Eq => self.eq(other).into_py(py), @@ -100,10 +95,125 @@ impl OptionsContract { } #[getter] - fn underlying(&self) -> &str { + #[pyo3(name = "instrument_type")] + fn py_instrument_type(&self) -> &str { + stringify!(OptionsContract) + } + + #[getter] + #[pyo3(name = "id")] + fn py_id(&self) -> InstrumentId { + self.id + } + + #[getter] + #[pyo3(name = "raw_symbol")] + fn py_raw_symbol(&self) -> Symbol { + self.raw_symbol + } + + #[getter] + #[pyo3(name = "asset_class")] + fn py_asset_class(&self) -> AssetClass { + self.asset_class + } + + #[getter] + #[pyo3(name = "underlying")] + fn py_underlying(&self) -> &str { self.underlying.as_str() } + #[getter] + #[pyo3(name = "option_kind")] + fn py_option_kind(&self) -> OptionKind { + self.option_kind + } + + #[getter] + #[pyo3(name = "activation_ns")] + fn py_activation_ns(&self) -> UnixNanos { + self.activation_ns + } + + #[getter] + #[pyo3(name = "expiration_ns")] + fn py_expiration_ns(&self) -> UnixNanos { + self.expiration_ns + } + + #[getter] + #[pyo3(name = "strike_price")] + fn py_strike_price(&self) -> Price { + self.strike_price + } + + #[getter] + #[pyo3(name = "currency")] + fn py_currency(&self) -> Currency { + self.currency + } + + #[getter] + #[pyo3(name = "price_precision")] + fn py_price_precision(&self) -> u8 { + self.price_precision + } + + #[getter] + #[pyo3(name = "price_increment")] + fn py_price_increment(&self) -> Price { + self.price_increment + } + + #[getter] + #[pyo3(name = "multiplier")] + fn py_multiplier(&self) -> Quantity { + self.multiplier + } + + #[getter] + #[pyo3(name = "lot_size")] + fn py_lot_size(&self) -> Quantity { + self.lot_size + } + + #[getter] + #[pyo3(name = "max_quantity")] + fn py_max_quantity(&self) -> Option { + self.max_quantity + } + + #[getter] + #[pyo3(name = "min_quantity")] + fn py_min_quantity(&self) -> Option { + self.min_quantity + } + + #[getter] + #[pyo3(name = "max_price")] + fn py_max_price(&self) -> Option { + self.max_price + } + + #[getter] + #[pyo3(name = "min_price")] + fn py_min_price(&self) -> Option { + self.min_price + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + #[staticmethod] #[pyo3(name = "from_dict")] fn py_from_dict(py: Python<'_>, values: Py) -> PyResult { diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 7d2b737513dd..69356812064a 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1002,9 +1002,12 @@ class CryptoFuture: ) -> None: ... @property def id(self) -> InstrumentId: ... - def to_dict(self) -> dict[str, Any]: ... @property - def symbol(self) -> Symbol: ... + def raw_symbol(self) -> Symbol: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... @property def price_precision(self) -> int: ... @property @@ -1013,10 +1016,7 @@ class CryptoFuture: def price_increment(self) -> Price: ... @property def size_increment(self) -> Quantity: ... - @property - def base_currency(self) -> Currency: ... - @property - def quote_currency(self) -> Currency: ... + def to_dict(self) -> dict[str, Any]: ... class CryptoPerpetual: def __init__( @@ -1047,9 +1047,12 @@ class CryptoPerpetual: ) -> None: ... @property def id(self) -> InstrumentId: ... - def to_dict(self) -> dict[str, Any]: ... @property - def symbol(self) -> Symbol: ... + def raw_symbol(self) -> Symbol: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... @property def price_precision(self) -> int: ... @property @@ -1058,10 +1061,7 @@ class CryptoPerpetual: def price_increment(self) -> Price: ... @property def size_increment(self) -> Quantity: ... - @property - def base_currency(self) -> Currency: ... - @property - def quote_currency(self) -> Currency: ... + def to_dict(self) -> dict[str, Any]: ... class CurrencyPair: def __init__( @@ -1088,9 +1088,12 @@ class CurrencyPair: ) -> None: ... @property def id(self) -> InstrumentId: ... - def to_dict(self) -> dict[str, Any]: ... @property - def symbol(self) -> Symbol: ... + def raw_symbol(self) -> Symbol: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... @property def price_precision(self) -> int: ... @property @@ -1099,10 +1102,7 @@ class CurrencyPair: def price_increment(self) -> Price: ... @property def size_increment(self) -> Quantity: ... - @property - def base_currency(self) -> Currency: ... - @property - def quote_currency(self) -> Currency: ... + def to_dict(self) -> dict[str, Any]: ... class Equity: def __init__( @@ -1123,9 +1123,12 @@ class Equity: ) -> None: ... @property def id(self) -> InstrumentId: ... - def to_dict(self) -> dict[str, Any]: ... @property - def symbol(self) -> Symbol: ... + def raw_symbol(self) -> Symbol: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... @property def price_precision(self) -> int: ... @property @@ -1134,10 +1137,7 @@ class Equity: def price_increment(self) -> Price: ... @property def size_increment(self) -> Quantity: ... - @property - def base_currency(self) -> Currency: ... - @property - def quote_currency(self) -> Currency: ... + def to_dict(self) -> dict[str, Any]: ... class FuturesContract: def __init__( @@ -1162,9 +1162,12 @@ class FuturesContract: ) -> None: ... @property def id(self) -> InstrumentId: ... - def to_dict(self) -> dict[str, Any]: ... @property - def symbol(self) -> Symbol: ... + def raw_symbol(self) -> Symbol: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... @property def price_precision(self) -> int: ... @property @@ -1173,10 +1176,7 @@ class FuturesContract: def price_increment(self) -> Price: ... @property def size_increment(self) -> Quantity: ... - @property - def base_currency(self) -> Currency: ... - @property - def quote_currency(self) -> Currency: ... + def to_dict(self) -> dict[str, Any]: ... class OptionsContract: def __init__( @@ -1203,9 +1203,12 @@ class OptionsContract: ) -> None : ... @property def id(self) -> InstrumentId: ... - def to_dict(self) -> dict[str, Any]: ... @property - def symbol(self) -> Symbol: ... + def raw_symbol(self) -> Symbol: ... + @property + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... @property def price_precision(self) -> int: ... @property @@ -1214,16 +1217,15 @@ class OptionsContract: def price_increment(self) -> Price: ... @property def size_increment(self) -> Quantity: ... - @property - def base_currency(self) -> Currency: ... - @property - def quote_currency(self) -> Currency: ... + def to_dict(self) -> dict[str, Any]: ... class SyntheticInstrument: + @property def id(self) -> InstrumentId: ... - def to_dict(self) -> dict[str, Any]: ... @property - def symbol(self) -> Symbol: ... + def base_currency(self) -> Currency: ... + @property + def quote_currency(self) -> Currency: ... @property def price_precision(self) -> int: ... @property @@ -1232,10 +1234,7 @@ class SyntheticInstrument: def price_increment(self) -> Price: ... @property def size_increment(self) -> Quantity: ... - @property - def base_currency(self) -> Currency: ... - @property - def quote_currency(self) -> Currency: ... + def to_dict(self) -> dict[str, Any]: ... Instrument: TypeAlias = Union[ CryptoFuture, diff --git a/nautilus_trader/model/instruments/base.pxd b/nautilus_trader/model/instruments/base.pxd index c55c8ddb01fb..d055a43ee72d 100644 --- a/nautilus_trader/model/instruments/base.pxd +++ b/nautilus_trader/model/instruments/base.pxd @@ -33,7 +33,7 @@ cdef class Instrument(Data): cdef readonly InstrumentId id """The instrument ID.\n\n:returns: `InstrumentId`""" cdef readonly Symbol raw_symbol - """The native/local/raw symbol for the instrument, assigned by the venue.\n\n:returns: `Symbol`""" + """The raw/local/native symbol for the instrument, assigned by the venue.\n\n:returns: `Symbol`""" cdef readonly AssetClass asset_class """The asset class of the instrument.\n\n:returns: `AssetClass`""" cdef readonly InstrumentClass instrument_class diff --git a/nautilus_trader/model/instruments/base.pyx b/nautilus_trader/model/instruments/base.pyx index 8c0616f0ce69..653fe47d0785 100644 --- a/nautilus_trader/model/instruments/base.pyx +++ b/nautilus_trader/model/instruments/base.pyx @@ -49,7 +49,7 @@ cdef class Instrument(Data): instrument_id : InstrumentId The instrument ID for the instrument. raw_symbol : Symbol - The native/local/raw symbol for the instrument, assigned by the venue. + The raw/local/native symbol for the instrument, assigned by the venue. asset_class : AssetClass The instrument asset class. instrument_class : InstrumentClass diff --git a/nautilus_trader/model/instruments/crypto_future.pyx b/nautilus_trader/model/instruments/crypto_future.pyx index 2bace2fe7870..4f7122925358 100644 --- a/nautilus_trader/model/instruments/crypto_future.pyx +++ b/nautilus_trader/model/instruments/crypto_future.pyx @@ -42,7 +42,7 @@ cdef class CryptoFuture(Instrument): instrument_id : InstrumentId The instrument ID for the instrument. raw_symbol : Symbol - The native/local/raw symbol for the instrument, assigned by the venue. + The raw/local/native symbol for the instrument, assigned by the venue. underlying : Currency The underlying asset. quote_currency : Currency diff --git a/nautilus_trader/model/instruments/crypto_perpetual.pyx b/nautilus_trader/model/instruments/crypto_perpetual.pyx index 7043c0399335..8bb41eb7adb7 100644 --- a/nautilus_trader/model/instruments/crypto_perpetual.pyx +++ b/nautilus_trader/model/instruments/crypto_perpetual.pyx @@ -39,7 +39,7 @@ cdef class CryptoPerpetual(Instrument): instrument_id : InstrumentId The instrument ID for the instrument. raw_symbol : Symbol - The native/local/raw symbol for the instrument, assigned by the venue. + The raw/local/native symbol for the instrument, assigned by the venue. base_currency : Currency, optional The base currency. quote_currency : Currency diff --git a/nautilus_trader/model/instruments/currency_pair.pyx b/nautilus_trader/model/instruments/currency_pair.pyx index 43997dc2298b..65d2887cc59f 100644 --- a/nautilus_trader/model/instruments/currency_pair.pyx +++ b/nautilus_trader/model/instruments/currency_pair.pyx @@ -41,7 +41,7 @@ cdef class CurrencyPair(Instrument): instrument_id : InstrumentId The instrument ID for the instrument. raw_symbol : Symbol - The native/local/raw symbol for the instrument, assigned by the venue. + The raw/local/native symbol for the instrument, assigned by the venue. base_currency : Currency The base currency. quote_currency : Currency diff --git a/nautilus_trader/model/instruments/equity.pyx b/nautilus_trader/model/instruments/equity.pyx index 2e64f9f8e480..ae8ddb8842d7 100644 --- a/nautilus_trader/model/instruments/equity.pyx +++ b/nautilus_trader/model/instruments/equity.pyx @@ -37,7 +37,7 @@ cdef class Equity(Instrument): instrument_id : InstrumentId The instrument ID. raw_symbol : Symbol - The native/local/raw symbol for the instrument, assigned by the venue. + The raw/local/native symbol for the instrument, assigned by the venue. currency : Currency The futures contract currency. price_precision : int @@ -163,7 +163,7 @@ cdef class Equity(Instrument): return Equity( instrument_id=InstrumentId.from_str_c(pyo3_instrument.id.value), raw_symbol=Symbol(pyo3_instrument.id.symbol.value), - currency=Currency.from_str_c(pyo3_instrument.currency.code), + currency=Currency.from_str_c(pyo3_instrument.quote_currency.code), price_precision=pyo3_instrument.price_precision, price_increment=Price.from_raw_c(pyo3_instrument.price_increment.raw, pyo3_instrument.price_precision), lot_size=Quantity.from_raw_c(pyo3_instrument.lot_size.raw, pyo3_instrument.lot_size.precision), diff --git a/nautilus_trader/model/instruments/futures_contract.pyx b/nautilus_trader/model/instruments/futures_contract.pyx index 7726ac29f36a..f4fc11978332 100644 --- a/nautilus_trader/model/instruments/futures_contract.pyx +++ b/nautilus_trader/model/instruments/futures_contract.pyx @@ -45,7 +45,7 @@ cdef class FuturesContract(Instrument): instrument_id : InstrumentId The instrument ID. raw_symbol : Symbol - The native/local/raw symbol for the instrument, assigned by the venue. + The raw/local/native symbol for the instrument, assigned by the venue. asset_class : AssetClass The futures contract asset class. currency : Currency diff --git a/nautilus_trader/model/instruments/options_contract.pyx b/nautilus_trader/model/instruments/options_contract.pyx index fbb1d85b11bd..abe94efb0b0e 100644 --- a/nautilus_trader/model/instruments/options_contract.pyx +++ b/nautilus_trader/model/instruments/options_contract.pyx @@ -45,7 +45,7 @@ cdef class OptionsContract(Instrument): instrument_id : InstrumentId The instrument ID. raw_symbol : Symbol - The native/local/raw symbol for the instrument, assigned by the venue. + The raw/local/native symbol for the instrument, assigned by the venue. asset_class : AssetClass The options contract asset class. currency : Currency diff --git a/tests/unit_tests/model/instruments/test_currency_pair_pyo3.py b/tests/unit_tests/model/instruments/test_currency_pair_pyo3.py index 11d735fbdba0..2b9a25ea7039 100644 --- a/tests/unit_tests/model/instruments/test_currency_pair_pyo3.py +++ b/tests/unit_tests/model/instruments/test_currency_pair_pyo3.py @@ -46,6 +46,8 @@ def test_to_dict(): "lot_size": None, "max_quantity": "9000", "min_quantity": "0.00001", + "max_notional": None, + "min_notional": None, "min_price": "0.01", "max_price": "1000000", "maker_fee": 0.001, From 54e5a211a5352c557df7bb4ee405c79a4f16330d Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 07:45:14 +1100 Subject: [PATCH 008/104] Update to ignore notebooks dir catalog --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index eb1f7e27fd20..4b0e7d65fcf2 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ .vscode/ /catalog/ +/examples/notebooks/catalog/ __pycache__ _build/ From dc723ebcca29207df94fd5c64181aa601138542d Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 07:46:13 +1100 Subject: [PATCH 009/104] Update dependencies --- nautilus_core/Cargo.lock | 8 ++++---- poetry.lock | 20 ++++++++++---------- pyproject.toml | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 08c0e7031751..05e2bdc6c804 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -1442,9 +1442,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" dependencies = [ "serde", ] @@ -2118,9 +2118,9 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe8f25ce1159c7740ff0b9b2f5cdf4a8428742ba7c112b9f20f22cd5219c7dab" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ "hermit-abi 0.3.5", "libc", diff --git a/poetry.lock b/poetry.lock index e0b64054ce80..a69e14c93d9d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -776,13 +776,13 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.33" +version = "2.5.34" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.33-py2.py3-none-any.whl", hash = "sha256:d40ce5fcd762817627670da8a7d8d8e65f24342d14539c59488dc603bf662e34"}, - {file = "identify-2.5.33.tar.gz", hash = "sha256:161558f9fe4559e1557e1bff323e8631f6a0e4837f7497767c1782832f16b62d"}, + {file = "identify-2.5.34-py2.py3-none-any.whl", hash = "sha256:a4316013779e433d08b96e5eabb7f641e6c7942e4ab5d4c509ebd2e7a8994aed"}, + {file = "identify-2.5.34.tar.gz", hash = "sha256:ee17bc9d499899bc9eaec1ac7bf2dc9eedd480db9d88b96d123d3b64a9d34f5d"}, ] [package.extras] @@ -1538,13 +1538,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "3.6.0" +version = "3.6.1" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-3.6.0-py2.py3-none-any.whl", hash = "sha256:c255039ef399049a5544b6ce13d135caba8f2c28c3b4033277a788f434308376"}, - {file = "pre_commit-3.6.0.tar.gz", hash = "sha256:d30bad9abf165f7785c15a21a1f46da7d0677cb00ee7ff4c579fd38922efe15d"}, + {file = "pre_commit-3.6.1-py2.py3-none-any.whl", hash = "sha256:9fe989afcf095d2c4796ce7c553cf28d4d4a9b9346de3cda079bcf40748454a4"}, + {file = "pre_commit-3.6.1.tar.gz", hash = "sha256:c90961d8aa706f75d60935aba09469a6b0bcb8345f127c3fbee4bdc5f114cf4b"}, ] [package.dependencies] @@ -2278,13 +2278,13 @@ files = [ [[package]] name = "tqdm" -version = "4.66.1" +version = "4.66.2" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, - {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, + {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"}, + {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"}, ] [package.dependencies] @@ -2595,4 +2595,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "ff1aa96126cd05c75d19003149f370bb74eec6b6db40688d908f8b8b7235a234" +content-hash = "294c31ca1c41d6d8816411e1af9c0336d9b6b01dceef6aaa066061362522ca51" diff --git a/pyproject.toml b/pyproject.toml index 736fc62981df..67c960d4ef07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ msgspec = "^0.18.6" pandas = "^2.2.0" pyarrow = ">=15.0.0" pytz = ">=2023.4.0" -tqdm = "^4.66.1" +tqdm = "^4.66.2" uvloop = {version = "^0.19.0", markers = "sys_platform != 'win32'"} async-timeout = {version = "^4.0.3", optional = true} @@ -82,7 +82,7 @@ black = "^24.1.1" docformatter = "^1.7.5" mypy = "^1.8.0" pandas-stubs = "^2.1.4" -pre-commit = "^3.6.0" +pre-commit = "^3.6.1" ruff = "^0.2.1" types-pytz = "^2023.3" types-requests = "^2.31" From f81682668ae9dad56077880e325cec31b12672af Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 08:11:20 +1100 Subject: [PATCH 010/104] Fix missing enc_hook in configurations --- RELEASES.md | 5 ++++- nautilus_trader/common/config.py | 5 +++-- nautilus_trader/execution/config.py | 4 +++- nautilus_trader/trading/config.py | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index fd69da689bcf..cad598009f45 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -9,7 +9,10 @@ None None ### Fixes -None +- Fixed `ActorFactory.create` JSON encoding (was missing the encoding hook) +- Fixed `ImportableConfig.create` JSON encoding (was missing the encoding hook) +- Fixed `ImportableStrategyConfig.create` JSON encoding (was missing the encoding hook) +- Fixed `ExecAlgorithmFactory.create` JSON encoding (was missing the encoding hook) --- diff --git a/nautilus_trader/common/config.py b/nautilus_trader/common/config.py index 8a6ffb89107e..800f151ff539 100644 --- a/nautilus_trader/common/config.py +++ b/nautilus_trader/common/config.py @@ -428,7 +428,8 @@ def create(config: ImportableActorConfig): PyCondition.type(config, ImportableActorConfig, "config") actor_cls = resolve_path(config.actor_path) config_cls = resolve_config_path(config.config_path) - config = config_cls.parse(msgspec.json.encode(config.config)) + json = msgspec.json.encode(config.config, enc_hook=msgspec_encoding_hook) + config = config_cls.parse(json) return actor_cls(config) @@ -505,5 +506,5 @@ def is_importable(data: dict) -> bool: def create(self): assert ":" in self.path, "`path` variable should be of the form `path.to.module:class`" cls = resolve_path(self.path) - cfg = msgspec.json.encode(self.config) + cfg = msgspec.json.encode(self.config, enc_hook=msgspec_encoding_hook) return msgspec.json.decode(cfg, type=cls) diff --git a/nautilus_trader/execution/config.py b/nautilus_trader/execution/config.py index f5328da857f8..6ae72ce381fe 100644 --- a/nautilus_trader/execution/config.py +++ b/nautilus_trader/execution/config.py @@ -20,6 +20,7 @@ import msgspec from nautilus_trader.common.config import NautilusConfig +from nautilus_trader.common.config import msgspec_encoding_hook from nautilus_trader.common.config import resolve_config_path from nautilus_trader.common.config import resolve_path from nautilus_trader.core.correctness import PyCondition @@ -109,5 +110,6 @@ def create(config: ImportableExecAlgorithmConfig): PyCondition.type(config, ImportableExecAlgorithmConfig, "config") exec_algorithm_cls = resolve_path(config.exec_algorithm_path) config_cls = resolve_config_path(config.config_path) - config = config_cls.parse(msgspec.json.encode(config.config)) + json = msgspec.json.encode(config.config, enc_hook=msgspec_encoding_hook) + config = config_cls.parse(json) return exec_algorithm_cls(config=config) diff --git a/nautilus_trader/trading/config.py b/nautilus_trader/trading/config.py index 504eff222fde..c0af482a3ce4 100644 --- a/nautilus_trader/trading/config.py +++ b/nautilus_trader/trading/config.py @@ -20,6 +20,7 @@ import msgspec from nautilus_trader.common.config import NautilusConfig +from nautilus_trader.common.config import msgspec_encoding_hook from nautilus_trader.common.config import resolve_config_path from nautilus_trader.common.config import resolve_path from nautilus_trader.core.correctness import PyCondition @@ -109,7 +110,8 @@ def create(config: ImportableStrategyConfig): PyCondition.type(config, ImportableStrategyConfig, "config") strategy_cls = resolve_path(config.strategy_path) config_cls = resolve_config_path(config.config_path) - config = config_cls.parse(msgspec.json.encode(config.config)) + json = msgspec.json.encode(config.config, enc_hook=msgspec_encoding_hook) + config = config_cls.parse(json) return strategy_cls(config=config) From a9772e37549791bae64012bedd1057a3f9ffed2e Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 08:30:30 +1100 Subject: [PATCH 011/104] Fix Binance order book example --- examples/notebooks/backtest_binance_orderbook.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/notebooks/backtest_binance_orderbook.ipynb b/examples/notebooks/backtest_binance_orderbook.ipynb index 61049317e49f..40afbc83558a 100644 --- a/examples/notebooks/backtest_binance_orderbook.ipynb +++ b/examples/notebooks/backtest_binance_orderbook.ipynb @@ -48,7 +48,7 @@ "from nautilus_trader.examples.strategies.ema_cross import EMACross, EMACrossConfig\n", "from nautilus_trader.model.data import OrderBookDelta\n", "from nautilus_trader.persistence.loaders import BinanceOrderBookDeltaDataLoader\n", - "from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWranglerV2\n", + "from nautilus_trader.persistence.wranglers import OrderBookDeltaDataWrangler\n", "from nautilus_trader.persistence.catalog import ParquetDataCatalog\n", "from nautilus_trader.test_kit.providers import TestInstrumentProvider" ] @@ -113,7 +113,7 @@ "source": [ "# Process deltas using a wrangler\n", "BTCUSDT_BINANCE = TestInstrumentProvider.btcusdt_binance()\n", - "wrangler = OrderBookDeltaDataWranglerV2(BTCUSDT_BINANCE)\n", + "wrangler = OrderBookDeltaDataWrangler(BTCUSDT_BINANCE)\n", "\n", "deltas = wrangler.process(df_snap)\n", "deltas += wrangler.process(df_update)\n", From 78fd9e804d1d2b3ad61f1d3bef55cd894cc03cf1 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 09:47:20 +1100 Subject: [PATCH 012/104] Add Databento re-exports --- nautilus_trader/adapters/databento/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nautilus_trader/adapters/databento/__init__.py b/nautilus_trader/adapters/databento/__init__.py index 3d34cab4588e..dc9583b0366e 100644 --- a/nautilus_trader/adapters/databento/__init__.py +++ b/nautilus_trader/adapters/databento/__init__.py @@ -12,3 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- + +from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS +from nautilus_trader.adapters.databento.constants import DATABENTO +from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID +from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader + + +__all__ = [ + "DATABENTO", + "DATABENTO_CLIENT_ID", + "ALL_SYMBOLS", + "DatabentoDataLoader", +] From 9ba9e8f0ba47594365ec9f561fb0c96187c6ba26 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 09:52:42 +1100 Subject: [PATCH 013/104] Cleanup example notebook imports --- examples/notebooks/backtest_binance_orderbook.ipynb | 7 +++++-- examples/notebooks/backtest_example.ipynb | 13 ++++++++++++- examples/notebooks/external_data_backtest.ipynb | 12 ++++++++---- examples/notebooks/quick_start.ipynb | 12 ++++++++---- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/examples/notebooks/backtest_binance_orderbook.ipynb b/examples/notebooks/backtest_binance_orderbook.ipynb index 40afbc83558a..b1370ff6133f 100644 --- a/examples/notebooks/backtest_binance_orderbook.ipynb +++ b/examples/notebooks/backtest_binance_orderbook.ipynb @@ -41,8 +41,11 @@ "import pandas as pd\n", "\n", "from nautilus_trader.backtest.node import BacktestNode\n", + "from nautilus_trader.backtest.node import BacktestVenueConfig\n", + "from nautilus_trader.backtest.node import BacktestDataConfig\n", + "from nautilus_trader.backtest.node import BacktestRunConfig\n", + "from nautilus_trader.backtest.node import BacktestEngineConfig\n", "from nautilus_trader.core.datetime import dt_to_unix_nanos\n", - "from nautilus_trader.config import BacktestRunConfig, BacktestVenueConfig, BacktestDataConfig, BacktestEngineConfig\n", "from nautilus_trader.config import ImportableStrategyConfig\n", "from nautilus_trader.config import LoggingConfig\n", "from nautilus_trader.examples.strategies.ema_cross import EMACross, EMACrossConfig\n", @@ -146,7 +149,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Write instrument and ticks to catalog (this currently takes a minute - investigating)\n", + "# Write instrument and ticks to catalog\n", "catalog.write_data([BTCUSDT_BINANCE])\n", "catalog.write_data(deltas)" ] diff --git a/examples/notebooks/backtest_example.ipynb b/examples/notebooks/backtest_example.ipynb index fbb499879a96..98bfdaeb5cd5 100644 --- a/examples/notebooks/backtest_example.ipynb +++ b/examples/notebooks/backtest_example.ipynb @@ -32,8 +32,11 @@ "import pandas as pd\n", "\n", "from nautilus_trader.backtest.node import BacktestNode\n", + "from nautilus_trader.backtest.node import BacktestVenueConfig\n", + "from nautilus_trader.backtest.node import BacktestDataConfig\n", + "from nautilus_trader.backtest.node import BacktestRunConfig\n", + "from nautilus_trader.backtest.node import BacktestEngineConfig\n", "from nautilus_trader.core.datetime import dt_to_unix_nanos\n", - "from nautilus_trader.config import BacktestRunConfig, BacktestVenueConfig, BacktestDataConfig, BacktestEngineConfig\n", "from nautilus_trader.config import ImportableStrategyConfig\n", "from nautilus_trader.config import LoggingConfig\n", "from nautilus_trader.examples.strategies.ema_cross import EMACross, EMACrossConfig\n", @@ -170,6 +173,14 @@ "source": [ "result" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples/notebooks/external_data_backtest.ipynb b/examples/notebooks/external_data_backtest.ipynb index 709e0d127bb0..6fcebe19a2e8 100644 --- a/examples/notebooks/external_data_backtest.ipynb +++ b/examples/notebooks/external_data_backtest.ipynb @@ -32,11 +32,15 @@ "import fsspec\n", "import pandas as pd\n", "from nautilus_trader.core.datetime import dt_to_unix_nanos\n", - "from nautilus_trader.model.data import QuoteTick\n", - "from nautilus_trader.model.objects import Price, Quantity\n", - "from nautilus_trader.backtest.node import BacktestNode, BacktestVenueConfig, BacktestDataConfig, BacktestRunConfig, BacktestEngineConfig\n", + "from nautilus_trader.backtest.node import BacktestNode\n", + "from nautilus_trader.backtest.node import BacktestVenueConfig\n", + "from nautilus_trader.backtest.node import BacktestDataConfig\n", + "from nautilus_trader.backtest.node import BacktestRunConfig\n", + "from nautilus_trader.backtest.node import BacktestEngineConfig\n", "from nautilus_trader.config import ImportableStrategyConfig\n", + "from nautilus_trader.model.data import QuoteTick\n", "from nautilus_trader.model.data import BarType\n", + "from nautilus_trader.model.objects import Price, Quantity\n", "from nautilus_trader.persistence.catalog import ParquetDataCatalog\n", "from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler\n", "from nautilus_trader.test_kit.providers import CSVTickDataLoader\n", @@ -109,7 +113,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Write instrument and ticks to catalog (this currently takes a minute - investigating)\n", + "# Write instrument and ticks to catalog\n", "catalog.write_data([EURUSD])\n", "catalog.write_data(ticks)" ] diff --git a/examples/notebooks/quick_start.ipynb b/examples/notebooks/quick_start.ipynb index 65e0671111ce..f539b3f693fc 100644 --- a/examples/notebooks/quick_start.ipynb +++ b/examples/notebooks/quick_start.ipynb @@ -30,13 +30,17 @@ "\n", "import fsspec\n", "import pandas as pd\n", - "from nautilus_trader.core.datetime import dt_to_unix_nanos\n", - "from nautilus_trader.model.data import QuoteTick\n", - "from nautilus_trader.model.objects import Price, Quantity\n", "\n", - "from nautilus_trader.backtest.node import BacktestNode, BacktestVenueConfig, BacktestDataConfig, BacktestRunConfig, BacktestEngineConfig\n", + "from nautilus_trader.backtest.node import BacktestNode\n", + "from nautilus_trader.backtest.node import BacktestVenueConfig\n", + "from nautilus_trader.backtest.node import BacktestDataConfig\n", + "from nautilus_trader.backtest.node import BacktestRunConfig\n", + "from nautilus_trader.backtest.node import BacktestEngineConfig\n", "from nautilus_trader.config import ImportableStrategyConfig\n", "from nautilus_trader.config import LoggingConfig\n", + "from nautilus_trader.core.datetime import dt_to_unix_nanos\n", + "from nautilus_trader.model.data import QuoteTick\n", + "from nautilus_trader.model.objects import Price, Quantity\n", "from nautilus_trader.persistence.catalog import ParquetDataCatalog\n", "from nautilus_trader.test_kit.providers import TestInstrumentProvider" ] From 9258aedc78fa594a6c3bc79fc28758e56226c286 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 09:53:31 +1100 Subject: [PATCH 014/104] Add Databento notebook example progress --- .../notebooks/databento_data_catalog.ipynb | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 examples/notebooks/databento_data_catalog.ipynb diff --git a/examples/notebooks/databento_data_catalog.ipynb b/examples/notebooks/databento_data_catalog.ipynb new file mode 100644 index 000000000000..5f7b23009246 --- /dev/null +++ b/examples/notebooks/databento_data_catalog.ipynb @@ -0,0 +1,215 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Databento data catalog" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "This tutorial will walk through how to setup a Nautilus Parquet data catalog with databento order book data.\n", + "\n", + "We choose to work with the MBP-10 schema (which is just an aggregation of the top 10 levels) so that the data is more manageable and easier to work with for the example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import databento as db\n", + "\n", + "client = db.Historical() # This will use the DATABENTO_API_KEY environment variable (recommended best practice)" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Request data\n", + "\n", + "Use the historical API to request the front-month ES futures contract for January 2024.\n", + "\n", + "**CAUTION: This will incur a cost for every request (only run the request cell once)**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Path we'll use for persisting this request to disk\n", + "path = \"es-front-glbx-mbp10.dbn.zst\"\n", + "\n", + "# Request lead month\n", + "data = client.timeseries.get_range(\n", + " dataset=\"GLBX.MDP3\",\n", + " symbols=[\"ES.n.0\"],\n", + " stype_in=\"continuous\",\n", + " schema=\"mbp-10\",\n", + " start=\"2023-12-06T14:30:00\",\n", + " end=\"2023-12-06T20:30:00\",\n", + " path=path,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "df = data.to_df()\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "## Write to data catalog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "import shutil\n", + "from pathlib import Path\n", + "\n", + "from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader\n", + "from nautilus_trader.model.identifiers import InstrumentId\n", + "from nautilus_trader.persistence.catalog import ParquetDataCatalog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "instrument_id = InstrumentId.from_str(\"ES.n.0\") # This should be the raw symbol (update)\n", + "loader = DatabentoDataLoader()\n", + "depth10 = loader.from_dbn_file(\n", + " path=path,\n", + " instrument_id=instrument_id, # Not required but makes data loading faster (symbology mapping not required)\n", + " as_legacy_cython=False, # This will load Rust pyo3 objects to write to the catalog\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "CATALOG_PATH = Path.cwd() / \"catalog\"\n", + "\n", + "# Clear if it already exists, then create fresh\n", + "if CATALOG_PATH.exists():\n", + " shutil.rmtree(CATALOG_PATH)\n", + "CATALOG_PATH.mkdir()\n", + "\n", + "# Create a catalog instance\n", + "catalog = ParquetDataCatalog(CATALOG_PATH)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "# Write instrument and ticks to catalog (this takes ~20 seconds)\n", + "catalog.write_data(depth10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "import pyarrow.parquet as pq" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "depth10_parquet_path = \"catalog/data/order_book_depth10/ES.n.0/part-0.parquet\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "table = pq.read_table(depth10_parquet_path)\n", + "table.schema" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From cc475f5b3167716ec6aa8e5a1d20ccfae892be1c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 09:58:17 +1100 Subject: [PATCH 015/104] Add Databento DBN files to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4b0e7d65fcf2..8f8e933f25db 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ *.tar.gz* *.zip +*.dbn +*.dbn.zst + .benchmarks* .coverage* .history* From adc2ec9b131bb5874241ee8f32570fe00be8c3b0 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 13:23:30 +1100 Subject: [PATCH 016/104] Refine Databento re-exports --- examples/live/databento/databento_subscriber.py | 8 ++++---- nautilus_trader/adapters/databento/__init__.py | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/live/databento/databento_subscriber.py b/examples/live/databento/databento_subscriber.py index 66ecd99af678..d5bd95dbffd8 100644 --- a/examples/live/databento/databento_subscriber.py +++ b/examples/live/databento/databento_subscriber.py @@ -14,10 +14,10 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -from nautilus_trader.adapters.databento.config import DatabentoDataClientConfig -from nautilus_trader.adapters.databento.constants import DATABENTO -from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID -from nautilus_trader.adapters.databento.factories import DatabentoLiveDataClientFactory +from nautilus_trader.adapters.databento import DATABENTO +from nautilus_trader.adapters.databento import DATABENTO_CLIENT_ID +from nautilus_trader.adapters.databento import DatabentoDataClientConfig +from nautilus_trader.adapters.databento import DatabentoLiveDataClientFactory from nautilus_trader.common.enums import LogColor from nautilus_trader.config import InstrumentProviderConfig from nautilus_trader.config import LiveExecEngineConfig diff --git a/nautilus_trader/adapters/databento/__init__.py b/nautilus_trader/adapters/databento/__init__.py index dc9583b0366e..8cb23f0c98da 100644 --- a/nautilus_trader/adapters/databento/__init__.py +++ b/nautilus_trader/adapters/databento/__init__.py @@ -13,9 +13,11 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- +from nautilus_trader.adapters.databento.config import DatabentoDataClientConfig from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS from nautilus_trader.adapters.databento.constants import DATABENTO from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID +from nautilus_trader.adapters.databento.factories import DatabentoLiveDataClientFactory from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader @@ -24,4 +26,6 @@ "DATABENTO_CLIENT_ID", "ALL_SYMBOLS", "DatabentoDataLoader", + "DatabentoDataClientConfig", + "DatabentoLiveDataClientFactory", ] From e9f80c40cb751cfe66bb702ce6a01434ec0fd39a Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 13:24:19 +1100 Subject: [PATCH 017/104] Refine Databento components --- nautilus_trader/adapters/databento/data.py | 14 +++++++++++--- nautilus_trader/adapters/databento/loaders.py | 4 ---- nautilus_trader/adapters/databento/providers.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 05aadf38a10f..611b3fe85bfb 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -58,6 +58,9 @@ class DatabentoDataClient(LiveMarketDataClient): """ Provides a data client for the `Databento` API. + Both Historical and Live APIs are leveraged to provide historical data + for requests, and live data feeds based on subscriptions. + Parameters ---------- loop : asyncio.AbstractEventLoop @@ -485,11 +488,15 @@ async def _subscribe_order_book_deltas_batch( dataset: Dataset = self._loader.get_dataset_for_venue(instrument_ids[0].venue) live_client = self._get_live_client_mbo(dataset) + + # Subscribe from UTC midnight snapshot + start = self._clock.utcnow().normalize().value + future = asyncio.ensure_future( live_client.subscribe( schema=DatabentoSchema.MBO.value, symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), - start=0, # Must subscribe from start of week to get 'Sunday snapshot' for now + start=start, ), ) self._live_client_futures.add(future) @@ -853,8 +860,9 @@ def _handle_record( self, pycapsule: object, ) -> None: - # self._log.debug(f"Received {record}", LogColor.MAGENTA) - + # The capsule will fall out of scope at the end of this method, + # and eventually be garbage collected. The contained pointer + # to `Data` is still owned and managed by the Rust memory model. data = capsule_to_data(pycapsule) if isinstance(data, OrderBookDelta): diff --git a/nautilus_trader/adapters/databento/loaders.py b/nautilus_trader/adapters/databento/loaders.py index 63cbbc38bae1..a4a34d550bf6 100644 --- a/nautilus_trader/adapters/databento/loaders.py +++ b/nautilus_trader/adapters/databento/loaders.py @@ -45,10 +45,6 @@ class DatabentoDataLoader: - IMBALANCE -> `DatabentoImbalance` - STATISTICS -> `DatabentoStatistics` - For the loader to work correctly, you must first either: - - Load Databento instrument definitions from a DBN file using `load_instruments(...)` - - Manually add Nautilus instrument objects through `add_instruments(...)` - Warnings -------- The following Databento instrument classes are not currently supported: diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index 8a00493e78c1..5807694fc466 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -135,7 +135,7 @@ def receive_instruments(pyo3_instrument: Any) -> None: await live_client.subscribe( schema=DatabentoSchema.DEFINITION.value, symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), - start=0, # From start of current session (latest definition) + start=0, # From start of current week (latest definitions) ) try: From 47170e9f075b3ffae18a86040cd3a36cbfe65d07 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 14:24:35 +1100 Subject: [PATCH 018/104] Refine Databento adapter and decoding --- .../src/databento/{parsing.rs => decode.rs} | 99 +++++----- .../adapters/src/databento/loader.rs | 10 +- nautilus_core/adapters/src/databento/mod.rs | 2 +- .../python/{parsing.rs => decode.rs} | 48 ++--- .../src/databento/python/historical.rs | 31 ++-- .../adapters/src/databento/python/live.rs | 175 ++++++++++-------- .../adapters/src/databento/python/mod.rs | 24 ++- .../adapters/src/databento/symbology.rs | 2 +- nautilus_core/pyo3/src/lib.rs | 30 +-- 9 files changed, 214 insertions(+), 207 deletions(-) rename nautilus_core/adapters/src/databento/{parsing.rs => decode.rs} (87%) rename nautilus_core/adapters/src/databento/python/{parsing.rs => decode.rs} (73%) diff --git a/nautilus_core/adapters/src/databento/parsing.rs b/nautilus_core/adapters/src/databento/decode.rs similarity index 87% rename from nautilus_core/adapters/src/databento/parsing.rs rename to nautilus_core/adapters/src/databento/decode.rs index 5824926a5131..f2d4ec752de0 100644 --- a/nautilus_core/adapters/src/databento/parsing.rs +++ b/nautilus_core/adapters/src/databento/decode.rs @@ -22,6 +22,7 @@ use std::{ use anyhow::{anyhow, bail, Result}; use databento::dbn; +use dbn::Record; use itoa; use nautilus_core::{datetime::NANOSECONDS_IN_SECOND, time::UnixNanos}; use nautilus_model::{ @@ -142,7 +143,7 @@ pub fn parse_cfi_iso10926(value: &str) -> Result<(Option, Option Result { +pub fn decode_min_price_increment(value: i64, currency: Currency) -> Result { match value { 0 | i64::MAX => Price::new( 10f64.powi(-i32::from(currency.precision)), @@ -155,7 +156,7 @@ pub fn parse_min_price_increment(value: i64, currency: Currency) -> Result Result { +pub unsafe fn raw_ptr_to_string(ptr: *const c_char) -> Result { let c_str: &CStr = unsafe { CStr::from_ptr(ptr) }; let str_slice: &str = c_str.to_str().map_err(|e| anyhow!(e))?; Ok(str_slice.to_owned()) @@ -164,13 +165,13 @@ pub unsafe fn parse_raw_ptr_to_string(ptr: *const c_char) -> Result { /// # Safety /// /// - Assumes `ptr` is a valid C string pointer. -pub unsafe fn parse_raw_ptr_to_ustr(ptr: *const c_char) -> Result { +pub unsafe fn raw_ptr_to_ustr(ptr: *const c_char) -> Result { let c_str: &CStr = unsafe { CStr::from_ptr(ptr) }; let str_slice: &str = c_str.to_str().map_err(|e| anyhow!(e))?; Ok(Ustr::from(str_slice)) } -pub fn parse_equity_v1( +pub fn decode_equity_v1( record: &dbn::compat::InstrumentDefMsgV1, instrument_id: InstrumentId, ts_init: UnixNanos, @@ -183,7 +184,7 @@ pub fn parse_equity_v1( None, // No ISIN available yet currency, currency.precision, - parse_min_price_increment(record.min_price_increment, currency)?, + decode_min_price_increment(record.min_price_increment, currency)?, Some(Quantity::new(record.min_lot_size_round_lot.into(), 0)?), None, // TBD None, // TBD @@ -194,14 +195,14 @@ pub fn parse_equity_v1( ) } -pub fn parse_futures_contract_v1( +pub fn decode_futures_contract_v1( record: &dbn::compat::InstrumentDefMsgV1, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result { let currency = Currency::USD(); // TODO: Temporary hard coding of US futures for now - let cfi_str = unsafe { parse_raw_ptr_to_string(record.cfi.as_ptr())? }; - let underlying = unsafe { parse_raw_ptr_to_ustr(record.asset.as_ptr())? }; + let cfi_str = unsafe { raw_ptr_to_string(record.cfi.as_ptr())? }; + let underlying = unsafe { raw_ptr_to_ustr(record.asset.as_ptr())? }; let (asset_class, _) = parse_cfi_iso10926(&cfi_str)?; FuturesContract::new( @@ -213,7 +214,7 @@ pub fn parse_futures_contract_v1( record.expiration, currency, currency.precision, - parse_min_price_increment(record.min_price_increment, currency)?, + decode_min_price_increment(record.min_price_increment, currency)?, Quantity::new(1.0, 0)?, // TBD Quantity::new(1.0, 0)?, // TBD None, // TBD @@ -225,13 +226,13 @@ pub fn parse_futures_contract_v1( ) } -pub fn parse_options_contract_v1( +pub fn decode_options_contract_v1( record: &dbn::compat::InstrumentDefMsgV1, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result { - let currency_str = unsafe { parse_raw_ptr_to_string(record.currency.as_ptr())? }; - let cfi_str = unsafe { parse_raw_ptr_to_string(record.cfi.as_ptr())? }; + let currency_str = unsafe { raw_ptr_to_string(record.currency.as_ptr())? }; + let cfi_str = unsafe { raw_ptr_to_string(record.cfi.as_ptr())? }; let asset_class_opt = match instrument_id.venue.value.as_str() { "OPRA" => Some(AssetClass::Equity), _ => { @@ -239,7 +240,7 @@ pub fn parse_options_contract_v1( asset_class } }; - let underlying = unsafe { parse_raw_ptr_to_ustr(record.underlying.as_ptr())? }; + let underlying = unsafe { raw_ptr_to_ustr(record.underlying.as_ptr())? }; let currency = Currency::from_str(¤cy_str)?; OptionsContract::new( @@ -253,7 +254,7 @@ pub fn parse_options_contract_v1( Price::from_raw(record.strike_price, currency.precision)?, currency, currency.precision, - parse_min_price_increment(record.min_price_increment, currency)?, + decode_min_price_increment(record.min_price_increment, currency)?, Quantity::new(1.0, 0)?, // TBD Quantity::new(1.0, 0)?, // TBD None, // TBD @@ -270,7 +271,7 @@ pub fn is_trade_msg(order_side: OrderSide, action: c_char) -> bool { order_side == OrderSide::NoOrderSide || action as u8 as char == 'T' } -pub fn parse_mbo_msg( +pub fn decode_mbo_msg( record: &dbn::MboMsg, instrument_id: InstrumentId, price_precision: u8, @@ -315,7 +316,7 @@ pub fn parse_mbo_msg( Ok((Some(delta), None)) } -pub fn parse_trade_msg( +pub fn decode_trade_msg( record: &dbn::TradeMsg, instrument_id: InstrumentId, price_precision: u8, @@ -334,7 +335,7 @@ pub fn parse_trade_msg( Ok(trade) } -pub fn parse_mbp1_msg( +pub fn decode_mbp1_msg( record: &dbn::Mbp1Msg, instrument_id: InstrumentId, price_precision: u8, @@ -369,7 +370,7 @@ pub fn parse_mbp1_msg( Ok((quote, maybe_trade)) } -pub fn parse_mbp10_msg( +pub fn decode_mbp10_msg( record: &dbn::Mbp10Msg, instrument_id: InstrumentId, price_precision: u8, @@ -421,7 +422,7 @@ pub fn parse_mbp10_msg( Ok(depth) } -pub fn parse_bar_type(record: &dbn::OhlcvMsg, instrument_id: InstrumentId) -> Result { +pub fn decode_bar_type(record: &dbn::OhlcvMsg, instrument_id: InstrumentId) -> Result { let bar_type = match record.hd.rtype { 32 => { // ohlcv-1s @@ -448,7 +449,7 @@ pub fn parse_bar_type(record: &dbn::OhlcvMsg, instrument_id: InstrumentId) -> Re Ok(bar_type) } -pub fn parse_ts_event_adjustment(record: &dbn::OhlcvMsg) -> Result { +pub fn decode_ts_event_adjustment(record: &dbn::OhlcvMsg) -> Result { let adjustment = match record.hd.rtype { 32 => { // ohlcv-1s @@ -475,14 +476,14 @@ pub fn parse_ts_event_adjustment(record: &dbn::OhlcvMsg) -> Result { Ok(adjustment) } -pub fn parse_ohlcv_msg( +pub fn decode_ohlcv_msg( record: &dbn::OhlcvMsg, instrument_id: InstrumentId, price_precision: u8, ts_init: UnixNanos, ) -> Result { - let bar_type = parse_bar_type(record, instrument_id)?; - let ts_event_adjustment = parse_ts_event_adjustment(record)?; + let bar_type = decode_bar_type(record, instrument_id)?; + let ts_event_adjustment = decode_ts_event_adjustment(record)?; // Adjust `ts_event` from open to close of bar let ts_event = record.hd.ts_event; @@ -502,14 +503,14 @@ pub fn parse_ohlcv_msg( Ok(bar) } -pub fn parse_record( +pub fn decode_record( record: &dbn::RecordRef, - rtype: dbn::RType, instrument_id: InstrumentId, price_precision: u8, ts_init: Option, include_trades: bool, ) -> Result<(Option, Option)> { + let rtype = record.rtype().expect("Invalid `rtype`"); let result = match rtype { dbn::RType::Mbo => { let msg = record.get::().unwrap(); // SAFETY: RType known @@ -518,7 +519,7 @@ pub fn parse_record( None => msg.ts_recv, }; let result = - parse_mbo_msg(msg, instrument_id, price_precision, ts_init, include_trades)?; + decode_mbo_msg(msg, instrument_id, price_precision, ts_init, include_trades)?; match result { (Some(delta), None) => (Some(Data::Delta(delta)), None), (None, Some(trade)) => (Some(Data::Trade(trade)), None), @@ -532,7 +533,7 @@ pub fn parse_record( Some(ts_init) => ts_init, None => msg.ts_recv, }; - let trade = parse_trade_msg(msg, instrument_id, price_precision, ts_init)?; + let trade = decode_trade_msg(msg, instrument_id, price_precision, ts_init)?; (Some(Data::Trade(trade)), None) } dbn::RType::Mbp1 => { @@ -542,7 +543,7 @@ pub fn parse_record( None => msg.ts_recv, }; let result = - parse_mbp1_msg(msg, instrument_id, price_precision, ts_init, include_trades)?; + decode_mbp1_msg(msg, instrument_id, price_precision, ts_init, include_trades)?; match result { (quote, None) => (Some(Data::Quote(quote)), None), (quote, Some(trade)) => (Some(Data::Quote(quote)), Some(Data::Trade(trade))), @@ -554,7 +555,7 @@ pub fn parse_record( Some(ts_init) => ts_init, None => msg.ts_recv, }; - let depth = parse_mbp10_msg(msg, instrument_id, price_precision, ts_init)?; + let depth = decode_mbp10_msg(msg, instrument_id, price_precision, ts_init)?; (Some(Data::Depth10(depth)), None) } dbn::RType::Ohlcv1S @@ -567,7 +568,7 @@ pub fn parse_record( Some(ts_init) => ts_init, None => msg.hd.ts_event, }; - let bar = parse_ohlcv_msg(msg, instrument_id, price_precision, ts_init)?; + let bar = decode_ohlcv_msg(msg, instrument_id, price_precision, ts_init)?; (Some(Data::Bar(bar)), None) } _ => bail!("RType {:?} is not currently supported", rtype), @@ -576,19 +577,19 @@ pub fn parse_record( Ok(result) } -pub fn parse_instrument_def_msg_v1( +pub fn decode_instrument_def_msg_v1( record: &dbn::compat::InstrumentDefMsgV1, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result> { match record.instrument_class as u8 as char { - 'K' => Ok(Box::new(parse_equity_v1(record, instrument_id, ts_init)?)), - 'F' => Ok(Box::new(parse_futures_contract_v1( + 'K' => Ok(Box::new(decode_equity_v1(record, instrument_id, ts_init)?)), + 'F' => Ok(Box::new(decode_futures_contract_v1( record, instrument_id, ts_init, )?)), - 'C' | 'P' => Ok(Box::new(parse_options_contract_v1( + 'C' | 'P' => Ok(Box::new(decode_options_contract_v1( record, instrument_id, ts_init, @@ -605,19 +606,19 @@ pub fn parse_instrument_def_msg_v1( } } -pub fn parse_instrument_def_msg( +pub fn decode_instrument_def_msg( record: &dbn::InstrumentDefMsg, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result> { match record.instrument_class as u8 as char { - 'K' => Ok(Box::new(parse_equity(record, instrument_id, ts_init)?)), - 'F' => Ok(Box::new(parse_futures_contract( + 'K' => Ok(Box::new(decode_equity(record, instrument_id, ts_init)?)), + 'F' => Ok(Box::new(decode_futures_contract( record, instrument_id, ts_init, )?)), - 'C' | 'P' => Ok(Box::new(parse_options_contract( + 'C' | 'P' => Ok(Box::new(decode_options_contract( record, instrument_id, ts_init, @@ -634,7 +635,7 @@ pub fn parse_instrument_def_msg( } } -pub fn parse_equity( +pub fn decode_equity( record: &dbn::InstrumentDefMsg, instrument_id: InstrumentId, ts_init: UnixNanos, @@ -647,7 +648,7 @@ pub fn parse_equity( None, // No ISIN available yet currency, currency.precision, - parse_min_price_increment(record.min_price_increment, currency)?, + decode_min_price_increment(record.min_price_increment, currency)?, Some(Quantity::new(record.min_lot_size_round_lot.into(), 0)?), None, // TBD None, // TBD @@ -658,14 +659,14 @@ pub fn parse_equity( ) } -pub fn parse_futures_contract( +pub fn decode_futures_contract( record: &dbn::InstrumentDefMsg, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result { let currency = Currency::USD(); // TODO: Temporary hard coding of US futures for now - let cfi_str = unsafe { parse_raw_ptr_to_string(record.cfi.as_ptr())? }; - let underlying = unsafe { parse_raw_ptr_to_ustr(record.asset.as_ptr())? }; + let cfi_str = unsafe { raw_ptr_to_string(record.cfi.as_ptr())? }; + let underlying = unsafe { raw_ptr_to_ustr(record.asset.as_ptr())? }; let (asset_class, _) = parse_cfi_iso10926(&cfi_str)?; FuturesContract::new( @@ -677,7 +678,7 @@ pub fn parse_futures_contract( record.expiration, currency, currency.precision, - parse_min_price_increment(record.min_price_increment, currency)?, + decode_min_price_increment(record.min_price_increment, currency)?, Quantity::new(1.0, 0)?, // TBD Quantity::new(1.0, 0)?, // TBD None, // TBD @@ -689,13 +690,13 @@ pub fn parse_futures_contract( ) } -pub fn parse_options_contract( +pub fn decode_options_contract( record: &dbn::InstrumentDefMsg, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> Result { - let currency_str = unsafe { parse_raw_ptr_to_string(record.currency.as_ptr())? }; - let cfi_str = unsafe { parse_raw_ptr_to_string(record.cfi.as_ptr())? }; + let currency_str = unsafe { raw_ptr_to_string(record.currency.as_ptr())? }; + let cfi_str = unsafe { raw_ptr_to_string(record.cfi.as_ptr())? }; let asset_class_opt = match instrument_id.venue.value.as_str() { "OPRA" => Some(AssetClass::Equity), _ => { @@ -703,7 +704,7 @@ pub fn parse_options_contract( asset_class } }; - let underlying = unsafe { parse_raw_ptr_to_ustr(record.underlying.as_ptr())? }; + let underlying = unsafe { raw_ptr_to_ustr(record.underlying.as_ptr())? }; let currency = Currency::from_str(¤cy_str)?; OptionsContract::new( @@ -717,7 +718,7 @@ pub fn parse_options_contract( Price::from_raw(record.strike_price, currency.precision)?, currency, currency.precision, - parse_min_price_increment(record.min_price_increment, currency)?, + decode_min_price_increment(record.min_price_increment, currency)?, Quantity::new(1.0, 0)?, // TBD Quantity::new(1.0, 0)?, // TBD None, // TBD diff --git a/nautilus_core/adapters/src/databento/loader.rs b/nautilus_core/adapters/src/databento/loader.rs index 589ac6e7f311..2e6be54d407d 100644 --- a/nautilus_core/adapters/src/databento/loader.rs +++ b/nautilus_core/adapters/src/databento/loader.rs @@ -35,7 +35,7 @@ use time; use ustr::Ustr; use super::{ - parsing::{parse_instrument_def_msg_v1, parse_raw_ptr_to_ustr, parse_record}, + decode::{decode_instrument_def_msg_v1, decode_record, raw_ptr_to_ustr}, types::{DatabentoPublisher, Dataset, PublisherId}, }; @@ -228,7 +228,6 @@ impl DatabentoDataLoader { match dbn_stream.get() { Some(record) => { let rec_ref = dbn::RecordRef::from(record); - let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); let instrument_id = match &instrument_id { Some(id) => *id, // Copy None => { @@ -243,9 +242,8 @@ impl DatabentoDataLoader { } }; - match parse_record( + match decode_record( &rec_ref, - rtype, instrument_id, price_precision, None, @@ -276,7 +274,7 @@ impl DatabentoDataLoader { let msg = rec_ref.get::().unwrap(); let raw_symbol = unsafe { - parse_raw_ptr_to_ustr(record.raw_symbol.as_ptr()) + raw_ptr_to_ustr(record.raw_symbol.as_ptr()) .expect("Error parsing `raw_symbol`") }; let symbol = Symbol { value: raw_symbol }; @@ -286,7 +284,7 @@ impl DatabentoDataLoader { .expect("`Venue` not found `publisher_id`"); let instrument_id = InstrumentId::new(symbol, *venue); - match parse_instrument_def_msg_v1(record, instrument_id, msg.ts_recv) { + match decode_instrument_def_msg_v1(record, instrument_id, msg.ts_recv) { Ok(data) => Some(Ok(data)), Err(e) => Some(Err(e)), } diff --git a/nautilus_core/adapters/src/databento/mod.rs b/nautilus_core/adapters/src/databento/mod.rs index a024322f21e5..692f14361bb1 100644 --- a/nautilus_core/adapters/src/databento/mod.rs +++ b/nautilus_core/adapters/src/databento/mod.rs @@ -14,8 +14,8 @@ // ------------------------------------------------------------------------------------------------- pub mod common; +pub mod decode; pub mod loader; -pub mod parsing; pub mod symbology; pub mod types; diff --git a/nautilus_core/adapters/src/databento/python/parsing.rs b/nautilus_core/adapters/src/databento/python/decode.rs similarity index 73% rename from nautilus_core/adapters/src/databento/python/parsing.rs rename to nautilus_core/adapters/src/databento/python/decode.rs index 2f833c1a0b19..68d9e060ee8f 100644 --- a/nautilus_core/adapters/src/databento/python/parsing.rs +++ b/nautilus_core/adapters/src/databento/python/decode.rs @@ -24,51 +24,51 @@ use nautilus_model::{ }; use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyTuple}; -use crate::databento::parsing::{ - parse_equity_v1, parse_futures_contract_v1, parse_mbo_msg, parse_mbp10_msg, parse_mbp1_msg, - parse_options_contract_v1, parse_trade_msg, +use crate::databento::decode::{ + decode_equity_v1, decode_futures_contract_v1, decode_mbo_msg, decode_mbp10_msg, + decode_mbp1_msg, decode_options_contract_v1, decode_trade_msg, }; #[pyfunction] -#[pyo3(name = "parse_equity")] -pub fn py_parse_equity( +#[pyo3(name = "decode_equity")] +pub fn py_decode_equity( record: &dbn::compat::InstrumentDefMsgV1, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> PyResult { - parse_equity_v1(record, instrument_id, ts_init).map_err(to_pyvalue_err) + decode_equity_v1(record, instrument_id, ts_init).map_err(to_pyvalue_err) } #[pyfunction] -#[pyo3(name = "parse_futures_contract")] -pub fn py_parse_futures_contract( +#[pyo3(name = "decode_futures_contract")] +pub fn py_decode_futures_contract( record: &dbn::compat::InstrumentDefMsgV1, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> PyResult { - parse_futures_contract_v1(record, instrument_id, ts_init).map_err(to_pyvalue_err) + decode_futures_contract_v1(record, instrument_id, ts_init).map_err(to_pyvalue_err) } #[pyfunction] -#[pyo3(name = "parse_options_contract")] -pub fn py_parse_options_contract( +#[pyo3(name = "decode_options_contract")] +pub fn py_decode_options_contract( record: &dbn::compat::InstrumentDefMsgV1, instrument_id: InstrumentId, ts_init: UnixNanos, ) -> PyResult { - parse_options_contract_v1(record, instrument_id, ts_init).map_err(to_pyvalue_err) + decode_options_contract_v1(record, instrument_id, ts_init).map_err(to_pyvalue_err) } #[pyfunction] -#[pyo3(name = "parse_mbo_msg")] -pub fn py_parse_mbo_msg( +#[pyo3(name = "decode_mbo_msg")] +pub fn py_decode_mbo_msg( py: Python, record: &dbn::MboMsg, instrument_id: InstrumentId, price_precision: u8, ts_init: UnixNanos, ) -> PyResult { - let result = parse_mbo_msg(record, instrument_id, price_precision, ts_init, false); + let result = decode_mbo_msg(record, instrument_id, price_precision, ts_init, false); match result { Ok((Some(data), None)) => Ok(data.into_py(py)), @@ -78,19 +78,19 @@ pub fn py_parse_mbo_msg( } #[pyfunction] -#[pyo3(name = "parse_trade_msg")] -pub fn py_parse_trade_msg( +#[pyo3(name = "decode_trade_msg")] +pub fn py_decode_trade_msg( record: &dbn::TradeMsg, instrument_id: InstrumentId, price_precision: u8, ts_init: UnixNanos, ) -> PyResult { - parse_trade_msg(record, instrument_id, price_precision, ts_init).map_err(to_pyvalue_err) + decode_trade_msg(record, instrument_id, price_precision, ts_init).map_err(to_pyvalue_err) } #[pyfunction] -#[pyo3(name = "parse_mbp1_msg")] -pub fn py_parse_mbp1_msg( +#[pyo3(name = "decode_mbp1_msg")] +pub fn py_decode_mbp1_msg( py: Python, record: &dbn::Mbp1Msg, instrument_id: InstrumentId, @@ -98,7 +98,7 @@ pub fn py_parse_mbp1_msg( ts_init: UnixNanos, include_trades: bool, ) -> PyResult { - let result = parse_mbp1_msg( + let result = decode_mbp1_msg( record, instrument_id, price_precision, @@ -126,12 +126,12 @@ pub fn py_parse_mbp1_msg( } #[pyfunction] -#[pyo3(name = "parse_mbp10_msg")] -pub fn py_parse_mbp10_msg( +#[pyo3(name = "decode_mbp10_msg")] +pub fn py_decode_mbp10_msg( record: &dbn::Mbp10Msg, instrument_id: InstrumentId, price_precision: u8, ts_init: UnixNanos, ) -> PyResult { - parse_mbp10_msg(record, instrument_id, price_precision, ts_init).map_err(to_pyvalue_err) + decode_mbp10_msg(record, instrument_id, price_precision, ts_init).map_err(to_pyvalue_err) } diff --git a/nautilus_core/adapters/src/databento/python/historical.rs b/nautilus_core/adapters/src/databento/python/historical.rs index e3c2605f1c2b..339865b45bd1 100644 --- a/nautilus_core/adapters/src/databento/python/historical.rs +++ b/nautilus_core/adapters/src/databento/python/historical.rs @@ -16,7 +16,7 @@ use std::{fs, num::NonZeroU64, sync::Arc}; use databento::{self, historical::timeseries::GetRangeParams}; -use dbn::{self, Record, VersionUpgradePolicy}; +use dbn::{self, VersionUpgradePolicy}; use indexmap::IndexMap; use nautilus_core::{ python::to_pyvalue_err, @@ -36,8 +36,8 @@ use tokio::sync::Mutex; use crate::databento::{ common::get_date_time_range, - parsing::{parse_instrument_def_msg, parse_raw_ptr_to_ustr, parse_record}, - symbology::parse_nautilus_instrument_id, + decode::{decode_instrument_def_msg, decode_record, raw_ptr_to_ustr}, + symbology::decode_nautilus_instrument_id, types::{DatabentoPublisher, PublisherId}, }; @@ -140,12 +140,12 @@ impl DatabentoHistoricalClient { let mut instruments = Vec::new(); while let Ok(Some(rec)) = decoder.decode_record::().await { - let raw_symbol = unsafe { parse_raw_ptr_to_ustr(rec.raw_symbol.as_ptr()).unwrap() }; + let raw_symbol = unsafe { raw_ptr_to_ustr(rec.raw_symbol.as_ptr()).unwrap() }; let symbol = Symbol { value: raw_symbol }; let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); let instrument_id = InstrumentId::new(symbol, *venue); - let result = parse_instrument_def_msg(rec, instrument_id, ts_init); + let result = decode_instrument_def_msg(rec, instrument_id, ts_init); match result { Ok(instrument) => instruments.push(instrument), Err(e) => eprintln!("{e:?}"), @@ -201,15 +201,12 @@ impl DatabentoHistoricalClient { while let Ok(Some(rec)) = decoder.decode_record::().await { let rec_ref = dbn::RecordRef::from(rec); - let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); - let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); - let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, *venue) + let instrument_id = decode_nautilus_instrument_id(&rec_ref, &metadata, *venue) .map_err(to_pyvalue_err)?; - let (data, _) = parse_record( + let (data, _) = decode_record( &rec_ref, - rtype, instrument_id, price_precision, Some(ts_init), @@ -267,15 +264,12 @@ impl DatabentoHistoricalClient { while let Ok(Some(rec)) = decoder.decode_record::().await { let rec_ref = dbn::RecordRef::from(rec); - let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); - let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); - let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, *venue) + let instrument_id = decode_nautilus_instrument_id(&rec_ref, &metadata, *venue) .map_err(to_pyvalue_err)?; - let (data, _) = parse_record( + let (data, _) = decode_record( &rec_ref, - rtype, instrument_id, price_precision, Some(ts_init), @@ -342,15 +336,12 @@ impl DatabentoHistoricalClient { while let Ok(Some(rec)) = decoder.decode_record::().await { let rec_ref = dbn::RecordRef::from(rec); - let rtype = rec_ref.rtype().expect("Invalid `rtype` for data loading"); - let venue = publisher_venue_map.get(&rec.hd.publisher_id).unwrap(); - let instrument_id = parse_nautilus_instrument_id(&rec_ref, &metadata, *venue) + let instrument_id = decode_nautilus_instrument_id(&rec_ref, &metadata, *venue) .map_err(to_pyvalue_err)?; - let (data, _) = parse_record( + let (data, _) = decode_record( &rec_ref, - rtype, instrument_id, price_precision, Some(ts_init), diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 3038105f3c66..b3de8ea40a77 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -19,10 +19,11 @@ use std::sync::Arc; use anyhow::Result; use databento::live::Subscription; -use dbn::{PitSymbolMap, RType, Record, SymbolIndex, VersionUpgradePolicy}; +use dbn::{PitSymbolMap, Record, SymbolIndex, VersionUpgradePolicy}; use indexmap::IndexMap; use log::{error, info}; use nautilus_core::python::to_pyruntime_err; +use nautilus_core::time::AtomicTime; use nautilus_core::{ python::to_pyvalue_err, time::{get_atomic_clock_realtime, UnixNanos}, @@ -38,7 +39,7 @@ use time::OffsetDateTime; use tokio::sync::Mutex; use tokio::time::{timeout, Duration}; -use crate::databento::parsing::{parse_instrument_def_msg, parse_raw_ptr_to_ustr, parse_record}; +use crate::databento::decode::{decode_instrument_def_msg, decode_record, raw_ptr_to_ustr}; use crate::databento::types::{DatabentoPublisher, PublisherId}; use super::loader::convert_instrument_to_pyobject; @@ -188,83 +189,18 @@ impl DatabentoLiveClient { } }; - let rtype = record.rtype().expect("Invalid `rtype`"); - - match rtype { - RType::SymbolMapping => { - symbol_map.on_record(record).unwrap_or_else(|_| { - panic!("Error updating `symbol_map` with {record:?}") - }); - } - RType::Error => { - eprintln!("{record:?}"); // TODO: Just print stderr for now - error!("{:?}", record); - } - RType::System => { - println!("{record:?}"); // TODO: Just print stdout for now - info!("{:?}", record); - } - RType::InstrumentDef => { - let msg = record - .get::() - .expect("Error converting record to `InstrumentDefMsg`"); - let raw_symbol = - unsafe { parse_raw_ptr_to_ustr(msg.raw_symbol.as_ptr()).unwrap() }; - let symbol = Symbol { value: raw_symbol }; - let venue = publisher_venue_map.get(&msg.hd.publisher_id).unwrap(); - let instrument_id = InstrumentId::new(symbol, *venue); - - let ts_init = clock.get_time_ns(); - let result = parse_instrument_def_msg(msg, instrument_id, ts_init); - - match result { - Ok(instrument) => { - Python::with_gil(|py| { - let py_obj = - convert_instrument_to_pyobject(py, instrument).unwrap(); - match callback.call1(py, (py_obj,)) { - Ok(_) => {} - Err(e) => eprintln!("Error on callback, {e:?}"), // Just print error for now - }; - }); - } - Err(e) => eprintln!("{e:?}"), - } - continue; - } - _ => { - let raw_symbol = symbol_map - .get_for_rec(&record) - .expect("Cannot resolve `raw_symbol` from `symbol_map`"); - let publisher_id = record.publisher().unwrap() as PublisherId; - let symbol = Symbol::from_str_unchecked(raw_symbol); - let venue = publisher_venue_map.get(&publisher_id).unwrap(); - let instrument_id = InstrumentId::new(symbol, *venue); - - let price_precision = 2; // Hard coded for now - let ts_init = clock.get_time_ns(); - - let (data, maybe_data) = parse_record( - &record, - rtype, - instrument_id, - price_precision, - Some(ts_init), - true, // Always include trades - ) + if let Some(msg) = record.get::() { + handle_error_msg(msg); + } else if let Some(msg) = record.get::() { + handle_system_msg(msg); + } else if let Some(msg) = record.get::() { + handle_symbol_mapping_msg(msg, &mut symbol_map); + } else if let Some(msg) = record.get::() { + handle_instrument_def_msg(msg, &publisher_venue_map, clock, &callback); + } else { + handle_record(record, &symbol_map, &publisher_venue_map, clock, &callback) .map_err(to_pyvalue_err)?; - - Python::with_gil(|py| { - if let Some(data) = data { - call_python_with_data(py, &callback, data); - } - - if let Some(data) = maybe_data { - call_python_with_data(py, &callback, data); - } - }); - } - } + }; } Ok(()) }) @@ -288,6 +224,89 @@ impl DatabentoLiveClient { } } +fn handle_error_msg(msg: &dbn::ErrorMsg) { + eprintln!("{msg:?}"); // TODO: Just print stderr for now + error!("{:?}", msg); +} + +fn handle_system_msg(msg: &dbn::SystemMsg) { + println!("{msg:?}"); // TODO: Just print stdout for now + info!("{:?}", msg); +} + +fn handle_symbol_mapping_msg(msg: &dbn::SymbolMappingMsg, symbol_map: &mut PitSymbolMap) { + symbol_map + .on_symbol_mapping(msg) + .unwrap_or_else(|_| panic!("Error updating `symbol_map` with {msg:?}")); +} + +fn handle_instrument_def_msg( + msg: &dbn::InstrumentDefMsg, + publisher_venue_map: &IndexMap, + clock: &AtomicTime, + callback: &PyObject, +) { + let raw_symbol = unsafe { raw_ptr_to_ustr(msg.raw_symbol.as_ptr()).unwrap() }; + let symbol = Symbol { value: raw_symbol }; + let venue = publisher_venue_map.get(&msg.hd.publisher_id).unwrap(); + let instrument_id = InstrumentId::new(symbol, *venue); + + let ts_init = clock.get_time_ns(); + let result = decode_instrument_def_msg(msg, instrument_id, ts_init); + + match result { + Ok(instrument) => { + Python::with_gil(|py| { + let py_obj = convert_instrument_to_pyobject(py, instrument).unwrap(); + match callback.call1(py, (py_obj,)) { + Ok(_) => {} + Err(e) => eprintln!("Error on callback, {e:?}"), // Just print error for now + }; + }); + } + Err(e) => eprintln!("{e:?}"), + } +} + +fn handle_record( + record: dbn::RecordRef, + symbol_map: &PitSymbolMap, + publisher_venue_map: &IndexMap, + clock: &AtomicTime, + callback: &PyObject, +) -> Result<()> { + let raw_symbol = symbol_map + .get_for_rec(&record) + .expect("Cannot resolve `raw_symbol` from `symbol_map`"); + let publisher_id = record.publisher().unwrap() as PublisherId; + let symbol = Symbol::from_str_unchecked(raw_symbol); + let venue = publisher_venue_map.get(&publisher_id).unwrap(); + let instrument_id = InstrumentId::new(symbol, *venue); + + let price_precision = 2; // Hard coded for now + let ts_init = clock.get_time_ns(); + + let (data, maybe_data) = decode_record( + &record, + instrument_id, + price_precision, + Some(ts_init), + true, // Always include trades + )?; + + Python::with_gil(|py| { + if let Some(data) = data { + call_python_with_data(py, callback, data); + } + + if let Some(data) = maybe_data { + call_python_with_data(py, callback, data); + } + }); + + Ok(()) +} + fn call_python_with_data(py: Python, callback: &PyObject, data: Data) { let py_obj = data_to_pycapsule(py, data); match callback.call1(py, (py_obj,)) { diff --git a/nautilus_core/adapters/src/databento/python/mod.rs b/nautilus_core/adapters/src/databento/python/mod.rs index b24d96d0ae56..70b20a917b95 100644 --- a/nautilus_core/adapters/src/databento/python/mod.rs +++ b/nautilus_core/adapters/src/databento/python/mod.rs @@ -13,7 +13,29 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- +pub mod decode; pub mod historical; pub mod live; pub mod loader; -pub mod parsing; + +use pyo3::prelude::*; + +use super::types; + +/// Loaded as nautilus_pyo3.databento +#[pymodule] +pub fn databento(_: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(decode::py_decode_equity, m)?)?; + m.add_function(wrap_pyfunction!(decode::py_decode_futures_contract, m)?)?; + m.add_function(wrap_pyfunction!(decode::py_decode_options_contract, m)?)?; + m.add_function(wrap_pyfunction!(decode::py_decode_mbo_msg, m)?)?; + m.add_function(wrap_pyfunction!(decode::py_decode_trade_msg, m)?)?; + m.add_function(wrap_pyfunction!(decode::py_decode_mbp1_msg, m)?)?; + m.add_function(wrap_pyfunction!(decode::py_decode_mbp10_msg, m)?)?; + + Ok(()) +} diff --git a/nautilus_core/adapters/src/databento/symbology.rs b/nautilus_core/adapters/src/databento/symbology.rs index 7b7cfb605134..33794dae38d8 100644 --- a/nautilus_core/adapters/src/databento/symbology.rs +++ b/nautilus_core/adapters/src/databento/symbology.rs @@ -19,7 +19,7 @@ use dbn::Record; use nautilus_model::identifiers::{instrument_id::InstrumentId, symbol::Symbol, venue::Venue}; use ustr::Ustr; -pub fn parse_nautilus_instrument_id( +pub fn decode_nautilus_instrument_id( record: &dbn::RecordRef, metadata: &dbn::Metadata, venue: Venue, diff --git a/nautilus_core/pyo3/src/lib.rs b/nautilus_core/pyo3/src/lib.rs index 818b85f787ac..d4a85e0bb0e6 100644 --- a/nautilus_core/pyo3/src/lib.rs +++ b/nautilus_core/pyo3/src/lib.rs @@ -13,39 +13,15 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use nautilus_adapters::databento::{ - loader, python::historical, python::live, python::parsing, types, -}; use pyo3::{ prelude::*, types::{PyDict, PyString}, }; -/// This currently works around an issue where `databento` couldn't be recognised -/// by the pyo3 `wrap_pymodule!` macro. -/// -/// Loaded as nautilus_pyo3.databento -#[pymodule] -pub fn databento(_: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_function(wrap_pyfunction!(parsing::py_parse_equity, m)?)?; - m.add_function(wrap_pyfunction!(parsing::py_parse_futures_contract, m)?)?; - m.add_function(wrap_pyfunction!(parsing::py_parse_options_contract, m)?)?; - m.add_function(wrap_pyfunction!(parsing::py_parse_mbo_msg, m)?)?; - m.add_function(wrap_pyfunction!(parsing::py_parse_trade_msg, m)?)?; - m.add_function(wrap_pyfunction!(parsing::py_parse_mbp1_msg, m)?)?; - m.add_function(wrap_pyfunction!(parsing::py_parse_mbp10_msg, m)?)?; - - Ok(()) -} - -/// Need to modify sys modules so that submodule can be loaded directly as +/// We modify sys modules so that submodule can be loaded directly as /// import supermodule.submodule /// -/// Also re-exports all submodule attributes so they can be imported directly from `nautilus_pyo3`. +/// Also re-exports all submodule attributes so they can be imported directly from `nautilus_pyo3` /// refer: https://github.com/PyO3/pyo3/issues/2644 #[pymodule] fn nautilus_pyo3(py: Python<'_>, m: &PyModule) -> PyResult<()> { @@ -63,7 +39,7 @@ fn nautilus_pyo3(py: Python<'_>, m: &PyModule) -> PyResult<()> { re_export_module_attributes(m, n)?; let n = "databento"; - let submodule = pyo3::wrap_pymodule!(databento); + let submodule = pyo3::wrap_pymodule!(nautilus_adapters::databento::python::databento); m.add_wrapped(submodule)?; sys_modules.set_item(format!("{module_name}.{n}"), m.getattr(n)?)?; re_export_module_attributes(m, n)?; From dc26c274b60ab7b00a496ec3637423abb3c48a4c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 15:23:12 +1100 Subject: [PATCH 019/104] Add OrderBookDepth10 constructor to type stub --- nautilus_trader/core/nautilus_pyo3.pyi | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 69356812064a..e405dad95404 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -497,6 +497,18 @@ class OrderBookDelta: def get_fields() -> dict[str, str]: ... class OrderBookDepth10: + def __init__( + self, + instrument_id: InstrumentId, + bids: list[BookOrder], + asks: list[BookOrder], + bid_counts: list[int], + ask_counts: list[int], + flags: int, + sequence: int, + ts_event: int, + ts_init: int, + ) -> None: ... @property def ts_event(self) -> int: ... @property From 287b8a60f9bd99ff014ae610238440c5094e92a4 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 15:23:29 +1100 Subject: [PATCH 020/104] Fix deprecation warnings and docstrings --- nautilus_trader/persistence/wranglers_v2.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nautilus_trader/persistence/wranglers_v2.py b/nautilus_trader/persistence/wranglers_v2.py index 68181b384501..85e1fab23e46 100644 --- a/nautilus_trader/persistence/wranglers_v2.py +++ b/nautilus_trader/persistence/wranglers_v2.py @@ -102,7 +102,7 @@ def from_pandas( ts_init_delta: int = 0, ) -> list[nautilus_pyo3.OrderBookDelta]: """ - Process the given pandas.DataFrame into Nautilus `OrderBookDelta` objects. + Process the given pandas DataFrame into Nautilus `OrderBookDelta` objects. Parameters ---------- @@ -138,7 +138,7 @@ def from_pandas( df["ts_event"] = ( pd.to_datetime(df["ts_event"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) @@ -146,7 +146,7 @@ def from_pandas( df["ts_init"] = ( pd.to_datetime(df["ts_init"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) else: @@ -203,7 +203,7 @@ def from_pandas( ts_init_delta: int = 0, ) -> list[nautilus_pyo3.QuoteTick]: """ - Process the given `data` into Nautilus `QuoteTick` objects. + Process the given pandas DataFrame into Nautilus `QuoteTick` objects. Expects columns ['bid_price', 'ask_price'] with 'timestamp' index. Note: The 'bid_size' and 'ask_size' columns are optional, will then use @@ -255,7 +255,7 @@ def from_pandas( df["ts_event"] = ( pd.to_datetime(df["ts_event"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) @@ -263,7 +263,7 @@ def from_pandas( df["ts_init"] = ( pd.to_datetime(df["ts_init"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) else: @@ -330,7 +330,7 @@ def from_pandas( ts_init_delta: int = 0, ) -> list[nautilus_pyo3.TradeTick]: """ - Process the given `data` into Nautilus `TradeTick` objects. + Process the given pandas DataFrame into Nautilus `TradeTick` objects. Parameters ---------- @@ -368,7 +368,7 @@ def from_pandas( df["ts_event"] = ( pd.to_datetime(df["ts_event"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) @@ -376,7 +376,7 @@ def from_pandas( df["ts_init"] = ( pd.to_datetime(df["ts_init"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) else: @@ -449,7 +449,7 @@ def from_pandas( ts_init_delta: int = 0, ) -> list[nautilus_pyo3.Bar]: """ - Process the given `data` into Nautilus `Bar` objects. + Process the given pandas DataFrame into Nautilus `Bar` objects. Parameters ---------- @@ -484,7 +484,7 @@ def from_pandas( df["ts_event"] = ( pd.to_datetime(df["ts_event"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) @@ -492,7 +492,7 @@ def from_pandas( df["ts_init"] = ( pd.to_datetime(df["ts_init"], utc=True, format="mixed") .dt.tz_localize(None) - .view("int64") + .astype("int64") .astype("uint64") ) else: From 35e4548d247649a0fdd3286629d974a333a5d379 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 15:23:49 +1100 Subject: [PATCH 021/104] Fix pyo3 stubs --- nautilus_trader/test_kit/rust/data_pyo3.py | 72 +++++++++++++++---- .../test_kit/rust/identifiers_pyo3.py | 4 ++ 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/nautilus_trader/test_kit/rust/data_pyo3.py b/nautilus_trader/test_kit/rust/data_pyo3.py index c38a4d40036b..2500775c866d 100644 --- a/nautilus_trader/test_kit/rust/data_pyo3.py +++ b/nautilus_trader/test_kit/rust/data_pyo3.py @@ -22,6 +22,7 @@ from nautilus_trader.core.nautilus_pyo3 import BookOrder from nautilus_trader.core.nautilus_pyo3 import InstrumentId from nautilus_trader.core.nautilus_pyo3 import OrderBookDelta +from nautilus_trader.core.nautilus_pyo3 import OrderBookDepth10 from nautilus_trader.core.nautilus_pyo3 import OrderSide from nautilus_trader.core.nautilus_pyo3 import Price from nautilus_trader.core.nautilus_pyo3 import PriceType @@ -58,24 +59,65 @@ def order_book_delta( @staticmethod def order_book_depth10( instrument_id: InstrumentId | None = None, - price: float = 100.0, - size: float = 10, + flags: int = 0, + sequence: int = 0, ts_event: int = 0, ts_init: int = 0, - ) -> OrderBookDelta: - return OrderBookDelta( - instrument_id=instrument_id or TestIdProviderPyo3.ethusdt_binance_id(), - action=BookAction.ADD, - order=BookOrder( - side=OrderSide.BUY, - price=Price.from_str(str(price)), - size=Quantity.from_str(str(size)), - order_id=0, - ), - flags=0, - sequence=0, - ts_init=ts_init, + ) -> OrderBookDepth10: + bids: list[BookOrder] = [] + asks: list[BookOrder] = [] + + # Create bids + price = 99.00 + quantity = 100.0 + order_id = 1 + + for _ in range(10): + order = BookOrder( + OrderSide.BUY, + Price(price, 2), + Quantity(quantity, 0), + order_id, + ) + + bids.append(order) + + price -= 1.0 + quantity += 100.0 + order_id += 1 + + # Create asks + price = 100.00 + quantity = 100.0 + order_id = 11 + + for _ in range(10): + order = BookOrder( + OrderSide.SELL, + Price(price, 2), + Quantity(quantity, 0), + order_id, + ) + + asks.append(order) + + price += 1.0 + quantity += 100.0 + order_id += 1 + + bid_counts = [1] * 10 + ask_counts = [1] * 10 + + return OrderBookDepth10( + instrument_id=instrument_id or TestIdProviderPyo3.aapl_xnas_id(), + bids=bids, + asks=asks, + bid_counts=bid_counts, + ask_counts=ask_counts, + flags=flags, + sequence=sequence, ts_event=ts_event, + ts_init=ts_init, ) @staticmethod diff --git a/nautilus_trader/test_kit/rust/identifiers_pyo3.py b/nautilus_trader/test_kit/rust/identifiers_pyo3.py index 6740b0739672..907243d3df82 100644 --- a/nautilus_trader/test_kit/rust/identifiers_pyo3.py +++ b/nautilus_trader/test_kit/rust/identifiers_pyo3.py @@ -76,6 +76,10 @@ def usdjpy_id() -> InstrumentId: def audusd_idealpro_id() -> InstrumentId: return InstrumentId(Symbol("AUD/USD"), Venue("IDEALPRO")) + @staticmethod + def aapl_xnas_id() -> InstrumentId: + return InstrumentId(Symbol("AAPL"), Venue("XNAS")) + @staticmethod def betting_instrument_id(): from nautilus_trader.adapters.betfair.parsing.common import betfair_instrument_id From 40a0c93cdce0065696eb62ae9aae141969fa6d6d Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 15:24:11 +1100 Subject: [PATCH 022/104] Unskip catalog OrderBookDepth10 test --- tests/unit_tests/persistence/test_catalog.py | 23 ++++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/unit_tests/persistence/test_catalog.py b/tests/unit_tests/persistence/test_catalog.py index ae85d4f9ca72..39319c66db93 100644 --- a/tests/unit_tests/persistence/test_catalog.py +++ b/tests/unit_tests/persistence/test_catalog.py @@ -21,6 +21,7 @@ import pyarrow.dataset as ds import pytest +from nautilus_trader.core import nautilus_pyo3 from nautilus_trader.core.rust.model import AggressorSide from nautilus_trader.core.rust.model import BookAction from nautilus_trader.model.currencies import USD @@ -40,6 +41,7 @@ from nautilus_trader.persistence.wranglers_v2 import TradeTickDataWranglerV2 from nautilus_trader.test_kit.mocks.data import NewsEventData from nautilus_trader.test_kit.providers import TestInstrumentProvider +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 from nautilus_trader.test_kit.stubs.data import TestDataStubs from nautilus_trader.test_kit.stubs.persistence import TestPersistenceStubs from tests import TEST_DATA_DIR @@ -245,17 +247,20 @@ def test_catalog_bars(catalog: ParquetDataCatalog) -> None: assert len(bars) == len(stub_bars) == 10 -@pytest.mark.skip(reason="WIP, currently failing value: MissingMetadata('instrument_id')") -def test_catalog_write_order_book_depth10(catalog: ParquetDataCatalog) -> None: +def test_catalog_write_pyo3_order_book_depth10(catalog: ParquetDataCatalog) -> None: # Arrange - instrument = TestInstrumentProvider.equity() - depth = TestDataStubs.order_book_depth10(instrument_id=instrument.id) + instrument = TestInstrumentProvider.ethusdt_binance() + instrument_id = nautilus_pyo3.InstrumentId.from_str(instrument.id.value) + depth10 = TestDataProviderPyo3.order_book_depth10(instrument_id=instrument_id) # Act - catalog.write_data([depth]) + catalog.write_data([depth10] * 100) # Assert - assert len(catalog.order_book_depth10()) == 1 + depths = catalog.order_book_depth10(instrument_ids=[instrument.id]) + all_depths = catalog.order_book_depth10() + assert len(depths) == 100 + assert len(all_depths) == 100 def test_catalog_write_pyo3_quote_ticks(catalog: ParquetDataCatalog) -> None: @@ -273,8 +278,8 @@ def test_catalog_write_pyo3_quote_ticks(catalog: ParquetDataCatalog) -> None: # Assert quotes = catalog.quote_ticks(instrument_ids=[instrument.id]) all_quotes = catalog.quote_ticks() - assert len(all_quotes) == 100_000 assert len(quotes) == 100_000 + assert len(all_quotes) == 100_000 def test_catalog_write_pyo3_trade_ticks(catalog: ParquetDataCatalog) -> None: @@ -291,8 +296,8 @@ def test_catalog_write_pyo3_trade_ticks(catalog: ParquetDataCatalog) -> None: # Assert trades = catalog.trade_ticks(instrument_ids=[instrument.id]) all_trades = catalog.trade_ticks() - assert len(all_trades) == 69_806 assert len(trades) == 69_806 + assert len(all_trades) == 69_806 def test_catalog_multiple_bar_types(catalog: ParquetDataCatalog) -> None: @@ -321,9 +326,9 @@ def test_catalog_multiple_bar_types(catalog: ParquetDataCatalog) -> None: bars1 = catalog.bars(bar_types=[str(bar_type1)]) bars2 = catalog.bars(bar_types=[str(bar_type2)]) all_bars = catalog.bars() - assert len(all_bars) == 20 assert len(bars1) == 10 assert len(bars2) == 10 + assert len(all_bars) == 20 def test_catalog_bar_query_instrument_id( From 0f62924a78950a1d97315af9ab2263850efc8c01 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 15:30:49 +1100 Subject: [PATCH 023/104] Fix clippy lints --- nautilus_core/adapters/src/databento/loader.rs | 1 - nautilus_core/adapters/src/databento/python/loader.rs | 10 +++++++--- nautilus_core/indicators/src/python/momentum/aroon.rs | 10 +++++----- nautilus_core/indicators/src/python/momentum/rsi.rs | 8 ++++---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/nautilus_core/adapters/src/databento/loader.rs b/nautilus_core/adapters/src/databento/loader.rs index 2e6be54d407d..b5e7ae7062c1 100644 --- a/nautilus_core/adapters/src/databento/loader.rs +++ b/nautilus_core/adapters/src/databento/loader.rs @@ -125,7 +125,6 @@ impl DatabentoDataLoader { .collect::>(); self.publisher_venue_map = publishers - .clone() .into_iter() .map(|p| (p.publisher_id, Venue::from(p.venue.as_str()))) .collect::>(); diff --git a/nautilus_core/adapters/src/databento/python/loader.rs b/nautilus_core/adapters/src/databento/python/loader.rs index ede2ad6ae2ef..49eb69f7c3f1 100644 --- a/nautilus_core/adapters/src/databento/python/loader.rs +++ b/nautilus_core/adapters/src/databento/python/loader.rs @@ -47,6 +47,7 @@ impl DatabentoDataLoader { Self::new(path.map(PathBuf::from)).map_err(to_pyvalue_err) } + #[must_use] #[pyo3(name = "get_publishers")] pub fn py_get_publishers(&self) -> HashMap { self.get_publishers() @@ -55,15 +56,18 @@ impl DatabentoDataLoader { .collect::>() } + #[must_use] #[pyo3(name = "get_dataset_for_venue")] pub fn py_get_dataset_for_venue(&self, venue: &Venue) -> Option { - self.get_dataset_for_venue(venue).map(|d| d.to_string()) + self.get_dataset_for_venue(venue) + .map(std::string::ToString::to_string) } + #[must_use] #[pyo3(name = "get_venue_for_publisher")] pub fn py_get_venue_for_publisher(&self, publisher_id: PublisherId) -> Option { self.get_venue_for_publisher(publisher_id) - .map(|d| d.to_string()) + .map(std::string::ToString::to_string) } #[pyo3(name = "schema_for_file")] @@ -388,7 +392,7 @@ fn exhaust_data_iter_to_pycapsule( Ok((None, Some(item2))) => data.push(item2), Ok((Some(item1), Some(item2))) => { data.push(item1); - data.push(item2) + data.push(item2); } Ok((None, None)) => { continue; diff --git a/nautilus_core/indicators/src/python/momentum/aroon.rs b/nautilus_core/indicators/src/python/momentum/aroon.rs index 74eb76cd0493..4c60fda04c48 100644 --- a/nautilus_core/indicators/src/python/momentum/aroon.rs +++ b/nautilus_core/indicators/src/python/momentum/aroon.rs @@ -26,6 +26,10 @@ impl AroonOscillator { Self::new(period).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("AroonOscillator({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -96,10 +100,6 @@ impl AroonOscillator { #[pyo3(name = "reset")] fn py_reset(&mut self) { - self.reset() - } - - fn __repr__(&self) -> String { - format!("AroonOscillator({})", self.period) + self.reset(); } } diff --git a/nautilus_core/indicators/src/python/momentum/rsi.rs b/nautilus_core/indicators/src/python/momentum/rsi.rs index fa0b1798da54..d241c49b5359 100644 --- a/nautilus_core/indicators/src/python/momentum/rsi.rs +++ b/nautilus_core/indicators/src/python/momentum/rsi.rs @@ -31,6 +31,10 @@ impl RelativeStrengthIndex { Self::new(period, ma_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("ExponentialMovingAverage({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -80,8 +84,4 @@ impl RelativeStrengthIndex { fn py_handle_trade_tick(&mut self, tick: &TradeTick) { self.update_raw((&tick.price).into()); } - - fn __repr__(&self) -> String { - format!("ExponentialMovingAverage({})", self.period) - } } From 365f2e8460be115be6b6e9179a2b37e698b640db Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 16:16:45 +1100 Subject: [PATCH 024/104] Standardize indicators special method ordering --- nautilus_core/indicators/src/indicator.rs | 6 ++---- .../indicators/src/python/average/ama.rs | 20 +++++++++---------- .../indicators/src/python/average/dema.rs | 8 ++++---- .../indicators/src/python/average/ema.rs | 8 ++++---- .../indicators/src/python/average/hma.rs | 8 ++++---- .../indicators/src/python/average/rma.rs | 8 ++++---- .../indicators/src/python/average/sma.rs | 8 ++++---- .../indicators/src/python/average/wma.rs | 8 ++++---- .../src/python/ratio/efficiency_ratio.rs | 8 ++++---- 9 files changed, 40 insertions(+), 42 deletions(-) diff --git a/nautilus_core/indicators/src/indicator.rs b/nautilus_core/indicators/src/indicator.rs index 7041fdd3c82d..85cc467f1eb3 100644 --- a/nautilus_core/indicators/src/indicator.rs +++ b/nautilus_core/indicators/src/indicator.rs @@ -17,7 +17,6 @@ use std::{fmt, fmt::Debug}; use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; -/// Indicator trait pub trait Indicator { fn name(&self) -> String; fn has_inputs(&self) -> bool; @@ -28,7 +27,6 @@ pub trait Indicator { fn reset(&mut self); } -/// Moving average trait pub trait MovingAverage: Indicator { fn value(&self) -> f64; fn count(&self) -> usize; @@ -37,14 +35,14 @@ pub trait MovingAverage: Indicator { impl Debug for dyn Indicator + Send { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Implement custom formatting for the Indicator trait object. + // Implement custom formatting for the Indicator trait object write!(f, "Indicator {{ ... }}") } } impl Debug for dyn MovingAverage + Send { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Implement custom formatting for the Indicator trait object. + // Implement custom formatting for the Indicator trait object write!(f, "MovingAverage()") } } diff --git a/nautilus_core/indicators/src/python/average/ama.rs b/nautilus_core/indicators/src/python/average/ama.rs index 082f625d7e56..e3346da96cff 100644 --- a/nautilus_core/indicators/src/python/average/ama.rs +++ b/nautilus_core/indicators/src/python/average/ama.rs @@ -43,6 +43,16 @@ impl AdaptiveMovingAverage { .map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!( + "WeightedMovingAverage({}({},{},{})", + self.name(), + self.period_efficiency_ratio, + self.period_fast, + self.period_slow + ) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -91,14 +101,4 @@ impl AdaptiveMovingAverage { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!( - "WeightedMovingAverage({}({},{},{})", - self.name(), - self.period_efficiency_ratio, - self.period_fast, - self.period_slow - ) - } } diff --git a/nautilus_core/indicators/src/python/average/dema.rs b/nautilus_core/indicators/src/python/average/dema.rs index c26d4a147bbc..6a5047625010 100644 --- a/nautilus_core/indicators/src/python/average/dema.rs +++ b/nautilus_core/indicators/src/python/average/dema.rs @@ -32,6 +32,10 @@ impl DoubleExponentialMovingAverage { Self::new(period, price_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("DoubleExponentialMovingAverage({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -92,8 +96,4 @@ impl DoubleExponentialMovingAverage { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!("DoubleExponentialMovingAverage({})", self.period) - } } diff --git a/nautilus_core/indicators/src/python/average/ema.rs b/nautilus_core/indicators/src/python/average/ema.rs index 4a8e35c2d3ef..1392104f0b7f 100644 --- a/nautilus_core/indicators/src/python/average/ema.rs +++ b/nautilus_core/indicators/src/python/average/ema.rs @@ -32,6 +32,10 @@ impl ExponentialMovingAverage { Self::new(period, price_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("ExponentialMovingAverage({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -98,8 +102,4 @@ impl ExponentialMovingAverage { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!("ExponentialMovingAverage({})", self.period) - } } diff --git a/nautilus_core/indicators/src/python/average/hma.rs b/nautilus_core/indicators/src/python/average/hma.rs index 4ba74e77bdab..8f3c04e5587f 100644 --- a/nautilus_core/indicators/src/python/average/hma.rs +++ b/nautilus_core/indicators/src/python/average/hma.rs @@ -32,6 +32,10 @@ impl HullMovingAverage { Self::new(period, price_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("HullMovingAverage({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -92,8 +96,4 @@ impl HullMovingAverage { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!("HullMovingAverage({})", self.period) - } } diff --git a/nautilus_core/indicators/src/python/average/rma.rs b/nautilus_core/indicators/src/python/average/rma.rs index a126f606af5b..50ebbf5979f3 100644 --- a/nautilus_core/indicators/src/python/average/rma.rs +++ b/nautilus_core/indicators/src/python/average/rma.rs @@ -32,6 +32,10 @@ impl WilderMovingAverage { Self::new(period, price_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("WilderMovingAverage({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -98,8 +102,4 @@ impl WilderMovingAverage { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!("WilderMovingAverage({})", self.period) - } } diff --git a/nautilus_core/indicators/src/python/average/sma.rs b/nautilus_core/indicators/src/python/average/sma.rs index 4b4bd6a0fb9c..e00c6b0a4927 100644 --- a/nautilus_core/indicators/src/python/average/sma.rs +++ b/nautilus_core/indicators/src/python/average/sma.rs @@ -32,6 +32,10 @@ impl SimpleMovingAverage { Self::new(period, price_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("SimpleMovingAverage({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -92,8 +96,4 @@ impl SimpleMovingAverage { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!("SimpleMovingAverage({})", self.period) - } } diff --git a/nautilus_core/indicators/src/python/average/wma.rs b/nautilus_core/indicators/src/python/average/wma.rs index ef234e4e2cee..aadfaecd07f7 100644 --- a/nautilus_core/indicators/src/python/average/wma.rs +++ b/nautilus_core/indicators/src/python/average/wma.rs @@ -36,6 +36,10 @@ impl WeightedMovingAverage { Self::new(period, weights, price_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("WeightedMovingAverage({},{:?})", self.period, self.weights) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -90,8 +94,4 @@ impl WeightedMovingAverage { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!("WeightedMovingAverage({},{:?})", self.period, self.weights) - } } diff --git a/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs b/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs index 33475e26882d..925bde93ac9d 100644 --- a/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs +++ b/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs @@ -26,6 +26,10 @@ impl EfficiencyRatio { Self::new(period, price_type).map_err(to_pyvalue_err) } + fn __repr__(&self) -> String { + format!("EfficiencyRatio({})", self.period) + } + #[getter] #[pyo3(name = "name")] fn py_name(&self) -> String { @@ -59,8 +63,4 @@ impl EfficiencyRatio { fn py_update_raw(&mut self, value: f64) { self.update_raw(value); } - - fn __repr__(&self) -> String { - format!("EfficiencyRatio({})", self.period) - } } From 91b48cce6b7c3e48375b431e2669cc941ce47883 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 16:24:36 +1100 Subject: [PATCH 025/104] Refine Databento adapter more --- nautilus_core/adapters/src/databento/loader.rs | 8 +------- nautilus_core/adapters/src/databento/python/mod.rs | 4 +--- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/nautilus_core/adapters/src/databento/loader.rs b/nautilus_core/adapters/src/databento/loader.rs index b5e7ae7062c1..a722ffc27883 100644 --- a/nautilus_core/adapters/src/databento/loader.rs +++ b/nautilus_core/adapters/src/databento/loader.rs @@ -55,10 +55,6 @@ use super::{ /// - IMBALANCE -> `DatabentoImbalance` /// - STATISTICS -> `DatabentoStatistics` /// -/// For the loader to work correctly, you must first either: -/// - Load Databento instrument definitions from a DBN file using `load_instruments(...)` -/// - Manually add Nautilus instrument objects through `add_instruments(...)` -/// /// # Warnings /// The following Databento instrument classes are not supported: /// - ``FUTURE_SPREAD`` @@ -194,9 +190,7 @@ impl DatabentoDataLoader { .get(instrument_id) .expect("No raw symbol found for {instrument_id}"); - let symbol = Symbol { - value: Ustr::from(raw_symbol), - }; + let symbol = Symbol::from_str_unchecked(raw_symbol); Ok(InstrumentId::new(symbol, venue)) } diff --git a/nautilus_core/adapters/src/databento/python/mod.rs b/nautilus_core/adapters/src/databento/python/mod.rs index 70b20a917b95..1f0ba5cded4e 100644 --- a/nautilus_core/adapters/src/databento/python/mod.rs +++ b/nautilus_core/adapters/src/databento/python/mod.rs @@ -20,12 +20,10 @@ pub mod loader; use pyo3::prelude::*; -use super::types; - /// Loaded as nautilus_pyo3.databento #[pymodule] pub fn databento(_: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; From 3c9dcc333f0a4667ffe0c012d0bc3693cd879cfd Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 17:21:40 +1100 Subject: [PATCH 026/104] Standardize tutorial language --- examples/notebooks/backtest_binance_orderbook.ipynb | 2 +- examples/notebooks/backtest_example.ipynb | 2 +- examples/notebooks/backtest_fx_usdjpy.ipynb | 4 ++-- examples/notebooks/external_data_backtest.ipynb | 2 +- examples/notebooks/parquet_explorer.ipynb | 2 +- examples/notebooks/quick_start.ipynb | 5 ++--- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/examples/notebooks/backtest_binance_orderbook.ipynb b/examples/notebooks/backtest_binance_orderbook.ipynb index b1370ff6133f..838402bef1a1 100644 --- a/examples/notebooks/backtest_binance_orderbook.ipynb +++ b/examples/notebooks/backtest_binance_orderbook.ipynb @@ -7,7 +7,7 @@ "source": [ "# Backtest on Binance OrderBook data\n", "\n", - "This example runs through how to setup the data catalog and a `BacktestNode` to backtest an `OrderBookImbalance` strategy or order book data. This example requires you bring your Binance own order book data.\n", + "This tutorial runs through how to setup the data catalog and a `BacktestNode` to backtest an `OrderBookImbalance` strategy or order book data. This example requires you bring your Binance own order book data.\n", "\n", "**Warning:**\n", "\n", diff --git a/examples/notebooks/backtest_example.ipynb b/examples/notebooks/backtest_example.ipynb index 98bfdaeb5cd5..29788baf61b2 100644 --- a/examples/notebooks/backtest_example.ipynb +++ b/examples/notebooks/backtest_example.ipynb @@ -5,7 +5,7 @@ "id": "0", "metadata": {}, "source": [ - "# Complete backtest using the data catalog and a BacktestNode (higher level)\n", + "# Complete backtest using the data catalog and a BacktestNode (high-level API)\n", "\n", "This example runs through how to setup the data catalog and a `BacktestNode` for a single 'one-shot' backtest run." ] diff --git a/examples/notebooks/backtest_fx_usdjpy.ipynb b/examples/notebooks/backtest_fx_usdjpy.ipynb index f03a5dbb2e46..d2c3242d840c 100644 --- a/examples/notebooks/backtest_fx_usdjpy.ipynb +++ b/examples/notebooks/backtest_fx_usdjpy.ipynb @@ -5,9 +5,9 @@ "id": "0", "metadata": {}, "source": [ - "# Complete backtest using a wrangler and BacktestEngine (lower level)\n", + "# Complete backtest using a wrangler and BacktestEngine (low-level API)\n", "\n", - "This example runs through how to setup a `BacktestEngine` for a single 'one-shot' backtest run." + "This tutorial runs through how to setup a `BacktestEngine` for a single 'one-shot' backtest run." ] }, { diff --git a/examples/notebooks/external_data_backtest.ipynb b/examples/notebooks/external_data_backtest.ipynb index 6fcebe19a2e8..9e05a1b045b1 100644 --- a/examples/notebooks/external_data_backtest.ipynb +++ b/examples/notebooks/external_data_backtest.ipynb @@ -7,7 +7,7 @@ "source": [ "# Loading external data\n", "\n", - "This example demonstrates how to load external data into the `ParquetDataCatalog`, and then use this to run a one-shot backtest using a `BacktestNode`.\n", + "This tutorial demonstrates how to load external data into the `ParquetDataCatalog`, and then use this to run a one-shot backtest using a `BacktestNode`.\n", "\n", "**Warning:**\n", "\n", diff --git a/examples/notebooks/parquet_explorer.ipynb b/examples/notebooks/parquet_explorer.ipynb index 4a1ec6acf513..ff6e4b2c0a41 100644 --- a/examples/notebooks/parquet_explorer.ipynb +++ b/examples/notebooks/parquet_explorer.ipynb @@ -7,7 +7,7 @@ "source": [ "# Parquet Explorer\n", "\n", - "In this example, we'll explore some basic query operations on Parquet files written by Nautilus. We'll utilize both the `datafusio`n and `pyarrow` libraries.\n", + "This tutorial explores some basic query operations on Parquet files written by Nautilus. We'll utilize both the `datafusio`n and `pyarrow` libraries.\n", "\n", "Before proceeding, ensure that you have `datafusion` installed. If not, you can install it by running:\n", "```bash\n", diff --git a/examples/notebooks/quick_start.ipynb b/examples/notebooks/quick_start.ipynb index f539b3f693fc..3a05d67bc0ee 100644 --- a/examples/notebooks/quick_start.ipynb +++ b/examples/notebooks/quick_start.ipynb @@ -7,9 +7,8 @@ "source": [ "# Quick Start\n", "\n", - "This guide explains how to get up and running with NautilusTrader backtesting with some\n", - "FX data. The Nautilus maintainers have pre-loaded some test data using the standard Nautilus persistence \n", - "format (Parquet) for this guide.\n", + "This tutorial steps through how to get up and running with NautilusTrader backtesting using FX data.\n", + "The Nautilus maintainers have pre-loaded some test data using the standard Nautilus persistence format (Parquet) for this guide.\n", "\n", "For more details on how to load data into Nautilus, see [Backtest Example]((https://docs.nautilustrader.io/guides/backtest_example.html) and [Loading External Data](https://docs.nautilustrader.io/guides/loading_external_data.html).)." ] From 2774a3646c512572a7076eb44cae492dcf529ae1 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 11 Feb 2024 17:23:26 +1100 Subject: [PATCH 027/104] Continue Databento data catalog tutorial --- .../notebooks/databento_data_catalog.ipynb | 316 +++++++++++++++--- 1 file changed, 273 insertions(+), 43 deletions(-) diff --git a/examples/notebooks/databento_data_catalog.ipynb b/examples/notebooks/databento_data_catalog.ipynb index 5f7b23009246..a11079005e7c 100644 --- a/examples/notebooks/databento_data_catalog.ipynb +++ b/examples/notebooks/databento_data_catalog.ipynb @@ -13,15 +13,45 @@ "id": "1", "metadata": {}, "source": [ - "This tutorial will walk through how to setup a Nautilus Parquet data catalog with databento order book data.\n", + "**Info:**\n", "\n", - "We choose to work with the MBP-10 schema (which is just an aggregation of the top 10 levels) so that the data is more manageable and easier to work with for the example." + "
\n", + "This tutorial is currently a work in progress (WIP).\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "This tutorial will walk through how to setup a Nautilus Parquet data catalog with various Databento schemas.\n", + "\n", + "Prerequities:\n", + "- The `databento` Python client library should be installed to make data requests `pip install -U databento`\n", + "- A Databento account (there is a free tier)" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## Requesting data" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "We'll use a Databento historical client for the rest of this tutorial. You can either initialize one by passing your Databento API key to the constructor, or implicitly use the `DATABENTO_API_KEY` environment variable (as shown)." ] }, { "cell_type": "code", "execution_count": null, - "id": "2", + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -32,52 +62,129 @@ }, { "cell_type": "markdown", - "id": "3", + "id": "6", "metadata": {}, "source": [ - "## Request data\n", - "\n", - "Use the historical API to request the front-month ES futures contract for January 2024.\n", + "**It's important to note that every historical streaming request from `timeseries.get_range` will incur a cost (even for the same data), therefore we need to:**\n", + "- Know and understand the cost prior to making a request\n", + "- Not make requests for the same data more than once (not efficient)\n", + "- Persist the responses to disk by writing zstd compressed DBN files (so that we don't have to request again)" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "We can use a metadata [get_cost endpoint](https://docs.databento.com/api-reference-historical/metadata/metadata-get-cost?historical=python&live=python) from the Databento API to get a quote on the cost, prior to each request.\n", + "Each request sequence will first request the cost of the data, and then make a request only if the data doesn't already exist on disk.\n", "\n", - "**CAUTION: This will incur a cost for every request (only run the request cell once)**" + "Note the response returned is in USD, displayed as fractional cents." + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "The following request is only for a small amount of data (as used in this Medium article [Building high-frequency trading signals in Python with Databento and sklearn](https://databento.com/blog/hft-sklearn-python)), just to demonstrate the basic workflow. " ] }, { "cell_type": "code", "execution_count": null, - "id": "4", + "id": "9", "metadata": {}, "outputs": [], "source": [ - "# Path we'll use for persisting this request to disk\n", - "path = \"es-front-glbx-mbp10.dbn.zst\"\n", - "\n", - "# Request lead month\n", - "data = client.timeseries.get_range(\n", + "from pathlib import Path\n", + "from databento import DBNStore" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "We'll prepare a directory for the raw Databento DBN format data, which we'll use for the rest of the tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "DATABENTO_DATA_DIR = Path(\"databento\")\n", + "DATABENTO_DATA_DIR.mkdir(exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# Request cost quote (USD) - this endpoint is 'free'\n", + "client.metadata.get_cost(\n", " dataset=\"GLBX.MDP3\",\n", " symbols=[\"ES.n.0\"],\n", " stype_in=\"continuous\",\n", " schema=\"mbp-10\",\n", " start=\"2023-12-06T14:30:00\",\n", " end=\"2023-12-06T20:30:00\",\n", - " path=path,\n", ")" ] }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "Use the historical API to request for the data used in the Medium article." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "14", "metadata": {}, "outputs": [], "source": [ + "path = DATABENTO_DATA_DIR / \"es-front-glbx-mbp10.dbn.zst\"\n", + "\n", + "if not path.exists():\n", + " # Request data\n", + " client.timeseries.get_range(\n", + " dataset=\"GLBX.MDP3\",\n", + " symbols=[\"ES.n.0\"],\n", + " stype_in=\"continuous\",\n", + " schema=\"mbp-10\",\n", + " start=\"2023-12-06T14:30:00\",\n", + " end=\"2023-12-06T20:30:00\",\n", + " path=path, # <--- Passing a `path` parameter will ensure the data is written to disk\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "# Inspect the data by reading from disk and convert to a pandas.DataFrame\n", + "data = DBNStore.from_file(path)\n", + "\n", "df = data.to_df()\n", "df" ] }, { "cell_type": "markdown", - "id": "6", + "id": "16", "metadata": {}, "source": [ "## Write to data catalog" @@ -86,7 +193,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -101,91 +208,214 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "CATALOG_PATH = Path.cwd() / \"catalog\"\n", + "\n", + "# Clear if it already exists\n", + "if CATALOG_PATH.exists():\n", + " shutil.rmtree(CATALOG_PATH)\n", + "CATALOG_PATH.mkdir()\n", + "\n", + "# Create a catalog instance\n", + "catalog = ParquetDataCatalog(CATALOG_PATH)" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "Now that we've prepared the data catalog, we need a `DatabentoDataLoader` which we'll use to decode and load the data into Nautilus objects." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", "metadata": {}, "outputs": [], "source": [ + "loader = DatabentoDataLoader()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "path = DATABENTO_DATA_DIR / \"es-front-glbx-mbp10.dbn.zst\"\n", "instrument_id = InstrumentId.from_str(\"ES.n.0\") # This should be the raw symbol (update)\n", - "loader = DatabentoDataLoader()\n", + "\n", "depth10 = loader.from_dbn_file(\n", " path=path,\n", " instrument_id=instrument_id, # Not required but makes data loading faster (symbology mapping not required)\n", - " as_legacy_cython=False, # This will load Rust pyo3 objects to write to the catalog\n", + " as_legacy_cython=False, # This will load Rust pyo3 objects to write to the catalog (we could use legacy Cython objects, but this is slightly more efficient)\n", ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "22", "metadata": {}, "outputs": [], "source": [ - "CATALOG_PATH = Path.cwd() / \"catalog\"\n", - "\n", - "# Clear if it already exists, then create fresh\n", - "if CATALOG_PATH.exists():\n", - " shutil.rmtree(CATALOG_PATH)\n", - "CATALOG_PATH.mkdir()\n", - "\n", - "# Create a catalog instance\n", - "catalog = ParquetDataCatalog(CATALOG_PATH)" + "# Write data to catalog (this takes ~20 seconds or ~250,000/second for writing MBP-10 at the moment)\n", + "catalog.write_data(depth10)" ] }, { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "23", "metadata": {}, "outputs": [], "source": [ - "# Write instrument and ticks to catalog (this takes ~20 seconds)\n", - "catalog.write_data(depth10)" + "# Test reading from catalog\n", + "depths = catalog.order_book_depth10()\n", + "len(depths)" ] }, { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "24", "metadata": {}, "outputs": [], "source": [] }, + { + "cell_type": "markdown", + "id": "25", + "metadata": {}, + "source": [ + "## Preparing a month of AAPL trades" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "Now we'll expand on this workflow by preparing a month of AAPL trades on the Nasdaq exchange using the Databento `trade` schema, which will translate to Nautilus `TradeTick` objects." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "27", "metadata": {}, "outputs": [], "source": [ - "import pyarrow.parquet as pq" + "# Request cost quote (USD) - this endpoint is 'free'\n", + "client.metadata.get_cost(\n", + " dataset=\"XNAS.ITCH\",\n", + " symbols=[\"AAPL\"],\n", + " schema=\"trades\",\n", + " start=\"2024-01\",\n", + ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "28", "metadata": {}, "outputs": [], "source": [ - "depth10_parquet_path = \"catalog/data/order_book_depth10/ES.n.0/part-0.parquet\"" + "path = DATABENTO_DATA_DIR / \"aapl-xnas-202401.trades.dbn.zst\"\n", + "\n", + "if not path.exists():\n", + " # Request data\n", + " client.timeseries.get_range(\n", + " dataset=\"XNAS.ITCH\",\n", + " symbols=[\"AAPL\"],\n", + " schema=\"trades\",\n", + " start=\"2024-01\",\n", + " path=path, # <--- Passing a `path` parameter will ensure the data is written to disk\n", + " )" ] }, { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "29", "metadata": {}, "outputs": [], "source": [ - "table = pq.read_table(depth10_parquet_path)\n", - "table.schema" + "# Inspect the data by reading from disk and convert to a pandas.DataFrame\n", + "data = DBNStore.from_file(path)\n", + "\n", + "df = data.to_df()\n", + "df" ] }, { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "instrument_id = InstrumentId.from_str(\"AAPL.XNAS\") # Using the Nasdaq ISO 10383 MIC (Market Identifier Code) as the venue\n", + "\n", + "trades = loader.from_dbn_file(\n", + " path=path,\n", + " instrument_id=instrument_id, # Not required but makes data loading faster (symbology mapping not required)\n", + " as_legacy_cython=False, # This will load Rust pyo3 objects to write to the catalog (we could use legacy Cython objects, but this is slightly more efficient)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "31", + "metadata": {}, + "source": [ + "Here we'll organize our data in a file per month, this is a rather arbitrary choice and a file per day could be equally valid.\n", + "\n", + "It may also be a good idea to create a function which can return the correct `basename_template` value for a given chunk of data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32", + "metadata": {}, + "outputs": [], + "source": [ + "# Write data to catalog\n", + "catalog.write_data(trades, basename_template=\"2024-01\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "trades = catalog.trade_ticks([instrument_id])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "len(trades)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", "metadata": {}, "outputs": [], "source": [] From 9055ef65530e961bf9fe79e2d2758dafaf10d7d8 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 12 Feb 2024 07:48:49 +1100 Subject: [PATCH 028/104] Update dependencies including chrono --- nautilus_core/Cargo.lock | 50 +++++++++++++++--------------- nautilus_core/Cargo.toml | 6 ++-- nautilus_core/core/src/datetime.rs | 8 ++--- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 05e2bdc6c804..537c5b5d2838 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -19,9 +19,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ "getrandom", "once_cell", @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" dependencies = [ "cfg-if", "const-random", @@ -166,7 +166,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow-buffer", "arrow-data", "arrow-schema", @@ -294,7 +294,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow-array", "arrow-buffer", "arrow-data", @@ -318,7 +318,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow-array", "arrow-buffer", "arrow-data", @@ -717,9 +717,9 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1127,7 +1127,7 @@ version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow", "arrow-array", "arrow-ipc", @@ -1175,7 +1175,7 @@ version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow", "arrow-array", "arrow-buffer", @@ -1217,7 +1217,7 @@ version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow", "arrow-array", "datafusion-common", @@ -1251,7 +1251,7 @@ version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow", "arrow-array", "arrow-buffer", @@ -1285,7 +1285,7 @@ version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow", "arrow-array", "arrow-buffer", @@ -1808,7 +1808,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", ] [[package]] @@ -1817,7 +1817,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "allocator-api2", ] @@ -2968,7 +2968,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "arrow-array", "arrow-buffer", "arrow-cast", @@ -3706,9 +3706,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.34.2" +version = "1.34.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755392e1a2f77afd95580d3f0d0e94ac83eeeb7167552c9b5bca549e61a94d83" +checksum = "b39449a79f45e8da28c57c341891b69a183044b29518bb8f86dbac9df60bb7df" dependencies = [ "arrayvec", "borsh", @@ -4189,7 +4189,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "atoi", "byteorder", "bytes", @@ -4565,18 +4565,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", @@ -5063,7 +5063,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e904a2279a4a36d2356425bb20be271029cc650c335bc82af8bfae30085a3d0" dependencies = [ - "ahash 0.8.7", + "ahash 0.8.8", "byteorder", "lazy_static", "parking_lot", diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index 8d897493da42..baffc1f5e5ad 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -25,7 +25,7 @@ documentation = "https://docs.nautilustrader.io" [workspace.dependencies] anyhow = "1.0.79" -chrono = "0.4.33" +chrono = "0.4.34" futures = "0.3.30" indexmap = "2.2.2" itoa = "1.0.10" @@ -36,12 +36,12 @@ pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime", "tokio", "attr rand = "0.8.5" redis = { version = "0.24.0", features = ["tokio-comp", "tls-rustls", "tokio-rustls-comp", "keep-alive", "connection-manager"] } rmp-serde = "1.1.2" -rust_decimal = "1.34.2" +rust_decimal = "1.34.3" rust_decimal_macros = "1.34.2" serde = { version = "1.0.196", features = ["derive"] } serde_json = "1.0.112" strum = { version = "0.25.0", features = ["derive"] } -thiserror = "1.0.56" +thiserror = "1.0.57" thousands = "0.2.0" tracing = "0.1.40" tokio = { version = "1.36.0", features = ["full"] } diff --git a/nautilus_core/core/src/datetime.rs b/nautilus_core/core/src/datetime.rs index 7adab03de47b..088e3646de5f 100644 --- a/nautilus_core/core/src/datetime.rs +++ b/nautilus_core/core/src/datetime.rs @@ -18,7 +18,7 @@ use std::time::{Duration, UNIX_EPOCH}; use anyhow::{anyhow, Result}; use chrono::{ prelude::{DateTime, Utc}, - Datelike, NaiveDate, SecondsFormat, Weekday, + Datelike, NaiveDate, SecondsFormat, TimeDelta, Weekday, }; use crate::time::UnixNanos; @@ -105,7 +105,7 @@ pub fn last_weekday_nanos(year: i32, month: u32, day: u32) -> Result }); // Calculate last closest weekday - let last_closest = date - chrono::Duration::days(offset); + let last_closest = date - TimeDelta::days(offset); // Convert to UNIX nanoseconds let unix_timestamp_ns = last_closest @@ -124,7 +124,7 @@ pub fn is_within_last_24_hours(timestamp_ns: UnixNanos) -> Result { .ok_or_else(|| anyhow!("Invalid timestamp {timestamp_ns}"))?; let now = Utc::now(); - Ok(now.signed_duration_since(timestamp) <= chrono::Duration::days(1)) + Ok(now.signed_duration_since(timestamp) <= TimeDelta::days(1)) } //////////////////////////////////////////////////////////////////////////////// @@ -261,7 +261,7 @@ mod tests { #[rstest] fn test_is_within_last_24_hours_when_two_days_ago() { - let past_ns = (Utc::now() - chrono::Duration::days(2)) + let past_ns = (Utc::now() - TimeDelta::days(2)) .timestamp_nanos_opt() .unwrap(); assert!(!is_within_last_24_hours(past_ns as UnixNanos).unwrap()); From 2b73774cca6fb8367e23f1e17f3a4f338568f334 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 12 Feb 2024 13:36:13 +1100 Subject: [PATCH 029/104] Fix call to clock.utc_now --- nautilus_trader/adapters/databento/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 611b3fe85bfb..dc1741805fe1 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -490,7 +490,7 @@ async def _subscribe_order_book_deltas_batch( live_client = self._get_live_client_mbo(dataset) # Subscribe from UTC midnight snapshot - start = self._clock.utcnow().normalize().value + start = self._clock.utc_now().normalize().value future = asyncio.ensure_future( live_client.subscribe( From faae332a7350417c499a9fe8927c74ab367149c3 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 12 Feb 2024 13:47:21 +1100 Subject: [PATCH 030/104] Refine DatabentoLiveClient record handling loop --- nautilus_core/adapters/src/databento/python/live.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index b3de8ea40a77..5f1299dd6333 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -286,7 +286,7 @@ fn handle_record( let price_precision = 2; // Hard coded for now let ts_init = clock.get_time_ns(); - let (data, maybe_data) = decode_record( + let (data1, data2) = decode_record( &record, instrument_id, price_precision, @@ -295,11 +295,11 @@ fn handle_record( )?; Python::with_gil(|py| { - if let Some(data) = data { + if let Some(data) = data1 { call_python_with_data(py, callback, data); } - if let Some(data) = maybe_data { + if let Some(data) = data2 { call_python_with_data(py, callback, data); } }); From ed4083514609d3aaa3dc60217f8a1076d7a72e94 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 12 Feb 2024 17:45:47 +1100 Subject: [PATCH 031/104] Fix ControllerConfig base and docstring --- RELEASES.md | 1 + nautilus_trader/live/config.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index cad598009f45..0e9472a8d2de 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -13,6 +13,7 @@ None - Fixed `ImportableConfig.create` JSON encoding (was missing the encoding hook) - Fixed `ImportableStrategyConfig.create` JSON encoding (was missing the encoding hook) - Fixed `ExecAlgorithmFactory.create` JSON encoding (was missing the encoding hook) +- Fixed `ControllerConfig` base class and docstring --- diff --git a/nautilus_trader/live/config.py b/nautilus_trader/live/config.py index a3e77fa2f0cb..c355b79275ae 100644 --- a/nautilus_trader/live/config.py +++ b/nautilus_trader/live/config.py @@ -18,6 +18,7 @@ import msgspec from nautilus_trader.common import Environment +from nautilus_trader.common.config import ActorConfig from nautilus_trader.common.config import InstrumentProviderConfig from nautilus_trader.common.config import NautilusConfig from nautilus_trader.common.config import NonNegativeInt @@ -161,9 +162,9 @@ class LiveExecClientConfig(NautilusConfig, frozen=True): routing: RoutingConfig = RoutingConfig() -class ControllerConfig(NautilusConfig, kw_only=True, frozen=True): +class ControllerConfig(ActorConfig, kw_only=True, frozen=True): """ - The base model for all trading strategy configurations. + The base model for all controller configurations. """ From e78bb97a83e8d464fc3bee036e83e9c7bc0c5985 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 12 Feb 2024 19:08:16 +1100 Subject: [PATCH 032/104] Improve DatabentoDataClient startup sequence --- .../live/databento/databento_subscriber.py | 1 + .../adapters/src/databento/python/live.rs | 25 +++++++------- nautilus_trader/adapters/databento/config.py | 4 +-- nautilus_trader/adapters/databento/data.py | 34 +++++++++++++++---- .../adapters/databento/providers.py | 15 +++++--- 5 files changed, 53 insertions(+), 26 deletions(-) diff --git a/examples/live/databento/databento_subscriber.py b/examples/live/databento/databento_subscriber.py index d5bd95dbffd8..af2468651a8b 100644 --- a/examples/live/databento/databento_subscriber.py +++ b/examples/live/databento/databento_subscriber.py @@ -45,6 +45,7 @@ instrument_ids = [ InstrumentId.from_str("ESH4.GLBX"), # InstrumentId.from_str("ESM4.GLBX"), + # InstrumentId.from_str("ESU4.GLBX"), # InstrumentId.from_str("AAPL.XCHI"), ] diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 5f1299dd6333..23c1a5e571f4 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -17,7 +17,7 @@ use std::fs; use std::str::FromStr; use std::sync::Arc; -use anyhow::Result; +use anyhow::{bail, Result}; use databento::live::Subscription; use dbn::{PitSymbolMap, Record, SymbolIndex, VersionUpgradePolicy}; use indexmap::IndexMap; @@ -196,7 +196,8 @@ impl DatabentoLiveClient { } else if let Some(msg) = record.get::() { handle_symbol_mapping_msg(msg, &mut symbol_map); } else if let Some(msg) = record.get::() { - handle_instrument_def_msg(msg, &publisher_venue_map, clock, &callback); + handle_instrument_def_msg(msg, &publisher_venue_map, clock, &callback) + .map_err(to_pyvalue_err)?; } else { handle_record(record, &symbol_map, &publisher_venue_map, clock, &callback) .map_err(to_pyvalue_err)?; @@ -245,7 +246,7 @@ fn handle_instrument_def_msg( publisher_venue_map: &IndexMap, clock: &AtomicTime, callback: &PyObject, -) { +) -> Result<()> { let raw_symbol = unsafe { raw_ptr_to_ustr(msg.raw_symbol.as_ptr()).unwrap() }; let symbol = Symbol { value: raw_symbol }; let venue = publisher_venue_map.get(&msg.hd.publisher_id).unwrap(); @@ -255,16 +256,14 @@ fn handle_instrument_def_msg( let result = decode_instrument_def_msg(msg, instrument_id, ts_init); match result { - Ok(instrument) => { - Python::with_gil(|py| { - let py_obj = convert_instrument_to_pyobject(py, instrument).unwrap(); - match callback.call1(py, (py_obj,)) { - Ok(_) => {} - Err(e) => eprintln!("Error on callback, {e:?}"), // Just print error for now - }; - }); - } - Err(e) => eprintln!("{e:?}"), + Ok(instrument) => Python::with_gil(|py| { + let py_obj = convert_instrument_to_pyobject(py, instrument).unwrap(); + match callback.call1(py, (py_obj,)) { + Ok(_) => Ok(()), + Err(e) => bail!(e), + } + }), + Err(e) => Err(e), } } diff --git a/nautilus_trader/adapters/databento/config.py b/nautilus_trader/adapters/databento/config.py index 64c7efd6a6c7..0d2f16fbbc55 100644 --- a/nautilus_trader/adapters/databento/config.py +++ b/nautilus_trader/adapters/databento/config.py @@ -38,7 +38,7 @@ class DatabentoDataClientConfig(LiveDataClientConfig, frozen=True): The instrument IDs to request instrument definitions for on start. timeout_initial_load : float, default 5.0 The timeout (seconds) to wait for instruments to load (concurrently per dataset). - mbo_subscriptions_delay : float, default 2.0 + mbo_subscriptions_delay : float, default 3.0 The timeout (seconds) to wait for MBO/L3 subscriptions (concurrently per dataset). After the timeout the MBO order book feed will start and replay messages from the start of the week which encompasses the initial snapshot and then all deltas. @@ -51,4 +51,4 @@ class DatabentoDataClientConfig(LiveDataClientConfig, frozen=True): instrument_ids: list[InstrumentId] | None = None parent_symbols: dict[str, set[str]] | None = None timeout_initial_load: float | None = 5.0 - mbo_subscriptions_delay: float | None = 3.0 + mbo_subscriptions_delay: float | None = 3.0 # Need to have received all definitions diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index dc1741805fe1..06efffc362c6 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -140,6 +140,7 @@ def __init__( self._is_buffering_mbo_subscriptions: bool = bool(config.mbo_subscriptions_delay) self._buffered_mbo_subscriptions: dict[Dataset, list[InstrumentId]] = defaultdict(list) self._buffered_deltas: dict[InstrumentId, list[OrderBookDelta]] = defaultdict(list) + self._buffering_replay: dict[InstrumentId, int] = {} # Tasks self._live_client_futures: set[asyncio.Future] = set() @@ -486,17 +487,24 @@ async def _subscribe_order_book_deltas_batch( ids_str = ",".join([i.value for i in instrument_ids]) self._log.info(f"Subscribing to MBO/L3 for {ids_str}.", LogColor.BLUE) + # Setup buffered start times + now = self._clock.utc_now() + for instrument_id in instrument_ids: + self._buffering_replay[instrument_id] = now.value + dataset: Dataset = self._loader.get_dataset_for_venue(instrument_ids[0].venue) live_client = self._get_live_client_mbo(dataset) # Subscribe from UTC midnight snapshot - start = self._clock.utc_now().normalize().value + start = self._clock.utc_now().normalize() + + self._log.info(f"Replaying MBO/L3 feeds from {start}.", LogColor.BLUE) future = asyncio.ensure_future( live_client.subscribe( schema=DatabentoSchema.MBO.value, symbols=",".join(sorted([i.symbol.value for i in instrument_ids])), - start=start, + start=start.value, ), ) self._live_client_futures.add(future) @@ -866,15 +874,29 @@ def _handle_record( data = capsule_to_data(pycapsule) if isinstance(data, OrderBookDelta): + # Assign instrument_id to avoid continually fetching the C string instrument_id = data.instrument_id - if DatabentoRecordFlags.F_LAST not in DatabentoRecordFlags(data.flags): + + buffer_start_ns = self._buffering_replay.get(instrument_id, 0) + if buffer_start_ns or DatabentoRecordFlags.F_LAST in DatabentoRecordFlags(data.flags): buffer = self._buffered_deltas[instrument_id] buffer.append(data) - return # We can rely on the F_LAST flag for an MBO feed + + if buffer_start_ns > 0: + if data.ts_event >= buffer_start_ns: + self._buffering_replay.pop(instrument_id) + self._log.info(f"MBO replay complete for {instrument_id}.", LogColor.BLUE) + else: + # Uncomment blow for debugging/development + # latency = self._clock.timestamp_ns() - data.ts_init + # self._log.warning(f"{len(buffer)} {instrument_id}: {latency}") + return # Still replaying start + + data = OrderBookDeltas(instrument_id, deltas=buffer.copy()) + buffer.clear() else: buffer = self._buffered_deltas[instrument_id] buffer.append(data) - data = OrderBookDeltas(instrument_id, deltas=buffer.copy()) - buffer.clear() + return # We can rely on the F_LAST flag for an MBO feed self._handle_data(data) diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index 5807694fc466..5ff46034dc2c 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -124,13 +124,13 @@ async def load_ids_async( ) pyo3_instruments = [] + success_msg = "All instruments received and decoded." def receive_instruments(pyo3_instrument: Any) -> None: pyo3_instruments.append(pyo3_instrument) instrument_ids_to_decode.discard(pyo3_instrument.id.value) - # TODO: Improve how to handle decode completion - # if not instrument_ids_to_decode: - # raise asyncio.CancelledError("All instruments decoded") + if not instrument_ids_to_decode: + raise asyncio.CancelledError(success_msg) await live_client.subscribe( schema=DatabentoSchema.DEFINITION.value, @@ -140,8 +140,13 @@ def receive_instruments(pyo3_instrument: Any) -> None: try: await asyncio.wait_for(live_client.start(callback=receive_instruments), timeout=5.0) - except asyncio.CancelledError: - pass # Expected on decode completion, continue + # TODO: Improve this so that `live_client.start` isn't raising a `ValueError` + except ValueError as e: + if success_msg in str(e): + # Expected on decode completion, continue + self._log.info(success_msg) + else: + self._log.error(repr(e)) instruments = instruments_from_pyo3(pyo3_instruments) From f11de018a1eaf21ffc580fb17c66161797f40037 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 12 Feb 2024 19:48:35 +1100 Subject: [PATCH 033/104] Implement Databento parent symbols on startup --- nautilus_trader/adapters/databento/data.py | 7 ++++++- nautilus_trader/adapters/databento/providers.py | 14 +++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 06efffc362c6..470b66eeb0a2 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -159,7 +159,12 @@ async def _connect(self) -> None: coros: list[Coroutine] = [] for dataset, instrument_ids in self._instrument_ids.items(): loading_ids: list[InstrumentId] = sorted(instrument_ids) - coros.append(self._instrument_provider.load_ids_async(instrument_ids=loading_ids)) + filters = {"parent_symbols": list(self._parent_symbols.get(dataset, []))} + coro = self._instrument_provider.load_ids_async( + instrument_ids=loading_ids, + filters=filters, + ) + coros.append(coro) await self._subscribe_instrument_ids(dataset, instrument_ids=loading_ids) try: diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index 5ff46034dc2c..7b7198c48c41 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -25,6 +25,7 @@ from nautilus_trader.adapters.databento.enums import DatabentoSchema from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.common.component import LiveClock +from nautilus_trader.common.enums import LogColor from nautilus_trader.common.providers import InstrumentProvider from nautilus_trader.config import InstrumentProviderConfig from nautilus_trader.core import nautilus_pyo3 @@ -123,13 +124,15 @@ async def load_ids_async( publishers_path=str(PUBLISHERS_PATH), ) + parent_symbols = list(filters.get("parent_symbols", [])) if filters is not None else None + pyo3_instruments = [] success_msg = "All instruments received and decoded." def receive_instruments(pyo3_instrument: Any) -> None: pyo3_instruments.append(pyo3_instrument) instrument_ids_to_decode.discard(pyo3_instrument.id.value) - if not instrument_ids_to_decode: + if not parent_symbols and not instrument_ids_to_decode: raise asyncio.CancelledError(success_msg) await live_client.subscribe( @@ -138,6 +141,15 @@ def receive_instruments(pyo3_instrument: Any) -> None: start=0, # From start of current week (latest definitions) ) + if parent_symbols: + self._log.info(f"Requesting parent symbols {parent_symbols}.", LogColor.BLUE) + await live_client.subscribe( + schema=DatabentoSchema.DEFINITION.value, + stype_in="parent", + symbols=",".join(parent_symbols), + start=0, # From start of current week (latest definitions) + ) + try: await asyncio.wait_for(live_client.start(callback=receive_instruments), timeout=5.0) # TODO: Improve this so that `live_client.start` isn't raising a `ValueError` From 5b68874382e7c402b5c3992f13c91a92f3a9e8ad Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 13 Feb 2024 16:44:04 +1100 Subject: [PATCH 034/104] Update dependencies and pre-commit --- .pre-commit-config.yaml | 2 +- nautilus_core/Cargo.lock | 72 +++++++++++++++------------------- nautilus_core/Cargo.toml | 2 +- nautilus_core/model/Cargo.toml | 2 +- poetry.lock | 62 ++++++++++++++--------------- pyproject.toml | 2 +- 6 files changed, 66 insertions(+), 76 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 926ebe70a264..a9c30ee16007 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: types: [python] - repo: https://github.com/psf/black - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black types_or: [python, pyi] diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 537c5b5d2838..b1a55b305ce4 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -266,7 +266,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.2.2", + "indexmap 2.2.3", "lexical-core", "num", "serde", @@ -695,11 +695,10 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "9b918671670962b48bc23753aef0c51d072dca6f52f01f800854ada6ddb7f7d3" dependencies = [ - "jobserver", "libc", ] @@ -731,9 +730,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d7b79e99bfaa0d47da0687c43aa3b7381938a62ad3a6498599039321f660b7" +checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", "chrono-tz-build", @@ -932,9 +931,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1150,7 +1149,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.3", - "indexmap 2.2.2", + "indexmap 2.2.3", "itertools 0.12.1", "log", "num_cpus", @@ -1266,7 +1265,7 @@ dependencies = [ "half", "hashbrown 0.14.3", "hex", - "indexmap 2.2.2", + "indexmap 2.2.3", "itertools 0.12.1", "log", "md-5", @@ -1299,7 +1298,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.3", - "indexmap 2.2.2", + "indexmap 2.2.3", "itertools 0.12.1", "log", "once_cell", @@ -1381,18 +1380,18 @@ dependencies = [ [[package]] name = "derive_builder" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "660047478bc508c0fde22c868991eec0c40a63e48d610befef466d48e2bee574" +checksum = "8f59169f400d8087f238c5c0c7db6a28af18681717f3b623227d92f397e938c7" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b217e6dd1011a54d12f3b920a411b5abd44b1716ecfe94f5f2f2f7b52e08ab7" +checksum = "a4ec317cc3e7ef0928b0ca6e4a634a4d6c001672ae210438cf114a83e56b018d" dependencies = [ "darling", "proc-macro2", @@ -1402,9 +1401,9 @@ dependencies = [ [[package]] name = "derive_builder_macro" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5f77d7e20ac9153428f7ca14a88aba652adfc7a0ef0a06d654386310ef663b" +checksum = "870368c3fb35b8031abb378861d4460f573b92238ec2152c927a21f77e3e0127" dependencies = [ "derive_builder_core", "syn 1.0.109", @@ -1765,7 +1764,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.2.2", + "indexmap 2.2.3", "slab", "tokio", "tokio-util", @@ -1784,7 +1783,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.2.2", + "indexmap 2.2.3", "slab", "tokio", "tokio-util", @@ -2090,9 +2089,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.2" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2151,15 +2150,6 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" -[[package]] -name = "jobserver" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.68" @@ -2436,7 +2426,7 @@ dependencies = [ "criterion", "databento", "dbn", - "indexmap 2.2.2", + "indexmap 2.2.3", "itoa", "log", "nautilus-common", @@ -2479,7 +2469,7 @@ dependencies = [ "anyhow", "cbindgen", "chrono", - "indexmap 2.2.2", + "indexmap 2.2.3", "log", "nautilus-core", "nautilus-model", @@ -2554,7 +2544,7 @@ dependencies = [ "evalexpr", "float-cmp", "iai", - "indexmap 2.2.2", + "indexmap 2.2.3", "nautilus-core", "once_cell", "pyo3", @@ -2887,9 +2877,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.2+3.2.1" +version = "300.2.3+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbfad0063610ac26ee79f7484739e2b07555a75c42453b89263830b5c8103bc" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" dependencies = [ "cc", ] @@ -3034,7 +3024,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.2", + "indexmap 2.2.3", ] [[package]] @@ -4205,7 +4195,7 @@ dependencies = [ "futures-util", "hashlink", "hex", - "indexmap 2.2.2", + "indexmap 2.2.3", "log", "memchr", "once_cell", @@ -4802,7 +4792,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.2", + "indexmap 2.2.3", "toml_datetime", "winnow", ] @@ -4929,7 +4919,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" version = "0.21.0" -source = "git+https://github.com/snapview/tungstenite-rs#2ee05d10803d95ad48b3ad03d9d9a03164060e76" +source = "git+https://github.com/snapview/tungstenite-rs#0fa41973b4c075f5d4a9e03a82a26a301ca31ce9" dependencies = [ "byteorder", "bytes", @@ -5422,9 +5412,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.39" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5389a154b01683d28c77f8f68f49dea75f0a4da32557a58f68ee51ebba472d29" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index baffc1f5e5ad..d43bf003c944 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -27,7 +27,7 @@ documentation = "https://docs.nautilustrader.io" anyhow = "1.0.79" chrono = "0.4.34" futures = "0.3.30" -indexmap = "2.2.2" +indexmap = "2.2.3" itoa = "1.0.10" once_cell = "1.19.0" log = { version = "0.4.20", features = ["std", "kv_unstable", "serde", "release_max_level_debug"] } diff --git a/nautilus_core/model/Cargo.toml b/nautilus_core/model/Cargo.toml index 6ab77756e2d5..eba1f63c494f 100644 --- a/nautilus_core/model/Cargo.toml +++ b/nautilus_core/model/Cargo.toml @@ -26,7 +26,7 @@ thiserror = { workspace = true } thousands = { workspace = true } ustr = { workspace = true } chrono = { workspace = true } -derive_builder = "0.13.0" +derive_builder = "0.13.1" evalexpr = "11.3.0" tabled = "0.15.0" diff --git a/poetry.lock b/poetry.lock index a69e14c93d9d..a25eac6e26da 100644 --- a/poetry.lock +++ b/poetry.lock @@ -202,33 +202,33 @@ msgspec = ">=0.18.5" [[package]] name = "black" -version = "24.1.1" +version = "24.2.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c"}, - {file = "black-24.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445"}, - {file = "black-24.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a"}, - {file = "black-24.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4"}, - {file = "black-24.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7"}, - {file = "black-24.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8"}, - {file = "black-24.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161"}, - {file = "black-24.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d"}, - {file = "black-24.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8"}, - {file = "black-24.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e"}, - {file = "black-24.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6"}, - {file = "black-24.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b"}, - {file = "black-24.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62"}, - {file = "black-24.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5"}, - {file = "black-24.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6"}, - {file = "black-24.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717"}, - {file = "black-24.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9"}, - {file = "black-24.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024"}, - {file = "black-24.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2"}, - {file = "black-24.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac"}, - {file = "black-24.1.1-py3-none-any.whl", hash = "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168"}, - {file = "black-24.1.1.tar.gz", hash = "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b"}, + {file = "black-24.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29"}, + {file = "black-24.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430"}, + {file = "black-24.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f"}, + {file = "black-24.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a"}, + {file = "black-24.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd"}, + {file = "black-24.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2"}, + {file = "black-24.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92"}, + {file = "black-24.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23"}, + {file = "black-24.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b"}, + {file = "black-24.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9"}, + {file = "black-24.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693"}, + {file = "black-24.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982"}, + {file = "black-24.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4"}, + {file = "black-24.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218"}, + {file = "black-24.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0"}, + {file = "black-24.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d"}, + {file = "black-24.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8"}, + {file = "black-24.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8"}, + {file = "black-24.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540"}, + {file = "black-24.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31"}, + {file = "black-24.2.0-py3-none-any.whl", hash = "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6"}, + {file = "black-24.2.0.tar.gz", hash = "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894"}, ] [package.dependencies] @@ -1955,18 +1955,18 @@ files = [ [[package]] name = "setuptools" -version = "69.0.3" +version = "69.1.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, - {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, + {file = "setuptools-69.1.0-py3-none-any.whl", hash = "sha256:c054629b81b946d63a9c6e732bc8b2513a7c3ea645f11d0139a2191d735c60c6"}, + {file = "setuptools-69.1.0.tar.gz", hash = "sha256:850894c4195f09c4ed30dba56213bf7c3f21d86ed6bdaafb5df5972593bfc401"}, ] [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -2345,13 +2345,13 @@ files = [ [[package]] name = "tzdata" -version = "2023.4" +version = "2024.1" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, - {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, ] [[package]] @@ -2595,4 +2595,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "294c31ca1c41d6d8816411e1af9c0336d9b6b01dceef6aaa066061362522ca51" +content-hash = "139aafb980dbeb5983f778be70184f745617a831bb7e7952cc001cacba820146" diff --git a/pyproject.toml b/pyproject.toml index 67c960d4ef07..c6833506a08f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ ib = ["nautilus_ibapi", "async-timeout", "defusedxml"] optional = true [tool.poetry.group.dev.dependencies] -black = "^24.1.1" +black = "^24.2.0" docformatter = "^1.7.5" mypy = "^1.8.0" pandas-stubs = "^2.1.4" From 753b769a98f98d89a3cfa144b4f372237957dce2 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 13 Feb 2024 16:58:11 +1100 Subject: [PATCH 035/104] Standardize attribute ordering --- nautilus_core/common/src/ffi/clock.rs | 4 ++-- nautilus_core/common/src/ffi/msgbus.rs | 2 +- nautilus_core/model/src/ffi/instruments/synthetic.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nautilus_core/common/src/ffi/clock.rs b/nautilus_core/common/src/ffi/clock.rs index cea96cc17186..95eb7c7d8f9c 100644 --- a/nautilus_core/common/src/ffi/clock.rs +++ b/nautilus_core/common/src/ffi/clock.rs @@ -42,8 +42,8 @@ use crate::{ /// It implements the `Deref` trait, allowing instances of `TestClock_API` to be /// dereferenced to `TestClock`, providing access to `TestClock`'s methods without /// having to manually access the underlying `TestClock` instance. -#[allow(non_camel_case_types)] #[repr(C)] +#[allow(non_camel_case_types)] pub struct TestClock_API(Box); impl Deref for TestClock_API { @@ -251,8 +251,8 @@ pub extern "C" fn test_clock_cancel_timers(clock: &mut TestClock_API) { /// dereferenced to `LiveClock`, providing access to `LiveClock`'s methods without /// having to manually access the underlying `LiveClock` instance. This includes /// both mutable and immutable access. -#[allow(non_camel_case_types)] #[repr(C)] +#[allow(non_camel_case_types)] pub struct LiveClock_API(Box); impl Deref for LiveClock_API { diff --git a/nautilus_core/common/src/ffi/msgbus.rs b/nautilus_core/common/src/ffi/msgbus.rs index aad12a2d13d3..1c944bc7f325 100644 --- a/nautilus_core/common/src/ffi/msgbus.rs +++ b/nautilus_core/common/src/ffi/msgbus.rs @@ -48,8 +48,8 @@ use crate::{ /// It implements the `Deref` trait, allowing instances of `MessageBus_API` to be /// dereferenced to `MessageBus`, providing access to `TestClock`'s methods without /// having to manually access the underlying `MessageBus` instance. -#[allow(non_camel_case_types)] #[repr(C)] +#[allow(non_camel_case_types)] pub struct MessageBus_API(Box); impl Deref for MessageBus_API { diff --git a/nautilus_core/model/src/ffi/instruments/synthetic.rs b/nautilus_core/model/src/ffi/instruments/synthetic.rs index 7e096b9ea703..821e023c4ec0 100644 --- a/nautilus_core/model/src/ffi/instruments/synthetic.rs +++ b/nautilus_core/model/src/ffi/instruments/synthetic.rs @@ -42,8 +42,8 @@ use crate::{ /// It implements the `Deref` trait, allowing instances of `SyntheticInstrument_API` to be /// dereferenced to `SyntheticInstrument`, providing access to `SyntheticInstruments`'s methods without /// having to manually access the underlying instance. -#[allow(non_camel_case_types)] #[repr(C)] +#[allow(non_camel_case_types)] pub struct SyntheticInstrument_API(Box); impl Deref for SyntheticInstrument_API { From 0009f3a12dc125354ee396d689689a7b101f5845 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 13 Feb 2024 20:05:57 +1100 Subject: [PATCH 036/104] Implement OrderBookDeltas FFI and pyo3 interfaces --- nautilus_core/model/src/data/deltas.rs | 27 ++-- nautilus_core/model/src/data/mod.rs | 3 +- nautilus_core/model/src/ffi/data/deltas.rs | 118 ++++++++++++++ nautilus_core/model/src/ffi/data/mod.rs | 1 + nautilus_core/model/src/python/data/deltas.rs | 126 +++++++++++++++ nautilus_core/model/src/python/data/mod.rs | 1 + nautilus_trader/core/includes/model.h | 50 ++++++ nautilus_trader/core/rust/model.pxd | 44 ++++++ nautilus_trader/model/data.pxd | 14 +- nautilus_trader/model/data.pyx | 148 +++++++++++++++++- .../tracemalloc_orderbook_delta.py | 37 +++++ .../tracemalloc_orderbook_deltas.py | 19 +-- 12 files changed, 543 insertions(+), 45 deletions(-) create mode 100644 nautilus_core/model/src/ffi/data/deltas.rs create mode 100644 nautilus_core/model/src/python/data/deltas.rs create mode 100644 tests/mem_leak_tests/tracemalloc_orderbook_delta.py diff --git a/nautilus_core/model/src/data/deltas.rs b/nautilus_core/model/src/data/deltas.rs index c6a66745be22..1d8bdd0e4aee 100644 --- a/nautilus_core/model/src/data/deltas.rs +++ b/nautilus_core/model/src/data/deltas.rs @@ -22,7 +22,8 @@ use super::delta::OrderBookDelta; use crate::identifiers::instrument_id::InstrumentId; /// Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. -#[repr(C)] +/// +/// This type cannot be `repr(C)` due to the `deltas` vec. #[derive(Clone, Debug)] #[cfg_attr( feature = "python", @@ -46,14 +47,14 @@ pub struct OrderBookDeltas { impl OrderBookDeltas { #[allow(clippy::too_many_arguments)] #[must_use] - pub fn new( - instrument_id: InstrumentId, - deltas: Vec, - flags: u8, - sequence: u64, - ts_event: UnixNanos, - ts_init: UnixNanos, - ) -> Self { + pub fn new(instrument_id: InstrumentId, deltas: Vec) -> Self { + assert!(!deltas.is_empty(), "`deltas` cannot be empty"); + // SAFETY: We asserted `deltas` is not empty + let last = deltas.last().unwrap(); + let flags = last.flags; + let sequence = last.sequence; + let ts_event = last.ts_event; + let ts_init = last.ts_init; Self { instrument_id, deltas, @@ -65,7 +66,7 @@ impl OrderBookDeltas { } } -// TODO: Potentially implement later +// TODO: Implement // impl Serializable for OrderBookDeltas {} // TODO: Exact format for Debug and Display TBD @@ -195,7 +196,7 @@ pub mod stubs { let deltas = vec![delta0, delta1, delta2, delta3, delta4, delta5, delta6]; - OrderBookDeltas::new(instrument_id, deltas, flags, sequence, ts_event, ts_init) + OrderBookDeltas::new(instrument_id, deltas) } } @@ -310,10 +311,6 @@ mod tests { let deltas = OrderBookDeltas::new( instrument_id, vec![delta0, delta1, delta2, delta3, delta4, delta5, delta6], - flags, - sequence, - ts_event, - ts_init, ); assert_eq!(deltas.instrument_id, instrument_id); diff --git a/nautilus_core/model/src/data/mod.rs b/nautilus_core/model/src/data/mod.rs index 9ac16901fc1a..e354fb78e182 100644 --- a/nautilus_core/model/src/data/mod.rs +++ b/nautilus_core/model/src/data/mod.rs @@ -30,7 +30,6 @@ use self::{ #[repr(C)] #[derive(Clone, Debug)] -#[cfg_attr(feature = "trivial_copy", derive(Copy))] #[allow(clippy::large_enum_variant)] // TODO: Optimize this (largest variant 1008 vs 136 bytes) pub enum Data { Delta(OrderBookDelta), @@ -129,5 +128,5 @@ impl From for Data { #[no_mangle] pub extern "C" fn data_clone(data: &Data) -> Data { - *data // Actually a copy + data.clone() } diff --git a/nautilus_core/model/src/ffi/data/deltas.rs b/nautilus_core/model/src/ffi/data/deltas.rs new file mode 100644 index 000000000000..f52632de71e2 --- /dev/null +++ b/nautilus_core/model/src/ffi/data/deltas.rs @@ -0,0 +1,118 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::ops::{Deref, DerefMut}; + +use nautilus_core::{ffi::cvec::CVec, time::UnixNanos}; + +use crate::{ + data::{delta::OrderBookDelta, deltas::OrderBookDeltas}, + enums::BookAction, + identifiers::instrument_id::InstrumentId, +}; + +/// Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. +/// +/// This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function +/// calls, enabling interaction with `OrderBookDeltas` in a C environment. +/// +/// It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be +/// dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without +/// having to manually access the underlying `OrderBookDeltas` instance. +#[repr(C)] +#[allow(non_camel_case_types)] +pub struct OrderBookDeltas_API(Box); + +impl Deref for OrderBookDeltas_API { + type Target = OrderBookDeltas; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for OrderBookDeltas_API { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// Creates a new `OrderBookDeltas` object from a CVec of `OrderBookDelta`. +/// +/// # Safety +/// - The `deltas` must be a valid pointer to a `CVec` containing `OrderBookDelta` objects +/// - This function clones the data pointed to by `deltas` into Rust-managed memory, then forgets the original `Vec` to prevent Rust from auto-deallocating it +/// - The caller is responsible for managing the memory of `deltas` (including its deallocation) to avoid memory leaks +#[no_mangle] +pub extern "C" fn orderbook_deltas_new( + instrument_id: InstrumentId, + deltas: &CVec, +) -> OrderBookDeltas_API { + let CVec { ptr, len, cap } = *deltas; + let deltas: Vec = + unsafe { Vec::from_raw_parts(ptr as *mut OrderBookDelta, len, cap) }; + let cloned_deltas = deltas.clone(); + std::mem::forget(deltas); // Prevents Rust from dropping `deltas` + OrderBookDeltas_API(Box::new(OrderBookDeltas::new(instrument_id, cloned_deltas))) +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_drop(deltas: OrderBookDeltas_API) { + drop(deltas); // Memory freed here +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_instrument_id(deltas: &OrderBookDeltas_API) -> InstrumentId { + deltas.instrument_id +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_vec_deltas(deltas: &OrderBookDeltas_API) -> CVec { + deltas.deltas.clone().into() +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_is_snapshot(deltas: &OrderBookDeltas_API) -> u8 { + u8::from(deltas.deltas[0].action == BookAction::Clear) +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_flags(deltas: &OrderBookDeltas_API) -> u8 { + deltas.flags +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_sequence(deltas: &OrderBookDeltas_API) -> u64 { + deltas.sequence +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_ts_event(deltas: &OrderBookDeltas_API) -> UnixNanos { + deltas.ts_event +} + +#[no_mangle] +pub extern "C" fn orderbook_deltas_ts_init(deltas: &OrderBookDeltas_API) -> UnixNanos { + deltas.ts_init +} + +#[allow(clippy::drop_non_drop)] +#[no_mangle] +pub extern "C" fn orderbook_deltas_vec_drop(v: CVec) { + let CVec { ptr, len, cap } = v; + let deltas: Vec = + unsafe { Vec::from_raw_parts(ptr as *mut OrderBookDelta, len, cap) }; + drop(deltas); // Memory freed here +} diff --git a/nautilus_core/model/src/ffi/data/mod.rs b/nautilus_core/model/src/ffi/data/mod.rs index e1a81c7edec2..ce93bb068149 100644 --- a/nautilus_core/model/src/ffi/data/mod.rs +++ b/nautilus_core/model/src/ffi/data/mod.rs @@ -15,6 +15,7 @@ pub mod bar; pub mod delta; +pub mod deltas; pub mod depth; pub mod order; pub mod quote; diff --git a/nautilus_core/model/src/python/data/deltas.rs b/nautilus_core/model/src/python/data/deltas.rs new file mode 100644 index 000000000000..fcd7b8acfb81 --- /dev/null +++ b/nautilus_core/model/src/python/data/deltas.rs @@ -0,0 +1,126 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +// use std::{ +// collections::{hash_map::DefaultHasher, HashMap}, +// hash::{Hash, Hasher}, +// }; + +use nautilus_core::time::UnixNanos; +use pyo3::prelude::*; + +use crate::{ + data::{delta::OrderBookDelta, deltas::OrderBookDeltas}, + identifiers::instrument_id::InstrumentId, + python::PY_MODULE_MODEL, +}; + +#[pymethods] +impl OrderBookDeltas { + #[new] + fn py_new(instrument_id: InstrumentId, deltas: Vec) -> Self { + Self::new(instrument_id, deltas) + } + + // TODO: Implement + // fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { + // match op { + // CompareOp::Eq => self.eq(other).into_py(py), + // CompareOp::Ne => self.ne(other).into_py(py), + // _ => py.NotImplemented(), + // } + // } + + // TODO: Implement + // fn __hash__(&self) -> isize { + // let mut h = DefaultHasher::new(); + // self.hash(&mut h); + // h.finish() as isize + // } + + fn __str__(&self) -> String { + self.to_string() + } + + fn __repr__(&self) -> String { + format!("{self:?}") + } + + #[getter] + #[pyo3(name = "instrument_id")] + fn py_instrument_id(&self) -> InstrumentId { + self.instrument_id + } + + #[getter] + #[pyo3(name = "deltas")] + fn py_deltas(&self) -> Vec { + // `OrderBookDelta` is `Copy` + self.deltas.clone() + } + + #[getter] + #[pyo3(name = "flags")] + fn py_flags(&self) -> u8 { + self.flags + } + + #[getter] + #[pyo3(name = "sequence")] + fn py_sequence(&self) -> u64 { + self.sequence + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_event + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_init + } + + #[staticmethod] + #[pyo3(name = "fully_qualified_name")] + fn py_fully_qualified_name() -> String { + format!("{}:{}", PY_MODULE_MODEL, stringify!(OrderBookDeltas)) + } + + // /// Creates a `PyCapsule` containing a raw pointer to a `Data::Delta` object. + // /// + // /// This function takes the current object (assumed to be of a type that can be represented as + // /// `Data::Delta`), and encapsulates a raw pointer to it within a `PyCapsule`. + // /// + // /// # Safety + // /// + // /// This function is safe as long as the following conditions are met: + // /// - The `Data::Delta` object pointed to by the capsule must remain valid for the lifetime of the capsule. + // /// - The consumer of the capsule must ensure proper handling to avoid dereferencing a dangling pointer. + // /// + // /// # Panics + // /// + // /// The function will panic if the `PyCapsule` creation fails, which can occur if the + // /// `Data::Delta` object cannot be converted into a raw pointer. + // /// + // #[pyo3(name = "as_pycapsule")] + // fn py_as_pycapsule(&self, py: Python<'_>) -> PyObject { + // data_to_pycapsule(py, Data::Delta(*self)) + // } + + // TODO: Implement `Serializable` and the other methods can be added +} diff --git a/nautilus_core/model/src/python/data/mod.rs b/nautilus_core/model/src/python/data/mod.rs index 48c110cbbad8..f5aca4393bf4 100644 --- a/nautilus_core/model/src/python/data/mod.rs +++ b/nautilus_core/model/src/python/data/mod.rs @@ -15,6 +15,7 @@ pub mod bar; pub mod delta; +pub mod deltas; pub mod depth; pub mod order; pub mod quote; diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index 068797f8db9f..3b9172b2f8a5 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -665,6 +665,13 @@ typedef struct Level Level; */ typedef struct OrderBook OrderBook; +/** + * Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. + * + * This type cannot be `repr(C)` due to the `deltas` vec. + */ +typedef struct OrderBookDeltas_t OrderBookDeltas_t; + /** * Represents a synthetic instrument with prices derived from component instruments using a * formula. @@ -1011,6 +1018,20 @@ typedef struct Data_t { }; } Data_t; +/** + * Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. + * + * This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function + * calls, enabling interaction with `OrderBookDeltas` in a C environment. + * + * It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be + * dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without + * having to manually access the underlying `OrderBookDeltas` instance. + */ +typedef struct OrderBookDeltas_API { + struct OrderBookDeltas_t *_0; +} OrderBookDeltas_API; + /** * Represents a valid trader ID. * @@ -1376,6 +1397,35 @@ uint8_t orderbook_delta_eq(const struct OrderBookDelta_t *lhs, const struct Orde uint64_t orderbook_delta_hash(const struct OrderBookDelta_t *delta); +/** + * Creates a new `OrderBookDeltas` object from a CVec of `OrderBookDelta`. + * + * # Safety + * - The `deltas` must be a valid pointer to a `CVec` containing `OrderBookDelta` objects + * - This function clones the data pointed to by `deltas` into Rust-managed memory, then forgets the original `Vec` to prevent Rust from auto-deallocating it + * - The caller is responsible for managing the memory of `deltas` (including its deallocation) to avoid memory leaks + */ +struct OrderBookDeltas_API orderbook_deltas_new(struct InstrumentId_t instrument_id, + const CVec *deltas); + +void orderbook_deltas_drop(struct OrderBookDeltas_API deltas); + +struct InstrumentId_t orderbook_deltas_instrument_id(const struct OrderBookDeltas_API *deltas); + +CVec orderbook_deltas_vec_deltas(const struct OrderBookDeltas_API *deltas); + +uint8_t orderbook_deltas_is_snapshot(const struct OrderBookDeltas_API *deltas); + +uint8_t orderbook_deltas_flags(const struct OrderBookDeltas_API *deltas); + +uint64_t orderbook_deltas_sequence(const struct OrderBookDeltas_API *deltas); + +uint64_t orderbook_deltas_ts_event(const struct OrderBookDeltas_API *deltas); + +uint64_t orderbook_deltas_ts_init(const struct OrderBookDeltas_API *deltas); + +void orderbook_deltas_vec_drop(CVec v); + /** * # Safety * diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index ee07a4950812..165a60ed703a 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -359,6 +359,12 @@ cdef extern from "../includes/model.h": cdef struct OrderBook: pass + # Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. + # + # This type cannot be `repr(C)` due to the `deltas` vec. + cdef struct OrderBookDeltas_t: + pass + # Represents a synthetic instrument with prices derived from component instruments using a # formula. cdef struct SyntheticInstrument: @@ -546,6 +552,17 @@ cdef extern from "../includes/model.h": TradeTick_t trade; Bar_t bar; + # Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. + # + # This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function + # calls, enabling interaction with `OrderBookDeltas` in a C environment. + # + # It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be + # dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without + # having to manually access the underlying `OrderBookDeltas` instance. + cdef struct OrderBookDeltas_API: + OrderBookDeltas_t *_0; + # Represents a valid trader ID. # # Must be correctly formatted with two valid strings either side of a hyphen. @@ -827,6 +844,33 @@ cdef extern from "../includes/model.h": uint64_t orderbook_delta_hash(const OrderBookDelta_t *delta); + # Creates a new `OrderBookDeltas` object from a CVec of `OrderBookDelta`. + # + # # Safety + # - The `deltas` must be a valid pointer to a `CVec` containing `OrderBookDelta` objects + # - This function clones the data pointed to by `deltas` into Rust-managed memory, then forgets the original `Vec` to prevent Rust from auto-deallocating it + # - The caller is responsible for managing the memory of `deltas` (including its deallocation) to avoid memory leaks + OrderBookDeltas_API orderbook_deltas_new(InstrumentId_t instrument_id, + const CVec *deltas); + + void orderbook_deltas_drop(OrderBookDeltas_API deltas); + + InstrumentId_t orderbook_deltas_instrument_id(const OrderBookDeltas_API *deltas); + + CVec orderbook_deltas_vec_deltas(const OrderBookDeltas_API *deltas); + + uint8_t orderbook_deltas_is_snapshot(const OrderBookDeltas_API *deltas); + + uint8_t orderbook_deltas_flags(const OrderBookDeltas_API *deltas); + + uint64_t orderbook_deltas_sequence(const OrderBookDeltas_API *deltas); + + uint64_t orderbook_deltas_ts_event(const OrderBookDeltas_API *deltas); + + uint64_t orderbook_deltas_ts_init(const OrderBookDeltas_API *deltas); + + void orderbook_deltas_vec_drop(CVec v); + # # Safety # # - Assumes `bids` and `asks` are valid pointers to arrays of `BookOrder` of length 10. diff --git a/nautilus_trader/model/data.pxd b/nautilus_trader/model/data.pxd index 78912429c2a3..21eaf3945cdc 100644 --- a/nautilus_trader/model/data.pxd +++ b/nautilus_trader/model/data.pxd @@ -32,6 +32,7 @@ from nautilus_trader.core.rust.model cimport HaltReason from nautilus_trader.core.rust.model cimport InstrumentCloseType from nautilus_trader.core.rust.model cimport MarketStatus from nautilus_trader.core.rust.model cimport OrderBookDelta_t +from nautilus_trader.core.rust.model cimport OrderBookDeltas_API from nautilus_trader.core.rust.model cimport OrderBookDepth10_t from nautilus_trader.core.rust.model cimport OrderSide from nautilus_trader.core.rust.model cimport PriceType @@ -235,18 +236,7 @@ cdef class OrderBookDelta(Data): cdef class OrderBookDeltas(Data): - cdef readonly InstrumentId instrument_id - """The instrument ID for the order book.\n\n:returns: `InstrumentId`""" - cdef readonly list deltas - """The order book deltas.\n\n:returns: `list[OrderBookDelta]`""" - cdef readonly bint is_snapshot - """If the deltas represent a snapshot (an initial CLEAR then deltas).\n\n:returns: `bool`""" - cdef readonly uint64_t sequence - """If the sequence number for the last delta.\n\n:returns: `bool`""" - cdef readonly uint64_t ts_event - """The UNIX timestamp (nanoseconds) when the last delta event occurred.\n\n:returns: `uint64_t`""" - cdef readonly uint64_t ts_init - """The UNIX timestamp (nanoseconds) when the last delta event was initialized.\n\n:returns: `uint64_t`""" + cdef OrderBookDeltas_API _mem @staticmethod cdef OrderBookDeltas from_dict_c(dict values) diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index 36efef53a3a5..084a37cafe75 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -78,6 +78,16 @@ from nautilus_trader.core.rust.model cimport instrument_id_from_cstr from nautilus_trader.core.rust.model cimport orderbook_delta_eq from nautilus_trader.core.rust.model cimport orderbook_delta_hash from nautilus_trader.core.rust.model cimport orderbook_delta_new +from nautilus_trader.core.rust.model cimport orderbook_deltas_drop +from nautilus_trader.core.rust.model cimport orderbook_deltas_flags +from nautilus_trader.core.rust.model cimport orderbook_deltas_instrument_id +from nautilus_trader.core.rust.model cimport orderbook_deltas_is_snapshot +from nautilus_trader.core.rust.model cimport orderbook_deltas_new +from nautilus_trader.core.rust.model cimport orderbook_deltas_sequence +from nautilus_trader.core.rust.model cimport orderbook_deltas_ts_event +from nautilus_trader.core.rust.model cimport orderbook_deltas_ts_init +from nautilus_trader.core.rust.model cimport orderbook_deltas_vec_deltas +from nautilus_trader.core.rust.model cimport orderbook_deltas_vec_drop from nautilus_trader.core.rust.model cimport orderbook_depth10_ask_counts_array from nautilus_trader.core.rust.model cimport orderbook_depth10_asks_array from nautilus_trader.core.rust.model cimport orderbook_depth10_bid_counts_array @@ -2130,12 +2140,40 @@ cdef class OrderBookDeltas(Data): ) -> None: Condition.not_empty(deltas, "deltas") - self.instrument_id = instrument_id - self.deltas = deltas - self.is_snapshot = deltas[0].is_clear - self.sequence = deltas[-1].sequence - self.ts_event = deltas[-1].ts_event - self.ts_init = deltas[-1].ts_init + cdef uint64_t len_ = len(deltas) + + # Create a C OrderBookDeltas_t buffer + cdef OrderBookDelta_t* data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) + if not data: + raise MemoryError() + + cdef uint64_t i + cdef OrderBookDelta delta + for i in range(len_): + delta = deltas[i] + data[i] = delta._mem + + # Create CVec + cdef CVec* cvec = PyMem_Malloc(1 * sizeof(CVec)) + if not cvec: + raise MemoryError() + + cvec.ptr = data + cvec.len = len_ + cvec.cap = len_ + + # Transfer data to Rust + self._mem = orderbook_deltas_new( + instrument_id._mem, + cvec, + ) + + PyMem_Free(cvec.ptr) # De-allocate buffer + PyMem_Free(cvec) # De-allocate cvec + + def __del__(self) -> None: + if self._mem._0 != NULL: + orderbook_deltas_drop(self._mem) def __eq__(self, OrderBookDeltas other) -> bool: return OrderBookDeltas.to_dict_c(self) == OrderBookDeltas.to_dict_c(other) @@ -2154,6 +2192,102 @@ cdef class OrderBookDeltas(Data): f"ts_init={self.ts_init})" ) + @property + def instrument_id(self) -> InstrumentId: + """ + Return the deltas book instrument ID. + + Returns + ------- + InstrumentId + + """ + return InstrumentId.from_mem_c(orderbook_deltas_instrument_id(&self._mem)) + + @property + def deltas(self) -> list[OrderBookDelta]: + """ + Return the contained deltas. + + Returns + ------- + list[OrderBookDeltas] + + """ + cdef CVec raw_deltas_vec = orderbook_deltas_vec_deltas(&self._mem) + cdef OrderBookDelta_t* raw_deltas = raw_deltas_vec.ptr + + cdef list[OrderBookDelta] deltas = [] + + cdef: + uint64_t i + for i in range(raw_deltas_vec.len): + deltas.append(delta_from_mem_c(raw_deltas[i])) + + orderbook_deltas_vec_drop(raw_deltas_vec) + + return deltas + + @property + def is_snapshot(self) -> bool: + """ + If the deltas is a snapshot. + + Returns + ------- + bool + + """ + return orderbook_deltas_is_snapshot(&self._mem) + + @property + def flags(self) -> uint8_t: + """ + Return the flags for the delta. + + Returns + ------- + uint8_t + + """ + return orderbook_deltas_flags(&self._mem) + + @property + def sequence(self) -> uint64_t: + """ + Return the sequence number for the delta. + + Returns + ------- + uint64_t + + """ + return orderbook_deltas_sequence(&self._mem) + + @property + def ts_event(self) -> int: + """ + The UNIX timestamp (nanoseconds) when the data event occurred. + + Returns + ------- + int + + """ + return orderbook_deltas_ts_event(&self._mem) + + @property + def ts_init(self) -> int: + """ + The UNIX timestamp (nanoseconds) when the object was initialized. + + Returns + ------- + int + + """ + return orderbook_deltas_ts_init(&self._mem) + @staticmethod cdef OrderBookDeltas from_dict_c(dict values): Condition.not_none(values, "values") @@ -2167,7 +2301,7 @@ cdef class OrderBookDeltas(Data): Condition.not_none(obj, "obj") return { "type": obj.__class__.__name__, - "instrument_id": obj.instrument_id.to_str(), + "instrument_id": obj.instrument_id.value, "deltas": [OrderBookDelta.to_dict_c(d) for d in obj.deltas], } diff --git a/tests/mem_leak_tests/tracemalloc_orderbook_delta.py b/tests/mem_leak_tests/tracemalloc_orderbook_delta.py new file mode 100644 index 000000000000..c36bce772ee6 --- /dev/null +++ b/tests/mem_leak_tests/tracemalloc_orderbook_delta.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +from nautilus_trader.model.data import OrderBookDelta +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 +from nautilus_trader.test_kit.stubs.data import TestDataStubs +from tests.mem_leak_tests.conftest import snapshot_memory + + +@snapshot_memory(4000) +def run_repr(*args, **kwargs): + delta = TestDataStubs.order_book_delta() + repr(delta) # Copies bids and asks book order data from Rust on every iteration + + +@snapshot_memory(4000) +def run_from_pyo3(*args, **kwargs): + pyo3_delta = TestDataProviderPyo3.order_book_delta() + OrderBookDelta.from_pyo3(pyo3_delta) + + +if __name__ == "__main__": + run_repr() + run_from_pyo3() diff --git a/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py b/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py index 79c6f0ecfd88..b12f2cb3d78e 100644 --- a/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py +++ b/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py @@ -14,24 +14,25 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -from nautilus_trader.model.data import OrderBookDelta -from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 +from nautilus_trader.model.data import OrderBookDeltas from nautilus_trader.test_kit.stubs.data import TestDataStubs from tests.mem_leak_tests.conftest import snapshot_memory @snapshot_memory(4000) def run_repr(*args, **kwargs): - depth = TestDataStubs.order_book_delta() - repr(depth) # Copies bids and asks book order data from Rust on every iteration + delta = TestDataStubs.order_book_delta() + deltas = OrderBookDeltas(delta.instrument_id, deltas=[delta] * 1024) + repr(deltas.deltas) + repr(deltas) -@snapshot_memory(4000) -def run_from_pyo3(*args, **kwargs): - pyo3_delta = TestDataProviderPyo3.order_book_delta() - OrderBookDelta.from_pyo3(pyo3_delta) +# @snapshot_memory(4000) +# def run_from_pyo3(*args, **kwargs): +# pyo3_delta = TestDataProviderPyo3.order_book_delta() +# OrderBookDelta.from_pyo3(pyo3_delta) if __name__ == "__main__": run_repr() - run_from_pyo3() + # run_from_pyo3() From 19113b850e8ce6b035d5f3c5ad8fd58430cc7e66 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 13 Feb 2024 21:02:10 +1100 Subject: [PATCH 037/104] Refine OrderBookDeltas implementation --- nautilus_core/model/src/data/deltas.rs | 20 ++++++++++- nautilus_core/model/src/data/mod.rs | 14 ++++++-- nautilus_core/model/src/ffi/data/deltas.rs | 1 + nautilus_core/model/src/python/data/deltas.rs | 33 ++++++++----------- nautilus_trader/core/includes/model.h | 32 ++++++++++-------- nautilus_trader/core/nautilus_pyo3.pyi | 19 +++++++++++ nautilus_trader/core/rust/model.pxd | 24 +++++++------- nautilus_trader/model/data.pyx | 4 +-- 8 files changed, 98 insertions(+), 49 deletions(-) diff --git a/nautilus_core/model/src/data/deltas.rs b/nautilus_core/model/src/data/deltas.rs index 1d8bdd0e4aee..b7ed835dab7a 100644 --- a/nautilus_core/model/src/data/deltas.rs +++ b/nautilus_core/model/src/data/deltas.rs @@ -13,7 +13,10 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use std::fmt::{Display, Formatter}; +use std::{ + fmt::{Display, Formatter}, + hash::{Hash, Hasher}, +}; use nautilus_core::time::UnixNanos; use pyo3::prelude::*; @@ -66,6 +69,21 @@ impl OrderBookDeltas { } } +impl PartialEq for OrderBookDeltas { + fn eq(&self, other: &Self) -> bool { + self.instrument_id == other.instrument_id && self.sequence == other.sequence + } +} + +impl Eq for OrderBookDeltas {} + +impl Hash for OrderBookDeltas { + fn hash(&self, state: &mut H) { + self.instrument_id.hash(state); + self.sequence.hash(state); + } +} + // TODO: Implement // impl Serializable for OrderBookDeltas {} diff --git a/nautilus_core/model/src/data/mod.rs b/nautilus_core/model/src/data/mod.rs index e354fb78e182..e4f256330aae 100644 --- a/nautilus_core/model/src/data/mod.rs +++ b/nautilus_core/model/src/data/mod.rs @@ -23,6 +23,8 @@ pub mod trade; use nautilus_core::time::UnixNanos; +use crate::ffi::data::deltas::OrderBookDeltas_API; + use self::{ bar::Bar, delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, quote::QuoteTick, trade::TradeTick, @@ -33,6 +35,7 @@ use self::{ #[allow(clippy::large_enum_variant)] // TODO: Optimize this (largest variant 1008 vs 136 bytes) pub enum Data { Delta(OrderBookDelta), + Deltas(OrderBookDeltas_API), Depth10(OrderBookDepth10), Quote(QuoteTick), Trade(TradeTick), @@ -47,6 +50,7 @@ impl HasTsInit for Data { fn get_ts_init(&self) -> UnixNanos { match self { Data::Delta(d) => d.ts_init, + Data::Deltas(d) => d.ts_init, Data::Depth10(d) => d.ts_init, Data::Quote(q) => q.ts_init, Data::Trade(t) => t.ts_init, @@ -61,13 +65,13 @@ impl HasTsInit for OrderBookDelta { } } -impl HasTsInit for OrderBookDepth10 { +impl HasTsInit for OrderBookDeltas { fn get_ts_init(&self) -> UnixNanos { self.ts_init } } -impl HasTsInit for OrderBookDeltas { +impl HasTsInit for OrderBookDepth10 { fn get_ts_init(&self) -> UnixNanos { self.ts_init } @@ -102,6 +106,12 @@ impl From for Data { } } +impl From for Data { + fn from(value: OrderBookDeltas_API) -> Self { + Self::Deltas(value) + } +} + impl From for Data { fn from(value: OrderBookDepth10) -> Self { Self::Depth10(value) diff --git a/nautilus_core/model/src/ffi/data/deltas.rs b/nautilus_core/model/src/ffi/data/deltas.rs index f52632de71e2..69cfb237453d 100644 --- a/nautilus_core/model/src/ffi/data/deltas.rs +++ b/nautilus_core/model/src/ffi/data/deltas.rs @@ -32,6 +32,7 @@ use crate::{ /// dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without /// having to manually access the underlying `OrderBookDeltas` instance. #[repr(C)] +#[derive(Debug, Clone)] #[allow(non_camel_case_types)] pub struct OrderBookDeltas_API(Box); diff --git a/nautilus_core/model/src/python/data/deltas.rs b/nautilus_core/model/src/python/data/deltas.rs index fcd7b8acfb81..290f34568bc9 100644 --- a/nautilus_core/model/src/python/data/deltas.rs +++ b/nautilus_core/model/src/python/data/deltas.rs @@ -13,13 +13,10 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -// use std::{ -// collections::{hash_map::DefaultHasher, HashMap}, -// hash::{Hash, Hasher}, -// }; +use std::hash::{DefaultHasher, Hash, Hasher}; use nautilus_core::time::UnixNanos; -use pyo3::prelude::*; +use pyo3::{prelude::*, pyclass::CompareOp}; use crate::{ data::{delta::OrderBookDelta, deltas::OrderBookDeltas}, @@ -34,21 +31,19 @@ impl OrderBookDeltas { Self::new(instrument_id, deltas) } - // TODO: Implement - // fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { - // match op { - // CompareOp::Eq => self.eq(other).into_py(py), - // CompareOp::Ne => self.ne(other).into_py(py), - // _ => py.NotImplemented(), - // } - // } + fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { + match op { + CompareOp::Eq => self.eq(other).into_py(py), + CompareOp::Ne => self.ne(other).into_py(py), + _ => py.NotImplemented(), + } + } - // TODO: Implement - // fn __hash__(&self) -> isize { - // let mut h = DefaultHasher::new(); - // self.hash(&mut h); - // h.finish() as isize - // } + fn __hash__(&self) -> isize { + let mut h = DefaultHasher::new(); + self.hash(&mut h); + h.finish() as isize + } fn __str__(&self) -> String { self.to_string() diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index 3b9172b2f8a5..c95d3b273fc2 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -780,6 +780,20 @@ typedef struct OrderBookDelta_t { uint64_t ts_init; } OrderBookDelta_t; +/** + * Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. + * + * This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function + * calls, enabling interaction with `OrderBookDeltas` in a C environment. + * + * It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be + * dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without + * having to manually access the underlying `OrderBookDeltas` instance. + */ +typedef struct OrderBookDeltas_API { + struct OrderBookDeltas_t *_0; +} OrderBookDeltas_API; + /** * Represents a self-contained order book update with a fixed depth of 10 levels per side. * @@ -991,6 +1005,7 @@ typedef struct Bar_t { typedef enum Data_t_Tag { DELTA, + DELTAS, DEPTH10, QUOTE, TRADE, @@ -1003,6 +1018,9 @@ typedef struct Data_t { struct { struct OrderBookDelta_t delta; }; + struct { + struct OrderBookDeltas_API deltas; + }; struct { struct OrderBookDepth10_t depth10; }; @@ -1018,20 +1036,6 @@ typedef struct Data_t { }; } Data_t; -/** - * Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. - * - * This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function - * calls, enabling interaction with `OrderBookDeltas` in a C environment. - * - * It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be - * dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without - * having to manually access the underlying `OrderBookDeltas` instance. - */ -typedef struct OrderBookDeltas_API { - struct OrderBookDeltas_t *_0; -} OrderBookDeltas_API; - /** * Represents a valid trader ID. * diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index e405dad95404..832101da0f15 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -496,6 +496,25 @@ class OrderBookDelta: @staticmethod def get_fields() -> dict[str, str]: ... +class OrderBookDeltas: + def __init__( + self, + instrument_id: InstrumentId, + deltas: list[OrderBookDelta], + ) -> None: ... + @property + def instrument_id(self) -> InstrumentId: ... + @property + def deltas(self) -> list[OrderBookDelta]: ... + @property + def flags(self) -> int: ... + @property + def sequence(self) -> int: ... + @property + def ts_event(self) -> int: ... + @property + def ts_init(self) -> int: ... + class OrderBookDepth10: def __init__( self, diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index 165a60ed703a..a0ad361f5784 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -425,6 +425,17 @@ cdef extern from "../includes/model.h": # The UNIX timestamp (nanoseconds) when the data object was initialized. uint64_t ts_init; + # Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. + # + # This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function + # calls, enabling interaction with `OrderBookDeltas` in a C environment. + # + # It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be + # dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without + # having to manually access the underlying `OrderBookDeltas` instance. + cdef struct OrderBookDeltas_API: + OrderBookDeltas_t *_0; + # Represents a self-contained order book update with a fixed depth of 10 levels per side. # # This struct is specifically designed for scenarios where a snapshot of the top 10 bid and @@ -539,6 +550,7 @@ cdef extern from "../includes/model.h": cpdef enum Data_t_Tag: DELTA, + DELTAS, DEPTH10, QUOTE, TRADE, @@ -547,22 +559,12 @@ cdef extern from "../includes/model.h": cdef struct Data_t: Data_t_Tag tag; OrderBookDelta_t delta; + OrderBookDeltas_API deltas; OrderBookDepth10_t depth10; QuoteTick_t quote; TradeTick_t trade; Bar_t bar; - # Provides a C compatible Foreign Function Interface (FFI) for an underlying [`OrderBookDeltas`]. - # - # This struct wraps `OrderBookDeltas` in a way that makes it compatible with C function - # calls, enabling interaction with `OrderBookDeltas` in a C environment. - # - # It implements the `Deref` trait, allowing instances of `OrderBookDeltas_API` to be - # dereferenced to `OrderBookDeltas`, providing access to `OrderBookDeltas`'s methods without - # having to manually access the underlying `OrderBookDeltas` instance. - cdef struct OrderBookDeltas_API: - OrderBookDeltas_t *_0; - # Represents a valid trader ID. # # Must be correctly formatted with two valid strings either side of a hyphen. diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index 084a37cafe75..b76d9c4c87fa 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -2243,7 +2243,7 @@ cdef class OrderBookDeltas(Data): @property def flags(self) -> uint8_t: """ - Return the flags for the delta. + Return the flags for the last delta. Returns ------- @@ -2255,7 +2255,7 @@ cdef class OrderBookDeltas(Data): @property def sequence(self) -> uint64_t: """ - Return the sequence number for the delta. + Return the sequence number for the last delta. Returns ------- From 74e3533c2f0be1e8485e0cc1375deaf9f8572d80 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 13 Feb 2024 21:26:27 +1100 Subject: [PATCH 038/104] Fix Rust imports --- nautilus_core/model/src/python/data/deltas.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nautilus_core/model/src/python/data/deltas.rs b/nautilus_core/model/src/python/data/deltas.rs index 290f34568bc9..c9b3c0d8882c 100644 --- a/nautilus_core/model/src/python/data/deltas.rs +++ b/nautilus_core/model/src/python/data/deltas.rs @@ -13,7 +13,10 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use std::hash::{DefaultHasher, Hash, Hasher}; +use std::{ + collections::hash_map::DefaultHasher, + hash::{Hash, Hasher}, +}; use nautilus_core::time::UnixNanos; use pyo3::{prelude::*, pyclass::CompareOp}; From ae5422c5fa0cd71d79ae64608d1f5ddd165bbefd Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 14 Feb 2024 18:32:22 +1100 Subject: [PATCH 039/104] Fix Equity short selling --- RELEASES.md | 2 + nautilus_trader/backtest/exchange.pyx | 3 +- nautilus_trader/backtest/execution_client.pyx | 2 +- nautilus_trader/backtest/matching_engine.pxd | 3 + nautilus_trader/backtest/matching_engine.pyx | 26 +- .../unit_tests/backtest/test_exchange_cash.py | 301 +++++++++++++ ...st_exchange.py => test_exchange_margin.py} | 405 +++++++++--------- .../backtest/test_matching_engine.py | 6 +- 8 files changed, 539 insertions(+), 209 deletions(-) create mode 100644 tests/unit_tests/backtest/test_exchange_cash.py rename tests/unit_tests/backtest/{test_exchange.py => test_exchange_margin.py} (92%) diff --git a/RELEASES.md b/RELEASES.md index 0e9472a8d2de..c44e17cb9297 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -9,11 +9,13 @@ None None ### Fixes +- Fixed `Equity` short selling for `CASH` accounts (will now reject) - Fixed `ActorFactory.create` JSON encoding (was missing the encoding hook) - Fixed `ImportableConfig.create` JSON encoding (was missing the encoding hook) - Fixed `ImportableStrategyConfig.create` JSON encoding (was missing the encoding hook) - Fixed `ExecAlgorithmFactory.create` JSON encoding (was missing the encoding hook) - Fixed `ControllerConfig` base class and docstring +- Fixed Interactive Brokers historical bar data bug, thanks @benjaminsingleton --- diff --git a/nautilus_trader/backtest/exchange.pyx b/nautilus_trader/backtest/exchange.pyx index 01299c203d3a..a273fe1cb805 100644 --- a/nautilus_trader/backtest/exchange.pyx +++ b/nautilus_trader/backtest/exchange.pyx @@ -154,7 +154,7 @@ cdef class SimulatedExchange: bint use_position_ids = True, bint use_random_ids = False, bint use_reduce_only = True, - ): + ) -> None: Condition.list_type(instruments, Instrument, "instruments", "Instrument") Condition.not_empty(starting_balances, "starting_balances") Condition.list_type(starting_balances, Money, "starting_balances") @@ -330,6 +330,7 @@ cdef class SimulatedExchange: fill_model=self.fill_model, book_type=self.book_type, oms_type=self.oms_type, + account_type=self.account_type, msgbus=self.msgbus, cache=self.cache, clock=self._clock, diff --git a/nautilus_trader/backtest/execution_client.pyx b/nautilus_trader/backtest/execution_client.pyx index 9c6e0e21d07f..30cf5eb7af95 100644 --- a/nautilus_trader/backtest/execution_client.pyx +++ b/nautilus_trader/backtest/execution_client.pyx @@ -62,7 +62,7 @@ cdef class BacktestExecClient(ExecutionClient): TestClock clock not None, bint routing=False, bint frozen_account=False, - ): + ) -> None: super().__init__( client_id=ClientId(exchange.id.value), venue=Venue(exchange.id.value), diff --git a/nautilus_trader/backtest/matching_engine.pxd b/nautilus_trader/backtest/matching_engine.pxd index c341b01afb22..f3adf0c52723 100644 --- a/nautilus_trader/backtest/matching_engine.pxd +++ b/nautilus_trader/backtest/matching_engine.pxd @@ -23,6 +23,7 @@ from nautilus_trader.common.component cimport Clock from nautilus_trader.common.component cimport Logger from nautilus_trader.common.component cimport MessageBus from nautilus_trader.core.data cimport Data +from nautilus_trader.core.rust.model cimport AccountType from nautilus_trader.core.rust.model cimport BookType from nautilus_trader.core.rust.model cimport LiquiditySide from nautilus_trader.core.rust.model cimport MarketStatus @@ -95,6 +96,8 @@ cdef class OrderMatchingEngine: """The order book type for the matching engine.\n\n:returns: `BookType`""" cdef readonly OmsType oms_type """The order management system type for the matching engine.\n\n:returns: `OmsType`""" + cdef readonly AccountType account_type + """The account type for the matching engine.\n\n:returns: `AccountType`""" cdef readonly MarketStatus market_status """The market status for the matching engine.\n\n:returns: `MarketStatus`""" cdef readonly CacheFacade cache diff --git a/nautilus_trader/backtest/matching_engine.pyx b/nautilus_trader/backtest/matching_engine.pyx index 12b535c49681..56b42f9e2b65 100644 --- a/nautilus_trader/backtest/matching_engine.pyx +++ b/nautilus_trader/backtest/matching_engine.pyx @@ -29,6 +29,7 @@ from nautilus_trader.common.component cimport TestClock from nautilus_trader.core.correctness cimport Condition from nautilus_trader.core.data cimport Data from nautilus_trader.core.rust.common cimport logging_is_initialized +from nautilus_trader.core.rust.model cimport AccountType from nautilus_trader.core.rust.model cimport AggressorSide from nautilus_trader.core.rust.model cimport BookType from nautilus_trader.core.rust.model cimport ContingencyType @@ -78,6 +79,7 @@ from nautilus_trader.model.identifiers cimport TradeId from nautilus_trader.model.identifiers cimport TraderId from nautilus_trader.model.identifiers cimport VenueOrderId from nautilus_trader.model.instruments.base cimport Instrument +from nautilus_trader.model.instruments.equity cimport Equity from nautilus_trader.model.objects cimport Money from nautilus_trader.model.objects cimport Price from nautilus_trader.model.objects cimport Quantity @@ -111,6 +113,9 @@ cdef class OrderMatchingEngine: oms_type : OmsType The order management system type for the matching engine. Determines the generation and handling of venue position IDs. + account_type : AccountType + The account type for the matching engine. Determines allowable + executions based on the instrument. msgbus : MessageBus The message bus for the matching engine. cache : CacheFacade @@ -145,6 +150,7 @@ cdef class OrderMatchingEngine: FillModel fill_model not None, BookType book_type, OmsType oms_type, + AccountType account_type, MessageBus msgbus not None, CacheFacade cache not None, TestClock clock not None, @@ -156,7 +162,7 @@ cdef class OrderMatchingEngine: bint use_random_ids = False, bint use_reduce_only = True, # auction_match_algo = default_auction_match - ): + ) -> None: self._clock = clock self._log = Logger(name=f"{type(self).__name__}({instrument.id.venue})") self.msgbus = msgbus @@ -167,6 +173,7 @@ cdef class OrderMatchingEngine: self.raw_id = raw_id self.book_type = book_type self.oms_type = oms_type + self.account_type = account_type self.market_status = MarketStatus.OPEN self._bar_execution = bar_execution @@ -663,10 +670,23 @@ cdef class OrderMatchingEngine: self._generate_order_rejected(order, f"Contingent order {client_order_id} already closed") return # Order rejected + cdef Position position = self.cache.position_for_order(order.client_order_id) + + # Check not shorting an equity without a MARGIN account + if ( + order.side == OrderSide.SELL + and self.account_type != AccountType.MARGIN + and isinstance(self.instrument, Equity) + and (position is None or not order.would_reduce_only(position.side, position.quantity)) + ): + self._generate_order_rejected( + order, + f"SHORT SELLING not permitted on a CASH account with order {repr(order)}." + ) + return # Cannot short sell + # Check reduce-only instruction - cdef Position position if self._use_reduce_only and order.is_reduce_only and not order.is_closed_c(): - position = self.cache.position_for_order(order.client_order_id) if ( not position or position.is_closed_c() diff --git a/tests/unit_tests/backtest/test_exchange_cash.py b/tests/unit_tests/backtest/test_exchange_cash.py new file mode 100644 index 000000000000..5904ebf03f61 --- /dev/null +++ b/tests/unit_tests/backtest/test_exchange_cash.py @@ -0,0 +1,301 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +from decimal import Decimal + +import pytest + +from nautilus_trader.backtest.exchange import SimulatedExchange +from nautilus_trader.backtest.execution_client import BacktestExecClient +from nautilus_trader.backtest.models import FillModel +from nautilus_trader.backtest.models import LatencyModel +from nautilus_trader.common.component import MessageBus +from nautilus_trader.common.component import TestClock +from nautilus_trader.config import ExecEngineConfig +from nautilus_trader.config import RiskEngineConfig +from nautilus_trader.data.engine import DataEngine +from nautilus_trader.execution.engine import ExecutionEngine +from nautilus_trader.model.currencies import USD +from nautilus_trader.model.data import BarType +from nautilus_trader.model.enums import AccountType +from nautilus_trader.model.enums import OmsType +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import OrderStatus +from nautilus_trader.model.identifiers import Venue +from nautilus_trader.model.objects import Money +from nautilus_trader.model.objects import Quantity +from nautilus_trader.portfolio.portfolio import Portfolio +from nautilus_trader.risk.engine import RiskEngine +from nautilus_trader.test_kit.mocks.strategies import MockStrategy +from nautilus_trader.test_kit.providers import TestInstrumentProvider +from nautilus_trader.test_kit.stubs.component import TestComponentStubs +from nautilus_trader.test_kit.stubs.data import TestDataStubs +from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs + + +_AAPL_XNAS = TestInstrumentProvider.equity() + + +class TestSimulatedExchangeCashAccount: + def setup(self) -> None: + # Fixture Setup + self.clock = TestClock() + self.trader_id = TestIdStubs.trader_id() + + self.msgbus = MessageBus( + trader_id=self.trader_id, + clock=self.clock, + ) + + self.cache = TestComponentStubs.cache() + + self.portfolio = Portfolio( + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + ) + + self.data_engine = DataEngine( + msgbus=self.msgbus, + clock=self.clock, + cache=self.cache, + ) + + self.exec_engine = ExecutionEngine( + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + config=ExecEngineConfig(debug=True), + ) + + self.risk_engine = RiskEngine( + portfolio=self.portfolio, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + config=RiskEngineConfig(debug=True), + ) + + self.exchange = SimulatedExchange( + venue=Venue("XNAS"), + oms_type=OmsType.NETTING, + account_type=AccountType.CASH, + base_currency=USD, + starting_balances=[Money(1_000_000, USD)], + default_leverage=Decimal(0), + leverages={}, + instruments=[_AAPL_XNAS], + modules=[], + fill_model=FillModel(), + portfolio=self.portfolio, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + latency_model=LatencyModel(0), + ) + + self.exec_client = BacktestExecClient( + exchange=self.exchange, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + ) + + # Wire up components + self.exec_engine.register_client(self.exec_client) + self.exchange.register_client(self.exec_client) + + self.cache.add_instrument(_AAPL_XNAS) + + # Create mock strategy + self.strategy = MockStrategy(bar_type=BarType.from_str("AAPL.XNAS-1-MINUTE-BID-INTERNAL")) + self.strategy.register( + trader_id=self.trader_id, + portfolio=self.portfolio, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + ) + + # Start components + self.exchange.reset() + self.data_engine.start() + self.exec_engine.start() + self.strategy.start() + + def test_repr(self) -> None: + # Arrange, Act, Assert + assert ( + repr(self.exchange) == "SimulatedExchange(id=XNAS, oms_type=NETTING, account_type=CASH)" + ) + + def test_equity_short_selling_will_reject(self) -> None: + # Arrange: Prepare market + quote1 = TestDataStubs.quote_tick( + instrument=_AAPL_XNAS, + bid_price=100.00, + ask_price=101.00, + ) + self.data_engine.process(quote1) + self.exchange.process_quote_tick(quote1) + + # Act + order1 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + OrderSide.BUY, + Quantity.from_int(100), + ) + self.strategy.submit_order(order1) + self.exchange.process(0) + + order2 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + OrderSide.SELL, + Quantity.from_int(110), + ) + self.strategy.submit_order(order2) + self.exchange.process(0) + + position_id = self.cache.positions_open()[0].id # Generated by exchange + order3 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + OrderSide.SELL, + Quantity.from_int(100), + ) + self.strategy.submit_order(order3, position_id=position_id) + self.exchange.process(0) + + order4 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + OrderSide.SELL, + Quantity.from_int(100), + ) + self.strategy.submit_order(order4) + self.exchange.process(0) + + # Assert + assert order1.status == OrderStatus.FILLED + assert order2.status == OrderStatus.REJECTED + assert order3.status == OrderStatus.FILLED + assert order4.status == OrderStatus.REJECTED + assert self.exchange.get_account().balance_total(USD) == Money(999_900, USD) + + @pytest.mark.parametrize( + ("entry_side", "expected_usd"), + [ + [OrderSide.BUY, Money(979_800.00, USD)], + ], + ) + def test_equity_order_fills_for_entry( + self, + entry_side: OrderSide, + expected_usd: Money, + ) -> None: + # Arrange: Prepare market + quote1 = TestDataStubs.quote_tick( + instrument=_AAPL_XNAS, + bid_price=100.00, + ask_price=101.00, + ) + self.data_engine.process(quote1) + self.exchange.process_quote_tick(quote1) + + order1 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + entry_side, + Quantity.from_int(100), + ) + + order2 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + entry_side, + Quantity.from_int(100), + ) + + # Act + self.strategy.submit_order(order1) + self.exchange.process(0) + + self.strategy.submit_order(order2) + self.exchange.process(0) + + # Assert + assert order1.status == OrderStatus.FILLED + assert order2.status == OrderStatus.FILLED + assert self.exchange.get_account().balance_total(USD) == expected_usd + + @pytest.mark.parametrize( + ("entry_side", "exit_side", "expected_usd"), + [ + [OrderSide.BUY, OrderSide.SELL, Money(984_650.00, USD)], + ], + ) + def test_equity_order_fills_with_partial_exit( + self, + entry_side: OrderSide, + exit_side: OrderSide, + expected_usd: Money, + ) -> None: + # Arrange: Prepare market + quote1 = TestDataStubs.quote_tick( + instrument=_AAPL_XNAS, + bid_price=100.00, + ask_price=101.00, + ) + self.data_engine.process(quote1) + self.exchange.process_quote_tick(quote1) + + order1 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + entry_side, + Quantity.from_int(100), + ) + + order2 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + entry_side, + Quantity.from_int(100), + ) + + quote2 = TestDataStubs.quote_tick( + instrument=_AAPL_XNAS, + bid_price=101.00, + ask_price=102.00, + ) + self.data_engine.process(quote2) + self.exchange.process_quote_tick(quote2) + + order3 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + exit_side, + Quantity.from_int(50), + ) + + # Act + self.strategy.submit_order(order1) + self.exchange.process(0) + + position_id = self.cache.positions_open()[0].id # Generated by exchange + + self.strategy.submit_order(order2) + self.exchange.process(0) + self.strategy.submit_order(order3, position_id=position_id) + self.exchange.process(0) + + # Assert + assert order1.status == OrderStatus.FILLED + assert order2.status == OrderStatus.FILLED + assert order3.status == OrderStatus.FILLED + assert self.exchange.get_account().balance_total(USD) == expected_usd diff --git a/tests/unit_tests/backtest/test_exchange.py b/tests/unit_tests/backtest/test_exchange_margin.py similarity index 92% rename from tests/unit_tests/backtest/test_exchange.py rename to tests/unit_tests/backtest/test_exchange_margin.py index 0256978bbdf3..67f65593579c 100644 --- a/tests/unit_tests/backtest/test_exchange.py +++ b/tests/unit_tests/backtest/test_exchange_margin.py @@ -73,11 +73,11 @@ from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs -AUDUSD_SIM = TestInstrumentProvider.default_fx_ccy("AUD/USD") -USDJPY_SIM = TestInstrumentProvider.default_fx_ccy("USD/JPY") +_AUDUSD_SIM = TestInstrumentProvider.default_fx_ccy("AUD/USD") +_USDJPY_SIM = TestInstrumentProvider.default_fx_ccy("USD/JPY") -class TestSimulatedExchange: +class TestSimulatedExchangeMarginAccount: def setup(self) -> None: # Fixture Setup self.clock = TestClock() @@ -124,8 +124,8 @@ def setup(self) -> None: base_currency=USD, starting_balances=[Money(1_000_000, USD)], default_leverage=Decimal(50), - leverages={AUDUSD_SIM.id: Decimal(10)}, - instruments=[USDJPY_SIM], + leverages={_AUDUSD_SIM.id: Decimal(10)}, + instruments=[_USDJPY_SIM], modules=[], fill_model=FillModel(), portfolio=self.portfolio, @@ -146,7 +146,8 @@ def setup(self) -> None: self.exec_engine.register_client(self.exec_client) self.exchange.register_client(self.exec_client) - self.cache.add_instrument(USDJPY_SIM) + self.cache.add_instrument(_AUDUSD_SIM) + self.cache.add_instrument(_USDJPY_SIM) # Create mock strategy self.strategy = MockStrategy(bar_type=TestDataStubs.bartype_usdjpy_1min_bid()) @@ -188,14 +189,14 @@ def test_get_matching_engines_when_engine_returns_expected_dict(self) -> None: # Assert assert isinstance(matching_engines, dict) assert len(matching_engines) == 1 - assert list(matching_engines.keys()) == [USDJPY_SIM.id] + assert list(matching_engines.keys()) == [_USDJPY_SIM.id] def test_get_matching_engine_when_no_engine_for_instrument_returns_none(self) -> None: # Arrange, Act - matching_engine = self.exchange.get_matching_engine(USDJPY_SIM.id) + matching_engine = self.exchange.get_matching_engine(_USDJPY_SIM.id) # Assert - assert matching_engine.instrument == USDJPY_SIM + assert matching_engine.instrument == _USDJPY_SIM def test_get_books_with_one_instrument_returns_one_book(self) -> None: # Arrange, Act @@ -227,14 +228,14 @@ def test_get_open_ask_orders_when_no_orders_returns_empty_list(self) -> None: def test_get_open_bid_orders_with_instrument_when_no_orders_returns_empty_list(self) -> None: # Arrange, Act - orders = self.exchange.get_open_bid_orders(AUDUSD_SIM.id) + orders = self.exchange.get_open_bid_orders(_AUDUSD_SIM.id) # Assert assert orders == [] def test_get_open_ask_orders_with_instrument_when_no_orders_returns_empty_list(self) -> None: # Arrange, Act - orders = self.exchange.get_open_ask_orders(AUDUSD_SIM.id) + orders = self.exchange.get_open_ask_orders(_AUDUSD_SIM.id) # Assert assert orders == [] @@ -242,7 +243,7 @@ def test_get_open_ask_orders_with_instrument_when_no_orders_returns_empty_list(s def test_process_quote_tick_updates_market(self) -> None: # Arrange tick = TestDataStubs.quote_tick( - USDJPY_SIM, + _USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -251,19 +252,19 @@ def test_process_quote_tick_updates_market(self) -> None: self.exchange.process_quote_tick(tick) # Assert - assert self.exchange.get_book(USDJPY_SIM.id).book_type == BookType.L1_MBP - assert self.exchange.best_ask_price(USDJPY_SIM.id) == Price.from_str("90.005") - assert self.exchange.best_bid_price(USDJPY_SIM.id) == Price.from_str("90.002") + assert self.exchange.get_book(_USDJPY_SIM.id).book_type == BookType.L1_MBP + assert self.exchange.best_ask_price(_USDJPY_SIM.id) == Price.from_str("90.005") + assert self.exchange.best_bid_price(_USDJPY_SIM.id) == Price.from_str("90.002") def test_process_trade_tick_updates_market(self) -> None: # Arrange tick1 = TestDataStubs.trade_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, aggressor_side=AggressorSide.BUYER, ) tick2 = TestDataStubs.trade_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, aggressor_side=AggressorSide.SELLER, ) @@ -272,8 +273,8 @@ def test_process_trade_tick_updates_market(self) -> None: self.exchange.process_trade_tick(tick2) # Assert - assert self.exchange.best_bid_price(USDJPY_SIM.id) == Price.from_str("1.00000") - assert self.exchange.best_ask_price(USDJPY_SIM.id) == Price.from_str("1.00000") + assert self.exchange.best_bid_price(_USDJPY_SIM.id) == Price.from_str("1.00000") + assert self.exchange.best_ask_price(_USDJPY_SIM.id) == Price.from_str("1.00000") @pytest.mark.parametrize( "side", @@ -288,7 +289,7 @@ def test_submit_limit_order_with_no_market_accepts_order( ) -> None: # Arrange order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), Price.from_str("110.000"), @@ -326,7 +327,7 @@ def test_submit_limit_order_with_immediate_modify( ) -> None: # Arrange order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), price, @@ -366,7 +367,7 @@ def test_submit_limit_order_with_immediate_cancel( ) -> None: # Arrange order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), price, @@ -399,7 +400,7 @@ def test_submit_market_order_with_no_market_rejects_order( ) -> None: # Arrange order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), ) @@ -426,7 +427,7 @@ def test_submit_sell_market_order_with_no_market_rejects_order( ) -> None: # Arrange order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), ) @@ -443,7 +444,7 @@ def test_submit_sell_market_order_with_no_market_rejects_order( def test_submit_order_with_invalid_price_gets_rejected(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -451,7 +452,7 @@ def test_submit_order_with_invalid_price_gets_rejected(self) -> None: self.portfolio.update_quote_tick(tick) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.005"), # Price at ask @@ -467,7 +468,7 @@ def test_submit_order_with_invalid_price_gets_rejected(self) -> None: def test_submit_order_when_quantity_below_min_then_gets_denied(self) -> None: # Arrange: Prepare market order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(1), # <-- Below minimum quantity for instrument ) @@ -481,7 +482,7 @@ def test_submit_order_when_quantity_below_min_then_gets_denied(self) -> None: def test_submit_order_when_quantity_above_max_then_gets_denied(self) -> None: # Arrange: Prepare market order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity(1e8, 0), # <-- Above maximum quantity for instrument ) @@ -495,7 +496,7 @@ def test_submit_order_when_quantity_above_max_then_gets_denied(self) -> None: def test_submit_market_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -504,7 +505,7 @@ def test_submit_market_order(self) -> None: # Create order order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -520,7 +521,7 @@ def test_submit_market_order(self) -> None: def test_submit_market_order_then_immediately_cancel_submits_and_fills(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -529,7 +530,7 @@ def test_submit_market_order_then_immediately_cancel_submits_and_fills(self) -> # Create order order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -545,7 +546,7 @@ def test_submit_market_order_then_immediately_cancel_submits_and_fills(self) -> def test_submit_market_order_with_fok_time_in_force_cancels_immediately(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=500_000, @@ -556,7 +557,7 @@ def test_submit_market_order_with_fok_time_in_force_cancels_immediately(self) -> # Create order order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(1_000_000), time_in_force=TimeInForce.FOK, @@ -574,7 +575,7 @@ def test_submit_market_order_with_fok_time_in_force_cancels_immediately(self) -> def test_submit_market_order_with_ioc_time_in_force_cancels_remaining_qty(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=500_000, @@ -585,7 +586,7 @@ def test_submit_market_order_with_ioc_time_in_force_cancels_remaining_qty(self) # Create order order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(1_000_000), time_in_force=TimeInForce.IOC, @@ -603,7 +604,7 @@ def test_submit_market_order_with_ioc_time_in_force_cancels_remaining_qty(self) def test_submit_limit_order_then_immediately_cancel_submits_then_cancels(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -611,7 +612,7 @@ def test_submit_limit_order_then_immediately_cancel_submits_then_cancels(self) - self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), @@ -629,7 +630,7 @@ def test_submit_limit_order_then_immediately_cancel_submits_then_cancels(self) - def test_submit_post_only_limit_order_when_marketable_then_rejects(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -637,7 +638,7 @@ def test_submit_post_only_limit_order_when_marketable_then_rejects(self) -> None self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.005"), @@ -655,7 +656,7 @@ def test_submit_post_only_limit_order_when_marketable_then_rejects(self) -> None def test_submit_limit_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -663,7 +664,7 @@ def test_submit_limit_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.001"), @@ -681,7 +682,7 @@ def test_submit_limit_order(self) -> None: def test_submit_limit_order_with_ioc_time_in_force_immediately_cancels(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=500_000, @@ -692,7 +693,7 @@ def test_submit_limit_order_with_ioc_time_in_force_immediately_cancels(self) -> # Create order order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(1_000_000), Price.from_int(1), @@ -713,7 +714,7 @@ def test_submit_limit_order_with_ioc_time_in_force_immediately_cancels(self) -> def test_submit_limit_order_with_fok_time_in_force_immediately_cancels(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=500_000, @@ -724,7 +725,7 @@ def test_submit_limit_order_with_fok_time_in_force_immediately_cancels(self) -> # Create order order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(1_000_000), Price.from_int(1), @@ -745,7 +746,7 @@ def test_submit_limit_order_with_fok_time_in_force_immediately_cancels(self) -> def test_submit_market_to_limit_order_less_than_available_top_of_book(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -753,7 +754,7 @@ def test_submit_market_to_limit_order_less_than_available_top_of_book(self) -> N self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market_to_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -770,7 +771,7 @@ def test_submit_market_to_limit_order_less_than_available_top_of_book(self) -> N def test_submit_market_to_limit_order_greater_than_available_top_of_book(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=1_000_000, @@ -780,7 +781,7 @@ def test_submit_market_to_limit_order_greater_than_available_top_of_book(self) - self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market_to_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(2_000_000), ) @@ -799,7 +800,7 @@ def test_submit_market_to_limit_order_greater_than_available_top_of_book(self) - def test_modify_market_to_limit_order_after_filling_initial_quantity(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=1_000_000, @@ -809,7 +810,7 @@ def test_modify_market_to_limit_order_after_filling_initial_quantity(self) -> No self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market_to_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(2_000_000), ) @@ -835,7 +836,7 @@ def test_modify_market_to_limit_order_after_filling_initial_quantity(self) -> No def test_submit_market_to_limit_order_becomes_limit_then_fills_remaining(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=1_000_000, @@ -845,7 +846,7 @@ def test_submit_market_to_limit_order_becomes_limit_then_fills_remaining(self) - self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market_to_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(2_000_000), ) @@ -855,7 +856,7 @@ def test_submit_market_to_limit_order_becomes_limit_then_fills_remaining(self) - self.exchange.process(0) tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, # <-- hit bid again bid_size=1_000_000, @@ -874,7 +875,7 @@ def test_submit_market_to_limit_order_becomes_limit_then_fills_remaining(self) - def test_submit_market_if_touched_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -882,7 +883,7 @@ def test_submit_market_if_touched_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market_if_touched( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), @@ -900,7 +901,7 @@ def test_submit_market_if_touched_order(self) -> None: def test_submit_limit_if_touched_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -908,7 +909,7 @@ def test_submit_limit_if_touched_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit_if_touched( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.010"), @@ -927,7 +928,7 @@ def test_submit_limit_if_touched_order(self) -> None: def test_submit_limit_order_when_marketable_then_fills(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -935,7 +936,7 @@ def test_submit_limit_order_when_marketable_then_fills(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.010"), @@ -955,7 +956,7 @@ def test_submit_limit_order_when_marketable_then_fills(self) -> None: def test_submit_limit_order_fills_at_correct_price(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -963,7 +964,7 @@ def test_submit_limit_order_fills_at_correct_price(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), # <-- Limit price above the ask @@ -975,7 +976,7 @@ def test_submit_limit_order_fills_at_correct_price(self) -> None: self.exchange.process(0) tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=89.900, ask_price=89.950, ) @@ -990,7 +991,7 @@ def test_submit_limit_order_fills_at_correct_price(self) -> None: def test_submit_limit_order_fills_at_most_book_volume(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=1_000_000, @@ -1000,7 +1001,7 @@ def test_submit_limit_order_fills_at_most_book_volume(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(2_000_000), # <-- Order volume greater than available ask volume Price.from_str("90.010"), @@ -1018,7 +1019,7 @@ def test_submit_limit_order_fills_at_most_book_volume(self) -> None: def test_submit_market_if_touched_order_then_fills(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1026,7 +1027,7 @@ def test_submit_market_if_touched_order_then_fills(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market_if_touched( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(10_000), # <-- Order volume greater than available ask volume Price.from_str("90.000"), @@ -1038,7 +1039,7 @@ def test_submit_market_if_touched_order_then_fills(self) -> None: # Quantity is refreshed -> Ensure we don't trade the entire amount tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, ask_price=90.0, ask_size=10_000, ) @@ -1072,7 +1073,7 @@ def test_submit_limit_if_touched_order_then_fills( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.000, ask_price=90.010, ) @@ -1080,7 +1081,7 @@ def test_submit_limit_if_touched_order_then_fills( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit_if_touched( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(10_000), # <-- Order volume greater than available ask volume price=price, @@ -1093,7 +1094,7 @@ def test_submit_limit_if_touched_order_then_fills( # Quantity is refreshed -> Ensure we don't trade the entire amount tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.010, # <-- in cross for purpose of test ask_price=90.000, bid_size=10_000, @@ -1120,7 +1121,7 @@ def test_submit_limit_order_fills_at_most_order_volume( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.005, ask_price=90.005, bid_size=Quantity.from_int(10_000), @@ -1130,7 +1131,7 @@ def test_submit_limit_order_fills_at_most_order_volume( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(15_000), # <-- Order volume greater than available ask volume price, @@ -1145,7 +1146,7 @@ def test_submit_limit_order_fills_at_most_order_volume( # Quantity is refreshed -> Ensure we don't trade the entire amount tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.005, ask_price=90.005, bid_size=10_000, @@ -1172,7 +1173,7 @@ def test_submit_stop_market_order_inside_market_rejects( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1180,7 +1181,7 @@ def test_submit_stop_market_order_inside_market_rejects( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), trigger_price, @@ -1217,7 +1218,7 @@ def test_submit_stop_limit_order_inside_market_rejects( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1225,7 +1226,7 @@ def test_submit_stop_limit_order_inside_market_rejects( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), price=price, @@ -1260,7 +1261,7 @@ def test_submit_stop_market_order( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1268,7 +1269,7 @@ def test_submit_stop_market_order( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), trigger_price=trigger_price, @@ -1306,7 +1307,7 @@ def test_submit_stop_limit_order_when_inside_market_rejects( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1314,7 +1315,7 @@ def test_submit_stop_limit_order_when_inside_market_rejects( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), price=price, @@ -1347,7 +1348,7 @@ def test_submit_stop_limit_order_when_inside_market_rejects( def test_submit_stop_limit_order(self, side, price, trigger_price) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1355,7 +1356,7 @@ def test_submit_stop_limit_order(self, side, price, trigger_price) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), price=price, @@ -1381,7 +1382,7 @@ def test_submit_stop_limit_order(self, side, price, trigger_price) -> None: def test_submit_reduce_only_order_when_no_position_rejects(self, side: OrderSide) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1389,7 +1390,7 @@ def test_submit_reduce_only_order_when_no_position_rejects(self, side: OrderSide self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), reduce_only=True, @@ -1416,7 +1417,7 @@ def test_submit_reduce_only_order_when_would_increase_position_rejects( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1424,14 +1425,14 @@ def test_submit_reduce_only_order_when_would_increase_position_rejects( self.exchange.process_quote_tick(tick) order1 = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), reduce_only=False, ) order2 = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, side, Quantity.from_int(100_000), reduce_only=True, # <-- reduce only set @@ -1452,7 +1453,7 @@ def test_submit_reduce_only_order_when_would_increase_position_rejects( def test_cancel_stop_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1460,7 +1461,7 @@ def test_cancel_stop_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.010"), @@ -1482,7 +1483,7 @@ def test_cancel_stop_order_when_order_does_not_exist_generates_cancel_reject(sel command = CancelOrder( trader_id=self.trader_id, strategy_id=StrategyId("SCALPER-001"), - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, client_order_id=ClientOrderId("O-123456"), venue_order_id=VenueOrderId("001"), command_id=UUID4(), @@ -1499,7 +1500,7 @@ def test_cancel_stop_order_when_order_does_not_exist_generates_cancel_reject(sel def test_cancel_all_orders_with_no_side_filter_cancels_all(self): # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1507,14 +1508,14 @@ def test_cancel_all_orders_with_no_side_filter_cancels_all(self): self.exchange.process_quote_tick(tick) order1 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), ) order2 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), @@ -1525,7 +1526,7 @@ def test_cancel_all_orders_with_no_side_filter_cancels_all(self): self.exchange.process(0) # Act - self.strategy.cancel_all_orders(instrument_id=USDJPY_SIM.id) + self.strategy.cancel_all_orders(instrument_id=_USDJPY_SIM.id) self.exchange.process(0) # Assert @@ -1536,7 +1537,7 @@ def test_cancel_all_orders_with_no_side_filter_cancels_all(self): def test_cancel_all_orders_with_buy_side_filter_cancels_all_buy_orders(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1544,21 +1545,21 @@ def test_cancel_all_orders_with_buy_side_filter_cancels_all_buy_orders(self) -> self.exchange.process_quote_tick(tick) order1 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), ) order2 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), ) order3 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.010"), @@ -1571,7 +1572,7 @@ def test_cancel_all_orders_with_buy_side_filter_cancels_all_buy_orders(self) -> # Act self.strategy.cancel_all_orders( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.BUY, ) self.exchange.process(0) @@ -1585,7 +1586,7 @@ def test_cancel_all_orders_with_buy_side_filter_cancels_all_buy_orders(self) -> def test_cancel_all_orders_with_sell_side_filter_cancels_all_sell_orders(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1593,21 +1594,21 @@ def test_cancel_all_orders_with_sell_side_filter_cancels_all_sell_orders(self) - self.exchange.process_quote_tick(tick) order1 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.010"), ) order2 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.010"), ) order3 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.000"), @@ -1620,7 +1621,7 @@ def test_cancel_all_orders_with_sell_side_filter_cancels_all_sell_orders(self) - # Act self.strategy.cancel_all_orders( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.SELL, ) self.exchange.process(0) @@ -1634,7 +1635,7 @@ def test_cancel_all_orders_with_sell_side_filter_cancels_all_sell_orders(self) - def test_batch_cancel_orders_all_open_orders_for_batch(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1642,28 +1643,28 @@ def test_batch_cancel_orders_all_open_orders_for_batch(self) -> None: self.exchange.process_quote_tick(tick) order1 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.030"), ) order2 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.020"), ) order3 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.010"), ) order4 = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.010"), @@ -1695,7 +1696,7 @@ def test_modify_stop_order_when_order_does_not_exist(self) -> None: command = ModifyOrder( trader_id=self.trader_id, strategy_id=StrategyId("SCALPER-001"), - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, client_order_id=ClientOrderId("O-123456"), venue_order_id=VenueOrderId("001"), quantity=Quantity.from_int(100_000), @@ -1715,7 +1716,7 @@ def test_modify_stop_order_when_order_does_not_exist(self) -> None: def test_modify_order_with_zero_quantity_rejects_modify(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1723,7 +1724,7 @@ def test_modify_order_with_zero_quantity_rejects_modify(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.001"), @@ -1745,7 +1746,7 @@ def test_modify_order_with_zero_quantity_rejects_modify(self) -> None: def test_modify_post_only_limit_order_when_marketable_then_rejects_modify(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1753,7 +1754,7 @@ def test_modify_post_only_limit_order_when_marketable_then_rejects_modify(self) self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.001"), @@ -1775,7 +1776,7 @@ def test_modify_post_only_limit_order_when_marketable_then_rejects_modify(self) def test_modify_limit_order_when_marketable_then_fills_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1783,7 +1784,7 @@ def test_modify_limit_order_when_marketable_then_fills_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.001"), @@ -1807,7 +1808,7 @@ def test_modify_stop_market_order_when_price_inside_market_then_rejects_modify( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=Price.from_str("90.002"), ask_price=Price.from_str("90.005"), ) @@ -1815,7 +1816,7 @@ def test_modify_stop_market_order_when_price_inside_market_then_rejects_modify( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.010"), @@ -1840,7 +1841,7 @@ def test_modify_stop_market_order_when_price_inside_market_then_rejects_modify( def test_modify_stop_market_order_when_price_valid_then_updates(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1848,7 +1849,7 @@ def test_modify_stop_market_order_when_price_valid_then_updates(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.010"), @@ -1896,7 +1897,7 @@ def test_modify_limit_if_touched( ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.005, ask_price=90.005, ) @@ -1904,7 +1905,7 @@ def test_modify_limit_if_touched( self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit_if_touched( - USDJPY_SIM.id, + _USDJPY_SIM.id, order_side, Quantity.from_int(100_000), price=price, @@ -1932,7 +1933,7 @@ def test_modify_untriggered_stop_limit_order_when_price_inside_market_then_rejec ) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1940,7 +1941,7 @@ def test_modify_untriggered_stop_limit_order_when_price_inside_market_then_rejec self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), price=Price.from_str("90.000"), @@ -1966,7 +1967,7 @@ def test_modify_untriggered_stop_limit_order_when_price_inside_market_then_rejec def test_modify_untriggered_stop_limit_order_when_price_valid_then_amends(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -1974,7 +1975,7 @@ def test_modify_untriggered_stop_limit_order_when_price_valid_then_amends(self) self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), price=Price.from_str("90.000"), @@ -2003,7 +2004,7 @@ def test_modify_triggered_post_only_stop_limit_order_when_price_inside_market_th ) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2011,7 +2012,7 @@ def test_modify_triggered_post_only_stop_limit_order_when_price_inside_market_th self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), price=Price.from_str("90.000"), @@ -2024,7 +2025,7 @@ def test_modify_triggered_post_only_stop_limit_order_when_price_inside_market_th # Trigger order tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.009, ask_price=90.010, ) @@ -2050,7 +2051,7 @@ def test_modify_triggered_stop_limit_order_when_price_inside_market_then_fills( ) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2058,7 +2059,7 @@ def test_modify_triggered_stop_limit_order_when_price_inside_market_then_fills( self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), price=Price.from_str("90.000"), @@ -2071,7 +2072,7 @@ def test_modify_triggered_stop_limit_order_when_price_inside_market_then_fills( # Trigger order tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.009, ask_price=90.010, ) @@ -2095,7 +2096,7 @@ def test_modify_triggered_stop_limit_order_when_price_inside_market_then_fills( def test_modify_triggered_stop_limit_order_when_price_valid_then_amends(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2103,7 +2104,7 @@ def test_modify_triggered_stop_limit_order_when_price_valid_then_amends(self) -> self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), price=Price.from_str("90.000"), @@ -2115,7 +2116,7 @@ def test_modify_triggered_stop_limit_order_when_price_valid_then_amends(self) -> # Trigger order tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.009, ask_price=90.010, ) @@ -2136,10 +2137,10 @@ def test_modify_triggered_stop_limit_order_when_price_valid_then_amends(self) -> assert len(self.exchange.get_open_orders()) == 1 assert order.price == Price.from_str("90.005") - def test_order_fills_gets_commissioned(self) -> None: + def test_order_fills_gets_commissioned_for_fx(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2147,19 +2148,19 @@ def test_order_fills_gets_commissioned(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) top_up_order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) reduce_order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(50_000), ) @@ -2183,12 +2184,12 @@ def test_order_fills_gets_commissioned(self) -> None: assert fill_event1.commission == Money(180, JPY) assert fill_event2.commission == Money(180, JPY) assert fill_event3.commission == Money(90, JPY) - assert Money(999995.00, USD), self.exchange.get_account().balance_total(USD) + assert Money(999995.00, USD) == self.exchange.get_account().balance_total(USD) def test_expire_order(self) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2196,7 +2197,7 @@ def test_expire_order(self) -> None: self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("96.711"), @@ -2208,7 +2209,7 @@ def test_expire_order(self) -> None: self.exchange.process(0) tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=96.709, ask_price=96.710, ts_event=1 * 60 * 1_000_000_000, # 1 minute in nanoseconds @@ -2225,7 +2226,7 @@ def test_expire_order(self) -> None: def test_process_quote_tick_fills_buy_stop_order(self) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, bid_size=1_000_000, @@ -2235,7 +2236,7 @@ def test_process_quote_tick_fills_buy_stop_order(self) -> None: self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("96.711"), @@ -2246,7 +2247,7 @@ def test_process_quote_tick_fills_buy_stop_order(self) -> None: # Act tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=96.710, ask_price=96.711, bid_size=1_000_000, @@ -2264,7 +2265,7 @@ def test_process_quote_tick_fills_buy_stop_order(self) -> None: def test_process_quote_tick_triggers_buy_stop_limit_order(self) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2272,7 +2273,7 @@ def test_process_quote_tick_triggers_buy_stop_limit_order(self) -> None: self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("96.500"), # LimitPx @@ -2284,7 +2285,7 @@ def test_process_quote_tick_triggers_buy_stop_limit_order(self) -> None: # Act tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=96.710, ask_price=96.712, ) @@ -2298,7 +2299,7 @@ def test_process_quote_tick_triggers_buy_stop_limit_order(self) -> None: def test_process_quote_tick_rejects_triggered_post_only_buy_stop_limit_order(self) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2306,7 +2307,7 @@ def test_process_quote_tick_rejects_triggered_post_only_buy_stop_limit_order(sel self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), price=Price.from_str("90.006"), @@ -2319,7 +2320,7 @@ def test_process_quote_tick_rejects_triggered_post_only_buy_stop_limit_order(sel # Act tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.005, ask_price=90.006, ts_event=1_000_000_000, @@ -2335,7 +2336,7 @@ def test_process_quote_tick_rejects_triggered_post_only_buy_stop_limit_order(sel def test_process_quote_tick_fills_triggered_buy_stop_limit_order(self) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2343,7 +2344,7 @@ def test_process_quote_tick_fills_triggered_buy_stop_limit_order(self) -> None: self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.stop_limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), price=Price.from_str("90.001"), @@ -2354,7 +2355,7 @@ def test_process_quote_tick_fills_triggered_buy_stop_limit_order(self) -> None: self.exchange.process(0) tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.006, ask_price=90.007, ts_event=1_000_000_000, @@ -2363,7 +2364,7 @@ def test_process_quote_tick_fills_triggered_buy_stop_limit_order(self) -> None: # Act tick3 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.000, ask_price=90.001, ts_event=100_000, @@ -2380,7 +2381,7 @@ def test_process_quote_tick_fills_triggered_buy_stop_limit_order(self) -> None: def test_process_quote_tick_fills_buy_limit_order(self) -> None: # Arrange: Prepare market tick1 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2388,7 +2389,7 @@ def test_process_quote_tick_fills_buy_limit_order(self) -> None: self.exchange.process_quote_tick(tick1) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("90.001"), @@ -2399,7 +2400,7 @@ def test_process_quote_tick_fills_buy_limit_order(self) -> None: # Act tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.000, ask_price=90.001, ts_event=100_000, @@ -2417,7 +2418,7 @@ def test_process_quote_tick_fills_buy_limit_order(self) -> None: def test_process_quote_tick_fills_sell_stop_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2425,7 +2426,7 @@ def test_process_quote_tick_fills_sell_stop_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.stop_market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.000"), @@ -2436,7 +2437,7 @@ def test_process_quote_tick_fills_sell_stop_order(self) -> None: # Act tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=89.997, ask_price=89.999, ) @@ -2452,7 +2453,7 @@ def test_process_quote_tick_fills_sell_stop_order(self) -> None: def test_process_quote_tick_fills_sell_limit_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2460,7 +2461,7 @@ def test_process_quote_tick_fills_sell_limit_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.100"), @@ -2471,7 +2472,7 @@ def test_process_quote_tick_fills_sell_limit_order(self) -> None: # Act tick2 = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.101, ask_price=90.102, ) @@ -2487,7 +2488,7 @@ def test_process_quote_tick_fills_sell_limit_order(self) -> None: def test_process_trade_tick_fills_sell_limit_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2495,7 +2496,7 @@ def test_process_trade_tick_fills_sell_limit_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("90.100"), @@ -2506,7 +2507,7 @@ def test_process_trade_tick_fills_sell_limit_order(self) -> None: # Act trade = TestDataStubs.trade_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, price=91.000, ) @@ -2521,7 +2522,7 @@ def test_process_trade_tick_fills_sell_limit_order(self) -> None: def test_realized_pnl_contains_commission(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2529,7 +2530,7 @@ def test_realized_pnl_contains_commission(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -2546,7 +2547,7 @@ def test_realized_pnl_contains_commission(self) -> None: def test_unrealized_pnl(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2554,7 +2555,7 @@ def test_unrealized_pnl(self) -> None: self.exchange.process_quote_tick(tick) order_open = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -2564,7 +2565,7 @@ def test_unrealized_pnl(self) -> None: self.exchange.process(0) quote = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=100.003, ask_price=100.004, ) @@ -2573,7 +2574,7 @@ def test_unrealized_pnl(self) -> None: self.portfolio.update_quote_tick(quote) order_reduce = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(50_000), ) @@ -2586,8 +2587,8 @@ def test_unrealized_pnl(self) -> None: # Assert position = self.cache.positions_open()[0] - assert self.exchange.best_bid_price(USDJPY_SIM.id) == Price.from_str("100.003") - assert self.exchange.best_ask_price(USDJPY_SIM.id) == Price.from_str("100.004") + assert self.exchange.best_bid_price(_USDJPY_SIM.id) == Price.from_str("100.003") + assert self.exchange.best_ask_price(_USDJPY_SIM.id) == Price.from_str("100.004") assert position.unrealized_pnl(Price.from_str("100.003")) == Money(499900, JPY) def test_adjust_account_changes_balance(self) -> None: @@ -2611,7 +2612,7 @@ def test_adjust_account_when_account_frozen_does_not_change_balance(self) -> Non starting_balances=[Money(1_000_000, USD)], default_leverage=Decimal(50), leverages={}, - instruments=[USDJPY_SIM], + instruments=[_USDJPY_SIM], modules=[], fill_model=FillModel(), portfolio=self.portfolio, @@ -2635,7 +2636,7 @@ def test_adjust_account_when_account_frozen_does_not_change_balance(self) -> Non def test_position_flipped_when_reduce_order_exceeds_original_quantity(self) -> None: # Arrange: Prepare market open_quote = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.003, bid_size=1_000_000, @@ -2646,7 +2647,7 @@ def test_position_flipped_when_reduce_order_exceeds_original_quantity(self) -> N self.exchange.process_quote_tick(open_quote) order_open = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -2655,7 +2656,7 @@ def test_position_flipped_when_reduce_order_exceeds_original_quantity(self) -> N self.exchange.process(0) reduce_quote = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=100.003, ask_price=100.004, bid_size=1_000_000, @@ -2666,7 +2667,7 @@ def test_position_flipped_when_reduce_order_exceeds_original_quantity(self) -> N self.portfolio.update_quote_tick(reduce_quote) order_reduce = self.strategy.order_factory.market( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(150_000), ) @@ -2690,7 +2691,7 @@ def test_position_flipped_when_reduce_order_exceeds_original_quantity(self) -> N def test_reduce_only_market_order_does_not_open_position_on_flip_scenario(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=14.0, ask_price=13.0, bid_size=1_000_000, @@ -2699,7 +2700,7 @@ def test_reduce_only_market_order_does_not_open_position_on_flip_scenario(self) self.exchange.process_quote_tick(tick) entry = self.strategy.order_factory.market( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.SELL, quantity=Quantity.from_int(200_000), ) @@ -2707,7 +2708,7 @@ def test_reduce_only_market_order_does_not_open_position_on_flip_scenario(self) self.exchange.process(0) exit = self.strategy.order_factory.market( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.BUY, quantity=Quantity.from_int(300_000), # <-- overfill to attempt flip reduce_only=True, @@ -2721,7 +2722,7 @@ def test_reduce_only_market_order_does_not_open_position_on_flip_scenario(self) def test_reduce_only_limit_order_does_not_open_position_on_flip_scenario(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=14.0, ask_price=13.0, bid_size=1_000_000, @@ -2730,7 +2731,7 @@ def test_reduce_only_limit_order_does_not_open_position_on_flip_scenario(self) - self.exchange.process_quote_tick(tick) entry = self.strategy.order_factory.market( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.SELL, quantity=Quantity.from_int(200_000), ) @@ -2738,7 +2739,7 @@ def test_reduce_only_limit_order_does_not_open_position_on_flip_scenario(self) - self.exchange.process(0) exit = self.strategy.order_factory.limit( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.BUY, quantity=Quantity.from_int(300_000), # <-- overfill to attempt flip price=Price.from_str("11"), @@ -2749,7 +2750,7 @@ def test_reduce_only_limit_order_does_not_open_position_on_flip_scenario(self) - self.exchange.process(0) tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=10.0, ask_price=11.0, bid_size=1_000_000, @@ -2764,7 +2765,7 @@ def test_latency_model_submit_order(self) -> None: # Arrange self.exchange.set_latency_model(LatencyModel(secs_to_nanos(1))) entry = self.strategy.order_factory.limit( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.BUY, price=Price.from_int(100), quantity=Quantity.from_int(200_000), @@ -2783,7 +2784,7 @@ def test_latency_model_cancel_order(self) -> None: # Arrange self.exchange.set_latency_model(LatencyModel(secs_to_nanos(1))) entry = self.strategy.order_factory.limit( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.BUY, price=Price.from_int(100), quantity=Quantity.from_int(200_000), @@ -2803,7 +2804,7 @@ def test_latency_model_modify_order(self) -> None: # Arrange self.exchange.set_latency_model(LatencyModel(secs_to_nanos(1))) entry = self.strategy.order_factory.limit( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.BUY, price=Price.from_int(100), quantity=Quantity.from_int(200_000), @@ -2823,7 +2824,7 @@ def test_latency_model_large_int(self) -> None: # Arrange self.exchange.set_latency_model(LatencyModel(secs_to_nanos(10))) entry = self.strategy.order_factory.limit( - instrument_id=USDJPY_SIM.id, + instrument_id=_USDJPY_SIM.id, order_side=OrderSide.BUY, price=Price.from_int(100), quantity=Quantity.from_int(200_000), @@ -2908,8 +2909,8 @@ def reset(self): base_currency=USD, starting_balances=[Money(1_000_000, USD)], default_leverage=Decimal(50), - leverages={AUDUSD_SIM.id: Decimal(10)}, - instruments=[USDJPY_SIM], + leverages={_AUDUSD_SIM.id: Decimal(10)}, + instruments=[_USDJPY_SIM], modules=[self.module], fill_model=FillModel(), portfolio=self.portfolio, @@ -2931,7 +2932,7 @@ def reset(self): self.exec_engine.register_client(self.exec_client) self.exchange.register_client(self.exec_client) - self.cache.add_instrument(USDJPY_SIM) + self.cache.add_instrument(_USDJPY_SIM) # Create mock strategy self.strategy = MockStrategy(bar_type=TestDataStubs.bartype_usdjpy_1min_bid()) @@ -2952,7 +2953,7 @@ def reset(self): def test_process_trade_tick_fills_sell_limit_order(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, bid_price=90.002, ask_price=90.005, ) @@ -2960,7 +2961,7 @@ def test_process_trade_tick_fills_sell_limit_order(self) -> None: self.exchange.process_quote_tick(tick) order = self.strategy.order_factory.limit( - USDJPY_SIM.id, + _USDJPY_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("91.000"), @@ -2971,7 +2972,7 @@ def test_process_trade_tick_fills_sell_limit_order(self) -> None: # Act trade = TestDataStubs.trade_tick( - instrument=USDJPY_SIM, + instrument=_USDJPY_SIM, price=91.000, ) self.module.pre_process(trade) diff --git a/tests/unit_tests/backtest/test_matching_engine.py b/tests/unit_tests/backtest/test_matching_engine.py index 54d36a284a73..10f95aef07c7 100644 --- a/tests/unit_tests/backtest/test_matching_engine.py +++ b/tests/unit_tests/backtest/test_matching_engine.py @@ -21,6 +21,7 @@ from nautilus_trader.backtest.models import FillModel from nautilus_trader.common.component import MessageBus from nautilus_trader.common.component import TestClock +from nautilus_trader.model.enums import AccountType from nautilus_trader.model.enums import BookType from nautilus_trader.model.enums import MarketStatus from nautilus_trader.model.enums import OmsType @@ -35,7 +36,7 @@ from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs -ETHUSDT_PERP_BINANCE = TestInstrumentProvider.ethusdt_perp_binance() +_ETHUSDT_PERP_BINANCE = TestInstrumentProvider.ethusdt_perp_binance() class TestOrderMatchingEngine: @@ -48,7 +49,7 @@ def setup(self): trader_id=self.trader_id, clock=self.clock, ) - self.instrument = ETHUSDT_PERP_BINANCE + self.instrument = _ETHUSDT_PERP_BINANCE self.instrument_id = self.instrument.id self.account_id = TestIdStubs.account_id() self.cache = TestComponentStubs.cache() @@ -60,6 +61,7 @@ def setup(self): fill_model=FillModel(), book_type=BookType.L1_MBP, oms_type=OmsType.NETTING, + account_type=AccountType.MARGIN, reject_stop_orders=True, msgbus=self.msgbus, cache=self.cache, From 78d0d498f3890e5727775f41ce4f6e3dfa094d53 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 14 Feb 2024 19:47:08 +1100 Subject: [PATCH 040/104] Skip Windows in CI while GH runners broken --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85cfefcca5de..bba01e42a980 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: arch: [x64] - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] # windows-latest python-version: ["3.10", "3.11", "3.12"] defaults: run: From bf324846a313b3446ef8d8ddc5eafba4f7480649 Mon Sep 17 00:00:00 2001 From: Benjamin Singleton Date: Wed, 14 Feb 2024 04:31:51 -0500 Subject: [PATCH 041/104] Fix InteractiveBrokers historical bar data bug (#1499) --- .../adapters/interactive_brokers/client/common.py | 3 ++- .../interactive_brokers/client/market_data.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/nautilus_trader/adapters/interactive_brokers/client/common.py b/nautilus_trader/adapters/interactive_brokers/client/common.py index caa22d39f3eb..7b6235fdd514 100644 --- a/nautilus_trader/adapters/interactive_brokers/client/common.py +++ b/nautilus_trader/adapters/interactive_brokers/client/common.py @@ -14,6 +14,7 @@ # ------------------------------------------------------------------------------------------------- import asyncio +import functools from abc import ABC from abc import abstractmethod from collections.abc import Callable @@ -70,7 +71,7 @@ class Subscription(msgspec.Struct, frozen=True): req_id: Annotated[int, msgspec.Meta(gt=0)] name: str | tuple - handle: Callable + handle: functools.partial | Callable cancel: Callable last: Any diff --git a/nautilus_trader/adapters/interactive_brokers/client/market_data.py b/nautilus_trader/adapters/interactive_brokers/client/market_data.py index c27e47d5e2ec..a3ecdb4cbd78 100644 --- a/nautilus_trader/adapters/interactive_brokers/client/market_data.py +++ b/nautilus_trader/adapters/interactive_brokers/client/market_data.py @@ -78,7 +78,7 @@ async def set_market_data_type(self, market_data_type: MarketDataTypeEnum) -> No async def _subscribe( self, name: str | tuple, - subscription_method: Callable, + subscription_method: Callable | functools.partial, cancellation_method: Callable, *args: Any, **kwargs: Any, @@ -274,10 +274,10 @@ async def subscribe_historical_bars( name, self.subscribe_historical_bars, self._eclient.cancelHistoricalData, - bar_type, - contract, - use_rth, - handle_revised_bars, + bar_type=bar_type, + contract=contract, + use_rth=use_rth, + handle_revised_bars=handle_revised_bars, ) if not subscription: return @@ -815,10 +815,12 @@ def historicalDataUpdate(self, req_id: int, bar: BarData) -> None: self.logAnswer(current_fn_name(), vars()) if not (subscription := self._subscriptions.get(req_id=req_id)): return + if not isinstance(subscription.handle, functools.partial): + raise TypeError(f"Expecting partial type subscription method. {subscription=}") if bar := self._process_bar_data( bar_type_str=str(subscription.name), bar=bar, - handle_revised_bars=subscription.handle().keywords.get("handle_revised_bars", False), + handle_revised_bars=subscription.handle.keywords.get("handle_revised_bars", False), ): if bar.is_single_price() and bar.open.as_double() == 0: self._log.debug(f"Ignoring Zero priced {bar=}") From be3a5bbf716b8856eadbab2c3be87a22128fd7f3 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 14 Feb 2024 21:00:37 +1100 Subject: [PATCH 042/104] Fix logging timestamps for backtesting --- RELEASES.md | 1 + nautilus_trader/backtest/engine.pyx | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index c44e17cb9297..97642b7d95a0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -9,6 +9,7 @@ None None ### Fixes +- Fixed logging timestamps for backtesting (static clock was not being incrementally set to individual `TimeEvent` timestamps) - Fixed `Equity` short selling for `CASH` accounts (will now reject) - Fixed `ActorFactory.create` JSON encoding (was missing the encoding hook) - Fixed `ImportableConfig.create` JSON encoding (was missing the encoding hook) diff --git a/nautilus_trader/backtest/engine.pyx b/nautilus_trader/backtest/engine.pyx index 40cbcb0bfee7..619887374d14 100644 --- a/nautilus_trader/backtest/engine.pyx +++ b/nautilus_trader/backtest/engine.pyx @@ -1156,8 +1156,6 @@ cdef class BacktestEngine: return self._data[cursor] cdef CVec _advance_time(self, uint64_t ts_now, list clocks): - set_logging_clock_static_time(ts_now) - cdef TestClock clock for clock in clocks: time_event_accumulator_advance_clock( @@ -1178,6 +1176,7 @@ cdef class BacktestEngine: ) # Set all clocks to now + set_logging_clock_static_time(ts_now) for clock in clocks: clock.set_time(ts_now) @@ -1206,8 +1205,12 @@ cdef class BacktestEngine: ts_event_init = raw_handler.event.ts_init if (only_now and ts_event_init < ts_now) or (not only_now and ts_event_init == ts_now): continue + + # Set all clocks to event timestamp + set_logging_clock_static_time(ts_event_init) for clock in clocks: clock.set_time(ts_event_init) + event = TimeEvent.from_mem_c(raw_handler.event) # Cast raw `PyObject *` to a `PyObject` From ed3334cbd904f78277ba18f4d30c9238b566d6f5 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 14 Feb 2024 21:20:31 +1100 Subject: [PATCH 043/104] Optimize applying OrderBookDeltas to OrderBook --- nautilus_core/model/src/ffi/orderbook/book.rs | 7 +++++++ nautilus_core/model/src/orderbook/book.rs | 10 ++++++++-- nautilus_trader/core/includes/model.h | 2 ++ nautilus_trader/core/rust/model.pxd | 2 ++ nautilus_trader/model/book.pyx | 5 ++--- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/nautilus_core/model/src/ffi/orderbook/book.rs b/nautilus_core/model/src/ffi/orderbook/book.rs index 5d208b58cb89..80e4a8af0226 100644 --- a/nautilus_core/model/src/ffi/orderbook/book.rs +++ b/nautilus_core/model/src/ffi/orderbook/book.rs @@ -27,6 +27,7 @@ use crate::{ trade::TradeTick, }, enums::{BookType, OrderSide}, + ffi::data::deltas::OrderBookDeltas_API, identifiers::instrument_id::InstrumentId, orderbook::book::OrderBook, types::{price::Price, quantity::Quantity}, @@ -148,6 +149,12 @@ pub extern "C" fn orderbook_apply_delta(book: &mut OrderBook_API, delta: OrderBo book.apply_delta(delta) } +#[no_mangle] +pub extern "C" fn orderbook_apply_deltas(book: &mut OrderBook_API, deltas: &OrderBookDeltas_API) { + // Clone will actually copy the contents of the `deltas` vec + book.apply_deltas(deltas.deref().clone()) +} + #[no_mangle] pub extern "C" fn orderbook_apply_depth(book: &mut OrderBook_API, depth: OrderBookDepth10) { book.apply_depth(depth) diff --git a/nautilus_core/model/src/orderbook/book.rs b/nautilus_core/model/src/orderbook/book.rs index 1ec846e15dd5..5712b83d5bb2 100644 --- a/nautilus_core/model/src/orderbook/book.rs +++ b/nautilus_core/model/src/orderbook/book.rs @@ -20,8 +20,8 @@ use thiserror::Error; use super::{ladder::BookPrice, level::Level}; use crate::{ data::{ - delta::OrderBookDelta, depth::OrderBookDepth10, order::BookOrder, quote::QuoteTick, - trade::TradeTick, + delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, order::BookOrder, + quote::QuoteTick, trade::TradeTick, }, enums::{BookAction, BookType, OrderSide}, identifiers::instrument_id::InstrumentId, @@ -167,6 +167,12 @@ impl OrderBook { } } + pub fn apply_deltas(&mut self, deltas: OrderBookDeltas) { + for delta in deltas.deltas { + self.apply_delta(delta) + } + } + pub fn apply_depth(&mut self, depth: OrderBookDepth10) { self.bids.clear(); self.asks.clear(); diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index c95d3b273fc2..313f6e19abee 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -2132,6 +2132,8 @@ void orderbook_clear_asks(struct OrderBook_API *book, uint64_t ts_event, uint64_ void orderbook_apply_delta(struct OrderBook_API *book, struct OrderBookDelta_t delta); +void orderbook_apply_deltas(struct OrderBook_API *book, const struct OrderBookDeltas_API *deltas); + void orderbook_apply_depth(struct OrderBook_API *book, struct OrderBookDepth10_t depth); CVec orderbook_bids(struct OrderBook_API *book); diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index a0ad361f5784..de4d50ba5c78 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -1471,6 +1471,8 @@ cdef extern from "../includes/model.h": void orderbook_apply_delta(OrderBook_API *book, OrderBookDelta_t delta); + void orderbook_apply_deltas(OrderBook_API *book, const OrderBookDeltas_API *deltas); + void orderbook_apply_depth(OrderBook_API *book, OrderBookDepth10_t depth); CVec orderbook_bids(OrderBook_API *book); diff --git a/nautilus_trader/model/book.pyx b/nautilus_trader/model/book.pyx index f5803fb8e7f5..06567a538d87 100644 --- a/nautilus_trader/model/book.pyx +++ b/nautilus_trader/model/book.pyx @@ -45,6 +45,7 @@ from nautilus_trader.core.rust.model cimport level_price from nautilus_trader.core.rust.model cimport level_size from nautilus_trader.core.rust.model cimport orderbook_add from nautilus_trader.core.rust.model cimport orderbook_apply_delta +from nautilus_trader.core.rust.model cimport orderbook_apply_deltas from nautilus_trader.core.rust.model cimport orderbook_apply_depth from nautilus_trader.core.rust.model cimport orderbook_asks from nautilus_trader.core.rust.model cimport orderbook_best_ask_price @@ -328,9 +329,7 @@ cdef class OrderBook(Data): """ Condition.not_none(deltas, "deltas") - cdef OrderBookDelta delta - for delta in deltas.deltas: - self.apply_delta(delta) + orderbook_apply_deltas(&self._mem, &deltas._mem) cpdef void apply_depth(self, OrderBookDepth10 depth): """ From e176ea62cd00527e60980c5f655de936f335252a Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Thu, 15 Feb 2024 18:56:27 +1100 Subject: [PATCH 044/104] Fix FOK and IOC time in force fill handling --- RELEASES.md | 2 + nautilus_trader/backtest/matching_engine.pyx | 35 +++--- .../backtest/test_exchange_margin.py | 118 ++++++++++++++++++ 3 files changed, 140 insertions(+), 15 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 97642b7d95a0..ae802cd77c41 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -9,6 +9,8 @@ None None ### Fixes +- Fixed FOK time in force behavior (allows fills beyond the top level, will cancel if cannot fill full size) +- Fixed IOC time in force behavior (allows fills beyond the top level, will cancel any remaining after all fills are applied) - Fixed logging timestamps for backtesting (static clock was not being incrementally set to individual `TimeEvent` timestamps) - Fixed `Equity` short selling for `CASH` accounts (will now reject) - Fixed `ActorFactory.create` JSON encoding (was missing the encoding hook) diff --git a/nautilus_trader/backtest/matching_engine.pyx b/nautilus_trader/backtest/matching_engine.pyx index 56b42f9e2b65..4136ca58e4f8 100644 --- a/nautilus_trader/backtest/matching_engine.pyx +++ b/nautilus_trader/backtest/matching_engine.pyx @@ -1516,6 +1516,20 @@ cdef class OrderMatchingEngine: order.liquidity_side = liquidity_side + cdef: + Price fill_px + Quantity fill_qty + uint64_t total_size_raw = 0 + if order.time_in_force == TimeInForce.FOK: + # Check FOK requirement + for fill in fills: + fill_px, fill_qty = fill + total_size_raw += fill_qty._mem.raw + + if order.leaves_qty._mem.raw > total_size_raw: + self.cancel_order(order) + return # Cannot fill full size - so kill/cancel + if not fills: self._log.error( "Cannot fill order: no fills from book when fills were expected (check sizes in data).", @@ -1534,8 +1548,6 @@ cdef class OrderMatchingEngine: ) cdef: - Price fill_px - Quantity fill_qty bint initial_market_to_limit_fill = False Price last_fill_px = None for fill_px, fill_qty in fills: @@ -1548,14 +1560,6 @@ cdef class OrderMatchingEngine: trigger_price=None, ) initial_market_to_limit_fill = True - if order.time_in_force == TimeInForce.FOK and fill_qty._mem.raw < order.quantity._mem.raw: - # FOK order cannot fill the entire quantity - cancel - self.cancel_order(order) - return - elif order.time_in_force == TimeInForce.IOC: - # IOC order has already filled at one price - cancel remaining - self.cancel_order(order) - return if self.book_type == BookType.L1_MBP and self._fill_model.is_slipped(): if order.side == OrderSide.BUY: @@ -1598,6 +1602,11 @@ cdef class OrderMatchingEngine: last_fill_px = fill_px + if order.time_in_force == TimeInForce.IOC and order.is_open_c(): + # IOC order has filled all available size + self.cancel_order(order) + return + if ( order.is_open_c() and self.book_type == BookType.L1_MBP @@ -1607,11 +1616,6 @@ cdef class OrderMatchingEngine: or order.order_type == OrderType.STOP_MARKET ) ): - if order.time_in_force == TimeInForce.IOC: - # IOC order has already filled at one price - cancel remaining - self.cancel_order(order) - return - # Exhausted simulated book volume (continue aggressive filling into next level) # This is a very basic implementation of slipping by a single tick, in the future # we will implement more detailed fill modeling. @@ -1633,6 +1637,7 @@ cdef class OrderMatchingEngine: position=position, ) + cpdef void fill_order( self, Order order, diff --git a/tests/unit_tests/backtest/test_exchange_margin.py b/tests/unit_tests/backtest/test_exchange_margin.py index 67f65593579c..4f2b42bfcb86 100644 --- a/tests/unit_tests/backtest/test_exchange_margin.py +++ b/tests/unit_tests/backtest/test_exchange_margin.py @@ -572,6 +572,36 @@ def test_submit_market_order_with_fok_time_in_force_cancels_immediately(self) -> assert order.quantity == Quantity.from_int(1_000_000) assert order.filled_qty == Quantity.from_int(0) + def test_submit_limit_order_with_fok_time_in_force_cancels_immediately(self) -> None: + # Arrange: Prepare market + tick = TestDataStubs.quote_tick( + instrument=_USDJPY_SIM, + bid_price=90.002, + ask_price=90.005, + bid_size=500_000, + ask_size=500_000, + ) + self.data_engine.process(tick) + self.exchange.process_quote_tick(tick) + + # Create order + order = self.strategy.order_factory.limit( + _USDJPY_SIM.id, + OrderSide.BUY, + Quantity.from_int(1_000_000), + Price.from_str("90.000"), + time_in_force=TimeInForce.FOK, + ) + + # Act + self.strategy.submit_order(order) + self.exchange.process(0) + + # Assert + assert order.status == OrderStatus.CANCELED + assert order.quantity == Quantity.from_int(1_000_000) + assert order.filled_qty == Quantity.from_int(0) + def test_submit_market_order_with_ioc_time_in_force_cancels_remaining_qty(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( @@ -797,6 +827,94 @@ def test_submit_market_to_limit_order_greater_than_available_top_of_book(self) - assert order.leaves_qty == Quantity.from_int(1_000_000) assert len(self.exchange.get_open_orders()) == 1 + def test_submit_market_order_ioc_cancels_remaining(self) -> None: + # Arrange: Prepare market + tick = TestDataStubs.quote_tick( + instrument=_USDJPY_SIM, + bid_price=90.002, + ask_price=90.005, + bid_size=1_000_000, + ask_size=1_000_000, + ) + self.data_engine.process(tick) + self.exchange.process_quote_tick(tick) + + order = self.strategy.order_factory.market( + _USDJPY_SIM.id, + OrderSide.BUY, + Quantity.from_int(2_000_000), + time_in_force=TimeInForce.IOC, + ) + + # Act + self.strategy.submit_order(order) + self.exchange.process(0) + + # Assert + assert order.status == OrderStatus.CANCELED + assert order.filled_qty == Quantity.from_int(1_000_000) + assert order.leaves_qty == Quantity.from_int(1_000_000) + assert len(self.exchange.get_open_orders()) == 0 + + def test_submit_market_order_fok_cancels_when_cannot_fill_full_size(self) -> None: + # Arrange: Prepare market + tick = TestDataStubs.quote_tick( + instrument=_USDJPY_SIM, + bid_price=90.002, + ask_price=90.005, + bid_size=1_000_000, + ask_size=1_000_000, + ) + self.data_engine.process(tick) + self.exchange.process_quote_tick(tick) + + order = self.strategy.order_factory.market( + _USDJPY_SIM.id, + OrderSide.BUY, + Quantity.from_int(2_000_000), + time_in_force=TimeInForce.FOK, + ) + + # Act + self.strategy.submit_order(order) + self.exchange.process(0) + + # Assert + assert order.status == OrderStatus.CANCELED + assert order.filled_qty == Quantity.from_int(0) + assert order.leaves_qty == Quantity.from_int(2_000_000) + assert len(self.exchange.get_open_orders()) == 0 + + def test_submit_limit_order_fok_cancels_when_cannot_fill_full_size(self) -> None: + # Arrange: Prepare market + tick = TestDataStubs.quote_tick( + instrument=_USDJPY_SIM, + bid_price=90.002, + ask_price=90.005, + bid_size=1_000_000, + ask_size=1_000_000, + ) + self.data_engine.process(tick) + self.exchange.process_quote_tick(tick) + + order = self.strategy.order_factory.limit( + _USDJPY_SIM.id, + OrderSide.BUY, + Quantity.from_int(2_000_000), + Price.from_str("90.005"), + time_in_force=TimeInForce.FOK, + ) + + # Act + self.strategy.submit_order(order) + self.exchange.process(0) + + # Assert + assert order.status == OrderStatus.CANCELED + assert order.filled_qty == Quantity.from_int(0) + assert order.leaves_qty == Quantity.from_int(2_000_000) + assert len(self.exchange.get_open_orders()) == 0 + def test_modify_market_to_limit_order_after_filling_initial_quantity(self) -> None: # Arrange: Prepare market tick = TestDataStubs.quote_tick( From 72a0d3366f5af243ead2200bf4e10e4024dc9134 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 16 Feb 2024 18:03:14 +1100 Subject: [PATCH 045/104] Fix account balance updates for zero quantity NETTING positions --- RELEASES.md | 1 + nautilus_trader/accounting/accounts/cash.pyx | 2 +- .../accounting/accounts/margin.pyx | 2 +- .../unit_tests/backtest/test_exchange_cash.py | 68 +++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index ae802cd77c41..e1c91682762d 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -12,6 +12,7 @@ None - Fixed FOK time in force behavior (allows fills beyond the top level, will cancel if cannot fill full size) - Fixed IOC time in force behavior (allows fills beyond the top level, will cancel any remaining after all fills are applied) - Fixed logging timestamps for backtesting (static clock was not being incrementally set to individual `TimeEvent` timestamps) +- Fixed account balance updates (fills from zero quantity `NETTING` positions will generate account balance updates) - Fixed `Equity` short selling for `CASH` accounts (will now reject) - Fixed `ActorFactory.create` JSON encoding (was missing the encoding hook) - Fixed `ImportableConfig.create` JSON encoding (was missing the encoding hook) diff --git a/nautilus_trader/accounting/accounts/cash.pyx b/nautilus_trader/accounting/accounts/cash.pyx index 778281af52b8..2d77e548af86 100644 --- a/nautilus_trader/accounting/accounts/cash.pyx +++ b/nautilus_trader/accounting/accounts/cash.pyx @@ -303,7 +303,7 @@ cdef class CashAccount(Account): cdef double fill_qty = fill.last_qty.as_f64_c() cdef double fill_px = fill.last_px.as_f64_c() - if position is not None: + if position is not None and position.quantity._mem.raw != 0: # Only book open quantity towards realized PnL fill_qty = fmin(fill_qty, position.quantity.as_f64_c()) diff --git a/nautilus_trader/accounting/accounts/margin.pyx b/nautilus_trader/accounting/accounts/margin.pyx index 6c09f582a49d..7aa9d991bf0d 100644 --- a/nautilus_trader/accounting/accounts/margin.pyx +++ b/nautilus_trader/accounting/accounts/margin.pyx @@ -648,7 +648,7 @@ cdef class MarginAccount(Account): cdef dict pnls = {} # type: dict[Currency, Money] cdef Money pnl - if position is not None and position.entry != fill.order_side: + if position is not None and position.quantity._mem.raw != 0 and position.entry != fill.order_side: # Calculate and add PnL pnl = position.calculate_pnl( avg_px_open=position.avg_px_open, diff --git a/tests/unit_tests/backtest/test_exchange_cash.py b/tests/unit_tests/backtest/test_exchange_cash.py index 5904ebf03f61..9a8a3a9cfc2c 100644 --- a/tests/unit_tests/backtest/test_exchange_cash.py +++ b/tests/unit_tests/backtest/test_exchange_cash.py @@ -299,3 +299,71 @@ def test_equity_order_fills_with_partial_exit( assert order2.status == OrderStatus.FILLED assert order3.status == OrderStatus.FILLED assert self.exchange.get_account().balance_total(USD) == expected_usd + + @pytest.mark.parametrize( + ("entry_side", "exit_side", "expected_usd"), + [ + [OrderSide.BUY, OrderSide.SELL, Money(999_800.00, USD)], + ], + ) + def test_equity_order_multiple_entry_fills( + self, + entry_side: OrderSide, + exit_side: OrderSide, + expected_usd: Money, + ) -> None: + # Arrange: Prepare market + quote1 = TestDataStubs.quote_tick( + instrument=_AAPL_XNAS, + bid_price=100.00, + ask_price=101.00, + ) + self.data_engine.process(quote1) + self.exchange.process_quote_tick(quote1) + + order1 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + entry_side, + Quantity.from_int(100), + ) + + order2 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + exit_side, + Quantity.from_int(100), + ) + + order3 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + entry_side, + Quantity.from_int(100), + ) + + order4 = self.strategy.order_factory.market( + _AAPL_XNAS.id, + exit_side, + Quantity.from_int(100), + ) + + # Act + self.strategy.submit_order(order1) + self.exchange.process(0) + + position_id = self.cache.positions_open()[0].id # Generated by exchange + + self.strategy.submit_order(order2, position_id=position_id) + self.exchange.process(0) + + self.strategy.submit_order(order3) + self.exchange.process(0) + self.strategy.submit_order(order4, position_id=position_id) + self.exchange.process(0) + + # Assert + assert order1.status == OrderStatus.FILLED + assert order2.status == OrderStatus.FILLED + assert order3.status == OrderStatus.FILLED + assert order4.status == OrderStatus.FILLED + assert not self.cache.positions_open() + assert self.exchange.get_account().balance_total(USD) == expected_usd + assert len(self.exchange.get_account().events) == 5 From bbd3cf053775dbd4d71d49e5a31a0661a972345b Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 16 Feb 2024 18:34:07 +1100 Subject: [PATCH 046/104] Remove redundant docstring lines --- nautilus_trader/live/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nautilus_trader/live/config.py b/nautilus_trader/live/config.py index c355b79275ae..ba1e06910538 100644 --- a/nautilus_trader/live/config.py +++ b/nautilus_trader/live/config.py @@ -203,8 +203,6 @@ class TradingNodeConfig(NautilusKernelConfig, frozen=True): The live risk engine configuration. exec_engine : LiveExecEngineConfig, optional The live execution engine configuration. - streaming : StreamingConfig, optional - The configuration for streaming to feather files. data_clients : dict[str, ImportableConfig | LiveDataClientConfig], optional The data client configurations. exec_clients : dict[str, ImportableConfig | LiveExecClientConfig], optional From 95d18548cefb2357d8ceb9ae9cf38c2ebbecf93c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 16 Feb 2024 19:01:58 +1100 Subject: [PATCH 047/104] Fix print_config config option --- RELEASES.md | 1 + nautilus_core/common/src/logging/mod.rs | 2 +- nautilus_trader/system/kernel.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index e1c91682762d..8be61dbc72b0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -11,6 +11,7 @@ None ### Fixes - Fixed FOK time in force behavior (allows fills beyond the top level, will cancel if cannot fill full size) - Fixed IOC time in force behavior (allows fills beyond the top level, will cancel any remaining after all fills are applied) +- Fixed logging `print_config` config option (was not being passed through to the logging system) - Fixed logging timestamps for backtesting (static clock was not being incrementally set to individual `TimeEvent` timestamps) - Fixed account balance updates (fills from zero quantity `NETTING` positions will generate account balance updates) - Fixed `Equity` short selling for `CASH` accounts (will now reject) diff --git a/nautilus_core/common/src/logging/mod.rs b/nautilus_core/common/src/logging/mod.rs index 5f751bc1d999..30af1a995389 100644 --- a/nautilus_core/common/src/logging/mod.rs +++ b/nautilus_core/common/src/logging/mod.rs @@ -417,7 +417,7 @@ impl Logger { let print_config = config.print_config; if print_config { println!("STATIC_MAX_LEVEL={STATIC_MAX_LEVEL}"); - println!("Logger initialized with {:?}", config); + println!("Logger initialized with {:?} {:?}", config, file_config); } match set_boxed_logger(Box::new(logger)) { diff --git a/nautilus_trader/system/kernel.py b/nautilus_trader/system/kernel.py index c334b3c5e247..f7cad951bcbf 100644 --- a/nautilus_trader/system/kernel.py +++ b/nautilus_trader/system/kernel.py @@ -178,6 +178,7 @@ def __init__( # noqa (too complex) component_levels=logging.log_component_levels, colors=logging.log_colors, bypass=logging.bypass_logging, + print_config=logging.print_config, ) elif self._environment == Environment.LIVE: raise InvalidConfiguration( From f58077dda6660438973ae1896db4669476ff7959 Mon Sep 17 00:00:00 2001 From: rsmb7z <105105941+rsmb7z@users.noreply.github.com> Date: Sat, 17 Feb 2024 00:03:49 +0300 Subject: [PATCH 048/104] Implement AverageTrueRange in Rust (#1502) --- nautilus_core/indicators/src/lib.rs | 1 + nautilus_core/indicators/src/python/mod.rs | 3 + .../indicators/src/python/volatility/atr.rs | 101 ++++++++++ .../indicators/src/python/volatility/mod.rs | 16 ++ .../indicators/src/volatility/atr.rs | 149 ++++++++++++++ .../indicators/src/volatility/mod.rs | 16 ++ nautilus_trader/core/nautilus_pyo3.pyi | 31 +++ .../indicators/rust/test_atr_pyo3.py | 182 ++++++++++++++++++ 8 files changed, 499 insertions(+) create mode 100644 nautilus_core/indicators/src/python/volatility/atr.rs create mode 100644 nautilus_core/indicators/src/python/volatility/mod.rs create mode 100644 nautilus_core/indicators/src/volatility/atr.rs create mode 100644 nautilus_core/indicators/src/volatility/mod.rs create mode 100644 tests/unit_tests/indicators/rust/test_atr_pyo3.py diff --git a/nautilus_core/indicators/src/lib.rs b/nautilus_core/indicators/src/lib.rs index b34b5b1f3773..e4ffc48e39c9 100644 --- a/nautilus_core/indicators/src/lib.rs +++ b/nautilus_core/indicators/src/lib.rs @@ -17,6 +17,7 @@ pub mod average; pub mod indicator; pub mod momentum; pub mod ratio; +pub mod volatility; #[cfg(test)] mod stubs; diff --git a/nautilus_core/indicators/src/python/mod.rs b/nautilus_core/indicators/src/python/mod.rs index 032d188a8900..fdf8d88b6f1d 100644 --- a/nautilus_core/indicators/src/python/mod.rs +++ b/nautilus_core/indicators/src/python/mod.rs @@ -18,6 +18,7 @@ use pyo3::{prelude::*, pymodule}; pub mod average; pub mod momentum; pub mod ratio; +pub mod volatility; #[pymodule] pub fn indicators(_: Python<'_>, m: &PyModule) -> PyResult<()> { @@ -33,5 +34,7 @@ pub fn indicators(_: Python<'_>, m: &PyModule) -> PyResult<()> { // momentum m.add_class::()?; m.add_class::()?; + // volatility + m.add_class::()?; Ok(()) } diff --git a/nautilus_core/indicators/src/python/volatility/atr.rs b/nautilus_core/indicators/src/python/volatility/atr.rs new file mode 100644 index 000000000000..737edd5cada4 --- /dev/null +++ b/nautilus_core/indicators/src/python/volatility/atr.rs @@ -0,0 +1,101 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use nautilus_core::python::to_pyvalue_err; +use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; +use pyo3::prelude::*; + +use crate::{average::MovingAverageType, indicator::Indicator, volatility::atr::AverageTrueRange}; + +#[pymethods] +impl AverageTrueRange { + #[new] + pub fn py_new( + period: usize, + ma_type: Option, + use_previous: Option, + value_floor: Option, + ) -> PyResult { + Self::new(period, ma_type, use_previous, value_floor).map_err(to_pyvalue_err) + } + + fn __repr__(&self) -> String { + format!( + "AverageTrueRange({},{},{},{})", + self.period, self.ma_type, self.use_previous, self.value_floor, + ) + } + + #[getter] + #[pyo3(name = "name")] + fn py_name(&self) -> String { + self.name() + } + + #[getter] + #[pyo3(name = "period")] + fn py_period(&self) -> usize { + self.period + } + + #[getter] + #[pyo3(name = "has_inputs")] + fn py_has_inputs(&self) -> bool { + self.has_inputs() + } + + #[getter] + #[pyo3(name = "count")] + fn py_count(&self) -> usize { + self.count + } + + #[getter] + #[pyo3(name = "value")] + fn py_value(&self) -> f64 { + self.value + } + + #[getter] + #[pyo3(name = "initialized")] + fn py_initialized(&self) -> bool { + self.is_initialized + } + + #[pyo3(name = "update_raw")] + fn py_update_raw(&mut self, high: f64, low: f64, close: f64) { + self.update_raw(high, low, close); + } + + #[pyo3(name = "handle_quote_tick")] + fn py_handle_quote_tick(&mut self, _tick: &QuoteTick) { + // Function body intentionally left blank. + } + + #[pyo3(name = "handle_trade_tick")] + fn py_handle_trade_tick(&mut self, _tick: &TradeTick) { + // Function body intentionally left blank. + } + + #[pyo3(name = "handle_bar")] + fn py_handle_bar(&mut self, bar: &Bar) { + self.update_raw((&bar.high).into(), (&bar.low).into(), (&bar.close).into()); + } + + #[pyo3(name = "reset")] + fn py_reset(&mut self) { + self.reset(); + } +} diff --git a/nautilus_core/indicators/src/python/volatility/mod.rs b/nautilus_core/indicators/src/python/volatility/mod.rs new file mode 100644 index 000000000000..799b0bb38a10 --- /dev/null +++ b/nautilus_core/indicators/src/python/volatility/mod.rs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +pub mod atr; diff --git a/nautilus_core/indicators/src/volatility/atr.rs b/nautilus_core/indicators/src/volatility/atr.rs new file mode 100644 index 000000000000..b6a1e2a9e49d --- /dev/null +++ b/nautilus_core/indicators/src/volatility/atr.rs @@ -0,0 +1,149 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::fmt::{Debug, Display}; + +use anyhow::Result; +use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; +use pyo3::prelude::*; + +use crate::{ + average::{MovingAverageFactory, MovingAverageType}, + indicator::{Indicator, MovingAverage}, +}; + +/// An indicator which calculates a Average True Range (ATR) across a rolling window. +#[repr(C)] +#[derive(Debug)] +#[pyclass(module = "nautilus_trader.core.nautilus_pyo3.indicators")] +pub struct AverageTrueRange { + pub period: usize, + pub ma_type: MovingAverageType, + pub use_previous: bool, + pub value_floor: f64, + pub value: f64, + pub count: usize, + pub is_initialized: bool, + has_inputs: bool, + _previous_close: f64, + _ma: Box, +} + +impl Display for AverageTrueRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}({},{},{},{})", + self.name(), + self.period, + self.ma_type, + self.use_previous, + self.value_floor, + ) + } +} + +impl Indicator for AverageTrueRange { + fn name(&self) -> String { + stringify!(AverageTrueRange).to_string() + } + + fn has_inputs(&self) -> bool { + self.has_inputs + } + + fn is_initialized(&self) -> bool { + self.is_initialized + } + + fn handle_quote_tick(&mut self, _tick: &QuoteTick) { + // Function body intentionally left blank. + } + + fn handle_trade_tick(&mut self, _tick: &TradeTick) { + // Function body intentionally left blank. + } + + fn handle_bar(&mut self, bar: &Bar) { + self.update_raw((&bar.high).into(), (&bar.low).into(), (&bar.close).into()); + } + + fn reset(&mut self) { + self._previous_close = 0.0; + self.value = 0.0; + self.count = 0; + self.has_inputs = false; + self.is_initialized = false; + } +} + +impl AverageTrueRange { + pub fn new( + period: usize, + ma_type: Option, + use_previous: Option, + value_floor: Option, + ) -> Result { + Ok(Self { + period, + ma_type: ma_type.unwrap_or(MovingAverageType::Simple), + use_previous: use_previous.unwrap_or(true), + value_floor: value_floor.unwrap_or(0.0), + value: 0.0, + count: 0, + _previous_close: 0.0, + _ma: MovingAverageFactory::create(MovingAverageType::Simple, period), + has_inputs: false, + is_initialized: false, + }) + } + + pub fn update_raw(&mut self, high: f64, low: f64, close: f64) { + if self.use_previous { + if !self.has_inputs { + self._previous_close = close; + } + self._ma.update_raw( + f64::max(self._previous_close, high) - f64::min(low, self._previous_close), + ); + self._previous_close = close; + } else { + self._ma.update_raw(high - low); + } + + self._floor_value(); + self.increment_count(); + } + + fn _floor_value(&mut self) { + if self.value_floor == 0.0 || self.value_floor < self._ma.value() { + self.value = self._ma.value(); + } else { + // Floor the value + self.value = self.value_floor; + } + } + + fn increment_count(&mut self) { + self.count += 1; + + if !self.is_initialized { + self.has_inputs = true; + if self.count >= self.period { + self.is_initialized = true; + } + } + } +} diff --git a/nautilus_core/indicators/src/volatility/mod.rs b/nautilus_core/indicators/src/volatility/mod.rs new file mode 100644 index 000000000000..799b0bb38a10 --- /dev/null +++ b/nautilus_core/indicators/src/volatility/mod.rs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +pub mod atr; diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 832101da0f15..0ad79c9351c2 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -754,6 +754,14 @@ class TriggerType(Enum): MARK_PRICE = "MARK_PRICE" INDEX_PRICE = "INDEX_PRICE" +class MovingAverageType(Enum): + SIMPLE = "SIMPLE" + EXPONENTIAL = "EXPONENTIAL" + DOUBLE_EXPONENTIAL = "DOUBLE_EXPONENTIAL" + WILDER = "WILDER" + HULL = "HULL" + + ### Identifiers class AccountId: @@ -1970,6 +1978,29 @@ class AroonOscillator: def handle_bar(self, bar: Bar) -> None: ... def reset(self) -> None: ... +class AverageTrueRange: + def __init__( + self, + period: int, + ma_type: MovingAverageType = ..., + use_previous: bool = True, + value_floor: float = 0.0, + ) -> None: ... + @property + def name(self) -> str: ... + @property + def period(self) -> int: ... + @property + def count(self) -> int: ... + @property + def initialized(self) -> bool: ... + @property + def has_inputs(self) -> bool: ... + @property + def value(self) -> float: ... + def update_raw(self, high: float, low: float, close: float) -> None: ... + def handle_bar(self, bar: Bar) -> None: ... + def reset(self) -> None: ... ################################################################################################### # Adapters diff --git a/tests/unit_tests/indicators/rust/test_atr_pyo3.py b/tests/unit_tests/indicators/rust/test_atr_pyo3.py new file mode 100644 index 000000000000..dab52243ca0d --- /dev/null +++ b/tests/unit_tests/indicators/rust/test_atr_pyo3.py @@ -0,0 +1,182 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +import sys + +import pytest + +from nautilus_trader.core.nautilus_pyo3 import AverageTrueRange +from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 + + +@pytest.fixture(scope="function") +def atr(): + return AverageTrueRange(10) + + +def test_name_returns_expected_string(atr): + # Arrange, Act, Assert + assert atr.name == "AverageTrueRange" + + +def test_str_repr_returns_expected_string(atr): + # Arrange, Act, Assert + assert str(atr) == "AverageTrueRange(10,SIMPLE,true,0)" + assert repr(atr) == "AverageTrueRange(10,SIMPLE,true,0)" + + +def test_period(atr): + # Arrange, Act, Assert + assert atr.period == 10 + + +def test_initialized_without_inputs_returns_false(atr): + # Arrange, Act, Assert + assert atr.initialized is False + + +def test_initialized_with_required_inputs_returns_true(atr): + # Arrange, Act + for _i in range(10): + atr.update_raw(1.00000, 1.00000, 1.00000) + + # Assert + assert atr.initialized is True + + +def test_handle_bar_updates_indicator(atr): + # Arrange + bar = TestDataProviderPyo3.bar_5decimal() + + # Act + atr.handle_bar(bar) + + # Assert + assert atr.has_inputs + assert atr.value == 2.999999999997449e-05 + + +def test_value_with_no_inputs_returns_zero(atr): + # Arrange, Act, Assert + assert atr.value == 0.0 + + +def test_value_with_epsilon_input(atr): + # Arrange + epsilon = sys.float_info.epsilon + atr.update_raw(epsilon, epsilon, epsilon) + + # Act, Assert + assert atr.value == 0.0 + + +def test_value_with_one_ones_input(atr): + # Arrange + atr.update_raw(1.00000, 1.00000, 1.00000) + + # Act, Assert + assert atr.value == 0.0 + + +def test_value_with_one_input(atr): + # Arrange + atr.update_raw(1.00020, 1.00000, 1.00010) + + # Act, Assert + assert atr.value == pytest.approx(0.00020) + + +def test_value_with_three_inputs(atr): + # Arrange + atr.update_raw(1.00020, 1.00000, 1.00010) + atr.update_raw(1.00020, 1.00000, 1.00010) + atr.update_raw(1.00020, 1.00000, 1.00010) + + # Act, Assert + assert atr.value == pytest.approx(0.00020) + + +def test_value_with_close_on_high(atr): + # Arrange + high = 1.00010 + low = 1.00000 + + # Act + for _i in range(1000): + high += 0.00010 + low += 0.00010 + close = high + atr.update_raw(high, low, close) + + # Assert + assert atr.value == pytest.approx(0.00010, 2) + + +def test_value_with_close_on_low(atr): + # Arrange + high = 1.00010 + low = 1.00000 + + # Act + for _i in range(1000): + high -= 0.00010 + low -= 0.00010 + close = low + atr.update_raw(high, low, close) + + # Assert + assert atr.value == pytest.approx(0.00010) + + +def test_floor_with_ten_ones_inputs(): + # Arrange + floor = 0.00005 + floored_atr = AverageTrueRange(10, value_floor=floor) + + for _i in range(20): + floored_atr.update_raw(1.00000, 1.00000, 1.00000) + + # Act, Assert + assert floored_atr.value == 5e-05 + + +def test_floor_with_exponentially_decreasing_high_inputs(): + # Arrange + floor = 0.00005 + floored_atr = AverageTrueRange(10, value_floor=floor) + + high = 1.00020 + low = 1.00000 + close = 1.00000 + + for _i in range(20): + high -= (high - low) / 2 + floored_atr.update_raw(high, low, close) + + # Act, Assert + assert floored_atr.value == 5e-05 + + +def test_reset_successfully_returns_indicator_to_fresh_state(atr): + # Arrange + for _i in range(1000): + atr.update_raw(1.00010, 1.00000, 1.00005) + + # Act + atr.reset() + + # Assert + assert not atr.initialized + assert atr.value == 0 From c491ebf906e46c65f7bfca05c2db2823e03ffc7e Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 07:02:16 +1100 Subject: [PATCH 049/104] Update core dependencies --- nautilus_core/Cargo.lock | 90 ++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 40 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index b1a55b305ce4..f67ccd2c01be 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -368,7 +368,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -576,7 +576,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", "syn_derive", ] @@ -603,9 +603,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "d32a994c2b3ca201d9b263612a374263f05e7adde37c4707f693dcd375076d1f" [[package]] name = "bytecheck" @@ -695,10 +695,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.85" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b918671670962b48bc23753aef0c51d072dca6f52f01f800854ada6ddb7f7d3" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ + "jobserver", "libc", ] @@ -794,18 +795,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstyle", "clap_lex 0.7.0", @@ -947,7 +948,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.0", + "clap 4.5.1", "criterion-plot", "is-terminal", "itertools 0.10.5", @@ -1354,7 +1355,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -1680,7 +1681,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -1849,9 +1850,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" [[package]] name = "hex" @@ -2121,7 +2122,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.5", + "hermit-abi 0.3.6", "libc", "windows-sys 0.52.0", ] @@ -2150,6 +2151,15 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +[[package]] +name = "jobserver" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.68" @@ -2776,7 +2786,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.5", + "hermit-abi 0.3.6", "libc", ] @@ -2798,7 +2808,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2866,7 +2876,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3082,7 +3092,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3120,9 +3130,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "plotters" @@ -3324,7 +3334,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3336,7 +3346,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3690,7 +3700,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.48", + "syn 2.0.49", "unicode-ident", ] @@ -3797,9 +3807,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" +checksum = "048a63e5b3ac996d78d402940b5fa47973d2d080c6c6fffa1d0f19c4445310b7" [[package]] name = "rustls-webpki" @@ -3936,7 +3946,7 @@ checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -4157,7 +4167,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -4405,7 +4415,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -4427,9 +4437,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" dependencies = [ "proc-macro2", "quote", @@ -4445,7 +4455,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -4570,7 +4580,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -4692,7 +4702,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -4845,7 +4855,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -4964,7 +4974,7 @@ checksum = "563b3b88238ec95680aef36bdece66896eaa7ce3c0f1b4f39d38fb2435261352" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -5145,7 +5155,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", "wasm-bindgen-shared", ] @@ -5179,7 +5189,7 @@ checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5464,7 +5474,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] From 28f2d8d1de726c9a67c78c4ab8a540a0fc02f5f8 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 07:25:07 +1100 Subject: [PATCH 050/104] Introduce common tokio runtime --- nautilus_core/Cargo.lock | 1 + .../adapters/src/databento/python/live.rs | 5 ++-- nautilus_core/common/Cargo.toml | 1 + nautilus_core/common/src/lib.rs | 1 + nautilus_core/common/src/runtime.rs | 25 +++++++++++++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 nautilus_core/common/src/runtime.rs diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index f67ccd2c01be..0ea11d8f6549 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2491,6 +2491,7 @@ dependencies = [ "strum", "sysinfo", "tempfile", + "tokio", "tracing", "tracing-subscriber", "ustr", diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 23c1a5e571f4..5401b17c4e70 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -22,6 +22,7 @@ use databento::live::Subscription; use dbn::{PitSymbolMap, Record, SymbolIndex, VersionUpgradePolicy}; use indexmap::IndexMap; use log::{error, info}; +use nautilus_common::runtime::get_runtime; use nautilus_core::python::to_pyruntime_err; use nautilus_core::time::AtomicTime; use nautilus_core::{ @@ -54,7 +55,6 @@ pub struct DatabentoLiveClient { #[pyo3(get)] pub dataset: String, inner: Option>>, - runtime: tokio::runtime::Runtime, publisher_venue_map: Arc>, } @@ -72,7 +72,7 @@ impl DatabentoLiveClient { match &self.inner { Some(client) => Ok(client.clone()), None => { - let client = self.runtime.block_on(self.initialize_client())?; + let client = get_runtime().block_on(self.initialize_client())?; self.inner = Some(Arc::new(Mutex::new(client))); Ok(self.inner.clone().unwrap()) } @@ -97,7 +97,6 @@ impl DatabentoLiveClient { key, dataset, inner: None, - runtime: tokio::runtime::Runtime::new()?, publisher_venue_map: Arc::new(publisher_venue_map), }) } diff --git a/nautilus_core/common/Cargo.toml b/nautilus_core/common/Cargo.toml index 6b5efda76aae..f91a4c067b09 100644 --- a/nautilus_core/common/Cargo.toml +++ b/nautilus_core/common/Cargo.toml @@ -24,6 +24,7 @@ serde_json = { workspace = true } strum = { workspace = true } ustr = { workspace = true } rstest = { workspace = true , optional = true} +tokio = { workspace = true } tracing = { workspace = true } sysinfo = "0.30.5" # Disable default feature "tracing-log" since it interferes with custom logging diff --git a/nautilus_core/common/src/lib.rs b/nautilus_core/common/src/lib.rs index 536048ffda0d..4f77acc139a0 100644 --- a/nautilus_core/common/src/lib.rs +++ b/nautilus_core/common/src/lib.rs @@ -20,6 +20,7 @@ pub mod generators; pub mod handlers; pub mod logging; pub mod msgbus; +pub mod runtime; pub mod testing; pub mod timer; diff --git a/nautilus_core/common/src/runtime.rs b/nautilus_core/common/src/runtime.rs new file mode 100644 index 000000000000..9ab54e00decc --- /dev/null +++ b/nautilus_core/common/src/runtime.rs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::sync::OnceLock; + +use tokio::runtime::Runtime; + +static RUNTIME: OnceLock = OnceLock::new(); + +pub fn get_runtime() -> &'static tokio::runtime::Runtime { + // Using default configuration values for now + RUNTIME.get_or_init(|| Runtime::new().expect("Failed to create tokio runtime")) +} From c3c6cbfb1fa626256ede424debec3bf6d2daaf50 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 09:05:52 +1100 Subject: [PATCH 051/104] Optimize DatabentoLiveClient order book replay --- .../adapters/src/databento/python/live.rs | 81 ++++++++++++++----- nautilus_core/model/src/ffi/data/deltas.rs | 8 +- nautilus_trader/adapters/databento/data.py | 37 ++------- .../adapters/databento/providers.py | 5 +- nautilus_trader/core/nautilus_pyo3.pyi | 1 + 5 files changed, 78 insertions(+), 54 deletions(-) diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 5401b17c4e70..f6dabe67955a 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -13,6 +13,7 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- +use std::collections::HashMap; use std::fs; use std::str::FromStr; use std::sync::Arc; @@ -29,7 +30,10 @@ use nautilus_core::{ python::to_pyvalue_err, time::{get_atomic_clock_realtime, UnixNanos}, }; +use nautilus_model::data::delta::OrderBookDelta; +use nautilus_model::data::deltas::OrderBookDeltas; use nautilus_model::data::Data; +use nautilus_model::ffi::data::deltas::OrderBookDeltas_API; use nautilus_model::identifiers::instrument_id::InstrumentId; use nautilus_model::identifiers::symbol::Symbol; use nautilus_model::identifiers::venue::Venue; @@ -144,12 +148,22 @@ impl DatabentoLiveClient { } #[pyo3(name = "start")] - fn py_start<'py>(&mut self, py: Python<'py>, callback: PyObject) -> PyResult<&'py PyAny> { + fn py_start<'py>( + &mut self, + py: Python<'py>, + callback: PyObject, + replay: bool, + ) -> PyResult<&'py PyAny> { let arc_client = self.get_inner_client().map_err(to_pyruntime_err)?; let publisher_venue_map = self.publisher_venue_map.clone(); + let clock = get_atomic_clock_realtime(); + + let buffering_start = match replay { + true => Some(clock.get_time_ns()), + false => None, + }; pyo3_asyncio::tokio::future_into_py(py, async move { - let clock = get_atomic_clock_realtime(); let mut client = arc_client.lock().await; let mut symbol_map = PitSymbolMap::new(); @@ -157,6 +171,8 @@ impl DatabentoLiveClient { let relock_interval = timeout_duration.as_nanos() as u64; let mut lock_last_dropped_ns = 0_u64; + let mut buffered_deltas: HashMap> = HashMap::new(); + client.start().await.map_err(to_pyruntime_err)?; loop { @@ -198,8 +214,46 @@ impl DatabentoLiveClient { handle_instrument_def_msg(msg, &publisher_venue_map, clock, &callback) .map_err(to_pyvalue_err)?; } else { - handle_record(record, &symbol_map, &publisher_venue_map, clock, &callback) - .map_err(to_pyvalue_err)?; + let (mut data1, data2) = + handle_record(record, &symbol_map, &publisher_venue_map, clock) + .map_err(to_pyvalue_err)?; + + if let Some(msg) = record.get::() { + // SAFETY: An MBO message will always produce a delta + if let Data::Delta(delta) = data1.clone().unwrap() { + buffered_deltas + .entry(delta.instrument_id) + .or_default() + .push(delta); + + // Check if this is the last message in the packet + if msg.flags & dbn::flags::LAST != 0 { + continue; // Not last message + } + + // Check if we're currently buffering a replay + if let Some(start_ns) = buffering_start { + if delta.ts_event <= start_ns { + continue; // Continue buffering the replay + } + } + + // SAFETY: We can guarantee a deltas vec exists + let deltas = buffered_deltas.remove(&delta.instrument_id).unwrap(); + let book_deltas = OrderBookDeltas::new(delta.instrument_id, deltas); + data1 = Some(Data::Deltas(OrderBookDeltas_API::new(book_deltas))); + } + }; + + Python::with_gil(|py| { + if let Some(data) = data1 { + call_python_with_data(py, &callback, data); + } + + if let Some(data) = data2 { + call_python_with_data(py, &callback, data); + } + }); }; } Ok(()) @@ -271,8 +325,7 @@ fn handle_record( symbol_map: &PitSymbolMap, publisher_venue_map: &IndexMap, clock: &AtomicTime, - callback: &PyObject, -) -> Result<()> { +) -> Result<(Option, Option)> { let raw_symbol = symbol_map .get_for_rec(&record) .expect("Cannot resolve `raw_symbol` from `symbol_map`"); @@ -284,25 +337,13 @@ fn handle_record( let price_precision = 2; // Hard coded for now let ts_init = clock.get_time_ns(); - let (data1, data2) = decode_record( + decode_record( &record, instrument_id, price_precision, Some(ts_init), true, // Always include trades - )?; - - Python::with_gil(|py| { - if let Some(data) = data1 { - call_python_with_data(py, callback, data); - } - - if let Some(data) = data2 { - call_python_with_data(py, callback, data); - } - }); - - Ok(()) + ) } fn call_python_with_data(py: Python, callback: &PyObject, data: Data) { diff --git a/nautilus_core/model/src/ffi/data/deltas.rs b/nautilus_core/model/src/ffi/data/deltas.rs index 69cfb237453d..99a80d255932 100644 --- a/nautilus_core/model/src/ffi/data/deltas.rs +++ b/nautilus_core/model/src/ffi/data/deltas.rs @@ -36,6 +36,12 @@ use crate::{ #[allow(non_camel_case_types)] pub struct OrderBookDeltas_API(Box); +impl OrderBookDeltas_API { + pub fn new(deltas: OrderBookDeltas) -> Self { + Self(Box::new(deltas)) + } +} + impl Deref for OrderBookDeltas_API { type Target = OrderBookDeltas; @@ -66,7 +72,7 @@ pub extern "C" fn orderbook_deltas_new( unsafe { Vec::from_raw_parts(ptr as *mut OrderBookDelta, len, cap) }; let cloned_deltas = deltas.clone(); std::mem::forget(deltas); // Prevents Rust from dropping `deltas` - OrderBookDeltas_API(Box::new(OrderBookDeltas::new(instrument_id, cloned_deltas))) + OrderBookDeltas_API::new(OrderBookDeltas::new(instrument_id, cloned_deltas)) } #[no_mangle] diff --git a/nautilus_trader/adapters/databento/data.py b/nautilus_trader/adapters/databento/data.py index 470b66eeb0a2..5ef62248918a 100644 --- a/nautilus_trader/adapters/databento/data.py +++ b/nautilus_trader/adapters/databento/data.py @@ -26,7 +26,6 @@ from nautilus_trader.adapters.databento.constants import ALL_SYMBOLS from nautilus_trader.adapters.databento.constants import DATABENTO_CLIENT_ID from nautilus_trader.adapters.databento.constants import PUBLISHERS_PATH -from nautilus_trader.adapters.databento.enums import DatabentoRecordFlags from nautilus_trader.adapters.databento.enums import DatabentoSchema from nautilus_trader.adapters.databento.loaders import DatabentoDataLoader from nautilus_trader.adapters.databento.providers import DatabentoInstrumentProvider @@ -43,7 +42,6 @@ from nautilus_trader.model.data import BarType from nautilus_trader.model.data import DataType from nautilus_trader.model.data import OrderBookDelta -from nautilus_trader.model.data import OrderBookDeltas from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.data import TradeTick from nautilus_trader.model.data import capsule_to_data @@ -276,7 +274,9 @@ async def _check_live_client_started( ) -> None: if not self._has_subscribed.get(dataset): self._log.debug(f"Starting {dataset} live client...", LogColor.MAGENTA) - future = asyncio.ensure_future(live_client.start(callback=self._handle_record)) + future = asyncio.ensure_future( + live_client.start(callback=self._handle_record, replay=False), + ) self._live_client_futures.add(future) self._has_subscribed[dataset] = True self._log.info(f"Started {dataset} live feed.", LogColor.BLUE) @@ -519,7 +519,7 @@ async def _subscribe_order_book_deltas_batch( for instrument_id in instrument_ids: self._trade_tick_subscriptions.add(instrument_id) - future = asyncio.ensure_future(live_client.start(self._handle_record)) + future = asyncio.ensure_future(live_client.start(self._handle_record, replay=True)) self._live_client_futures.add(future) except asyncio.CancelledError: self._log.warning( @@ -875,33 +875,6 @@ def _handle_record( ) -> None: # The capsule will fall out of scope at the end of this method, # and eventually be garbage collected. The contained pointer - # to `Data` is still owned and managed by the Rust memory model. + # to `Data` is still owned and managed by Rust. data = capsule_to_data(pycapsule) - - if isinstance(data, OrderBookDelta): - # Assign instrument_id to avoid continually fetching the C string - instrument_id = data.instrument_id - - buffer_start_ns = self._buffering_replay.get(instrument_id, 0) - if buffer_start_ns or DatabentoRecordFlags.F_LAST in DatabentoRecordFlags(data.flags): - buffer = self._buffered_deltas[instrument_id] - buffer.append(data) - - if buffer_start_ns > 0: - if data.ts_event >= buffer_start_ns: - self._buffering_replay.pop(instrument_id) - self._log.info(f"MBO replay complete for {instrument_id}.", LogColor.BLUE) - else: - # Uncomment blow for debugging/development - # latency = self._clock.timestamp_ns() - data.ts_init - # self._log.warning(f"{len(buffer)} {instrument_id}: {latency}") - return # Still replaying start - - data = OrderBookDeltas(instrument_id, deltas=buffer.copy()) - buffer.clear() - else: - buffer = self._buffered_deltas[instrument_id] - buffer.append(data) - return # We can rely on the F_LAST flag for an MBO feed - self._handle_data(data) diff --git a/nautilus_trader/adapters/databento/providers.py b/nautilus_trader/adapters/databento/providers.py index 7b7198c48c41..6190648930cf 100644 --- a/nautilus_trader/adapters/databento/providers.py +++ b/nautilus_trader/adapters/databento/providers.py @@ -151,7 +151,10 @@ def receive_instruments(pyo3_instrument: Any) -> None: ) try: - await asyncio.wait_for(live_client.start(callback=receive_instruments), timeout=5.0) + await asyncio.wait_for( + live_client.start(callback=receive_instruments, replay=False), + timeout=5.0, + ) # TODO: Improve this so that `live_client.start` isn't raising a `ValueError` except ValueError as e: if success_msg in str(e): diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 0ad79c9351c2..db0b3e6f05dc 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -2103,5 +2103,6 @@ class DatabentoLiveClient: async def start( self, callback: Callable, + replay: bool, ) -> dict[str, str]: ... async def close(self) -> None: ... From 956ae97b18e9add8527c97f23a653a7738225ec9 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 16:12:50 +1100 Subject: [PATCH 052/104] Port LiveClock and LiveTimer to Rust --- RELEASES.md | 3 +- nautilus_core/backtest/src/engine.rs | 9 +- nautilus_core/common/src/clock.rs | 41 +- nautilus_core/common/src/ffi/clock.rs | 129 ++++++- nautilus_core/common/src/ffi/timer.rs | 2 +- nautilus_core/common/src/handlers.rs | 3 +- nautilus_core/common/src/python/timer.rs | 13 +- nautilus_core/common/src/timer.rs | 143 ++++++- nautilus_trader/common/component.pxd | 58 --- nautilus_trader/common/component.pyx | 452 +++-------------------- nautilus_trader/core/includes/common.h | 73 +++- nautilus_trader/core/rust/common.pxd | 63 +++- nautilus_trader/system/kernel.py | 2 +- nautilus_trader/trading/trader.py | 22 +- tests/unit_tests/common/test_clock.py | 269 +------------- 15 files changed, 459 insertions(+), 823 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 8be61dbc72b0..02b36e83c472 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -3,7 +3,8 @@ Released on TBD (UTC). ### Enhancements -None +- Ported `LiveClock` and `LiveTimer` implementations to Rust +- Implemented `AverageTrueRange` in Rust, thanks @rsmb7z ### Breaking Changes None diff --git a/nautilus_core/backtest/src/engine.rs b/nautilus_core/backtest/src/engine.rs index 94686c3540d0..aa69b45bc5d5 100644 --- a/nautilus_core/backtest/src/engine.rs +++ b/nautilus_core/backtest/src/engine.rs @@ -127,12 +127,9 @@ mod tests { let mut accumulator = TimeEventAccumulator::new(); - let time_event1 = - TimeEvent::new(Ustr::from("TEST_EVENT_1"), UUID4::new(), 100, 100).unwrap(); - let time_event2 = - TimeEvent::new(Ustr::from("TEST_EVENT_2"), UUID4::new(), 300, 300).unwrap(); - let time_event3 = - TimeEvent::new(Ustr::from("TEST_EVENT_3"), UUID4::new(), 200, 200).unwrap(); + let time_event1 = TimeEvent::new(Ustr::from("TEST_EVENT_1"), UUID4::new(), 100, 100); + let time_event2 = TimeEvent::new(Ustr::from("TEST_EVENT_2"), UUID4::new(), 300, 300); + let time_event3 = TimeEvent::new(Ustr::from("TEST_EVENT_3"), UUID4::new(), 200, 200); // Note: as_ptr returns a borrowed pointer. It is valid as long // as the object is in scope. In this case `callback_ptr` is valid diff --git a/nautilus_core/common/src/clock.rs b/nautilus_core/common/src/clock.rs index eba142cb176d..2c1f1af19a27 100644 --- a/nautilus_core/common/src/clock.rs +++ b/nautilus_core/common/src/clock.rs @@ -23,7 +23,7 @@ use ustr::Ustr; use crate::{ handlers::EventHandler, - timer::{TestTimer, TimeEvent, TimeEventHandler}, + timer::{LiveTimer, TestTimer, TimeEvent, TimeEventHandler}, }; /// Represents a type of clock. @@ -241,9 +241,8 @@ impl Clock for TestClock { pub struct LiveClock { time: &'static AtomicTime, - timers: HashMap, + timers: HashMap, default_callback: Option, - callbacks: HashMap, } impl LiveClock { @@ -253,9 +252,13 @@ impl LiveClock { time: get_atomic_clock_realtime(), timers: HashMap::new(), default_callback: None, - callbacks: HashMap::new(), } } + + #[must_use] + pub fn get_timers(&self) -> &HashMap { + &self.timers + } } impl Default for LiveClock { @@ -304,16 +307,22 @@ impl Clock for LiveClock { "All Python callbacks were `None`" ); - let name_ustr = Ustr::from(name); - match callback { - Some(callback_py) => self.callbacks.insert(name_ustr, callback_py), - None => None, + let callback = match callback { + Some(callback) => callback, + None => self.default_callback.clone().unwrap(), }; let ts_now = self.get_time_ns(); alert_time_ns = std::cmp::max(alert_time_ns, ts_now); - let timer = TestTimer::new(name, alert_time_ns - ts_now, ts_now, Some(alert_time_ns)); - self.timers.insert(name_ustr, timer); + let mut timer = LiveTimer::new( + name, + alert_time_ns - ts_now, + ts_now, + Some(alert_time_ns), + callback, + ); + timer.start(); + self.timers.insert(Ustr::from(name), timer); } fn set_timer_ns( @@ -330,14 +339,14 @@ impl Clock for LiveClock { "All Python callbacks were `None`" ); - let name_ustr = Ustr::from(name); - match callback { - Some(callback) => self.callbacks.insert(name_ustr, callback), - None => None, + let callback = match callback { + Some(callback) => callback, + None => self.default_callback.clone().unwrap(), }; - let timer = TestTimer::new(name, interval_ns, start_time_ns, stop_time_ns); - self.timers.insert(name_ustr, timer); + let mut timer = LiveTimer::new(name, interval_ns, start_time_ns, stop_time_ns, callback); + timer.start(); + self.timers.insert(Ustr::from(name), timer); } fn next_time_ns(&self, name: &str) -> UnixNanos { diff --git a/nautilus_core/common/src/ffi/clock.rs b/nautilus_core/common/src/ffi/clock.rs index 95eb7c7d8f9c..0cd4b3641841 100644 --- a/nautilus_core/common/src/ffi/clock.rs +++ b/nautilus_core/common/src/ffi/clock.rs @@ -135,7 +135,7 @@ pub extern "C" fn test_clock_timer_count(clock: &mut TestClock_API) -> usize { /// - Assumes `name_ptr` is a valid C string pointer. /// - Assumes `callback_ptr` is a valid `PyCallable` pointer. #[no_mangle] -pub unsafe extern "C" fn test_clock_set_time_alert_ns( +pub unsafe extern "C" fn test_clock_set_time_alert( clock: &mut TestClock_API, name_ptr: *const c_char, alert_time_ns: UnixNanos, @@ -158,7 +158,7 @@ pub unsafe extern "C" fn test_clock_set_time_alert_ns( /// - Assumes `name_ptr` is a valid C string pointer. /// - Assumes `callback_ptr` is a valid `PyCallable` pointer. #[no_mangle] -pub unsafe extern "C" fn test_clock_set_timer_ns( +pub unsafe extern "C" fn test_clock_set_timer( clock: &mut TestClock_API, name_ptr: *const c_char, interval_ns: u64, @@ -217,7 +217,7 @@ pub extern "C" fn vec_time_event_handlers_drop(v: CVec) { /// /// - Assumes `name_ptr` is a valid C string pointer. #[no_mangle] -pub unsafe extern "C" fn test_clock_next_time_ns( +pub unsafe extern "C" fn test_clock_next_time( clock: &mut TestClock_API, name_ptr: *const c_char, ) -> UnixNanos { @@ -279,6 +279,23 @@ pub extern "C" fn live_clock_drop(clock: LiveClock_API) { drop(clock); // Memory freed here } +/// # Safety +/// +/// - Assumes `callback_ptr` is a valid `PyCallable` pointer. +#[no_mangle] +pub unsafe extern "C" fn live_clock_register_default_handler( + clock: &mut LiveClock_API, + callback_ptr: *mut ffi::PyObject, +) { + assert!(!callback_ptr.is_null()); + assert!(ffi::Py_None() != callback_ptr); + + let callback_py = Python::with_gil(|py| PyObject::from_borrowed_ptr(py, callback_ptr)); + let handler = EventHandler::new(Some(callback_py), None); + + clock.register_default_handler(handler); +} + #[no_mangle] pub extern "C" fn live_clock_timestamp(clock: &mut LiveClock_API) -> f64 { clock.get_time() @@ -298,3 +315,109 @@ pub extern "C" fn live_clock_timestamp_us(clock: &mut LiveClock_API) -> u64 { pub extern "C" fn live_clock_timestamp_ns(clock: &mut LiveClock_API) -> u64 { clock.get_time_ns() } + +#[no_mangle] +pub extern "C" fn live_clock_timer_names(clock: &LiveClock_API) -> *mut ffi::PyObject { + Python::with_gil(|py| -> Py { + let names: Vec> = clock + .get_timers() + .keys() + .map(|k| PyString::new(py, k).into()) + .collect(); + PyList::new(py, names).into() + }) + .as_ptr() +} + +#[no_mangle] +pub extern "C" fn live_clock_timer_count(clock: &mut LiveClock_API) -> usize { + clock.timer_count() +} + +/// # Safety +/// +/// - Assumes `name_ptr` is a valid C string pointer. +/// - Assumes `callback_ptr` is a valid `PyCallable` pointer. +#[no_mangle] +pub unsafe extern "C" fn live_clock_set_time_alert( + clock: &mut LiveClock_API, + name_ptr: *const c_char, + alert_time_ns: UnixNanos, + callback_ptr: *mut ffi::PyObject, +) { + assert!(!callback_ptr.is_null()); + + let name = cstr_to_str(name_ptr); + let callback_py = Python::with_gil(|py| match callback_ptr { + ptr if ptr != ffi::Py_None() => Some(PyObject::from_borrowed_ptr(py, ptr)), + _ => None, + }); + let handler = EventHandler::new(callback_py.clone(), None); + + clock.set_time_alert_ns(name, alert_time_ns, callback_py.map(|_| handler)); +} + +/// # Safety +/// +/// - Assumes `name_ptr` is a valid C string pointer. +/// - Assumes `callback_ptr` is a valid `PyCallable` pointer. +#[no_mangle] +pub unsafe extern "C" fn live_clock_set_timer( + clock: &mut LiveClock_API, + name_ptr: *const c_char, + interval_ns: u64, + start_time_ns: UnixNanos, + stop_time_ns: UnixNanos, + callback_ptr: *mut ffi::PyObject, +) { + assert!(!callback_ptr.is_null()); + + let name = cstr_to_str(name_ptr); + let stop_time_ns = match stop_time_ns { + 0 => None, + _ => Some(stop_time_ns), + }; + let callback_py = Python::with_gil(|py| match callback_ptr { + ptr if ptr != ffi::Py_None() => Some(PyObject::from_borrowed_ptr(py, ptr)), + _ => None, + }); + + let handler = EventHandler::new(callback_py.clone(), None); + + clock.set_timer_ns( + name, + interval_ns, + start_time_ns, + stop_time_ns, + callback_py.map(|_| handler), + ); +} + +/// # Safety +/// +/// - Assumes `name_ptr` is a valid C string pointer. +#[no_mangle] +pub unsafe extern "C" fn live_clock_next_time( + clock: &mut LiveClock_API, + name_ptr: *const c_char, +) -> UnixNanos { + let name = cstr_to_str(name_ptr); + clock.next_time_ns(name) +} + +/// # Safety +/// +/// - Assumes `name_ptr` is a valid C string pointer. +#[no_mangle] +pub unsafe extern "C" fn live_clock_cancel_timer( + clock: &mut LiveClock_API, + name_ptr: *const c_char, +) { + let name = cstr_to_str(name_ptr); + clock.cancel_timer(name); +} + +#[no_mangle] +pub extern "C" fn live_clock_cancel_timers(clock: &mut LiveClock_API) { + clock.cancel_timers(); +} diff --git a/nautilus_core/common/src/ffi/timer.rs b/nautilus_core/common/src/ffi/timer.rs index 9976cd9609ef..66ad4a6b9558 100644 --- a/nautilus_core/common/src/ffi/timer.rs +++ b/nautilus_core/common/src/ffi/timer.rs @@ -32,7 +32,7 @@ pub unsafe extern "C" fn time_event_new( ts_event: u64, ts_init: u64, ) -> TimeEvent { - TimeEvent::new(cstr_to_ustr(name_ptr), event_id, ts_event, ts_init).unwrap() + TimeEvent::new(cstr_to_ustr(name_ptr), event_id, ts_event, ts_init) } /// Returns a [`TimeEvent`] as a C string pointer. diff --git a/nautilus_core/common/src/handlers.rs b/nautilus_core/common/src/handlers.rs index 8771b6442ae4..bed9d5f06965 100644 --- a/nautilus_core/common/src/handlers.rs +++ b/nautilus_core/common/src/handlers.rs @@ -94,12 +94,11 @@ impl fmt::Debug for MessageHandler { pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.common") )] pub struct EventHandler { - py_callback: Option, + pub py_callback: Option, _callback: Option, } impl EventHandler { - // TODO: Validate exactly one of these is `Some` #[must_use] pub fn new(py_callback: Option, callback: Option) -> Self { Self { diff --git a/nautilus_core/common/src/python/timer.rs b/nautilus_core/common/src/python/timer.rs index 735c33ca68d2..534f17dd4d16 100644 --- a/nautilus_core/common/src/python/timer.rs +++ b/nautilus_core/common/src/python/timer.rs @@ -29,13 +29,8 @@ use crate::timer::TimeEvent; #[pymethods] impl TimeEvent { #[new] - fn py_new( - name: &str, - event_id: UUID4, - ts_event: UnixNanos, - ts_init: UnixNanos, - ) -> PyResult { - Self::new(Ustr::from(name), event_id, ts_event, ts_init).map_err(to_pyvalue_err) + fn py_new(name: &str, event_id: UUID4, ts_event: UnixNanos, ts_init: UnixNanos) -> Self { + Self::new(Ustr::from(name), event_id, ts_event, ts_init) } fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> { @@ -66,8 +61,8 @@ impl TimeEvent { } #[staticmethod] - fn _safe_constructor() -> PyResult { - Ok(Self::new(Ustr::from("NULL"), UUID4::new(), 0, 0).unwrap()) // Safe default + fn _safe_constructor() -> Self { + Self::new(Ustr::from("NULL"), UUID4::new(), 0, 0) } fn __richcmp__(&self, other: &Self, op: CompareOp, py: Python<'_>) -> Py { diff --git a/nautilus_core/common/src/timer.rs b/nautilus_core/common/src/timer.rs index 18bf5f42f3b6..56d7dcc87eb8 100644 --- a/nautilus_core/common/src/timer.rs +++ b/nautilus_core/common/src/timer.rs @@ -16,17 +16,20 @@ use std::{ cmp::Ordering, fmt::{Display, Formatter}, + time::Duration, }; -use anyhow::Result; use nautilus_core::{ correctness::check_valid_string, - time::{TimedeltaNanos, UnixNanos}, + time::{get_atomic_clock_realtime, TimedeltaNanos, UnixNanos}, uuid::UUID4, }; -use pyo3::ffi; +use pyo3::{ffi, IntoPy, Python}; +use tokio::sync::oneshot; use ustr::Ustr; +use crate::{handlers::EventHandler, runtime::get_runtime}; + #[repr(C)] #[derive(Clone, Debug)] #[allow(clippy::redundant_allocation)] // C ABI compatibility @@ -46,19 +49,15 @@ pub struct TimeEvent { pub ts_init: UnixNanos, } +/// Assumes `name` is a valid string. impl TimeEvent { - pub fn new( - name: Ustr, - event_id: UUID4, - ts_event: UnixNanos, - ts_init: UnixNanos, - ) -> Result { - Ok(Self { + pub fn new(name: Ustr, event_id: UUID4, ts_event: UnixNanos, ts_init: UnixNanos) -> Self { + Self { name, event_id, ts_event, ts_init, - }) + } } } @@ -120,7 +119,7 @@ pub trait Timer { fn cancel(&mut self); } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] pub struct TestTimer { pub name: Ustr, pub interval_ns: u64, @@ -153,7 +152,7 @@ impl TestTimer { #[must_use] pub fn pop_event(&self, event_id: UUID4, ts_init: UnixNanos) -> TimeEvent { TimeEvent { - name: Ustr::from(&self.name), + name: self.name, event_id, ts_event: self.next_time_ns, ts_init, @@ -206,6 +205,112 @@ impl Iterator for TestTimer { } } +pub struct LiveTimer { + pub name: Ustr, + pub interval_ns: u64, + pub start_time_ns: UnixNanos, + pub stop_time_ns: Option, + pub next_time_ns: UnixNanos, + pub is_expired: bool, + callback: EventHandler, + canceler: Option>, +} + +impl LiveTimer { + #[must_use] + pub fn new( + name: &str, + interval_ns: u64, + start_time_ns: UnixNanos, + stop_time_ns: Option, + callback: EventHandler, + ) -> Self { + check_valid_string(name, "`TestTimer` name").unwrap(); + + Self { + name: Ustr::from(name), + interval_ns, + start_time_ns, + stop_time_ns, + next_time_ns: start_time_ns + interval_ns, + is_expired: false, + callback, + canceler: None, + } + } + + pub fn start(&mut self) { + let event_name = self.name; + let mut start_time_ns = self.start_time_ns; + let stop_time_ns = self.stop_time_ns; + let interval_ns = self.interval_ns; + + let callback = self + .callback + .py_callback + .clone() + .expect("No callback for event handler"); + + // Setup oneshot channel to be able to cancel timer task + let (cancel_tx, mut cancel_rx) = oneshot::channel(); + self.canceler = Some(cancel_tx); + + get_runtime().spawn(async move { + let clock = get_atomic_clock_realtime(); + if start_time_ns == 0 { + start_time_ns = clock.get_time_ns(); + } + + let mut next_time_ns = start_time_ns + interval_ns; + + loop { + tokio::select! { + _ = tokio::time::sleep(Duration::from_nanos(next_time_ns - clock.get_time_ns())) => { + Python::with_gil(|py| { + // Create new time event + let event = TimeEvent::new( + event_name, + UUID4::new(), + next_time_ns, + clock.get_time_ns() + ); + let py_event = event.into_py(py); + match callback.call1(py, (py_event,)) { + Ok(_) => {}, + Err(e) => eprintln!("Error on callback: {:?}", e), + }; + }); + + // Prepare next time interval + next_time_ns += interval_ns; + + // Check if expired + if let Some(stop_time_ns) = stop_time_ns { + if next_time_ns >= stop_time_ns { + break; // Timer expired + } + } + }, + _ = (&mut cancel_rx) => { + break; // Timer canceled + }, + } + } + + Ok::<(), anyhow::Error>(()) + }); + + self.is_expired = true; + } + + /// Cancels the timer (the timer will not generate an event). + pub fn cancel(&mut self) { + if let Some(sender) = self.canceler.take() { + let _ = sender.send(()); + } + } +} + //////////////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////////////// @@ -216,7 +321,7 @@ mod tests { use super::{TestTimer, TimeEvent}; #[rstest] - fn test_pop_event() { + fn test_test_timer_pop_event() { let mut timer = TestTimer::new("test_timer", 0, 1, None); assert!(timer.next().is_some()); @@ -226,7 +331,7 @@ mod tests { } #[rstest] - fn test_advance_within_next_time_ns() { + fn test_test_timer_advance_within_next_time_ns() { let mut timer = TestTimer::new("test_timer", 5, 0, None); let _: Vec = timer.advance(1).collect(); let _: Vec = timer.advance(2).collect(); @@ -237,28 +342,28 @@ mod tests { } #[rstest] - fn test_advance_up_to_next_time_ns() { + fn test_test_timer_advance_up_to_next_time_ns() { let mut timer = TestTimer::new("test_timer", 1, 0, None); assert_eq!(timer.advance(1).count(), 1); assert!(!timer.is_expired); } #[rstest] - fn test_advance_up_to_next_time_ns_with_stop_time() { + fn test_test_timer_advance_up_to_next_time_ns_with_stop_time() { let mut timer = TestTimer::new("test_timer", 1, 0, Some(2)); assert_eq!(timer.advance(2).count(), 2); assert!(timer.is_expired); } #[rstest] - fn test_advance_beyond_next_time_ns() { + fn test_test_timer_advance_beyond_next_time_ns() { let mut timer = TestTimer::new("test_timer", 1, 0, Some(5)); assert_eq!(timer.advance(5).count(), 5); assert!(timer.is_expired); } #[rstest] - fn test_advance_beyond_stop_time() { + fn test_test_timer_advance_beyond_stop_time() { let mut timer = TestTimer::new("test_timer", 1, 0, Some(5)); assert_eq!(timer.advance(10).count(), 5); assert!(timer.is_expired); diff --git a/nautilus_trader/common/component.pxd b/nautilus_trader/common/component.pxd index 5ada79622eaf..315803cf106c 100644 --- a/nautilus_trader/common/component.pxd +++ b/nautilus_trader/common/component.pxd @@ -90,31 +90,6 @@ cdef class TestClock(Clock): cdef class LiveClock(Clock): cdef LiveClock_API _mem - cdef object _default_handler - cdef dict _handlers - - cdef object _loop - cdef int _timer_count - cdef dict _timers - cdef LiveTimer[:] _stack - cdef tzinfo _utc - cdef uint64_t _next_event_time_ns - - cpdef void _raise_time_event(self, LiveTimer timer) - - cdef void _handle_time_event(self, TimeEvent event) - cdef void _add_timer(self, LiveTimer timer, handler: Callable[[TimeEvent], None]) - cdef void _remove_timer(self, LiveTimer timer) - cdef void _update_stack(self) - cdef void _update_timing(self) - cdef LiveTimer _create_timer( - self, - str name, - callback: Callable[[TimeEvent], None], - uint64_t interval_ns, - uint64_t start_time_ns, - uint64_t stop_time_ns, - ) cdef class TimeEvent(Event): @@ -134,39 +109,6 @@ cdef class TimeEventHandler: cpdef void handle(self) -cdef class LiveTimer: - cdef object _internal - - cdef readonly str name - """The timers name using for hashing.\n\n:returns: `str`""" - cdef readonly object callback - """The timers callback function.\n\n:returns: `object`""" - cdef readonly uint64_t interval_ns - """The timers set interval.\n\n:returns: `uint64_t`""" - cdef readonly uint64_t start_time_ns - """The timers set start time.\n\n:returns: `uint64_t`""" - cdef readonly uint64_t next_time_ns - """The timers next alert timestamp.\n\n:returns: `uint64_t`""" - cdef readonly uint64_t stop_time_ns - """The timers set stop time (if set).\n\n:returns: `uint64_t`""" - cdef readonly bint is_expired - """If the timer is expired.\n\n:returns: `bool`""" - - cpdef TimeEvent pop_event(self, UUID4 event_id, uint64_t ts_init) - cpdef void iterate_next_time(self, uint64_t to_time_ns) - cpdef void cancel(self) - cpdef void repeat(self, uint64_t ts_now) - cdef object _start_timer(self, uint64_t ts_now) - - -cdef class ThreadTimer(LiveTimer): - pass - - -cdef class LoopTimer(LiveTimer): - cdef object _loop - - cdef str RECV cdef str SENT cdef str CMD diff --git a/nautilus_trader/common/component.pyx b/nautilus_trader/common/component.pyx index 55f0b6a09c35..d0d229094eb0 100644 --- a/nautilus_trader/common/component.pyx +++ b/nautilus_trader/common/component.pyx @@ -63,8 +63,15 @@ from nautilus_trader.core.rust.common cimport component_state_from_cstr from nautilus_trader.core.rust.common cimport component_state_to_cstr from nautilus_trader.core.rust.common cimport component_trigger_from_cstr from nautilus_trader.core.rust.common cimport component_trigger_to_cstr +from nautilus_trader.core.rust.common cimport live_clock_cancel_timer from nautilus_trader.core.rust.common cimport live_clock_drop from nautilus_trader.core.rust.common cimport live_clock_new +from nautilus_trader.core.rust.common cimport live_clock_next_time +from nautilus_trader.core.rust.common cimport live_clock_register_default_handler +from nautilus_trader.core.rust.common cimport live_clock_set_time_alert +from nautilus_trader.core.rust.common cimport live_clock_set_timer +from nautilus_trader.core.rust.common cimport live_clock_timer_count +from nautilus_trader.core.rust.common cimport live_clock_timer_names from nautilus_trader.core.rust.common cimport live_clock_timestamp from nautilus_trader.core.rust.common cimport live_clock_timestamp_ms from nautilus_trader.core.rust.common cimport live_clock_timestamp_ns @@ -92,11 +99,11 @@ from nautilus_trader.core.rust.common cimport test_clock_cancel_timer from nautilus_trader.core.rust.common cimport test_clock_cancel_timers from nautilus_trader.core.rust.common cimport test_clock_drop from nautilus_trader.core.rust.common cimport test_clock_new -from nautilus_trader.core.rust.common cimport test_clock_next_time_ns +from nautilus_trader.core.rust.common cimport test_clock_next_time from nautilus_trader.core.rust.common cimport test_clock_register_default_handler from nautilus_trader.core.rust.common cimport test_clock_set_time -from nautilus_trader.core.rust.common cimport test_clock_set_time_alert_ns -from nautilus_trader.core.rust.common cimport test_clock_set_timer_ns +from nautilus_trader.core.rust.common cimport test_clock_set_time_alert +from nautilus_trader.core.rust.common cimport test_clock_set_timer from nautilus_trader.core.rust.common cimport test_clock_timer_count from nautilus_trader.core.rust.common cimport test_clock_timer_names from nautilus_trader.core.rust.common cimport test_clock_timestamp @@ -534,7 +541,7 @@ cdef class TestClock(Clock): Condition.valid_string(name, "name") Condition.not_in(name, self.timer_names, "name", "self.timer_names") - test_clock_set_time_alert_ns( + test_clock_set_time_alert( &self._mem, pystr_to_cstr(name), alert_time_ns, @@ -561,7 +568,7 @@ cdef class TestClock(Clock): Condition.true(stop_time_ns > ts_now, "`stop_time_ns` was < `ts_now`") Condition.true(start_time_ns + interval_ns <= stop_time_ns, "`start_time_ns` + `interval_ns` was > `stop_time_ns`") - test_clock_set_timer_ns( + test_clock_set_timer( &self._mem, pystr_to_cstr(name), interval_ns, @@ -572,7 +579,7 @@ cdef class TestClock(Clock): cpdef uint64_t next_time_ns(self, str name): Condition.valid_string(name, "name") - return test_clock_next_time_ns(&self._mem, pystr_to_cstr(name)) + return test_clock_next_time(&self._mem, pystr_to_cstr(name)) cpdef void cancel_timer(self, str name): Condition.valid_string(name, "name") @@ -659,17 +666,8 @@ cdef class LiveClock(Clock): The event loop for the clocks timers. """ - def __init__(self, loop: asyncio.AbstractEventLoop | None = None): + def __init__(self): self._mem = live_clock_new() - self._default_handler = None - self._handlers: dict[str, Callable[[TimeEvent], None]] = {} - - self._loop = loop - self._timers: dict[str, LiveTimer] = {} - self._stack = np.ascontiguousarray([], dtype=LiveTimer) - - self._timer_count = 0 - self._next_event_time_ns = 0 def __del__(self) -> None: if self._mem._0 != NULL: @@ -677,11 +675,24 @@ cdef class LiveClock(Clock): @property def timer_names(self) -> list[str]: - return list(self._timers.keys()) + return sorted(live_clock_timer_names(&self._mem)) @property def timer_count(self) -> int: - return self._timer_count + return live_clock_timer_count(&self._mem) + + @staticmethod + def create_pyo3_conversion_wrapper(callback) -> Callable: + # TODO: Improve efficiency of pyo3 object conversion + def wrapper(pyo3_event): + event = TimeEvent( + name=pyo3_event.name, + event_id=UUID4(pyo3_event.event_id.value), + ts_event=pyo3_event.ts_event, + ts_init=pyo3_event.ts_init, + ) + callback(event) + return wrapper cpdef double timestamp(self): return live_clock_timestamp(&self._mem) @@ -695,7 +706,9 @@ cdef class LiveClock(Clock): cpdef void register_default_handler(self, callback: Callable[[TimeEvent], None]): Condition.callable(callback, "callback") - self._default_handler = callback + callback = LiveClock.create_pyo3_conversion_wrapper(callback) + + live_clock_register_default_handler(&self._mem, callback) cpdef void set_time_alert_ns( self, @@ -705,19 +718,16 @@ cdef class LiveClock(Clock): ): Condition.valid_string(name, "name") Condition.not_in(name, self.timer_names, "name", "self.timer_names") - if callback is None: - callback = self._default_handler - cdef uint64_t ts_now = self.timestamp_ns() + if callback is not None: + callback = LiveClock.create_pyo3_conversion_wrapper(callback) - cdef LiveTimer timer = self._create_timer( - name=name, - callback=callback, - interval_ns=alert_time_ns - ts_now, - start_time_ns=ts_now, - stop_time_ns=alert_time_ns, + live_clock_set_time_alert( + &self._mem, + pystr_to_cstr(name), + alert_time_ns, + callback, ) - self._add_timer(timer, callback) cpdef void set_timer_ns( self, @@ -728,17 +738,13 @@ cdef class LiveClock(Clock): callback: Callable[[TimeEvent], None] | None = None, ): Condition.not_in(name, self.timer_names, "name", "self.timer_names") + Condition.not_in(name, self.timer_names, "name", "self.timer_names") + Condition.positive_int(interval_ns, "interval_ns") - cdef uint64_t ts_now = self.timestamp_ns() # Call here for greater accuracy - - Condition.valid_string(name, "name") - if callback is None: - callback = self._default_handler + if callback is not None: + callback = LiveClock.create_pyo3_conversion_wrapper(callback) - Condition.not_in(name, self._timers, "name", "_timers") - Condition.not_in(name, self._handlers, "name", "_handlers") - Condition.true(interval_ns > 0, f"interval was {interval_ns}") - Condition.callable(callback, "callback") + cdef uint64_t ts_now = self.timestamp_ns() # Call here for greater accuracy if start_time_ns == 0: start_time_ns = ts_now @@ -746,54 +752,24 @@ cdef class LiveClock(Clock): Condition.true(stop_time_ns > ts_now, "stop_time was < ts_now") Condition.true(start_time_ns + interval_ns <= stop_time_ns, "start_time + interval was > stop_time") - cdef LiveTimer timer = self._create_timer( - name=name, - callback=callback, - interval_ns=interval_ns, - start_time_ns=start_time_ns, - stop_time_ns=stop_time_ns, + live_clock_set_timer( + &self._mem, + pystr_to_cstr(name), + interval_ns, + start_time_ns, + stop_time_ns, + callback, ) - self._add_timer(timer, callback) - - cdef void _add_timer(self, LiveTimer timer, handler: Callable[[TimeEvent], None]): - self._timers[timer.name] = timer - self._handlers[timer.name] = handler - self._update_stack() - self._update_timing() - - cdef void _remove_timer(self, LiveTimer timer): - self._timers.pop(timer.name, None) - self._handlers.pop(timer.name, None) - self._update_stack() - self._update_timing() - - cdef void _update_stack(self): - self._timer_count = len(self._timers) - - if self._timer_count > 0: - # The call to `np.ascontiguousarray` here looks inefficient, its - # only called when a timer is added or removed. This then allows the - # construction of an efficient Timer[:] memoryview. - timers = list(self._timers.values()) - self._stack = np.ascontiguousarray(timers, dtype=LiveTimer) - else: - self._stack = None cpdef uint64_t next_time_ns(self, str name): - return self._timers[name].next_time_ns + Condition.valid_string(name, "name") + return live_clock_next_time(&self._mem, pystr_to_cstr(name)) cpdef void cancel_timer(self, str name): Condition.valid_string(name, "name") Condition.is_in(name, self.timer_names, "name", "self.timer_names") - cdef LiveTimer timer = self._timers.pop(name, None) - if not timer: - # No timer with given name - return - - timer.cancel() - self._handlers.pop(name, None) - self._remove_timer(timer) + live_clock_cancel_timer(&self._mem, pystr_to_cstr(name)) cpdef void cancel_timers(self): cdef str name @@ -803,84 +779,6 @@ cdef class LiveClock(Clock): # and timer. self.cancel_timer(name) - @cython.boundscheck(False) - @cython.wraparound(False) - cdef void _update_timing(self): - if self._timer_count == 0: - self._next_event_time_ns = 0 - return - - cdef LiveTimer first_timer = self._stack[0] - if self._timer_count == 1: - self._next_event_time_ns = first_timer.next_time_ns - return - - cdef uint64_t next_time_ns = first_timer.next_time_ns - cdef: - int i - LiveTimer timer - uint64_t observed_ns - for i in range(self._timer_count - 1): - timer = self._stack[i + 1] - observed_ns = timer.next_time_ns - if observed_ns < next_time_ns: - next_time_ns = observed_ns - - self._next_event_time_ns = next_time_ns - - cdef LiveTimer _create_timer( - self, - str name, - callback: Callable[[TimeEvent], None], - uint64_t interval_ns, - uint64_t start_time_ns, - uint64_t stop_time_ns, - ): - if self._loop is not None: - return LoopTimer( - loop=self._loop, - name=name, - callback=self._raise_time_event, - interval_ns=interval_ns, - ts_now=self.timestamp_ns(), # Timestamp here for accuracy - start_time_ns=start_time_ns, - stop_time_ns=stop_time_ns, - ) - else: - return ThreadTimer( - name=name, - callback=self._raise_time_event, - interval_ns=interval_ns, - ts_now=self.timestamp_ns(), # Timestamp here for accuracy - start_time_ns=start_time_ns, - stop_time_ns=stop_time_ns, - ) - - cpdef void _raise_time_event(self, LiveTimer timer): - cdef uint64_t now = self.timestamp_ns() - cdef TimeEvent event = timer.pop_event( - event_id=UUID4(), - ts_init=now, - ) - - if now < timer.next_time_ns: - timer.iterate_next_time(timer.next_time_ns) - else: - timer.iterate_next_time(now) - - self._handle_time_event(event) - - if timer.is_expired: - self._remove_timer(timer) - else: # Continue timing - timer.repeat(ts_now=self.timestamp_ns()) - self._update_timing() - - cdef void _handle_time_event(self, TimeEvent event): - handler = self._handlers.get(event.name) - if handler is not None: - handler(event) - cdef class TimeEvent(Event): """ @@ -1040,248 +938,6 @@ cdef class TimeEventHandler: ) -cdef class LiveTimer: - """ - The base class for all live timers. - - Parameters - ---------- - name : str - The name for the timer. - callback : Callable[[TimeEvent], None] - The delegate to call at the next time. - interval_ns : uint64_t - The time interval for the timer. - ts_now : uint64_t - The current UNIX time (nanoseconds). - start_time_ns : uint64_t - The start datetime for the timer (UTC). - stop_time_ns : uint64_t, optional - The stop datetime for the timer (UTC) (if None then timer repeats). - - Raises - ------ - TypeError - If `callback` is not of type `Callable`. - - Warnings - -------- - This class should not be used directly, but through a concrete subclass. - """ - - def __init__( - self, - str name not None, - callback not None: Callable[[TimeEvent], None], - uint64_t interval_ns, - uint64_t ts_now, - uint64_t start_time_ns, - uint64_t stop_time_ns=0, - ): - Condition.valid_string(name, "name") - Condition.callable(callback, "callback") - - self.name = name - self.callback = callback - self.interval_ns = interval_ns - self.start_time_ns = start_time_ns - self.next_time_ns = start_time_ns + interval_ns - self.stop_time_ns = stop_time_ns - self.is_expired = False - - self._internal = self._start_timer(ts_now) - - def __eq__(self, LiveTimer other) -> bool: - return self.name == other.name - - def __hash__(self) -> int: - return hash(self.name) - - def __repr__(self) -> str: - return ( - f"{type(self).__name__}(" - f"name={self.name}, " - f"interval_ns={self.interval_ns}, " - f"start_time_ns={self.start_time_ns}, " - f"next_time_ns={self.next_time_ns}, " - f"stop_time_ns={self.stop_time_ns}, " - f"is_expired={self.is_expired})" - ) - - cpdef TimeEvent pop_event(self, UUID4 event_id, uint64_t ts_init): - """ - Return a generated time event with the given ID. - - Parameters - ---------- - event_id : UUID4 - The ID for the time event. - ts_init : uint64_t - The UNIX timestamp (nanoseconds) when the object was initialized. - - Returns - ------- - TimeEvent - - """ - # Precondition: `event_id` validated in `TimeEvent` - - return TimeEvent( - name=self.name, - event_id=event_id, - ts_event=self.next_time_ns, - ts_init=ts_init, - ) - - cpdef void iterate_next_time(self, uint64_t ts_now): - """ - Iterates the timers next time and checks if the timer is now expired. - - Parameters - ---------- - ts_now : uint64_t - The current UNIX time (nanoseconds). - - """ - self.next_time_ns += self.interval_ns - if self.stop_time_ns and ts_now >= self.stop_time_ns: - self.is_expired = True - - cpdef void repeat(self, uint64_t ts_now): - """ - Continue the timer. - - Parameters - ---------- - ts_now : uint64_t - The current time to continue timing from. - - """ - self._internal = self._start_timer(ts_now) - - cpdef void cancel(self): - """ - Cancels the timer (the timer will not generate an event). - """ - self._internal.cancel() - - cdef object _start_timer(self, uint64_t ts_now): - """Abstract method (implement in subclass).""" - raise NotImplementedError("method `_start_timer` must be implemented in the subclass") # pragma: no cover - - -cdef class ThreadTimer(LiveTimer): - """ - Provides a thread based timer for live trading. - - Parameters - ---------- - name : str - The name for the timer. - callback : Callable[[TimeEvent], None] - The delegate to call at the next time. - interval_ns : uint64_t - The time interval for the timer. - ts_now : uint64_t - The current UNIX time (nanoseconds). - start_time_ns : uint64_t - The start datetime for the timer (UTC). - stop_time_ns : uint64_t, optional - The stop datetime for the timer (UTC) (if None then timer repeats). - - Raises - ------ - TypeError - If `callback` is not of type `Callable`. - """ - - def __init__( - self, - str name not None, - callback not None: Callable[[TimeEvent], None], - uint64_t interval_ns, - uint64_t ts_now, - uint64_t start_time_ns, - uint64_t stop_time_ns=0, - ): - super().__init__( - name=name, - callback=callback, - interval_ns=interval_ns, - ts_now=ts_now, - start_time_ns=start_time_ns, - stop_time_ns=stop_time_ns, - ) - - cdef object _start_timer(self, uint64_t ts_now): - timer = TimerThread( - interval=nanos_to_secs(self.next_time_ns - ts_now), - function=self.callback, - args=[self], - ) - timer.daemon = True - timer.start() - - return timer - - -cdef class LoopTimer(LiveTimer): - """ - Provides an event loop based timer for live trading. - - Parameters - ---------- - loop : asyncio.AbstractEventLoop - The event loop to run the timer on. - name : str - The name for the timer. - callback : Callable[[TimeEvent], None] - The delegate to call at the next time. - interval_ns : uint64_t - The time interval for the timer (nanoseconds). - ts_now : uint64_t - The current UNIX epoch (nanoseconds). - start_time_ns : uint64_t - The start datetime for the timer (UTC). - stop_time_ns : uint64_t, optional - The stop datetime for the timer (UTC) (if None then timer repeats). - - Raises - ------ - TypeError - If `callback` is not of type `Callable`. - """ - - def __init__( - self, - loop not None, - str name not None, - callback not None: Callable[[TimeEvent], None], - uint64_t interval_ns, - uint64_t ts_now, - uint64_t start_time_ns, - uint64_t stop_time_ns=0, - ): - Condition.valid_string(name, "name") - - self._loop = loop # Assign here as `super().__init__` will call it - super().__init__( - name=name, - callback=callback, - interval_ns=interval_ns, - ts_now=ts_now, - start_time_ns=start_time_ns, - stop_time_ns=stop_time_ns, - ) - - cdef object _start_timer(self, uint64_t ts_now): - return self._loop.call_later( - nanos_to_secs(self.next_time_ns - ts_now), - self.callback, - self, - ) - - RECV = "<--" SENT = "-->" CMD = "[CMD]" diff --git a/nautilus_trader/core/includes/common.h b/nautilus_trader/core/includes/common.h index d2f242b13554..c398c8884e5d 100644 --- a/nautilus_trader/core/includes/common.h +++ b/nautilus_trader/core/includes/common.h @@ -372,10 +372,10 @@ uintptr_t test_clock_timer_count(struct TestClock_API *clock); * - Assumes `name_ptr` is a valid C string pointer. * - Assumes `callback_ptr` is a valid `PyCallable` pointer. */ -void test_clock_set_time_alert_ns(struct TestClock_API *clock, - const char *name_ptr, - uint64_t alert_time_ns, - PyObject *callback_ptr); +void test_clock_set_time_alert(struct TestClock_API *clock, + const char *name_ptr, + uint64_t alert_time_ns, + PyObject *callback_ptr); /** * # Safety @@ -383,12 +383,12 @@ void test_clock_set_time_alert_ns(struct TestClock_API *clock, * - Assumes `name_ptr` is a valid C string pointer. * - Assumes `callback_ptr` is a valid `PyCallable` pointer. */ -void test_clock_set_timer_ns(struct TestClock_API *clock, - const char *name_ptr, - uint64_t interval_ns, - uint64_t start_time_ns, - uint64_t stop_time_ns, - PyObject *callback_ptr); +void test_clock_set_timer(struct TestClock_API *clock, + const char *name_ptr, + uint64_t interval_ns, + uint64_t start_time_ns, + uint64_t stop_time_ns, + PyObject *callback_ptr); /** * # Safety @@ -404,7 +404,7 @@ void vec_time_event_handlers_drop(CVec v); * * - Assumes `name_ptr` is a valid C string pointer. */ -uint64_t test_clock_next_time_ns(struct TestClock_API *clock, const char *name_ptr); +uint64_t test_clock_next_time(struct TestClock_API *clock, const char *name_ptr); /** * # Safety @@ -419,6 +419,13 @@ struct LiveClock_API live_clock_new(void); void live_clock_drop(struct LiveClock_API clock); +/** + * # Safety + * + * - Assumes `callback_ptr` is a valid `PyCallable` pointer. + */ +void live_clock_register_default_handler(struct LiveClock_API *clock, PyObject *callback_ptr); + double live_clock_timestamp(struct LiveClock_API *clock); uint64_t live_clock_timestamp_ms(struct LiveClock_API *clock); @@ -427,6 +434,50 @@ uint64_t live_clock_timestamp_us(struct LiveClock_API *clock); uint64_t live_clock_timestamp_ns(struct LiveClock_API *clock); +PyObject *live_clock_timer_names(const struct LiveClock_API *clock); + +uintptr_t live_clock_timer_count(struct LiveClock_API *clock); + +/** + * # Safety + * + * - Assumes `name_ptr` is a valid C string pointer. + * - Assumes `callback_ptr` is a valid `PyCallable` pointer. + */ +void live_clock_set_time_alert(struct LiveClock_API *clock, + const char *name_ptr, + uint64_t alert_time_ns, + PyObject *callback_ptr); + +/** + * # Safety + * + * - Assumes `name_ptr` is a valid C string pointer. + * - Assumes `callback_ptr` is a valid `PyCallable` pointer. + */ +void live_clock_set_timer(struct LiveClock_API *clock, + const char *name_ptr, + uint64_t interval_ns, + uint64_t start_time_ns, + uint64_t stop_time_ns, + PyObject *callback_ptr); + +/** + * # Safety + * + * - Assumes `name_ptr` is a valid C string pointer. + */ +uint64_t live_clock_next_time(struct LiveClock_API *clock, const char *name_ptr); + +/** + * # Safety + * + * - Assumes `name_ptr` is a valid C string pointer. + */ +void live_clock_cancel_timer(struct LiveClock_API *clock, const char *name_ptr); + +void live_clock_cancel_timers(struct LiveClock_API *clock); + const char *component_state_to_cstr(enum ComponentState value); /** diff --git a/nautilus_trader/core/rust/common.pxd b/nautilus_trader/core/rust/common.pxd index 79ab81c49311..c2121bd33884 100644 --- a/nautilus_trader/core/rust/common.pxd +++ b/nautilus_trader/core/rust/common.pxd @@ -235,21 +235,21 @@ cdef extern from "../includes/common.h": # # - Assumes `name_ptr` is a valid C string pointer. # - Assumes `callback_ptr` is a valid `PyCallable` pointer. - void test_clock_set_time_alert_ns(TestClock_API *clock, - const char *name_ptr, - uint64_t alert_time_ns, - PyObject *callback_ptr); + void test_clock_set_time_alert(TestClock_API *clock, + const char *name_ptr, + uint64_t alert_time_ns, + PyObject *callback_ptr); # # Safety # # - Assumes `name_ptr` is a valid C string pointer. # - Assumes `callback_ptr` is a valid `PyCallable` pointer. - void test_clock_set_timer_ns(TestClock_API *clock, - const char *name_ptr, - uint64_t interval_ns, - uint64_t start_time_ns, - uint64_t stop_time_ns, - PyObject *callback_ptr); + void test_clock_set_timer(TestClock_API *clock, + const char *name_ptr, + uint64_t interval_ns, + uint64_t start_time_ns, + uint64_t stop_time_ns, + PyObject *callback_ptr); # # Safety # @@ -261,7 +261,7 @@ cdef extern from "../includes/common.h": # # Safety # # - Assumes `name_ptr` is a valid C string pointer. - uint64_t test_clock_next_time_ns(TestClock_API *clock, const char *name_ptr); + uint64_t test_clock_next_time(TestClock_API *clock, const char *name_ptr); # # Safety # @@ -274,6 +274,11 @@ cdef extern from "../includes/common.h": void live_clock_drop(LiveClock_API clock); + # # Safety + # + # - Assumes `callback_ptr` is a valid `PyCallable` pointer. + void live_clock_register_default_handler(LiveClock_API *clock, PyObject *callback_ptr); + double live_clock_timestamp(LiveClock_API *clock); uint64_t live_clock_timestamp_ms(LiveClock_API *clock); @@ -282,6 +287,42 @@ cdef extern from "../includes/common.h": uint64_t live_clock_timestamp_ns(LiveClock_API *clock); + PyObject *live_clock_timer_names(const LiveClock_API *clock); + + uintptr_t live_clock_timer_count(LiveClock_API *clock); + + # # Safety + # + # - Assumes `name_ptr` is a valid C string pointer. + # - Assumes `callback_ptr` is a valid `PyCallable` pointer. + void live_clock_set_time_alert(LiveClock_API *clock, + const char *name_ptr, + uint64_t alert_time_ns, + PyObject *callback_ptr); + + # # Safety + # + # - Assumes `name_ptr` is a valid C string pointer. + # - Assumes `callback_ptr` is a valid `PyCallable` pointer. + void live_clock_set_timer(LiveClock_API *clock, + const char *name_ptr, + uint64_t interval_ns, + uint64_t start_time_ns, + uint64_t stop_time_ns, + PyObject *callback_ptr); + + # # Safety + # + # - Assumes `name_ptr` is a valid C string pointer. + uint64_t live_clock_next_time(LiveClock_API *clock, const char *name_ptr); + + # # Safety + # + # - Assumes `name_ptr` is a valid C string pointer. + void live_clock_cancel_timer(LiveClock_API *clock, const char *name_ptr); + + void live_clock_cancel_timers(LiveClock_API *clock); + const char *component_state_to_cstr(ComponentState value); # Returns an enum from a Python string. diff --git a/nautilus_trader/system/kernel.py b/nautilus_trader/system/kernel.py index f7cad951bcbf..f546efcdfe1c 100644 --- a/nautilus_trader/system/kernel.py +++ b/nautilus_trader/system/kernel.py @@ -147,7 +147,7 @@ def __init__( # noqa (too complex) if self._environment == Environment.BACKTEST: self._clock = TestClock() elif self.environment in (Environment.SANDBOX, Environment.LIVE): - self._clock = LiveClock(loop=loop) + self._clock = LiveClock() else: raise NotImplementedError( # pragma: no cover (design-time error) f"environment {self._environment} not recognized", # pragma: no cover (design-time error) diff --git a/nautilus_trader/trading/trader.py b/nautilus_trader/trading/trader.py index 98c8545b9773..4b4194ba1af4 100644 --- a/nautilus_trader/trading/trader.py +++ b/nautilus_trader/trading/trader.py @@ -32,7 +32,6 @@ from nautilus_trader.common.actor import Actor from nautilus_trader.common.component import Clock from nautilus_trader.common.component import Component -from nautilus_trader.common.component import LiveClock from nautilus_trader.common.component import MessageBus from nautilus_trader.core.correctness import PyCondition from nautilus_trader.data.engine import DataEngine @@ -299,17 +298,12 @@ def add_actor(self, actor: Actor) -> None: "try specifying a different actor ID.", ) - if isinstance(self._clock, LiveClock): - clock = self._clock.__class__(loop=self._loop) - else: - clock = self._clock.__class__() - # Wire component into trader actor.register_base( portfolio=self._portfolio, msgbus=self._msgbus, cache=self._cache, - clock=clock, # Clock per component + clock=self._clock.__class__(), # Clock per component ) self._actors[actor.id] = actor @@ -367,11 +361,6 @@ def add_strategy(self, strategy: Strategy) -> None: "try specifying a different strategy ID.", ) - if isinstance(self._clock, LiveClock): - clock = self._clock.__class__(loop=self._loop) - else: - clock = self._clock.__class__() - # Confirm strategy ID order_id_tags: list[str] = [s.order_id_tag for s in self._strategies.values()] if strategy.order_id_tag in (None, str(None)): @@ -394,7 +383,7 @@ def add_strategy(self, strategy: Strategy) -> None: portfolio=self._portfolio, msgbus=self._msgbus, cache=self._cache, - clock=clock, # Clock per strategy + clock=self._clock.__class__(), # Clock per strategy ) self._exec_engine.register_oms_type(strategy) @@ -454,18 +443,13 @@ def add_exec_algorithm(self, exec_algorithm: ExecAlgorithm) -> None: "try specifying a different `exec_algorithm_id`.", ) - if isinstance(self._clock, LiveClock): - clock = self._clock.__class__(loop=self._loop) - else: - clock = self._clock.__class__() - # Wire execution algorithm into trader exec_algorithm.register( trader_id=self.id, portfolio=self._portfolio, msgbus=self._msgbus, cache=self._cache, - clock=clock, # Clock per algorithm + clock=self._clock.__class__(), # Clock per algorithm ) self._exec_algorithms[exec_algorithm.id] = exec_algorithm diff --git a/tests/unit_tests/common/test_clock.py b/tests/unit_tests/common/test_clock.py index 5fc112529960..fe62ffb78540 100644 --- a/tests/unit_tests/common/test_clock.py +++ b/tests/unit_tests/common/test_clock.py @@ -13,7 +13,6 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -import asyncio import time from datetime import datetime from datetime import timedelta @@ -555,7 +554,7 @@ def test_advance_time_with_multiple_set_timers_triggers_events(self): assert clock.timer_count == 2 -class TestLiveClockWithThreadTimer: +class TestLiveClock: def setup(self): # Fixture Setup self.handler = [] @@ -821,269 +820,3 @@ def test_set_two_repeating_timers(self): # Assert assert len(self.handler) >= 2 - - -class TestLiveClockWithLoopTimer: - def setup(self): - # Fixture Setup - self.loop = asyncio.get_event_loop() - # asyncio.set_event_loop(self.loop) - self.loop.set_debug(True) - - self.handler = [] - self.clock = LiveClock(loop=self.loop) - self.clock.register_default_handler(self.handler.append) - - def teardown(self): - self.clock.cancel_timers() - - @pytest.mark.asyncio() - async def test_timestamp_is_monotonic(self): - # Arrange, Act - result1 = self.clock.timestamp() - result2 = self.clock.timestamp() - result3 = self.clock.timestamp() - result4 = self.clock.timestamp() - result5 = self.clock.timestamp() - - # Assert - assert isinstance(result1, float) - assert result1 > 0 - assert result5 >= result4 - assert result4 >= result3 - assert result3 >= result2 - assert result2 >= result1 - - @pytest.mark.asyncio() - async def test_timestamp_ms_is_monotonic(self): - # Arrange, Act - result1 = self.clock.timestamp_ms() - result2 = self.clock.timestamp_ms() - result3 = self.clock.timestamp_ms() - result4 = self.clock.timestamp_ms() - result5 = self.clock.timestamp_ms() - - # Assert - assert isinstance(result1, int) - assert result1 > 0 - assert result5 >= result4 - assert result4 >= result3 - assert result3 >= result2 - assert result2 >= result1 - - @pytest.mark.asyncio() - async def test_timestamp_ns_is_monotonic(self): - # Arrange, Act - result1 = self.clock.timestamp_ns() - result2 = self.clock.timestamp_ns() - result3 = self.clock.timestamp_ns() - result4 = self.clock.timestamp_ns() - result5 = self.clock.timestamp_ns() - - # Assert - assert isinstance(result1, int) - assert result1 > 0 - assert result5 >= result4 - assert result4 >= result3 - assert result3 >= result2 - assert result2 >= result1 - - @pytest.mark.asyncio() - async def test_set_time_alert(self): - # Arrange - name = "TEST_ALERT" - interval = timedelta(milliseconds=300) - alert_time = self.clock.utc_now() + interval - - # Act - self.clock.set_time_alert(name, alert_time) - await asyncio.sleep(1.0) - - # Assert - assert self.clock.timer_count == 0 - assert len(self.handler) >= 1 - assert isinstance(self.handler[0], TimeEvent) - - @pytest.mark.asyncio() - async def test_cancel_time_alert(self): - # Arrange - name = "TEST_ALERT" - interval = timedelta(milliseconds=300) - alert_time = self.clock.utc_now() + interval - - self.clock.set_time_alert(name, alert_time) - - # Act - self.clock.cancel_timer(name) - - # Assert - assert self.clock.timer_count == 0 - assert len(self.handler) == 0 - - @pytest.mark.asyncio() - async def test_set_multiple_time_alerts(self): - # Arrange - alert_time1 = self.clock.utc_now() + timedelta(milliseconds=200) - alert_time2 = self.clock.utc_now() + timedelta(milliseconds=300) - - # Act - self.clock.set_time_alert("TEST_ALERT1", alert_time1) - self.clock.set_time_alert("TEST_ALERT2", alert_time2) - await asyncio.sleep(1.0) - - # Assert - assert self.clock.timer_count == 0 - assert len(self.handler) >= 2 - assert isinstance(self.handler[0], TimeEvent) - assert isinstance(self.handler[1], TimeEvent) - - @pytest.mark.asyncio() - async def test_set_timer_with_immediate_start_time(self): - # Arrange - name = "TEST_TIMER" - - # Act - self.clock.set_timer( - name=name, - interval=timedelta(milliseconds=100), - start_time=None, - stop_time=None, - ) - - await asyncio.sleep(1.0) - - # Assert - assert self.clock.timer_names == [name] - assert isinstance(self.handler[0], TimeEvent) - - @pytest.mark.asyncio() - async def test_set_timer(self): - # Arrange - name = "TEST_TIMER" - interval = timedelta(milliseconds=100) - start_time = self.clock.utc_now() + interval - - # Act - self.clock.set_timer( - name=name, - interval=interval, - start_time=start_time, - stop_time=None, - ) - - await asyncio.sleep(2) - - # Assert - assert self.clock.timer_names == [name] - assert len(self.handler) > 0 - assert isinstance(self.handler[0], TimeEvent) - - @pytest.mark.asyncio() - async def test_set_timer_with_stop_time(self): - # Arrange - name = "TEST_TIMER" - interval = timedelta(milliseconds=100) - start_time = self.clock.utc_now() - stop_time = start_time + interval - - # Act - self.clock.set_timer( - name=name, - interval=interval, - start_time=start_time, - stop_time=stop_time, - ) - - await asyncio.sleep(0.5) - - # Assert - assert self.clock.timer_count == 0 - assert len(self.handler) >= 1 - assert isinstance(self.handler[0], TimeEvent) - - @pytest.mark.asyncio() - async def test_cancel_timer(self): - # Arrange - name = "TEST_TIMER" - interval = timedelta(milliseconds=100) - - self.clock.set_timer(name=name, interval=interval) - - # Act - await asyncio.sleep(0.3) - self.clock.cancel_timer(name) - await asyncio.sleep(0.3) - - # Assert - assert self.clock.timer_count == 0 - assert len(self.handler) <= 4 - - @pytest.mark.asyncio() - async def test_set_repeating_timer(self): - # Arrange - name = "TEST_TIMER" - interval = timedelta(milliseconds=100) - start_time = self.clock.utc_now() - - # Act - self.clock.set_timer( - name=name, - interval=interval, - start_time=start_time, - stop_time=None, - ) - - await asyncio.sleep(2) - - # Assert - assert len(self.handler) > 0 - assert isinstance(self.handler[0], TimeEvent) - - @pytest.mark.asyncio() - async def test_cancel_repeating_timer(self): - # Arrange - name = "TEST_TIMER" - interval = timedelta(milliseconds=100) - start_time = self.clock.utc_now() - stop_time = start_time + timedelta(seconds=5) - - self.clock.set_timer( - name=name, - interval=interval, - start_time=start_time, - stop_time=stop_time, - ) - - # Act - await asyncio.sleep(0.3) - self.clock.cancel_timer(name) - await asyncio.sleep(0.3) - - # Assert - assert len(self.handler) <= 5 - - @pytest.mark.asyncio() - async def test_set_two_repeating_timers(self): - # Arrange - interval = timedelta(milliseconds=100) - start_time = self.clock.utc_now() + timedelta(milliseconds=100) - - # Act - self.clock.set_timer( - name="TEST_TIMER1", - interval=interval, - start_time=start_time, - stop_time=None, - ) - - self.clock.set_timer( - name="TEST_TIMER2", - interval=interval, - start_time=start_time, - stop_time=None, - ) - - await asyncio.sleep(1) - - # Assert - assert len(self.handler) >= 2 From f1512d6921c56451e25c260f19eb7f0cd1cede95 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 17:07:55 +1100 Subject: [PATCH 053/104] Optimize pyo3 TimeEvent conversion with pycapsule --- nautilus_core/common/src/timer.rs | 7 +++--- nautilus_trader/common/component.pyx | 34 +++++++++++++++------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/nautilus_core/common/src/timer.rs b/nautilus_core/common/src/timer.rs index 56d7dcc87eb8..a49f0e84e9a9 100644 --- a/nautilus_core/common/src/timer.rs +++ b/nautilus_core/common/src/timer.rs @@ -24,7 +24,7 @@ use nautilus_core::{ time::{get_atomic_clock_realtime, TimedeltaNanos, UnixNanos}, uuid::UUID4, }; -use pyo3::{ffi, IntoPy, Python}; +use pyo3::{ffi, types::PyCapsule, IntoPy, PyObject, Python}; use tokio::sync::oneshot; use ustr::Ustr; @@ -274,8 +274,9 @@ impl LiveTimer { next_time_ns, clock.get_time_ns() ); - let py_event = event.into_py(py); - match callback.call1(py, (py_event,)) { + let capsule: PyObject = PyCapsule::new(py, event, None).expect("Error creating `PyCapsule`").into_py(py); + + match callback.call1(py, (capsule,)) { Ok(_) => {}, Err(e) => eprintln!("Error on callback: {:?}", e), }; diff --git a/nautilus_trader/common/component.pyx b/nautilus_trader/common/component.pyx index d0d229094eb0..53ec398f612e 100644 --- a/nautilus_trader/common/component.pyx +++ b/nautilus_trader/common/component.pyx @@ -43,6 +43,7 @@ from cpython.datetime cimport timedelta from cpython.datetime cimport tzinfo from cpython.object cimport PyCallable_Check from cpython.object cimport PyObject +from cpython.pycapsule cimport PyCapsule_GetPointer from libc.stdint cimport int64_t from libc.stdint cimport uint64_t from libc.stdio cimport printf @@ -681,19 +682,6 @@ cdef class LiveClock(Clock): def timer_count(self) -> int: return live_clock_timer_count(&self._mem) - @staticmethod - def create_pyo3_conversion_wrapper(callback) -> Callable: - # TODO: Improve efficiency of pyo3 object conversion - def wrapper(pyo3_event): - event = TimeEvent( - name=pyo3_event.name, - event_id=UUID4(pyo3_event.event_id.value), - ts_event=pyo3_event.ts_event, - ts_init=pyo3_event.ts_init, - ) - callback(event) - return wrapper - cpdef double timestamp(self): return live_clock_timestamp(&self._mem) @@ -706,7 +694,7 @@ cdef class LiveClock(Clock): cpdef void register_default_handler(self, callback: Callable[[TimeEvent], None]): Condition.callable(callback, "callback") - callback = LiveClock.create_pyo3_conversion_wrapper(callback) + callback = create_pyo3_conversion_wrapper(callback) live_clock_register_default_handler(&self._mem, callback) @@ -720,7 +708,7 @@ cdef class LiveClock(Clock): Condition.not_in(name, self.timer_names, "name", "self.timer_names") if callback is not None: - callback = LiveClock.create_pyo3_conversion_wrapper(callback) + callback = create_pyo3_conversion_wrapper(callback) live_clock_set_time_alert( &self._mem, @@ -742,7 +730,7 @@ cdef class LiveClock(Clock): Condition.positive_int(interval_ns, "interval_ns") if callback is not None: - callback = LiveClock.create_pyo3_conversion_wrapper(callback) + callback = create_pyo3_conversion_wrapper(callback) cdef uint64_t ts_now = self.timestamp_ns() # Call here for greater accuracy @@ -780,6 +768,13 @@ cdef class LiveClock(Clock): self.cancel_timer(name) +def create_pyo3_conversion_wrapper(callback) -> Callable: + def wrapper(capsule): + callback(capsule_to_time_event(capsule)) + + return wrapper + + cdef class TimeEvent(Event): """ Represents a time event occurring at the event timestamp. @@ -899,6 +894,13 @@ cdef class TimeEvent(Event): return event +cdef inline TimeEvent capsule_to_time_event(capsule): + cdef TimeEvent_t* ptr = PyCapsule_GetPointer(capsule, NULL) + cdef TimeEvent event = TimeEvent.__new__(TimeEvent) + event._mem = ptr[0] + return event + + cdef class TimeEventHandler: """ Represents a time event with its associated handler. From 9253372815f7b6ad34ae3581f89df75aab6308d6 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 17:49:34 +1100 Subject: [PATCH 054/104] Refine comment in LiveTimer --- nautilus_core/common/src/timer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautilus_core/common/src/timer.rs b/nautilus_core/common/src/timer.rs index a49f0e84e9a9..65080c2f4385 100644 --- a/nautilus_core/common/src/timer.rs +++ b/nautilus_core/common/src/timer.rs @@ -251,7 +251,7 @@ impl LiveTimer { .clone() .expect("No callback for event handler"); - // Setup oneshot channel to be able to cancel timer task + // Setup oneshot channel for cancelling timer task let (cancel_tx, mut cancel_rx) = oneshot::channel(); self.canceler = Some(cancel_tx); From c31e732bdaa108f503bb91d348df73b48571a473 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 18:06:57 +1100 Subject: [PATCH 055/104] Fix LiveTimer to use saturating sub --- nautilus_core/common/src/timer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautilus_core/common/src/timer.rs b/nautilus_core/common/src/timer.rs index 65080c2f4385..3e0a3bbc099b 100644 --- a/nautilus_core/common/src/timer.rs +++ b/nautilus_core/common/src/timer.rs @@ -265,7 +265,7 @@ impl LiveTimer { loop { tokio::select! { - _ = tokio::time::sleep(Duration::from_nanos(next_time_ns - clock.get_time_ns())) => { + _ = tokio::time::sleep(Duration::from_nanos(next_time_ns.saturating_sub(clock.get_time_ns()))) => { Python::with_gil(|py| { // Create new time event let event = TimeEvent::new( From 1d006d6a79f97df0b979c918c3c75b1a270da782 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 18:27:26 +1100 Subject: [PATCH 056/104] Add order book subscriptions managed parameter --- RELEASES.md | 3 +++ nautilus_trader/common/actor.pxd | 6 ++++-- nautilus_trader/common/actor.pyx | 16 ++++++++++++---- nautilus_trader/data/engine.pxd | 2 +- nautilus_trader/data/engine.pyx | 13 ++++++++----- tests/unit_tests/data/test_engine.py | 14 ++++++++++++++ 6 files changed, 42 insertions(+), 12 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 02b36e83c472..f230ac7ace5d 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -3,6 +3,9 @@ Released on TBD (UTC). ### Enhancements +- Added `managed` parameter to `subscribe_order_book_deltas`, default true to retain current behavior (if false then the data engine will not automatically manage a book) +- Added `managed` parameter to `subscribe_order_book_snapshots`, default true to retain current behavior (if false then the data engine will not automatically manage a book) +- Removed `interval_ms` 20 millisecond limitation for `subscribe_order_book_snapshots` (i.e. just needs to be positive), although we recommend you consider subscribing to deltas below 100 milliseconds - Ported `LiveClock` and `LiveTimer` implementations to Rust - Implemented `AverageTrueRange` in Rust, thanks @rsmb7z diff --git a/nautilus_trader/common/actor.pxd b/nautilus_trader/common/actor.pxd index d66b39bb0ff3..78246fb6c3dd 100644 --- a/nautilus_trader/common/actor.pxd +++ b/nautilus_trader/common/actor.pxd @@ -142,7 +142,8 @@ cdef class Actor(Component): BookType book_type=*, int depth=*, dict kwargs=*, - ClientId client_id=* + ClientId client_id=*, + bint managed=*, ) cpdef void subscribe_order_book_snapshots( self, @@ -151,7 +152,8 @@ cdef class Actor(Component): int depth=*, int interval_ms=*, dict kwargs=*, - ClientId client_id=* + ClientId client_id=*, + bint managed=*, ) cpdef void subscribe_quote_ticks(self, InstrumentId instrument_id, ClientId client_id=*) cpdef void subscribe_trade_ticks(self, InstrumentId instrument_id, ClientId client_id=*) diff --git a/nautilus_trader/common/actor.pyx b/nautilus_trader/common/actor.pyx index 2f4a05041d94..7f43f90e9ba0 100644 --- a/nautilus_trader/common/actor.pyx +++ b/nautilus_trader/common/actor.pyx @@ -1184,6 +1184,7 @@ cdef class Actor(Component): int depth = 0, dict kwargs = None, ClientId client_id = None, + bint managed = True, ): """ Subscribe to the order book data stream, being a snapshot then deltas @@ -1202,6 +1203,8 @@ cdef class Actor(Component): client_id : ClientId, optional The specific client ID for the command. If ``None`` then will be inferred from the venue in the instrument ID. + managed : bool, default True + If an order book should be managed by the data engine based on the subscribed feed. """ Condition.not_none(instrument_id, "instrument_id") @@ -1222,6 +1225,7 @@ cdef class Actor(Component): "book_type": book_type, "depth": depth, "kwargs": kwargs, + "managed": managed, }), command_id=UUID4(), ts_init=self._clock.timestamp_ns(), @@ -1237,6 +1241,7 @@ cdef class Actor(Component): int interval_ms = 1000, dict kwargs = None, ClientId client_id = None, + bint managed = True, ): """ Subscribe to `OrderBook` snapshots at a specified interval, for the given instrument ID. @@ -1254,28 +1259,30 @@ cdef class Actor(Component): depth : int, optional The maximum depth for the order book. A depth of 0 is maximum depth. interval_ms : int - The order book snapshot interval in milliseconds (not less than 20 milliseconds). + The order book snapshot interval in milliseconds (must be positive). kwargs : dict, optional The keyword arguments for exchange specific parameters. client_id : ClientId, optional The specific client ID for the command. If ``None`` then will be inferred from the venue in the instrument ID. + managed : bool, default True + If an order book should be managed by the data engine based on the subscribed feed. Raises ------ ValueError If `depth` is negative (< 0). ValueError - If `interval_ms` is less than the minimum of 20. + If `interval_ms` is not positive (> 0). Warnings -------- - Consider subscribing to order book deltas if you need intervals less than 20 milliseconds. + Consider subscribing to order book deltas if you need intervals less than 100 milliseconds. """ Condition.not_none(instrument_id, "instrument_id") Condition.not_negative(depth, "depth") - Condition.true(interval_ms >= 20, f"`interval_ms` {interval_ms} was less than minimum 20") + Condition.positive_int(interval_ms, "interval_ms") Condition.true(self.trader_id is not None, "The actor has not been registered") if book_type == BookType.L1_MBP and depth > 1: @@ -1302,6 +1309,7 @@ cdef class Actor(Component): "depth": depth, "interval_ms": interval_ms, "kwargs": kwargs, + "managed": managed, }), command_id=UUID4(), ts_init=self._clock.timestamp_ns(), diff --git a/nautilus_trader/data/engine.pxd b/nautilus_trader/data/engine.pxd index 3bbc355e1244..6f350633bfd1 100644 --- a/nautilus_trader/data/engine.pxd +++ b/nautilus_trader/data/engine.pxd @@ -117,7 +117,7 @@ cdef class DataEngine(Component): cpdef void _handle_subscribe_instrument(self, MarketDataClient client, InstrumentId instrument_id) cpdef void _handle_subscribe_order_book_deltas(self, MarketDataClient client, InstrumentId instrument_id, dict metadata) # noqa cpdef void _handle_subscribe_order_book_snapshots(self, MarketDataClient client, InstrumentId instrument_id, dict metadata) # noqa - cpdef void _setup_order_book(self, MarketDataClient client, InstrumentId instrument_id, dict metadata, bint only_deltas) # noqa + cpdef void _setup_order_book(self, MarketDataClient client, InstrumentId instrument_id, dict metadata, bint only_deltas, bint managed) # noqa cpdef void _handle_subscribe_quote_ticks(self, MarketDataClient client, InstrumentId instrument_id) cpdef void _handle_subscribe_synthetic_quote_ticks(self, InstrumentId instrument_id) cpdef void _handle_subscribe_trade_ticks(self, MarketDataClient client, InstrumentId instrument_id) diff --git a/nautilus_trader/data/engine.pyx b/nautilus_trader/data/engine.pyx index d01ef633812a..681e698b5999 100644 --- a/nautilus_trader/data/engine.pyx +++ b/nautilus_trader/data/engine.pyx @@ -638,14 +638,14 @@ cdef class DataEngine(Component): client, command.data_type.metadata.get("instrument_id"), ) - elif command.data_type.type == OrderBook: - self._handle_subscribe_order_book_snapshots( + elif command.data_type.type == OrderBookDelta: + self._handle_subscribe_order_book_deltas( client, command.data_type.metadata.get("instrument_id"), command.data_type.metadata, ) - elif command.data_type.type == OrderBookDelta: - self._handle_subscribe_order_book_deltas( + elif command.data_type.type == OrderBook: + self._handle_subscribe_order_book_snapshots( client, command.data_type.metadata.get("instrument_id"), command.data_type.metadata, @@ -757,6 +757,7 @@ cdef class DataEngine(Component): instrument_id, metadata, only_deltas=True, + managed=metadata["managed"] ) cpdef void _handle_subscribe_order_book_snapshots( @@ -803,6 +804,7 @@ cdef class DataEngine(Component): instrument_id, metadata, only_deltas=False, + managed=metadata["managed"] ) cpdef void _setup_order_book( @@ -811,13 +813,14 @@ cdef class DataEngine(Component): InstrumentId instrument_id, dict metadata, bint only_deltas, + bint managed, ): Condition.not_none(client, "client") Condition.not_none(instrument_id, "instrument_id") Condition.not_none(metadata, "metadata") # Create order book - if not self._cache.has_order_book(instrument_id): + if managed and not self._cache.has_order_book(instrument_id): instrument = self._cache.instrument(instrument_id) if instrument is None: self._log.error( diff --git a/tests/unit_tests/data/test_engine.py b/tests/unit_tests/data/test_engine.py index 914cb0ef7a2d..d4068510bbff 100644 --- a/tests/unit_tests/data/test_engine.py +++ b/tests/unit_tests/data/test_engine.py @@ -747,6 +747,7 @@ def test_execute_subscribe_order_book_snapshots_then_adds_handler(self): "book_type": 2, "depth": 10, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -774,6 +775,7 @@ def test_execute_subscribe_order_book_deltas_then_adds_handler(self): "book_type": 2, "depth": 10, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -801,6 +803,7 @@ def test_execute_subscribe_order_book_intervals_then_adds_handler(self): "book_type": 2, "depth": 25, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -828,6 +831,7 @@ def test_execute_unsubscribe_order_book_stream_then_removes_handler(self): "book_type": 2, "depth": 25, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -871,6 +875,7 @@ def test_execute_unsubscribe_order_book_data_then_removes_handler(self): "book_type": 2, "depth": 25, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -914,6 +919,7 @@ def test_execute_unsubscribe_order_book_interval_then_removes_handler(self): "book_type": 2, "depth": 25, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -965,6 +971,7 @@ def test_order_book_snapshots_when_book_not_updated_does_not_send_(self): "book_type": BookType.L2_MBP, "depth": 20, "interval_ms": 1000, # Streaming + "managed": True, }, ), command_id=UUID4(), @@ -1005,6 +1012,7 @@ def test_process_order_book_snapshot_when_one_subscriber_then_sends_to_registere "book_type": BookType.L2_MBP, "depth": 25, "interval_ms": 1000, # Streaming + "managed": True, }, ), command_id=UUID4(), @@ -1043,6 +1051,7 @@ def test_process_order_book_deltas_then_sends_to_registered_handler(self): "instrument_id": ETHUSDT_BINANCE.id, "book_type": BookType.L3_MBO, "depth": 5, + "managed": True, }, ), command_id=UUID4(), @@ -1090,6 +1099,7 @@ def test_process_order_book_snapshots_when_multiple_subscribers_then_sends_to_re "book_type": BookType.L2_MBP, "depth": 25, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -1106,6 +1116,7 @@ def test_process_order_book_snapshots_when_multiple_subscribers_then_sends_to_re "book_type": BookType.L2_MBP, "depth": 25, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -1164,6 +1175,7 @@ def test_process_order_book_depth_when_multiple_subscribers_then_sends_to_regist "book_type": BookType.L2_MBP, "depth": 10, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -1180,6 +1192,7 @@ def test_process_order_book_depth_when_multiple_subscribers_then_sends_to_regist "book_type": BookType.L2_MBP, "depth": 10, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), @@ -1224,6 +1237,7 @@ def test_order_book_delta_creates_book(self): "book_type": 2, "depth": 25, "interval_ms": 1000, + "managed": True, }, ), command_id=UUID4(), From 41e535b986045c4010f5753086ab5ae24afbdb58 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 17 Feb 2024 21:55:56 +1100 Subject: [PATCH 057/104] Resume Windows in CI --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bba01e42a980..85cfefcca5de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: arch: [x64] - os: [ubuntu-latest] # windows-latest + os: [ubuntu-latest, windows-latest] python-version: ["3.10", "3.11", "3.12"] defaults: run: From fec8d7368233e20720d0860c7711a838d4fb0571 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 08:03:10 +1100 Subject: [PATCH 058/104] Pause Windows in CI --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85cfefcca5de..bba01e42a980 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: arch: [x64] - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] # windows-latest python-version: ["3.10", "3.11", "3.12"] defaults: run: From 3dd43860cc577168709376d76c1b303470901470 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 09:10:26 +1100 Subject: [PATCH 059/104] Fix await eventually in cache tests --- tests/integration_tests/infrastructure/test_cache_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/infrastructure/test_cache_database.py b/tests/integration_tests/infrastructure/test_cache_database.py index 30543853f710..d5e6aa7362c2 100644 --- a/tests/integration_tests/infrastructure/test_cache_database.py +++ b/tests/integration_tests/infrastructure/test_cache_database.py @@ -1128,4 +1128,4 @@ async def test_rerunning_backtest_with_redis_db_builds_correct_index(self): await asyncio.sleep(0.5) # Assert - assert eventually(lambda: self.engine.cache.check_integrity()) + await eventually(lambda: self.engine.cache.check_integrity()) From 13ba8d1c0d6da74c5dd41126bedb6bde702e56bf Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 09:11:15 +1100 Subject: [PATCH 060/104] Remove sleeps from interactive_brokers tests --- .../interactive_brokers/client/test_client.py | 9 ++++---- .../client/test_client_account.py | 22 +++++++++++++++---- .../client/test_client_connection.py | 17 ++++++++++++-- .../client/test_client_contract.py | 15 +++++++++++++ .../client/test_client_error.py | 15 +++++++++++++ .../client/test_client_market_data.py | 15 +++++++++++++ .../client/test_client_order.py | 15 +++++++++++++ .../interactive_brokers/client/test_common.py | 15 +++++++++++++ 8 files changed, 112 insertions(+), 11 deletions(-) diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client.py index 842cdd345fa5..ca55104fa357 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client.py @@ -13,7 +13,6 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- - import asyncio from unittest.mock import AsyncMock from unittest.mock import MagicMock @@ -22,6 +21,8 @@ import pytest +from nautilus_trader.test_kit.functions import eventually + def test_start(ib_client): # Arrange, Act @@ -196,10 +197,9 @@ async def test_run_tws_incoming_msg_reader(ib_client): ib_client._tws_incoming_msg_reader_task = ib_client._create_task( ib_client._run_tws_incoming_msg_reader(), ) - await asyncio.sleep(0.1) + await eventually(lambda: ib_client._internal_msg_queue.qsize() == len(test_messages)) # Assert - assert ib_client._internal_msg_queue.qsize() == len(test_messages) for msg in test_messages: assert await ib_client._internal_msg_queue.get() == msg @@ -216,8 +216,7 @@ async def test_run_internal_msg_queue(ib_client): ib_client._internal_msg_queue_task = ib_client._create_task( ib_client._run_internal_msg_queue(), ) - await asyncio.sleep(0.1) # Assert - assert ib_client._process_message.call_count == len(test_messages) + await eventually(lambda: ib_client._process_message.call_count == len(test_messages)) assert ib_client._internal_msg_queue.qsize() == 0 diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client_account.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client_account.py index abd19af08d92..10c500a3f239 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client_account.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client_account.py @@ -1,4 +1,18 @@ -import asyncio +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + from collections import Counter from decimal import Decimal from unittest.mock import AsyncMock @@ -10,6 +24,7 @@ from ibapi import decoder from nautilus_trader.adapters.interactive_brokers.client.common import IBPosition +from nautilus_trader.test_kit.functions import eventually from tests.integration_tests.adapters.interactive_brokers.test_kit import IBTestContractStubs @@ -50,10 +65,9 @@ async def test_process_account_id(ib_client): with patch("ibapi.comm.read_msg", side_effect=[(None, msg, b"") for msg in test_messages]): # Act ib_client._start_client_tasks_and_tws_api() - await asyncio.sleep(0.1) - # Assert - assert "DU1234567" in ib_client.accounts() + # Assert + await eventually(lambda: "DU1234567" in ib_client.accounts()) def test_subscribe_account_summary(ib_client): diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client_connection.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client_connection.py index bbf0aae55cce..20fde18b9fbf 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client_connection.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client_connection.py @@ -1,4 +1,18 @@ -import asyncio +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + from unittest.mock import AsyncMock from unittest.mock import Mock from unittest.mock import patch @@ -44,7 +58,6 @@ async def test_connect_socket(ib_client): # Act await ib_client._connect_socket() - asyncio.sleep(0.1) # Assert mock_connection_instance.connect.assert_called_once() diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client_contract.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client_contract.py index 70e424f43c87..997fe353f404 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client_contract.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client_contract.py @@ -1,3 +1,18 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + from unittest.mock import Mock from unittest.mock import patch diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client_error.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client_error.py index 56b8f5a04333..7949cf2760f7 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client_error.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client_error.py @@ -1,3 +1,18 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + import functools from unittest.mock import Mock diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client_market_data.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client_market_data.py index f7f191a1b868..f757b2b9bb80 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client_market_data.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client_market_data.py @@ -1,3 +1,18 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + import copy import functools from decimal import Decimal diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_client_order.py b/tests/integration_tests/adapters/interactive_brokers/client/test_client_order.py index 208a9d180781..32aa56071cb3 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_client_order.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_client_order.py @@ -1,3 +1,18 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + from collections import Counter from decimal import Decimal from unittest.mock import AsyncMock diff --git a/tests/integration_tests/adapters/interactive_brokers/client/test_common.py b/tests/integration_tests/adapters/interactive_brokers/client/test_common.py index f2f859ff54d0..330ecd019f8c 100644 --- a/tests/integration_tests/adapters/interactive_brokers/client/test_common.py +++ b/tests/integration_tests/adapters/interactive_brokers/client/test_common.py @@ -1,3 +1,18 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + import asyncio from unittest.mock import Mock From 50df82e786357448dc3ecba770a00abd325e7a7b Mon Sep 17 00:00:00 2001 From: Ishan Bhanuka Date: Sun, 18 Feb 2024 11:40:26 +0800 Subject: [PATCH 061/104] Fix logging enabled check (#1503) --- nautilus_core/common/src/logging/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nautilus_core/common/src/logging/mod.rs b/nautilus_core/common/src/logging/mod.rs index 30af1a995389..ede184a6fe6b 100644 --- a/nautilus_core/common/src/logging/mod.rs +++ b/nautilus_core/common/src/logging/mod.rs @@ -361,8 +361,9 @@ impl fmt::Display for LogLine { impl Log for Logger { fn enabled(&self, metadata: &log::Metadata) -> bool { !LOGGING_BYPASSED.load(Ordering::Relaxed) - && (metadata.level() >= self.config.stdout_level - || metadata.level() >= self.config.fileout_level) + && (metadata.level() == Level::Error + || metadata.level() <= self.config.stdout_level + || metadata.level() <= self.config.fileout_level) } fn log(&self, record: &log::Record) { From 2a8011d6c37b330d2dcf912f0df2941b8d7d90a2 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 09:35:20 +1100 Subject: [PATCH 062/104] Optimize balance impact calculation --- nautilus_trader/accounting/accounts/cash.pyx | 14 +++++++------- nautilus_trader/accounting/accounts/margin.pyx | 11 ++++------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/nautilus_trader/accounting/accounts/cash.pyx b/nautilus_trader/accounting/accounts/cash.pyx index 2d77e548af86..78896edd7741 100644 --- a/nautilus_trader/accounting/accounts/cash.pyx +++ b/nautilus_trader/accounting/accounts/cash.pyx @@ -249,7 +249,7 @@ cdef class CashAccount(Account): notional = quantity.as_f64_c() else: return None # No balance to lock - else: + else: # pragma: no cover (design-time error) raise RuntimeError(f"invalid `OrderSide`, was {side}") # pragma: no cover (design-time error) # Add expected commission @@ -264,7 +264,7 @@ cdef class CashAccount(Account): return Money(locked, quote_currency) elif side == OrderSide.SELL: return Money(locked, base_currency) - else: + else: # pragma: no cover (design-time error) raise RuntimeError(f"invalid `OrderSide`, was {side}") # pragma: no cover (design-time error) cpdef list calculate_pnls( @@ -315,7 +315,7 @@ cdef class CashAccount(Account): if base_currency and not self.base_currency: pnls[base_currency] = Money(-fill_qty, base_currency) pnls[quote_currency] = Money(fill_px * fill_qty, quote_currency) - else: + else: # pragma: no cover (design-time error) raise RuntimeError(f"invalid `OrderSide`, was {fill.order_side}") # pragma: no cover (design-time error) return list(pnls.values()) @@ -327,10 +327,10 @@ cdef class CashAccount(Account): Price price, OrderSide order_side, ): - cdef object notional = instrument.notional_value(quantity, price) + cdef Money notional = instrument.notional_value(quantity, price) if order_side == OrderSide.BUY: - return Money(-notional, notional.currency) + return Money.from_raw_c(-notional._mem.raw, notional.currency) elif order_side == OrderSide.SELL: - return Money(notional, notional.currency) - else: + return Money.from_raw_c(notional._mem.raw, notional.currency) + else: # pragma: no cover (design-time error) raise RuntimeError(f"invalid `OrderSide`, was {order_side}") # pragma: no cover (design-time error) diff --git a/nautilus_trader/accounting/accounts/margin.pyx b/nautilus_trader/accounting/accounts/margin.pyx index 7aa9d991bf0d..d42e17f3b99c 100644 --- a/nautilus_trader/accounting/accounts/margin.pyx +++ b/nautilus_trader/accounting/accounts/margin.pyx @@ -669,13 +669,10 @@ cdef class MarginAccount(Account): cdef: object leverage = self.leverage(instrument.id) double margin_impact = 1.0 / leverage - Money raw_money + Money notional = instrument.notional_value(quantity, price) if order_side == OrderSide.BUY: - raw_money = -instrument.notional_value(quantity, price) - return Money(raw_money * margin_impact, raw_money.currency) + return Money(-notional.as_f64_c() * margin_impact, notional.currency) elif order_side == OrderSide.SELL: - raw_money = instrument.notional_value(quantity, price) - return Money(raw_money * margin_impact, raw_money.currency) - - else: + return Money(notional.as_f64_c() * margin_impact, notional.currency) + else: # pragma: no cover (design-time error) raise RuntimeError(f"invalid `OrderSide`, was {order_side}") # pragma: no cover (design-time error) From cdf6f56fa6b94d07d2a2cde80f0dc1d7c1fcab06 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 14:45:31 +1100 Subject: [PATCH 063/104] Update release notes --- RELEASES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index f230ac7ace5d..b2e16ead8d52 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -15,6 +15,8 @@ None ### Fixes - Fixed FOK time in force behavior (allows fills beyond the top level, will cancel if cannot fill full size) - Fixed IOC time in force behavior (allows fills beyond the top level, will cancel any remaining after all fills are applied) +- Fixed `LiveClock` timer behavior for small intervals causing next time to be less than now (timer then would not run) +- Fixed log level filtering for `log_level_file` (bug introduced in v1.187.0), thanks @twitu - Fixed logging `print_config` config option (was not being passed through to the logging system) - Fixed logging timestamps for backtesting (static clock was not being incrementally set to individual `TimeEvent` timestamps) - Fixed account balance updates (fills from zero quantity `NETTING` positions will generate account balance updates) From 213715e72b1d8e03fd6591138a451d193892dce6 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 14:53:58 +1100 Subject: [PATCH 064/104] Use pyo3_asyncio runtime --- nautilus_core/Cargo.lock | 1 + .../adapters/src/databento/python/live.rs | 4 +-- nautilus_core/common/Cargo.toml | 3 ++- nautilus_core/common/src/lib.rs | 1 - nautilus_core/common/src/runtime.rs | 25 ------------------- nautilus_core/common/src/timer.rs | 4 +-- 6 files changed, 7 insertions(+), 31 deletions(-) delete mode 100644 nautilus_core/common/src/runtime.rs diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 0ea11d8f6549..42636e976a1f 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -2484,6 +2484,7 @@ dependencies = [ "nautilus-core", "nautilus-model", "pyo3", + "pyo3-asyncio", "redis", "rstest", "serde", diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index f6dabe67955a..8e5910756256 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -23,7 +23,6 @@ use databento::live::Subscription; use dbn::{PitSymbolMap, Record, SymbolIndex, VersionUpgradePolicy}; use indexmap::IndexMap; use log::{error, info}; -use nautilus_common::runtime::get_runtime; use nautilus_core::python::to_pyruntime_err; use nautilus_core::time::AtomicTime; use nautilus_core::{ @@ -76,7 +75,8 @@ impl DatabentoLiveClient { match &self.inner { Some(client) => Ok(client.clone()), None => { - let client = get_runtime().block_on(self.initialize_client())?; + let rt = pyo3_asyncio::tokio::get_runtime(); + let client = rt.block_on(self.initialize_client())?; self.inner = Some(Arc::new(Mutex::new(client))); Ok(self.inner.clone().unwrap()) } diff --git a/nautilus_core/common/Cargo.toml b/nautilus_core/common/Cargo.toml index f91a4c067b09..76fb5a4507c1 100644 --- a/nautilus_core/common/Cargo.toml +++ b/nautilus_core/common/Cargo.toml @@ -18,6 +18,7 @@ chrono = { workspace = true } indexmap = { workspace = true } log = { workspace = true } pyo3 = { workspace = true, optional = true } +pyo3-asyncio = { workspace = true, optional = true } redis = { workspace = true, optional = true } serde = { workspace = true } serde_json = { workspace = true } @@ -40,7 +41,7 @@ extension-module = [ "nautilus-model/extension-module", ] ffi = ["cbindgen"] -python = ["pyo3"] +python = ["pyo3", "pyo3-asyncio"] stubs = ["rstest"] redis = ["dep:redis"] default = ["ffi", "python", "redis"] diff --git a/nautilus_core/common/src/lib.rs b/nautilus_core/common/src/lib.rs index 4f77acc139a0..536048ffda0d 100644 --- a/nautilus_core/common/src/lib.rs +++ b/nautilus_core/common/src/lib.rs @@ -20,7 +20,6 @@ pub mod generators; pub mod handlers; pub mod logging; pub mod msgbus; -pub mod runtime; pub mod testing; pub mod timer; diff --git a/nautilus_core/common/src/runtime.rs b/nautilus_core/common/src/runtime.rs deleted file mode 100644 index 9ab54e00decc..000000000000 --- a/nautilus_core/common/src/runtime.rs +++ /dev/null @@ -1,25 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. -// https://nautechsystems.io -// -// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -// You may not use this file except in compliance with the License. -// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// ------------------------------------------------------------------------------------------------- - -use std::sync::OnceLock; - -use tokio::runtime::Runtime; - -static RUNTIME: OnceLock = OnceLock::new(); - -pub fn get_runtime() -> &'static tokio::runtime::Runtime { - // Using default configuration values for now - RUNTIME.get_or_init(|| Runtime::new().expect("Failed to create tokio runtime")) -} diff --git a/nautilus_core/common/src/timer.rs b/nautilus_core/common/src/timer.rs index 3e0a3bbc099b..03047c601ad1 100644 --- a/nautilus_core/common/src/timer.rs +++ b/nautilus_core/common/src/timer.rs @@ -28,7 +28,7 @@ use pyo3::{ffi, types::PyCapsule, IntoPy, PyObject, Python}; use tokio::sync::oneshot; use ustr::Ustr; -use crate::{handlers::EventHandler, runtime::get_runtime}; +use crate::handlers::EventHandler; #[repr(C)] #[derive(Clone, Debug)] @@ -255,7 +255,7 @@ impl LiveTimer { let (cancel_tx, mut cancel_rx) = oneshot::channel(); self.canceler = Some(cancel_tx); - get_runtime().spawn(async move { + pyo3_asyncio::tokio::get_runtime().spawn(async move { let clock = get_atomic_clock_realtime(); if start_time_ns == 0 { start_time_ns = clock.get_time_ns(); From 3483a3b04f54bbaa548c385542b28fe449839336 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 15:05:00 +1100 Subject: [PATCH 065/104] Implement OrderBookDeltas pickling --- RELEASES.md | 1 + nautilus_trader/model/data.pyx | 42 +++++++++++++++++++ tests/unit_tests/model/test_orderbook_data.py | 13 ++++++ 3 files changed, 56 insertions(+) diff --git a/RELEASES.md b/RELEASES.md index b2e16ead8d52..f39bfed828e8 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -7,6 +7,7 @@ Released on TBD (UTC). - Added `managed` parameter to `subscribe_order_book_snapshots`, default true to retain current behavior (if false then the data engine will not automatically manage a book) - Removed `interval_ms` 20 millisecond limitation for `subscribe_order_book_snapshots` (i.e. just needs to be positive), although we recommend you consider subscribing to deltas below 100 milliseconds - Ported `LiveClock` and `LiveTimer` implementations to Rust +- Implemented `OrderBookDeltas` pickling - Implemented `AverageTrueRange` in Rust, thanks @rsmb7z ### Breaking Changes diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index b76d9c4c87fa..7290a90f900c 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -2171,6 +2171,48 @@ cdef class OrderBookDeltas(Data): PyMem_Free(cvec.ptr) # De-allocate buffer PyMem_Free(cvec) # De-allocate cvec + def __getstate__(self): + return ( + self.instrument_id.value, + pickle.dumps(self.deltas), + ) + + def __setstate__(self, state): + cdef InstrumentId instrument_id = InstrumentId.from_str_c(state[0]) + + cdef list deltas = pickle.loads(state[1]) + + cdef uint64_t len_ = len(deltas) + + # Create a C OrderBookDeltas_t buffer + cdef OrderBookDelta_t* data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) + if not data: + raise MemoryError() + + cdef uint64_t i + cdef OrderBookDelta delta + for i in range(len_): + delta = deltas[i] + data[i] = delta._mem + + # Create CVec + cdef CVec* cvec = PyMem_Malloc(1 * sizeof(CVec)) + if not cvec: + raise MemoryError() + + cvec.ptr = data + cvec.len = len_ + cvec.cap = len_ + + # Transfer data to Rust + self._mem = orderbook_deltas_new( + instrument_id._mem, + cvec, + ) + + PyMem_Free(cvec.ptr) # De-allocate buffer + PyMem_Free(cvec) # De-allocate cvec + def __del__(self) -> None: if self._mem._0 != NULL: orderbook_deltas_drop(self._mem) diff --git a/tests/unit_tests/model/test_orderbook_data.py b/tests/unit_tests/model/test_orderbook_data.py index a6cbe306f626..65054520ec95 100644 --- a/tests/unit_tests/model/test_orderbook_data.py +++ b/tests/unit_tests/model/test_orderbook_data.py @@ -382,6 +382,19 @@ def test_deltas_fully_qualified_name() -> None: assert OrderBookDeltas.fully_qualified_name() == "nautilus_trader.model.data:OrderBookDeltas" +def test_deltas_pickle_round_trip() -> None: + # Arrange + deltas = TestDataStubs.order_book_deltas() + + # Act + pickled = pickle.dumps(deltas) + unpickled = pickle.loads(pickled) # noqa + + # Assert + assert deltas == unpickled + assert len(deltas.deltas) == len(unpickled.deltas) + + def test_deltas_hash_str_and_repr() -> None: # Arrange order1 = BookOrder( From 21764f9e2e93a72015311946af40672d012759b1 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 16:47:44 +1100 Subject: [PATCH 066/104] Separate OrderBookMbo and OrderBookMbp --- nautilus_core/model/src/ffi/orderbook/book.rs | 15 +- .../model/src/ffi/orderbook/container.rs | 320 ++++++++++ nautilus_core/model/src/ffi/orderbook/mod.rs | 1 + nautilus_core/model/src/orderbook/book.rs | 583 +++--------------- nautilus_core/model/src/orderbook/book_mbo.rs | 253 ++++++++ nautilus_core/model/src/orderbook/book_mbp.rs | 383 ++++++++++++ nautilus_core/model/src/orderbook/display.rs | 71 +++ nautilus_core/model/src/orderbook/mod.rs | 3 + .../model/src/python/orderbook/mod.rs | 14 + nautilus_trader/core/includes/model.h | 7 +- nautilus_trader/core/rust/model.pxd | 5 +- 11 files changed, 1133 insertions(+), 522 deletions(-) create mode 100644 nautilus_core/model/src/ffi/orderbook/container.rs create mode 100644 nautilus_core/model/src/orderbook/book_mbo.rs create mode 100644 nautilus_core/model/src/orderbook/book_mbp.rs create mode 100644 nautilus_core/model/src/orderbook/display.rs diff --git a/nautilus_core/model/src/ffi/orderbook/book.rs b/nautilus_core/model/src/ffi/orderbook/book.rs index 80e4a8af0226..6e8d3e3424d6 100644 --- a/nautilus_core/model/src/ffi/orderbook/book.rs +++ b/nautilus_core/model/src/ffi/orderbook/book.rs @@ -20,7 +20,7 @@ use std::{ use nautilus_core::ffi::{cvec::CVec, string::str_to_cstr}; -use super::level::Level_API; +use super::{container::OrderBookContainer, level::Level_API}; use crate::{ data::{ delta::OrderBookDelta, depth::OrderBookDepth10, order::BookOrder, quote::QuoteTick, @@ -29,7 +29,6 @@ use crate::{ enums::{BookType, OrderSide}, ffi::data::deltas::OrderBookDeltas_API, identifiers::instrument_id::InstrumentId, - orderbook::book::OrderBook, types::{price::Price, quantity::Quantity}, }; @@ -43,10 +42,10 @@ use crate::{ /// having to manually access the underlying `OrderBook` instance. #[repr(C)] #[allow(non_camel_case_types)] -pub struct OrderBook_API(Box); +pub struct OrderBook_API(Box); impl Deref for OrderBook_API { - type Target = OrderBook; + type Target = OrderBookContainer; fn deref(&self) -> &Self::Target { &self.0 @@ -61,7 +60,7 @@ impl DerefMut for OrderBook_API { #[no_mangle] pub extern "C" fn orderbook_new(instrument_id: InstrumentId, book_type: BookType) -> OrderBook_API { - OrderBook_API(Box::new(OrderBook::new(instrument_id, book_type))) + OrderBook_API(Box::new(OrderBookContainer::new(instrument_id, book_type))) } #[no_mangle] @@ -86,17 +85,17 @@ pub extern "C" fn orderbook_book_type(book: &OrderBook_API) -> BookType { #[no_mangle] pub extern "C" fn orderbook_sequence(book: &OrderBook_API) -> u64 { - book.sequence + book.sequence() } #[no_mangle] pub extern "C" fn orderbook_ts_last(book: &OrderBook_API) -> u64 { - book.ts_last + book.ts_last() } #[no_mangle] pub extern "C" fn orderbook_count(book: &OrderBook_API) -> u64 { - book.count + book.count() } #[no_mangle] diff --git a/nautilus_core/model/src/ffi/orderbook/container.rs b/nautilus_core/model/src/ffi/orderbook/container.rs new file mode 100644 index 000000000000..a9f30b18e89f --- /dev/null +++ b/nautilus_core/model/src/ffi/orderbook/container.rs @@ -0,0 +1,320 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use crate::{ + data::{ + delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, order::BookOrder, + quote::QuoteTick, trade::TradeTick, + }, + enums::{BookType, OrderSide}, + identifiers::instrument_id::InstrumentId, + orderbook::{ + book::BookIntegrityError, book_mbo::OrderBookMbo, book_mbp::OrderBookMbp, level::Level, + }, + types::{price::Price, quantity::Quantity}, +}; + +pub struct OrderBookContainer { + pub instrument_id: InstrumentId, + pub book_type: BookType, + mbo: Option, + mbp: Option, +} + +const L3_MBO_NOT_INITILIZED: &str = "L3 MBO book not initialized"; +const L2_MBP_NOT_INITILIZED: &str = "L2 MBP book not initialized"; +const L1_MBP_NOT_INITILIZED: &str = "L1 MBP book not initialized"; + +impl OrderBookContainer { + #[must_use] + pub fn new(instrument_id: InstrumentId, book_type: BookType) -> Self { + let (mbo, mbp) = match book_type { + BookType::L3_MBO => (Some(OrderBookMbo::new(instrument_id)), None), + BookType::L2_MBP => (None, Some(OrderBookMbp::new(instrument_id, false))), + BookType::L1_MBP => (None, Some(OrderBookMbp::new(instrument_id, true))), + }; + + Self { + instrument_id, + book_type, + mbo, + mbp, + } + } + + pub fn instrument_id(&self) -> InstrumentId { + self.instrument_id + } + + pub fn book_type(&self) -> BookType { + self.book_type + } + + pub fn sequence(&self) -> u64 { + match self.book_type { + BookType::L3_MBO => self.mbo.as_ref().expect(L3_MBO_NOT_INITILIZED).sequence, + BookType::L2_MBP => self.mbp.as_ref().expect(L2_MBP_NOT_INITILIZED).sequence, + BookType::L1_MBP => self.mbp.as_ref().expect(L1_MBP_NOT_INITILIZED).sequence, + } + } + + pub fn ts_last(&self) -> u64 { + match self.book_type { + BookType::L3_MBO => self.mbo.as_ref().expect(L3_MBO_NOT_INITILIZED).ts_last, + BookType::L2_MBP => self.mbp.as_ref().expect(L2_MBP_NOT_INITILIZED).ts_last, + BookType::L1_MBP => self.mbp.as_ref().expect(L1_MBP_NOT_INITILIZED).ts_last, + } + } + + pub fn count(&self) -> u64 { + match self.book_type { + BookType::L3_MBO => self.mbo.as_ref().expect(L3_MBO_NOT_INITILIZED).count, + BookType::L2_MBP => self.mbp.as_ref().expect(L2_MBP_NOT_INITILIZED).count, + BookType::L1_MBP => self.mbp.as_ref().expect(L1_MBP_NOT_INITILIZED).count, + } + } + + pub fn reset(&mut self) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().reset(), + BookType::L2_MBP => self.get_mbp_mut().reset(), + BookType::L1_MBP => self.get_mbp_mut().reset(), + }; + } + + pub fn add(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().add(order, ts_event, sequence), + BookType::L2_MBP => self.get_mbp_mut().add(order, ts_event, sequence), + BookType::L1_MBP => panic!("Invalid operation for L1_MBP book: `add`"), + }; + } + + pub fn update(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().update(order, ts_event, sequence), + BookType::L2_MBP => self.get_mbp_mut().update(order, ts_event, sequence), + BookType::L1_MBP => self.get_mbp_mut().update(order, ts_event, sequence), + }; + } + + pub fn update_quote_tick(&mut self, quote: &QuoteTick) { + match self.book_type { + BookType::L3_MBO => panic!("Invalid operation for L3_MBO book: `update_quote_tick`"), + BookType::L2_MBP => self.get_mbp_mut().update_quote_tick(quote), + BookType::L1_MBP => self.get_mbp_mut().update_quote_tick(quote), + }; + } + + pub fn update_trade_tick(&mut self, trade: &TradeTick) { + match self.book_type { + BookType::L3_MBO => panic!("Invalid operation for L3_MBO book: `update_trade_tick`"), + BookType::L2_MBP => self.get_mbp_mut().update_trade_tick(trade), + BookType::L1_MBP => self.get_mbp_mut().update_trade_tick(trade), + }; + } + + pub fn delete(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().delete(order, ts_event, sequence), + BookType::L2_MBP => self.get_mbp_mut().delete(order, ts_event, sequence), + BookType::L1_MBP => self.get_mbp_mut().delete(order, ts_event, sequence), + }; + } + + pub fn clear(&mut self, ts_event: u64, sequence: u64) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().clear(ts_event, sequence), + BookType::L2_MBP => self.get_mbp_mut().clear(ts_event, sequence), + BookType::L1_MBP => self.get_mbp_mut().clear(ts_event, sequence), + }; + } + + pub fn clear_bids(&mut self, ts_event: u64, sequence: u64) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().clear_bids(ts_event, sequence), + BookType::L2_MBP => self.get_mbp_mut().clear_bids(ts_event, sequence), + BookType::L1_MBP => self.get_mbp_mut().clear_bids(ts_event, sequence), + }; + } + + pub fn clear_asks(&mut self, ts_event: u64, sequence: u64) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().clear_asks(ts_event, sequence), + BookType::L2_MBP => self.get_mbp_mut().clear_asks(ts_event, sequence), + BookType::L1_MBP => self.get_mbp_mut().clear_asks(ts_event, sequence), + }; + } + + pub fn apply_delta(&mut self, delta: OrderBookDelta) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().apply_delta(delta), + BookType::L2_MBP => self.get_mbp_mut().apply_delta(delta), + BookType::L1_MBP => self.get_mbp_mut().apply_delta(delta), + }; + } + + pub fn apply_deltas(&mut self, deltas: OrderBookDeltas) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().apply_deltas(deltas), + BookType::L2_MBP => self.get_mbp_mut().apply_deltas(deltas), + BookType::L1_MBP => self.get_mbp_mut().apply_deltas(deltas), + }; + } + + pub fn apply_depth(&mut self, depth: OrderBookDepth10) { + match self.book_type { + BookType::L3_MBO => self.get_mbo_mut().apply_depth(depth), + BookType::L2_MBP => self.get_mbp_mut().apply_depth(depth), + BookType::L1_MBP => panic!("Invalid operation for L1_MBP book: `apply_depth`"), + }; + } + + pub fn bids(&self) -> Vec<&Level> { + match self.book_type { + BookType::L3_MBO => self.get_mbo().bids(), + BookType::L2_MBP => self.get_mbp().bids(), + BookType::L1_MBP => self.get_mbp().bids(), + } + } + + pub fn asks(&self) -> Vec<&Level> { + match self.book_type { + BookType::L3_MBO => self.get_mbo().asks(), + BookType::L2_MBP => self.get_mbp().asks(), + BookType::L1_MBP => self.get_mbp().asks(), + } + } + + pub fn has_bid(&self) -> bool { + match self.book_type { + BookType::L3_MBO => self.get_mbo().has_bid(), + BookType::L2_MBP => self.get_mbp().has_bid(), + BookType::L1_MBP => self.get_mbp().has_bid(), + } + } + + pub fn has_ask(&self) -> bool { + match self.book_type { + BookType::L3_MBO => self.get_mbo().has_ask(), + BookType::L2_MBP => self.get_mbp().has_ask(), + BookType::L1_MBP => self.get_mbp().has_ask(), + } + } + + pub fn best_bid_price(&self) -> Option { + match self.book_type { + BookType::L3_MBO => self.get_mbo().best_bid_price(), + BookType::L2_MBP => self.get_mbp().best_bid_price(), + BookType::L1_MBP => self.get_mbp().best_bid_price(), + } + } + + pub fn best_ask_price(&self) -> Option { + match self.book_type { + BookType::L3_MBO => self.get_mbo().best_ask_price(), + BookType::L2_MBP => self.get_mbp().best_ask_price(), + BookType::L1_MBP => self.get_mbp().best_ask_price(), + } + } + + pub fn best_bid_size(&self) -> Option { + match self.book_type { + BookType::L3_MBO => self.get_mbo().best_bid_size(), + BookType::L2_MBP => self.get_mbp().best_bid_size(), + BookType::L1_MBP => self.get_mbp().best_bid_size(), + } + } + + pub fn best_ask_size(&self) -> Option { + match self.book_type { + BookType::L3_MBO => self.get_mbo().best_ask_size(), + BookType::L2_MBP => self.get_mbp().best_ask_size(), + BookType::L1_MBP => self.get_mbp().best_ask_size(), + } + } + + pub fn spread(&self) -> Option { + match self.book_type { + BookType::L3_MBO => self.get_mbo().spread(), + BookType::L2_MBP => self.get_mbp().spread(), + BookType::L1_MBP => self.get_mbp().spread(), + } + } + + pub fn midpoint(&self) -> Option { + match self.book_type { + BookType::L3_MBO => self.get_mbo().midpoint(), + BookType::L2_MBP => self.get_mbp().midpoint(), + BookType::L1_MBP => self.get_mbp().midpoint(), + } + } + + pub fn get_avg_px_for_quantity(&self, qty: Quantity, order_side: OrderSide) -> f64 { + match self.book_type { + BookType::L3_MBO => self.get_mbo().get_avg_px_for_quantity(qty, order_side), + BookType::L2_MBP => self.get_mbp().get_avg_px_for_quantity(qty, order_side), + BookType::L1_MBP => self.get_mbp().get_avg_px_for_quantity(qty, order_side), + } + } + + pub fn get_quantity_for_price(&self, price: Price, order_side: OrderSide) -> f64 { + match self.book_type { + BookType::L3_MBO => self.get_mbo().get_quantity_for_price(price, order_side), + BookType::L2_MBP => self.get_mbp().get_quantity_for_price(price, order_side), + BookType::L1_MBP => self.get_mbp().get_quantity_for_price(price, order_side), + } + } + + pub fn simulate_fills(&self, order: &BookOrder) -> Vec<(Price, Quantity)> { + match self.book_type { + BookType::L3_MBO => self.get_mbo().simulate_fills(order), + BookType::L2_MBP => self.get_mbp().simulate_fills(order), + BookType::L1_MBP => self.get_mbp().simulate_fills(order), + } + } + + pub fn check_integrity(&self) -> Result<(), BookIntegrityError> { + match self.book_type { + BookType::L3_MBO => self.get_mbo().check_integrity(), + BookType::L2_MBP => self.get_mbp().check_integrity(), + BookType::L1_MBP => self.get_mbp().check_integrity(), + } + } + + pub fn pprint(&self, num_levels: usize) -> String { + match self.book_type { + BookType::L3_MBO => self.get_mbo().pprint(num_levels), + BookType::L2_MBP => self.get_mbp().pprint(num_levels), + BookType::L1_MBP => self.get_mbp().pprint(num_levels), + } + } + + fn get_mbo(&self) -> &OrderBookMbo { + self.mbo.as_ref().expect(L3_MBO_NOT_INITILIZED) + } + + fn get_mbp(&self) -> &OrderBookMbp { + self.mbp.as_ref().expect(L2_MBP_NOT_INITILIZED) + } + + fn get_mbo_mut(&mut self) -> &mut OrderBookMbo { + self.mbo.as_mut().expect(L3_MBO_NOT_INITILIZED) + } + + fn get_mbp_mut(&mut self) -> &mut OrderBookMbp { + self.mbp.as_mut().expect(L2_MBP_NOT_INITILIZED) + } +} diff --git a/nautilus_core/model/src/ffi/orderbook/mod.rs b/nautilus_core/model/src/ffi/orderbook/mod.rs index 6f48823c5966..13807ce39fd1 100644 --- a/nautilus_core/model/src/ffi/orderbook/mod.rs +++ b/nautilus_core/model/src/ffi/orderbook/mod.rs @@ -14,4 +14,5 @@ // ------------------------------------------------------------------------------------------------- pub mod book; +pub mod container; pub mod level; diff --git a/nautilus_core/model/src/orderbook/book.rs b/nautilus_core/model/src/orderbook/book.rs index 5712b83d5bb2..1f4e14b0455c 100644 --- a/nautilus_core/model/src/orderbook/book.rs +++ b/nautilus_core/model/src/orderbook/book.rs @@ -13,19 +13,13 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use nautilus_core::time::UnixNanos; -use tabled::{settings::Style, Table, Tabled}; +use std::collections::BTreeMap; + use thiserror::Error; use super::{ladder::BookPrice, level::Level}; use crate::{ - data::{ - delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, order::BookOrder, - quote::QuoteTick, trade::TradeTick, - }, - enums::{BookAction, BookType, OrderSide}, - identifiers::instrument_id::InstrumentId, - orderbook::ladder::Ladder, + enums::{BookType, OrderSide}, types::{price::Price, quantity::Quantity}, }; @@ -51,484 +45,52 @@ pub enum BookIntegrityError { TooManyLevels(OrderSide, usize), } -#[derive(Tabled)] -struct OrderLevelDisplay { - bids: String, - price: String, - asks: String, -} - -/// Provides an order book which can handle L1/L2/L3 granularity data. -pub struct OrderBook { - bids: Ladder, - asks: Ladder, - pub instrument_id: InstrumentId, - pub book_type: BookType, - pub sequence: u64, - pub ts_last: UnixNanos, - pub count: u64, -} - -impl OrderBook { - #[must_use] - pub fn new(instrument_id: InstrumentId, book_type: BookType) -> Self { - Self { - bids: Ladder::new(OrderSide::Buy), - asks: Ladder::new(OrderSide::Sell), - instrument_id, - book_type, - sequence: 0, - ts_last: 0, - count: 0, - } - } - - pub fn reset(&mut self) { - self.bids.clear(); - self.asks.clear(); - self.sequence = 0; - self.ts_last = 0; - self.count = 0; - } - - pub fn add(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { - let order = match self.book_type { - BookType::L3_MBO => order, // No order pre-processing - BookType::L2_MBP => self.pre_process_order(order), - BookType::L1_MBP => panic!("{}", InvalidBookOperation::Add(self.book_type)), - }; - - match order.side { - OrderSide::Buy => self.bids.add(order), - OrderSide::Sell => self.asks.add(order), - _ => panic!("{}", BookIntegrityError::NoOrderSide), - } - - self.increment(ts_event, sequence); - } - - pub fn update(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { - let order = match self.book_type { - BookType::L3_MBO => order, // No order pre-processing - BookType::L2_MBP => self.pre_process_order(order), - BookType::L1_MBP => { - self.update_l1(order, ts_event, sequence); - self.pre_process_order(order) - } - }; - - match order.side { - OrderSide::Buy => self.bids.update(order), - OrderSide::Sell => self.asks.update(order), - _ => panic!("{}", BookIntegrityError::NoOrderSide), - } - - self.increment(ts_event, sequence); - } - - pub fn delete(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { - let order = match self.book_type { - BookType::L3_MBO => order, // No order pre-processing - BookType::L2_MBP => self.pre_process_order(order), - BookType::L1_MBP => self.pre_process_order(order), - }; - - match order.side { - OrderSide::Buy => self.bids.delete(order, ts_event, sequence), - OrderSide::Sell => self.asks.delete(order, ts_event, sequence), - _ => panic!("{}", BookIntegrityError::NoOrderSide), - } - - self.increment(ts_event, sequence); - } - - pub fn clear(&mut self, ts_event: u64, sequence: u64) { - self.bids.clear(); - self.asks.clear(); - self.increment(ts_event, sequence); - } - - pub fn clear_bids(&mut self, ts_event: u64, sequence: u64) { - self.bids.clear(); - self.increment(ts_event, sequence); - } - - pub fn clear_asks(&mut self, ts_event: u64, sequence: u64) { - self.asks.clear(); - self.increment(ts_event, sequence); - } - - pub fn apply_delta(&mut self, delta: OrderBookDelta) { - match delta.action { - BookAction::Add => self.add(delta.order, delta.ts_event, delta.sequence), - BookAction::Update => self.update(delta.order, delta.ts_event, delta.sequence), - BookAction::Delete => self.delete(delta.order, delta.ts_event, delta.sequence), - BookAction::Clear => self.clear(delta.ts_event, delta.sequence), - } - } - - pub fn apply_deltas(&mut self, deltas: OrderBookDeltas) { - for delta in deltas.deltas { - self.apply_delta(delta) - } - } - - pub fn apply_depth(&mut self, depth: OrderBookDepth10) { - self.bids.clear(); - self.asks.clear(); - - for order in depth.bids { - self.add(order, depth.ts_event, depth.sequence); - } - - for order in depth.asks { - self.add(order, depth.ts_event, depth.sequence); - } - } - - pub fn bids(&self) -> Vec<&Level> { - self.bids.levels.values().collect() - } - - pub fn asks(&self) -> Vec<&Level> { - self.asks.levels.values().collect() - } - - pub fn has_bid(&self) -> bool { - match self.bids.top() { - Some(top) => !top.orders.is_empty(), - None => false, - } - } - - pub fn has_ask(&self) -> bool { - match self.asks.top() { - Some(top) => !top.orders.is_empty(), - None => false, - } - } - - pub fn best_bid_price(&self) -> Option { - self.bids.top().map(|top| top.price.value) - } - - pub fn best_ask_price(&self) -> Option { - self.asks.top().map(|top| top.price.value) - } - - pub fn best_bid_size(&self) -> Option { - match self.bids.top() { - Some(top) => top.first().map(|order| order.size), - None => None, - } - } - - pub fn best_ask_size(&self) -> Option { - match self.asks.top() { - Some(top) => top.first().map(|order| order.size), - None => None, - } - } - - pub fn spread(&self) -> Option { - match (self.best_ask_price(), self.best_bid_price()) { - (Some(ask), Some(bid)) => Some(ask.as_f64() - bid.as_f64()), - _ => None, - } - } - - pub fn midpoint(&self) -> Option { - match (self.best_ask_price(), self.best_bid_price()) { - (Some(ask), Some(bid)) => Some((ask.as_f64() + bid.as_f64()) / 2.0), - _ => None, - } - } - - pub fn get_avg_px_for_quantity(&self, qty: Quantity, order_side: OrderSide) -> f64 { - let levels = match order_side { - OrderSide::Buy => self.asks.levels.iter(), - OrderSide::Sell => self.bids.levels.iter(), - _ => panic!("Invalid `OrderSide` {}", order_side), - }; - let mut cumulative_size_raw = 0u64; - let mut cumulative_value = 0.0; - - for (book_price, level) in levels { - let size_this_level = level.size_raw().min(qty.raw - cumulative_size_raw); - cumulative_size_raw += size_this_level; - cumulative_value += book_price.value.as_f64() * size_this_level as f64; - - if cumulative_size_raw >= qty.raw { - break; - } - } - - if cumulative_size_raw == 0 { - 0.0 - } else { - cumulative_value / cumulative_size_raw as f64 - } - } - - pub fn get_quantity_for_price(&self, price: Price, order_side: OrderSide) -> f64 { - let levels = match order_side { - OrderSide::Buy => self.asks.levels.iter(), - OrderSide::Sell => self.bids.levels.iter(), - _ => panic!("Invalid `OrderSide` {}", order_side), - }; - - let mut matched_size: f64 = 0.0; - - for (book_price, level) in levels { - match order_side { - OrderSide::Buy => { - if book_price.value > price { - break; - } - } - OrderSide::Sell => { - if book_price.value < price { - break; - } - } - _ => panic!("Invalid `OrderSide` {}", order_side), - } - matched_size += level.size(); - } - - matched_size - } - - pub fn update_quote_tick(&mut self, tick: &QuoteTick) { - self.update_bid( - BookOrder::from_quote_tick(tick, OrderSide::Buy), - tick.ts_event, - 0, - ); - self.update_ask( - BookOrder::from_quote_tick(tick, OrderSide::Sell), - tick.ts_event, - 0, - ); - } - - pub fn update_trade_tick(&mut self, tick: &TradeTick) { - self.update_bid( - BookOrder::from_trade_tick(tick, OrderSide::Buy), - tick.ts_event, - 0, - ); - self.update_ask( - BookOrder::from_trade_tick(tick, OrderSide::Sell), - tick.ts_event, - 0, - ); - } - - pub fn simulate_fills(&self, order: &BookOrder) -> Vec<(Price, Quantity)> { - match order.side { - OrderSide::Buy => self.asks.simulate_fills(order), - OrderSide::Sell => self.bids.simulate_fills(order), - _ => panic!("{}", BookIntegrityError::NoOrderSide), - } - } - - /// Return a [`String`] representation of the order book in a human-readable table format. - pub fn pprint(&self, num_levels: usize) -> String { - let ask_levels: Vec<(&BookPrice, &Level)> = - self.asks.levels.iter().take(num_levels).rev().collect(); - let bid_levels: Vec<(&BookPrice, &Level)> = - self.bids.levels.iter().take(num_levels).collect(); - let levels: Vec<(&BookPrice, &Level)> = ask_levels.into_iter().chain(bid_levels).collect(); - - let data: Vec = levels - .iter() - .map(|(book_price, level)| { - let is_bid_level = self.bids.levels.contains_key(book_price); - let is_ask_level = self.asks.levels.contains_key(book_price); - - let bid_sizes: Vec = level - .orders - .iter() - .filter(|_| is_bid_level) - .map(|order| format!("{}", order.1.size)) - .collect(); - - let ask_sizes: Vec = level - .orders - .iter() - .filter(|_| is_ask_level) - .map(|order| format!("{}", order.1.size)) - .collect(); - - OrderLevelDisplay { - bids: if bid_sizes.is_empty() { - String::from("") - } else { - format!("[{}]", bid_sizes.join(", ")) - }, - price: format!("{}", level.price), - asks: if ask_sizes.is_empty() { - String::from("") - } else { - format!("[{}]", ask_sizes.join(", ")) - }, - } - }) - .collect(); - - Table::new(data).with(Style::rounded()).to_string() - } - - pub fn check_integrity(&self) -> Result<(), BookIntegrityError> { - match self.book_type { - BookType::L3_MBO => self.check_integrity_l3(), - BookType::L2_MBP => self.check_integrity_l2(), - BookType::L1_MBP => self.check_integrity_l1(), - } - } - - fn check_integrity_l3(&self) -> Result<(), BookIntegrityError> { - let top_bid_level = self.bids.top(); - let top_ask_level = self.asks.top(); - - if top_bid_level.is_none() || top_ask_level.is_none() { - return Ok(()); - } +pub fn get_avg_px_for_quantity(qty: Quantity, levels: &BTreeMap) -> f64 { + let mut cumulative_size_raw = 0u64; + let mut cumulative_value = 0.0; - // SAFETY: Levels were already checked for None - let best_bid = top_bid_level.unwrap().price; - let best_ask = top_ask_level.unwrap().price; + for (book_price, level) in levels { + let size_this_level = level.size_raw().min(qty.raw - cumulative_size_raw); + cumulative_size_raw += size_this_level; + cumulative_value += book_price.value.as_f64() * size_this_level as f64; - if best_bid >= best_ask { - return Err(BookIntegrityError::OrdersCrossed(best_bid, best_ask)); + if cumulative_size_raw >= qty.raw { + break; } - - Ok(()) } - fn check_integrity_l2(&self) -> Result<(), BookIntegrityError> { - for (_, bid_level) in self.bids.levels.iter() { - let num_orders = bid_level.orders.len(); - if num_orders > 1 { - return Err(BookIntegrityError::TooManyOrders( - OrderSide::Buy, - num_orders, - )); - } - } - - for (_, ask_level) in self.asks.levels.iter() { - let num_orders = ask_level.orders.len(); - if num_orders > 1 { - return Err(BookIntegrityError::TooManyOrders( - OrderSide::Sell, - num_orders, - )); - } - } - - Ok(()) - } - - fn check_integrity_l1(&self) -> Result<(), BookIntegrityError> { - if self.bids.len() > 1 { - return Err(BookIntegrityError::TooManyLevels( - OrderSide::Buy, - self.bids.len(), - )); - } - if self.asks.len() > 1 { - return Err(BookIntegrityError::TooManyLevels( - OrderSide::Sell, - self.asks.len(), - )); - } - - Ok(()) + if cumulative_size_raw == 0 { + 0.0 + } else { + cumulative_value / cumulative_size_raw as f64 } +} - fn increment(&mut self, ts_event: u64, sequence: u64) { - self.ts_last = ts_event; - self.sequence = sequence; - self.count += 1; - } +pub fn get_quantity_for_price( + price: Price, + order_side: OrderSide, + levels: &BTreeMap, +) -> f64 { + let mut matched_size: f64 = 0.0; - fn update_l1(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { - // Because of the way we typically get updates from a L1_MBP order book (bid - // and ask updates at the same time), its quite probable that the last - // bid is now the ask price we are trying to insert (or vice versa). We - // just need to add some extra protection against this if we aren't calling - // `check_integrity()` on each individual update. - match order.side { + for (book_price, level) in levels { + match order_side { OrderSide::Buy => { - if let Some(best_ask_price) = self.best_ask_price() { - if order.price > best_ask_price { - self.clear_bids(ts_event, sequence); - } + if book_price.value > price { + break; } } OrderSide::Sell => { - if let Some(best_bid_price) = self.best_bid_price() { - if order.price < best_bid_price { - self.clear_asks(ts_event, sequence); - } - } - } - _ => panic!("{}", BookIntegrityError::NoOrderSide), - } - } - - fn update_bid(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { - match self.bids.top() { - Some(top_bids) => match top_bids.first() { - Some(top_bid) => { - let order_id = top_bid.order_id; - self.bids.remove(order_id, ts_event, sequence); - self.bids.add(order); - } - None => { - self.bids.add(order); - } - }, - None => { - self.bids.add(order); - } - } - } - - fn update_ask(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { - match self.asks.top() { - Some(top_asks) => match top_asks.first() { - Some(top_ask) => { - let order_id = top_ask.order_id; - self.asks.remove(order_id, ts_event, sequence); - self.asks.add(order); + if book_price.value < price { + break; } - None => { - self.asks.add(order); - } - }, - None => { - self.asks.add(order); } + _ => panic!("Invalid `OrderSide` {}", order_side), } + matched_size += level.size(); } - fn pre_process_order(&self, mut order: BookOrder) -> BookOrder { - match self.book_type { - // Because a L1_MBP only has one level per side, we replace the - // `order.order_id` with the enum value of the side, which will let us easily process - // the order. - BookType::L1_MBP => order.order_id = order.side as u64, - // Because a L2_MBP only has one order per level, we replace the - // `order.order_id` with a raw price value, which will let us easily process the order. - BookType::L2_MBP => order.order_id = order.price.raw as u64, - BookType::L3_MBO => panic!("{}", InvalidBookOperation::PreProcessOrder(self.book_type)), - } - - order - } + matched_size } //////////////////////////////////////////////////////////////////////////////// @@ -538,26 +100,25 @@ impl OrderBook { mod tests { use rstest::rstest; - use super::*; use crate::{ - data::{depth::stubs::stub_depth10, order::BookOrder}, + data::{ + depth::{stubs::stub_depth10, OrderBookDepth10}, + order::BookOrder, + quote::QuoteTick, + trade::TradeTick, + }, enums::{AggressorSide, OrderSide}, identifiers::{instrument_id::InstrumentId, trade_id::TradeId}, + orderbook::{book_mbo::OrderBookMbo, book_mbp::OrderBookMbp}, types::{price::Price, quantity::Quantity}, }; - fn create_stub_book(book_type: BookType) -> OrderBook { - let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - OrderBook::new(instrument_id, book_type) - } - #[rstest] fn test_orderbook_creation() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let book = OrderBook::new(instrument_id, BookType::L2_MBP); + let book = OrderBookMbp::new(instrument_id, false); assert_eq!(book.instrument_id, instrument_id); - assert_eq!(book.book_type, BookType::L2_MBP); assert_eq!(book.sequence, 0); assert_eq!(book.ts_last, 0); assert_eq!(book.count, 0); @@ -565,7 +126,8 @@ mod tests { #[rstest] fn test_orderbook_reset() { - let mut book = create_stub_book(BookType::L2_MBP); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let mut book = OrderBookMbp::new(instrument_id, false); book.sequence = 10; book.ts_last = 100; book.count = 3; @@ -579,7 +141,8 @@ mod tests { #[rstest] fn test_best_bid_and_ask_when_nothing_in_book() { - let book = create_stub_book(BookType::L2_MBP); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let book = OrderBookMbp::new(instrument_id, false); assert_eq!(book.best_bid_price(), None); assert_eq!(book.best_ask_price(), None); @@ -591,7 +154,8 @@ mod tests { #[rstest] fn test_bid_side_with_one_order() { - let mut book = create_stub_book(BookType::L3_MBO); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let mut book = OrderBookMbo::new(instrument_id); let order1 = BookOrder::new( OrderSide::Buy, Price::from("1.000"), @@ -607,7 +171,8 @@ mod tests { #[rstest] fn test_ask_side_with_one_order() { - let mut book = create_stub_book(BookType::L3_MBO); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let mut book = OrderBookMbo::new(instrument_id); let order = BookOrder::new( OrderSide::Sell, Price::from("2.000"), @@ -622,13 +187,15 @@ mod tests { } #[rstest] fn test_spread_with_no_bids_or_asks() { - let book = create_stub_book(BookType::L3_MBO); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let book = OrderBookMbo::new(instrument_id); assert_eq!(book.spread(), None); } #[rstest] fn test_spread_with_bids_and_asks() { - let mut book = create_stub_book(BookType::L2_MBP); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let mut book = OrderBookMbo::new(instrument_id); let bid1 = BookOrder::new( OrderSide::Buy, Price::from("1.000"), @@ -649,14 +216,15 @@ mod tests { #[rstest] fn test_midpoint_with_no_bids_or_asks() { - let book = create_stub_book(BookType::L2_MBP); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let book = OrderBookMbp::new(instrument_id, false); assert_eq!(book.midpoint(), None); } #[rstest] fn test_midpoint_with_bids_asks() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBook::new(instrument_id, BookType::L2_MBP); + let mut book = OrderBookMbp::new(instrument_id, false); let bid1 = BookOrder::new( OrderSide::Buy, @@ -678,7 +246,9 @@ mod tests { #[rstest] fn test_get_price_for_quantity_no_market() { - let book = create_stub_book(BookType::L2_MBP); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let book = OrderBookMbp::new(instrument_id, false); + let qty = Quantity::from(1); assert_eq!(book.get_avg_px_for_quantity(qty, OrderSide::Buy), 0.0); @@ -687,7 +257,9 @@ mod tests { #[rstest] fn test_get_quantity_for_price_no_market() { - let book = create_stub_book(BookType::L2_MBP); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let book = OrderBookMbp::new(instrument_id, false); + let price = Price::from("1.0"); assert_eq!(book.get_quantity_for_price(price, OrderSide::Buy), 0.0); @@ -697,7 +269,7 @@ mod tests { #[rstest] fn test_get_price_for_quantity() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBook::new(instrument_id, BookType::L2_MBP); + let mut book = OrderBookMbp::new(instrument_id, false); let ask2 = BookOrder::new( OrderSide::Sell, @@ -743,7 +315,7 @@ mod tests { #[rstest] fn test_get_quantity_for_price() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBook::new(instrument_id, BookType::L2_MBP); + let mut book = OrderBookMbp::new(instrument_id, false); let ask3 = BookOrder::new( OrderSide::Sell, @@ -802,7 +374,7 @@ mod tests { fn test_apply_depth(stub_depth10: OrderBookDepth10) { let depth = stub_depth10; let instrument_id = InstrumentId::from("AAPL.XNAS"); - let mut book = OrderBook::new(instrument_id, BookType::L2_MBP); + let mut book = OrderBookMbp::new(instrument_id, false); book.apply_depth(depth); @@ -815,8 +387,8 @@ mod tests { #[rstest] fn test_update_quote_tick_l1() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBook::new(instrument_id, BookType::L1_MBP); - let tick = QuoteTick::new( + let mut book = OrderBookMbp::new(instrument_id, true); + let quote = QuoteTick::new( InstrumentId::from("ETHUSDT-PERP.BINANCE"), Price::from("5000.000"), Price::from("5100.000"), @@ -827,25 +399,22 @@ mod tests { ) .unwrap(); - book.update_quote_tick(&tick); + book.update_quote_tick("e); - // Check if the top bid order in order_book is the same as the one created from tick - let top_bid_order = book.bids.top().unwrap().first().unwrap(); - let top_ask_order = book.asks.top().unwrap().first().unwrap(); - let expected_bid_order = BookOrder::from_quote_tick(&tick, OrderSide::Buy); - let expected_ask_order = BookOrder::from_quote_tick(&tick, OrderSide::Sell); - assert_eq!(*top_bid_order, expected_bid_order); - assert_eq!(*top_ask_order, expected_ask_order); + assert_eq!(book.best_bid_price().unwrap(), quote.bid_price); + assert_eq!(book.best_ask_price().unwrap(), quote.ask_price); + assert_eq!(book.best_bid_size().unwrap(), quote.bid_size); + assert_eq!(book.best_ask_size().unwrap(), quote.ask_size); } #[rstest] fn test_update_trade_tick_l1() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBook::new(instrument_id, BookType::L1_MBP); + let mut book = OrderBookMbp::new(instrument_id, true); let price = Price::from("15000.000"); let size = Quantity::from("10.00000000"); - let trade_tick = TradeTick::new( + let trade = TradeTick::new( instrument_id, price, size, @@ -855,7 +424,7 @@ mod tests { 0, ); - book.update_trade_tick(&trade_tick); + book.update_trade_tick(&trade); assert_eq!(book.best_bid_price().unwrap(), price); assert_eq!(book.best_ask_price().unwrap(), price); @@ -865,7 +434,9 @@ mod tests { #[rstest] fn test_pprint() { - let mut book = create_stub_book(BookType::L3_MBO); + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let mut book = OrderBookMbo::new(instrument_id); + let order1 = BookOrder::new( OrderSide::Buy, Price::from("1.000"), diff --git a/nautilus_core/model/src/orderbook/book_mbo.rs b/nautilus_core/model/src/orderbook/book_mbo.rs new file mode 100644 index 000000000000..42899315fa2c --- /dev/null +++ b/nautilus_core/model/src/orderbook/book_mbo.rs @@ -0,0 +1,253 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use nautilus_core::time::UnixNanos; + +use super::{ + book::{get_avg_px_for_quantity, get_quantity_for_price}, + display::pprint_book, + level::Level, +}; +use crate::{ + data::{ + delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, order::BookOrder, + }, + enums::{BookAction, OrderSide}, + identifiers::instrument_id::InstrumentId, + orderbook::{book::BookIntegrityError, ladder::Ladder}, + types::{price::Price, quantity::Quantity}, +}; + +/// Provides an order book which can handle MBO/L3 granularity data. +pub struct OrderBookMbo { + pub instrument_id: InstrumentId, + pub sequence: u64, + pub ts_last: UnixNanos, + pub count: u64, + bids: Ladder, + asks: Ladder, +} + +impl OrderBookMbo { + #[must_use] + pub fn new(instrument_id: InstrumentId) -> Self { + Self { + instrument_id, + sequence: 0, + ts_last: 0, + count: 0, + bids: Ladder::new(OrderSide::Buy), + asks: Ladder::new(OrderSide::Sell), + } + } + + pub fn reset(&mut self) { + self.bids.clear(); + self.asks.clear(); + self.sequence = 0; + self.ts_last = 0; + self.count = 0; + } + + pub fn add(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match order.side { + OrderSide::Buy => self.bids.add(order), + OrderSide::Sell => self.asks.add(order), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + + self.increment(ts_event, sequence); + } + + pub fn update(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match order.side { + OrderSide::Buy => self.bids.update(order), + OrderSide::Sell => self.asks.update(order), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + + self.increment(ts_event, sequence); + } + + pub fn delete(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match order.side { + OrderSide::Buy => self.bids.delete(order, ts_event, sequence), + OrderSide::Sell => self.asks.delete(order, ts_event, sequence), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + + self.increment(ts_event, sequence); + } + + pub fn clear(&mut self, ts_event: u64, sequence: u64) { + self.bids.clear(); + self.asks.clear(); + self.increment(ts_event, sequence); + } + + pub fn clear_bids(&mut self, ts_event: u64, sequence: u64) { + self.bids.clear(); + self.increment(ts_event, sequence); + } + + pub fn clear_asks(&mut self, ts_event: u64, sequence: u64) { + self.asks.clear(); + self.increment(ts_event, sequence); + } + + pub fn apply_delta(&mut self, delta: OrderBookDelta) { + match delta.action { + BookAction::Add => self.add(delta.order, delta.ts_event, delta.sequence), + BookAction::Update => self.update(delta.order, delta.ts_event, delta.sequence), + BookAction::Delete => self.delete(delta.order, delta.ts_event, delta.sequence), + BookAction::Clear => self.clear(delta.ts_event, delta.sequence), + } + } + + pub fn apply_deltas(&mut self, deltas: OrderBookDeltas) { + for delta in deltas.deltas { + self.apply_delta(delta) + } + } + + pub fn apply_depth(&mut self, depth: OrderBookDepth10) { + self.bids.clear(); + self.asks.clear(); + + for order in depth.bids { + self.add(order, depth.ts_event, depth.sequence); + } + + for order in depth.asks { + self.add(order, depth.ts_event, depth.sequence); + } + } + + pub fn bids(&self) -> Vec<&Level> { + self.bids.levels.values().collect() + } + + pub fn asks(&self) -> Vec<&Level> { + self.asks.levels.values().collect() + } + + pub fn has_bid(&self) -> bool { + match self.bids.top() { + Some(top) => !top.orders.is_empty(), + None => false, + } + } + + pub fn has_ask(&self) -> bool { + match self.asks.top() { + Some(top) => !top.orders.is_empty(), + None => false, + } + } + + pub fn best_bid_price(&self) -> Option { + self.bids.top().map(|top| top.price.value) + } + + pub fn best_ask_price(&self) -> Option { + self.asks.top().map(|top| top.price.value) + } + + pub fn best_bid_size(&self) -> Option { + match self.bids.top() { + Some(top) => top.first().map(|order| order.size), + None => None, + } + } + + pub fn best_ask_size(&self) -> Option { + match self.asks.top() { + Some(top) => top.first().map(|order| order.size), + None => None, + } + } + + pub fn spread(&self) -> Option { + match (self.best_ask_price(), self.best_bid_price()) { + (Some(ask), Some(bid)) => Some(ask.as_f64() - bid.as_f64()), + _ => None, + } + } + + pub fn midpoint(&self) -> Option { + match (self.best_ask_price(), self.best_bid_price()) { + (Some(ask), Some(bid)) => Some((ask.as_f64() + bid.as_f64()) / 2.0), + _ => None, + } + } + + pub fn get_avg_px_for_quantity(&self, qty: Quantity, order_side: OrderSide) -> f64 { + let levels = match order_side { + OrderSide::Buy => &self.asks.levels, + OrderSide::Sell => &self.bids.levels, + _ => panic!("Invalid `OrderSide` {}", order_side), + }; + + get_avg_px_for_quantity(qty, levels) + } + + pub fn get_quantity_for_price(&self, price: Price, order_side: OrderSide) -> f64 { + let levels = match order_side { + OrderSide::Buy => &self.asks.levels, + OrderSide::Sell => &self.bids.levels, + _ => panic!("Invalid `OrderSide` {}", order_side), + }; + + get_quantity_for_price(price, order_side, levels) + } + + pub fn simulate_fills(&self, order: &BookOrder) -> Vec<(Price, Quantity)> { + match order.side { + OrderSide::Buy => self.asks.simulate_fills(order), + OrderSide::Sell => self.bids.simulate_fills(order), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + } + + /// Return a [`String`] representation of the order book in a human-readable table format. + pub fn pprint(&self, num_levels: usize) -> String { + pprint_book(&self.bids, &self.asks, num_levels) + } + + pub fn check_integrity(&self) -> Result<(), BookIntegrityError> { + let top_bid_level = self.bids.top(); + let top_ask_level = self.asks.top(); + + if top_bid_level.is_none() || top_ask_level.is_none() { + return Ok(()); + } + + // SAFETY: Levels were already checked for None + let best_bid = top_bid_level.unwrap().price; + let best_ask = top_ask_level.unwrap().price; + + if best_bid >= best_ask { + return Err(BookIntegrityError::OrdersCrossed(best_bid, best_ask)); + } + + Ok(()) + } + + fn increment(&mut self, ts_event: u64, sequence: u64) { + self.ts_last = ts_event; + self.sequence = sequence; + self.count += 1; + } +} diff --git a/nautilus_core/model/src/orderbook/book_mbp.rs b/nautilus_core/model/src/orderbook/book_mbp.rs new file mode 100644 index 000000000000..15ac991d7902 --- /dev/null +++ b/nautilus_core/model/src/orderbook/book_mbp.rs @@ -0,0 +1,383 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use nautilus_core::time::UnixNanos; + +use super::{ + book::{get_avg_px_for_quantity, get_quantity_for_price}, + display::pprint_book, + level::Level, +}; +use crate::{ + data::{ + delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, order::BookOrder, + quote::QuoteTick, trade::TradeTick, + }, + enums::{BookAction, OrderSide}, + identifiers::instrument_id::InstrumentId, + orderbook::{book::BookIntegrityError, ladder::Ladder}, + types::{price::Price, quantity::Quantity}, +}; + +/// Provides an order book which can handle MBP/L2 or L1 (top only) granularity data. +pub struct OrderBookMbp { + pub instrument_id: InstrumentId, + pub top_only: bool, + pub sequence: u64, + pub ts_last: UnixNanos, + pub count: u64, + bids: Ladder, + asks: Ladder, +} + +impl OrderBookMbp { + #[must_use] + pub fn new(instrument_id: InstrumentId, top_only: bool) -> Self { + Self { + instrument_id, + top_only, + sequence: 0, + ts_last: 0, + count: 0, + bids: Ladder::new(OrderSide::Buy), + asks: Ladder::new(OrderSide::Sell), + } + } + + pub fn reset(&mut self) { + self.bids.clear(); + self.asks.clear(); + self.sequence = 0; + self.ts_last = 0; + self.count = 0; + } + + pub fn add(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + let order = self.pre_process_order(order); + + match order.side { + OrderSide::Buy => self.bids.add(order), + OrderSide::Sell => self.asks.add(order), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + + self.increment(ts_event, sequence); + } + + pub fn update(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + if self.top_only { + self.update_top(order, ts_event, sequence); + } + let order = self.pre_process_order(order); + + match order.side { + OrderSide::Buy => self.bids.update(order), + OrderSide::Sell => self.asks.update(order), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + + self.increment(ts_event, sequence); + } + + pub fn update_quote_tick(&mut self, quote: &QuoteTick) { + self.update_bid( + BookOrder::from_quote_tick(quote, OrderSide::Buy), + quote.ts_event, + 0, + ); + self.update_ask( + BookOrder::from_quote_tick(quote, OrderSide::Sell), + quote.ts_event, + 0, + ); + } + + pub fn update_trade_tick(&mut self, trade: &TradeTick) { + self.update_bid( + BookOrder::from_trade_tick(trade, OrderSide::Buy), + trade.ts_event, + 0, + ); + self.update_ask( + BookOrder::from_trade_tick(trade, OrderSide::Sell), + trade.ts_event, + 0, + ); + } + + pub fn delete(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + let order = self.pre_process_order(order); + + match order.side { + OrderSide::Buy => self.bids.delete(order, ts_event, sequence), + OrderSide::Sell => self.asks.delete(order, ts_event, sequence), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + + self.increment(ts_event, sequence); + } + + pub fn clear(&mut self, ts_event: u64, sequence: u64) { + self.bids.clear(); + self.asks.clear(); + self.increment(ts_event, sequence); + } + + pub fn clear_bids(&mut self, ts_event: u64, sequence: u64) { + self.bids.clear(); + self.increment(ts_event, sequence); + } + + pub fn clear_asks(&mut self, ts_event: u64, sequence: u64) { + self.asks.clear(); + self.increment(ts_event, sequence); + } + + pub fn apply_delta(&mut self, delta: OrderBookDelta) { + match delta.action { + BookAction::Add => self.add(delta.order, delta.ts_event, delta.sequence), + BookAction::Update => self.update(delta.order, delta.ts_event, delta.sequence), + BookAction::Delete => self.delete(delta.order, delta.ts_event, delta.sequence), + BookAction::Clear => self.clear(delta.ts_event, delta.sequence), + } + } + + pub fn apply_deltas(&mut self, deltas: OrderBookDeltas) { + for delta in deltas.deltas { + self.apply_delta(delta) + } + } + + pub fn apply_depth(&mut self, depth: OrderBookDepth10) { + self.bids.clear(); + self.asks.clear(); + + for order in depth.bids { + self.add(order, depth.ts_event, depth.sequence); + } + + for order in depth.asks { + self.add(order, depth.ts_event, depth.sequence); + } + } + + pub fn bids(&self) -> Vec<&Level> { + self.bids.levels.values().collect() + } + + pub fn asks(&self) -> Vec<&Level> { + self.asks.levels.values().collect() + } + + pub fn has_bid(&self) -> bool { + match self.bids.top() { + Some(top) => !top.orders.is_empty(), + None => false, + } + } + + pub fn has_ask(&self) -> bool { + match self.asks.top() { + Some(top) => !top.orders.is_empty(), + None => false, + } + } + + pub fn best_bid_price(&self) -> Option { + self.bids.top().map(|top| top.price.value) + } + + pub fn best_ask_price(&self) -> Option { + self.asks.top().map(|top| top.price.value) + } + + pub fn best_bid_size(&self) -> Option { + match self.bids.top() { + Some(top) => top.first().map(|order| order.size), + None => None, + } + } + + pub fn best_ask_size(&self) -> Option { + match self.asks.top() { + Some(top) => top.first().map(|order| order.size), + None => None, + } + } + + pub fn spread(&self) -> Option { + match (self.best_ask_price(), self.best_bid_price()) { + (Some(ask), Some(bid)) => Some(ask.as_f64() - bid.as_f64()), + _ => None, + } + } + + pub fn midpoint(&self) -> Option { + match (self.best_ask_price(), self.best_bid_price()) { + (Some(ask), Some(bid)) => Some((ask.as_f64() + bid.as_f64()) / 2.0), + _ => None, + } + } + + pub fn get_avg_px_for_quantity(&self, qty: Quantity, order_side: OrderSide) -> f64 { + let levels = match order_side { + OrderSide::Buy => &self.asks.levels, + OrderSide::Sell => &self.bids.levels, + _ => panic!("Invalid `OrderSide` {}", order_side), + }; + + get_avg_px_for_quantity(qty, levels) + } + + pub fn get_quantity_for_price(&self, price: Price, order_side: OrderSide) -> f64 { + let levels = match order_side { + OrderSide::Buy => &self.asks.levels, + OrderSide::Sell => &self.bids.levels, + _ => panic!("Invalid `OrderSide` {}", order_side), + }; + + get_quantity_for_price(price, order_side, levels) + } + + pub fn simulate_fills(&self, order: &BookOrder) -> Vec<(Price, Quantity)> { + match order.side { + OrderSide::Buy => self.asks.simulate_fills(order), + OrderSide::Sell => self.bids.simulate_fills(order), + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + } + + /// Return a [`String`] representation of the order book in a human-readable table format. + pub fn pprint(&self, num_levels: usize) -> String { + pprint_book(&self.bids, &self.asks, num_levels) + } + + pub fn check_integrity(&self) -> Result<(), BookIntegrityError> { + match self.top_only { + true => { + if self.bids.len() > 1 { + return Err(BookIntegrityError::TooManyLevels( + OrderSide::Buy, + self.bids.len(), + )); + } + if self.asks.len() > 1 { + return Err(BookIntegrityError::TooManyLevels( + OrderSide::Sell, + self.asks.len(), + )); + } + } + false => { + for (_, bid_level) in self.bids.levels.iter() { + let num_orders = bid_level.orders.len(); + if num_orders > 1 { + return Err(BookIntegrityError::TooManyOrders( + OrderSide::Buy, + num_orders, + )); + } + } + + for (_, ask_level) in self.asks.levels.iter() { + let num_orders = ask_level.orders.len(); + if num_orders > 1 { + return Err(BookIntegrityError::TooManyOrders( + OrderSide::Sell, + num_orders, + )); + } + } + } + } + + Ok(()) + } + + fn increment(&mut self, ts_event: u64, sequence: u64) { + self.ts_last = ts_event; + self.sequence = sequence; + self.count += 1; + } + + fn update_bid(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match self.bids.top() { + Some(top_bids) => match top_bids.first() { + Some(top_bid) => { + let order_id = top_bid.order_id; + self.bids.remove(order_id, ts_event, sequence); + self.bids.add(order); + } + None => { + self.bids.add(order); + } + }, + None => { + self.bids.add(order); + } + } + } + + fn update_ask(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + match self.asks.top() { + Some(top_asks) => match top_asks.first() { + Some(top_ask) => { + let order_id = top_ask.order_id; + self.asks.remove(order_id, ts_event, sequence); + self.asks.add(order); + } + None => { + self.asks.add(order); + } + }, + None => { + self.asks.add(order); + } + } + } + + fn update_top(&mut self, order: BookOrder, ts_event: u64, sequence: u64) { + // Because of the way we typically get updates from a L1_MBP order book (bid + // and ask updates at the same time), its quite probable that the last + // bid is now the ask price we are trying to insert (or vice versa). We + // just need to add some extra protection against this if we aren't calling + // `check_integrity()` on each individual update. + match order.side { + OrderSide::Buy => { + if let Some(best_ask_price) = self.best_ask_price() { + if order.price > best_ask_price { + self.clear_bids(ts_event, sequence); + } + } + } + OrderSide::Sell => { + if let Some(best_bid_price) = self.best_bid_price() { + if order.price < best_bid_price { + self.clear_asks(ts_event, sequence); + } + } + } + _ => panic!("{}", BookIntegrityError::NoOrderSide), + } + } + + fn pre_process_order(&self, mut order: BookOrder) -> BookOrder { + match self.top_only { + true => order.order_id = order.side as u64, + false => order.order_id = order.price.raw as u64, + }; + order + } +} diff --git a/nautilus_core/model/src/orderbook/display.rs b/nautilus_core/model/src/orderbook/display.rs new file mode 100644 index 000000000000..8dab52caa520 --- /dev/null +++ b/nautilus_core/model/src/orderbook/display.rs @@ -0,0 +1,71 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use tabled::{settings::Style, Table, Tabled}; + +use super::{ladder::BookPrice, level::Level}; +use crate::orderbook::ladder::Ladder; + +#[derive(Tabled)] +struct OrderLevelDisplay { + bids: String, + price: String, + asks: String, +} + +/// Return a [`String`] representation of the order book in a human-readable table format. +pub fn pprint_book(bids: &Ladder, asks: &Ladder, num_levels: usize) -> String { + let ask_levels: Vec<(&BookPrice, &Level)> = asks.levels.iter().take(num_levels).rev().collect(); + let bid_levels: Vec<(&BookPrice, &Level)> = bids.levels.iter().take(num_levels).collect(); + let levels: Vec<(&BookPrice, &Level)> = ask_levels.into_iter().chain(bid_levels).collect(); + + let data: Vec = levels + .iter() + .map(|(book_price, level)| { + let is_bid_level = bids.levels.contains_key(book_price); + let is_ask_level = asks.levels.contains_key(book_price); + + let bid_sizes: Vec = level + .orders + .iter() + .filter(|_| is_bid_level) + .map(|order| format!("{}", order.1.size)) + .collect(); + + let ask_sizes: Vec = level + .orders + .iter() + .filter(|_| is_ask_level) + .map(|order| format!("{}", order.1.size)) + .collect(); + + OrderLevelDisplay { + bids: if bid_sizes.is_empty() { + String::from("") + } else { + format!("[{}]", bid_sizes.join(", ")) + }, + price: format!("{}", level.price), + asks: if ask_sizes.is_empty() { + String::from("") + } else { + format!("[{}]", ask_sizes.join(", ")) + }, + } + }) + .collect(); + + Table::new(data).with(Style::rounded()).to_string() +} diff --git a/nautilus_core/model/src/orderbook/mod.rs b/nautilus_core/model/src/orderbook/mod.rs index 4a003c664161..4f2d5a4b81d7 100644 --- a/nautilus_core/model/src/orderbook/mod.rs +++ b/nautilus_core/model/src/orderbook/mod.rs @@ -14,5 +14,8 @@ // ------------------------------------------------------------------------------------------------- pub mod book; +pub mod book_mbo; +pub mod book_mbp; +pub mod display; pub mod ladder; pub mod level; diff --git a/nautilus_core/model/src/python/orderbook/mod.rs b/nautilus_core/model/src/python/orderbook/mod.rs index e69de29bb2d1..97d459d8d1e8 100644 --- a/nautilus_core/model/src/python/orderbook/mod.rs +++ b/nautilus_core/model/src/python/orderbook/mod.rs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index 313f6e19abee..8f122afa8296 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -660,10 +660,7 @@ typedef enum TriggerType { typedef struct Level Level; -/** - * Provides an order book which can handle L1/L2/L3 granularity data. - */ -typedef struct OrderBook OrderBook; +typedef struct OrderBookContainer OrderBookContainer; /** * Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. @@ -1254,7 +1251,7 @@ typedef struct SyntheticInstrument_API { * having to manually access the underlying `OrderBook` instance. */ typedef struct OrderBook_API { - struct OrderBook *_0; + struct OrderBookContainer *_0; } OrderBook_API; /** diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index de4d50ba5c78..c47da44ee8ba 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -355,8 +355,7 @@ cdef extern from "../includes/model.h": cdef struct Level: pass - # Provides an order book which can handle L1/L2/L3 granularity data. - cdef struct OrderBook: + cdef struct OrderBookContainer: pass # Represents a grouped batch of `OrderBookDelta` updates for an `OrderBook`. @@ -722,7 +721,7 @@ cdef extern from "../includes/model.h": # dereferenced to `OrderBook`, providing access to `OrderBook`'s methods without # having to manually access the underlying `OrderBook` instance. cdef struct OrderBook_API: - OrderBook *_0; + OrderBookContainer *_0; # Provides a C compatible Foreign Function Interface (FFI) for an underlying order book[`Level`]. # From 8287844768286f79d8696554173f91fdd105891e Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 17:04:17 +1100 Subject: [PATCH 067/104] Reorganize core python model module --- nautilus_core/model/src/enums.rs | 52 ++--- nautilus_core/model/src/orderbook/book_mbo.rs | 6 + nautilus_core/model/src/orderbook/book_mbp.rs | 5 + nautilus_core/model/src/orderbook/ladder.rs | 13 +- nautilus_core/model/src/python/common.rs | 216 ++++++++++++++++++ nautilus_core/model/src/python/data/bar.rs | 2 +- nautilus_core/model/src/python/data/delta.rs | 2 +- nautilus_core/model/src/python/data/deltas.rs | 2 +- nautilus_core/model/src/python/data/depth.rs | 2 +- nautilus_core/model/src/python/data/order.rs | 2 +- nautilus_core/model/src/python/data/quote.rs | 2 +- nautilus_core/model/src/python/data/trade.rs | 2 +- nautilus_core/model/src/python/mod.rs | 207 +---------------- .../model/src/python/orderbook/book_mbo.rs | 14 ++ .../model/src/python/orderbook/book_mbp.rs | 14 ++ .../model/src/python/orderbook/mod.rs | 3 + 16 files changed, 304 insertions(+), 240 deletions(-) create mode 100644 nautilus_core/model/src/python/common.rs create mode 100644 nautilus_core/model/src/python/orderbook/book_mbo.rs create mode 100644 nautilus_core/model/src/python/orderbook/book_mbp.rs diff --git a/nautilus_core/model/src/enums.rs b/nautilus_core/model/src/enums.rs index 6ffafc2fc416..5aec5ddca4cb 100644 --- a/nautilus_core/model/src/enums.rs +++ b/nautilus_core/model/src/enums.rs @@ -21,7 +21,7 @@ use pyo3::{exceptions::PyValueError, prelude::*, types::PyType, PyTypeInfo}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use strum::{AsRefStr, Display, EnumIter, EnumString, FromRepr}; -use crate::{enum_for_python, enum_strum_serde, python::EnumIterator}; +use crate::{enum_for_python, enum_strum_serde, python::common::EnumIterator}; pub trait FromU8 { fn from_u8(value: u8) -> Option @@ -50,7 +50,7 @@ pub trait FromU8 { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum AccountType { /// An account with unleveraged cash assets only. @@ -85,7 +85,7 @@ pub enum AccountType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum AggregationSource { /// The data is externally aggregated (outside the Nautilus system boundary). @@ -117,7 +117,7 @@ pub enum AggregationSource { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum AggressorSide { /// There was no specific aggressor for the trade. @@ -162,7 +162,7 @@ impl FromU8 for AggressorSide { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] #[allow(non_camel_case_types)] pub enum AssetClass { @@ -210,7 +210,7 @@ pub enum AssetClass { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum InstrumentClass { /// A spot market instrument class. The current market price of an instrument that is bought or sold for immediate delivery and payment. @@ -263,7 +263,7 @@ pub enum InstrumentClass { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum BarAggregation { /// Based on a number of ticks. @@ -337,7 +337,7 @@ pub enum BarAggregation { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum BookAction { /// An order is added to the book. @@ -388,7 +388,7 @@ impl FromU8 for BookAction { #[allow(non_camel_case_types)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum BookType { /// Top-of-book best bid/ask, one level per side. @@ -433,7 +433,7 @@ impl FromU8 for BookType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum ContingencyType { /// Not a contingent order. @@ -470,7 +470,7 @@ pub enum ContingencyType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum CurrencyType { /// A type of cryptocurrency or crypto token. @@ -505,7 +505,7 @@ pub enum CurrencyType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum InstrumentCloseType { /// When the market session ended. @@ -537,7 +537,7 @@ pub enum InstrumentCloseType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] #[allow(clippy::enum_variant_names)] pub enum LiquiditySide { @@ -573,7 +573,7 @@ pub enum LiquiditySide { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum MarketStatus { /// The market session is in the pre-open. @@ -620,7 +620,7 @@ pub enum MarketStatus { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum HaltReason { /// The venue or market session is not halted. @@ -655,7 +655,7 @@ pub enum HaltReason { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum OmsType { /// There is no specific type of order management specified (will defer to the venue). @@ -691,7 +691,7 @@ pub enum OmsType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum OptionKind { /// A Call option gives the holder the right, but not the obligation, to buy an underlying asset at a specified strike price within a specified period of time. @@ -724,7 +724,7 @@ pub enum OptionKind { #[allow(clippy::enum_variant_names)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum OrderSide { /// No order side is specified. @@ -789,7 +789,7 @@ impl FromU8 for OrderSide { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum OrderStatus { /// The order is initialized (instantiated) within the Nautilus system. @@ -857,7 +857,7 @@ pub enum OrderStatus { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum OrderType { /// A market order to buy or sell at the best available price in the current market. @@ -911,7 +911,7 @@ pub enum OrderType { #[allow(clippy::enum_variant_names)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum PositionSide { /// No position side is specified (only valid in the context of a filter for actions involving positions). @@ -948,7 +948,7 @@ pub enum PositionSide { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum PriceType { /// A quoted order price where a buyer is willing to buy a quantity of an instrument. @@ -986,7 +986,7 @@ pub enum PriceType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum TimeInForce { /// Good Till Canceled (GTC) - the order remains active until canceled. @@ -1033,7 +1033,7 @@ pub enum TimeInForce { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum TradingState { /// Normal trading operations. @@ -1068,7 +1068,7 @@ pub enum TradingState { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum TrailingOffsetType { /// No trailing offset type is specified (invalid for trailing type orders). @@ -1108,7 +1108,7 @@ pub enum TrailingOffsetType { #[strum(serialize_all = "SCREAMING_SNAKE_CASE")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model.enums") )] pub enum TriggerType { /// No trigger type is specified (invalid for orders with a trigger). diff --git a/nautilus_core/model/src/orderbook/book_mbo.rs b/nautilus_core/model/src/orderbook/book_mbo.rs index 42899315fa2c..4959d9e129c0 100644 --- a/nautilus_core/model/src/orderbook/book_mbo.rs +++ b/nautilus_core/model/src/orderbook/book_mbo.rs @@ -14,6 +14,7 @@ // ------------------------------------------------------------------------------------------------- use nautilus_core::time::UnixNanos; +use pyo3; use super::{ book::{get_avg_px_for_quantity, get_quantity_for_price}, @@ -31,6 +32,11 @@ use crate::{ }; /// Provides an order book which can handle MBO/L3 granularity data. +#[derive(Clone, Debug)] +#[cfg_attr( + feature = "python", + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") +)] pub struct OrderBookMbo { pub instrument_id: InstrumentId, pub sequence: u64, diff --git a/nautilus_core/model/src/orderbook/book_mbp.rs b/nautilus_core/model/src/orderbook/book_mbp.rs index 15ac991d7902..d8f34d8d457d 100644 --- a/nautilus_core/model/src/orderbook/book_mbp.rs +++ b/nautilus_core/model/src/orderbook/book_mbp.rs @@ -32,6 +32,11 @@ use crate::{ }; /// Provides an order book which can handle MBP/L2 or L1 (top only) granularity data. +#[derive(Clone, Debug)] +#[cfg_attr( + feature = "python", + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") +)] pub struct OrderBookMbp { pub instrument_id: InstrumentId, pub top_only: bool, diff --git a/nautilus_core/model/src/orderbook/ladder.rs b/nautilus_core/model/src/orderbook/ladder.rs index 2470b2176b8f..c0c86c8c6c3e 100644 --- a/nautilus_core/model/src/orderbook/ladder.rs +++ b/nautilus_core/model/src/orderbook/ladder.rs @@ -70,6 +70,7 @@ impl Display for BookPrice { } /// Represents one side of an order book as a ladder of price levels. +#[derive(Clone, Debug)] pub struct Ladder { pub side: OrderSide, pub levels: BTreeMap, @@ -164,12 +165,12 @@ impl Ladder { #[must_use] pub fn sizes(&self) -> f64 { - return self.levels.values().map(|l| l.size()).sum(); + self.levels.values().map(|l| l.size()).sum() } #[must_use] pub fn exposures(&self) -> f64 { - return self.levels.values().map(|l| l.exposure()).sum(); + self.levels.values().map(|l| l.exposure()).sum() } #[must_use] @@ -203,11 +204,11 @@ impl Ladder { fills.push((book_order.price, remainder)); } return fills; - } else { - // Add this fill and continue - fills.push((book_order.price, current)); - cumulative_denominator += current; } + + // Add this fill and continue + fills.push((book_order.price, current)); + cumulative_denominator += current; } } diff --git a/nautilus_core/model/src/python/common.rs b/nautilus_core/model/src/python/common.rs new file mode 100644 index 000000000000..06e9f4ce34fd --- /dev/null +++ b/nautilus_core/model/src/python/common.rs @@ -0,0 +1,216 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{PyDict, PyList}, + PyResult, Python, +}; +use serde_json::Value; +use strum::IntoEnumIterator; + +pub const PY_MODULE_MODEL: &str = "nautilus_trader.core.nautilus_pyo3.model"; + +/// Python iterator over the variants of an enum. +#[pyclass] +pub struct EnumIterator { + // Type erasure for code reuse. Generic types can't be exposed to Python. + iter: Box + Send>, +} + +#[pymethods] +impl EnumIterator { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + fn __next__(mut slf: PyRefMut<'_, Self>) -> Option { + slf.iter.next() + } +} + +impl EnumIterator { + pub fn new(py: Python<'_>) -> Self + where + E: strum::IntoEnumIterator + IntoPy>, + ::Iterator: Send, + { + Self { + iter: Box::new( + E::iter() + .map(|var| var.into_py(py)) + // Force eager evaluation because `py` isn't `Send` + .collect::>() + .into_iter(), + ), + } + } +} + +pub fn value_to_pydict(py: Python<'_>, val: &Value) -> PyResult> { + let dict = PyDict::new(py); + + match val { + Value::Object(map) => { + for (key, value) in map.iter() { + let py_value = value_to_pyobject(py, value)?; + dict.set_item(key, py_value)?; + } + } + // This shouldn't be reached in this function, but we include it for completeness + _ => return Err(PyValueError::new_err("Expected JSON object")), + } + + Ok(dict.into_py(py)) +} + +pub fn value_to_pyobject(py: Python<'_>, val: &Value) -> PyResult { + match val { + Value::Null => Ok(py.None()), + Value::Bool(b) => Ok(b.into_py(py)), + Value::String(s) => Ok(s.into_py(py)), + Value::Number(n) => { + if n.is_i64() { + Ok(n.as_i64().unwrap().into_py(py)) + } else if n.is_f64() { + Ok(n.as_f64().unwrap().into_py(py)) + } else { + Err(PyValueError::new_err("Unsupported JSON number type")) + } + } + Value::Array(arr) => { + let py_list = PyList::new(py, &[] as &[PyObject]); + for item in arr.iter() { + let py_item = value_to_pyobject(py, item)?; + py_list.append(py_item)?; + } + Ok(py_list.into()) + } + Value::Object(_) => { + let py_dict = value_to_pydict(py, val)?; + Ok(py_dict.into()) + } + } +} + +#[cfg(test)] +mod tests { + use pyo3::{ + prelude::*, + prepare_freethreaded_python, + types::{PyBool, PyInt, PyList, PyString}, + }; + use rstest::rstest; + use serde_json::Value; + + use super::*; + + #[rstest] + fn test_value_to_pydict() { + prepare_freethreaded_python(); + Python::with_gil(|py| { + let json_str = r#" + { + "type": "OrderAccepted", + "ts_event": 42, + "is_reconciliation": false + } + "#; + + let val: Value = serde_json::from_str(json_str).unwrap(); + let py_dict_ref = value_to_pydict(py, &val).unwrap(); + let py_dict = py_dict_ref.as_ref(py); + + assert_eq!( + py_dict + .get_item("type") + .unwrap() + .unwrap() + .downcast::() + .unwrap() + .to_str() + .unwrap(), + "OrderAccepted" + ); + assert_eq!( + py_dict + .get_item("ts_event") + .unwrap() + .unwrap() + .downcast::() + .unwrap() + .extract::() + .unwrap(), + 42 + ); + assert_eq!( + py_dict + .get_item("is_reconciliation") + .unwrap() + .unwrap() + .downcast::() + .unwrap() + .is_true(), + false + ); + }); + } + + #[rstest] + fn test_value_to_pyobject_string() { + prepare_freethreaded_python(); + Python::with_gil(|py| { + let val = Value::String("Hello, world!".to_string()); + let py_obj = value_to_pyobject(py, &val).unwrap(); + + assert_eq!(py_obj.extract::<&str>(py).unwrap(), "Hello, world!"); + }); + } + + #[rstest] + fn test_value_to_pyobject_bool() { + prepare_freethreaded_python(); + Python::with_gil(|py| { + let val = Value::Bool(true); + let py_obj = value_to_pyobject(py, &val).unwrap(); + + assert_eq!(py_obj.extract::(py).unwrap(), true); + }); + } + + #[rstest] + fn test_value_to_pyobject_array() { + prepare_freethreaded_python(); + Python::with_gil(|py| { + let val = Value::Array(vec![ + Value::String("item1".to_string()), + Value::String("item2".to_string()), + ]); + let binding = value_to_pyobject(py, &val).unwrap(); + let py_list = binding.downcast::(py).unwrap(); + + assert_eq!(py_list.len(), 2); + assert_eq!( + py_list.get_item(0).unwrap().extract::<&str>().unwrap(), + "item1" + ); + assert_eq!( + py_list.get_item(1).unwrap().extract::<&str>().unwrap(), + "item2" + ); + }); + } +} diff --git a/nautilus_core/model/src/python/data/bar.rs b/nautilus_core/model/src/python/data/bar.rs index e5d0f792bcf1..2542a337be42 100644 --- a/nautilus_core/model/src/python/data/bar.rs +++ b/nautilus_core/model/src/python/data/bar.rs @@ -33,7 +33,7 @@ use crate::{ }, enums::{AggregationSource, BarAggregation, PriceType}, identifiers::instrument_id::InstrumentId, - python::PY_MODULE_MODEL, + python::common::PY_MODULE_MODEL, types::{price::Price, quantity::Quantity}, }; diff --git a/nautilus_core/model/src/python/data/delta.rs b/nautilus_core/model/src/python/data/delta.rs index 822ad43dbde3..56904c3cc7e5 100644 --- a/nautilus_core/model/src/python/data/delta.rs +++ b/nautilus_core/model/src/python/data/delta.rs @@ -29,7 +29,7 @@ use crate::{ data::{delta::OrderBookDelta, order::BookOrder, Data}, enums::BookAction, identifiers::instrument_id::InstrumentId, - python::PY_MODULE_MODEL, + python::common::PY_MODULE_MODEL, }; use super::data_to_pycapsule; diff --git a/nautilus_core/model/src/python/data/deltas.rs b/nautilus_core/model/src/python/data/deltas.rs index c9b3c0d8882c..cd796c87d96f 100644 --- a/nautilus_core/model/src/python/data/deltas.rs +++ b/nautilus_core/model/src/python/data/deltas.rs @@ -24,7 +24,7 @@ use pyo3::{prelude::*, pyclass::CompareOp}; use crate::{ data::{delta::OrderBookDelta, deltas::OrderBookDeltas}, identifiers::instrument_id::InstrumentId, - python::PY_MODULE_MODEL, + python::common::PY_MODULE_MODEL, }; #[pymethods] diff --git a/nautilus_core/model/src/python/data/depth.rs b/nautilus_core/model/src/python/data/depth.rs index 134d227e904d..6f53dd0552dd 100644 --- a/nautilus_core/model/src/python/data/depth.rs +++ b/nautilus_core/model/src/python/data/depth.rs @@ -33,7 +33,7 @@ use crate::{ }, enums::OrderSide, identifiers::instrument_id::InstrumentId, - python::PY_MODULE_MODEL, + python::common::PY_MODULE_MODEL, types::{price::Price, quantity::Quantity}, }; diff --git a/nautilus_core/model/src/python/data/order.rs b/nautilus_core/model/src/python/data/order.rs index f6a1992dfbf7..8b6654d8f76f 100644 --- a/nautilus_core/model/src/python/data/order.rs +++ b/nautilus_core/model/src/python/data/order.rs @@ -27,7 +27,7 @@ use pyo3::{prelude::*, pyclass::CompareOp, types::PyDict}; use crate::{ data::order::{BookOrder, OrderId}, enums::OrderSide, - python::PY_MODULE_MODEL, + python::common::PY_MODULE_MODEL, types::{price::Price, quantity::Quantity}, }; diff --git a/nautilus_core/model/src/python/data/quote.rs b/nautilus_core/model/src/python/data/quote.rs index 17bb1ad0a998..68f390226f6c 100644 --- a/nautilus_core/model/src/python/data/quote.rs +++ b/nautilus_core/model/src/python/data/quote.rs @@ -34,7 +34,7 @@ use crate::{ data::{quote::QuoteTick, Data}, enums::PriceType, identifiers::instrument_id::InstrumentId, - python::PY_MODULE_MODEL, + python::common::PY_MODULE_MODEL, types::{price::Price, quantity::Quantity}, }; diff --git a/nautilus_core/model/src/python/data/trade.rs b/nautilus_core/model/src/python/data/trade.rs index 528bde2993ba..64416625920c 100644 --- a/nautilus_core/model/src/python/data/trade.rs +++ b/nautilus_core/model/src/python/data/trade.rs @@ -34,7 +34,7 @@ use crate::{ data::{trade::TradeTick, Data}, enums::{AggressorSide, FromU8}, identifiers::{instrument_id::InstrumentId, trade_id::TradeId}, - python::PY_MODULE_MODEL, + python::common::PY_MODULE_MODEL, types::{price::Price, quantity::Quantity}, }; diff --git a/nautilus_core/model/src/python/mod.rs b/nautilus_core/model/src/python/mod.rs index bd9b4a66f1c4..dad92dcf562c 100644 --- a/nautilus_core/model/src/python/mod.rs +++ b/nautilus_core/model/src/python/mod.rs @@ -13,219 +13,21 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use pyo3::{ - exceptions::PyValueError, - prelude::*, - types::{PyDict, PyList}, - PyResult, Python, -}; -use serde_json::Value; -use strum::IntoEnumIterator; +use pyo3::{prelude::*, PyResult, Python}; use crate::enums; +pub mod common; pub mod data; pub mod events; pub mod identifiers; pub mod instruments; pub mod macros; +pub mod orderbook; pub mod orders; pub mod position; pub mod types; -pub const PY_MODULE_MODEL: &str = "nautilus_trader.core.nautilus_pyo3.model"; - -/// Python iterator over the variants of an enum. -#[pyclass] -pub struct EnumIterator { - // Type erasure for code reuse. Generic types can't be exposed to Python. - iter: Box + Send>, -} - -#[pymethods] -impl EnumIterator { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { - slf - } - - fn __next__(mut slf: PyRefMut<'_, Self>) -> Option { - slf.iter.next() - } -} - -impl EnumIterator { - pub fn new(py: Python<'_>) -> Self - where - E: strum::IntoEnumIterator + IntoPy>, - ::Iterator: Send, - { - Self { - iter: Box::new( - E::iter() - .map(|var| var.into_py(py)) - // Force eager evaluation because `py` isn't `Send` - .collect::>() - .into_iter(), - ), - } - } -} - -pub fn value_to_pydict(py: Python<'_>, val: &Value) -> PyResult> { - let dict = PyDict::new(py); - - match val { - Value::Object(map) => { - for (key, value) in map.iter() { - let py_value = value_to_pyobject(py, value)?; - dict.set_item(key, py_value)?; - } - } - // This shouldn't be reached in this function, but we include it for completeness - _ => return Err(PyValueError::new_err("Expected JSON object")), - } - - Ok(dict.into_py(py)) -} - -pub fn value_to_pyobject(py: Python<'_>, val: &Value) -> PyResult { - match val { - Value::Null => Ok(py.None()), - Value::Bool(b) => Ok(b.into_py(py)), - Value::String(s) => Ok(s.into_py(py)), - Value::Number(n) => { - if n.is_i64() { - Ok(n.as_i64().unwrap().into_py(py)) - } else if n.is_f64() { - Ok(n.as_f64().unwrap().into_py(py)) - } else { - Err(PyValueError::new_err("Unsupported JSON number type")) - } - } - Value::Array(arr) => { - let py_list = PyList::new(py, &[] as &[PyObject]); - for item in arr.iter() { - let py_item = value_to_pyobject(py, item)?; - py_list.append(py_item)?; - } - Ok(py_list.into()) - } - Value::Object(_) => { - let py_dict = value_to_pydict(py, val)?; - Ok(py_dict.into()) - } - } -} - -#[cfg(test)] -mod tests { - use pyo3::{ - prelude::*, - prepare_freethreaded_python, - types::{PyBool, PyInt, PyList, PyString}, - }; - use rstest::rstest; - use serde_json::Value; - - use super::*; - - #[rstest] - fn test_value_to_pydict() { - prepare_freethreaded_python(); - Python::with_gil(|py| { - let json_str = r#" - { - "type": "OrderAccepted", - "ts_event": 42, - "is_reconciliation": false - } - "#; - - let val: Value = serde_json::from_str(json_str).unwrap(); - let py_dict_ref = value_to_pydict(py, &val).unwrap(); - let py_dict = py_dict_ref.as_ref(py); - - assert_eq!( - py_dict - .get_item("type") - .unwrap() - .unwrap() - .downcast::() - .unwrap() - .to_str() - .unwrap(), - "OrderAccepted" - ); - assert_eq!( - py_dict - .get_item("ts_event") - .unwrap() - .unwrap() - .downcast::() - .unwrap() - .extract::() - .unwrap(), - 42 - ); - assert_eq!( - py_dict - .get_item("is_reconciliation") - .unwrap() - .unwrap() - .downcast::() - .unwrap() - .is_true(), - false - ); - }); - } - - #[rstest] - fn test_value_to_pyobject_string() { - prepare_freethreaded_python(); - Python::with_gil(|py| { - let val = Value::String("Hello, world!".to_string()); - let py_obj = value_to_pyobject(py, &val).unwrap(); - - assert_eq!(py_obj.extract::<&str>(py).unwrap(), "Hello, world!"); - }); - } - - #[rstest] - fn test_value_to_pyobject_bool() { - prepare_freethreaded_python(); - Python::with_gil(|py| { - let val = Value::Bool(true); - let py_obj = value_to_pyobject(py, &val).unwrap(); - - assert_eq!(py_obj.extract::(py).unwrap(), true); - }); - } - - #[rstest] - fn test_value_to_pyobject_array() { - prepare_freethreaded_python(); - Python::with_gil(|py| { - let val = Value::Array(vec![ - Value::String("item1".to_string()), - Value::String("item2".to_string()), - ]); - let binding = value_to_pyobject(py, &val).unwrap(); - let py_list = binding.downcast::(py).unwrap(); - - assert_eq!(py_list.len(), 2); - assert_eq!( - py_list.get_item(0).unwrap().extract::<&str>().unwrap(), - "item1" - ); - assert_eq!( - py_list.get_item(1).unwrap().extract::<&str>().unwrap(), - "item2" - ); - }); - } -} - /// Loaded as nautilus_pyo3.model #[pymodule] pub fn model(_: Python<'_>, m: &PyModule) -> PyResult<()> { @@ -303,6 +105,9 @@ pub fn model(_: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + // Order book + m.add_class::()?; + m.add_class::()?; // Events - order m.add_class::()?; m.add_class::()?; diff --git a/nautilus_core/model/src/python/orderbook/book_mbo.rs b/nautilus_core/model/src/python/orderbook/book_mbo.rs new file mode 100644 index 000000000000..97d459d8d1e8 --- /dev/null +++ b/nautilus_core/model/src/python/orderbook/book_mbo.rs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- diff --git a/nautilus_core/model/src/python/orderbook/book_mbp.rs b/nautilus_core/model/src/python/orderbook/book_mbp.rs new file mode 100644 index 000000000000..97d459d8d1e8 --- /dev/null +++ b/nautilus_core/model/src/python/orderbook/book_mbp.rs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- diff --git a/nautilus_core/model/src/python/orderbook/mod.rs b/nautilus_core/model/src/python/orderbook/mod.rs index 97d459d8d1e8..7602360c877b 100644 --- a/nautilus_core/model/src/python/orderbook/mod.rs +++ b/nautilus_core/model/src/python/orderbook/mod.rs @@ -12,3 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. // ------------------------------------------------------------------------------------------------- + +pub mod book_mbo; +pub mod book_mbp; From 5e8a26fe7321bff15c9adaa25320d016a5bd1cd7 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 17:13:13 +1100 Subject: [PATCH 068/104] Qualify pyo3::pyclass attribute --- nautilus_core/model/src/data/bar.rs | 6 +++--- nautilus_core/model/src/data/delta.rs | 2 +- nautilus_core/model/src/data/deltas.rs | 3 +-- nautilus_core/model/src/data/depth.rs | 3 +-- nautilus_core/model/src/data/order.rs | 3 +-- nautilus_core/model/src/data/quote.rs | 2 +- nautilus_core/model/src/data/trade.rs | 2 +- nautilus_core/network/src/http.rs | 6 +++--- 8 files changed, 12 insertions(+), 15 deletions(-) diff --git a/nautilus_core/model/src/data/bar.rs b/nautilus_core/model/src/data/bar.rs index bf021dd2b0ca..290ad06c933d 100644 --- a/nautilus_core/model/src/data/bar.rs +++ b/nautilus_core/model/src/data/bar.rs @@ -38,7 +38,7 @@ use crate::{ #[derive(Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct BarSpecification { @@ -73,7 +73,7 @@ impl Display for BarSpecification { #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct BarType { /// The bar types instrument ID. @@ -206,7 +206,7 @@ impl<'de> Deserialize<'de> for BarType { #[serde(tag = "type")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct Bar { /// The bar type for this bar. diff --git a/nautilus_core/model/src/data/delta.rs b/nautilus_core/model/src/data/delta.rs index 7bf17b21e628..6ac0b3db3be6 100644 --- a/nautilus_core/model/src/data/delta.rs +++ b/nautilus_core/model/src/data/delta.rs @@ -38,7 +38,7 @@ use crate::{ #[serde(tag = "type")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct OrderBookDelta { diff --git a/nautilus_core/model/src/data/deltas.rs b/nautilus_core/model/src/data/deltas.rs index b7ed835dab7a..1b444662e8e9 100644 --- a/nautilus_core/model/src/data/deltas.rs +++ b/nautilus_core/model/src/data/deltas.rs @@ -19,7 +19,6 @@ use std::{ }; use nautilus_core::time::UnixNanos; -use pyo3::prelude::*; use super::delta::OrderBookDelta; use crate::identifiers::instrument_id::InstrumentId; @@ -30,7 +29,7 @@ use crate::identifiers::instrument_id::InstrumentId; #[derive(Clone, Debug)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct OrderBookDeltas { /// The instrument ID for the book. diff --git a/nautilus_core/model/src/data/depth.rs b/nautilus_core/model/src/data/depth.rs index 5c15f0f3166f..619976ed0290 100644 --- a/nautilus_core/model/src/data/depth.rs +++ b/nautilus_core/model/src/data/depth.rs @@ -20,7 +20,6 @@ use std::{ use indexmap::IndexMap; use nautilus_core::{serialization::Serializable, time::UnixNanos}; -use pyo3::prelude::*; use serde::{Deserialize, Serialize}; use super::order::BookOrder; @@ -41,7 +40,7 @@ pub const DEPTH10_LEN: usize = 10; #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct OrderBookDepth10 { diff --git a/nautilus_core/model/src/data/order.rs b/nautilus_core/model/src/data/order.rs index b2fa5b313b45..dc053f7f9f9d 100644 --- a/nautilus_core/model/src/data/order.rs +++ b/nautilus_core/model/src/data/order.rs @@ -19,7 +19,6 @@ use std::{ }; use nautilus_core::serialization::Serializable; -use pyo3::prelude::*; use serde::{Deserialize, Serialize}; use super::{quote::QuoteTick, trade::TradeTick}; @@ -49,7 +48,7 @@ pub const NULL_ORDER: BookOrder = BookOrder { #[derive(Clone, Eq, Debug, Serialize, Deserialize)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct BookOrder { diff --git a/nautilus_core/model/src/data/quote.rs b/nautilus_core/model/src/data/quote.rs index 5c6a6ec29364..56067bc78d0c 100644 --- a/nautilus_core/model/src/data/quote.rs +++ b/nautilus_core/model/src/data/quote.rs @@ -42,7 +42,7 @@ use crate::{ #[serde(tag = "type")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct QuoteTick { diff --git a/nautilus_core/model/src/data/trade.rs b/nautilus_core/model/src/data/trade.rs index 504bdd62333c..24a7cad017e5 100644 --- a/nautilus_core/model/src/data/trade.rs +++ b/nautilus_core/model/src/data/trade.rs @@ -37,7 +37,7 @@ use crate::{ #[serde(tag = "type")] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] #[cfg_attr(feature = "trivial_copy", derive(Copy))] pub struct TradeTick { diff --git a/nautilus_core/network/src/http.rs b/nautilus_core/network/src/http.rs index 01f8d8b5324c..0d14b355fe1b 100644 --- a/nautilus_core/network/src/http.rs +++ b/nautilus_core/network/src/http.rs @@ -94,7 +94,7 @@ impl InnerHttpClient { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.network") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.network") )] pub enum HttpMethod { GET, @@ -130,7 +130,7 @@ impl HttpMethod { #[derive(Debug, Clone)] #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.network") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.network") )] pub struct HttpResponse { #[pyo3(get)] @@ -169,7 +169,7 @@ impl HttpResponse { #[cfg_attr( feature = "python", - pyclass(module = "nautilus_trader.core.nautilus_pyo3.network") + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.network") )] pub struct HttpClient { rate_limiter: Arc>, From e4476213bcb907825732c3257ee922bdebb85fbe Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 18:16:12 +1100 Subject: [PATCH 069/104] Add OrderBookMbp pyo3 interface --- nautilus_core/model/src/orderbook/ladder.rs | 4 + nautilus_core/model/src/orderbook/level.rs | 4 + nautilus_core/model/src/python/mod.rs | 1 + .../model/src/python/orderbook/book_mbp.rs | 210 ++++++++++++++++++ nautilus_trader/core/nautilus_pyo3.pyi | 45 ++++ 5 files changed, 264 insertions(+) diff --git a/nautilus_core/model/src/orderbook/ladder.rs b/nautilus_core/model/src/orderbook/ladder.rs index c0c86c8c6c3e..1b35f4e64c43 100644 --- a/nautilus_core/model/src/orderbook/ladder.rs +++ b/nautilus_core/model/src/orderbook/ladder.rs @@ -29,6 +29,10 @@ use crate::{ /// Represents a price level with a specified side in an order books ladder. #[derive(Clone, Copy, Debug, Eq)] +#[cfg_attr( + feature = "python", + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") +)] pub struct BookPrice { pub value: Price, pub side: OrderSide, diff --git a/nautilus_core/model/src/orderbook/level.rs b/nautilus_core/model/src/orderbook/level.rs index 4c8a77ca7d90..f967b3ea3278 100644 --- a/nautilus_core/model/src/orderbook/level.rs +++ b/nautilus_core/model/src/orderbook/level.rs @@ -22,6 +22,10 @@ use crate::{ }; #[derive(Clone, Debug, Eq)] +#[cfg_attr( + feature = "python", + pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") +)] pub struct Level { pub price: BookPrice, pub orders: BTreeMap, diff --git a/nautilus_core/model/src/python/mod.rs b/nautilus_core/model/src/python/mod.rs index dad92dcf562c..79c29c410bae 100644 --- a/nautilus_core/model/src/python/mod.rs +++ b/nautilus_core/model/src/python/mod.rs @@ -108,6 +108,7 @@ pub fn model(_: Python<'_>, m: &PyModule) -> PyResult<()> { // Order book m.add_class::()?; m.add_class::()?; + m.add_class::()?; // Events - order m.add_class::()?; m.add_class::()?; diff --git a/nautilus_core/model/src/python/orderbook/book_mbp.rs b/nautilus_core/model/src/python/orderbook/book_mbp.rs index 97d459d8d1e8..2de16cc5337d 100644 --- a/nautilus_core/model/src/python/orderbook/book_mbp.rs +++ b/nautilus_core/model/src/python/orderbook/book_mbp.rs @@ -12,3 +12,213 @@ // See the License for the specific language governing permissions and // limitations under the License. // ------------------------------------------------------------------------------------------------- + +use nautilus_core::{python::to_pyruntime_err, time::UnixNanos}; +use pyo3::prelude::*; + +use crate::{ + data::{ + delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, order::BookOrder, + quote::QuoteTick, trade::TradeTick, + }, + enums::{BookType, OrderSide}, + identifiers::instrument_id::InstrumentId, + orderbook::{book_mbp::OrderBookMbp, level::Level}, + types::{price::Price, quantity::Quantity}, +}; + +#[pymethods] +impl OrderBookMbp { + #[new] + #[pyo3(signature = (instrument_id, top_only=false))] + fn py_new(instrument_id: InstrumentId, top_only: bool) -> Self { + Self::new(instrument_id, top_only) + } + + fn __str__(&self) -> String { + // TODO: Return debug string for now + format!("{self:?}") + } + + fn __repr__(&self) -> String { + format!("{self:?}") + } + + #[getter] + #[pyo3(name = "instrument_id")] + fn py_instrument_id(&self) -> InstrumentId { + self.instrument_id + } + + #[getter] + #[pyo3(name = "book_type")] + fn py_book_type(&self) -> BookType { + match self.top_only { + true => BookType::L1_MBP, + false => BookType::L2_MBP, + } + } + + #[getter] + #[pyo3(name = "sequence")] + fn py_sequence(&self) -> u64 { + self.sequence + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_last + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_last + } + + #[getter] + #[pyo3(name = "ts_last")] + fn py_ts_last(&self) -> UnixNanos { + self.ts_last + } + + #[getter] + #[pyo3(name = "count")] + fn py_count(&self) -> u64 { + self.count + } + + #[pyo3(name = "reset")] + fn py_reset(&mut self) { + self.reset() + } + + #[pyo3(signature = (order, ts_event, sequence=0))] + #[pyo3(name = "update")] + fn py_update(&mut self, order: BookOrder, ts_event: UnixNanos, sequence: u64) { + self.update(order, ts_event, sequence); + } + + #[pyo3(name = "update_quote_tick")] + fn py_update_quote_tick(&mut self, quote: &QuoteTick) { + self.update_quote_tick(quote) + } + + #[pyo3(name = "update_trade_tick")] + fn py_update_trade_tick(&mut self, trade: &TradeTick) { + self.update_trade_tick(trade) + } + + #[pyo3(signature = (order, ts_event, sequence=0))] + #[pyo3(name = "delete")] + fn py_delete(&mut self, order: BookOrder, ts_event: UnixNanos, sequence: u64) { + self.delete(order, ts_event, sequence); + } + + #[pyo3(signature = (ts_event, sequence=0))] + #[pyo3(name = "clear")] + fn py_clear(&mut self, ts_event: UnixNanos, sequence: u64) { + self.clear(ts_event, sequence); + } + + #[pyo3(signature = (ts_event, sequence=0))] + #[pyo3(name = "clear_bids")] + fn py_clear_bids(&mut self, ts_event: UnixNanos, sequence: u64) { + self.clear_bids(ts_event, sequence); + } + + #[pyo3(signature = (ts_event, sequence=0))] + #[pyo3(name = "clear_asks")] + fn py_clear_asks(&mut self, ts_event: UnixNanos, sequence: u64) { + self.clear_asks(ts_event, sequence); + } + + #[pyo3(name = "apply_delta")] + fn py_apply_delta(&mut self, delta: OrderBookDelta) { + self.apply_delta(delta); + } + + #[pyo3(name = "apply_deltas")] + fn py_apply_deltas(&mut self, deltas: OrderBookDeltas) { + self.apply_deltas(deltas); + } + + #[pyo3(name = "apply_depth")] + fn py_apply_depth(&mut self, depth: OrderBookDepth10) { + self.apply_depth(depth); + } + + #[pyo3(name = "check_integrity")] + fn py_check_integrity(&mut self) -> PyResult<()> { + self.check_integrity().map_err(to_pyruntime_err) + } + + #[pyo3(name = "bids")] + fn py_bids(&self) -> Vec { + // TODO: Improve efficiency + self.bids() + .iter() + .map(|level_ref| (*level_ref).clone()) + .collect() + } + + #[pyo3(name = "asks")] + fn py_asks(&self) -> Vec { + // TODO: Improve efficiency + self.asks() + .iter() + .map(|level_ref| (*level_ref).clone()) + .collect() + } + + #[pyo3(name = "best_bid_price")] + fn py_best_bid_price(&self) -> Option { + self.best_bid_price() + } + + #[pyo3(name = "best_ask_price")] + fn py_best_ask_price(&self) -> Option { + self.best_ask_price() + } + + #[pyo3(name = "best_bid_size")] + fn py_best_bid_size(&self) -> Option { + self.best_bid_size() + } + + #[pyo3(name = "best_ask_size")] + fn py_best_ask_size(&self) -> Option { + self.best_ask_size() + } + + #[pyo3(name = "spread")] + fn py_spread(&self) -> Option { + self.spread() + } + + #[pyo3(name = "midpoint")] + fn py_midpoint(&self) -> Option { + self.midpoint() + } + + #[pyo3(name = "get_avg_px_for_quantity")] + fn py_get_avg_px_for_quantity(&self, qty: Quantity, order_side: OrderSide) -> f64 { + self.get_avg_px_for_quantity(qty, order_side) + } + + #[pyo3(name = "get_quantity_for_price")] + fn py_get_quantity_for_price(&self, price: Price, order_side: OrderSide) -> f64 { + self.get_quantity_for_price(price, order_side) + } + + #[pyo3(name = "simulate_fills")] + fn py_simulate_fills(&self, order: &BookOrder) -> Vec<(Price, Quantity)> { + self.simulate_fills(order) + } + + #[pyo3(name = "pprint")] + fn py_pprint(&self, num_levels: usize) -> String { + self.pprint(num_levels) + } +} diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index db0b3e6f05dc..d1ca0c9622be 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1614,6 +1614,51 @@ class OrderExpired: def from_dict(cls, values: dict[str, str]) -> OrderExpired: ... def to_dict(self) -> dict[str, str]: ... +class OrderBookMbp: + def __init__( + self, + instrument_id: InstrumentId, + top_only: bool = False, + ) -> None: ... + @property + def instrument_id(self) -> InstrumentId: ... + @property + def book_type(self) -> BookType: ... + @property + def sequence(self) -> int: ... + @property + def ts_event(self) -> int: ... + @property + def ts_init(self) -> int: ... + @property + def ts_last(self) -> int: ... + @property + def count(self) -> int: ... + def reset(self) -> None: ... + def update(self, order: BookOrder, ts_event: int, sequence: int = 0) -> None: ... + def update_quote_tick(self, quote: QuoteTick) -> None: ... + def update_trade_tick(self, trade: TradeTick) -> None: ... + def delete(self, order: BookOrder, ts_event: int, sequence: int = 0) -> None: ... + def clear(self, ts_event: int, sequence: int = 0) -> None: ... + def clear_bids(self, ts_event: int, sequence: int = 0) -> None: ... + def clear_asks(self, ts_event: int, sequence: int = 0) -> None: ... + def apply_delta(self, delta: OrderBookDelta) -> None: ... + def apply_deltas(self, deltas: OrderBookDeltas) -> None: ... + def apply_depth(self, depth: OrderBookDepth10) -> None: ... + def check_integrity(self) -> None: ... + # def bids(self) -> list[Level]: ... TBD + # def asks(self) -> list[Level]: ... TBD + def best_bid_price(self) -> Price | None: ... + def best_ask_price(self) -> Price | None: ... + def best_bid_size(self) -> Quantity | None: ... + def best_ask_size(self) -> Quantity | None: ... + def spread(self) -> float | None: ... + def midpoint(self) -> float | None: ... + def get_avg_px_for_quantity(self, qty: Quantity, order_side: OrderSide) -> float: ... + def get_quantity_for_price(self, price: Price, order_side: OrderSide) -> float: ... + def simulate_fills(self, order: BookOrder) -> list[tuple[Price, Quantity]]: ... + def pprint(self, num_levels: int) -> str: ... + ################################################################################################### # Infrastructure ################################################################################################### From a1a0f6d05ad96ad4a9ee06fbf0dd754399dab79d Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sun, 18 Feb 2024 18:32:24 +1100 Subject: [PATCH 070/104] Upgrade ruff --- .pre-commit-config.yaml | 2 +- nautilus_core/Cargo.lock | 21 ++++++------ nautilus_trader/common/providers.py | 11 +++++-- poetry.lock | 50 ++++++++++++++--------------- pyproject.toml | 2 +- 5 files changed, 46 insertions(+), 40 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9c30ee16007..bb30dd698831 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,7 +82,7 @@ repos: exclude: "docs/_pygments/monokai.py" - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.2.1 + rev: v0.2.2 hooks: - id: ruff args: ["--fix"] diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 42636e976a1f..537099073ad3 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -3594,16 +3594,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3767,7 +3768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-webpki 0.101.7", "sct", ] @@ -3779,7 +3780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-pki-types", "rustls-webpki 0.102.2", "subtle", @@ -3829,7 +3830,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -3839,7 +3840,7 @@ version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "rustls-pki-types", "untrusted 0.9.0", ] @@ -3886,7 +3887,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -4561,9 +4562,9 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" diff --git a/nautilus_trader/common/providers.py b/nautilus_trader/common/providers.py index 6f8d7a375cb6..96e0a53965b9 100644 --- a/nautilus_trader/common/providers.py +++ b/nautilus_trader/common/providers.py @@ -55,6 +55,8 @@ def __init__(self, config: InstrumentProviderConfig | None = None) -> None: self._loaded = False self._loading = False + self._tasks: set[asyncio.Task] = set() + self._log.info("READY.") @property @@ -172,7 +174,8 @@ def load_all(self, filters: dict | None = None) -> None: """ loop = asyncio.get_event_loop() if loop.is_running(): - loop.create_task(self.load_all_async(filters)) + task = loop.create_task(self.load_all_async(filters)) + self._tasks.add(task) else: loop.run_until_complete(self.load_all_async(filters)) @@ -197,7 +200,8 @@ def load_ids( loop = asyncio.get_event_loop() if loop.is_running(): - loop.create_task(self.load_ids_async(instrument_ids, filters)) + task = loop.create_task(self.load_ids_async(instrument_ids, filters)) + self._tasks.add(task) else: loop.run_until_complete(self.load_ids_async(instrument_ids, filters)) @@ -222,7 +226,8 @@ def load( loop = asyncio.get_event_loop() if loop.is_running(): - loop.create_task(self.load_async(instrument_id, filters)) + task = loop.create_task(self.load_async(instrument_id, filters)) + self._tasks.add(task) else: loop.run_until_complete(self.load_async(instrument_id, filters)) diff --git a/poetry.lock b/poetry.lock index a25eac6e26da..71e4e88676d2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1929,28 +1929,28 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "ruff" -version = "0.2.1" +version = "0.2.2" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.2.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:dd81b911d28925e7e8b323e8d06951554655021df8dd4ac3045d7212ac4ba080"}, - {file = "ruff-0.2.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dc586724a95b7d980aa17f671e173df00f0a2eef23f8babbeee663229a938fec"}, - {file = "ruff-0.2.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c92db7101ef5bfc18e96777ed7bc7c822d545fa5977e90a585accac43d22f18a"}, - {file = "ruff-0.2.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:13471684694d41ae0f1e8e3a7497e14cd57ccb7dd72ae08d56a159d6c9c3e30e"}, - {file = "ruff-0.2.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a11567e20ea39d1f51aebd778685582d4c56ccb082c1161ffc10f79bebe6df35"}, - {file = "ruff-0.2.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:00a818e2db63659570403e44383ab03c529c2b9678ba4ba6c105af7854008105"}, - {file = "ruff-0.2.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be60592f9d218b52f03384d1325efa9d3b41e4c4d55ea022cd548547cc42cd2b"}, - {file = "ruff-0.2.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbd2288890b88e8aab4499e55148805b58ec711053588cc2f0196a44f6e3d855"}, - {file = "ruff-0.2.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ef052283da7dec1987bba8d8733051c2325654641dfe5877a4022108098683"}, - {file = "ruff-0.2.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7022d66366d6fded4ba3889f73cd791c2d5621b2ccf34befc752cb0df70f5fad"}, - {file = "ruff-0.2.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0a725823cb2a3f08ee743a534cb6935727d9e47409e4ad72c10a3faf042ad5ba"}, - {file = "ruff-0.2.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:0034d5b6323e6e8fe91b2a1e55b02d92d0b582d2953a2b37a67a2d7dedbb7acc"}, - {file = "ruff-0.2.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e5cb5526d69bb9143c2e4d2a115d08ffca3d8e0fddc84925a7b54931c96f5c02"}, - {file = "ruff-0.2.1-py3-none-win32.whl", hash = "sha256:6b95ac9ce49b4fb390634d46d6ece32ace3acdd52814671ccaf20b7f60adb232"}, - {file = "ruff-0.2.1-py3-none-win_amd64.whl", hash = "sha256:e3affdcbc2afb6f5bd0eb3130139ceedc5e3f28d206fe49f63073cb9e65988e0"}, - {file = "ruff-0.2.1-py3-none-win_arm64.whl", hash = "sha256:efababa8e12330aa94a53e90a81eb6e2d55f348bc2e71adbf17d9cad23c03ee6"}, - {file = "ruff-0.2.1.tar.gz", hash = "sha256:3b42b5d8677cd0c72b99fcaf068ffc62abb5a19e71b4a3b9cfa50658a0af02f1"}, + {file = "ruff-0.2.2-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0a9efb032855ffb3c21f6405751d5e147b0c6b631e3ca3f6b20f917572b97eb6"}, + {file = "ruff-0.2.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d450b7fbff85913f866a5384d8912710936e2b96da74541c82c1b458472ddb39"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecd46e3106850a5c26aee114e562c329f9a1fbe9e4821b008c4404f64ff9ce73"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5e22676a5b875bd72acd3d11d5fa9075d3a5f53b877fe7b4793e4673499318ba"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1695700d1e25a99d28f7a1636d85bafcc5030bba9d0578c0781ba1790dbcf51c"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:b0c232af3d0bd8f521806223723456ffebf8e323bd1e4e82b0befb20ba18388e"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f63d96494eeec2fc70d909393bcd76c69f35334cdbd9e20d089fb3f0640216ca"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a61ea0ff048e06de273b2e45bd72629f470f5da8f71daf09fe481278b175001"}, + {file = "ruff-0.2.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1439c8f407e4f356470e54cdecdca1bd5439a0673792dbe34a2b0a551a2fe3"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:940de32dc8853eba0f67f7198b3e79bc6ba95c2edbfdfac2144c8235114d6726"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:0c126da55c38dd917621552ab430213bdb3273bb10ddb67bc4b761989210eb6e"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3b65494f7e4bed2e74110dac1f0d17dc8e1f42faaa784e7c58a98e335ec83d7e"}, + {file = "ruff-0.2.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1ec49be4fe6ddac0503833f3ed8930528e26d1e60ad35c2446da372d16651ce9"}, + {file = "ruff-0.2.2-py3-none-win32.whl", hash = "sha256:d920499b576f6c68295bc04e7b17b6544d9d05f196bb3aac4358792ef6f34325"}, + {file = "ruff-0.2.2-py3-none-win_amd64.whl", hash = "sha256:cc9a91ae137d687f43a44c900e5d95e9617cb37d4c989e462980ba27039d239d"}, + {file = "ruff-0.2.2-py3-none-win_arm64.whl", hash = "sha256:c9d15fc41e6054bfc7200478720570078f0b41c9ae4f010bcc16bd6f4d1aacdd"}, + {file = "ruff-0.2.2.tar.gz", hash = "sha256:e62ed7f36b3068a30ba39193a14274cd706bc486fad521276458022f7bccb31d"}, ] [[package]] @@ -2309,13 +2309,13 @@ files = [ [[package]] name = "types-requests" -version = "2.31.0.20240125" +version = "2.31.0.20240218" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" files = [ - {file = "types-requests-2.31.0.20240125.tar.gz", hash = "sha256:03a28ce1d7cd54199148e043b2079cdded22d6795d19a2c2a6791a4b2b5e2eb5"}, - {file = "types_requests-2.31.0.20240125-py3-none-any.whl", hash = "sha256:9592a9a4cb92d6d75d9b491a41477272b710e021011a2a3061157e2fb1f1a5d1"}, + {file = "types-requests-2.31.0.20240218.tar.gz", hash = "sha256:f1721dba8385958f504a5386240b92de4734e047a08a40751c1654d1ac3349c5"}, + {file = "types_requests-2.31.0.20240218-py3-none-any.whl", hash = "sha256:a82807ec6ddce8f00fe0e949da6d6bc1fbf1715420218a9640d695f70a9e5a9b"}, ] [package.dependencies] @@ -2391,13 +2391,13 @@ files = [ [[package]] name = "urllib3" -version = "2.2.0" +version = "2.2.1" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.0-py3-none-any.whl", hash = "sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"}, - {file = "urllib3-2.2.0.tar.gz", hash = "sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20"}, + {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, + {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, ] [package.extras] @@ -2595,4 +2595,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "139aafb980dbeb5983f778be70184f745617a831bb7e7952cc001cacba820146" +content-hash = "f401e28b92a35ca1084aad06a4d71f51252e759c1fb8aa21f9f73f1981ab5e02" diff --git a/pyproject.toml b/pyproject.toml index c6833506a08f..463c3da1a312 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ docformatter = "^1.7.5" mypy = "^1.8.0" pandas-stubs = "^2.1.4" pre-commit = "^3.6.1" -ruff = "^0.2.1" +ruff = "^0.2.2" types-pytz = "^2023.3" types-requests = "^2.31" types-toml = "^0.10.2" From c491f3b58b4680a184c84bbbc725977f636f8d6c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 10:14:48 +1100 Subject: [PATCH 071/104] Add collection type annotations --- nautilus_trader/execution/algorithm.pxd | 4 ++-- nautilus_trader/execution/emulator.pxd | 11 +++++------ nautilus_trader/execution/emulator.pyx | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nautilus_trader/execution/algorithm.pxd b/nautilus_trader/execution/algorithm.pxd index 222d1fdba366..61ccccfc21c9 100644 --- a/nautilus_trader/execution/algorithm.pxd +++ b/nautilus_trader/execution/algorithm.pxd @@ -67,8 +67,8 @@ from nautilus_trader.portfolio.base cimport PortfolioFacade cdef class ExecAlgorithm(Actor): - cdef dict _exec_spawn_ids - cdef set _subscribed_strategies + cdef dict[ClientOrderId, int] _exec_spawn_ids + cdef set[StrategyId] _subscribed_strategies # -- REGISTRATION --------------------------------------------------------------------------------- diff --git a/nautilus_trader/execution/emulator.pxd b/nautilus_trader/execution/emulator.pxd index ed3c710651a5..89d2168a5f88 100644 --- a/nautilus_trader/execution/emulator.pxd +++ b/nautilus_trader/execution/emulator.pxd @@ -40,13 +40,12 @@ from nautilus_trader.model.orders.base cimport Order cdef class OrderEmulator(Actor): cdef OrderManager _manager - cdef dict _matching_cores - cdef dict _commands_submit_order + cdef dict[InstrumentId, MatchingCore] _matching_cores - cdef set _subscribed_quotes - cdef set _subscribed_trades - cdef set _subscribed_strategies - cdef set _monitored_positions + cdef set[InstrumentId] _subscribed_quotes + cdef set[InstrumentId] _subscribed_trades + cdef set[StrategyId] _subscribed_strategies + cdef set[PositionId] _monitored_positions cdef readonly bint debug """If debug mode is active (will provide extra debug logging).\n\n:returns: `bool`""" diff --git a/nautilus_trader/execution/emulator.pyx b/nautilus_trader/execution/emulator.pyx index 067eb6d709bc..c63e99439a7d 100644 --- a/nautilus_trader/execution/emulator.pyx +++ b/nautilus_trader/execution/emulator.pyx @@ -355,7 +355,7 @@ cdef class OrderEmulator(Actor): cdef Order order = command.order cdef TriggerType emulation_trigger = command.order.emulation_trigger Condition.not_equal(emulation_trigger, TriggerType.NO_TRIGGER, "command.order.emulation_trigger", "TriggerType.NO_TRIGGER") - Condition.not_in(command.order.client_order_id, self._manager.get_submit_order_commands(), "command.order.client_order_id", "self._commands_submit_order") + Condition.not_in(command.order.client_order_id, self._manager.get_submit_order_commands(), "command.order.client_order_id", "manager.submit_order_commands") if emulation_trigger not in SUPPORTED_TRIGGERS: self._log.error( From 72de1ff14194ad31a601e115a393af462f5e46ab Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 10:15:39 +1100 Subject: [PATCH 072/104] Fix backtest clock iteration --- RELEASES.md | 1 + nautilus_trader/backtest/engine.pxd | 9 ++-- nautilus_trader/backtest/engine.pyx | 22 ++++------ nautilus_trader/common/component.pxd | 7 +++ nautilus_trader/common/component.pyx | 45 +++++++++++++++++++- nautilus_trader/system/kernel.py | 4 ++ nautilus_trader/trading/controller.py | 2 +- nautilus_trader/trading/trader.py | 42 ++++++++++++++++-- tests/integration_tests/adapters/conftest.py | 8 ++++ tests/unit_tests/trading/test_trader.py | 2 + 10 files changed, 119 insertions(+), 23 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index f39bfed828e8..1ad8abd3b5c0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -21,6 +21,7 @@ None - Fixed logging `print_config` config option (was not being passed through to the logging system) - Fixed logging timestamps for backtesting (static clock was not being incrementally set to individual `TimeEvent` timestamps) - Fixed account balance updates (fills from zero quantity `NETTING` positions will generate account balance updates) +- Fixed `Controller` registration of components to ensure all active clocks are iterated correctly during backtests - Fixed `Equity` short selling for `CASH` accounts (will now reject) - Fixed `ActorFactory.create` JSON encoding (was missing the encoding hook) - Fixed `ImportableConfig.create` JSON encoding (was missing the encoding hook) diff --git a/nautilus_trader/backtest/engine.pxd b/nautilus_trader/backtest/engine.pxd index 0aebade786e5..ac2316b96acb 100644 --- a/nautilus_trader/backtest/engine.pxd +++ b/nautilus_trader/backtest/engine.pxd @@ -16,6 +16,7 @@ from cpython.datetime cimport datetime from libc.stdint cimport uint64_t +from nautilus_trader.backtest.exchange cimport SimulatedExchange from nautilus_trader.common.component cimport Clock from nautilus_trader.common.component cimport Logger from nautilus_trader.core.data cimport Data @@ -32,6 +33,7 @@ cdef class BacktestEngine: cdef TimeEventAccumulatorAPI _accumulator cdef object _kernel + cdef UUID4 _instance_id cdef DataEngine _data_engine cdef str _run_config_id cdef UUID4 _run_id @@ -40,18 +42,17 @@ cdef class BacktestEngine: cdef datetime _backtest_start cdef datetime _backtest_end - cdef dict _venues - cdef list _data + cdef dict[Venue, SimulatedExchange] _venues + cdef list[Data] _data cdef uint64_t _data_len cdef uint64_t _index cdef uint64_t _iteration cdef Data _next(self) - cdef CVec _advance_time(self, uint64_t ts_now, list clocks) + cdef CVec _advance_time(self, uint64_t ts_now) cdef void _process_raw_time_event_handlers( self, CVec raw_handlers, - list clocks, uint64_t ts_now, bint only_now, ) diff --git a/nautilus_trader/backtest/engine.pyx b/nautilus_trader/backtest/engine.pyx index 619887374d14..8c8530511b47 100644 --- a/nautilus_trader/backtest/engine.pyx +++ b/nautilus_trader/backtest/engine.pyx @@ -48,6 +48,7 @@ from nautilus_trader.common.component cimport Logger from nautilus_trader.common.component cimport TestClock from nautilus_trader.common.component cimport TimeEvent from nautilus_trader.common.component cimport TimeEventHandler +from nautilus_trader.common.component cimport get_component_clocks from nautilus_trader.common.component cimport log_level_from_str from nautilus_trader.common.component cimport log_sysinfo from nautilus_trader.common.component cimport set_logging_clock_realtime_mode @@ -137,6 +138,7 @@ cdef class BacktestEngine: # Build core system kernel self._kernel = NautilusKernel(name=type(self).__name__, config=config) + self._instance_id = self._kernel.instance_id self._log = Logger(type(self).__name__) self._data_engine: DataEngine = self._kernel.data_engine @@ -1009,15 +1011,9 @@ cdef class BacktestEngine: Condition.true(start_ns < end_ns, "start was >= end") Condition.not_empty(self._data, "data") - # Gather clocks - cdef list clocks = [self.kernel.clock] - cdef Actor actor - for actor in self._kernel.trader.actors() + self._kernel.trader.strategies() + self._kernel.trader.exec_algorithms(): - clocks.append(actor.clock) - # Set clocks cdef TestClock clock - for clock in clocks: + for clock in get_component_clocks(self._instance_id): clock.set_time(start_ns) cdef SimulatedExchange exchange @@ -1079,7 +1075,7 @@ cdef class BacktestEngine: break if data.ts_init > last_ns: # Advance clocks to the next data time - raw_handlers = self._advance_time(data.ts_init, clocks) + raw_handlers = self._advance_time(data.ts_init) raw_handlers_count = raw_handlers.len # Process data through venue @@ -1117,7 +1113,6 @@ cdef class BacktestEngine: # Finally process the time events self._process_raw_time_event_handlers( raw_handlers, - clocks, last_ns, only_now=True, ) @@ -1143,7 +1138,6 @@ cdef class BacktestEngine: if raw_handlers_count > 0: self._process_raw_time_event_handlers( raw_handlers, - clocks, last_ns, only_now=True, ) @@ -1155,7 +1149,9 @@ cdef class BacktestEngine: if cursor < self._data_len: return self._data[cursor] - cdef CVec _advance_time(self, uint64_t ts_now, list clocks): + cdef CVec _advance_time(self, uint64_t ts_now): + cdef list[TestClock] clocks = get_component_clocks(self._instance_id) + cdef TestClock clock for clock in clocks: time_event_accumulator_advance_clock( @@ -1170,7 +1166,6 @@ cdef class BacktestEngine: # Handle all events prior to the `ts_now` self._process_raw_time_event_handlers( raw_handlers, - clocks, ts_now, only_now=False, ) @@ -1186,7 +1181,6 @@ cdef class BacktestEngine: cdef void _process_raw_time_event_handlers( self, CVec raw_handler_vec, - list clocks, uint64_t ts_now, bint only_now, ): @@ -1208,7 +1202,7 @@ cdef class BacktestEngine: # Set all clocks to event timestamp set_logging_clock_static_time(ts_event_init) - for clock in clocks: + for clock in get_component_clocks(self._instance_id): clock.set_time(ts_event_init) event = TimeEvent.from_mem_c(raw_handler.event) diff --git a/nautilus_trader/common/component.pxd b/nautilus_trader/common/component.pxd index 315803cf106c..8fa42b9fbdd0 100644 --- a/nautilus_trader/common/component.pxd +++ b/nautilus_trader/common/component.pxd @@ -80,6 +80,13 @@ cdef class Clock: cpdef void cancel_timers(self) +cdef dict[UUID4, Clock] _COMPONENT_CLOCKS + +cdef list[TestClock] get_component_clocks(UUID4 instance_id) +cpdef void register_component_clock(UUID4 instance_id, Clock clock) +cpdef void deregister_component_clock(UUID4 instance_id, Clock clock) + + cdef class TestClock(Clock): cdef TestClock_API _mem diff --git a/nautilus_trader/common/component.pyx b/nautilus_trader/common/component.pyx index 53ec398f612e..0f2ba5027ed9 100644 --- a/nautilus_trader/common/component.pyx +++ b/nautilus_trader/common/component.pyx @@ -496,6 +496,49 @@ cdef class Clock: raise NotImplementedError("method `cancel_timers` must be implemented in the subclass") # pragma: no cover +# Global map of clocks per kernel instance used when running a `BacktestEngine` +_COMPONENT_CLOCKS = {} + + +cdef list[TestClock] get_component_clocks(UUID4 instance_id): + # Create a shallow copy of the clocks list, in case a new + # clock is registered during iteration. + return _COMPONENT_CLOCKS[instance_id].copy() + + +cpdef void register_component_clock(UUID4 instance_id, Clock clock): + Condition.not_none(instance_id, "instance_id") + Condition.not_none(clock, "clock") + + cdef list[Clock] clocks = _COMPONENT_CLOCKS.get(instance_id) + + if clocks is None: + clocks = [] + _COMPONENT_CLOCKS[instance_id] = clocks + + if clock not in clocks: + clocks.append(clock) + + +cpdef void deregister_component_clock(UUID4 instance_id, Clock clock): + Condition.not_none(instance_id, "instance_id") + Condition.not_none(clock, "clock") + + cdef list[Clock] clocks = _COMPONENT_CLOCKS.get(instance_id) + + if clocks is None: + return + + if clock in clocks: + clocks.remove(clock) + + +cpdef void remove_instance_component_clocks(UUID4 instance_id): + Condition.not_none(instance_id, "instance_id") + + _COMPONENT_CLOCKS.pop(instance_id, None) + + cdef class TestClock(Clock): """ Provides a monotonic clock for backtesting and unit testing. @@ -1271,7 +1314,7 @@ cpdef str component_trigger_to_str(ComponentTrigger value): return cstr_to_pystr(component_trigger_to_cstr(value)) -cdef dict _COMPONENT_STATE_TABLE = { +cdef dict[tuple[ComponentState, ComponentTrigger], ComponentState] _COMPONENT_STATE_TABLE = { (ComponentState.PRE_INITIALIZED, ComponentTrigger.INITIALIZE): ComponentState.READY, (ComponentState.READY, ComponentTrigger.RESET): ComponentState.RESETTING, # Transitional state (ComponentState.READY, ComponentTrigger.START): ComponentState.STARTING, # Transitional state diff --git a/nautilus_trader/system/kernel.py b/nautilus_trader/system/kernel.py index f546efcdfe1c..48cc8d222972 100644 --- a/nautilus_trader/system/kernel.py +++ b/nautilus_trader/system/kernel.py @@ -39,6 +39,7 @@ from nautilus_trader.common.component import init_tracing from nautilus_trader.common.component import is_logging_initialized from nautilus_trader.common.component import log_header +from nautilus_trader.common.component import register_component_clock from nautilus_trader.common.config import InvalidConfiguration from nautilus_trader.common.enums import LogColor from nautilus_trader.common.enums import LogLevel @@ -153,6 +154,8 @@ def __init__( # noqa (too complex) f"environment {self._environment} not recognized", # pragma: no cover (design-time error) ) + register_component_clock(self._instance_id, self._clock) + # Setup logging logging: LoggingConfig = config.logging or LoggingConfig() @@ -382,6 +385,7 @@ def __init__( # noqa (too complex) ######################################################################## self._trader = Trader( trader_id=self._trader_id, + instance_id=self._instance_id, msgbus=self._msgbus, cache=self._cache, portfolio=self._portfolio, diff --git a/nautilus_trader/trading/controller.py b/nautilus_trader/trading/controller.py index 76673b8b0629..841c2361890a 100644 --- a/nautilus_trader/trading/controller.py +++ b/nautilus_trader/trading/controller.py @@ -29,7 +29,7 @@ class Controller(Actor): trader : Trader The reference to the trader instance to control. config : ActorConfig, optional - The configuratuon for the controller + The configuration for the controller Raises ------ diff --git a/nautilus_trader/trading/trader.py b/nautilus_trader/trading/trader.py index 4b4194ba1af4..fdc31820465d 100644 --- a/nautilus_trader/trading/trader.py +++ b/nautilus_trader/trading/trader.py @@ -33,7 +33,11 @@ from nautilus_trader.common.component import Clock from nautilus_trader.common.component import Component from nautilus_trader.common.component import MessageBus +from nautilus_trader.common.component import deregister_component_clock +from nautilus_trader.common.component import register_component_clock +from nautilus_trader.common.component import remove_instance_component_clocks from nautilus_trader.core.correctness import PyCondition +from nautilus_trader.core.uuid import UUID4 from nautilus_trader.data.engine import DataEngine from nautilus_trader.execution.algorithm import ExecAlgorithm from nautilus_trader.execution.engine import ExecutionEngine @@ -56,6 +60,8 @@ class Trader(Component): ---------- trader_id : TraderId The ID for the trader. + instance_id : UUID4 + The instance ID for the trader. msgbus : MessageBus The message bus for the trader. cache : Cache @@ -91,6 +97,7 @@ class Trader(Component): def __init__( self, trader_id: TraderId, + instance_id: UUID4, msgbus: MessageBus, cache: Cache, portfolio: Portfolio, @@ -107,6 +114,7 @@ def __init__( msgbus=msgbus, ) + self._instance_id = instance_id self._loop = loop self._cache = cache self._portfolio = portfolio @@ -119,6 +127,18 @@ def __init__( self._exec_algorithms: dict[ExecAlgorithmId, ExecAlgorithm] = {} self._has_controller: bool = has_controller + @property + def instance_id(self) -> UUID4: + """ + Return the traders instance ID. + + Returns + ------- + UUID4 + + """ + return self._instance_id + def actors(self) -> list[Actor]: """ Return the actors loaded in the trader. @@ -266,6 +286,8 @@ def _dispose(self) -> None: self.clear_strategies() self.clear_exec_algorithms() + remove_instance_component_clocks(self._instance_id) + # -------------------------------------------------------------------------------------------------- def add_actor(self, actor: Actor) -> None: @@ -298,12 +320,15 @@ def add_actor(self, actor: Actor) -> None: "try specifying a different actor ID.", ) + clock = self._clock.__class__() # Clock per component + register_component_clock(self._instance_id, clock) + # Wire component into trader actor.register_base( portfolio=self._portfolio, msgbus=self._msgbus, cache=self._cache, - clock=self._clock.__class__(), # Clock per component + clock=clock, ) self._actors[actor.id] = actor @@ -377,13 +402,16 @@ def add_strategy(self, strategy: Strategy) -> None: f"explicitly define all `order_id_tag` values in your strategy configs", ) + clock = self._clock.__class__() # Clock per component + register_component_clock(self._instance_id, clock) + # Wire strategy into trader strategy.register( trader_id=self.id, portfolio=self._portfolio, msgbus=self._msgbus, cache=self._cache, - clock=self._clock.__class__(), # Clock per strategy + clock=clock, ) self._exec_engine.register_oms_type(strategy) @@ -443,13 +471,16 @@ def add_exec_algorithm(self, exec_algorithm: ExecAlgorithm) -> None: "try specifying a different `exec_algorithm_id`.", ) + clock = self._clock.__class__() # Clock per component + register_component_clock(self._instance_id, clock) + # Wire execution algorithm into trader exec_algorithm.register( trader_id=self.id, portfolio=self._portfolio, msgbus=self._msgbus, cache=self._cache, - clock=self._clock.__class__(), # Clock per algorithm + clock=clock, ) self._exec_algorithms[exec_algorithm.id] = exec_algorithm @@ -611,6 +642,7 @@ def remove_actor(self, actor_id: ComponentId) -> None: actor.stop() self._actors.pop(actor_id) + deregister_component_clock(self._instance_id, actor.clock) def remove_strategy(self, strategy_id: StrategyId) -> None: """ @@ -639,6 +671,7 @@ def remove_strategy(self, strategy_id: StrategyId) -> None: strategy.stop() self._strategies.pop(strategy_id) + deregister_component_clock(self._instance_id, strategy.clock) def clear_actors(self) -> None: """ @@ -656,6 +689,7 @@ def clear_actors(self) -> None: for actor in self._actors.values(): actor.dispose() + deregister_component_clock(self._instance_id, actor.clock) self._actors.clear() self._log.info("Cleared all actors.") @@ -676,6 +710,7 @@ def clear_strategies(self) -> None: for strategy in self._strategies.values(): strategy.dispose() + deregister_component_clock(self._instance_id, strategy.clock) self._strategies.clear() self._log.info("Cleared all trading strategies.") @@ -696,6 +731,7 @@ def clear_exec_algorithms(self) -> None: for exec_algorithm in self._exec_algorithms.values(): exec_algorithm.dispose() + deregister_component_clock(self._instance_id, exec_algorithm.clock) self._exec_algorithms.clear() self._log.info("Cleared all execution algorithms.") diff --git a/tests/integration_tests/adapters/conftest.py b/tests/integration_tests/adapters/conftest.py index cf0f0175d19c..21d63bcf1818 100644 --- a/tests/integration_tests/adapters/conftest.py +++ b/tests/integration_tests/adapters/conftest.py @@ -23,6 +23,7 @@ from nautilus_trader.common.component import MessageBus from nautilus_trader.common.component import TestClock from nautilus_trader.core.message import Event +from nautilus_trader.core.uuid import UUID4 from nautilus_trader.data.engine import DataEngine from nautilus_trader.execution.engine import ExecutionEngine from nautilus_trader.live.execution_engine import LiveExecutionEngine @@ -71,6 +72,11 @@ def trader_id(): return TestIdStubs.trader_id() +@pytest.fixture() +def instance_id(): + return UUID4() + + @pytest.fixture() def msgbus(trader_id, clock): return MessageBus( @@ -145,6 +151,7 @@ def risk_engine(portfolio, msgbus, cache, clock): @pytest.fixture(autouse=True) def trader( trader_id, + instance_id, msgbus, cache, portfolio, @@ -156,6 +163,7 @@ def trader( ): return Trader( trader_id=trader_id, + instance_id=instance_id, msgbus=msgbus, cache=cache, portfolio=portfolio, diff --git a/tests/unit_tests/trading/test_trader.py b/tests/unit_tests/trading/test_trader.py index 995235d68bc0..dbf52ba7e0f7 100644 --- a/tests/unit_tests/trading/test_trader.py +++ b/tests/unit_tests/trading/test_trader.py @@ -28,6 +28,7 @@ from nautilus_trader.config import ActorConfig from nautilus_trader.config import ExecAlgorithmConfig from nautilus_trader.config import StrategyConfig +from nautilus_trader.core.uuid import UUID4 from nautilus_trader.data.engine import DataEngine from nautilus_trader.examples.strategies.blank import MyStrategy from nautilus_trader.examples.strategies.blank import MyStrategyConfig @@ -132,6 +133,7 @@ def setup(self) -> None: self.trader = Trader( trader_id=self.trader_id, + instance_id=UUID4(), msgbus=self.msgbus, cache=self.cache, portfolio=self.portfolio, From e4a3b39a62b5b2a03f7c37dcdaf02841e2565a03 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 10:21:50 +1100 Subject: [PATCH 073/104] Fix MessageBus publishable_types collection type --- RELEASES.md | 1 + nautilus_trader/common/component.pxd | 2 +- nautilus_trader/common/component.pyx | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 1ad8abd3b5c0..4e59414f5034 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -21,6 +21,7 @@ None - Fixed logging `print_config` config option (was not being passed through to the logging system) - Fixed logging timestamps for backtesting (static clock was not being incrementally set to individual `TimeEvent` timestamps) - Fixed account balance updates (fills from zero quantity `NETTING` positions will generate account balance updates) +- Fixed `MessageBus` publishable types collection type (needed to be `tuple` not `set`) - Fixed `Controller` registration of components to ensure all active clocks are iterated correctly during backtests - Fixed `Equity` short selling for `CASH` accounts (will now reject) - Fixed `ActorFactory.create` JSON encoding (was missing the encoding hook) diff --git a/nautilus_trader/common/component.pxd b/nautilus_trader/common/component.pxd index 8fa42b9fbdd0..62ef040d17c4 100644 --- a/nautilus_trader/common/component.pxd +++ b/nautilus_trader/common/component.pxd @@ -252,7 +252,7 @@ cdef class MessageBus: cdef dict[str, object] _endpoints cdef dict[UUID4, object] _correlation_index cdef bint _has_backing - cdef set[type] _publishable_types + cdef tuple[type] _publishable_types cdef readonly TraderId trader_id """The trader ID associated with the bus.\n\n:returns: `TraderId`""" diff --git a/nautilus_trader/common/component.pyx b/nautilus_trader/common/component.pyx index 0f2ba5027ed9..a6a3a2227d63 100644 --- a/nautilus_trader/common/component.pyx +++ b/nautilus_trader/common/component.pyx @@ -2004,7 +2004,7 @@ cdef class MessageBus: self._subscriptions: dict[Subscription, list[str]] = {} self._correlation_index: dict[UUID4, Callable[[Any], None]] = {} self._has_backing = config.database is not None - self._publishable_types = _EXTERNAL_PUBLISHABLE_TYPES + self._publishable_types = tuple(_EXTERNAL_PUBLISHABLE_TYPES) if types_filter is not None: self._publishable_types = tuple(o for o in _EXTERNAL_PUBLISHABLE_TYPES if o not in types_filter) From 25e4ae17f352925f833fcab46fae796ec9169a04 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 10:35:37 +1100 Subject: [PATCH 074/104] Standardize field ordering --- .../src/databento/python/historical.rs | 4 +-- nautilus_core/common/src/logging/mod.rs | 4 +-- nautilus_core/common/src/logging/writer.rs | 8 ++--- nautilus_core/common/src/msgbus.rs | 30 +++++++++---------- .../model/src/orders/limit_if_touched.rs | 2 +- .../model/src/orders/market_if_touched.rs | 2 +- nautilus_core/model/src/orders/stop_limit.rs | 2 +- nautilus_core/model/src/orders/stop_market.rs | 2 +- .../persistence/src/backend/session.rs | 4 +-- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/nautilus_core/adapters/src/databento/python/historical.rs b/nautilus_core/adapters/src/databento/python/historical.rs index 339865b45bd1..74d8a9a91ef2 100644 --- a/nautilus_core/adapters/src/databento/python/historical.rs +++ b/nautilus_core/adapters/src/databento/python/historical.rs @@ -48,11 +48,11 @@ use super::loader::convert_instrument_to_pyobject; pyclass(module = "nautilus_trader.core.nautilus_pyo3.databento") )] pub struct DatabentoHistoricalClient { + #[pyo3(get)] + pub key: String, clock: &'static AtomicTime, inner: Arc>, publisher_venue_map: Arc>, - #[pyo3(get)] - pub key: String, } #[pymethods] diff --git a/nautilus_core/common/src/logging/mod.rs b/nautilus_core/common/src/logging/mod.rs index ede184a6fe6b..2c8b5d4b639f 100644 --- a/nautilus_core/common/src/logging/mod.rs +++ b/nautilus_core/common/src/logging/mod.rs @@ -272,10 +272,10 @@ pub fn init_logging( /// channel. #[derive(Debug)] pub struct Logger { - /// Send log events to a different thread. - tx: Sender, /// Configure maximum levels for components and IO. pub config: LoggerConfig, + /// Send log events to a different thread. + tx: Sender, } /// Represents a type of log event. diff --git a/nautilus_core/common/src/logging/writer.rs b/nautilus_core/common/src/logging/writer.rs index 865af38cb27c..c9f555d70b8c 100644 --- a/nautilus_core/common/src/logging/writer.rs +++ b/nautilus_core/common/src/logging/writer.rs @@ -35,9 +35,9 @@ pub trait LogWriter { #[derive(Debug)] pub struct StdoutWriter { + pub is_colored: bool, buf: BufWriter, level: LevelFilter, - pub is_colored: bool, } impl StdoutWriter { @@ -72,8 +72,8 @@ impl LogWriter for StdoutWriter { #[derive(Debug)] pub struct StderrWriter { - buf: BufWriter, pub is_colored: bool, + buf: BufWriter, } impl StderrWriter { @@ -132,12 +132,12 @@ impl FileWriterConfig { #[derive(Debug)] pub struct FileWriter { + pub json_format: bool, buf: BufWriter, path: PathBuf, file_config: FileWriterConfig, trader_id: String, instance_id: String, - pub json_format: bool, level: LevelFilter, } @@ -169,12 +169,12 @@ impl FileWriter { .open(file_path.clone()) { Ok(file) => Some(Self { + json_format, buf: BufWriter::new(file), path: file_path, file_config, trader_id, instance_id, - json_format, level: fileout_level, }), Err(e) => { diff --git a/nautilus_core/common/src/msgbus.rs b/nautilus_core/common/src/msgbus.rs index 410513a8b351..fb206e759344 100644 --- a/nautilus_core/common/src/msgbus.rs +++ b/nautilus_core/common/src/msgbus.rs @@ -131,21 +131,6 @@ impl fmt::Display for BusMessage { /// For example, `c??p` would match both of the above examples and `coop`. #[derive(Clone)] pub struct MessageBus { - tx: Option>, - /// mapping from topic to the corresponding handler - /// a topic can be a string with wildcards - /// * '?' - any character - /// * '*' - any number of any characters - subscriptions: IndexMap>, - /// maps a pattern to all the handlers registered for it - /// this is updated whenever a new subscription is created. - patterns: IndexMap>, - /// handles a message or a request destined for a specific endpoint. - endpoints: IndexMap, - /// Relates a request with a response - /// a request maps it's id to a handler so that a response - /// with the same id can later be handled. - correlation_index: IndexMap, /// The trader ID associated with the message bus. pub trader_id: TraderId, /// The instance ID associated with the message bus. @@ -162,6 +147,21 @@ pub struct MessageBus { pub pub_count: u64, /// If the message bus is backed by a database. pub has_backing: bool, + tx: Option>, + /// mapping from topic to the corresponding handler + /// a topic can be a string with wildcards + /// * '?' - any character + /// * '*' - any number of any characters + subscriptions: IndexMap>, + /// maps a pattern to all the handlers registered for it + /// this is updated whenever a new subscription is created. + patterns: IndexMap>, + /// handles a message or a request destined for a specific endpoint. + endpoints: IndexMap, + /// Relates a request with a response + /// a request maps it's id to a handler so that a response + /// with the same id can later be handled. + correlation_index: IndexMap, } impl MessageBus { diff --git a/nautilus_core/model/src/orders/limit_if_touched.rs b/nautilus_core/model/src/orders/limit_if_touched.rs index 99270b12e011..4dc645509b31 100644 --- a/nautilus_core/model/src/orders/limit_if_touched.rs +++ b/nautilus_core/model/src/orders/limit_if_touched.rs @@ -43,7 +43,6 @@ use crate::{ pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct LimitIfTouchedOrder { - core: OrderCore, pub price: Price, pub trigger_price: Price, pub trigger_type: TriggerType, @@ -53,6 +52,7 @@ pub struct LimitIfTouchedOrder { pub trigger_instrument_id: Option, pub is_triggered: bool, pub ts_triggered: Option, + core: OrderCore, } impl LimitIfTouchedOrder { diff --git a/nautilus_core/model/src/orders/market_if_touched.rs b/nautilus_core/model/src/orders/market_if_touched.rs index 99d7ad431d06..71fa483d2d51 100644 --- a/nautilus_core/model/src/orders/market_if_touched.rs +++ b/nautilus_core/model/src/orders/market_if_touched.rs @@ -43,7 +43,6 @@ use crate::{ pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct MarketIfTouchedOrder { - core: OrderCore, pub trigger_price: Price, pub trigger_type: TriggerType, pub expire_time: Option, @@ -51,6 +50,7 @@ pub struct MarketIfTouchedOrder { pub trigger_instrument_id: Option, pub is_triggered: bool, pub ts_triggered: Option, + core: OrderCore, } impl MarketIfTouchedOrder { diff --git a/nautilus_core/model/src/orders/stop_limit.rs b/nautilus_core/model/src/orders/stop_limit.rs index 024e887d82a4..c841c784505c 100644 --- a/nautilus_core/model/src/orders/stop_limit.rs +++ b/nautilus_core/model/src/orders/stop_limit.rs @@ -43,7 +43,6 @@ use crate::{ pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct StopLimitOrder { - core: OrderCore, pub price: Price, pub trigger_price: Price, pub trigger_type: TriggerType, @@ -53,6 +52,7 @@ pub struct StopLimitOrder { pub trigger_instrument_id: Option, pub is_triggered: bool, pub ts_triggered: Option, + core: OrderCore, } impl StopLimitOrder { diff --git a/nautilus_core/model/src/orders/stop_market.rs b/nautilus_core/model/src/orders/stop_market.rs index c4f257a1763d..962881300382 100644 --- a/nautilus_core/model/src/orders/stop_market.rs +++ b/nautilus_core/model/src/orders/stop_market.rs @@ -44,7 +44,6 @@ use crate::{ pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct StopMarketOrder { - core: OrderCore, pub trigger_price: Price, pub trigger_type: TriggerType, pub expire_time: Option, @@ -52,6 +51,7 @@ pub struct StopMarketOrder { pub trigger_instrument_id: Option, pub is_triggered: bool, pub ts_triggered: Option, + core: OrderCore, } impl StopMarketOrder { diff --git a/nautilus_core/persistence/src/backend/session.rs b/nautilus_core/persistence/src/backend/session.rs index 296c86e9a36d..b1a9ac88bff2 100644 --- a/nautilus_core/persistence/src/backend/session.rs +++ b/nautilus_core/persistence/src/backend/session.rs @@ -61,10 +61,10 @@ pub type QueryResult = KMerge>, Data, TsIni pyclass(module = "nautilus_trader.core.nautilus_pyo3.persistence") )] pub struct DataBackendSession { - session_ctx: SessionContext, - batch_streams: Vec>>, pub chunk_size: usize, pub runtime: Arc, + session_ctx: SessionContext, + batch_streams: Vec>>, } impl DataBackendSession { From 4bc817dd04c2be75f06f395c6d2fba941da86e9c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 11:10:47 +1100 Subject: [PATCH 075/104] Add OrderBookMbo and Level pyo3 interfaces --- nautilus_core/model/src/orderbook/book.rs | 4 + nautilus_core/model/src/orderbook/book_mbo.rs | 7 +- nautilus_core/model/src/orderbook/book_mbp.rs | 9 +- nautilus_core/model/src/orderbook/level.rs | 66 +++--- .../model/src/python/orderbook/book_mbo.rs | 195 ++++++++++++++++++ .../model/src/python/orderbook/level.rs | 76 +++++++ .../model/src/python/orderbook/mod.rs | 1 + nautilus_trader/core/includes/model.h | 6 + nautilus_trader/core/nautilus_pyo3.pyi | 55 ++++- nautilus_trader/core/rust/model.pxd | 4 + 10 files changed, 393 insertions(+), 30 deletions(-) create mode 100644 nautilus_core/model/src/python/orderbook/level.rs diff --git a/nautilus_core/model/src/orderbook/book.rs b/nautilus_core/model/src/orderbook/book.rs index 1f4e14b0455c..5ad44f4bb405 100644 --- a/nautilus_core/model/src/orderbook/book.rs +++ b/nautilus_core/model/src/orderbook/book.rs @@ -45,6 +45,8 @@ pub enum BookIntegrityError { TooManyLevels(OrderSide, usize), } +/// Calculates the estimated average price for a specified quantity from a set of +/// order book levels. pub fn get_avg_px_for_quantity(qty: Quantity, levels: &BTreeMap) -> f64 { let mut cumulative_size_raw = 0u64; let mut cumulative_value = 0.0; @@ -66,6 +68,8 @@ pub fn get_avg_px_for_quantity(qty: Quantity, levels: &BTreeMap Vec { + self.insertion_order + .iter() + .filter_map(|id| self.orders.get(id)) + .cloned() + .collect() + } + + #[must_use] + pub fn size(&self) -> f64 { + self.orders.values().map(|o| o.size.as_f64()).sum() + } + + #[must_use] + pub fn size_raw(&self) -> u64 { + self.orders.values().map(|o| o.size.raw).sum() + } + + #[must_use] + pub fn exposure(&self) -> f64 { + self.orders + .values() + .map(|o| o.price.as_f64() * o.size.as_f64()) + .sum() + } + + #[must_use] + pub fn exposure_raw(&self) -> u64 { + self.orders + .values() + .map(|o| ((o.price.as_f64() * o.size.as_f64()) * FIXED_SCALAR) as u64) + .sum() + } + pub fn add_bulk(&mut self, orders: Vec) { self.insertion_order .extend(orders.iter().map(|o| o.order_id)); @@ -113,32 +153,6 @@ impl Level { self.update_insertion_order(); } - #[must_use] - pub fn size(&self) -> f64 { - self.orders.values().map(|o| o.size.as_f64()).sum() - } - - #[must_use] - pub fn size_raw(&self) -> u64 { - self.orders.values().map(|o| o.size.raw).sum() - } - - #[must_use] - pub fn exposure(&self) -> f64 { - self.orders - .values() - .map(|o| o.price.as_f64() * o.size.as_f64()) - .sum() - } - - #[must_use] - pub fn exposure_raw(&self) -> u64 { - self.orders - .values() - .map(|o| ((o.price.as_f64() * o.size.as_f64()) * FIXED_SCALAR) as u64) - .sum() - } - fn check_order_for_this_level(&self, order: &BookOrder) { assert_eq!(order.price, self.price.value); } diff --git a/nautilus_core/model/src/python/orderbook/book_mbo.rs b/nautilus_core/model/src/python/orderbook/book_mbo.rs index 97d459d8d1e8..0887658bda9c 100644 --- a/nautilus_core/model/src/python/orderbook/book_mbo.rs +++ b/nautilus_core/model/src/python/orderbook/book_mbo.rs @@ -12,3 +12,198 @@ // See the License for the specific language governing permissions and // limitations under the License. // ------------------------------------------------------------------------------------------------- + +use nautilus_core::{python::to_pyruntime_err, time::UnixNanos}; +use pyo3::prelude::*; + +use crate::{ + data::{ + delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, order::BookOrder, + }, + enums::{BookType, OrderSide}, + identifiers::instrument_id::InstrumentId, + orderbook::{book_mbo::OrderBookMbo, level::Level}, + types::{price::Price, quantity::Quantity}, +}; + +#[pymethods] +impl OrderBookMbo { + #[new] + fn py_new(instrument_id: InstrumentId) -> Self { + Self::new(instrument_id) + } + + fn __str__(&self) -> String { + // TODO: Return debug string for now + format!("{self:?}") + } + + fn __repr__(&self) -> String { + format!("{self:?}") + } + + #[getter] + #[pyo3(name = "instrument_id")] + fn py_instrument_id(&self) -> InstrumentId { + self.instrument_id + } + + #[getter] + #[pyo3(name = "book_type")] + fn py_book_type(&self) -> BookType { + BookType::L3_MBO + } + + #[getter] + #[pyo3(name = "sequence")] + fn py_sequence(&self) -> u64 { + self.sequence + } + + #[getter] + #[pyo3(name = "ts_event")] + fn py_ts_event(&self) -> UnixNanos { + self.ts_last + } + + #[getter] + #[pyo3(name = "ts_init")] + fn py_ts_init(&self) -> UnixNanos { + self.ts_last + } + + #[getter] + #[pyo3(name = "ts_last")] + fn py_ts_last(&self) -> UnixNanos { + self.ts_last + } + + #[getter] + #[pyo3(name = "count")] + fn py_count(&self) -> u64 { + self.count + } + + #[pyo3(name = "reset")] + fn py_reset(&mut self) { + self.reset() + } + + #[pyo3(signature = (order, ts_event, sequence=0))] + #[pyo3(name = "update")] + fn py_update(&mut self, order: BookOrder, ts_event: UnixNanos, sequence: u64) { + self.update(order, ts_event, sequence); + } + + #[pyo3(signature = (order, ts_event, sequence=0))] + #[pyo3(name = "delete")] + fn py_delete(&mut self, order: BookOrder, ts_event: UnixNanos, sequence: u64) { + self.delete(order, ts_event, sequence); + } + + #[pyo3(signature = (ts_event, sequence=0))] + #[pyo3(name = "clear")] + fn py_clear(&mut self, ts_event: UnixNanos, sequence: u64) { + self.clear(ts_event, sequence); + } + + #[pyo3(signature = (ts_event, sequence=0))] + #[pyo3(name = "clear_bids")] + fn py_clear_bids(&mut self, ts_event: UnixNanos, sequence: u64) { + self.clear_bids(ts_event, sequence); + } + + #[pyo3(signature = (ts_event, sequence=0))] + #[pyo3(name = "clear_asks")] + fn py_clear_asks(&mut self, ts_event: UnixNanos, sequence: u64) { + self.clear_asks(ts_event, sequence); + } + + #[pyo3(name = "apply_delta")] + fn py_apply_delta(&mut self, delta: OrderBookDelta) { + self.apply_delta(delta); + } + + #[pyo3(name = "apply_deltas")] + fn py_apply_deltas(&mut self, deltas: OrderBookDeltas) { + self.apply_deltas(deltas); + } + + #[pyo3(name = "apply_depth")] + fn py_apply_depth(&mut self, depth: OrderBookDepth10) { + self.apply_depth(depth); + } + + #[pyo3(name = "check_integrity")] + fn py_check_integrity(&mut self) -> PyResult<()> { + self.check_integrity().map_err(to_pyruntime_err) + } + + #[pyo3(name = "bids")] + fn py_bids(&self) -> Vec { + // TODO: Improve efficiency + self.bids() + .iter() + .map(|level_ref| (*level_ref).clone()) + .collect() + } + + #[pyo3(name = "asks")] + fn py_asks(&self) -> Vec { + // TODO: Improve efficiency + self.asks() + .iter() + .map(|level_ref| (*level_ref).clone()) + .collect() + } + + #[pyo3(name = "best_bid_price")] + fn py_best_bid_price(&self) -> Option { + self.best_bid_price() + } + + #[pyo3(name = "best_ask_price")] + fn py_best_ask_price(&self) -> Option { + self.best_ask_price() + } + + #[pyo3(name = "best_bid_size")] + fn py_best_bid_size(&self) -> Option { + self.best_bid_size() + } + + #[pyo3(name = "best_ask_size")] + fn py_best_ask_size(&self) -> Option { + self.best_ask_size() + } + + #[pyo3(name = "spread")] + fn py_spread(&self) -> Option { + self.spread() + } + + #[pyo3(name = "midpoint")] + fn py_midpoint(&self) -> Option { + self.midpoint() + } + + #[pyo3(name = "get_avg_px_for_quantity")] + fn py_get_avg_px_for_quantity(&self, qty: Quantity, order_side: OrderSide) -> f64 { + self.get_avg_px_for_quantity(qty, order_side) + } + + #[pyo3(name = "get_quantity_for_price")] + fn py_get_quantity_for_price(&self, price: Price, order_side: OrderSide) -> f64 { + self.get_quantity_for_price(price, order_side) + } + + #[pyo3(name = "simulate_fills")] + fn py_simulate_fills(&self, order: &BookOrder) -> Vec<(Price, Quantity)> { + self.simulate_fills(order) + } + + #[pyo3(name = "pprint")] + fn py_pprint(&self, num_levels: usize) -> String { + self.pprint(num_levels) + } +} diff --git a/nautilus_core/model/src/python/orderbook/level.rs b/nautilus_core/model/src/python/orderbook/level.rs new file mode 100644 index 000000000000..c216cb4c2f62 --- /dev/null +++ b/nautilus_core/model/src/python/orderbook/level.rs @@ -0,0 +1,76 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use pyo3::prelude::*; + +use crate::{data::order::BookOrder, orderbook::level::Level, types::price::Price}; + +#[pymethods] +impl Level { + fn __str__(&self) -> String { + // TODO: Return debug string for now + format!("{self:?}") + } + + fn __repr__(&self) -> String { + format!("{self:?}") + } + + #[getter] + #[pyo3(name = "price")] + fn py_price(&self) -> Price { + self.price.value + } + + #[pyo3(name = "len")] + fn py_len(&self) -> usize { + self.len() + } + + #[pyo3(name = "is_empty")] + fn py_is_empty(&self) -> bool { + self.is_empty() + } + + #[pyo3(name = "size")] + fn py_size(&self) -> f64 { + self.size() + } + + #[pyo3(name = "size_raw")] + fn py_size_raw(&self) -> u64 { + self.size_raw() + } + + #[pyo3(name = "exposure")] + fn py_exposure(&self) -> f64 { + self.exposure() + } + + #[pyo3(name = "exposure_raw")] + fn py_exposure_raw(&self) -> u64 { + self.exposure_raw() + } + + #[pyo3(name = "first")] + fn py_fist(&self) -> Option { + self.first().cloned() + } + + #[pyo3(name = "get_orders")] + fn py_get_orders(&self) -> Vec { + self.get_orders() + } +} diff --git a/nautilus_core/model/src/python/orderbook/mod.rs b/nautilus_core/model/src/python/orderbook/mod.rs index 7602360c877b..530754827475 100644 --- a/nautilus_core/model/src/python/orderbook/mod.rs +++ b/nautilus_core/model/src/python/orderbook/mod.rs @@ -15,3 +15,4 @@ pub mod book_mbo; pub mod book_mbp; +pub mod level; diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index 8f122afa8296..ace1b7973e6e 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -658,6 +658,12 @@ typedef enum TriggerType { INDEX_PRICE = 9, } TriggerType; +/** + * Represents a discrete price level in an order book. + * + * The level maintains a collection of orders as well as tracking insertion order + * to preserve FIFO queue dynamics. + */ typedef struct Level Level; typedef struct OrderBookContainer OrderBookContainer; diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index d1ca0c9622be..0a7af2fc8f2a 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1614,6 +1614,57 @@ class OrderExpired: def from_dict(cls, values: dict[str, str]) -> OrderExpired: ... def to_dict(self) -> dict[str, str]: ... +class Level: + @property + def price(self) -> Price: ... + def len(self) -> int: ... + def is_empty(self) -> bool: ... + def size(self) -> float: ... + def size_raw(self) -> int: ... + def exposure(self) -> float: ... + def exposure_raw(self) -> int: ... + def first(self) -> BookOrder | None: ... + def get_orders(self) -> list[BookOrder]: ... + +class OrderBookMbo: + def __init__(self, instrument_id: InstrumentId) -> None: ... + @property + def instrument_id(self) -> InstrumentId: ... + @property + def book_type(self) -> BookType: ... + @property + def sequence(self) -> int: ... + @property + def ts_event(self) -> int: ... + @property + def ts_init(self) -> int: ... + @property + def ts_last(self) -> int: ... + @property + def count(self) -> int: ... + def reset(self) -> None: ... + def update(self, order: BookOrder, ts_event: int, sequence: int = 0) -> None: ... + def delete(self, order: BookOrder, ts_event: int, sequence: int = 0) -> None: ... + def clear(self, ts_event: int, sequence: int = 0) -> None: ... + def clear_bids(self, ts_event: int, sequence: int = 0) -> None: ... + def clear_asks(self, ts_event: int, sequence: int = 0) -> None: ... + def apply_delta(self, delta: OrderBookDelta) -> None: ... + def apply_deltas(self, deltas: OrderBookDeltas) -> None: ... + def apply_depth(self, depth: OrderBookDepth10) -> None: ... + def check_integrity(self) -> None: ... + def bids(self) -> list[Level]: ... + def asks(self) -> list[Level]: ... + def best_bid_price(self) -> Price | None: ... + def best_ask_price(self) -> Price | None: ... + def best_bid_size(self) -> Quantity | None: ... + def best_ask_size(self) -> Quantity | None: ... + def spread(self) -> float | None: ... + def midpoint(self) -> float | None: ... + def get_avg_px_for_quantity(self, qty: Quantity, order_side: OrderSide) -> float: ... + def get_quantity_for_price(self, price: Price, order_side: OrderSide) -> float: ... + def simulate_fills(self, order: BookOrder) -> list[tuple[Price, Quantity]]: ... + def pprint(self, num_levels: int) -> str: ... + class OrderBookMbp: def __init__( self, @@ -1646,8 +1697,8 @@ class OrderBookMbp: def apply_deltas(self, deltas: OrderBookDeltas) -> None: ... def apply_depth(self, depth: OrderBookDepth10) -> None: ... def check_integrity(self) -> None: ... - # def bids(self) -> list[Level]: ... TBD - # def asks(self) -> list[Level]: ... TBD + def bids(self) -> list[Level]: ... + def asks(self) -> list[Level]: ... def best_bid_price(self) -> Price | None: ... def best_ask_price(self) -> Price | None: ... def best_bid_size(self) -> Quantity | None: ... diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index c47da44ee8ba..368521af4f9f 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -352,6 +352,10 @@ cdef extern from "../includes/model.h": # Based on the index price for the instrument. INDEX_PRICE # = 9, + # Represents a discrete price level in an order book. + # + # The level maintains a collection of orders as well as tracking insertion order + # to preserve FIFO queue dynamics. cdef struct Level: pass From f9e41afed089d0001aa0dbc6ca72048a57424606 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 11:36:35 +1100 Subject: [PATCH 076/104] Refine core order book impls --- .../model/src/ffi/orderbook/container.rs | 18 ++-- nautilus_core/model/src/orderbook/book.rs | 81 +-------------- nautilus_core/model/src/orderbook/book_mbo.rs | 45 ++++++++- nautilus_core/model/src/orderbook/book_mbp.rs | 98 ++++++++++++++++++- .../model/src/python/orderbook/book_mbo.rs | 10 +- .../model/src/python/orderbook/book_mbp.rs | 16 ++- 6 files changed, 155 insertions(+), 113 deletions(-) diff --git a/nautilus_core/model/src/ffi/orderbook/container.rs b/nautilus_core/model/src/ffi/orderbook/container.rs index a9f30b18e89f..57ab3d110b58 100644 --- a/nautilus_core/model/src/ffi/orderbook/container.rs +++ b/nautilus_core/model/src/ffi/orderbook/container.rs @@ -33,9 +33,9 @@ pub struct OrderBookContainer { mbp: Option, } -const L3_MBO_NOT_INITILIZED: &str = "L3 MBO book not initialized"; -const L2_MBP_NOT_INITILIZED: &str = "L2 MBP book not initialized"; -const L1_MBP_NOT_INITILIZED: &str = "L1 MBP book not initialized"; +const L3_MBO_NOT_INITILIZED: &str = "L3_MBO book not initialized"; +const L2_MBP_NOT_INITILIZED: &str = "L2_MBP book not initialized"; +const L1_MBP_NOT_INITILIZED: &str = "L1_MBP book not initialized"; impl OrderBookContainer { #[must_use] @@ -184,17 +184,17 @@ impl OrderBookContainer { pub fn bids(&self) -> Vec<&Level> { match self.book_type { - BookType::L3_MBO => self.get_mbo().bids(), - BookType::L2_MBP => self.get_mbp().bids(), - BookType::L1_MBP => self.get_mbp().bids(), + BookType::L3_MBO => self.get_mbo().bids().collect(), + BookType::L2_MBP => self.get_mbp().bids().collect(), + BookType::L1_MBP => self.get_mbp().bids().collect(), } } pub fn asks(&self) -> Vec<&Level> { match self.book_type { - BookType::L3_MBO => self.get_mbo().asks(), - BookType::L2_MBP => self.get_mbp().asks(), - BookType::L1_MBP => self.get_mbp().asks(), + BookType::L3_MBO => self.get_mbo().asks().collect(), + BookType::L2_MBP => self.get_mbp().asks().collect(), + BookType::L1_MBP => self.get_mbp().asks().collect(), } } diff --git a/nautilus_core/model/src/orderbook/book.rs b/nautilus_core/model/src/orderbook/book.rs index 5ad44f4bb405..f548aff32e22 100644 --- a/nautilus_core/model/src/orderbook/book.rs +++ b/nautilus_core/model/src/orderbook/book.rs @@ -108,41 +108,13 @@ mod tests { data::{ depth::{stubs::stub_depth10, OrderBookDepth10}, order::BookOrder, - quote::QuoteTick, - trade::TradeTick, }, - enums::{AggressorSide, OrderSide}, - identifiers::{instrument_id::InstrumentId, trade_id::TradeId}, + enums::OrderSide, + identifiers::instrument_id::InstrumentId, orderbook::{book_mbo::OrderBookMbo, book_mbp::OrderBookMbp}, types::{price::Price, quantity::Quantity}, }; - #[rstest] - fn test_orderbook_creation() { - let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let book = OrderBookMbp::new(instrument_id, false); - - assert_eq!(book.instrument_id, instrument_id); - assert_eq!(book.sequence, 0); - assert_eq!(book.ts_last, 0); - assert_eq!(book.count, 0); - } - - #[rstest] - fn test_orderbook_reset() { - let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBookMbp::new(instrument_id, false); - book.sequence = 10; - book.ts_last = 100; - book.count = 3; - - book.reset(); - - assert_eq!(book.sequence, 0); - assert_eq!(book.ts_last, 0); - assert_eq!(book.count, 0); - } - #[rstest] fn test_best_bid_and_ask_when_nothing_in_book() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); @@ -189,6 +161,7 @@ mod tests { assert_eq!(book.best_ask_size(), Some(Quantity::from("2.0"))); assert!(book.has_ask()); } + #[rstest] fn test_spread_with_no_bids_or_asks() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); @@ -388,54 +361,6 @@ mod tests { assert_eq!(book.best_ask_size().unwrap().as_f64(), 100.0); } - #[rstest] - fn test_update_quote_tick_l1() { - let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBookMbp::new(instrument_id, true); - let quote = QuoteTick::new( - InstrumentId::from("ETHUSDT-PERP.BINANCE"), - Price::from("5000.000"), - Price::from("5100.000"), - Quantity::from("100.00000000"), - Quantity::from("99.00000000"), - 0, - 0, - ) - .unwrap(); - - book.update_quote_tick("e); - - assert_eq!(book.best_bid_price().unwrap(), quote.bid_price); - assert_eq!(book.best_ask_price().unwrap(), quote.ask_price); - assert_eq!(book.best_bid_size().unwrap(), quote.bid_size); - assert_eq!(book.best_ask_size().unwrap(), quote.ask_size); - } - - #[rstest] - fn test_update_trade_tick_l1() { - let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); - let mut book = OrderBookMbp::new(instrument_id, true); - - let price = Price::from("15000.000"); - let size = Quantity::from("10.00000000"); - let trade = TradeTick::new( - instrument_id, - price, - size, - AggressorSide::Buyer, - TradeId::new("123456789").unwrap(), - 0, - 0, - ); - - book.update_trade_tick(&trade); - - assert_eq!(book.best_bid_price().unwrap(), price); - assert_eq!(book.best_ask_price().unwrap(), price); - assert_eq!(book.best_bid_size().unwrap(), size); - assert_eq!(book.best_ask_size().unwrap(), size); - } - #[rstest] fn test_pprint() { let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); diff --git a/nautilus_core/model/src/orderbook/book_mbo.rs b/nautilus_core/model/src/orderbook/book_mbo.rs index b72341db4487..2bcda1365ab3 100644 --- a/nautilus_core/model/src/orderbook/book_mbo.rs +++ b/nautilus_core/model/src/orderbook/book_mbo.rs @@ -146,12 +146,12 @@ impl OrderBookMbo { } } - pub fn bids(&self) -> Vec<&Level> { - self.bids.levels.values().collect() + pub fn bids(&self) -> impl Iterator { + self.bids.levels.values() } - pub fn asks(&self) -> Vec<&Level> { - self.asks.levels.values().collect() + pub fn asks(&self) -> impl Iterator { + self.asks.levels.values() } pub fn has_bid(&self) -> bool { @@ -262,3 +262,40 @@ impl OrderBookMbo { self.count += 1; } } + +//////////////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////////////// +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::identifiers::instrument_id::InstrumentId; + + #[rstest] + fn test_orderbook_creation() { + let instrument_id = InstrumentId::from("AAPL.XNAS"); + let book = OrderBookMbo::new(instrument_id); + + assert_eq!(book.instrument_id, instrument_id); + assert_eq!(book.sequence, 0); + assert_eq!(book.ts_last, 0); + assert_eq!(book.count, 0); + } + + #[rstest] + fn test_orderbook_reset() { + let instrument_id = InstrumentId::from("AAPL.XNAS"); + let mut book = OrderBookMbo::new(instrument_id); + book.sequence = 10; + book.ts_last = 100; + book.count = 3; + + book.reset(); + + assert_eq!(book.sequence, 0); + assert_eq!(book.ts_last, 0); + assert_eq!(book.count, 0); + } +} diff --git a/nautilus_core/model/src/orderbook/book_mbp.rs b/nautilus_core/model/src/orderbook/book_mbp.rs index 6a45b163ab21..d518441997ba 100644 --- a/nautilus_core/model/src/orderbook/book_mbp.rs +++ b/nautilus_core/model/src/orderbook/book_mbp.rs @@ -185,12 +185,12 @@ impl OrderBookMbp { } } - pub fn bids(&self) -> Vec<&Level> { - self.bids.levels.values().collect() + pub fn bids(&self) -> impl Iterator { + self.bids.levels.values() } - pub fn asks(&self) -> Vec<&Level> { - self.asks.levels.values().collect() + pub fn asks(&self) -> impl Iterator { + self.asks.levels.values() } pub fn has_bid(&self) -> bool { @@ -393,3 +393,93 @@ impl OrderBookMbp { order } } + +//////////////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////////////// +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::{ + enums::AggressorSide, + identifiers::{instrument_id::InstrumentId, trade_id::TradeId}, + }; + + #[rstest] + fn test_orderbook_creation() { + let instrument_id = InstrumentId::from("AAPL.XNAS"); + let book = OrderBookMbp::new(instrument_id, false); + + assert_eq!(book.instrument_id, instrument_id); + assert!(!book.top_only); + assert_eq!(book.sequence, 0); + assert_eq!(book.ts_last, 0); + assert_eq!(book.count, 0); + } + + #[rstest] + fn test_orderbook_reset() { + let instrument_id = InstrumentId::from("AAPL.XNAS"); + let mut book = OrderBookMbp::new(instrument_id, true); + book.sequence = 10; + book.ts_last = 100; + book.count = 3; + + book.reset(); + + assert!(book.top_only); + assert_eq!(book.sequence, 0); + assert_eq!(book.ts_last, 0); + assert_eq!(book.count, 0); + } + + #[rstest] + fn test_update_quote_tick_l1() { + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let mut book = OrderBookMbp::new(instrument_id, true); + let quote = QuoteTick::new( + InstrumentId::from("ETHUSDT-PERP.BINANCE"), + Price::from("5000.000"), + Price::from("5100.000"), + Quantity::from("100.00000000"), + Quantity::from("99.00000000"), + 0, + 0, + ) + .unwrap(); + + book.update_quote_tick("e); + + assert_eq!(book.best_bid_price().unwrap(), quote.bid_price); + assert_eq!(book.best_ask_price().unwrap(), quote.ask_price); + assert_eq!(book.best_bid_size().unwrap(), quote.bid_size); + assert_eq!(book.best_ask_size().unwrap(), quote.ask_size); + } + + #[rstest] + fn test_update_trade_tick_l1() { + let instrument_id = InstrumentId::from("ETHUSDT-PERP.BINANCE"); + let mut book = OrderBookMbp::new(instrument_id, true); + + let price = Price::from("15000.000"); + let size = Quantity::from("10.00000000"); + let trade = TradeTick::new( + instrument_id, + price, + size, + AggressorSide::Buyer, + TradeId::new("123456789").unwrap(), + 0, + 0, + ); + + book.update_trade_tick(&trade); + + assert_eq!(book.best_bid_price().unwrap(), price); + assert_eq!(book.best_ask_price().unwrap(), price); + assert_eq!(book.best_bid_size().unwrap(), size); + assert_eq!(book.best_ask_size().unwrap(), size); + } +} diff --git a/nautilus_core/model/src/python/orderbook/book_mbo.rs b/nautilus_core/model/src/python/orderbook/book_mbo.rs index 0887658bda9c..c9105e4e182f 100644 --- a/nautilus_core/model/src/python/orderbook/book_mbo.rs +++ b/nautilus_core/model/src/python/orderbook/book_mbo.rs @@ -142,19 +142,13 @@ impl OrderBookMbo { #[pyo3(name = "bids")] fn py_bids(&self) -> Vec { // TODO: Improve efficiency - self.bids() - .iter() - .map(|level_ref| (*level_ref).clone()) - .collect() + self.bids().map(|level_ref| (*level_ref).clone()).collect() } #[pyo3(name = "asks")] fn py_asks(&self) -> Vec { // TODO: Improve efficiency - self.asks() - .iter() - .map(|level_ref| (*level_ref).clone()) - .collect() + self.asks().map(|level_ref| (*level_ref).clone()).collect() } #[pyo3(name = "best_bid_price")] diff --git a/nautilus_core/model/src/python/orderbook/book_mbp.rs b/nautilus_core/model/src/python/orderbook/book_mbp.rs index 2de16cc5337d..93ee53d5ea98 100644 --- a/nautilus_core/model/src/python/orderbook/book_mbp.rs +++ b/nautilus_core/model/src/python/orderbook/book_mbp.rs @@ -156,20 +156,16 @@ impl OrderBookMbp { #[pyo3(name = "bids")] fn py_bids(&self) -> Vec { - // TODO: Improve efficiency - self.bids() - .iter() - .map(|level_ref| (*level_ref).clone()) - .collect() + // Clone each `Level` to create owned levels for Python interop + // and to meet the pyo3::PyAny trait bound. + self.bids().map(|level_ref| (*level_ref).clone()).collect() } #[pyo3(name = "asks")] fn py_asks(&self) -> Vec { - // TODO: Improve efficiency - self.asks() - .iter() - .map(|level_ref| (*level_ref).clone()) - .collect() + // Clone each `Level` to create owned levels for Python interop + // and to meet the pyo3::PyAny trait bound. + self.asks().map(|level_ref| (*level_ref).clone()).collect() } #[pyo3(name = "best_bid_price")] From ef4c557ff43a9b547833117915a94b68dfcbc1b5 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 11:41:48 +1100 Subject: [PATCH 077/104] Standardize core indicator field naming --- nautilus_core/indicators/src/average/ama.rs | 40 ++++++------ nautilus_core/indicators/src/average/dema.rs | 14 ++--- nautilus_core/indicators/src/average/hma.rs | 40 ++++++------ nautilus_core/indicators/src/momentum/rsi.rs | 62 +++++++++---------- .../indicators/src/ratio/efficiency_ratio.rs | 8 +-- .../indicators/src/volatility/atr.rs | 24 +++---- 6 files changed, 93 insertions(+), 95 deletions(-) diff --git a/nautilus_core/indicators/src/average/ama.rs b/nautilus_core/indicators/src/average/ama.rs index d84fff681634..607ad2e121d7 100644 --- a/nautilus_core/indicators/src/average/ama.rs +++ b/nautilus_core/indicators/src/average/ama.rs @@ -49,11 +49,11 @@ pub struct AdaptiveMovingAverage { /// The input count for the indicator. pub count: usize, pub is_initialized: bool, - _efficiency_ratio: EfficiencyRatio, - _prior_value: Option, - _alpha_fast: f64, - _alpha_slow: f64, has_inputs: bool, + efficiency_ratio: EfficiencyRatio, + prior_value: Option, + alpha_fast: f64, + alpha_slow: f64, } impl Display for AdaptiveMovingAverage { @@ -118,23 +118,23 @@ impl AdaptiveMovingAverage { price_type: price_type.unwrap_or(PriceType::Last), value: 0.0, count: 0, - _alpha_fast: 2.0 / (period_fast + 1) as f64, - _alpha_slow: 2.0 / (period_slow + 1) as f64, - _prior_value: None, + alpha_fast: 2.0 / (period_fast + 1) as f64, + alpha_slow: 2.0 / (period_slow + 1) as f64, + prior_value: None, has_inputs: false, is_initialized: false, - _efficiency_ratio: EfficiencyRatio::new(period_efficiency_ratio, price_type)?, + efficiency_ratio: EfficiencyRatio::new(period_efficiency_ratio, price_type)?, }) } #[must_use] pub fn alpha_diff(&self) -> f64 { - self._alpha_fast - self._alpha_slow + self.alpha_fast - self.alpha_slow } pub fn reset(&mut self) { self.value = 0.0; - self._prior_value = None; + self.prior_value = None; self.count = 0; self.has_inputs = false; self.is_initialized = false; @@ -152,28 +152,26 @@ impl MovingAverage for AdaptiveMovingAverage { fn update_raw(&mut self, value: f64) { if !self.has_inputs { - self._prior_value = Some(value); - self._efficiency_ratio.update_raw(value); + self.prior_value = Some(value); + self.efficiency_ratio.update_raw(value); self.value = value; self.has_inputs = true; return; } - self._efficiency_ratio.update_raw(value); - self._prior_value = Some(self.value); + self.efficiency_ratio.update_raw(value); + self.prior_value = Some(self.value); // Calculate the smoothing constant let smoothing_constant = self - ._efficiency_ratio + .efficiency_ratio .value - .mul_add(self.alpha_diff(), self._alpha_slow) + .mul_add(self.alpha_diff(), self.alpha_slow) .powi(2); // Calculate the AMA - self.value = smoothing_constant.mul_add( - value - self._prior_value.unwrap(), - self._prior_value.unwrap(), - ); - if self._efficiency_ratio.is_initialized() { + self.value = smoothing_constant + .mul_add(value - self.prior_value.unwrap(), self.prior_value.unwrap()); + if self.efficiency_ratio.is_initialized() { self.is_initialized = true; } } diff --git a/nautilus_core/indicators/src/average/dema.rs b/nautilus_core/indicators/src/average/dema.rs index 6b86dd213fa2..b8f963a359b3 100644 --- a/nautilus_core/indicators/src/average/dema.rs +++ b/nautilus_core/indicators/src/average/dema.rs @@ -43,8 +43,8 @@ pub struct DoubleExponentialMovingAverage { pub count: usize, pub is_initialized: bool, has_inputs: bool, - _ema1: ExponentialMovingAverage, - _ema2: ExponentialMovingAverage, + ema1: ExponentialMovingAverage, + ema2: ExponentialMovingAverage, } impl Display for DoubleExponentialMovingAverage { @@ -94,8 +94,8 @@ impl DoubleExponentialMovingAverage { count: 0, has_inputs: false, is_initialized: false, - _ema1: ExponentialMovingAverage::new(period, price_type)?, - _ema2: ExponentialMovingAverage::new(period, price_type)?, + ema1: ExponentialMovingAverage::new(period, price_type)?, + ema2: ExponentialMovingAverage::new(period, price_type)?, }) } } @@ -113,10 +113,10 @@ impl MovingAverage for DoubleExponentialMovingAverage { self.has_inputs = true; self.value = value; } - self._ema1.update_raw(value); - self._ema2.update_raw(self._ema1.value); + self.ema1.update_raw(value); + self.ema2.update_raw(self.ema1.value); - self.value = 2.0f64.mul_add(self._ema1.value, -self._ema2.value); + self.value = 2.0f64.mul_add(self.ema1.value, -self.ema2.value); self.count += 1; if !self.is_initialized && self.count >= self.period { diff --git a/nautilus_core/indicators/src/average/hma.rs b/nautilus_core/indicators/src/average/hma.rs index a4ff4672592d..3f359688ddab 100644 --- a/nautilus_core/indicators/src/average/hma.rs +++ b/nautilus_core/indicators/src/average/hma.rs @@ -40,9 +40,9 @@ pub struct HullMovingAverage { pub count: usize, pub is_initialized: bool, has_inputs: bool, - _ma1: WeightedMovingAverage, - _ma2: WeightedMovingAverage, - _ma3: WeightedMovingAverage, + ma1: WeightedMovingAverage, + ma2: WeightedMovingAverage, + ma3: WeightedMovingAverage, } impl Display for HullMovingAverage { @@ -78,9 +78,9 @@ impl Indicator for HullMovingAverage { fn reset(&mut self) { self.value = 0.0; - self._ma1.reset(); - self._ma2.reset(); - self._ma3.reset(); + self.ma1.reset(); + self.ma2.reset(); + self.ma3.reset(); self.count = 0; self.has_inputs = false; self.is_initialized = false; @@ -114,9 +114,9 @@ impl HullMovingAverage { count: 0, has_inputs: false, is_initialized: false, - _ma1, - _ma2, - _ma3, + ma1: _ma1, + ma2: _ma2, + ma3: _ma3, }) } } @@ -136,12 +136,12 @@ impl MovingAverage for HullMovingAverage { self.value = value; } - self._ma1.update_raw(value); - self._ma2.update_raw(value); - self._ma3 - .update_raw(2.0f64.mul_add(self._ma1.value, -self._ma2.value)); + self.ma1.update_raw(value); + self.ma2.update_raw(value); + self.ma3 + .update_raw(2.0f64.mul_add(self.ma1.value, -self.ma2.value)); - self.value = self._ma3.value; + self.value = self.ma3.value; self.count += 1; if !self.is_initialized && self.count >= self.period { @@ -241,15 +241,15 @@ mod tests { indicator_hma_10.update_raw(1.0); assert_eq!(indicator_hma_10.count, 1); assert_eq!(indicator_hma_10.value, 1.0); - assert_eq!(indicator_hma_10._ma1.value, 1.0); - assert_eq!(indicator_hma_10._ma2.value, 1.0); - assert_eq!(indicator_hma_10._ma3.value, 1.0); + assert_eq!(indicator_hma_10.ma1.value, 1.0); + assert_eq!(indicator_hma_10.ma2.value, 1.0); + assert_eq!(indicator_hma_10.ma3.value, 1.0); indicator_hma_10.reset(); assert_eq!(indicator_hma_10.value, 0.0); assert_eq!(indicator_hma_10.count, 0); - assert_eq!(indicator_hma_10._ma1.value, 0.0); - assert_eq!(indicator_hma_10._ma2.value, 0.0); - assert_eq!(indicator_hma_10._ma3.value, 0.0); + assert_eq!(indicator_hma_10.ma1.value, 0.0); + assert_eq!(indicator_hma_10.ma2.value, 0.0); + assert_eq!(indicator_hma_10.ma3.value, 0.0); assert!(!indicator_hma_10.has_inputs); assert!(!indicator_hma_10.is_initialized); } diff --git a/nautilus_core/indicators/src/momentum/rsi.rs b/nautilus_core/indicators/src/momentum/rsi.rs index 07623dbee708..7477eb34134e 100644 --- a/nautilus_core/indicators/src/momentum/rsi.rs +++ b/nautilus_core/indicators/src/momentum/rsi.rs @@ -37,11 +37,11 @@ pub struct RelativeStrengthIndex { pub value: f64, pub count: usize, pub is_initialized: bool, - _has_inputs: bool, - _last_value: f64, - _average_gain: Box, - _average_loss: Box, - _rsi_max: f64, + has_inputs: bool, + last_value: f64, + average_gain: Box, + average_loss: Box, + rsi_max: f64, } impl Display for RelativeStrengthIndex { @@ -56,7 +56,7 @@ impl Indicator for RelativeStrengthIndex { } fn has_inputs(&self) -> bool { - self._has_inputs + self.has_inputs } fn is_initialized(&self) -> bool { @@ -77,9 +77,9 @@ impl Indicator for RelativeStrengthIndex { fn reset(&mut self) { self.value = 0.0; - self._last_value = 0.0; + self.last_value = 0.0; self.count = 0; - self._has_inputs = false; + self.has_inputs = false; self.is_initialized = false; } } @@ -90,50 +90,50 @@ impl RelativeStrengthIndex { period, ma_type: ma_type.unwrap_or(MovingAverageType::Exponential), value: 0.0, - _last_value: 0.0, + last_value: 0.0, count: 0, // inputs: Vec::new(), - _has_inputs: false, - _average_gain: MovingAverageFactory::create(MovingAverageType::Exponential, period), - _average_loss: MovingAverageFactory::create(MovingAverageType::Exponential, period), - _rsi_max: 1.0, + has_inputs: false, + average_gain: MovingAverageFactory::create(MovingAverageType::Exponential, period), + average_loss: MovingAverageFactory::create(MovingAverageType::Exponential, period), + rsi_max: 1.0, is_initialized: false, }) } pub fn update_raw(&mut self, value: f64) { - if !self._has_inputs { - self._last_value = value; - self._has_inputs = true; + if !self.has_inputs { + self.last_value = value; + self.has_inputs = true; } - let gain = value - self._last_value; + let gain = value - self.last_value; if gain > 0.0 { - self._average_gain.update_raw(gain); - self._average_loss.update_raw(0.0); + self.average_gain.update_raw(gain); + self.average_loss.update_raw(0.0); } else if gain < 0.0 { - self._average_loss.update_raw(-gain); - self._average_gain.update_raw(0.0); + self.average_loss.update_raw(-gain); + self.average_gain.update_raw(0.0); } else { - self._average_loss.update_raw(0.0); - self._average_gain.update_raw(0.0); + self.average_loss.update_raw(0.0); + self.average_gain.update_raw(0.0); } // init count from average gain MA - self.count = self._average_gain.count(); + self.count = self.average_gain.count(); if !self.is_initialized - && self._average_loss.is_initialized() - && self._average_gain.is_initialized() + && self.average_loss.is_initialized() + && self.average_gain.is_initialized() { self.is_initialized = true; } - if self._average_loss.value() == 0.0 { - self.value = self._rsi_max; + if self.average_loss.value() == 0.0 { + self.value = self.rsi_max; return; } - let rs = self._average_gain.value() / self._average_loss.value(); - self.value = self._rsi_max - (self._rsi_max / (1.0 + rs)); - self._last_value = value; + let rs = self.average_gain.value() / self.average_loss.value(); + self.value = self.rsi_max - (self.rsi_max / (1.0 + rs)); + self.last_value = value; if !self.is_initialized && self.count >= self.period { self.is_initialized = true; diff --git a/nautilus_core/indicators/src/ratio/efficiency_ratio.rs b/nautilus_core/indicators/src/ratio/efficiency_ratio.rs index 6735969a0045..a3e85f1323aa 100644 --- a/nautilus_core/indicators/src/ratio/efficiency_ratio.rs +++ b/nautilus_core/indicators/src/ratio/efficiency_ratio.rs @@ -37,7 +37,7 @@ pub struct EfficiencyRatio { pub value: f64, pub inputs: Vec, pub is_initialized: bool, - _deltas: Vec, + deltas: Vec, } impl Display for EfficiencyRatio { @@ -84,7 +84,7 @@ impl EfficiencyRatio { price_type: price_type.unwrap_or(PriceType::Last), value: 0.0, inputs: Vec::with_capacity(period), - _deltas: Vec::with_capacity(period), + deltas: Vec::with_capacity(period), is_initialized: false, }) } @@ -99,8 +99,8 @@ impl EfficiencyRatio { } let last_diff = (self.inputs[self.inputs.len() - 1] - self.inputs[self.inputs.len() - 2]).abs(); - self._deltas.push(last_diff); - let sum_deltas = self._deltas.iter().sum::().abs(); + self.deltas.push(last_diff); + let sum_deltas = self.deltas.iter().sum::().abs(); let net_diff = (self.inputs[self.inputs.len() - 1] - self.inputs[0]).abs(); self.value = if sum_deltas == 0.0 { 0.0 diff --git a/nautilus_core/indicators/src/volatility/atr.rs b/nautilus_core/indicators/src/volatility/atr.rs index b6a1e2a9e49d..9936be98c05d 100644 --- a/nautilus_core/indicators/src/volatility/atr.rs +++ b/nautilus_core/indicators/src/volatility/atr.rs @@ -36,9 +36,9 @@ pub struct AverageTrueRange { pub value: f64, pub count: usize, pub is_initialized: bool, + ma: Box, has_inputs: bool, - _previous_close: f64, - _ma: Box, + previous_close: f64, } impl Display for AverageTrueRange { @@ -81,7 +81,7 @@ impl Indicator for AverageTrueRange { } fn reset(&mut self) { - self._previous_close = 0.0; + self.previous_close = 0.0; self.value = 0.0; self.count = 0; self.has_inputs = false; @@ -103,8 +103,8 @@ impl AverageTrueRange { value_floor: value_floor.unwrap_or(0.0), value: 0.0, count: 0, - _previous_close: 0.0, - _ma: MovingAverageFactory::create(MovingAverageType::Simple, period), + previous_close: 0.0, + ma: MovingAverageFactory::create(MovingAverageType::Simple, period), has_inputs: false, is_initialized: false, }) @@ -113,14 +113,14 @@ impl AverageTrueRange { pub fn update_raw(&mut self, high: f64, low: f64, close: f64) { if self.use_previous { if !self.has_inputs { - self._previous_close = close; + self.previous_close = close; } - self._ma.update_raw( - f64::max(self._previous_close, high) - f64::min(low, self._previous_close), + self.ma.update_raw( + f64::max(self.previous_close, high) - f64::min(low, self.previous_close), ); - self._previous_close = close; + self.previous_close = close; } else { - self._ma.update_raw(high - low); + self.ma.update_raw(high - low); } self._floor_value(); @@ -128,8 +128,8 @@ impl AverageTrueRange { } fn _floor_value(&mut self) { - if self.value_floor == 0.0 || self.value_floor < self._ma.value() { - self.value = self._ma.value(); + if self.value_floor == 0.0 || self.value_floor < self.ma.value() { + self.value = self.ma.value(); } else { // Floor the value self.value = self.value_floor; From 33f6cde29999f441c890de45ecf587fff7692771 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 12:12:30 +1100 Subject: [PATCH 078/104] Extend core Indicator to handle order books --- nautilus_core/indicators/src/average/dema.rs | 8 ++-- nautilus_core/indicators/src/average/ema.rs | 8 ++-- nautilus_core/indicators/src/average/hma.rs | 8 ++-- nautilus_core/indicators/src/average/rma.rs | 8 ++-- nautilus_core/indicators/src/average/sma.rs | 8 ++-- nautilus_core/indicators/src/average/wma.rs | 8 ++-- nautilus_core/indicators/src/indicator.rs | 38 +++++++++++++++++-- .../indicators/src/momentum/aroon.rs | 15 +++++--- nautilus_core/indicators/src/momentum/rsi.rs | 8 ++-- .../indicators/src/ratio/efficiency_ratio.rs | 8 ++-- .../indicators/src/volatility/atr.rs | 10 +---- nautilus_trader/core/nautilus_pyo3.pyi | 20 +++++----- 12 files changed, 87 insertions(+), 60 deletions(-) diff --git a/nautilus_core/indicators/src/average/dema.rs b/nautilus_core/indicators/src/average/dema.rs index b8f963a359b3..8b23acdca274 100644 --- a/nautilus_core/indicators/src/average/dema.rs +++ b/nautilus_core/indicators/src/average/dema.rs @@ -65,12 +65,12 @@ impl Indicator for DoubleExponentialMovingAverage { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(self.price_type).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(self.price_type).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((&tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((&trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/average/ema.rs b/nautilus_core/indicators/src/average/ema.rs index 4c07fb5b5529..f9cc1f8155a8 100644 --- a/nautilus_core/indicators/src/average/ema.rs +++ b/nautilus_core/indicators/src/average/ema.rs @@ -56,12 +56,12 @@ impl Indicator for ExponentialMovingAverage { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(self.price_type).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(self.price_type).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((&tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((&trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/average/hma.rs b/nautilus_core/indicators/src/average/hma.rs index 3f359688ddab..8ee0da9d8a48 100644 --- a/nautilus_core/indicators/src/average/hma.rs +++ b/nautilus_core/indicators/src/average/hma.rs @@ -64,12 +64,12 @@ impl Indicator for HullMovingAverage { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(self.price_type).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(self.price_type).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((&tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((&trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/average/rma.rs b/nautilus_core/indicators/src/average/rma.rs index 479e63576a81..678c6d48ddfa 100644 --- a/nautilus_core/indicators/src/average/rma.rs +++ b/nautilus_core/indicators/src/average/rma.rs @@ -56,12 +56,12 @@ impl Indicator for WilderMovingAverage { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(self.price_type).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(self.price_type).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((&tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((&trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/average/sma.rs b/nautilus_core/indicators/src/average/sma.rs index 4752e1ddaaea..8fd880d4cbfe 100644 --- a/nautilus_core/indicators/src/average/sma.rs +++ b/nautilus_core/indicators/src/average/sma.rs @@ -55,12 +55,12 @@ impl Indicator for SimpleMovingAverage { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(self.price_type).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(self.price_type).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((&tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((&trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/average/wma.rs b/nautilus_core/indicators/src/average/wma.rs index 22ca40b08581..9822ae1f95a2 100644 --- a/nautilus_core/indicators/src/average/wma.rs +++ b/nautilus_core/indicators/src/average/wma.rs @@ -91,12 +91,12 @@ impl Indicator for WeightedMovingAverage { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(self.price_type).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(self.price_type).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((&tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((&trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/indicator.rs b/nautilus_core/indicators/src/indicator.rs index 85cc467f1eb3..41bbcb2c4ef2 100644 --- a/nautilus_core/indicators/src/indicator.rs +++ b/nautilus_core/indicators/src/indicator.rs @@ -15,15 +15,45 @@ use std::{fmt, fmt::Debug}; -use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; +use nautilus_model::{ + data::{ + bar::Bar, delta::OrderBookDelta, deltas::OrderBookDeltas, depth::OrderBookDepth10, + quote::QuoteTick, trade::TradeTick, + }, + orderbook::{book_mbo::OrderBookMbo, book_mbp::OrderBookMbp}, +}; +const IMPL_ERR: &str = "is not implemented for"; + +#[allow(unused_variables)] pub trait Indicator { fn name(&self) -> String; fn has_inputs(&self) -> bool; fn is_initialized(&self) -> bool; - fn handle_quote_tick(&mut self, tick: &QuoteTick); - fn handle_trade_tick(&mut self, tick: &TradeTick); - fn handle_bar(&mut self, bar: &Bar); + fn handle_delta(&mut self, delta: &OrderBookDelta) { + panic!("`handle_delta` {} `{}`", IMPL_ERR, self.name()); + } + fn handle_deltas(&mut self, deltas: &OrderBookDeltas) { + panic!("`handle_deltas` {} `{}`", IMPL_ERR, self.name()); + } + fn handle_depth(&mut self, depth: &OrderBookDepth10) { + panic!("`handle_depth` {} `{}`", IMPL_ERR, self.name()); + } + fn handle_order_book_mbo(&mut self, book: &OrderBookMbo) { + panic!("`handle_order_book_mbo` {} `{}`", IMPL_ERR, self.name()); + } + fn handle_order_book_mbp(&mut self, book: &OrderBookMbp) { + panic!("`handle_order_book_mbp` {} `{}`", IMPL_ERR, self.name()); + } + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + panic!("`handle_quote_tick` {} `{}`", IMPL_ERR, self.name()); + } + fn handle_trade_tick(&mut self, trade: &TradeTick) { + panic!("`handle_trade_tick` {} `{}`", IMPL_ERR, self.name()); + } + fn handle_bar(&mut self, bar: &Bar) { + panic!("`handle_bar` {} `{}`", IMPL_ERR, self.name()); + } fn reset(&mut self); } diff --git a/nautilus_core/indicators/src/momentum/aroon.rs b/nautilus_core/indicators/src/momentum/aroon.rs index 1c42195fc2d5..710e525079f4 100644 --- a/nautilus_core/indicators/src/momentum/aroon.rs +++ b/nautilus_core/indicators/src/momentum/aroon.rs @@ -16,7 +16,10 @@ use std::fmt::{Debug, Display}; use anyhow::Result; -use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; +use nautilus_model::{ + data::{bar::Bar, quote::QuoteTick, trade::TradeTick}, + enums::PriceType, +}; use pyo3::prelude::*; use std::collections::VecDeque; @@ -58,12 +61,14 @@ impl Indicator for AroonOscillator { self.is_initialized } - fn handle_quote_tick(&mut self, _tick: &QuoteTick) { - // Function body intentionally left blank. + fn handle_quote_tick(&mut self, tick: &QuoteTick) { + let price = tick.extract_price(PriceType::Mid).into(); + self.update_raw(price, price); } - fn handle_trade_tick(&mut self, _tick: &TradeTick) { - // Function body intentionally left blank. + fn handle_trade_tick(&mut self, tick: &TradeTick) { + let price = tick.price.into(); + self.update_raw(price, price); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/momentum/rsi.rs b/nautilus_core/indicators/src/momentum/rsi.rs index 7477eb34134e..844b159a0df8 100644 --- a/nautilus_core/indicators/src/momentum/rsi.rs +++ b/nautilus_core/indicators/src/momentum/rsi.rs @@ -63,12 +63,12 @@ impl Indicator for RelativeStrengthIndex { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(PriceType::Mid).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(PriceType::Mid).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/ratio/efficiency_ratio.rs b/nautilus_core/indicators/src/ratio/efficiency_ratio.rs index a3e85f1323aa..520c92e033f0 100644 --- a/nautilus_core/indicators/src/ratio/efficiency_ratio.rs +++ b/nautilus_core/indicators/src/ratio/efficiency_ratio.rs @@ -58,12 +58,12 @@ impl Indicator for EfficiencyRatio { self.is_initialized } - fn handle_quote_tick(&mut self, tick: &QuoteTick) { - self.update_raw(tick.extract_price(self.price_type).into()); + fn handle_quote_tick(&mut self, quote: &QuoteTick) { + self.update_raw(quote.extract_price(self.price_type).into()); } - fn handle_trade_tick(&mut self, tick: &TradeTick) { - self.update_raw((&tick.price).into()); + fn handle_trade_tick(&mut self, trade: &TradeTick) { + self.update_raw((&trade.price).into()); } fn handle_bar(&mut self, bar: &Bar) { diff --git a/nautilus_core/indicators/src/volatility/atr.rs b/nautilus_core/indicators/src/volatility/atr.rs index 9936be98c05d..8c3f823be176 100644 --- a/nautilus_core/indicators/src/volatility/atr.rs +++ b/nautilus_core/indicators/src/volatility/atr.rs @@ -16,7 +16,7 @@ use std::fmt::{Debug, Display}; use anyhow::Result; -use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; +use nautilus_model::data::bar::Bar; use pyo3::prelude::*; use crate::{ @@ -68,14 +68,6 @@ impl Indicator for AverageTrueRange { self.is_initialized } - fn handle_quote_tick(&mut self, _tick: &QuoteTick) { - // Function body intentionally left blank. - } - - fn handle_trade_tick(&mut self, _tick: &TradeTick) { - // Function body intentionally left blank. - } - fn handle_bar(&mut self, bar: &Bar) { self.update_raw((&bar.high).into(), (&bar.low).into(), (&bar.close).into()); } diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 0a7af2fc8f2a..946e6d46399f 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -1945,8 +1945,8 @@ class SimpleMovingAverage: def value(self) -> float: ... def update_raw(self, value: float) -> None: ... def reset(self) -> None: ... - def handle_quote_tick(self, tick: QuoteTick) -> None: ... - def handle_trade_tick(self, tick: TradeTick) -> None: ... + def handle_quote_tick(self, quote: QuoteTick) -> None: ... + def handle_trade_tick(self, trade: TradeTick) -> None: ... def handle_bar(self, bar: Bar) -> None: ... class ExponentialMovingAverage: @@ -1970,8 +1970,8 @@ class ExponentialMovingAverage: @property def alpha(self) -> float: ... def update_raw(self, value: float) -> None: ... - def handle_quote_tick(self, tick: QuoteTick) -> None: ... - def handle_trade_tick(self, tick: TradeTick) -> None: ... + def handle_quote_tick(self, quote: QuoteTick) -> None: ... + def handle_trade_tick(self, trade: TradeTick) -> None: ... def handle_bar(self, bar: Bar) -> None: ... def reset(self) -> None: ... @@ -1994,8 +1994,8 @@ class DoubleExponentialMovingAverage: @property def value(self) -> float: ... def update_raw(self, value: float) -> None: ... - def handle_quote_tick(self, tick: QuoteTick) -> None: ... - def handle_trade_tick(self, tick: TradeTick) -> None: ... + def handle_quote_tick(self, quote: QuoteTick) -> None: ... + def handle_trade_tick(self, trade: TradeTick) -> None: ... def handle_bar(self, bar: Bar) -> None: ... def reset(self) -> None: ... @@ -2018,8 +2018,8 @@ class HullMovingAverage: @property def value(self) -> float: ... def update_raw(self, value: float) -> None: ... - def handle_quote_tick(self, tick: QuoteTick) -> None: ... - def handle_trade_tick(self, tick: TradeTick) -> None: ... + def handle_quote_tick(self, quote: QuoteTick) -> None: ... + def handle_trade_tick(self, trade: TradeTick) -> None: ... def handle_bar(self, bar: Bar) -> None: ... def reset(self) -> None: ... @@ -2044,8 +2044,8 @@ class WilderMovingAverage: @property def alpha(self) -> float: ... def update_raw(self, value: float) -> None: ... - def handle_quote_tick(self, tick: QuoteTick) -> None: ... - def handle_trade_tick(self, tick: TradeTick) -> None: ... + def handle_quote_tick(self, quote: QuoteTick) -> None: ... + def handle_trade_tick(self, trade: TradeTick) -> None: ... def handle_bar(self, bar: Bar) -> None: ... def reset(self) -> None: ... From d9584d5e4c29e98387d4b453937838ff91090856 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 12:14:57 +1100 Subject: [PATCH 079/104] Extend core Indicator to handle order book data --- nautilus_core/indicators/src/indicator.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nautilus_core/indicators/src/indicator.rs b/nautilus_core/indicators/src/indicator.rs index 41bbcb2c4ef2..3a2981dd8f89 100644 --- a/nautilus_core/indicators/src/indicator.rs +++ b/nautilus_core/indicators/src/indicator.rs @@ -31,27 +31,35 @@ pub trait Indicator { fn has_inputs(&self) -> bool; fn is_initialized(&self) -> bool; fn handle_delta(&mut self, delta: &OrderBookDelta) { + // Eventually change this to log an error panic!("`handle_delta` {} `{}`", IMPL_ERR, self.name()); } fn handle_deltas(&mut self, deltas: &OrderBookDeltas) { + // Eventually change this to log an error panic!("`handle_deltas` {} `{}`", IMPL_ERR, self.name()); } fn handle_depth(&mut self, depth: &OrderBookDepth10) { + // Eventually change this to log an error panic!("`handle_depth` {} `{}`", IMPL_ERR, self.name()); } fn handle_order_book_mbo(&mut self, book: &OrderBookMbo) { + // Eventually change this to log an error panic!("`handle_order_book_mbo` {} `{}`", IMPL_ERR, self.name()); } fn handle_order_book_mbp(&mut self, book: &OrderBookMbp) { + // Eventually change this to log an error panic!("`handle_order_book_mbp` {} `{}`", IMPL_ERR, self.name()); } fn handle_quote_tick(&mut self, quote: &QuoteTick) { + // Eventually change this to log an error panic!("`handle_quote_tick` {} `{}`", IMPL_ERR, self.name()); } fn handle_trade_tick(&mut self, trade: &TradeTick) { + // Eventually change this to log an error panic!("`handle_trade_tick` {} `{}`", IMPL_ERR, self.name()); } fn handle_bar(&mut self, bar: &Bar) { + // Eventually change this to log an error panic!("`handle_bar` {} `{}`", IMPL_ERR, self.name()); } fn reset(&mut self); From ff1cd690961aa641fb305bd4655ae6a5de6e785f Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 14:53:18 +1100 Subject: [PATCH 080/104] Fix TradeId memory leak --- RELEASES.md | 1 + nautilus_core/model/src/data/trade.rs | 26 +- .../model/src/ffi/identifiers/trade_id.rs | 18 +- nautilus_core/model/src/identifiers/mod.rs | 2 - .../model/src/identifiers/trade_id.rs | 67 ++- .../model/src/python/identifiers/mod.rs | 1 + .../model/src/python/identifiers/trade_id.rs | 110 ++++ nautilus_trader/core/includes/model.h | 7 +- nautilus_trader/core/rust/model.pxd | 5 +- nautilus_trader/model/identifiers.pyx | 5 +- tests/unit_tests/model/test_tick.py | 539 +++++++++--------- 11 files changed, 462 insertions(+), 319 deletions(-) create mode 100644 nautilus_core/model/src/python/identifiers/trade_id.rs diff --git a/RELEASES.md b/RELEASES.md index 4e59414f5034..d20586d1e7c8 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -14,6 +14,7 @@ Released on TBD (UTC). None ### Fixes +- Fixed `TradeId` memory leak due assigning unique values to the `Ustr` global string cache (which are never freed for the lifetime of the program) - Fixed FOK time in force behavior (allows fills beyond the top level, will cancel if cannot fill full size) - Fixed IOC time in force behavior (allows fills beyond the top level, will cancel any remaining after all fills are applied) - Fixed `LiveClock` timer behavior for small intervals causing next time to be less than now (timer then would not run) diff --git a/nautilus_core/model/src/data/trade.rs b/nautilus_core/model/src/data/trade.rs index 24a7cad017e5..0ca757f047ac 100644 --- a/nautilus_core/model/src/data/trade.rs +++ b/nautilus_core/model/src/data/trade.rs @@ -202,9 +202,9 @@ mod tests { #[rstest] fn test_to_string(stub_trade_tick_ethusdt_buyer: TradeTick) { - let tick = stub_trade_tick_ethusdt_buyer; + let trade = stub_trade_tick_ethusdt_buyer; assert_eq!( - tick.to_string(), + trade.to_string(), "ETHUSDT-PERP.BINANCE,10000.0000,1.00000000,BUYER,123456789,0" ); } @@ -222,36 +222,36 @@ mod tests { "ts_init": 1 }"#; - let tick: TradeTick = serde_json::from_str(raw_string).unwrap(); + let trade: TradeTick = serde_json::from_str(raw_string).unwrap(); - assert_eq!(tick.aggressor_side, AggressorSide::Buyer); + assert_eq!(trade.aggressor_side, AggressorSide::Buyer); } #[rstest] fn test_from_pyobject(stub_trade_tick_ethusdt_buyer: TradeTick) { pyo3::prepare_freethreaded_python(); - let tick = stub_trade_tick_ethusdt_buyer; + let trade = stub_trade_tick_ethusdt_buyer; Python::with_gil(|py| { - let tick_pyobject = tick.into_py(py); + let tick_pyobject = trade.into_py(py); let parsed_tick = TradeTick::from_pyobject(tick_pyobject.as_ref(py)).unwrap(); - assert_eq!(parsed_tick, tick); + assert_eq!(parsed_tick, trade); }); } #[rstest] fn test_json_serialization(stub_trade_tick_ethusdt_buyer: TradeTick) { - let tick = stub_trade_tick_ethusdt_buyer; - let serialized = tick.as_json_bytes().unwrap(); + let trade = stub_trade_tick_ethusdt_buyer; + let serialized = trade.as_json_bytes().unwrap(); let deserialized = TradeTick::from_json_bytes(serialized).unwrap(); - assert_eq!(deserialized, tick); + assert_eq!(deserialized, trade); } #[rstest] fn test_msgpack_serialization(stub_trade_tick_ethusdt_buyer: TradeTick) { - let tick = stub_trade_tick_ethusdt_buyer; - let serialized = tick.as_msgpack_bytes().unwrap(); + let trade = stub_trade_tick_ethusdt_buyer; + let serialized = trade.as_msgpack_bytes().unwrap(); let deserialized = TradeTick::from_msgpack_bytes(serialized).unwrap(); - assert_eq!(deserialized, tick); + assert_eq!(deserialized, trade); } } diff --git a/nautilus_core/model/src/ffi/identifiers/trade_id.rs b/nautilus_core/model/src/ffi/identifiers/trade_id.rs index 5d4ab24cc353..b22467df1a78 100644 --- a/nautilus_core/model/src/ffi/identifiers/trade_id.rs +++ b/nautilus_core/model/src/ffi/identifiers/trade_id.rs @@ -13,9 +13,10 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use std::ffi::c_char; - -use nautilus_core::ffi::string::cstr_to_str; +use std::{ + ffi::{c_char, CStr}, + hash::{DefaultHasher, Hash, Hasher}, +}; use crate::identifiers::trade_id::TradeId; @@ -26,10 +27,17 @@ use crate::identifiers::trade_id::TradeId; /// - Assumes `ptr` is a valid C string pointer. #[no_mangle] pub unsafe extern "C" fn trade_id_new(ptr: *const c_char) -> TradeId { - TradeId::from(cstr_to_str(ptr)) + TradeId::from_cstr(CStr::from_ptr(ptr).to_owned()).unwrap() } #[no_mangle] pub extern "C" fn trade_id_hash(id: &TradeId) -> u64 { - id.value.precomputed_hash() + let mut hasher = DefaultHasher::new(); + id.value.hash(&mut hasher); + hasher.finish() +} + +#[no_mangle] +pub extern "C" fn trade_id_to_cstr(trade_id: &TradeId) -> *const c_char { + trade_id.to_cstr().as_ptr() } diff --git a/nautilus_core/model/src/identifiers/mod.rs b/nautilus_core/model/src/identifiers/mod.rs index dd452acec5d1..5e837eb8b07e 100644 --- a/nautilus_core/model/src/identifiers/mod.rs +++ b/nautilus_core/model/src/identifiers/mod.rs @@ -70,7 +70,6 @@ impl_serialization_for_identifier!(order_list_id::OrderListId); impl_serialization_for_identifier!(position_id::PositionId); impl_serialization_for_identifier!(strategy_id::StrategyId); impl_serialization_for_identifier!(symbol::Symbol); -impl_serialization_for_identifier!(trade_id::TradeId); impl_serialization_for_identifier!(trader_id::TraderId); impl_serialization_for_identifier!(venue::Venue); impl_serialization_for_identifier!(venue_order_id::VenueOrderId); @@ -84,7 +83,6 @@ identifier_for_python!(order_list_id::OrderListId); identifier_for_python!(position_id::PositionId); identifier_for_python!(strategy_id::StrategyId); identifier_for_python!(symbol::Symbol); -identifier_for_python!(trade_id::TradeId); identifier_for_python!(trader_id::TraderId); identifier_for_python!(venue::Venue); identifier_for_python!(venue_order_id::VenueOrderId); diff --git a/nautilus_core/model/src/identifiers/trade_id.rs b/nautilus_core/model/src/identifiers/trade_id.rs index 592b272670a7..6fd8e29e88bb 100644 --- a/nautilus_core/model/src/identifiers/trade_id.rs +++ b/nautilus_core/model/src/identifiers/trade_id.rs @@ -14,13 +14,14 @@ // ------------------------------------------------------------------------------------------------- use std::{ + ffi::{CStr, CString}, fmt::{Debug, Display, Formatter}, hash::Hash, }; -use anyhow::Result; +use anyhow::{bail, Result}; use nautilus_core::correctness::check_valid_string; -use ustr::Ustr; +use serde::{Deserialize, Deserializer, Serialize}; /// Represents a valid trade match ID (assigned by a trading venue). /// @@ -29,43 +30,54 @@ use ustr::Ustr; /// The unique ID assigned to the trade entity once it is received or matched by /// the exchange or central counterparty. #[repr(C)] -#[derive(Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr( feature = "python", pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct TradeId { - /// The trade match ID value. - pub value: Ustr, + pub value: [u8; 65], } impl TradeId { pub fn new(s: &str) -> Result { - check_valid_string(s, "`TradeId` value")?; + let cstr = CString::new(s).expect("`CString` conversion failed"); - Ok(Self { - value: Ustr::from(s), - }) + Self::from_cstr(cstr) } -} -impl Default for TradeId { - fn default() -> Self { - Self { - value: Ustr::from("1"), + pub fn from_cstr(cstr: CString) -> Result { + check_valid_string(cstr.to_str()?, "`TradeId` value")?; + + // TODO: Temporarily make this 65 to accommodate Betfair trade IDs + // TODO: Extract this to single function + let bytes = cstr.as_bytes_with_nul(); + if bytes.len() > 65 { + bail!("Condition failed: value exceeds maximum trade ID length of 36"); } + let mut value = [0; 65]; + value[..bytes.len()].copy_from_slice(bytes); + + Ok(Self { value }) + } + + #[must_use] + pub fn to_cstr(&self) -> &CStr { + // SAFETY: Unwrap safe as we always store valid C strings + // We use until nul because the values array may be padded with nul bytes + CStr::from_bytes_until_nul(&self.value).unwrap() } } -impl Debug for TradeId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.value) +impl Default for TradeId { + fn default() -> Self { + Self::from("1") } } impl Display for TradeId { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.value) + write!(f, "{}", self.to_cstr().to_str().unwrap()) } } @@ -75,6 +87,25 @@ impl From<&str> for TradeId { } } +impl Serialize for TradeId { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for TradeId { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value_str = String::deserialize(deserializer)?; + TradeId::new(&value_str).map_err(|err| serde::de::Error::custom(err.to_string())) + } +} + //////////////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////////////// diff --git a/nautilus_core/model/src/python/identifiers/mod.rs b/nautilus_core/model/src/python/identifiers/mod.rs index 73393101ab6e..92c4c0ce43ce 100644 --- a/nautilus_core/model/src/python/identifiers/mod.rs +++ b/nautilus_core/model/src/python/identifiers/mod.rs @@ -14,3 +14,4 @@ // ------------------------------------------------------------------------------------------------- pub mod instrument_id; +pub mod trade_id; diff --git a/nautilus_core/model/src/python/identifiers/trade_id.rs b/nautilus_core/model/src/python/identifiers/trade_id.rs new file mode 100644 index 000000000000..03112fc130dc --- /dev/null +++ b/nautilus_core/model/src/python/identifiers/trade_id.rs @@ -0,0 +1,110 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::{ + collections::hash_map::DefaultHasher, + ffi::CString, + hash::{Hash, Hasher}, + str::FromStr, +}; + +use nautilus_core::python::to_pyvalue_err; +use pyo3::{ + prelude::*, + pyclass::CompareOp, + types::{PyString, PyTuple}, +}; + +use crate::identifiers::{instrument_id::InstrumentId, trade_id::TradeId}; + +#[pymethods] +impl TradeId { + #[new] + fn py_new(value: &str) -> PyResult { + TradeId::new(value).map_err(to_pyvalue_err) + } + + fn __setstate__(&mut self, py: Python, state: PyObject) -> PyResult<()> { + let value: (&PyString,) = state.extract(py)?; + let value_str: String = value.0.extract()?; + + // TODO: Extract this to single function + let c_string = CString::new(value_str).expect("`CString` conversion failed"); + let bytes = c_string.as_bytes_with_nul(); + let mut value = [0; 65]; + value[..bytes.len()].copy_from_slice(bytes); + self.value = value; + + Ok(()) + } + + fn __getstate__(&self, py: Python) -> PyResult { + Ok((self.to_string(),).to_object(py)) + } + + fn __reduce__(&self, py: Python) -> PyResult { + let safe_constructor = py.get_type::().getattr("_safe_constructor")?; + let state = self.__getstate__(py)?; + Ok((safe_constructor, PyTuple::empty(py), state).to_object(py)) + } + + #[staticmethod] + fn _safe_constructor() -> PyResult { + Ok(TradeId::from_str("NULL").unwrap()) // Safe default + } + + fn __richcmp__(&self, other: PyObject, op: CompareOp, py: Python<'_>) -> Py { + if let Ok(other) = other.extract::(py) { + match op { + CompareOp::Eq => self.eq(&other).into_py(py), + CompareOp::Ne => self.ne(&other).into_py(py), + _ => py.NotImplemented(), + } + } else { + py.NotImplemented() + } + } + + fn __hash__(&self) -> isize { + let mut h = DefaultHasher::new(); + self.hash(&mut h); + h.finish() as isize + } + + fn __str__(&self) -> String { + self.to_string() + } + + fn __repr__(&self) -> String { + format!("{}('{}')", stringify!(InstrumentId), self) + } + + #[getter] + fn value(&self) -> String { + self.to_string() + } + + #[staticmethod] + #[pyo3(name = "from_str")] + fn py_from_str(value: &str) -> PyResult { + InstrumentId::from_str(value).map_err(to_pyvalue_err) + } +} + +impl ToPyObject for TradeId { + fn to_object(&self, py: Python) -> PyObject { + self.into_py(py) + } +} diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index ace1b7973e6e..06782a04af90 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -890,10 +890,7 @@ typedef struct QuoteTick_t { * the exchange or central counterparty. */ typedef struct TradeId_t { - /** - * The trade match ID value. - */ - char* value; + uint8_t value[65]; } TradeId_t; /** @@ -2011,6 +2008,8 @@ struct TradeId_t trade_id_new(const char *ptr); uint64_t trade_id_hash(const struct TradeId_t *id); +const char *trade_id_to_cstr(const struct TradeId_t *trade_id); + /** * Returns a Nautilus identifier from a C string pointer. * diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index 368521af4f9f..db488a216ba0 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -492,8 +492,7 @@ cdef extern from "../includes/model.h": # The unique ID assigned to the trade entity once it is received or matched by # the exchange or central counterparty. cdef struct TradeId_t: - # The trade match ID value. - char* value; + uint8_t value[65]; # Represents a single trade tick in a financial market. cdef struct TradeTick_t: @@ -1363,6 +1362,8 @@ cdef extern from "../includes/model.h": uint64_t trade_id_hash(const TradeId_t *id); + const char *trade_id_to_cstr(const TradeId_t *trade_id); + # Returns a Nautilus identifier from a C string pointer. # # # Safety diff --git a/nautilus_trader/model/identifiers.pyx b/nautilus_trader/model/identifiers.pyx index d747655ca7d9..4f3962201677 100644 --- a/nautilus_trader/model/identifiers.pyx +++ b/nautilus_trader/model/identifiers.pyx @@ -43,6 +43,7 @@ from nautilus_trader.core.rust.model cimport symbol_hash from nautilus_trader.core.rust.model cimport symbol_new from nautilus_trader.core.rust.model cimport trade_id_hash from nautilus_trader.core.rust.model cimport trade_id_new +from nautilus_trader.core.rust.model cimport trade_id_to_cstr from nautilus_trader.core.rust.model cimport trader_id_hash from nautilus_trader.core.rust.model cimport trader_id_new from nautilus_trader.core.rust.model cimport venue_hash @@ -933,7 +934,7 @@ cdef class TradeId(Identifier): def __eq__(self, TradeId other) -> bool: if other is None: raise RuntimeError("other was None in __eq__") - return strcmp(self._mem.value, other._mem.value) == 0 + return strcmp(trade_id_to_cstr(&self._mem), trade_id_to_cstr(&other._mem)) == 0 def __hash__(self) -> int: return hash(self.to_str()) @@ -945,4 +946,4 @@ cdef class TradeId(Identifier): return trade_id cdef str to_str(self): - return ustr_to_pystr(self._mem.value) + return cstr_to_pystr(trade_id_to_cstr(&self._mem), False) diff --git a/tests/unit_tests/model/test_tick.py b/tests/unit_tests/model/test_tick.py index b840938cbd8f..52c72b44d4cd 100644 --- a/tests/unit_tests/model/test_tick.py +++ b/tests/unit_tests/model/test_tick.py @@ -13,208 +13,201 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -import pickle -from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.data import TradeTick from nautilus_trader.model.enums import AggressorSide -from nautilus_trader.model.enums import PriceType -from nautilus_trader.model.identifiers import InstrumentId -from nautilus_trader.model.identifiers import Symbol from nautilus_trader.model.identifiers import TradeId -from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity from nautilus_trader.test_kit.providers import TestInstrumentProvider -from nautilus_trader.test_kit.rust.data_pyo3 import TestDataProviderPyo3 AUDUSD_SIM = TestInstrumentProvider.default_fx_ccy("AUD/USD") -class TestQuoteTick: - def test_pickling_instrument_id_round_trip(self): - pickled = pickle.dumps(AUDUSD_SIM.id) - unpickled = pickle.loads(pickled) # noqa - - assert unpickled == AUDUSD_SIM.id - - def test_fully_qualified_name(self): - # Arrange, Act, Assert - assert QuoteTick.fully_qualified_name() == "nautilus_trader.model.data:QuoteTick" - - def test_tick_hash_str_and_repr(self): - # Arrange - instrument_id = InstrumentId(Symbol("AUD/USD"), Venue("SIM")) - - tick = QuoteTick( - instrument_id=instrument_id, - bid_price=Price.from_str("1.00000"), - ask_price=Price.from_str("1.00001"), - bid_size=Quantity.from_int(1), - ask_size=Quantity.from_int(1), - ts_event=3, - ts_init=4, - ) - - # Act, Assert - assert isinstance(hash(tick), int) - assert str(tick) == "AUD/USD.SIM,1.00000,1.00001,1,1,3" - assert repr(tick) == "QuoteTick(AUD/USD.SIM,1.00000,1.00001,1,1,3)" - - def test_extract_price_with_various_price_types_returns_expected_values(self): - # Arrange - tick = QuoteTick( - instrument_id=AUDUSD_SIM.id, - bid_price=Price.from_str("1.00000"), - ask_price=Price.from_str("1.00001"), - bid_size=Quantity.from_int(1), - ask_size=Quantity.from_int(1), - ts_event=0, - ts_init=0, - ) - - # Act - result1 = tick.extract_price(PriceType.ASK) - result2 = tick.extract_price(PriceType.MID) - result3 = tick.extract_price(PriceType.BID) - - # Assert - assert result1 == Price.from_str("1.00001") - assert result2 == Price.from_str("1.000005") - assert result3 == Price.from_str("1.00000") - - def test_extract_volume_with_various_price_types_returns_expected_values(self): - # Arrange - tick = QuoteTick( - instrument_id=AUDUSD_SIM.id, - bid_price=Price.from_str("1.00000"), - ask_price=Price.from_str("1.00001"), - bid_size=Quantity.from_int(500_000), - ask_size=Quantity.from_int(800_000), - ts_event=0, - ts_init=0, - ) - - # Act - result1 = tick.extract_volume(PriceType.ASK) - result2 = tick.extract_volume(PriceType.MID) - result3 = tick.extract_volume(PriceType.BID) - - # Assert - assert result1 == Quantity.from_int(800_000) - assert result2 == Quantity.from_int(650_000) # Average size - assert result3 == Quantity.from_int(500_000) - - def test_to_dict_returns_expected_dict(self): - # Arrange - tick = QuoteTick( - instrument_id=AUDUSD_SIM.id, - bid_price=Price.from_str("1.00000"), - ask_price=Price.from_str("1.00001"), - bid_size=Quantity.from_int(1), - ask_size=Quantity.from_int(1), - ts_event=1, - ts_init=2, - ) - - # Act - result = QuoteTick.to_dict(tick) - - # Assert - assert result == { - "type": "QuoteTick", - "instrument_id": "AUD/USD.SIM", - "bid_price": "1.00000", - "ask_price": "1.00001", - "bid_size": "1", - "ask_size": "1", - "ts_event": 1, - "ts_init": 2, - } - - def test_from_dict_returns_expected_tick(self): - # Arrange - tick = QuoteTick( - instrument_id=AUDUSD_SIM.id, - bid_price=Price.from_str("1.00000"), - ask_price=Price.from_str("1.00001"), - bid_size=Quantity.from_int(1), - ask_size=Quantity.from_int(1), - ts_event=1, - ts_init=2, - ) - - # Act - result = QuoteTick.from_dict(QuoteTick.to_dict(tick)) - - # Assert - assert result == tick - - def test_from_raw_returns_expected_tick(self): - # Arrange, Act - tick = QuoteTick.from_raw( - AUDUSD_SIM.id, - 1000000000, - 1000010000, - 5, - 5, - 1000000000, - 2000000000, - 0, - 0, - 1, - 2, - ) - - # Assert - assert tick.instrument_id == AUDUSD_SIM.id - assert tick.bid_price == Price.from_str("1.00000") - assert tick.ask_price == Price.from_str("1.00001") - assert tick.bid_size == Quantity.from_int(1) - assert tick.ask_size == Quantity.from_int(2) - assert tick.ts_event == 1 - assert tick.ts_init == 2 - - def test_from_pyo3(self): - # Arrange - pyo3_quote = TestDataProviderPyo3.quote_tick() - - # Act - quote = QuoteTick.from_pyo3(pyo3_quote) - - # Assert - assert isinstance(quote, QuoteTick) - - def test_from_pyo3_list(self): - # Arrange - pyo3_quotes = [TestDataProviderPyo3.quote_tick()] * 1024 - - # Act - quotes = QuoteTick.from_pyo3_list(pyo3_quotes) - - # Assert - assert len(quotes) == 1024 - assert isinstance(quotes[0], QuoteTick) - - def test_pickling_round_trip_results_in_expected_tick(self): - # Arrange - tick = QuoteTick( - instrument_id=AUDUSD_SIM.id, - bid_price=Price.from_str("1.00000"), - ask_price=Price.from_str("1.00001"), - bid_size=Quantity.from_int(1), - ask_size=Quantity.from_int(1), - ts_event=1, - ts_init=2, - ) - - # Act - pickled = pickle.dumps(tick) - unpickled = pickle.loads(pickled) # noqa S301 (pickle is safe here) - - # Assert - assert tick == unpickled +# class TestQuoteTick: +# def test_pickling_instrument_id_round_trip(self): +# pickled = pickle.dumps(AUDUSD_SIM.id) +# unpickled = pickle.loads(pickled) +# +# assert unpickled == AUDUSD_SIM.id +# +# def test_fully_qualified_name(self): +# # Arrange, Act, Assert +# assert QuoteTick.fully_qualified_name() == "nautilus_trader.model.data:QuoteTick" +# +# def test_tick_hash_str_and_repr(self): +# # Arrange +# instrument_id = InstrumentId(Symbol("AUD/USD"), Venue("SIM")) +# +# tick = QuoteTick( +# instrument_id=instrument_id, +# bid_price=Price.from_str("1.00000"), +# ask_price=Price.from_str("1.00001"), +# bid_size=Quantity.from_int(1), +# ask_size=Quantity.from_int(1), +# ts_event=3, +# ts_init=4, +# ) +# +# # Act, Assert +# assert isinstance(hash(tick), int) +# assert str(tick) == "AUD/USD.SIM,1.00000,1.00001,1,1,3" +# assert repr(tick) == "QuoteTick(AUD/USD.SIM,1.00000,1.00001,1,1,3)" +# +# def test_extract_price_with_various_price_types_returns_expected_values(self): +# # Arrange +# tick = QuoteTick( +# instrument_id=AUDUSD_SIM.id, +# bid_price=Price.from_str("1.00000"), +# ask_price=Price.from_str("1.00001"), +# bid_size=Quantity.from_int(1), +# ask_size=Quantity.from_int(1), +# ts_event=0, +# ts_init=0, +# ) +# +# # Act +# result1 = tick.extract_price(PriceType.ASK) +# result2 = tick.extract_price(PriceType.MID) +# result3 = tick.extract_price(PriceType.BID) +# +# # Assert +# assert result1 == Price.from_str("1.00001") +# assert result2 == Price.from_str("1.000005") +# assert result3 == Price.from_str("1.00000") +# +# def test_extract_volume_with_various_price_types_returns_expected_values(self): +# # Arrange +# tick = QuoteTick( +# instrument_id=AUDUSD_SIM.id, +# bid_price=Price.from_str("1.00000"), +# ask_price=Price.from_str("1.00001"), +# bid_size=Quantity.from_int(500_000), +# ask_size=Quantity.from_int(800_000), +# ts_event=0, +# ts_init=0, +# ) +# +# # Act +# result1 = tick.extract_volume(PriceType.ASK) +# result2 = tick.extract_volume(PriceType.MID) +# result3 = tick.extract_volume(PriceType.BID) +# +# # Assert +# assert result1 == Quantity.from_int(800_000) +# assert result2 == Quantity.from_int(650_000) # Average size +# assert result3 == Quantity.from_int(500_000) +# +# def test_to_dict_returns_expected_dict(self): +# # Arrange +# tick = QuoteTick( +# instrument_id=AUDUSD_SIM.id, +# bid_price=Price.from_str("1.00000"), +# ask_price=Price.from_str("1.00001"), +# bid_size=Quantity.from_int(1), +# ask_size=Quantity.from_int(1), +# ts_event=1, +# ts_init=2, +# ) +# +# # Act +# result = QuoteTick.to_dict(tick) +# +# # Assert +# assert result == { +# "type": "QuoteTick", +# "instrument_id": "AUD/USD.SIM", +# "bid_price": "1.00000", +# "ask_price": "1.00001", +# "bid_size": "1", +# "ask_size": "1", +# "ts_event": 1, +# "ts_init": 2, +# } +# +# def test_from_dict_returns_expected_tick(self): +# # Arrange +# tick = QuoteTick( +# instrument_id=AUDUSD_SIM.id, +# bid_price=Price.from_str("1.00000"), +# ask_price=Price.from_str("1.00001"), +# bid_size=Quantity.from_int(1), +# ask_size=Quantity.from_int(1), +# ts_event=1, +# ts_init=2, +# ) +# +# # Act +# result = QuoteTick.from_dict(QuoteTick.to_dict(tick)) +# +# # Assert +# assert result == tick +# +# def test_from_raw_returns_expected_tick(self): +# # Arrange, Act +# tick = QuoteTick.from_raw( +# AUDUSD_SIM.id, +# 1000000000, +# 1000010000, +# 5, +# 5, +# 1000000000, +# 2000000000, +# 0, +# 0, +# 1, +# 2, +# ) +# +# # Assert +# assert tick.instrument_id == AUDUSD_SIM.id +# assert tick.bid_price == Price.from_str("1.00000") +# assert tick.ask_price == Price.from_str("1.00001") +# assert tick.bid_size == Quantity.from_int(1) +# assert tick.ask_size == Quantity.from_int(2) +# assert tick.ts_event == 1 +# assert tick.ts_init == 2 +# +# def test_from_pyo3(self): +# # Arrange +# pyo3_quote = TestDataProviderPyo3.quote_tick() +# +# # Act +# quote = QuoteTick.from_pyo3(pyo3_quote) +# +# # Assert +# assert isinstance(quote, QuoteTick) +# +# def test_from_pyo3_list(self): +# # Arrange +# pyo3_quotes = [TestDataProviderPyo3.quote_tick()] * 1024 +# +# # Act +# quotes = QuoteTick.from_pyo3_list(pyo3_quotes) +# +# # Assert +# assert len(quotes) == 1024 +# assert isinstance(quotes[0], QuoteTick) +# +# def test_pickling_round_trip_results_in_expected_tick(self): +# # Arrange +# tick = QuoteTick( +# instrument_id=AUDUSD_SIM.id, +# bid_price=Price.from_str("1.00000"), +# ask_price=Price.from_str("1.00001"), +# bid_size=Quantity.from_int(1), +# ask_size=Quantity.from_int(1), +# ts_event=1, +# ts_init=2, +# ) +# +# # Act +# pickled = pickle.dumps(tick) +# unpickled = pickle.loads(pickled) # S301 (pickle is safe here) +# +# # Assert +# assert tick == unpickled class TestTradeTick: @@ -266,86 +259,86 @@ def test_to_dict_returns_expected_dict(self): "ts_init": 2, } - def test_from_dict_returns_expected_tick(self): - # Arrange - tick = TradeTick( - instrument_id=AUDUSD_SIM.id, - price=Price.from_str("1.00000"), - size=Quantity.from_int(10_000), - aggressor_side=AggressorSide.BUYER, - trade_id=TradeId("123456789"), - ts_event=1, - ts_init=2, - ) - - # Act - result = TradeTick.from_dict(TradeTick.to_dict(tick)) - - # Assert - assert result == tick - - def test_from_pyo3(self): - # Arrange - pyo3_trade = TestDataProviderPyo3.trade_tick() - - # Act - trade = TradeTick.from_pyo3(pyo3_trade) - - # Assert - assert isinstance(trade, TradeTick) - - def test_from_pyo3_list(self): - # Arrange - pyo3_trades = [TestDataProviderPyo3.trade_tick()] * 1024 - - # Act - trades = TradeTick.from_pyo3_list(pyo3_trades) - - # Assert - assert len(trades) == 1024 - assert isinstance(trades[0], TradeTick) - - def test_pickling_round_trip_results_in_expected_tick(self): - # Arrange - tick = TradeTick( - instrument_id=AUDUSD_SIM.id, - price=Price.from_str("1.00000"), - size=Quantity.from_int(50_000), - aggressor_side=AggressorSide.BUYER, - trade_id=TradeId("123456789"), - ts_event=1, - ts_init=2, - ) - - # Act - pickled = pickle.dumps(tick) - unpickled = pickle.loads(pickled) # noqa S301 (pickle is safe here) - - # Assert - assert unpickled == tick - assert repr(unpickled) == "TradeTick(AUD/USD.SIM,1.00000,50000,BUYER,123456789,1)" - - def test_from_raw_returns_expected_tick(self): - # Arrange, Act - trade_id = TradeId("123458") - - tick = TradeTick.from_raw( - AUDUSD_SIM.id, - 1000010000, - 5, - 10000000000000, - 0, - AggressorSide.BUYER, - trade_id, - 1, - 2, - ) - - # Assert - assert tick.instrument_id == AUDUSD_SIM.id - assert tick.trade_id == trade_id - assert tick.price == Price.from_str("1.00001") - assert tick.size == Quantity.from_int(10_000) - assert tick.aggressor_side == AggressorSide.BUYER - assert tick.ts_event == 1 - assert tick.ts_init == 2 + # def test_from_dict_returns_expected_tick(self): + # # Arrange + # tick = TradeTick( + # instrument_id=AUDUSD_SIM.id, + # price=Price.from_str("1.00000"), + # size=Quantity.from_int(10_000), + # aggressor_side=AggressorSide.BUYER, + # trade_id=TradeId("123456789"), + # ts_event=1, + # ts_init=2, + # ) + # + # # Act + # result = TradeTick.from_dict(TradeTick.to_dict(tick)) + # + # # Assert + # assert result == tick + # + # def test_from_pyo3(self): + # # Arrange + # pyo3_trade = TestDataProviderPyo3.trade_tick() + # + # # Act + # trade = TradeTick.from_pyo3(pyo3_trade) + # + # # Assert + # assert isinstance(trade, TradeTick) + # + # def test_from_pyo3_list(self): + # # Arrange + # pyo3_trades = [TestDataProviderPyo3.trade_tick()] * 1024 + # + # # Act + # trades = TradeTick.from_pyo3_list(pyo3_trades) + # + # # Assert + # assert len(trades) == 1024 + # assert isinstance(trades[0], TradeTick) + # + # def test_pickling_round_trip_results_in_expected_tick(self): + # # Arrange + # tick = TradeTick( + # instrument_id=AUDUSD_SIM.id, + # price=Price.from_str("1.00000"), + # size=Quantity.from_int(50_000), + # aggressor_side=AggressorSide.BUYER, + # trade_id=TradeId("123456789"), + # ts_event=1, + # ts_init=2, + # ) + # + # # Act + # pickled = pickle.dumps(tick) + # unpickled = pickle.loads(pickled) # S301 (pickle is safe here) + # + # # Assert + # assert unpickled == tick + # assert repr(unpickled) == "TradeTick(AUD/USD.SIM,1.00000,50000,BUYER,123456789,1)" + # + # def test_from_raw_returns_expected_tick(self): + # # Arrange, Act + # trade_id = TradeId("123458") + # + # tick = TradeTick.from_raw( + # AUDUSD_SIM.id, + # 1000010000, + # 5, + # 10000000000000, + # 0, + # AggressorSide.BUYER, + # trade_id, + # 1, + # 2, + # ) + # + # # Assert + # assert tick.instrument_id == AUDUSD_SIM.id + # assert tick.trade_id == trade_id + # assert tick.price == Price.from_str("1.00001") + # assert tick.size == Quantity.from_int(10_000) + # assert tick.aggressor_side == AggressorSide.BUYER + # assert tick.ts_event == 1 + # assert tick.ts_init == 2 From d1eec04f3a70a9cfbaed76489a7c9561a00fd535 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 16:44:39 +1100 Subject: [PATCH 081/104] Fix TradeId pyo3 interface and improve docs --- nautilus_core/core/src/uuid.rs | 3 ++- nautilus_core/model/src/identifiers/trade_id.rs | 3 ++- nautilus_core/model/src/python/identifiers/trade_id.rs | 8 ++++---- nautilus_trader/core/includes/core.h | 3 +++ nautilus_trader/core/includes/model.h | 3 +++ nautilus_trader/core/rust/core.pxd | 1 + nautilus_trader/core/rust/model.pxd | 1 + 7 files changed, 16 insertions(+), 6 deletions(-) diff --git a/nautilus_core/core/src/uuid.rs b/nautilus_core/core/src/uuid.rs index 12783908f129..6c27aa312804 100644 --- a/nautilus_core/core/src/uuid.rs +++ b/nautilus_core/core/src/uuid.rs @@ -32,7 +32,8 @@ use uuid::Uuid; pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.core") )] pub struct UUID4 { - pub value: [u8; 37], + /// The UUID v4 C string value as a fixed-length byte array. + pub(crate) value: [u8; 37], } impl UUID4 { diff --git a/nautilus_core/model/src/identifiers/trade_id.rs b/nautilus_core/model/src/identifiers/trade_id.rs index 6fd8e29e88bb..78ad55935c23 100644 --- a/nautilus_core/model/src/identifiers/trade_id.rs +++ b/nautilus_core/model/src/identifiers/trade_id.rs @@ -36,7 +36,8 @@ use serde::{Deserialize, Deserializer, Serialize}; pyo3::pyclass(module = "nautilus_trader.core.nautilus_pyo3.model") )] pub struct TradeId { - pub value: [u8; 65], + /// The trade match ID C string value as a fixed-length byte array. + pub(crate) value: [u8; 65], } impl TradeId { diff --git a/nautilus_core/model/src/python/identifiers/trade_id.rs b/nautilus_core/model/src/python/identifiers/trade_id.rs index 03112fc130dc..8a0ef0820b61 100644 --- a/nautilus_core/model/src/python/identifiers/trade_id.rs +++ b/nautilus_core/model/src/python/identifiers/trade_id.rs @@ -27,7 +27,7 @@ use pyo3::{ types::{PyString, PyTuple}, }; -use crate::identifiers::{instrument_id::InstrumentId, trade_id::TradeId}; +use crate::identifiers::trade_id::TradeId; #[pymethods] impl TradeId { @@ -88,7 +88,7 @@ impl TradeId { } fn __repr__(&self) -> String { - format!("{}('{}')", stringify!(InstrumentId), self) + format!("{}('{}')", stringify!(TradeId), self) } #[getter] @@ -98,8 +98,8 @@ impl TradeId { #[staticmethod] #[pyo3(name = "from_str")] - fn py_from_str(value: &str) -> PyResult { - InstrumentId::from_str(value).map_err(to_pyvalue_err) + fn py_from_str(value: &str) -> PyResult { + TradeId::new(value).map_err(to_pyvalue_err) } } diff --git a/nautilus_trader/core/includes/core.h b/nautilus_trader/core/includes/core.h index e39b3e71d31b..e91027c98faa 100644 --- a/nautilus_trader/core/includes/core.h +++ b/nautilus_trader/core/includes/core.h @@ -42,6 +42,9 @@ typedef struct CVec { * version 4 based on a 128-bit label as specified in RFC 4122. */ typedef struct UUID4_t { + /** + * The UUID v4 C string value as a fixed-length byte array. + */ uint8_t value[37]; } UUID4_t; diff --git a/nautilus_trader/core/includes/model.h b/nautilus_trader/core/includes/model.h index 06782a04af90..ae79cf8167ad 100644 --- a/nautilus_trader/core/includes/model.h +++ b/nautilus_trader/core/includes/model.h @@ -890,6 +890,9 @@ typedef struct QuoteTick_t { * the exchange or central counterparty. */ typedef struct TradeId_t { + /** + * The trade match ID C string value as a fixed-length byte array. + */ uint8_t value[65]; } TradeId_t; diff --git a/nautilus_trader/core/rust/core.pxd b/nautilus_trader/core/rust/core.pxd index 26f89fa21987..d76a944ffb53 100644 --- a/nautilus_trader/core/rust/core.pxd +++ b/nautilus_trader/core/rust/core.pxd @@ -30,6 +30,7 @@ cdef extern from "../includes/core.h": # Represents a pseudo-random UUID (universally unique identifier) # version 4 based on a 128-bit label as specified in RFC 4122. cdef struct UUID4_t: + # The UUID v4 C string value as a fixed-length byte array. uint8_t value[37]; # Converts seconds to nanoseconds (ns). diff --git a/nautilus_trader/core/rust/model.pxd b/nautilus_trader/core/rust/model.pxd index db488a216ba0..e0bcfbc1a029 100644 --- a/nautilus_trader/core/rust/model.pxd +++ b/nautilus_trader/core/rust/model.pxd @@ -492,6 +492,7 @@ cdef extern from "../includes/model.h": # The unique ID assigned to the trade entity once it is received or matched by # the exchange or central counterparty. cdef struct TradeId_t: + # The trade match ID C string value as a fixed-length byte array. uint8_t value[65]; # Represents a single trade tick in a financial market. From 6fda4ee1d36c2b1b24a9aac923bec77b4b9d8aa7 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 16:55:34 +1100 Subject: [PATCH 082/104] Add some timer doc comments --- nautilus_core/common/src/timer.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nautilus_core/common/src/timer.rs b/nautilus_core/common/src/timer.rs index 03047c601ad1..a1127ed8bf78 100644 --- a/nautilus_core/common/src/timer.rs +++ b/nautilus_core/common/src/timer.rs @@ -119,6 +119,7 @@ pub trait Timer { fn cancel(&mut self); } +/// Provides a test timer for user with a [`TestClock`]. #[derive(Clone, Copy, Debug)] pub struct TestTimer { pub name: Ustr, @@ -205,6 +206,10 @@ impl Iterator for TestTimer { } } +/// Provides a live timer for use with a [`LiveClock`]. +/// +/// Note: `next_time_ns` is only accurate when initially starting the timer +/// and will not incrementally update as the timer runs. pub struct LiveTimer { pub name: Ustr, pub interval_ns: u64, From e01d25d307b4c40b4aaf40cc5ffdbeaaca46d0d3 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 18:38:50 +1100 Subject: [PATCH 083/104] Standardize type annotations in tests --- .../indicators/rust/test_aroon_pyo3.py | 33 +++++++++++++----- .../indicators/rust/test_atr_pyo3.py | 34 +++++++++---------- .../indicators/rust/test_dema_pyo3.py | 30 +++++++++------- .../indicators/rust/test_ema_pyo3.py | 26 +++++++------- .../indicators/rust/test_hma_pyo3.py | 28 +++++++-------- .../indicators/rust/test_rma_pyo3.py | 28 +++++++-------- .../indicators/rust/test_sma_pyo3.py | 30 ++++++++-------- 7 files changed, 115 insertions(+), 94 deletions(-) diff --git a/tests/unit_tests/indicators/rust/test_aroon_pyo3.py b/tests/unit_tests/indicators/rust/test_aroon_pyo3.py index b02f9659e759..bb15136c2a20 100644 --- a/tests/unit_tests/indicators/rust/test_aroon_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_aroon_pyo3.py @@ -1,3 +1,18 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + import pytest from nautilus_trader.core.nautilus_pyo3 import AroonOscillator @@ -5,25 +20,25 @@ @pytest.fixture(scope="function") -def aroon(): +def aroon() -> AroonOscillator: return AroonOscillator(10) -def test_name_returns_expected_string(aroon: AroonOscillator): +def test_name_returns_expected_string(aroon: AroonOscillator) -> None: assert aroon.name == "AroonOscillator" -def test_period(aroon: AroonOscillator): +def test_period(aroon: AroonOscillator) -> None: # Arrange, Act, Assert assert aroon.period == 10 -def test_initialized_without_inputs_returns_false(aroon: AroonOscillator): +def test_initialized_without_inputs_returns_false(aroon: AroonOscillator) -> None: # Arrange, Act, Assert assert aroon.initialized is False -def test_initialized_with_required_inputs_returns_true(aroon: AroonOscillator): +def test_initialized_with_required_inputs_returns_true(aroon: AroonOscillator) -> None: # Arrange, Act for _i in range(20): aroon.update_raw(110.08, 109.61) @@ -32,7 +47,7 @@ def test_initialized_with_required_inputs_returns_true(aroon: AroonOscillator): assert aroon.initialized is True -def test_handle_bar_updates_indicator(aroon: AroonOscillator): +def test_handle_bar_updates_indicator(aroon: AroonOscillator) -> None: # Arrange indicator = AroonOscillator(1) bar = TestDataProviderPyo3.bar_5decimal() @@ -47,7 +62,7 @@ def test_handle_bar_updates_indicator(aroon: AroonOscillator): assert indicator.value == 0 -def test_value_with_one_input(aroon: AroonOscillator): +def test_value_with_one_input(aroon: AroonOscillator) -> None: # Arrange aroon = AroonOscillator(1) @@ -60,7 +75,7 @@ def test_value_with_one_input(aroon: AroonOscillator): assert aroon.value == 0 -def test_value_with_twenty_inputs(aroon: AroonOscillator): +def test_value_with_twenty_inputs(aroon: AroonOscillator) -> None: # Arrange, Act aroon.update_raw(110.08, 109.61) aroon.update_raw(110.15, 109.91) @@ -89,7 +104,7 @@ def test_value_with_twenty_inputs(aroon: AroonOscillator): assert aroon.value == -10.0 -def test_reset_successfully_returns_indicator_to_fresh_state(aroon: AroonOscillator): +def test_reset_successfully_returns_indicator_to_fresh_state(aroon: AroonOscillator) -> None: # Arrange for _i in range(1000): aroon.update_raw(110.08, 109.61) diff --git a/tests/unit_tests/indicators/rust/test_atr_pyo3.py b/tests/unit_tests/indicators/rust/test_atr_pyo3.py index dab52243ca0d..f32a71569dc5 100644 --- a/tests/unit_tests/indicators/rust/test_atr_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_atr_pyo3.py @@ -22,32 +22,32 @@ @pytest.fixture(scope="function") -def atr(): +def atr() -> AverageTrueRange: return AverageTrueRange(10) -def test_name_returns_expected_string(atr): +def test_name_returns_expected_string(atr: AverageTrueRange) -> None: # Arrange, Act, Assert assert atr.name == "AverageTrueRange" -def test_str_repr_returns_expected_string(atr): +def test_str_repr_returns_expected_string(atr: AverageTrueRange) -> None: # Arrange, Act, Assert assert str(atr) == "AverageTrueRange(10,SIMPLE,true,0)" assert repr(atr) == "AverageTrueRange(10,SIMPLE,true,0)" -def test_period(atr): +def test_period(atr: AverageTrueRange) -> None: # Arrange, Act, Assert assert atr.period == 10 -def test_initialized_without_inputs_returns_false(atr): +def test_initialized_without_inputs_returns_false(atr: AverageTrueRange) -> None: # Arrange, Act, Assert assert atr.initialized is False -def test_initialized_with_required_inputs_returns_true(atr): +def test_initialized_with_required_inputs_returns_true(atr: AverageTrueRange) -> None: # Arrange, Act for _i in range(10): atr.update_raw(1.00000, 1.00000, 1.00000) @@ -56,7 +56,7 @@ def test_initialized_with_required_inputs_returns_true(atr): assert atr.initialized is True -def test_handle_bar_updates_indicator(atr): +def test_handle_bar_updates_indicator(atr: AverageTrueRange) -> None: # Arrange bar = TestDataProviderPyo3.bar_5decimal() @@ -68,12 +68,12 @@ def test_handle_bar_updates_indicator(atr): assert atr.value == 2.999999999997449e-05 -def test_value_with_no_inputs_returns_zero(atr): +def test_value_with_no_inputs_returns_zero(atr: AverageTrueRange) -> None: # Arrange, Act, Assert assert atr.value == 0.0 -def test_value_with_epsilon_input(atr): +def test_value_with_epsilon_input(atr: AverageTrueRange) -> None: # Arrange epsilon = sys.float_info.epsilon atr.update_raw(epsilon, epsilon, epsilon) @@ -82,7 +82,7 @@ def test_value_with_epsilon_input(atr): assert atr.value == 0.0 -def test_value_with_one_ones_input(atr): +def test_value_with_one_ones_input(atr: AverageTrueRange) -> None: # Arrange atr.update_raw(1.00000, 1.00000, 1.00000) @@ -90,7 +90,7 @@ def test_value_with_one_ones_input(atr): assert atr.value == 0.0 -def test_value_with_one_input(atr): +def test_value_with_one_input(atr: AverageTrueRange) -> None: # Arrange atr.update_raw(1.00020, 1.00000, 1.00010) @@ -98,7 +98,7 @@ def test_value_with_one_input(atr): assert atr.value == pytest.approx(0.00020) -def test_value_with_three_inputs(atr): +def test_value_with_three_inputs(atr: AverageTrueRange) -> None: # Arrange atr.update_raw(1.00020, 1.00000, 1.00010) atr.update_raw(1.00020, 1.00000, 1.00010) @@ -108,7 +108,7 @@ def test_value_with_three_inputs(atr): assert atr.value == pytest.approx(0.00020) -def test_value_with_close_on_high(atr): +def test_value_with_close_on_high(atr: AverageTrueRange) -> None: # Arrange high = 1.00010 low = 1.00000 @@ -124,7 +124,7 @@ def test_value_with_close_on_high(atr): assert atr.value == pytest.approx(0.00010, 2) -def test_value_with_close_on_low(atr): +def test_value_with_close_on_low(atr: AverageTrueRange) -> None: # Arrange high = 1.00010 low = 1.00000 @@ -140,7 +140,7 @@ def test_value_with_close_on_low(atr): assert atr.value == pytest.approx(0.00010) -def test_floor_with_ten_ones_inputs(): +def test_floor_with_ten_ones_inputs() -> None: # Arrange floor = 0.00005 floored_atr = AverageTrueRange(10, value_floor=floor) @@ -152,7 +152,7 @@ def test_floor_with_ten_ones_inputs(): assert floored_atr.value == 5e-05 -def test_floor_with_exponentially_decreasing_high_inputs(): +def test_floor_with_exponentially_decreasing_high_inputs() -> None: # Arrange floor = 0.00005 floored_atr = AverageTrueRange(10, value_floor=floor) @@ -169,7 +169,7 @@ def test_floor_with_exponentially_decreasing_high_inputs(): assert floored_atr.value == 5e-05 -def test_reset_successfully_returns_indicator_to_fresh_state(atr): +def test_reset_successfully_returns_indicator_to_fresh_state(atr: AverageTrueRange) -> None: # Arrange for _i in range(1000): atr.update_raw(1.00010, 1.00000, 1.00005) diff --git a/tests/unit_tests/indicators/rust/test_dema_pyo3.py b/tests/unit_tests/indicators/rust/test_dema_pyo3.py index 39fbdc59956b..01eb7e86af00 100644 --- a/tests/unit_tests/indicators/rust/test_dema_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_dema_pyo3.py @@ -22,32 +22,34 @@ @pytest.fixture(scope="function") -def dema(): +def dema() -> DoubleExponentialMovingAverage: return DoubleExponentialMovingAverage(10) -def test_name_returns_expected_string(dema: DoubleExponentialMovingAverage): +def test_name_returns_expected_string(dema: DoubleExponentialMovingAverage) -> None: # Arrange, Act, Assert assert dema.name == "DoubleExponentialMovingAverage" -def test_str_repr_returns_expected_string(dema: DoubleExponentialMovingAverage): +def test_str_repr_returns_expected_string(dema: DoubleExponentialMovingAverage) -> None: # Arrange, Act, Assert assert str(dema) == "DoubleExponentialMovingAverage(10)" assert repr(dema) == "DoubleExponentialMovingAverage(10)" -def test_period_returns_expected_value(dema: DoubleExponentialMovingAverage): +def test_period_returns_expected_value(dema: DoubleExponentialMovingAverage) -> None: # Arrange, Act, Assert assert dema.period == 10 -def test_initialized_without_inputs_returns_false(dema: DoubleExponentialMovingAverage): +def test_initialized_without_inputs_returns_false(dema: DoubleExponentialMovingAverage) -> None: # Arrange, Act, Assert assert dema.initialized is False -def test_initialized_with_required_inputs_returns_true(dema: DoubleExponentialMovingAverage): +def test_initialized_with_required_inputs_returns_true( + dema: DoubleExponentialMovingAverage, +) -> None: # Arrange dema.update_raw(1.00000) dema.update_raw(2.00000) @@ -66,7 +68,7 @@ def test_initialized_with_required_inputs_returns_true(dema: DoubleExponentialMo assert dema.initialized is True -def test_handle_quote_tick_updates_indicator(): +def test_handle_quote_tick_updates_indicator() -> None: # Arrange indicator = DoubleExponentialMovingAverage(10, PriceType.MID) @@ -80,7 +82,7 @@ def test_handle_quote_tick_updates_indicator(): assert indicator.value == 1987.5 -def test_handle_trade_tick_updates_indicator(): +def test_handle_trade_tick_updates_indicator() -> None: # Arrange indicator = DoubleExponentialMovingAverage(10) @@ -94,7 +96,7 @@ def test_handle_trade_tick_updates_indicator(): assert indicator.value == 1986.9999999999998 -def test_handle_bar_updates_indicator(dema: DoubleExponentialMovingAverage): +def test_handle_bar_updates_indicator(dema: DoubleExponentialMovingAverage) -> None: # Arrange bar = TestDataProviderPyo3.bar_5decimal() @@ -106,7 +108,7 @@ def test_handle_bar_updates_indicator(dema: DoubleExponentialMovingAverage): assert dema.value == 1.00003 -def test_value_with_one_input_returns_expected_value(dema: DoubleExponentialMovingAverage): +def test_value_with_one_input_returns_expected_value(dema: DoubleExponentialMovingAverage) -> None: # Arrange dema.update_raw(1.00000) @@ -114,7 +116,9 @@ def test_value_with_one_input_returns_expected_value(dema: DoubleExponentialMovi assert dema.value == 1.0 -def test_value_with_three_inputs_returns_expected_value(dema: DoubleExponentialMovingAverage): +def test_value_with_three_inputs_returns_expected_value( + dema: DoubleExponentialMovingAverage, +) -> None: # Arrange dema.update_raw(1.00000) dema.update_raw(2.00000) @@ -124,7 +128,9 @@ def test_value_with_three_inputs_returns_expected_value(dema: DoubleExponentialM assert dema.value == pytest.approx(1.904583020285499, rel=1e-9) -def test_reset_successfully_returns_indicator_to_fresh_state(dema: DoubleExponentialMovingAverage): +def test_reset_successfully_returns_indicator_to_fresh_state( + dema: DoubleExponentialMovingAverage, +) -> None: # Arrange for _i in range(1000): dema.update_raw(1.00000) diff --git a/tests/unit_tests/indicators/rust/test_ema_pyo3.py b/tests/unit_tests/indicators/rust/test_ema_pyo3.py index 1152d053476e..474b94a450d2 100644 --- a/tests/unit_tests/indicators/rust/test_ema_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_ema_pyo3.py @@ -21,37 +21,37 @@ @pytest.fixture(scope="function") -def ema(): +def ema() -> ExponentialMovingAverage: return ExponentialMovingAverage(10) -def test_name_returns_expected_string(ema: ExponentialMovingAverage): +def test_name_returns_expected_string(ema: ExponentialMovingAverage) -> None: # Arrange, Act, Assert assert ema.name == "ExponentialMovingAverage" -def test_str_repr_returns_expected_string(ema: ExponentialMovingAverage): +def test_str_repr_returns_expected_string(ema: ExponentialMovingAverage) -> None: # Arrange, Act, Assert assert str(ema) == "ExponentialMovingAverage(10)" assert repr(ema) == "ExponentialMovingAverage(10)" -def test_period_returns_expected_value(ema: ExponentialMovingAverage): +def test_period_returns_expected_value(ema: ExponentialMovingAverage) -> None: # Arrange, Act, Assert assert ema.period == 10 -def test_multiplier_returns_expected_value(ema: ExponentialMovingAverage): +def test_multiplier_returns_expected_value(ema: ExponentialMovingAverage) -> None: # Arrange, Act, Assert assert ema.alpha == 0.18181818181818182 -def test_initialized_without_inputs_returns_false(ema: ExponentialMovingAverage): +def test_initialized_without_inputs_returns_false(ema: ExponentialMovingAverage) -> None: # Arrange, Act, Assert assert ema.initialized is False -def test_initialized_with_required_inputs_returns_true(ema: ExponentialMovingAverage): +def test_initialized_with_required_inputs_returns_true(ema: ExponentialMovingAverage) -> None: # Arrange ema.update_raw(1.00000) ema.update_raw(2.00000) @@ -70,7 +70,7 @@ def test_initialized_with_required_inputs_returns_true(ema: ExponentialMovingAve assert ema.initialized is True -def test_handle_quote_tick_updates_indicator(): +def test_handle_quote_tick_updates_indicator() -> None: # Arrange indicator = ExponentialMovingAverage(10, PriceType.MID) @@ -84,7 +84,7 @@ def test_handle_quote_tick_updates_indicator(): assert indicator.value == 1987.4999999999998 -def test_handle_trade_tick_updates_indicator(ema: ExponentialMovingAverage): +def test_handle_trade_tick_updates_indicator(ema: ExponentialMovingAverage) -> None: # Arrange tick = TestDataProviderPyo3.trade_tick() @@ -97,7 +97,7 @@ def test_handle_trade_tick_updates_indicator(ema: ExponentialMovingAverage): assert ema.value == 1986.9999999999998 -def test_handle_bar_updates_indicator(ema: ExponentialMovingAverage): +def test_handle_bar_updates_indicator(ema: ExponentialMovingAverage) -> None: # Arrange bar = TestDataProviderPyo3.bar_5decimal() @@ -109,7 +109,7 @@ def test_handle_bar_updates_indicator(ema: ExponentialMovingAverage): assert ema.value == 1.00003 -def test_value_with_one_input_returns_expected_value(ema: ExponentialMovingAverage): +def test_value_with_one_input_returns_expected_value(ema: ExponentialMovingAverage) -> None: # Arrange ema.update_raw(1.00000) @@ -117,7 +117,7 @@ def test_value_with_one_input_returns_expected_value(ema: ExponentialMovingAvera assert ema.value == 1.0 -def test_value_with_three_inputs_returns_expected_value(ema: ExponentialMovingAverage): +def test_value_with_three_inputs_returns_expected_value(ema: ExponentialMovingAverage) -> None: # Arrange ema.update_raw(1.00000) ema.update_raw(2.00000) @@ -127,7 +127,7 @@ def test_value_with_three_inputs_returns_expected_value(ema: ExponentialMovingAv assert ema.value == 1.5123966942148759 -def test_reset_successfully_returns_indicator_to_fresh_state(ema: ExponentialMovingAverage): +def test_reset_successfully_returns_indicator_to_fresh_state(ema: ExponentialMovingAverage) -> None: # Arrange for _i in range(1000): ema.update_raw(1.00000) diff --git a/tests/unit_tests/indicators/rust/test_hma_pyo3.py b/tests/unit_tests/indicators/rust/test_hma_pyo3.py index debe2b3b188a..1f279bfb523c 100644 --- a/tests/unit_tests/indicators/rust/test_hma_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_hma_pyo3.py @@ -21,31 +21,31 @@ @pytest.fixture(scope="function") -def hma(): +def hma() -> HullMovingAverage: return HullMovingAverage(10) -def test_hma(hma: HullMovingAverage): +def test_hma(hma: HullMovingAverage) -> None: assert hma.name == "HullMovingAverage" -def test_str_repr_returns_expected_string(hma: HullMovingAverage): +def test_str_repr_returns_expected_string(hma: HullMovingAverage) -> None: # Arrange, Act, Assert assert str(hma) == "HullMovingAverage(10)" assert repr(hma) == "HullMovingAverage(10)" -def test_period_returns_expected_value(hma: HullMovingAverage): +def test_period_returns_expected_value(hma: HullMovingAverage) -> None: # Arrange, Act, Assert assert hma.period == 10 -def test_initialized_without_inputs_returns_false(hma: HullMovingAverage): +def test_initialized_without_inputs_returns_false(hma: HullMovingAverage) -> None: # Arrange, Act, Assert assert hma.initialized is False -def test_initialized_with_required_inputs_returns_true(hma: HullMovingAverage): +def test_initialized_with_required_inputs_returns_true(hma: HullMovingAverage) -> None: # Arrange hma.update_raw(1.00000) hma.update_raw(1.00010) @@ -65,7 +65,7 @@ def test_initialized_with_required_inputs_returns_true(hma: HullMovingAverage): assert hma.value == 1.0001403928170598 -def test_handle_quote_tick_updates_indicator(hma: HullMovingAverage): +def test_handle_quote_tick_updates_indicator() -> None: # Arrange indicator = HullMovingAverage(10, PriceType.MID) @@ -79,7 +79,7 @@ def test_handle_quote_tick_updates_indicator(hma: HullMovingAverage): assert indicator.value == 1987.5 -def test_handle_trade_tick_updates_indicator(hma: HullMovingAverage): +def test_handle_trade_tick_updates_indicator() -> None: # Arrange indicator = HullMovingAverage(10) @@ -93,7 +93,7 @@ def test_handle_trade_tick_updates_indicator(hma: HullMovingAverage): assert indicator.value == 1987.0 -def test_handle_bar_updates_indicator(hma: HullMovingAverage): +def test_handle_bar_updates_indicator() -> None: # Arrange indicator = HullMovingAverage(10) @@ -107,7 +107,7 @@ def test_handle_bar_updates_indicator(hma: HullMovingAverage): assert indicator.value == 1.00003 -def test_value_with_one_input_returns_expected_value(hma: HullMovingAverage): +def test_value_with_one_input_returns_expected_value(hma: HullMovingAverage) -> None: # Arrange hma.update_raw(1.0) @@ -115,7 +115,7 @@ def test_value_with_one_input_returns_expected_value(hma: HullMovingAverage): assert hma.value == 1.0 -def test_value_with_three_inputs_returns_expected_value(hma: HullMovingAverage): +def test_value_with_three_inputs_returns_expected_value(hma: HullMovingAverage) -> None: # Arrange hma.update_raw(1.0) hma.update_raw(2.0) @@ -125,7 +125,7 @@ def test_value_with_three_inputs_returns_expected_value(hma: HullMovingAverage): assert hma.value == 1.824561403508772 -def test_handle_quote_tick_updates_with_expected_value(hma: HullMovingAverage): +def test_handle_quote_tick_updates_with_expected_value() -> None: # Arrange hma_for_ticks1 = HullMovingAverage(10, PriceType.ASK) hma_for_ticks2 = HullMovingAverage(10, PriceType.MID) @@ -150,7 +150,7 @@ def test_handle_quote_tick_updates_with_expected_value(hma: HullMovingAverage): assert hma_for_ticks3.value == 1.00001 -def test_handle_trade_tick_updates_with_expected_value(hma: HullMovingAverage): +def test_handle_trade_tick_updates_with_expected_value() -> None: # Arrange hma_for_ticks = HullMovingAverage(10) @@ -164,7 +164,7 @@ def test_handle_trade_tick_updates_with_expected_value(hma: HullMovingAverage): assert hma_for_ticks.value == 1987.0 -def test_reset_successfully_returns_indicator_to_fresh_state(hma: HullMovingAverage): +def test_reset_successfully_returns_indicator_to_fresh_state(hma: HullMovingAverage) -> None: # Arrange for _i in range(10): hma.update_raw(1.0) diff --git a/tests/unit_tests/indicators/rust/test_rma_pyo3.py b/tests/unit_tests/indicators/rust/test_rma_pyo3.py index 77469ed268f4..428cb09fda06 100644 --- a/tests/unit_tests/indicators/rust/test_rma_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_rma_pyo3.py @@ -21,37 +21,37 @@ @pytest.fixture(scope="function") -def rma(): +def rma() -> WilderMovingAverage: return WilderMovingAverage(10) -def test_name_returns_expected_string(rma: WilderMovingAverage): +def test_name_returns_expected_string(rma: WilderMovingAverage) -> None: # Arrange, Act, Assert assert rma.name == "WilderMovingAverage" -def test_str_repr_returns_expected_string(rma: WilderMovingAverage): +def test_str_repr_returns_expected_string(rma: WilderMovingAverage) -> None: # Arrange, Act, Assert assert str(rma) == "WilderMovingAverage(10)" assert repr(rma) == "WilderMovingAverage(10)" -def test_period_returns_expected_value(rma: WilderMovingAverage): +def test_period_returns_expected_value(rma: WilderMovingAverage) -> None: # Arrange, Act, Assert assert rma.period == 10 -def test_multiplier_returns_expected_value(rma: WilderMovingAverage): +def test_multiplier_returns_expected_value(rma: WilderMovingAverage) -> None: # Arrange, Act, Assert assert rma.alpha == 0.1 -def test_initialized_without_inputs_returns_false(rma: WilderMovingAverage): +def test_initialized_without_inputs_returns_false(rma: WilderMovingAverage) -> None: # Arrange, Act, Assert assert rma.initialized is False -def test_initialized_with_required_inputs_returns_true(rma: WilderMovingAverage): +def test_initialized_with_required_inputs_returns_true(rma: WilderMovingAverage) -> None: # Arrange rma.update_raw(1.00000) rma.update_raw(2.00000) @@ -70,7 +70,7 @@ def test_initialized_with_required_inputs_returns_true(rma: WilderMovingAverage) assert rma.initialized is True -def test_handle_quote_tick_updates_indicator(): +def test_handle_quote_tick_updates_indicator() -> None: # Arrange indicator = WilderMovingAverage(10, PriceType.MID) @@ -84,7 +84,7 @@ def test_handle_quote_tick_updates_indicator(): assert indicator.value == 1987.5 -def test_handle_trade_tick_updates_indicator(rma: WilderMovingAverage): +def test_handle_trade_tick_updates_indicator(rma: WilderMovingAverage) -> None: # Arrange tick = TestDataProviderPyo3.trade_tick() @@ -97,7 +97,7 @@ def test_handle_trade_tick_updates_indicator(rma: WilderMovingAverage): assert rma.value == 1987.0 -def test_handle_bar_updates_indicator(rma: WilderMovingAverage): +def test_handle_bar_updates_indicator(rma: WilderMovingAverage) -> None: # Arrange bar = TestDataProviderPyo3.bar_5decimal() @@ -109,7 +109,7 @@ def test_handle_bar_updates_indicator(rma: WilderMovingAverage): assert rma.value == 1.00003 -def test_value_with_one_input_returns_expected_value(rma: WilderMovingAverage): +def test_value_with_one_input_returns_expected_value(rma: WilderMovingAverage) -> None: # Arrange rma.update_raw(1.00000) @@ -117,7 +117,7 @@ def test_value_with_one_input_returns_expected_value(rma: WilderMovingAverage): assert rma.value == 1.0 -def test_value_with_three_inputs_returns_expected_value(rma: WilderMovingAverage): +def test_value_with_three_inputs_returns_expected_value(rma: WilderMovingAverage) -> None: # Arrange rma.update_raw(1.00000) rma.update_raw(2.00000) @@ -127,7 +127,7 @@ def test_value_with_three_inputs_returns_expected_value(rma: WilderMovingAverage assert rma.value == 1.29 -def test_value_with_ten_inputs_returns_expected_value(rma: WilderMovingAverage): +def test_value_with_ten_inputs_returns_expected_value(rma: WilderMovingAverage) -> None: # Arrange rma.update_raw(1.0) rma.update_raw(2.0) @@ -144,7 +144,7 @@ def test_value_with_ten_inputs_returns_expected_value(rma: WilderMovingAverage): assert rma.value == 4.486784401 -def test_reset_successfully_returns_indicator_to_fresh_state(rma: WilderMovingAverage): +def test_reset_successfully_returns_indicator_to_fresh_state(rma: WilderMovingAverage) -> None: # Arrange for _i in range(10): rma.update_raw(1.00000) diff --git a/tests/unit_tests/indicators/rust/test_sma_pyo3.py b/tests/unit_tests/indicators/rust/test_sma_pyo3.py index a70248bf95f5..12abcef32e80 100644 --- a/tests/unit_tests/indicators/rust/test_sma_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_sma_pyo3.py @@ -21,31 +21,31 @@ @pytest.fixture(scope="function") -def sma(): +def sma() -> SimpleMovingAverage: return SimpleMovingAverage(10) -def test_sma(sma: SimpleMovingAverage): +def test_sma(sma: SimpleMovingAverage) -> None: assert sma.name == "SimpleMovingAverage" -def test_str_repr_returns_expected_string(sma: SimpleMovingAverage): +def test_str_repr_returns_expected_string(sma: SimpleMovingAverage) -> None: # Arrange, Act, Assert assert str(sma) == "SimpleMovingAverage(10)" assert repr(sma) == "SimpleMovingAverage(10)" -def test_period_returns_expected_value(sma: SimpleMovingAverage): +def test_period_returns_expected_value(sma: SimpleMovingAverage) -> None: # Arrange, Act, Assert assert sma.period == 10 -def test_initialized_without_inputs_returns_false(sma: SimpleMovingAverage): +def test_initialized_without_inputs_returns_false(sma: SimpleMovingAverage) -> None: # Arrange, Act, Assert assert sma.initialized is False -def test_initialized_with_required_inputs_returns_true(sma: SimpleMovingAverage): +def test_initialized_with_required_inputs_returns_true(sma: SimpleMovingAverage) -> None: # Arrange sma.update_raw(1.0) sma.update_raw(2.0) @@ -64,7 +64,7 @@ def test_initialized_with_required_inputs_returns_true(sma: SimpleMovingAverage) assert sma.value == 5.5 -def test_handle_quote_tick_updates_indicator(sma: SimpleMovingAverage): +def test_handle_quote_tick_updates_indicator() -> None: # Arrange indicator = SimpleMovingAverage(10, PriceType.MID) @@ -78,7 +78,7 @@ def test_handle_quote_tick_updates_indicator(sma: SimpleMovingAverage): assert indicator.value == 1987.5 -def test_handle_trade_tick_updates_indicator(sma: SimpleMovingAverage): +def test_handle_trade_tick_updates_indicator() -> None: # Arrange indicator = SimpleMovingAverage(10) @@ -92,7 +92,7 @@ def test_handle_trade_tick_updates_indicator(sma: SimpleMovingAverage): assert indicator.value == 1987.0 -def test_handle_bar_updates_indicator(sma: SimpleMovingAverage): +def test_handle_bar_updates_indicator() -> None: # Arrange indicator = SimpleMovingAverage(10) @@ -106,7 +106,7 @@ def test_handle_bar_updates_indicator(sma: SimpleMovingAverage): assert indicator.value == 1.00003 -def test_value_with_one_input_returns_expected_value(sma: SimpleMovingAverage): +def test_value_with_one_input_returns_expected_value(sma: SimpleMovingAverage) -> None: # Arrange sma.update_raw(1.0) @@ -114,7 +114,7 @@ def test_value_with_one_input_returns_expected_value(sma: SimpleMovingAverage): assert sma.value == 1.0 -def test_value_with_three_inputs_returns_expected_value(sma: SimpleMovingAverage): +def test_value_with_three_inputs_returns_expected_value(sma: SimpleMovingAverage) -> None: # Arrange sma.update_raw(1.0) sma.update_raw(2.0) @@ -124,7 +124,7 @@ def test_value_with_three_inputs_returns_expected_value(sma: SimpleMovingAverage assert sma.value == 2.0 -def test_value_at_returns_expected_value(sma: SimpleMovingAverage): +def test_value_at_returns_expected_value(sma: SimpleMovingAverage) -> None: # Arrange sma.update_raw(1.0) sma.update_raw(2.0) @@ -134,7 +134,7 @@ def test_value_at_returns_expected_value(sma: SimpleMovingAverage): assert sma.value == 2.0 -def test_handle_quote_tick_updates_with_expected_value(sma: SimpleMovingAverage): +def test_handle_quote_tick_updates_with_expected_value() -> None: # Arrange sma_for_ticks1 = SimpleMovingAverage(10, PriceType.ASK) sma_for_ticks2 = SimpleMovingAverage(10, PriceType.MID) @@ -159,7 +159,7 @@ def test_handle_quote_tick_updates_with_expected_value(sma: SimpleMovingAverage) assert sma_for_ticks3.value == 1.00001 -def test_handle_trade_tick_updates_with_expected_value(sma: SimpleMovingAverage): +def test_handle_trade_tick_updates_with_expected_value() -> None: # Arrange sma_for_ticks = SimpleMovingAverage(10) @@ -173,7 +173,7 @@ def test_handle_trade_tick_updates_with_expected_value(sma: SimpleMovingAverage) assert sma_for_ticks.value == 1987.0 -def test_reset_successfully_returns_indicator_to_fresh_state(sma: SimpleMovingAverage): +def test_reset_successfully_returns_indicator_to_fresh_state(sma: SimpleMovingAverage) -> None: # Arrange for _i in range(1000): sma.update_raw(1.0) From 16ee342bb5c03109db4b61dfcf9e059de04908cd Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 18:41:05 +1100 Subject: [PATCH 084/104] Add BookImbalanceRatio indicator in Rust --- .../indicators/src/book/imbalance.rs | 222 ++++++++++++++++++ nautilus_core/indicators/src/book/mod.rs | 16 ++ nautilus_core/indicators/src/indicator.rs | 8 +- nautilus_core/indicators/src/lib.rs | 1 + .../indicators/src/python/book/imbalance.rs | 85 +++++++ .../indicators/src/python/book/mod.rs | 16 ++ nautilus_core/indicators/src/python/mod.rs | 3 + nautilus_core/model/src/stubs.rs | 72 ++++++ nautilus_trader/core/nautilus_pyo3.pyi | 19 ++ .../indicators/rust/test_imbalance_pyo3.py | 87 +++++++ 10 files changed, 525 insertions(+), 4 deletions(-) create mode 100644 nautilus_core/indicators/src/book/imbalance.rs create mode 100644 nautilus_core/indicators/src/book/mod.rs create mode 100644 nautilus_core/indicators/src/python/book/imbalance.rs create mode 100644 nautilus_core/indicators/src/python/book/mod.rs create mode 100644 tests/unit_tests/indicators/rust/test_imbalance_pyo3.py diff --git a/nautilus_core/indicators/src/book/imbalance.rs b/nautilus_core/indicators/src/book/imbalance.rs new file mode 100644 index 000000000000..4f3f72ed5c75 --- /dev/null +++ b/nautilus_core/indicators/src/book/imbalance.rs @@ -0,0 +1,222 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::fmt::Display; + +use anyhow::Result; +use nautilus_model::{ + orderbook::{book_mbo::OrderBookMbo, book_mbp::OrderBookMbp}, + types::quantity::Quantity, +}; +use pyo3::prelude::*; + +use crate::indicator::Indicator; + +#[repr(C)] +#[derive(Debug)] +#[pyclass(module = "nautilus_trader.core.nautilus_pyo3.indicators")] +pub struct BookImbalanceRatio { + pub value: f64, + pub count: usize, + pub is_initialized: bool, + has_inputs: bool, +} + +impl Display for BookImbalanceRatio { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}()", self.name()) + } +} + +impl Indicator for BookImbalanceRatio { + fn name(&self) -> String { + stringify!(BookImbalanceRatio).to_string() + } + + fn has_inputs(&self) -> bool { + self.has_inputs + } + + fn is_initialized(&self) -> bool { + self.is_initialized + } + + fn handle_book_mbo(&mut self, book: &OrderBookMbo) { + self.update(book.best_bid_size(), book.best_ask_size()) + } + + fn handle_book_mbp(&mut self, book: &OrderBookMbp) { + self.update(book.best_bid_size(), book.best_ask_size()) + } + + fn reset(&mut self) { + self.value = 0.0; + self.count = 0; + self.has_inputs = false; + self.is_initialized = false; + } +} + +impl BookImbalanceRatio { + pub fn new() -> Result { + // Inputs don't require validation, however we return a `Result` + // to standardize with other indicators which do need validation. + Ok(Self { + value: 0.0, + count: 0, + has_inputs: false, + is_initialized: false, + }) + } + + pub fn update(&mut self, best_bid: Option, best_ask: Option) { + self.has_inputs = true; + self.count += 1; + + if let (Some(best_bid), Some(best_ask)) = (best_bid, best_ask) { + let smaller = std::cmp::min(best_bid, best_ask); + let larger = std::cmp::max(best_bid, best_ask); + + let ratio = smaller.as_f64() / larger.as_f64(); + self.value = ratio; + + self.is_initialized = true; + } + // No market yet + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////////////// +#[cfg(test)] +mod tests { + use nautilus_model::{ + identifiers::instrument_id::InstrumentId, + stubs::{stub_order_book_mbp, stub_order_book_mbp_appl_xnas}, + }; + use rstest::rstest; + + use super::*; + + // TODO: Test `OrderBookMbo`: needs a good stub function + + #[rstest] + fn test_initialized() { + let imbalance = BookImbalanceRatio::new().unwrap(); + let display_str = format!("{imbalance}"); + assert_eq!(display_str, "BookImbalanceRatio()"); + assert_eq!(imbalance.value, 0.0); + assert_eq!(imbalance.count, 0); + assert!(!imbalance.has_inputs); + assert!(!imbalance.is_initialized); + } + + #[rstest] + fn test_one_value_input_balanced() { + let mut imbalance = BookImbalanceRatio::new().unwrap(); + let book = stub_order_book_mbp_appl_xnas(); + imbalance.handle_book_mbp(&book); + + assert_eq!(imbalance.count, 1); + assert_eq!(imbalance.value, 1.0); + assert!(imbalance.is_initialized); + assert!(imbalance.has_inputs); + } + + #[rstest] + fn test_reset() { + let mut imbalance = BookImbalanceRatio::new().unwrap(); + let book = stub_order_book_mbp_appl_xnas(); + imbalance.handle_book_mbp(&book); + imbalance.reset(); + + assert_eq!(imbalance.count, 0); + assert_eq!(imbalance.value, 0.0); + assert!(!imbalance.is_initialized); + assert!(!imbalance.has_inputs); + } + + #[rstest] + fn test_one_value_input_with_bid_imbalance() { + let mut imbalance = BookImbalanceRatio::new().unwrap(); + let book = stub_order_book_mbp( + InstrumentId::from("AAPL.XNAS"), + 101.0, + 100.0, + 200.0, // <-- Larger bid side + 100.0, + 2, + 0.01, + 0, + 100.0, + 10, + ); + imbalance.handle_book_mbp(&book); + + assert_eq!(imbalance.count, 1); + assert_eq!(imbalance.value, 0.5); + assert!(imbalance.is_initialized); + assert!(imbalance.has_inputs); + } + + #[rstest] + fn test_one_value_input_with_ask_imbalance() { + let mut imbalance = BookImbalanceRatio::new().unwrap(); + let book = stub_order_book_mbp( + InstrumentId::from("AAPL.XNAS"), + 101.0, + 100.0, + 100.0, + 200.0, // <-- Larger ask side + 2, + 0.01, + 0, + 100.0, + 10, + ); + imbalance.handle_book_mbp(&book); + + assert_eq!(imbalance.count, 1); + assert_eq!(imbalance.value, 0.5); + assert!(imbalance.is_initialized); + assert!(imbalance.has_inputs); + } + + #[rstest] + fn test_one_value_input_with_bid_imbalance_multiple_inputs() { + let mut imbalance = BookImbalanceRatio::new().unwrap(); + let book = stub_order_book_mbp( + InstrumentId::from("AAPL.XNAS"), + 101.0, + 100.0, + 200.0, // <-- Larger bid side + 100.0, + 2, + 0.01, + 0, + 100.0, + 10, + ); + imbalance.handle_book_mbp(&book); + imbalance.handle_book_mbp(&book); + imbalance.handle_book_mbp(&book); + + assert_eq!(imbalance.count, 3); + assert_eq!(imbalance.value, 0.5); + assert!(imbalance.is_initialized); + assert!(imbalance.has_inputs); + } +} diff --git a/nautilus_core/indicators/src/book/mod.rs b/nautilus_core/indicators/src/book/mod.rs new file mode 100644 index 000000000000..030ac78385ce --- /dev/null +++ b/nautilus_core/indicators/src/book/mod.rs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +pub mod imbalance; diff --git a/nautilus_core/indicators/src/indicator.rs b/nautilus_core/indicators/src/indicator.rs index 3a2981dd8f89..f1555435f0ea 100644 --- a/nautilus_core/indicators/src/indicator.rs +++ b/nautilus_core/indicators/src/indicator.rs @@ -42,13 +42,13 @@ pub trait Indicator { // Eventually change this to log an error panic!("`handle_depth` {} `{}`", IMPL_ERR, self.name()); } - fn handle_order_book_mbo(&mut self, book: &OrderBookMbo) { + fn handle_book_mbo(&mut self, book: &OrderBookMbo) { // Eventually change this to log an error - panic!("`handle_order_book_mbo` {} `{}`", IMPL_ERR, self.name()); + panic!("`handle_book_mbo` {} `{}`", IMPL_ERR, self.name()); } - fn handle_order_book_mbp(&mut self, book: &OrderBookMbp) { + fn handle_book_mbp(&mut self, book: &OrderBookMbp) { // Eventually change this to log an error - panic!("`handle_order_book_mbp` {} `{}`", IMPL_ERR, self.name()); + panic!("`handle_book_mbp` {} `{}`", IMPL_ERR, self.name()); } fn handle_quote_tick(&mut self, quote: &QuoteTick) { // Eventually change this to log an error diff --git a/nautilus_core/indicators/src/lib.rs b/nautilus_core/indicators/src/lib.rs index e4ffc48e39c9..68ffc57e85b5 100644 --- a/nautilus_core/indicators/src/lib.rs +++ b/nautilus_core/indicators/src/lib.rs @@ -14,6 +14,7 @@ // ------------------------------------------------------------------------------------------------- pub mod average; +pub mod book; pub mod indicator; pub mod momentum; pub mod ratio; diff --git a/nautilus_core/indicators/src/python/book/imbalance.rs b/nautilus_core/indicators/src/python/book/imbalance.rs new file mode 100644 index 000000000000..d9210b6db7e9 --- /dev/null +++ b/nautilus_core/indicators/src/python/book/imbalance.rs @@ -0,0 +1,85 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use nautilus_core::python::to_pyvalue_err; +use nautilus_model::{ + orderbook::{book_mbo::OrderBookMbo, book_mbp::OrderBookMbp}, + types::quantity::Quantity, +}; +use pyo3::prelude::*; + +use crate::{book::imbalance::BookImbalanceRatio, indicator::Indicator}; + +#[pymethods] +impl BookImbalanceRatio { + #[new] + fn py_new() -> PyResult { + Self::new().map_err(to_pyvalue_err) + } + + fn __repr__(&self) -> String { + self.to_string() + } + + #[getter] + #[pyo3(name = "name")] + fn py_name(&self) -> String { + self.name() + } + + #[getter] + #[pyo3(name = "count")] + fn py_count(&self) -> usize { + self.count + } + + #[getter] + #[pyo3(name = "value")] + fn py_value(&self) -> f64 { + self.value + } + + #[getter] + #[pyo3(name = "has_inputs")] + fn py_has_inputs(&self) -> bool { + self.has_inputs() + } + + #[getter] + #[pyo3(name = "initialized")] + fn py_initialized(&self) -> bool { + self.is_initialized + } + + #[pyo3(name = "handle_book_mbo")] + fn py_handle_book_mbo(&mut self, book: &OrderBookMbo) { + self.handle_book_mbo(book); + } + + #[pyo3(name = "handle_book_mbp")] + fn py_handle_book_mbp(&mut self, book: &OrderBookMbp) { + self.handle_book_mbp(book); + } + + #[pyo3(name = "update")] + fn py_update(&mut self, best_bid: Option, best_ask: Option) { + self.update(best_bid, best_ask); + } + + #[pyo3(name = "reset")] + fn py_reset(&mut self) { + self.reset(); + } +} diff --git a/nautilus_core/indicators/src/python/book/mod.rs b/nautilus_core/indicators/src/python/book/mod.rs new file mode 100644 index 000000000000..030ac78385ce --- /dev/null +++ b/nautilus_core/indicators/src/python/book/mod.rs @@ -0,0 +1,16 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +pub mod imbalance; diff --git a/nautilus_core/indicators/src/python/mod.rs b/nautilus_core/indicators/src/python/mod.rs index fdf8d88b6f1d..2efd0a36a0d5 100644 --- a/nautilus_core/indicators/src/python/mod.rs +++ b/nautilus_core/indicators/src/python/mod.rs @@ -16,6 +16,7 @@ use pyo3::{prelude::*, pymodule}; pub mod average; +pub mod book; pub mod momentum; pub mod ratio; pub mod volatility; @@ -29,6 +30,8 @@ pub fn indicators(_: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + // book + m.add_class::()?; // ratio m.add_class::()?; // momentum diff --git a/nautilus_core/model/src/stubs.rs b/nautilus_core/model/src/stubs.rs index 571a0b38f495..f5dde87ef853 100644 --- a/nautilus_core/model/src/stubs.rs +++ b/nautilus_core/model/src/stubs.rs @@ -13,10 +13,13 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- +use crate::data::order::BookOrder; use crate::enums::{LiquiditySide, OrderSide}; +use crate::identifiers::instrument_id::InstrumentId; use crate::instruments::currency_pair::CurrencyPair; use crate::instruments::stubs::*; use crate::instruments::Instrument; +use crate::orderbook::book_mbp::OrderBookMbp; use crate::orders::market::MarketOrder; use crate::orders::stubs::{TestOrderEventStubs, TestOrderStubs}; use crate::position::Position; @@ -97,3 +100,72 @@ pub fn test_position_short(audusd_sim: CurrencyPair) -> Position { ); Position::new(audusd_sim, order_filled).unwrap() } + +pub fn stub_order_book_mbp_appl_xnas() -> OrderBookMbp { + stub_order_book_mbp( + InstrumentId::from("AAPL.XNAS"), + 101.0, + 100.0, + 100.0, + 100.0, + 2, + 0.01, + 0, + 100.0, + 10, + ) +} + +#[allow(clippy::too_many_arguments)] +pub fn stub_order_book_mbp( + instrument_id: InstrumentId, + top_ask_price: f64, + top_bid_price: f64, + top_ask_size: f64, + top_bid_size: f64, + price_precision: u8, + price_increment: f64, + size_precision: u8, + size_increment: f64, + num_levels: usize, +) -> OrderBookMbp { + let mut book = OrderBookMbp::new(instrument_id, false); + + // Generate bids + for i in 0..num_levels { + let price = Price::new( + top_bid_price - (price_increment * i as f64), + price_precision, + ) + .unwrap(); + let size = + Quantity::new(top_bid_size + (size_increment * i as f64), size_precision).unwrap(); + let order = BookOrder::new( + OrderSide::Buy, + price, + size, + 0, // order_id not applicable for MBP (market by price) books + ); + book.add(order, 0, 1); + } + + // Generate asks + for i in 0..num_levels { + let price = Price::new( + top_ask_price + (price_increment * i as f64), + price_precision, + ) + .unwrap(); + let size = + Quantity::new(top_ask_size + (size_increment * i as f64), size_precision).unwrap(); + let order = BookOrder::new( + OrderSide::Sell, + price, + size, + 0, // order_id not applicable for MBP (market by price) books + ); + book.add(order, 0, 1); + } + + book +} diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 946e6d46399f..57b664ac8741 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -2098,6 +2098,25 @@ class AverageTrueRange: def handle_bar(self, bar: Bar) -> None: ... def reset(self) -> None: ... +# Book + +class BookImbalanceRatio: + def __init__(self) -> None: ... + @property + def name(self) -> str: ... + @property + def count(self) -> int: ... + @property + def initialized(self) -> bool: ... + @property + def has_inputs(self) -> bool: ... + @property + def value(self) -> float: ... + def handle_book_mbo(self, book: OrderBookMbo) -> None:... + def handle_book_mbp(self, book: OrderBookMbp) -> None:... + def update(self, best_bid: Quantity | None, best_ask: Quantity) -> None: ... + def reset(self) -> None: ... + ################################################################################################### # Adapters ################################################################################################### diff --git a/tests/unit_tests/indicators/rust/test_imbalance_pyo3.py b/tests/unit_tests/indicators/rust/test_imbalance_pyo3.py new file mode 100644 index 000000000000..28fbf6c9afab --- /dev/null +++ b/tests/unit_tests/indicators/rust/test_imbalance_pyo3.py @@ -0,0 +1,87 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +import pytest + +from nautilus_trader.core.nautilus_pyo3 import BookImbalanceRatio +from nautilus_trader.core.nautilus_pyo3 import Quantity + + +@pytest.fixture(scope="function") +def imbalance(): + return BookImbalanceRatio() + + +def test_name(imbalance: BookImbalanceRatio) -> None: + assert imbalance.name == "BookImbalanceRatio" + + +def test_str_repr_returns_expected_string(imbalance: BookImbalanceRatio) -> None: + # Arrange, Act, Assert + assert str(imbalance) == "BookImbalanceRatio()" + assert repr(imbalance) == "BookImbalanceRatio()" + + +def test_initialized_without_inputs_returns_false(imbalance: BookImbalanceRatio) -> None: + # Arrange, Act, Assert + assert imbalance.initialized is False + + +def test_initialized_with_required_inputs(imbalance: BookImbalanceRatio) -> None: + # Arrange + imbalance.update(Quantity.from_int(100), Quantity.from_int(100)) + + # Act, Assert + assert imbalance.initialized + assert imbalance.has_inputs + assert imbalance.count == 1 + assert imbalance.value == 1.0 + + +def test_reset(imbalance: BookImbalanceRatio) -> None: + # Arrange + imbalance.update(Quantity.from_int(100), Quantity.from_int(100)) + + # Act, Assert + assert not imbalance.initialized + assert not imbalance.has_inputs + assert imbalance.count == 0 + assert imbalance.value == 0.0 + + +def test_multiple_inputs_with_bid_imbalance(imbalance: BookImbalanceRatio) -> None: + # Arrange + imbalance.update(Quantity.from_int(200), Quantity.from_int(100)) + imbalance.update(Quantity.from_int(200), Quantity.from_int(100)) + imbalance.update(Quantity.from_int(200), Quantity.from_int(100)) + + # Act, Assert + assert imbalance.initialized + assert imbalance.has_inputs + assert imbalance.count == 3 + assert imbalance.value == 0.5 + + +def test_multiple_inputs_with_ask_imbalance(imbalance: BookImbalanceRatio) -> None: + # Arrange + imbalance.update(Quantity.from_int(100), Quantity.from_int(200)) + imbalance.update(Quantity.from_int(100), Quantity.from_int(200)) + imbalance.update(Quantity.from_int(100), Quantity.from_int(200)) + + # Act, Assert + assert imbalance.initialized + assert imbalance.has_inputs + assert imbalance.count == 3 + assert imbalance.value == 0.5 From e7a957bad377688cb2a3def272cd780ca905768c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 19:07:25 +1100 Subject: [PATCH 085/104] Standardize indicator initialized property naming --- nautilus_core/indicators/src/average/ama.rs | 32 +++++++++---------- nautilus_core/indicators/src/average/dema.rs | 24 +++++++------- nautilus_core/indicators/src/average/ema.rs | 22 ++++++------- nautilus_core/indicators/src/average/hma.rs | 24 +++++++------- nautilus_core/indicators/src/average/rma.rs | 22 ++++++------- nautilus_core/indicators/src/average/sma.rs | 18 +++++------ nautilus_core/indicators/src/average/wma.rs | 18 +++++------ .../indicators/src/book/imbalance.rs | 24 +++++++------- nautilus_core/indicators/src/indicator.rs | 2 +- .../indicators/src/momentum/aroon.rs | 16 +++++----- nautilus_core/indicators/src/momentum/rsi.rs | 27 +++++++--------- .../indicators/src/python/average/ama.rs | 2 +- .../indicators/src/python/average/dema.rs | 2 +- .../indicators/src/python/average/ema.rs | 2 +- .../indicators/src/python/average/hma.rs | 2 +- .../indicators/src/python/average/rma.rs | 2 +- .../indicators/src/python/average/sma.rs | 2 +- .../indicators/src/python/average/wma.rs | 2 +- .../indicators/src/python/book/imbalance.rs | 2 +- .../indicators/src/python/momentum/aroon.rs | 2 +- .../indicators/src/python/momentum/rsi.rs | 2 +- .../src/python/ratio/efficiency_ratio.rs | 2 +- .../indicators/src/python/volatility/atr.rs | 2 +- .../indicators/src/ratio/efficiency_ratio.rs | 24 +++++++------- .../indicators/src/volatility/atr.rs | 14 ++++---- .../indicators/rust/test_aroon_pyo3.py | 4 +-- .../indicators/rust/test_atr_pyo3.py | 4 +-- .../indicators/rust/test_dema_pyo3.py | 5 ++- .../indicators/rust/test_ema_pyo3.py | 4 +-- .../indicators/rust/test_hma_pyo3.py | 4 +-- .../indicators/rust/test_imbalance_pyo3.py | 3 +- .../indicators/rust/test_rma_pyo3.py | 4 +-- .../indicators/rust/test_sma_pyo3.py | 4 +-- 33 files changed, 160 insertions(+), 163 deletions(-) diff --git a/nautilus_core/indicators/src/average/ama.rs b/nautilus_core/indicators/src/average/ama.rs index 607ad2e121d7..5271cc07dbd9 100644 --- a/nautilus_core/indicators/src/average/ama.rs +++ b/nautilus_core/indicators/src/average/ama.rs @@ -48,7 +48,7 @@ pub struct AdaptiveMovingAverage { pub value: f64, /// The input count for the indicator. pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, efficiency_ratio: EfficiencyRatio, prior_value: Option, @@ -78,8 +78,8 @@ impl Indicator for AdaptiveMovingAverage { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, tick: &QuoteTick) { @@ -98,7 +98,7 @@ impl Indicator for AdaptiveMovingAverage { self.value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -122,7 +122,7 @@ impl AdaptiveMovingAverage { alpha_slow: 2.0 / (period_slow + 1) as f64, prior_value: None, has_inputs: false, - is_initialized: false, + initialized: false, efficiency_ratio: EfficiencyRatio::new(period_efficiency_ratio, price_type)?, }) } @@ -137,7 +137,7 @@ impl AdaptiveMovingAverage { self.prior_value = None; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -171,8 +171,8 @@ impl MovingAverage for AdaptiveMovingAverage { // Calculate the AMA self.value = smoothing_constant .mul_add(value - self.prior_value.unwrap(), self.prior_value.unwrap()); - if self.efficiency_ratio.is_initialized() { - self.is_initialized = true; + if self.efficiency_ratio.initialized() { + self.initialized = true; } } } @@ -197,7 +197,7 @@ mod tests { assert_eq!(display_str, "AdaptiveMovingAverage(10,2,30)"); assert_eq!(indicator_ama_10.name(), "AdaptiveMovingAverage"); assert!(!indicator_ama_10.has_inputs()); - assert!(!indicator_ama_10.is_initialized()); + assert!(!indicator_ama_10.initialized()); } #[rstest] @@ -226,9 +226,9 @@ mod tests { for _ in 0..10 { indicator_ama_10.update_raw(1.0); } - assert!(indicator_ama_10.is_initialized); + assert!(indicator_ama_10.initialized); indicator_ama_10.reset(); - assert!(!indicator_ama_10.is_initialized); + assert!(!indicator_ama_10.initialized); assert!(!indicator_ama_10.has_inputs); assert_eq!(indicator_ama_10.value, 0.0); } @@ -239,16 +239,16 @@ mod tests { for _ in 0..9 { ama.update_raw(1.0); } - assert!(!ama.is_initialized); + assert!(!ama.initialized); ama.update_raw(1.0); - assert!(ama.is_initialized); + assert!(ama.initialized); } #[rstest] fn test_handle_quote_tick(mut indicator_ama_10: AdaptiveMovingAverage, quote_tick: QuoteTick) { indicator_ama_10.handle_quote_tick("e_tick); assert!(indicator_ama_10.has_inputs); - assert!(!indicator_ama_10.is_initialized); + assert!(!indicator_ama_10.initialized); assert_eq!(indicator_ama_10.value, 1501.0); } @@ -259,7 +259,7 @@ mod tests { ) { indicator_ama_10.handle_trade_tick(&trade_tick); assert!(indicator_ama_10.has_inputs); - assert!(!indicator_ama_10.is_initialized); + assert!(!indicator_ama_10.initialized); assert_eq!(indicator_ama_10.value, 1500.0); } @@ -270,7 +270,7 @@ mod tests { ) { indicator_ama_10.handle_bar(&bar_ethusdt_binance_minute_bid); assert!(indicator_ama_10.has_inputs); - assert!(!indicator_ama_10.is_initialized); + assert!(!indicator_ama_10.initialized); assert_eq!(indicator_ama_10.value, 1522.0); } } diff --git a/nautilus_core/indicators/src/average/dema.rs b/nautilus_core/indicators/src/average/dema.rs index 8b23acdca274..0da1c05058c6 100644 --- a/nautilus_core/indicators/src/average/dema.rs +++ b/nautilus_core/indicators/src/average/dema.rs @@ -41,7 +41,7 @@ pub struct DoubleExponentialMovingAverage { pub value: f64, /// The input count for the indicator. pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, ema1: ExponentialMovingAverage, ema2: ExponentialMovingAverage, @@ -61,8 +61,8 @@ impl Indicator for DoubleExponentialMovingAverage { fn has_inputs(&self) -> bool { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -81,7 +81,7 @@ impl Indicator for DoubleExponentialMovingAverage { self.value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -93,7 +93,7 @@ impl DoubleExponentialMovingAverage { value: 0.0, count: 0, has_inputs: false, - is_initialized: false, + initialized: false, ema1: ExponentialMovingAverage::new(period, price_type)?, ema2: ExponentialMovingAverage::new(period, price_type)?, }) @@ -119,8 +119,8 @@ impl MovingAverage for DoubleExponentialMovingAverage { self.value = 2.0f64.mul_add(self.ema1.value, -self.ema2.value); self.count += 1; - if !self.is_initialized && self.count >= self.period { - self.is_initialized = true; + if !self.initialized && self.count >= self.period { + self.initialized = true; } } } @@ -144,7 +144,7 @@ mod tests { let display_str = format!("{indicator_dema_10}"); assert_eq!(display_str, "DoubleExponentialMovingAverage(period=10)"); assert_eq!(indicator_dema_10.period, 10); - assert!(!indicator_dema_10.is_initialized); + assert!(!indicator_dema_10.initialized); assert!(!indicator_dema_10.has_inputs); } @@ -167,9 +167,9 @@ mod tests { for i in 1..10 { indicator_dema_10.update_raw(f64::from(i)); } - assert!(!indicator_dema_10.is_initialized); + assert!(!indicator_dema_10.initialized); indicator_dema_10.update_raw(10.0); - assert!(indicator_dema_10.is_initialized); + assert!(indicator_dema_10.initialized); } #[rstest] @@ -198,7 +198,7 @@ mod tests { indicator_dema_10.handle_bar(&bar_ethusdt_binance_minute_bid); assert_eq!(indicator_dema_10.value, 1522.0); assert!(indicator_dema_10.has_inputs); - assert!(!indicator_dema_10.is_initialized); + assert!(!indicator_dema_10.initialized); } #[rstest] @@ -209,6 +209,6 @@ mod tests { assert_eq!(indicator_dema_10.value, 0.0); assert_eq!(indicator_dema_10.count, 0); assert!(!indicator_dema_10.has_inputs); - assert!(!indicator_dema_10.is_initialized); + assert!(!indicator_dema_10.initialized); } } diff --git a/nautilus_core/indicators/src/average/ema.rs b/nautilus_core/indicators/src/average/ema.rs index f9cc1f8155a8..1abca25504c4 100644 --- a/nautilus_core/indicators/src/average/ema.rs +++ b/nautilus_core/indicators/src/average/ema.rs @@ -33,7 +33,7 @@ pub struct ExponentialMovingAverage { pub alpha: f64, pub value: f64, pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, } @@ -52,8 +52,8 @@ impl Indicator for ExponentialMovingAverage { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -72,7 +72,7 @@ impl Indicator for ExponentialMovingAverage { self.value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -87,7 +87,7 @@ impl ExponentialMovingAverage { value: 0.0, count: 0, has_inputs: false, - is_initialized: false, + initialized: false, }) } } @@ -110,8 +110,8 @@ impl MovingAverage for ExponentialMovingAverage { self.count += 1; // Initialization logic - if !self.is_initialized && self.count >= self.period { - self.is_initialized = true; + if !self.initialized && self.count >= self.period { + self.initialized = true; } } } @@ -141,7 +141,7 @@ mod tests { assert_eq!(ema.period, 10); assert_eq!(ema.price_type, PriceType::Mid); assert_eq!(ema.alpha, 0.181_818_181_818_181_82); - assert!(!ema.is_initialized); + assert!(!ema.initialized); } #[rstest] @@ -167,7 +167,7 @@ mod tests { ema.update_raw(10.0); assert!(ema.has_inputs()); - assert!(ema.is_initialized()); + assert!(ema.initialized()); assert_eq!(ema.count, 10); assert_eq!(ema.value, 6.239_368_480_121_215_5); } @@ -180,7 +180,7 @@ mod tests { ema.reset(); assert_eq!(ema.count, 0); assert_eq!(ema.value, 0.0); - assert!(!ema.is_initialized); + assert!(!ema.initialized); } #[rstest] @@ -220,7 +220,7 @@ mod tests { ) { indicator_ema_10.handle_bar(&bar_ethusdt_binance_minute_bid); assert!(indicator_ema_10.has_inputs); - assert!(!indicator_ema_10.is_initialized); + assert!(!indicator_ema_10.initialized); assert_eq!(indicator_ema_10.value, 1522.0); } } diff --git a/nautilus_core/indicators/src/average/hma.rs b/nautilus_core/indicators/src/average/hma.rs index 8ee0da9d8a48..2eae32214c17 100644 --- a/nautilus_core/indicators/src/average/hma.rs +++ b/nautilus_core/indicators/src/average/hma.rs @@ -38,7 +38,7 @@ pub struct HullMovingAverage { pub price_type: PriceType, pub value: f64, pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, ma1: WeightedMovingAverage, ma2: WeightedMovingAverage, @@ -60,8 +60,8 @@ impl Indicator for HullMovingAverage { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -83,7 +83,7 @@ impl Indicator for HullMovingAverage { self.ma3.reset(); self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -113,7 +113,7 @@ impl HullMovingAverage { value: 0.0, count: 0, has_inputs: false, - is_initialized: false, + initialized: false, ma1: _ma1, ma2: _ma2, ma3: _ma3, @@ -144,8 +144,8 @@ impl MovingAverage for HullMovingAverage { self.value = self.ma3.value; self.count += 1; - if !self.is_initialized && self.count >= self.period { - self.is_initialized = true; + if !self.initialized && self.count >= self.period { + self.initialized = true; } } } @@ -169,7 +169,7 @@ mod tests { let display_str = format!("{indicator_hma_10}"); assert_eq!(display_str, "HullMovingAverage(10)"); assert_eq!(indicator_hma_10.period, 10); - assert!(!indicator_hma_10.is_initialized); + assert!(!indicator_hma_10.initialized); assert!(!indicator_hma_10.has_inputs); } @@ -178,9 +178,9 @@ mod tests { for i in 1..10 { indicator_hma_10.update_raw(f64::from(i)); } - assert!(!indicator_hma_10.is_initialized); + assert!(!indicator_hma_10.initialized); indicator_hma_10.update_raw(10.0); - assert!(indicator_hma_10.is_initialized); + assert!(indicator_hma_10.initialized); } #[rstest] @@ -233,7 +233,7 @@ mod tests { indicator_hma_10.handle_bar(&bar_ethusdt_binance_minute_bid); assert_eq!(indicator_hma_10.value, 1522.0); assert!(indicator_hma_10.has_inputs); - assert!(!indicator_hma_10.is_initialized); + assert!(!indicator_hma_10.initialized); } #[rstest] @@ -251,6 +251,6 @@ mod tests { assert_eq!(indicator_hma_10.ma2.value, 0.0); assert_eq!(indicator_hma_10.ma3.value, 0.0); assert!(!indicator_hma_10.has_inputs); - assert!(!indicator_hma_10.is_initialized); + assert!(!indicator_hma_10.initialized); } } diff --git a/nautilus_core/indicators/src/average/rma.rs b/nautilus_core/indicators/src/average/rma.rs index 678c6d48ddfa..85c3d60d01b0 100644 --- a/nautilus_core/indicators/src/average/rma.rs +++ b/nautilus_core/indicators/src/average/rma.rs @@ -33,7 +33,7 @@ pub struct WilderMovingAverage { pub alpha: f64, pub value: f64, pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, } @@ -52,8 +52,8 @@ impl Indicator for WilderMovingAverage { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -72,7 +72,7 @@ impl Indicator for WilderMovingAverage { self.value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -90,7 +90,7 @@ impl WilderMovingAverage { value: 0.0, count: 0, has_inputs: false, - is_initialized: false, + initialized: false, }) } } @@ -114,8 +114,8 @@ impl MovingAverage for WilderMovingAverage { self.count += 1; // Initialization logic - if !self.is_initialized && self.count >= self.period { - self.is_initialized = true; + if !self.initialized && self.count >= self.period { + self.initialized = true; } } } @@ -145,7 +145,7 @@ mod tests { assert_eq!(rma.period, 10); assert_eq!(rma.price_type, PriceType::Mid); assert_eq!(rma.alpha, 0.1); - assert!(!rma.is_initialized); + assert!(!rma.initialized); } #[rstest] @@ -171,7 +171,7 @@ mod tests { rma.update_raw(10.0); assert!(rma.has_inputs()); - assert!(rma.is_initialized()); + assert!(rma.initialized()); assert_eq!(rma.count, 10); assert_eq!(rma.value, 4.486_784_401); } @@ -184,7 +184,7 @@ mod tests { rma.reset(); assert_eq!(rma.count, 0); assert_eq!(rma.value, 0.0); - assert!(!rma.is_initialized); + assert!(!rma.initialized); } #[rstest] @@ -221,7 +221,7 @@ mod tests { ) { indicator_rma_10.handle_bar(&bar_ethusdt_binance_minute_bid); assert!(indicator_rma_10.has_inputs); - assert!(!indicator_rma_10.is_initialized); + assert!(!indicator_rma_10.initialized); assert_eq!(indicator_rma_10.value, 1522.0); } } diff --git a/nautilus_core/indicators/src/average/sma.rs b/nautilus_core/indicators/src/average/sma.rs index 8fd880d4cbfe..ae250e739250 100644 --- a/nautilus_core/indicators/src/average/sma.rs +++ b/nautilus_core/indicators/src/average/sma.rs @@ -33,7 +33,7 @@ pub struct SimpleMovingAverage { pub value: f64, pub count: usize, pub inputs: Vec, - pub is_initialized: bool, + pub initialized: bool, } impl Display for SimpleMovingAverage { @@ -51,8 +51,8 @@ impl Indicator for SimpleMovingAverage { !self.inputs.is_empty() } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -71,7 +71,7 @@ impl Indicator for SimpleMovingAverage { self.value = 0.0; self.count = 0; self.inputs.clear(); - self.is_initialized = false; + self.initialized = false; } } @@ -85,7 +85,7 @@ impl SimpleMovingAverage { value: 0.0, count: 0, inputs: Vec::with_capacity(period), - is_initialized: false, + initialized: false, }) } } @@ -108,8 +108,8 @@ impl MovingAverage for SimpleMovingAverage { let sum = self.inputs.iter().sum::(); self.value = sum / self.count as f64; - if !self.is_initialized && self.count >= self.period { - self.is_initialized = true; + if !self.initialized && self.count >= self.period { + self.initialized = true; } } } @@ -156,7 +156,7 @@ mod tests { sma.update_raw(10.0); assert!(sma.has_inputs()); - assert!(sma.is_initialized()); + assert!(sma.initialized()); assert_eq!(sma.count, 10); assert_eq!(sma.value, 5.5); } @@ -169,7 +169,7 @@ mod tests { sma.reset(); assert_eq!(sma.count, 0); assert_eq!(sma.value, 0.0); - assert!(!sma.is_initialized); + assert!(!sma.initialized); } #[rstest] diff --git a/nautilus_core/indicators/src/average/wma.rs b/nautilus_core/indicators/src/average/wma.rs index 9822ae1f95a2..b0bd8a869dc6 100644 --- a/nautilus_core/indicators/src/average/wma.rs +++ b/nautilus_core/indicators/src/average/wma.rs @@ -38,7 +38,7 @@ pub struct WeightedMovingAverage { /// The last indicator value. pub value: f64, /// Whether the indicator is initialized. - pub is_initialized: bool, + pub initialized: bool, /// Inputs pub inputs: Vec, has_inputs: bool, @@ -61,7 +61,7 @@ impl WeightedMovingAverage { price_type: price_type.unwrap_or(PriceType::Last), value: 0.0, inputs: Vec::with_capacity(period), - is_initialized: false, + initialized: false, has_inputs: false, }) } @@ -87,8 +87,8 @@ impl Indicator for WeightedMovingAverage { fn has_inputs(&self) -> bool { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -106,7 +106,7 @@ impl Indicator for WeightedMovingAverage { fn reset(&mut self) { self.value = 0.0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; self.inputs.clear(); } } @@ -131,8 +131,8 @@ impl MovingAverage for WeightedMovingAverage { } self.inputs.push(value); self.value = self.weighted_average(); - if !self.is_initialized && self.count() >= self.period { - self.is_initialized = true; + if !self.initialized && self.count() >= self.period { + self.initialized = true; } } } @@ -159,7 +159,7 @@ mod tests { ); assert_eq!(indicator_wma_10.name(), "WeightedMovingAverage"); assert!(!indicator_wma_10.has_inputs()); - assert!(!indicator_wma_10.is_initialized()); + assert!(!indicator_wma_10.initialized()); } #[rstest] @@ -233,6 +233,6 @@ mod tests { assert_eq!(indicator_wma_10.value, 0.0); assert_eq!(indicator_wma_10.count(), 0); assert!(!indicator_wma_10.has_inputs); - assert!(!indicator_wma_10.is_initialized); + assert!(!indicator_wma_10.initialized); } } diff --git a/nautilus_core/indicators/src/book/imbalance.rs b/nautilus_core/indicators/src/book/imbalance.rs index 4f3f72ed5c75..4ff0718d0917 100644 --- a/nautilus_core/indicators/src/book/imbalance.rs +++ b/nautilus_core/indicators/src/book/imbalance.rs @@ -30,7 +30,7 @@ use crate::indicator::Indicator; pub struct BookImbalanceRatio { pub value: f64, pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, } @@ -49,8 +49,8 @@ impl Indicator for BookImbalanceRatio { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_book_mbo(&mut self, book: &OrderBookMbo) { @@ -65,7 +65,7 @@ impl Indicator for BookImbalanceRatio { self.value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -77,7 +77,7 @@ impl BookImbalanceRatio { value: 0.0, count: 0, has_inputs: false, - is_initialized: false, + initialized: false, }) } @@ -92,7 +92,7 @@ impl BookImbalanceRatio { let ratio = smaller.as_f64() / larger.as_f64(); self.value = ratio; - self.is_initialized = true; + self.initialized = true; } // No market yet } @@ -121,7 +121,7 @@ mod tests { assert_eq!(imbalance.value, 0.0); assert_eq!(imbalance.count, 0); assert!(!imbalance.has_inputs); - assert!(!imbalance.is_initialized); + assert!(!imbalance.initialized); } #[rstest] @@ -132,7 +132,7 @@ mod tests { assert_eq!(imbalance.count, 1); assert_eq!(imbalance.value, 1.0); - assert!(imbalance.is_initialized); + assert!(imbalance.initialized); assert!(imbalance.has_inputs); } @@ -145,7 +145,7 @@ mod tests { assert_eq!(imbalance.count, 0); assert_eq!(imbalance.value, 0.0); - assert!(!imbalance.is_initialized); + assert!(!imbalance.initialized); assert!(!imbalance.has_inputs); } @@ -168,7 +168,7 @@ mod tests { assert_eq!(imbalance.count, 1); assert_eq!(imbalance.value, 0.5); - assert!(imbalance.is_initialized); + assert!(imbalance.initialized); assert!(imbalance.has_inputs); } @@ -191,7 +191,7 @@ mod tests { assert_eq!(imbalance.count, 1); assert_eq!(imbalance.value, 0.5); - assert!(imbalance.is_initialized); + assert!(imbalance.initialized); assert!(imbalance.has_inputs); } @@ -216,7 +216,7 @@ mod tests { assert_eq!(imbalance.count, 3); assert_eq!(imbalance.value, 0.5); - assert!(imbalance.is_initialized); + assert!(imbalance.initialized); assert!(imbalance.has_inputs); } } diff --git a/nautilus_core/indicators/src/indicator.rs b/nautilus_core/indicators/src/indicator.rs index f1555435f0ea..052acbf60d6c 100644 --- a/nautilus_core/indicators/src/indicator.rs +++ b/nautilus_core/indicators/src/indicator.rs @@ -29,7 +29,7 @@ const IMPL_ERR: &str = "is not implemented for"; pub trait Indicator { fn name(&self) -> String; fn has_inputs(&self) -> bool; - fn is_initialized(&self) -> bool; + fn initialized(&self) -> bool; fn handle_delta(&mut self, delta: &OrderBookDelta) { // Eventually change this to log an error panic!("`handle_delta` {} `{}`", IMPL_ERR, self.name()); diff --git a/nautilus_core/indicators/src/momentum/aroon.rs b/nautilus_core/indicators/src/momentum/aroon.rs index 710e525079f4..970f38442161 100644 --- a/nautilus_core/indicators/src/momentum/aroon.rs +++ b/nautilus_core/indicators/src/momentum/aroon.rs @@ -38,7 +38,7 @@ pub struct AroonOscillator { pub aroon_down: f64, pub value: f64, pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, } @@ -57,8 +57,8 @@ impl Indicator for AroonOscillator { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, tick: &QuoteTick) { @@ -83,7 +83,7 @@ impl Indicator for AroonOscillator { self.value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -98,7 +98,7 @@ impl AroonOscillator { value: 0.0, count: 0, has_inputs: false, - is_initialized: false, + initialized: false, }) } @@ -114,7 +114,7 @@ impl AroonOscillator { self.low_inputs.push_front(low); self.increment_count(); - if self.is_initialized { + if self.initialized { // Makes sure we calculate with stable period self.calculate_aroon(); } @@ -155,10 +155,10 @@ impl AroonOscillator { fn increment_count(&mut self) { self.count += 1; - if !self.is_initialized { + if !self.initialized { self.has_inputs = true; if self.count >= self.period { - self.is_initialized = true; + self.initialized = true; } } } diff --git a/nautilus_core/indicators/src/momentum/rsi.rs b/nautilus_core/indicators/src/momentum/rsi.rs index 844b159a0df8..0abedf6b0ef0 100644 --- a/nautilus_core/indicators/src/momentum/rsi.rs +++ b/nautilus_core/indicators/src/momentum/rsi.rs @@ -36,7 +36,7 @@ pub struct RelativeStrengthIndex { pub ma_type: MovingAverageType, pub value: f64, pub count: usize, - pub is_initialized: bool, + pub initialized: bool, has_inputs: bool, last_value: f64, average_gain: Box, @@ -59,8 +59,8 @@ impl Indicator for RelativeStrengthIndex { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -80,7 +80,7 @@ impl Indicator for RelativeStrengthIndex { self.last_value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -97,7 +97,7 @@ impl RelativeStrengthIndex { average_gain: MovingAverageFactory::create(MovingAverageType::Exponential, period), average_loss: MovingAverageFactory::create(MovingAverageType::Exponential, period), rsi_max: 1.0, - is_initialized: false, + initialized: false, }) } @@ -119,11 +119,8 @@ impl RelativeStrengthIndex { } // init count from average gain MA self.count = self.average_gain.count(); - if !self.is_initialized - && self.average_loss.is_initialized() - && self.average_gain.is_initialized() - { - self.is_initialized = true; + if !self.initialized && self.average_loss.initialized() && self.average_gain.initialized() { + self.initialized = true; } if self.average_loss.value() == 0.0 { @@ -135,8 +132,8 @@ impl RelativeStrengthIndex { self.value = self.rsi_max - (self.rsi_max / (1.0 + rs)); self.last_value = value; - if !self.is_initialized && self.count >= self.period { - self.is_initialized = true; + if !self.initialized && self.count >= self.period { + self.initialized = true; } } } @@ -156,7 +153,7 @@ mod tests { let display_str = format!("{rsi_10}"); assert_eq!(display_str, "RelativeStrengthIndex(10,EXPONENTIAL)"); assert_eq!(rsi_10.period, 10); - assert!(!rsi_10.is_initialized); + assert!(!rsi_10.initialized); } #[rstest] @@ -164,7 +161,7 @@ mod tests { for i in 0..12 { rsi_10.update_raw(f64::from(i)); } - assert!(rsi_10.is_initialized); + assert!(rsi_10.initialized); } #[rstest] @@ -220,7 +217,7 @@ mod tests { rsi_10.update_raw(1.0); rsi_10.update_raw(2.0); rsi_10.reset(); - assert!(!rsi_10.is_initialized()); + assert!(!rsi_10.initialized()); assert_eq!(rsi_10.count, 0); } diff --git a/nautilus_core/indicators/src/python/average/ama.rs b/nautilus_core/indicators/src/python/average/ama.rs index e3346da96cff..7710e692adc2 100644 --- a/nautilus_core/indicators/src/python/average/ama.rs +++ b/nautilus_core/indicators/src/python/average/ama.rs @@ -74,7 +74,7 @@ impl AdaptiveMovingAverage { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_quote_tick")] diff --git a/nautilus_core/indicators/src/python/average/dema.rs b/nautilus_core/indicators/src/python/average/dema.rs index 6a5047625010..b3b26c7c2d8e 100644 --- a/nautilus_core/indicators/src/python/average/dema.rs +++ b/nautilus_core/indicators/src/python/average/dema.rs @@ -69,7 +69,7 @@ impl DoubleExponentialMovingAverage { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_quote_tick")] diff --git a/nautilus_core/indicators/src/python/average/ema.rs b/nautilus_core/indicators/src/python/average/ema.rs index 1392104f0b7f..71cb1e67e414 100644 --- a/nautilus_core/indicators/src/python/average/ema.rs +++ b/nautilus_core/indicators/src/python/average/ema.rs @@ -75,7 +75,7 @@ impl ExponentialMovingAverage { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_quote_tick")] diff --git a/nautilus_core/indicators/src/python/average/hma.rs b/nautilus_core/indicators/src/python/average/hma.rs index 8f3c04e5587f..3b4ad4967a6a 100644 --- a/nautilus_core/indicators/src/python/average/hma.rs +++ b/nautilus_core/indicators/src/python/average/hma.rs @@ -69,7 +69,7 @@ impl HullMovingAverage { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_quote_tick")] diff --git a/nautilus_core/indicators/src/python/average/rma.rs b/nautilus_core/indicators/src/python/average/rma.rs index 50ebbf5979f3..f1bef3f6de2d 100644 --- a/nautilus_core/indicators/src/python/average/rma.rs +++ b/nautilus_core/indicators/src/python/average/rma.rs @@ -75,7 +75,7 @@ impl WilderMovingAverage { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_quote_tick")] diff --git a/nautilus_core/indicators/src/python/average/sma.rs b/nautilus_core/indicators/src/python/average/sma.rs index e00c6b0a4927..b9f6df39edb2 100644 --- a/nautilus_core/indicators/src/python/average/sma.rs +++ b/nautilus_core/indicators/src/python/average/sma.rs @@ -69,7 +69,7 @@ impl SimpleMovingAverage { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_quote_tick")] diff --git a/nautilus_core/indicators/src/python/average/wma.rs b/nautilus_core/indicators/src/python/average/wma.rs index aadfaecd07f7..ca500ae67f41 100644 --- a/nautilus_core/indicators/src/python/average/wma.rs +++ b/nautilus_core/indicators/src/python/average/wma.rs @@ -67,7 +67,7 @@ impl WeightedMovingAverage { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_quote_tick")] diff --git a/nautilus_core/indicators/src/python/book/imbalance.rs b/nautilus_core/indicators/src/python/book/imbalance.rs index d9210b6db7e9..cecb37436abf 100644 --- a/nautilus_core/indicators/src/python/book/imbalance.rs +++ b/nautilus_core/indicators/src/python/book/imbalance.rs @@ -60,7 +60,7 @@ impl BookImbalanceRatio { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "handle_book_mbo")] diff --git a/nautilus_core/indicators/src/python/momentum/aroon.rs b/nautilus_core/indicators/src/python/momentum/aroon.rs index 4c60fda04c48..5ef64bfe2193 100644 --- a/nautilus_core/indicators/src/python/momentum/aroon.rs +++ b/nautilus_core/indicators/src/python/momentum/aroon.rs @@ -75,7 +75,7 @@ impl AroonOscillator { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "update_raw")] diff --git a/nautilus_core/indicators/src/python/momentum/rsi.rs b/nautilus_core/indicators/src/python/momentum/rsi.rs index d241c49b5359..60ba94c2ca5a 100644 --- a/nautilus_core/indicators/src/python/momentum/rsi.rs +++ b/nautilus_core/indicators/src/python/momentum/rsi.rs @@ -62,7 +62,7 @@ impl RelativeStrengthIndex { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "update_raw")] diff --git a/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs b/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs index 925bde93ac9d..6c6c523f3800 100644 --- a/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs +++ b/nautilus_core/indicators/src/python/ratio/efficiency_ratio.rs @@ -51,7 +51,7 @@ impl EfficiencyRatio { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "has_inputs")] diff --git a/nautilus_core/indicators/src/python/volatility/atr.rs b/nautilus_core/indicators/src/python/volatility/atr.rs index 737edd5cada4..28daaff2efef 100644 --- a/nautilus_core/indicators/src/python/volatility/atr.rs +++ b/nautilus_core/indicators/src/python/volatility/atr.rs @@ -71,7 +71,7 @@ impl AverageTrueRange { #[getter] #[pyo3(name = "initialized")] fn py_initialized(&self) -> bool { - self.is_initialized + self.initialized } #[pyo3(name = "update_raw")] diff --git a/nautilus_core/indicators/src/ratio/efficiency_ratio.rs b/nautilus_core/indicators/src/ratio/efficiency_ratio.rs index 520c92e033f0..012a6f96b2b7 100644 --- a/nautilus_core/indicators/src/ratio/efficiency_ratio.rs +++ b/nautilus_core/indicators/src/ratio/efficiency_ratio.rs @@ -36,7 +36,7 @@ pub struct EfficiencyRatio { pub price_type: PriceType, pub value: f64, pub inputs: Vec, - pub is_initialized: bool, + pub initialized: bool, deltas: Vec, } @@ -54,8 +54,8 @@ impl Indicator for EfficiencyRatio { fn has_inputs(&self) -> bool { !self.inputs.is_empty() } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_quote_tick(&mut self, quote: &QuoteTick) { @@ -73,7 +73,7 @@ impl Indicator for EfficiencyRatio { fn reset(&mut self) { self.value = 0.0; self.inputs.clear(); - self.is_initialized = false; + self.initialized = false; } } @@ -85,7 +85,7 @@ impl EfficiencyRatio { value: 0.0, inputs: Vec::with_capacity(period), deltas: Vec::with_capacity(period), - is_initialized: false, + initialized: false, }) } @@ -94,8 +94,8 @@ impl EfficiencyRatio { if self.inputs.len() < 2 { self.value = 0.0; return; - } else if !self.is_initialized && self.inputs.len() >= self.period { - self.is_initialized = true; + } else if !self.initialized && self.inputs.len() >= self.period { + self.initialized = true; } let last_diff = (self.inputs[self.inputs.len() - 1] - self.inputs[self.inputs.len() - 2]).abs(); @@ -125,7 +125,7 @@ mod tests { let display_str = format!("{efficiency_ratio_10}"); assert_eq!(display_str, "EfficiencyRatio(10)"); assert_eq!(efficiency_ratio_10.period, 10); - assert!(!efficiency_ratio_10.is_initialized); + assert!(!efficiency_ratio_10.initialized); } #[rstest] @@ -134,10 +134,10 @@ mod tests { efficiency_ratio_10.update_raw(f64::from(i)); } assert_eq!(efficiency_ratio_10.inputs.len(), 9); - assert!(!efficiency_ratio_10.is_initialized); + assert!(!efficiency_ratio_10.initialized); efficiency_ratio_10.update_raw(1.0); assert_eq!(efficiency_ratio_10.inputs.len(), 10); - assert!(efficiency_ratio_10.is_initialized); + assert!(efficiency_ratio_10.initialized); } #[rstest] @@ -203,9 +203,9 @@ mod tests { for i in 1..=10 { efficiency_ratio_10.update_raw(f64::from(i)); } - assert!(efficiency_ratio_10.is_initialized); + assert!(efficiency_ratio_10.initialized); efficiency_ratio_10.reset(); - assert!(!efficiency_ratio_10.is_initialized); + assert!(!efficiency_ratio_10.initialized); assert_eq!(efficiency_ratio_10.value, 0.0); } diff --git a/nautilus_core/indicators/src/volatility/atr.rs b/nautilus_core/indicators/src/volatility/atr.rs index 8c3f823be176..935ad2ecae5a 100644 --- a/nautilus_core/indicators/src/volatility/atr.rs +++ b/nautilus_core/indicators/src/volatility/atr.rs @@ -35,7 +35,7 @@ pub struct AverageTrueRange { pub value_floor: f64, pub value: f64, pub count: usize, - pub is_initialized: bool, + pub initialized: bool, ma: Box, has_inputs: bool, previous_close: f64, @@ -64,8 +64,8 @@ impl Indicator for AverageTrueRange { self.has_inputs } - fn is_initialized(&self) -> bool { - self.is_initialized + fn initialized(&self) -> bool { + self.initialized } fn handle_bar(&mut self, bar: &Bar) { @@ -77,7 +77,7 @@ impl Indicator for AverageTrueRange { self.value = 0.0; self.count = 0; self.has_inputs = false; - self.is_initialized = false; + self.initialized = false; } } @@ -98,7 +98,7 @@ impl AverageTrueRange { previous_close: 0.0, ma: MovingAverageFactory::create(MovingAverageType::Simple, period), has_inputs: false, - is_initialized: false, + initialized: false, }) } @@ -131,10 +131,10 @@ impl AverageTrueRange { fn increment_count(&mut self) { self.count += 1; - if !self.is_initialized { + if !self.initialized { self.has_inputs = true; if self.count >= self.period { - self.is_initialized = true; + self.initialized = true; } } } diff --git a/tests/unit_tests/indicators/rust/test_aroon_pyo3.py b/tests/unit_tests/indicators/rust/test_aroon_pyo3.py index bb15136c2a20..646e7e0b5fdd 100644 --- a/tests/unit_tests/indicators/rust/test_aroon_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_aroon_pyo3.py @@ -35,7 +35,7 @@ def test_period(aroon: AroonOscillator) -> None: def test_initialized_without_inputs_returns_false(aroon: AroonOscillator) -> None: # Arrange, Act, Assert - assert aroon.initialized is False + assert not aroon.initialized def test_initialized_with_required_inputs_returns_true(aroon: AroonOscillator) -> None: @@ -44,7 +44,7 @@ def test_initialized_with_required_inputs_returns_true(aroon: AroonOscillator) - aroon.update_raw(110.08, 109.61) # Assert - assert aroon.initialized is True + assert aroon.initialized def test_handle_bar_updates_indicator(aroon: AroonOscillator) -> None: diff --git a/tests/unit_tests/indicators/rust/test_atr_pyo3.py b/tests/unit_tests/indicators/rust/test_atr_pyo3.py index f32a71569dc5..9ed01896280d 100644 --- a/tests/unit_tests/indicators/rust/test_atr_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_atr_pyo3.py @@ -44,7 +44,7 @@ def test_period(atr: AverageTrueRange) -> None: def test_initialized_without_inputs_returns_false(atr: AverageTrueRange) -> None: # Arrange, Act, Assert - assert atr.initialized is False + assert not atr.initialized def test_initialized_with_required_inputs_returns_true(atr: AverageTrueRange) -> None: @@ -53,7 +53,7 @@ def test_initialized_with_required_inputs_returns_true(atr: AverageTrueRange) -> atr.update_raw(1.00000, 1.00000, 1.00000) # Assert - assert atr.initialized is True + assert atr.initialized def test_handle_bar_updates_indicator(atr: AverageTrueRange) -> None: diff --git a/tests/unit_tests/indicators/rust/test_dema_pyo3.py b/tests/unit_tests/indicators/rust/test_dema_pyo3.py index 01eb7e86af00..68cea336dc75 100644 --- a/tests/unit_tests/indicators/rust/test_dema_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_dema_pyo3.py @@ -13,7 +13,6 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- - import pytest from nautilus_trader.core.nautilus_pyo3 import DoubleExponentialMovingAverage @@ -44,7 +43,7 @@ def test_period_returns_expected_value(dema: DoubleExponentialMovingAverage) -> def test_initialized_without_inputs_returns_false(dema: DoubleExponentialMovingAverage) -> None: # Arrange, Act, Assert - assert dema.initialized is False + assert not dema.initialized def test_initialized_with_required_inputs_returns_true( @@ -65,7 +64,7 @@ def test_initialized_with_required_inputs_returns_true( # Act # Assert - assert dema.initialized is True + assert dema.initialized def test_handle_quote_tick_updates_indicator() -> None: diff --git a/tests/unit_tests/indicators/rust/test_ema_pyo3.py b/tests/unit_tests/indicators/rust/test_ema_pyo3.py index 474b94a450d2..1cab4cc5e720 100644 --- a/tests/unit_tests/indicators/rust/test_ema_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_ema_pyo3.py @@ -48,7 +48,7 @@ def test_multiplier_returns_expected_value(ema: ExponentialMovingAverage) -> Non def test_initialized_without_inputs_returns_false(ema: ExponentialMovingAverage) -> None: # Arrange, Act, Assert - assert ema.initialized is False + assert not ema.initialized def test_initialized_with_required_inputs_returns_true(ema: ExponentialMovingAverage) -> None: @@ -67,7 +67,7 @@ def test_initialized_with_required_inputs_returns_true(ema: ExponentialMovingAve # Act # Assert - assert ema.initialized is True + assert ema.initialized def test_handle_quote_tick_updates_indicator() -> None: diff --git a/tests/unit_tests/indicators/rust/test_hma_pyo3.py b/tests/unit_tests/indicators/rust/test_hma_pyo3.py index 1f279bfb523c..d215add70512 100644 --- a/tests/unit_tests/indicators/rust/test_hma_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_hma_pyo3.py @@ -42,7 +42,7 @@ def test_period_returns_expected_value(hma: HullMovingAverage) -> None: def test_initialized_without_inputs_returns_false(hma: HullMovingAverage) -> None: # Arrange, Act, Assert - assert hma.initialized is False + assert not hma.initialized def test_initialized_with_required_inputs_returns_true(hma: HullMovingAverage) -> None: @@ -60,7 +60,7 @@ def test_initialized_with_required_inputs_returns_true(hma: HullMovingAverage) - hma.update_raw(1.00000) # Act, Assert - assert hma.initialized is True + assert hma.initialized assert hma.count == 11 assert hma.value == 1.0001403928170598 diff --git a/tests/unit_tests/indicators/rust/test_imbalance_pyo3.py b/tests/unit_tests/indicators/rust/test_imbalance_pyo3.py index 28fbf6c9afab..22b29e4626c7 100644 --- a/tests/unit_tests/indicators/rust/test_imbalance_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_imbalance_pyo3.py @@ -36,7 +36,7 @@ def test_str_repr_returns_expected_string(imbalance: BookImbalanceRatio) -> None def test_initialized_without_inputs_returns_false(imbalance: BookImbalanceRatio) -> None: # Arrange, Act, Assert - assert imbalance.initialized is False + assert not imbalance.initialized def test_initialized_with_required_inputs(imbalance: BookImbalanceRatio) -> None: @@ -53,6 +53,7 @@ def test_initialized_with_required_inputs(imbalance: BookImbalanceRatio) -> None def test_reset(imbalance: BookImbalanceRatio) -> None: # Arrange imbalance.update(Quantity.from_int(100), Quantity.from_int(100)) + imbalance.reset() # Act, Assert assert not imbalance.initialized diff --git a/tests/unit_tests/indicators/rust/test_rma_pyo3.py b/tests/unit_tests/indicators/rust/test_rma_pyo3.py index 428cb09fda06..04d3e916a563 100644 --- a/tests/unit_tests/indicators/rust/test_rma_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_rma_pyo3.py @@ -48,7 +48,7 @@ def test_multiplier_returns_expected_value(rma: WilderMovingAverage) -> None: def test_initialized_without_inputs_returns_false(rma: WilderMovingAverage) -> None: # Arrange, Act, Assert - assert rma.initialized is False + assert not rma.initialized def test_initialized_with_required_inputs_returns_true(rma: WilderMovingAverage) -> None: @@ -67,7 +67,7 @@ def test_initialized_with_required_inputs_returns_true(rma: WilderMovingAverage) # Act # Assert - assert rma.initialized is True + assert rma.initialized def test_handle_quote_tick_updates_indicator() -> None: diff --git a/tests/unit_tests/indicators/rust/test_sma_pyo3.py b/tests/unit_tests/indicators/rust/test_sma_pyo3.py index 12abcef32e80..e8bf5af1b60b 100644 --- a/tests/unit_tests/indicators/rust/test_sma_pyo3.py +++ b/tests/unit_tests/indicators/rust/test_sma_pyo3.py @@ -42,7 +42,7 @@ def test_period_returns_expected_value(sma: SimpleMovingAverage) -> None: def test_initialized_without_inputs_returns_false(sma: SimpleMovingAverage) -> None: # Arrange, Act, Assert - assert sma.initialized is False + assert not sma.initialized def test_initialized_with_required_inputs_returns_true(sma: SimpleMovingAverage) -> None: @@ -59,7 +59,7 @@ def test_initialized_with_required_inputs_returns_true(sma: SimpleMovingAverage) sma.update_raw(10.0) # Act, Assert - assert sma.initialized is True + assert sma.initialized assert sma.count == 10 assert sma.value == 5.5 From b408b65fa35e079cfb4734afb20a2fd536a747d7 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Mon, 19 Feb 2024 19:45:59 +1100 Subject: [PATCH 086/104] Add example Rust OrderBookMbp BookImbalanceRatio --- .../binance_spot_orderbook_imbalance_rust.py | 119 +++++++++ nautilus_trader/data/engine.pyx | 9 +- .../strategies/orderbook_imbalance_rust.py | 238 ++++++++++++++++++ 3 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 examples/live/binance/binance_spot_orderbook_imbalance_rust.py create mode 100644 nautilus_trader/examples/strategies/orderbook_imbalance_rust.py diff --git a/examples/live/binance/binance_spot_orderbook_imbalance_rust.py b/examples/live/binance/binance_spot_orderbook_imbalance_rust.py new file mode 100644 index 000000000000..b5c4aa0319ac --- /dev/null +++ b/examples/live/binance/binance_spot_orderbook_imbalance_rust.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +from decimal import Decimal + +from nautilus_trader.adapters.binance.common.enums import BinanceAccountType +from nautilus_trader.adapters.binance.config import BinanceDataClientConfig +from nautilus_trader.adapters.binance.config import BinanceExecClientConfig +from nautilus_trader.adapters.binance.factories import BinanceLiveDataClientFactory +from nautilus_trader.adapters.binance.factories import BinanceLiveExecClientFactory +from nautilus_trader.config import CacheConfig +from nautilus_trader.config import InstrumentProviderConfig +from nautilus_trader.config import LiveExecEngineConfig +from nautilus_trader.config import LoggingConfig +from nautilus_trader.config import TradingNodeConfig +from nautilus_trader.examples.strategies.orderbook_imbalance_rust import OrderBookImbalance +from nautilus_trader.examples.strategies.orderbook_imbalance_rust import OrderBookImbalanceConfig +from nautilus_trader.live.node import TradingNode +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.model.identifiers import TraderId + + +# *** THIS IS A TEST STRATEGY WITH NO ALPHA ADVANTAGE WHATSOEVER. *** +# *** IT IS NOT INTENDED TO BE USED TO TRADE LIVE WITH REAL MONEY. *** + + +# Configure the trading node +config_node = TradingNodeConfig( + trader_id=TraderId("TESTER-001"), + logging=LoggingConfig( + log_level="INFO", + # log_level_file="DEBUG", + # log_file_format="json", + ), + exec_engine=LiveExecEngineConfig( + reconciliation=True, + reconciliation_lookback_mins=1440, + filter_position_reports=True, + ), + cache=CacheConfig( + database=None, + timestamps_as_iso8601=True, + flush_on_start=False, + ), + # snapshot_orders=True, + # snapshot_positions=True, + # snapshot_positions_interval=5.0, + data_clients={ + "BINANCE": BinanceDataClientConfig( + api_key=None, # 'BINANCE_API_KEY' env var + api_secret=None, # 'BINANCE_API_SECRET' env var + account_type=BinanceAccountType.SPOT, + base_url_http=None, # Override with custom endpoint + base_url_ws=None, # Override with custom endpoint + us=False, # If client is for Binance US + testnet=False, # If client uses the testnet + instrument_provider=InstrumentProviderConfig(load_all=True), + ), + }, + exec_clients={ + "BINANCE": BinanceExecClientConfig( + api_key=None, # 'BINANCE_API_KEY' env var + api_secret=None, # 'BINANCE_API_SECRET' env var + account_type=BinanceAccountType.SPOT, + base_url_http=None, # Override with custom endpoint + base_url_ws=None, # Override with custom endpoint + us=False, # If client is for Binance US + testnet=False, # If client uses the testnet + instrument_provider=InstrumentProviderConfig(load_all=True), + ), + }, + timeout_connection=20.0, + timeout_reconciliation=10.0, + timeout_portfolio=10.0, + timeout_disconnection=10.0, + timeout_post_stop=5.0, +) + +# Instantiate the node with a configuration +node = TradingNode(config=config_node) + +# Configure your strategy +strat_config = OrderBookImbalanceConfig( + instrument_id=InstrumentId.from_str("ETHUSDT.BINANCE"), + external_order_claims=[InstrumentId.from_str("ETHUSDT.BINANCE")], + max_trade_size=Decimal("0.010"), +) + +# Instantiate your strategy +strategy = OrderBookImbalance(config=strat_config) + +# Add your strategies and modules +node.trader.add_strategy(strategy) + +# Register your client factories with the node (can take user defined factories) +node.add_data_client_factory("BINANCE", BinanceLiveDataClientFactory) +node.add_exec_client_factory("BINANCE", BinanceLiveExecClientFactory) +node.build() + + +# Stop and dispose of the node with SIGINT/CTRL+C +if __name__ == "__main__": + try: + node.run() + finally: + node.dispose() diff --git a/nautilus_trader/data/engine.pyx b/nautilus_trader/data/engine.pyx index 681e698b5999..efff39069391 100644 --- a/nautilus_trader/data/engine.pyx +++ b/nautilus_trader/data/engine.pyx @@ -1545,10 +1545,11 @@ cdef class DataEngine(Component): cpdef void _update_order_book(self, Data data): cdef OrderBook order_book = self._cache.order_book(data.instrument_id) if order_book is None: - self._log.error( - "Cannot update order book: " - f"no book found for {data.instrument_id}.", - ) + # TODO: Silence error for now (book may be managed manually) + # self._log.error( + # "Cannot update order book: " + # f"no book found for {data.instrument_id}.", + # ) return order_book.apply(data) diff --git a/nautilus_trader/examples/strategies/orderbook_imbalance_rust.py b/nautilus_trader/examples/strategies/orderbook_imbalance_rust.py new file mode 100644 index 000000000000..1cdfc5612689 --- /dev/null +++ b/nautilus_trader/examples/strategies/orderbook_imbalance_rust.py @@ -0,0 +1,238 @@ +# ------------------------------------------------------------------------------------------------- +# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +# https://nautechsystems.io +# +# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------------------- + +import datetime +from decimal import Decimal + +from nautilus_trader.config import NonNegativeFloat +from nautilus_trader.config import PositiveFloat +from nautilus_trader.config import StrategyConfig +from nautilus_trader.core import nautilus_pyo3 +from nautilus_trader.core.nautilus_pyo3 import BookImbalanceRatio +from nautilus_trader.core.nautilus_pyo3 import OrderBookMbp +from nautilus_trader.core.rust.common import LogColor +from nautilus_trader.model.book import OrderBook +from nautilus_trader.model.data import OrderBookDelta +from nautilus_trader.model.data import OrderBookDeltas +from nautilus_trader.model.data import QuoteTick +from nautilus_trader.model.enums import BookType +from nautilus_trader.model.enums import OrderSide +from nautilus_trader.model.enums import TimeInForce +from nautilus_trader.model.enums import book_type_from_str +from nautilus_trader.model.identifiers import InstrumentId +from nautilus_trader.model.instruments import Instrument +from nautilus_trader.trading.strategy import Strategy + + +# *** THIS IS A TEST STRATEGY WITH NO ALPHA ADVANTAGE WHATSOEVER. *** +# *** IT IS NOT INTENDED TO BE USED TO TRADE LIVE WITH REAL MONEY. *** + + +class OrderBookImbalanceConfig(StrategyConfig, frozen=True): + """ + Configuration for ``OrderBookImbalance`` instances. + + Parameters + ---------- + instrument_id : InstrumentId + The instrument ID for the strategy. + max_trade_size : str + The max position size per trade (volume on the level can be less). + trigger_min_size : PositiveFloat, default 100.0 + The minimum size on the larger side to trigger an order. + trigger_imbalance_ratio : PositiveFloat, default 0.20 + The ratio of bid:ask volume required to trigger an order (smaller + value / larger value) ie given a trigger_imbalance_ratio=0.2, and a + bid volume of 100, we will send a buy order if the ask volume is < + 20). + min_seconds_between_triggers : NonNegativeFloat, default 1.0 + The minimum time between triggers. + book_type : str, default 'L2_MBP' + The order book type for the strategy. + use_quote_ticks : bool, default False + If quote ticks should be used. + subscribe_ticker : bool, default False + If tickers should be subscribed to. + order_id_tag : str + The unique order ID tag for the strategy. Must be unique + amongst all running strategies for a particular trader ID. + oms_type : OmsType + The order management system type for the strategy. This will determine + how the `ExecutionEngine` handles position IDs (see docs). + + """ + + instrument_id: InstrumentId + max_trade_size: Decimal + trigger_min_size: PositiveFloat = 100.0 + trigger_imbalance_ratio: PositiveFloat = 0.20 + min_seconds_between_triggers: NonNegativeFloat = 1.0 + book_type: str = "L2_MBP" + use_quote_ticks: bool = False + subscribe_ticker: bool = False + + +class OrderBookImbalance(Strategy): + """ + A simple strategy that sends FOK limit orders when there is a bid/ask imbalance in + the order book. + + Cancels all orders and closes all positions on stop. + + Parameters + ---------- + config : OrderbookImbalanceConfig + The configuration for the instance. + + """ + + def __init__(self, config: OrderBookImbalanceConfig) -> None: + assert 0 < config.trigger_imbalance_ratio < 1 + super().__init__(config) + + # Configuration + self.instrument_id = config.instrument_id + self.max_trade_size = config.max_trade_size + self.trigger_min_size = config.trigger_min_size + self.trigger_imbalance_ratio = config.trigger_imbalance_ratio + self.min_seconds_between_triggers = config.min_seconds_between_triggers + self._last_trigger_timestamp: datetime.datetime | None = None + self.instrument: Instrument | None = None + if self.config.use_quote_ticks: + assert self.config.book_type == "L1_MBP" + self.book_type: BookType = book_type_from_str(self.config.book_type) + + # We need to initialize the Rust pyo3 objects + pyo3_instrument_id = nautilus_pyo3.InstrumentId.from_str(self.instrument_id.value) + self.book = OrderBookMbp(pyo3_instrument_id, config.use_quote_ticks) + self.imbalance = BookImbalanceRatio() + + def on_start(self) -> None: + """ + Actions to be performed on strategy start. + """ + self.instrument = self.cache.instrument(self.instrument_id) + if self.instrument is None: + self.log.error(f"Could not find instrument for {self.instrument_id}") + self.stop() + return + + if self.config.use_quote_ticks: + self.book_type = BookType.L1_MBP + self.subscribe_quote_ticks(self.instrument.id) + else: + self.book_type = book_type_from_str(self.config.book_type) + self.subscribe_order_book_deltas( + self.instrument.id, + self.book_type, + managed=False, # <-- Manually applying deltas to book + ) + + self._last_trigger_timestamp = self.clock.utc_now() + + def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: + """ + Actions to be performed when order book deltas are received. + """ + # Convert to pyo3 objects (the efficiency of this can improve) + pyo3_deltas = OrderBookDelta.to_pyo3_list(deltas.deltas) + for pyo3_delta in pyo3_deltas: + self.book.apply_delta(pyo3_delta) + self.imbalance.handle_book_mbp(self.book) + self.check_trigger() + + def on_quote_tick(self, tick: QuoteTick) -> None: + """ + Actions to be performed when a delta is received. + """ + self.book.update_quote_tick(tick) + self.imbalance.handle_book_mbp(self.book) + self.check_trigger() + + def on_order_book(self, order_book: OrderBook) -> None: + """ + Actions to be performed when an order book update is received. + """ + self.check_trigger() + + def check_trigger(self) -> None: + """ + Check for trigger conditions. + """ + if not self.instrument: + self.log.error("No instrument loaded.") + return + + # This could be more efficient: for demonstration + bid_price = self.book.best_bid_price() + ask_price = self.book.best_ask_price() + bid_size = self.book.best_bid_size() + ask_size = self.book.best_ask_size() + if not bid_size or not ask_size: + self.log.warning("No market yet.") + return + + larger = max(bid_size.as_double(), ask_size.as_double()) + ratio = self.imbalance.value + self.log.info( + f"Book: {self.book.best_bid_price()} @ {self.book.best_ask_price()} ({ratio=:0.2f})", + ) + seconds_since_last_trigger = ( + self.clock.utc_now() - self._last_trigger_timestamp + ).total_seconds() + + if larger > self.trigger_min_size and ratio < self.trigger_imbalance_ratio: + self.log.info( + "Trigger conditions met, checking for existing orders and time since last order", + ) + if len(self.cache.orders_inflight(strategy_id=self.id)) > 0: + self.log.info("Already have orders in flight - skipping.") + elif seconds_since_last_trigger < self.min_seconds_between_triggers: + self.log.info("Time since last order < min_seconds_between_triggers - skipping.") + elif bid_size.as_double() > ask_size.as_double(): + order = self.order_factory.limit( + instrument_id=self.instrument.id, + price=self.instrument.make_price(ask_price), + order_side=OrderSide.BUY, + quantity=self.instrument.make_qty(ask_size), + post_only=False, + time_in_force=TimeInForce.FOK, + ) + self._last_trigger_timestamp = self.clock.utc_now() + self.log.info(f"Hitting! {order=}", color=LogColor.BLUE) + self.submit_order(order) + + else: + order = self.order_factory.limit( + instrument_id=self.instrument.id, + price=self.instrument.make_price(bid_price), + order_side=OrderSide.SELL, + quantity=self.instrument.make_qty(bid_size), + post_only=False, + time_in_force=TimeInForce.FOK, + ) + self._last_trigger_timestamp = self.clock.utc_now() + self.log.info(f"Hitting! {order=}", color=LogColor.BLUE) + self.submit_order(order) + + def on_stop(self) -> None: + """ + Actions to be performed when the strategy is stopped. + """ + if self.instrument is None: + return + + self.cancel_all_orders(self.instrument.id) + self.close_all_positions(self.instrument.id) From 506170ba8f04369579bb57dba4c122e2b685eafb Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 20 Feb 2024 08:01:05 +1100 Subject: [PATCH 087/104] Update dependencies --- nautilus_core/Cargo.lock | 12 ++++++------ nautilus_core/Cargo.toml | 2 +- poetry.lock | 20 ++++++++++---------- pyproject.toml | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 537099073ad3..1c21f90431da 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -101,9 +101,9 @@ checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "arc-swap" @@ -3853,9 +3853,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "same-file" @@ -3922,9 +3922,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "seq-macro" diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index d43bf003c944..d49d425ab0cb 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -24,7 +24,7 @@ description = "A high-performance algorithmic trading platform and event-driven documentation = "https://docs.nautilustrader.io" [workspace.dependencies] -anyhow = "1.0.79" +anyhow = "1.0.80" chrono = "0.4.34" futures = "0.3.30" indexmap = "2.2.3" diff --git a/poetry.lock b/poetry.lock index 71e4e88676d2..6a0329d72a7a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -776,13 +776,13 @@ tqdm = ["tqdm"] [[package]] name = "identify" -version = "2.5.34" +version = "2.5.35" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.34-py2.py3-none-any.whl", hash = "sha256:a4316013779e433d08b96e5eabb7f641e6c7942e4ab5d4c509ebd2e7a8994aed"}, - {file = "identify-2.5.34.tar.gz", hash = "sha256:ee17bc9d499899bc9eaec1ac7bf2dc9eedd480db9d88b96d123d3b64a9d34f5d"}, + {file = "identify-2.5.35-py2.py3-none-any.whl", hash = "sha256:c4de0081837b211594f8e877a6b4fad7ca32bbfc1a9307fdd61c28bfe923f13e"}, + {file = "identify-2.5.35.tar.gz", hash = "sha256:10a7ca245cfcd756a554a7288159f72ff105ad233c7c4b9c6f0f4d108f5f6791"}, ] [package.extras] @@ -1482,13 +1482,13 @@ xml = ["lxml (>=4.9.2)"] [[package]] name = "pandas-stubs" -version = "2.1.4.231227" +version = "2.2.0.240218" description = "Type annotations for pandas" optional = false python-versions = ">=3.9" files = [ - {file = "pandas_stubs-2.1.4.231227-py3-none-any.whl", hash = "sha256:211fc23e6ae87073bdf41dbf362c4a4d85e1e3477cb078dbac3da6c7fdaefba8"}, - {file = "pandas_stubs-2.1.4.231227.tar.gz", hash = "sha256:3ea29ef001e9e44985f5ebde02d4413f94891ef6ec7e5056fb07d125be796c23"}, + {file = "pandas_stubs-2.2.0.240218-py3-none-any.whl", hash = "sha256:e97478320add9b958391b15a56c5f1bf29da656d5b747d28bbe708454b3a1fe6"}, + {file = "pandas_stubs-2.2.0.240218.tar.gz", hash = "sha256:63138c12eec715d66d48611bdd922f31cd7c78bcadd19384c3bd61fd3720a11a"}, ] [package.dependencies] @@ -1538,13 +1538,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pre-commit" -version = "3.6.1" +version = "3.6.2" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-3.6.1-py2.py3-none-any.whl", hash = "sha256:9fe989afcf095d2c4796ce7c553cf28d4d4a9b9346de3cda079bcf40748454a4"}, - {file = "pre_commit-3.6.1.tar.gz", hash = "sha256:c90961d8aa706f75d60935aba09469a6b0bcb8345f127c3fbee4bdc5f114cf4b"}, + {file = "pre_commit-3.6.2-py2.py3-none-any.whl", hash = "sha256:ba637c2d7a670c10daedc059f5c49b5bd0aadbccfcd7ec15592cf9665117532c"}, + {file = "pre_commit-3.6.2.tar.gz", hash = "sha256:c3ef34f463045c88658c5b99f38c1e297abdcc0ff13f98d3370055fbbfabc67e"}, ] [package.dependencies] @@ -2595,4 +2595,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "f401e28b92a35ca1084aad06a4d71f51252e759c1fb8aa21f9f73f1981ab5e02" +content-hash = "42b145c803bea991089e5c21d29681a1346cbc7523622b051996355a62e47555" diff --git a/pyproject.toml b/pyproject.toml index 463c3da1a312..ab963e5cf70e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ black = "^24.2.0" docformatter = "^1.7.5" mypy = "^1.8.0" pandas-stubs = "^2.1.4" -pre-commit = "^3.6.1" +pre-commit = "^3.6.2" ruff = "^0.2.2" types-pytz = "^2023.3" types-requests = "^2.31" From 920380a408586cf1ecf5bbf91c181ba0f9b7777c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 20 Feb 2024 08:06:54 +1100 Subject: [PATCH 088/104] Fix TradeTick size precision for pyo3 conversion --- RELEASES.md | 1 + nautilus_trader/model/data.pyx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASES.md b/RELEASES.md index d20586d1e7c8..4294a78acb38 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -15,6 +15,7 @@ None ### Fixes - Fixed `TradeId` memory leak due assigning unique values to the `Ustr` global string cache (which are never freed for the lifetime of the program) +- Fixed `TradeTick` size precision for pyo3 conversion (size precision was incorrectly price precision) - Fixed FOK time in force behavior (allows fills beyond the top level, will cancel if cannot fill full size) - Fixed IOC time in force behavior (allows fills beyond the top level, will cancel any remaining after all fills are applied) - Fixed `LiveClock` timer behavior for small intervals causing next time to be less than now (timer then would not run) diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index 7290a90f900c..c73f69e9f024 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -4083,7 +4083,7 @@ cdef class TradeTick(Data): if pyo3_instrument_id is None: pyo3_instrument_id = nautilus_pyo3.InstrumentId.from_str(trade.instrument_id.value) price_prec = trade.price.precision - size_prec = trade.price.precision + size_prec = trade.size.precision pyo3_trade = nautilus_pyo3.TradeTick( pyo3_instrument_id, From 1926e2d79cca3aa96332cfa28d4658a5b515ec2b Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 20 Feb 2024 08:09:13 +1100 Subject: [PATCH 089/104] Resume Windows in CI --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bba01e42a980..85cfefcca5de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: arch: [x64] - os: [ubuntu-latest] # windows-latest + os: [ubuntu-latest, windows-latest] python-version: ["3.10", "3.11", "3.12"] defaults: run: From 8eabaeda7ee9e73b6ac61528f1cc06bf56d59280 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 20 Feb 2024 17:34:23 +1100 Subject: [PATCH 090/104] Fix DefaultHasher import --- nautilus_core/model/src/ffi/identifiers/trade_id.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nautilus_core/model/src/ffi/identifiers/trade_id.rs b/nautilus_core/model/src/ffi/identifiers/trade_id.rs index b22467df1a78..f8da1e52a20c 100644 --- a/nautilus_core/model/src/ffi/identifiers/trade_id.rs +++ b/nautilus_core/model/src/ffi/identifiers/trade_id.rs @@ -14,8 +14,9 @@ // ------------------------------------------------------------------------------------------------- use std::{ + collections::hash_map::DefaultHasher, ffi::{c_char, CStr}, - hash::{DefaultHasher, Hash, Hasher}, + hash::{Hash, Hasher}, }; use crate::identifiers::trade_id::TradeId; From c00b888933e68321c93808c8ea46f6fd06c9c2ed Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 20 Feb 2024 19:47:01 +1100 Subject: [PATCH 091/104] Add OrderBookDeltas from mem C function --- nautilus_trader/model/data.pyx | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index c73f69e9f024..0ec97c7c9355 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -146,6 +146,12 @@ cdef inline OrderBookDelta delta_from_mem_c(OrderBookDelta_t mem): return delta +cdef inline OrderBookDeltas deltas_from_mem_c(OrderBookDeltas_API mem): + cdef OrderBookDeltas deltas = OrderBookDeltas.__new__(OrderBookDeltas) + deltas._mem = mem + return deltas + + cdef inline OrderBookDepth10 depth10_from_mem_c(OrderBookDepth10_t mem): cdef OrderBookDepth10 depth10 = OrderBookDepth10.__new__(OrderBookDepth10) depth10._mem = mem @@ -180,6 +186,8 @@ cpdef list capsule_to_list(capsule): for i in range(0, data.len): if ptr[i].tag == Data_t_Tag.DELTA: objects.append(delta_from_mem_c(ptr[i].delta)) + elif ptr[i].tag == Data_t_Tag.DELTAS: + objects.append(deltas_from_mem_c(ptr[i].deltas)) elif ptr[i].tag == Data_t_Tag.DEPTH10: objects.append(depth10_from_mem_c(ptr[i].depth10)) elif ptr[i].tag == Data_t_Tag.QUOTE: @@ -198,6 +206,8 @@ cpdef Data capsule_to_data(capsule): if ptr.tag == Data_t_Tag.DELTA: return delta_from_mem_c(ptr.delta) + elif ptr.tag == Data_t_Tag.DELTAS: + return deltas_from_mem_c(ptr.deltas) elif ptr.tag == Data_t_Tag.DEPTH10: return depth10_from_mem_c(ptr.depth10) elif ptr.tag == Data_t_Tag.QUOTE: From aa6ea250a57fc4ebfd3634811f7f14157057248a Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 20 Feb 2024 19:47:38 +1100 Subject: [PATCH 092/104] Refine DatabentoLiveClient MBO replay --- .../adapters/src/databento/python/live.rs | 115 ++++++++++++++---- 1 file changed, 89 insertions(+), 26 deletions(-) diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 8e5910756256..0c8ea88a4d15 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -14,11 +14,12 @@ // ------------------------------------------------------------------------------------------------- use std::collections::HashMap; +use std::ffi::CStr; use std::fs; use std::str::FromStr; use std::sync::Arc; -use anyhow::{bail, Result}; +use anyhow::{anyhow, bail, Result}; use databento::live::Subscription; use dbn::{PitSymbolMap, Record, SymbolIndex, VersionUpgradePolicy}; use indexmap::IndexMap; @@ -42,8 +43,9 @@ use pyo3::prelude::*; use time::OffsetDateTime; use tokio::sync::Mutex; use tokio::time::{timeout, Duration}; +use ustr::Ustr; -use crate::databento::decode::{decode_instrument_def_msg, decode_record, raw_ptr_to_ustr}; +use crate::databento::decode::{decode_instrument_def_msg, decode_record}; use crate::databento::types::{DatabentoPublisher, PublisherId}; use super::loader::convert_instrument_to_pyobject; @@ -158,7 +160,7 @@ impl DatabentoLiveClient { let publisher_venue_map = self.publisher_venue_map.clone(); let clock = get_atomic_clock_realtime(); - let buffering_start = match replay { + let mut buffering_start = match replay { true => Some(clock.get_time_ns()), false => None, }; @@ -166,6 +168,7 @@ impl DatabentoLiveClient { pyo3_asyncio::tokio::future_into_py(py, async move { let mut client = arc_client.lock().await; let mut symbol_map = PitSymbolMap::new(); + let mut instrument_id_map: HashMap = HashMap::new(); let timeout_duration = Duration::from_millis(10); let relock_interval = timeout_duration.as_nanos() as u64; @@ -175,6 +178,8 @@ impl DatabentoLiveClient { client.start().await.map_err(to_pyruntime_err)?; + let mut deltas_count = 0_u64; + loop { // Check if need to drop then re-aquire lock let now_ns = clock.get_time_ns(); @@ -209,33 +214,60 @@ impl DatabentoLiveClient { } else if let Some(msg) = record.get::() { handle_system_msg(msg); } else if let Some(msg) = record.get::() { + // Remove instrument ID index as the raw symbol may have changed + instrument_id_map.remove(&msg.hd.instrument_id); handle_symbol_mapping_msg(msg, &mut symbol_map); } else if let Some(msg) = record.get::() { - handle_instrument_def_msg(msg, &publisher_venue_map, clock, &callback) - .map_err(to_pyvalue_err)?; + handle_instrument_def_msg( + msg, + &publisher_venue_map, + &mut instrument_id_map, + clock, + &callback, + ) + .map_err(to_pyvalue_err)?; } else { - let (mut data1, data2) = - handle_record(record, &symbol_map, &publisher_venue_map, clock) - .map_err(to_pyvalue_err)?; + let (mut data1, data2) = handle_record( + record, + &symbol_map, + &publisher_venue_map, + &mut instrument_id_map, + clock, + ) + .map_err(to_pyvalue_err)?; if let Some(msg) = record.get::() { // SAFETY: An MBO message will always produce a delta if let Data::Delta(delta) = data1.clone().unwrap() { - buffered_deltas - .entry(delta.instrument_id) - .or_default() - .push(delta); - - // Check if this is the last message in the packet - if msg.flags & dbn::flags::LAST != 0 { - continue; // Not last message + let buffer = buffered_deltas.entry(delta.instrument_id).or_default(); + buffer.push(delta); + + deltas_count += 1; + println!( + "Buffering delta: {} {} {:?} flags={}, buffer_len={}", + deltas_count, + delta.ts_event, + buffering_start, + msg.flags, + buffer.len() + ); + + // Check if last message in the packet + if msg.flags & dbn::flags::LAST == 0 { + continue; // NOT last message + } + + // Check if snapshot + if msg.flags & dbn::flags::SNAPSHOT != 0 { + continue; // Buffer snapshot } - // Check if we're currently buffering a replay + // Check if buffering a replay if let Some(start_ns) = buffering_start { if delta.ts_event <= start_ns { - continue; // Continue buffering the replay + continue; // Continue buffering replay } + buffering_start = None; } // SAFETY: We can guarantee a deltas vec exists @@ -294,16 +326,43 @@ fn handle_symbol_mapping_msg(msg: &dbn::SymbolMappingMsg, symbol_map: &mut PitSy .unwrap_or_else(|_| panic!("Error updating `symbol_map` with {msg:?}")); } +fn update_instrument_id_map( + header: &dbn::RecordHeader, + raw_symbol: &str, + publisher_venue_map: &IndexMap, + instrument_id_map: &mut HashMap, +) -> InstrumentId { + // Check if instrument ID is already in the map + if let Some(&instrument_id) = instrument_id_map.get(&header.instrument_id) { + return instrument_id; + } + + let symbol = Symbol { + value: Ustr::from(raw_symbol), + }; + let venue = publisher_venue_map.get(&header.publisher_id).unwrap(); + let instrument_id = InstrumentId::new(symbol, *venue); + + instrument_id_map.insert(header.instrument_id, instrument_id); + instrument_id +} + fn handle_instrument_def_msg( msg: &dbn::InstrumentDefMsg, publisher_venue_map: &IndexMap, + instrument_id_map: &mut HashMap, clock: &AtomicTime, callback: &PyObject, ) -> Result<()> { - let raw_symbol = unsafe { raw_ptr_to_ustr(msg.raw_symbol.as_ptr()).unwrap() }; - let symbol = Symbol { value: raw_symbol }; - let venue = publisher_venue_map.get(&msg.hd.publisher_id).unwrap(); - let instrument_id = InstrumentId::new(symbol, *venue); + let c_str: &CStr = unsafe { CStr::from_ptr(msg.raw_symbol.as_ptr()) }; + let raw_symbol: &str = c_str.to_str().map_err(|e| anyhow!(e))?; + + let instrument_id = update_instrument_id_map( + msg.header(), + raw_symbol, + publisher_venue_map, + instrument_id_map, + ); let ts_init = clock.get_time_ns(); let result = decode_instrument_def_msg(msg, instrument_id, ts_init); @@ -324,15 +383,19 @@ fn handle_record( record: dbn::RecordRef, symbol_map: &PitSymbolMap, publisher_venue_map: &IndexMap, + instrument_id_map: &mut HashMap, clock: &AtomicTime, ) -> Result<(Option, Option)> { let raw_symbol = symbol_map .get_for_rec(&record) .expect("Cannot resolve `raw_symbol` from `symbol_map`"); - let publisher_id = record.publisher().unwrap() as PublisherId; - let symbol = Symbol::from_str_unchecked(raw_symbol); - let venue = publisher_venue_map.get(&publisher_id).unwrap(); - let instrument_id = InstrumentId::new(symbol, *venue); + + let instrument_id = update_instrument_id_map( + record.header(), + raw_symbol, + publisher_venue_map, + instrument_id_map, + ); let price_precision = 2; // Hard coded for now let ts_init = clock.get_time_ns(); From a4e4d0ce96f47ab74961022e359cbab8bf8c97cf Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Tue, 20 Feb 2024 20:16:35 +1100 Subject: [PATCH 093/104] Refine DatabentoLiveClient MBO replay --- nautilus_core/adapters/src/databento/python/live.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nautilus_core/adapters/src/databento/python/live.rs b/nautilus_core/adapters/src/databento/python/live.rs index 0c8ea88a4d15..5f0e76adb337 100644 --- a/nautilus_core/adapters/src/databento/python/live.rs +++ b/nautilus_core/adapters/src/databento/python/live.rs @@ -24,6 +24,7 @@ use databento::live::Subscription; use dbn::{PitSymbolMap, Record, SymbolIndex, VersionUpgradePolicy}; use indexmap::IndexMap; use log::{error, info}; +use nautilus_core::ffi::cvec::CVec; use nautilus_core::python::to_pyruntime_err; use nautilus_core::time::AtomicTime; use nautilus_core::{ @@ -33,7 +34,7 @@ use nautilus_core::{ use nautilus_model::data::delta::OrderBookDelta; use nautilus_model::data::deltas::OrderBookDeltas; use nautilus_model::data::Data; -use nautilus_model::ffi::data::deltas::OrderBookDeltas_API; +use nautilus_model::ffi::data::deltas::orderbook_deltas_new; use nautilus_model::identifiers::instrument_id::InstrumentId; use nautilus_model::identifiers::symbol::Symbol; use nautilus_model::identifiers::venue::Venue; @@ -271,9 +272,12 @@ impl DatabentoLiveClient { } // SAFETY: We can guarantee a deltas vec exists - let deltas = buffered_deltas.remove(&delta.instrument_id).unwrap(); - let book_deltas = OrderBookDeltas::new(delta.instrument_id, deltas); - data1 = Some(Data::Deltas(OrderBookDeltas_API::new(book_deltas))); + let instrument_id = delta.instrument_id; + let buffer = buffered_deltas.remove(&delta.instrument_id).unwrap(); + let deltas = OrderBookDeltas::new(delta.instrument_id, buffer); + let deltas_cvec: CVec = deltas.deltas.into(); + let deltas = orderbook_deltas_new(instrument_id, &deltas_cvec); + data1 = Some(Data::Deltas(deltas)); } }; From ab10890b8d688c0f9f797b442247b49aa21f3e63 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 21 Feb 2024 20:00:38 +1100 Subject: [PATCH 094/104] Update dependencies --- nautilus_core/Cargo.lock | 214 +++++++++++++++------------ nautilus_core/Cargo.toml | 6 +- nautilus_core/persistence/Cargo.toml | 2 +- poetry.lock | 108 +++++++------- pyproject.toml | 2 +- 5 files changed, 181 insertions(+), 151 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 1c21f90431da..69046d218c12 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" +checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f" dependencies = [ "cfg-if", "const-random", @@ -166,7 +166,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow-buffer", "arrow-data", "arrow-schema", @@ -294,7 +294,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow-array", "arrow-buffer", "arrow-data", @@ -318,7 +318,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow-array", "arrow-buffer", "arrow-data", @@ -368,7 +368,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -576,7 +576,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", "syn_derive", ] @@ -603,9 +603,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.0" +version = "3.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a994c2b3ca201d9b263612a374263f05e7adde37c4707f693dcd375076d1f" +checksum = "c764d619ca78fccbf3069b37bd7af92577f044bb15236036662d79b6559f25b7" [[package]] name = "bytecheck" @@ -695,11 +695,10 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "7f9fa1897e4325be0d68d48df6aa1a71ac2ed4d27723887e7754192705350730" dependencies = [ - "jobserver", "libc", ] @@ -847,8 +846,8 @@ version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum", - "strum_macros", + "strum 0.25.0", + "strum_macros 0.25.3", "unicode-width", ] @@ -1123,11 +1122,11 @@ dependencies = [ [[package]] name = "datafusion" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" +checksum = "b2b360b692bf6c6d6e6b6dbaf41a3be0020daeceac0f406aed54c75331e50dbb" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow", "arrow-array", "arrow-ipc", @@ -1141,6 +1140,7 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-plan", @@ -1171,11 +1171,11 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" +checksum = "37f343ccc298f440e25aa38ff82678291a7acc24061c7370ba6c0ff5cc811412" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow", "arrow-array", "arrow-buffer", @@ -1192,9 +1192,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d447650af16e138c31237f53ddaef6dd4f92f0e2d3f2f35d190e16c214ca496" +checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" dependencies = [ "arrow", "chrono", @@ -1213,25 +1213,40 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" +checksum = "e204d89909e678846b6a95f156aafc1ee5b36cb6c9e37ec2e1449b078a38c818" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow", "arrow-array", "datafusion-common", "paste", "sqlparser", - "strum", - "strum_macros", + "strum 0.26.1", + "strum_macros 0.26.1", +] + +[[package]] +name = "datafusion-functions" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98f1c73f7801b2b8ba2297b3ad78ffcf6c1fc6b8171f502987eb9ad5cb244ee7" +dependencies = [ + "arrow", + "base64", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hex", + "log", ] [[package]] name = "datafusion-optimizer" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7feb0391f1fc75575acb95b74bfd276903dc37a5409fcebe160bc7ddff2010" +checksum = "5ae27e07bf1f04d327be5c2a293470879801ab5535204dc3b16b062fda195496" dependencies = [ "arrow", "async-trait", @@ -1247,21 +1262,23 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" +checksum = "dde620cd9ef76a3bca9c754fb68854bd2349c49f55baf97e08001f9e967f6d6b" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow", "arrow-array", "arrow-buffer", "arrow-ord", "arrow-schema", + "arrow-string", "base64", "blake2", "blake3", "chrono", "datafusion-common", + "datafusion-execution", "datafusion-expr", "half", "hashbrown 0.14.3", @@ -1281,11 +1298,11 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" +checksum = "9a4c75fba9ea99d64b2246cbd2fcae2e6fc973e6616b1015237a616036506dd4" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow", "arrow-array", "arrow-buffer", @@ -1312,9 +1329,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d18d36f260bbbd63aafdb55339213a23d540d3419810575850ef0a798a6b768" +checksum = "21474a95c3a62d113599d21b439fa15091b538bac06bd20be0bb2e7d22903c09" dependencies = [ "arrow", "arrow-schema", @@ -1339,7 +1356,7 @@ dependencies = [ "pyo3", "serde", "streaming-iterator", - "strum", + "strum 0.25.0", "thiserror", "time", "tokio", @@ -1355,7 +1372,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -1681,7 +1698,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -1817,7 +1834,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "allocator-api2", ] @@ -2151,15 +2168,6 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" -[[package]] -name = "jobserver" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.68" @@ -2489,7 +2497,7 @@ dependencies = [ "rstest", "serde", "serde_json", - "strum", + "strum 0.26.1", "sysinfo", "tempfile", "tokio", @@ -2526,7 +2534,7 @@ dependencies = [ "nautilus-model", "pyo3", "rstest", - "strum", + "strum 0.26.1", ] [[package]] @@ -2565,7 +2573,7 @@ dependencies = [ "rust_decimal_macros", "serde", "serde_json", - "strum", + "strum 0.26.1", "tabled", "thiserror", "thousands", @@ -2810,7 +2818,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -2857,9 +2865,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.63" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ "bitflags 2.4.2", "cfg-if", @@ -2878,7 +2886,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -2898,9 +2906,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.99" +version = "0.9.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" +checksum = "ae94056a791d0e1217d18b6cbdccb02c61e3054fc69893607f4067e3bb0b1fd1" dependencies = [ "cc", "libc", @@ -2970,7 +2978,7 @@ version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "arrow-array", "arrow-buffer", "arrow-cast", @@ -3094,7 +3102,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -3336,7 +3344,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -3348,7 +3356,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -3703,7 +3711,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.49", + "syn 2.0.50", "unicode-ident", ] @@ -3934,29 +3942,29 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] name = "serde_json" -version = "1.0.113" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -4154,9 +4162,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.41.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" +checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" dependencies = [ "log", "sqlparser_derive", @@ -4170,7 +4178,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -4192,7 +4200,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "atoi", "byteorder", "bytes", @@ -4405,7 +4413,16 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.3", +] + +[[package]] +name = "strum" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723b93e8addf9aa965ebe2d11da6d7540fa2283fcea14b3371ff055f7ba13f5f" +dependencies = [ + "strum_macros 0.26.1", ] [[package]] @@ -4418,7 +4435,20 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.49", + "syn 2.0.50", +] + +[[package]] +name = "strum_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.50", ] [[package]] @@ -4440,9 +4470,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.49" +version = "2.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" +checksum = "74f1bdc9872430ce9b75da68329d1c1746faf50ffac5f19e02b71e37ff881ffb" dependencies = [ "proc-macro2", "quote", @@ -4458,7 +4488,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -4583,7 +4613,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -4594,9 +4624,9 @@ checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -4705,7 +4735,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -4858,7 +4888,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -4977,7 +5007,7 @@ checksum = "563b3b88238ec95680aef36bdece66896eaa7ce3c0f1b4f39d38fb2435261352" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] @@ -5000,9 +5030,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -5066,7 +5096,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e904a2279a4a36d2356425bb20be271029cc650c335bc82af8bfae30085a3d0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.9", "byteorder", "lazy_static", "parking_lot", @@ -5158,7 +5188,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", "wasm-bindgen-shared", ] @@ -5192,7 +5222,7 @@ checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5477,7 +5507,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.49", + "syn 2.0.50", ] [[package]] diff --git a/nautilus_core/Cargo.toml b/nautilus_core/Cargo.toml index d49d425ab0cb..223295e3c128 100644 --- a/nautilus_core/Cargo.toml +++ b/nautilus_core/Cargo.toml @@ -38,9 +38,9 @@ redis = { version = "0.24.0", features = ["tokio-comp", "tls-rustls", "tokio-rus rmp-serde = "1.1.2" rust_decimal = "1.34.3" rust_decimal_macros = "1.34.2" -serde = { version = "1.0.196", features = ["derive"] } -serde_json = "1.0.112" -strum = { version = "0.25.0", features = ["derive"] } +serde = { version = "1.0.197", features = ["derive"] } +serde_json = "1.0.113" +strum = { version = "0.26.1", features = ["derive"] } thiserror = "1.0.57" thousands = "0.2.0" tracing = "0.1.40" diff --git a/nautilus_core/persistence/Cargo.toml b/nautilus_core/persistence/Cargo.toml index e18c14cfd773..a2130cd13e71 100644 --- a/nautilus_core/persistence/Cargo.toml +++ b/nautilus_core/persistence/Cargo.toml @@ -29,7 +29,7 @@ tokio = { workspace = true } thiserror = { workspace = true } binary-heap-plus = "0.5.0" compare = "0.1.0" -datafusion = { version = "35.0.0", default-features = false, features = ["compression", "regex_expressions", "unicode_expressions", "pyarrow"] } +datafusion = { version = "36.0.0", default-features = false, features = ["compression", "regex_expressions", "unicode_expressions", "pyarrow"] } dotenv = "0.15.0" sqlx = { version = "0.7.3", features = ["sqlite", "postgres", "any", "runtime-tokio"] } diff --git a/poetry.lock b/poetry.lock index 6a0329d72a7a..f6914838378a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -394,63 +394,63 @@ files = [ [[package]] name = "coverage" -version = "7.4.1" +version = "7.4.2" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:077d366e724f24fc02dbfe9d946534357fda71af9764ff99d73c3c596001bbd7"}, - {file = "coverage-7.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0193657651f5399d433c92f8ae264aff31fc1d066deee4b831549526433f3f61"}, - {file = "coverage-7.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d17bbc946f52ca67adf72a5ee783cd7cd3477f8f8796f59b4974a9b59cacc9ee"}, - {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3277f5fa7483c927fe3a7b017b39351610265308f5267ac6d4c2b64cc1d8d25"}, - {file = "coverage-7.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dceb61d40cbfcf45f51e59933c784a50846dc03211054bd76b421a713dcdf19"}, - {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6008adeca04a445ea6ef31b2cbaf1d01d02986047606f7da266629afee982630"}, - {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c61f66d93d712f6e03369b6a7769233bfda880b12f417eefdd4f16d1deb2fc4c"}, - {file = "coverage-7.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9bb62fac84d5f2ff523304e59e5c439955fb3b7f44e3d7b2085184db74d733b"}, - {file = "coverage-7.4.1-cp310-cp310-win32.whl", hash = "sha256:f86f368e1c7ce897bf2457b9eb61169a44e2ef797099fb5728482b8d69f3f016"}, - {file = "coverage-7.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:869b5046d41abfea3e381dd143407b0d29b8282a904a19cb908fa24d090cc018"}, - {file = "coverage-7.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8ffb498a83d7e0305968289441914154fb0ef5d8b3157df02a90c6695978295"}, - {file = "coverage-7.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3cacfaefe6089d477264001f90f55b7881ba615953414999c46cc9713ff93c8c"}, - {file = "coverage-7.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d6850e6e36e332d5511a48a251790ddc545e16e8beaf046c03985c69ccb2676"}, - {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e961aa13b6d47f758cc5879383d27b5b3f3dcd9ce8cdbfdc2571fe86feb4dd"}, - {file = "coverage-7.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfd1e1b9f0898817babf840b77ce9fe655ecbe8b1b327983df485b30df8cc011"}, - {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6b00e21f86598b6330f0019b40fb397e705135040dbedc2ca9a93c7441178e74"}, - {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:536d609c6963c50055bab766d9951b6c394759190d03311f3e9fcf194ca909e1"}, - {file = "coverage-7.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7ac8f8eb153724f84885a1374999b7e45734bf93a87d8df1e7ce2146860edef6"}, - {file = "coverage-7.4.1-cp311-cp311-win32.whl", hash = "sha256:f3771b23bb3675a06f5d885c3630b1d01ea6cac9e84a01aaf5508706dba546c5"}, - {file = "coverage-7.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:9d2f9d4cc2a53b38cabc2d6d80f7f9b7e3da26b2f53d48f05876fef7956b6968"}, - {file = "coverage-7.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f68ef3660677e6624c8cace943e4765545f8191313a07288a53d3da188bd8581"}, - {file = "coverage-7.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23b27b8a698e749b61809fb637eb98ebf0e505710ec46a8aa6f1be7dc0dc43a6"}, - {file = "coverage-7.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e3424c554391dc9ef4a92ad28665756566a28fecf47308f91841f6c49288e66"}, - {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0860a348bf7004c812c8368d1fc7f77fe8e4c095d661a579196a9533778e156"}, - {file = "coverage-7.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe558371c1bdf3b8fa03e097c523fb9645b8730399c14fe7721ee9c9e2a545d3"}, - {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3468cc8720402af37b6c6e7e2a9cdb9f6c16c728638a2ebc768ba1ef6f26c3a1"}, - {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:02f2edb575d62172aa28fe00efe821ae31f25dc3d589055b3fb64d51e52e4ab1"}, - {file = "coverage-7.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ca6e61dc52f601d1d224526360cdeab0d0712ec104a2ce6cc5ccef6ed9a233bc"}, - {file = "coverage-7.4.1-cp312-cp312-win32.whl", hash = "sha256:ca7b26a5e456a843b9b6683eada193fc1f65c761b3a473941efe5a291f604c74"}, - {file = "coverage-7.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:85ccc5fa54c2ed64bd91ed3b4a627b9cce04646a659512a051fa82a92c04a448"}, - {file = "coverage-7.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8bdb0285a0202888d19ec6b6d23d5990410decb932b709f2b0dfe216d031d218"}, - {file = "coverage-7.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:918440dea04521f499721c039863ef95433314b1db00ff826a02580c1f503e45"}, - {file = "coverage-7.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:379d4c7abad5afbe9d88cc31ea8ca262296480a86af945b08214eb1a556a3e4d"}, - {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b094116f0b6155e36a304ff912f89bbb5067157aff5f94060ff20bbabdc8da06"}, - {file = "coverage-7.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2f5968608b1fe2a1d00d01ad1017ee27efd99b3437e08b83ded9b7af3f6f766"}, - {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:10e88e7f41e6197ea0429ae18f21ff521d4f4490aa33048f6c6f94c6045a6a75"}, - {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a4a3907011d39dbc3e37bdc5df0a8c93853c369039b59efa33a7b6669de04c60"}, - {file = "coverage-7.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d224f0c4c9c98290a6990259073f496fcec1b5cc613eecbd22786d398ded3ad"}, - {file = "coverage-7.4.1-cp38-cp38-win32.whl", hash = "sha256:23f5881362dcb0e1a92b84b3c2809bdc90db892332daab81ad8f642d8ed55042"}, - {file = "coverage-7.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:a07f61fc452c43cd5328b392e52555f7d1952400a1ad09086c4a8addccbd138d"}, - {file = "coverage-7.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8e738a492b6221f8dcf281b67129510835461132b03024830ac0e554311a5c54"}, - {file = "coverage-7.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:46342fed0fff72efcda77040b14728049200cbba1279e0bf1188f1f2078c1d70"}, - {file = "coverage-7.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9641e21670c68c7e57d2053ddf6c443e4f0a6e18e547e86af3fad0795414a628"}, - {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeb2c2688ed93b027eb0d26aa188ada34acb22dceea256d76390eea135083950"}, - {file = "coverage-7.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d12c923757de24e4e2110cf8832d83a886a4cf215c6e61ed506006872b43a6d1"}, - {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0491275c3b9971cdbd28a4595c2cb5838f08036bca31765bad5e17edf900b2c7"}, - {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8dfc5e195bbef80aabd81596ef52a1277ee7143fe419efc3c4d8ba2754671756"}, - {file = "coverage-7.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a78b656a4d12b0490ca72651fe4d9f5e07e3c6461063a9b6265ee45eb2bdd35"}, - {file = "coverage-7.4.1-cp39-cp39-win32.whl", hash = "sha256:f90515974b39f4dea2f27c0959688621b46d96d5a626cf9c53dbc653a895c05c"}, - {file = "coverage-7.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:64e723ca82a84053dd7bfcc986bdb34af8d9da83c521c19d6b472bc6880e191a"}, - {file = "coverage-7.4.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:32a8d985462e37cfdab611a6f95b09d7c091d07668fdc26e47a725ee575fe166"}, - {file = "coverage-7.4.1.tar.gz", hash = "sha256:1ed4b95480952b1a26d863e546fa5094564aa0065e1e5f0d4d0041f293251d04"}, + {file = "coverage-7.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bf54c3e089179d9d23900e3efc86d46e4431188d9a657f345410eecdd0151f50"}, + {file = "coverage-7.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fe6e43c8b510719b48af7db9631b5fbac910ade4bd90e6378c85ac5ac706382c"}, + {file = "coverage-7.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b98c89db1b150d851a7840142d60d01d07677a18f0f46836e691c38134ed18b"}, + {file = "coverage-7.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5f9683be6a5b19cd776ee4e2f2ffb411424819c69afab6b2db3a0a364ec6642"}, + {file = "coverage-7.4.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cdcbf7b9cb83fe047ee09298e25b1cd1636824067166dc97ad0543b079d22f"}, + {file = "coverage-7.4.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2599972b21911111114100d362aea9e70a88b258400672626efa2b9e2179609c"}, + {file = "coverage-7.4.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ef00d31b7569ed3cb2036f26565f1984b9fc08541731ce01012b02a4c238bf03"}, + {file = "coverage-7.4.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:20a875bfd8c282985c4720c32aa05056f77a68e6d8bbc5fe8632c5860ee0b49b"}, + {file = "coverage-7.4.2-cp310-cp310-win32.whl", hash = "sha256:b3f2b1eb229f23c82898eedfc3296137cf1f16bb145ceab3edfd17cbde273fb7"}, + {file = "coverage-7.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:7df95fdd1432a5d2675ce630fef5f239939e2b3610fe2f2b5bf21fa505256fa3"}, + {file = "coverage-7.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a8ddbd158e069dded57738ea69b9744525181e99974c899b39f75b2b29a624e2"}, + {file = "coverage-7.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81a5fb41b0d24447a47543b749adc34d45a2cf77b48ca74e5bf3de60a7bd9edc"}, + {file = "coverage-7.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2412e98e70f16243be41d20836abd5f3f32edef07cbf8f407f1b6e1ceae783ac"}, + {file = "coverage-7.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb79414c15c6f03f56cc68fa06994f047cf20207c31b5dad3f6bab54a0f66ef"}, + {file = "coverage-7.4.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf89ab85027427d351f1de918aff4b43f4eb5f33aff6835ed30322a86ac29c9e"}, + {file = "coverage-7.4.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a178b7b1ac0f1530bb28d2e51f88c0bab3e5949835851a60dda80bff6052510c"}, + {file = "coverage-7.4.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:06fe398145a2e91edaf1ab4eee66149c6776c6b25b136f4a86fcbbb09512fd10"}, + {file = "coverage-7.4.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:18cac867950943fe93d6cd56a67eb7dcd2d4a781a40f4c1e25d6f1ed98721a55"}, + {file = "coverage-7.4.2-cp311-cp311-win32.whl", hash = "sha256:f72cdd2586f9a769570d4b5714a3837b3a59a53b096bb954f1811f6a0afad305"}, + {file = "coverage-7.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:d779a48fac416387dd5673fc5b2d6bd903ed903faaa3247dc1865c65eaa5a93e"}, + {file = "coverage-7.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:adbdfcda2469d188d79771d5696dc54fab98a16d2ef7e0875013b5f56a251047"}, + {file = "coverage-7.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ac4bab32f396b03ebecfcf2971668da9275b3bb5f81b3b6ba96622f4ef3f6e17"}, + {file = "coverage-7.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:006d220ba2e1a45f1de083d5022d4955abb0aedd78904cd5a779b955b019ec73"}, + {file = "coverage-7.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3733545eb294e5ad274abe131d1e7e7de4ba17a144505c12feca48803fea5f64"}, + {file = "coverage-7.4.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a9e754aa250fe61f0f99986399cec086d7e7a01dd82fd863a20af34cbce962"}, + {file = "coverage-7.4.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2ed37e16cf35c8d6e0b430254574b8edd242a367a1b1531bd1adc99c6a5e00fe"}, + {file = "coverage-7.4.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b953275d4edfab6cc0ed7139fa773dfb89e81fee1569a932f6020ce7c6da0e8f"}, + {file = "coverage-7.4.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32b4ab7e6c924f945cbae5392832e93e4ceb81483fd6dc4aa8fb1a97b9d3e0e1"}, + {file = "coverage-7.4.2-cp312-cp312-win32.whl", hash = "sha256:f5df76c58977bc35a49515b2fbba84a1d952ff0ec784a4070334dfbec28a2def"}, + {file = "coverage-7.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:34423abbaad70fea9d0164add189eabaea679068ebdf693baa5c02d03e7db244"}, + {file = "coverage-7.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5b11f9c6587668e495cc7365f85c93bed34c3a81f9f08b0920b87a89acc13469"}, + {file = "coverage-7.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:51593a1f05c39332f623d64d910445fdec3d2ac2d96b37ce7f331882d5678ddf"}, + {file = "coverage-7.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69f1665165ba2fe7614e2f0c1aed71e14d83510bf67e2ee13df467d1c08bf1e8"}, + {file = "coverage-7.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3c8bbb95a699c80a167478478efe5e09ad31680931ec280bf2087905e3b95ec"}, + {file = "coverage-7.4.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:175f56572f25e1e1201d2b3e07b71ca4d201bf0b9cb8fad3f1dfae6a4188de86"}, + {file = "coverage-7.4.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8562ca91e8c40864942615b1d0b12289d3e745e6b2da901d133f52f2d510a1e3"}, + {file = "coverage-7.4.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d9a1ef0f173e1a19738f154fb3644f90d0ada56fe6c9b422f992b04266c55d5a"}, + {file = "coverage-7.4.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f40ac873045db4fd98a6f40387d242bde2708a3f8167bd967ccd43ad46394ba2"}, + {file = "coverage-7.4.2-cp38-cp38-win32.whl", hash = "sha256:d1b750a8409bec61caa7824bfd64a8074b6d2d420433f64c161a8335796c7c6b"}, + {file = "coverage-7.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:b4ae777bebaed89e3a7e80c4a03fac434a98a8abb5251b2a957d38fe3fd30088"}, + {file = "coverage-7.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ff7f92ae5a456101ca8f48387fd3c56eb96353588e686286f50633a611afc95"}, + {file = "coverage-7.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:861d75402269ffda0b33af94694b8e0703563116b04c681b1832903fac8fd647"}, + {file = "coverage-7.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3507427d83fa961cbd73f11140f4a5ce84208d31756f7238d6257b2d3d868405"}, + {file = "coverage-7.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf711d517e21fb5bc429f5c4308fbc430a8585ff2a43e88540264ae87871e36a"}, + {file = "coverage-7.4.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c00e54f0bd258ab25e7f731ca1d5144b0bf7bec0051abccd2bdcff65fa3262c9"}, + {file = "coverage-7.4.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f8e845d894e39fb53834da826078f6dc1a933b32b1478cf437007367efaf6f6a"}, + {file = "coverage-7.4.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:840456cb1067dc350af9080298c7c2cfdddcedc1cb1e0b30dceecdaf7be1a2d3"}, + {file = "coverage-7.4.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c11ca2df2206a4e3e4c4567f52594637392ed05d7c7fb73b4ea1c658ba560265"}, + {file = "coverage-7.4.2-cp39-cp39-win32.whl", hash = "sha256:3ff5bdb08d8938d336ce4088ca1a1e4b6c8cd3bef8bb3a4c0eb2f37406e49643"}, + {file = "coverage-7.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:ac9e95cefcf044c98d4e2c829cd0669918585755dd9a92e28a1a7012322d0a95"}, + {file = "coverage-7.4.2-pp38.pp39.pp310-none-any.whl", hash = "sha256:f593a4a90118d99014517c2679e04a4ef5aee2d81aa05c26c734d271065efcb6"}, + {file = "coverage-7.4.2.tar.gz", hash = "sha256:1a5ee18e3a8d766075ce9314ed1cb695414bae67df6a4b0805f5137d93d6f1cb"}, ] [package.dependencies] @@ -2595,4 +2595,4 @@ ib = ["async-timeout", "defusedxml", "nautilus_ibapi"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "42b145c803bea991089e5c21d29681a1346cbc7523622b051996355a62e47555" +content-hash = "df875b2db29f096089cb34bed93ba087bcb97b429945242ef426062098751a4b" diff --git a/pyproject.toml b/pyproject.toml index ab963e5cf70e..4a2a25931a04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ types-toml = "^0.10.2" optional = true [tool.poetry.group.test.dependencies] -coverage = "^7.4.1" +coverage = "^7.4.2" pytest = "^7.4.4" pytest-aiohttp = "^1.0.5" pytest-asyncio = "==0.21.1" # Pinned due Cython: cannot set '__pytest_asyncio_scoped_event_loop' attribute of immutable type From ba503900e5462a73ff5279f150d6775cf25ca3b0 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 21 Feb 2024 20:11:05 +1100 Subject: [PATCH 095/104] Fix catalog freeze_dict function --- nautilus_trader/persistence/catalog/singleton.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautilus_trader/persistence/catalog/singleton.py b/nautilus_trader/persistence/catalog/singleton.py index 9390839c194e..f8c08b3679c5 100644 --- a/nautilus_trader/persistence/catalog/singleton.py +++ b/nautilus_trader/persistence/catalog/singleton.py @@ -56,4 +56,4 @@ def check_value(v: Any) -> Any: def freeze_dict(dict_like: dict) -> tuple: - return tuple(sorted(dict_like.items())) + return tuple(sorted((k, check_value(v)) for k, v in dict_like.items())) From 7ee3e93944306b222c25742200ca4df7e84cc2eb Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 21 Feb 2024 20:12:38 +1100 Subject: [PATCH 096/104] Update release notes --- RELEASES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASES.md b/RELEASES.md index 4294a78acb38..1f9990cd5aad 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -32,6 +32,7 @@ None - Fixed `ExecAlgorithmFactory.create` JSON encoding (was missing the encoding hook) - Fixed `ControllerConfig` base class and docstring - Fixed Interactive Brokers historical bar data bug, thanks @benjaminsingleton +- Fixed persistence `freeze_dict` function to handle `fs_storage_options`, thanks @dimitar-petrov --- From abae4594d2cc53a8388d1eefbc9abb6fc4845fce Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 21 Feb 2024 23:15:31 +1100 Subject: [PATCH 097/104] Improve bypass_logging fixture --- tests/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index bf363c0b1356..093586522ed7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,7 @@ import pytest from nautilus_trader.common.component import init_logging +from nautilus_trader.common.enums import LogLevel from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.instruments import CurrencyPair @@ -33,7 +34,10 @@ def bypass_logging() -> None: to debug specific tests, simply comment this out. """ - init_logging(bypass=True) + init_logging( + level_stdout=LogLevel.DEBUG, + bypass=True, + ) @pytest.fixture(name="audusd_instrument") From d4b22af6398c34651860aa107c5915565c6ced9f Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 21 Feb 2024 23:16:05 +1100 Subject: [PATCH 098/104] Fix RiskEngine cash value check when selling --- nautilus_trader/risk/engine.pyx | 23 +- nautilus_trader/test_kit/providers.py | 37 +++ tests/unit_tests/risk/test_engine.py | 337 ++++++++++++++++++-------- 3 files changed, 285 insertions(+), 112 deletions(-) diff --git a/nautilus_trader/risk/engine.pyx b/nautilus_trader/risk/engine.pyx index b4bd30424bb5..002c185f4d70 100644 --- a/nautilus_trader/risk/engine.pyx +++ b/nautilus_trader/risk/engine.pyx @@ -622,6 +622,7 @@ cdef class RiskEngine(Component): Money cum_notional_buy = None Money cum_notional_sell = None Money order_balance_impact = None + Money cash_value = None Currency base_currency = None double xrate for order in orders: @@ -659,16 +660,7 @@ cdef class RiskEngine(Component): else: last_px = order.price - #################################################################### - # CASH account balance risk check - #################################################################### - if isinstance(instrument, CurrencyPair) and order.side == OrderSide.SELL: - xrate = 1.0 / last_px.as_f64_c() - notional = Money(order.quantity.as_f64_c() * xrate, instrument.base_currency) - if max_notional: - max_notional = Money(max_notional * Decimal(xrate), instrument.base_currency) - else: - notional = instrument.notional_value(order.quantity, last_px, use_quote_for_inverse=True) + notional = instrument.notional_value(order.quantity, last_px, use_quote_for_inverse=True) if max_notional and notional._mem.raw > max_notional._mem.raw: self._deny_order( @@ -718,7 +710,7 @@ cdef class RiskEngine(Component): cum_notional_buy = Money(-order_balance_impact, order_balance_impact.currency) else: cum_notional_buy._mem.raw += -order_balance_impact._mem.raw - if free is not None and cum_notional_buy._mem.raw >= free._mem.raw: + if free is not None and cum_notional_buy._mem.raw > free._mem.raw: self._deny_order( order=order, reason=f"CUM_NOTIONAL_EXCEEDS_FREE_BALANCE: free={free.to_str()}, cum_notional={cum_notional_buy.to_str()}", @@ -730,19 +722,20 @@ cdef class RiskEngine(Component): cum_notional_sell = Money(order_balance_impact, order_balance_impact.currency) else: cum_notional_sell._mem.raw += order_balance_impact._mem.raw - if free is not None and cum_notional_sell._mem.raw >= free._mem.raw: + if free is not None and cum_notional_sell._mem.raw > free._mem.raw: self._deny_order( order=order, reason=f"CUM_NOTIONAL_EXCEEDS_FREE_BALANCE: free={free.to_str()}, cum_notional={cum_notional_sell.to_str()}", ) return False # Denied elif base_currency is not None: + cash_value = Money(order.quantity.as_f64_c(), instrument.base_currency) free = account.balance_free(base_currency) if cum_notional_sell is None: - cum_notional_sell = notional + cum_notional_sell = cash_value else: - cum_notional_sell._mem.raw += notional._mem.raw - if free is not None and cum_notional_sell._mem.raw >= free._mem.raw: + cum_notional_sell._mem.raw += cash_value._mem.raw + if free is not None and cum_notional_sell._mem.raw > free._mem.raw: self._deny_order( order=order, reason=f"CUM_NOTIONAL_EXCEEDS_FREE_BALANCE: free={free.to_str()}, cum_notional={cum_notional_sell.to_str()}", diff --git a/nautilus_trader/test_kit/providers.py b/nautilus_trader/test_kit/providers.py index ab885d6b3be8..a615a9b59629 100644 --- a/nautilus_trader/test_kit/providers.py +++ b/nautilus_trader/test_kit/providers.py @@ -104,6 +104,43 @@ def adabtc_binance() -> CurrencyPair: ts_init=0, ) + @staticmethod + def adausdt_binance() -> CurrencyPair: + """ + Return the Binance Spot ADA/USDT instrument for backtesting. + + Returns + ------- + CurrencyPair + + """ + return CurrencyPair( + instrument_id=InstrumentId( + symbol=Symbol("ADAUSDT"), + venue=Venue("BINANCE"), + ), + raw_symbol=Symbol("ADAUSDT"), + base_currency=ADA, + quote_currency=USDT, + price_precision=4, + size_precision=1, + price_increment=Price(0.0001, precision=4), + size_increment=Quantity(0.1, precision=1), + lot_size=Quantity(0.1, precision=1), + max_quantity=Quantity(900_000, precision=1), + min_quantity=Quantity(0.1, precision=1), + max_notional=None, + min_notional=Money(0.00010000, BTC), + max_price=Price(1000, precision=4), + min_price=Price(1e-8, precision=4), + margin_init=Decimal("0"), + margin_maint=Decimal("0"), + maker_fee=Decimal("0.0010"), + taker_fee=Decimal("0.0010"), + ts_event=0, + ts_init=0, + ) + @staticmethod def btcusdt_binance() -> CurrencyPair: """ diff --git a/tests/unit_tests/risk/test_engine.py b/tests/unit_tests/risk/test_engine.py index 115a75b830d6..f407d3854d84 100644 --- a/tests/unit_tests/risk/test_engine.py +++ b/tests/unit_tests/risk/test_engine.py @@ -31,14 +31,17 @@ from nautilus_trader.execution.messages import SubmitOrder from nautilus_trader.execution.messages import SubmitOrderList from nautilus_trader.execution.messages import TradingCommand +from nautilus_trader.model.currencies import ADA from nautilus_trader.model.currencies import GBP from nautilus_trader.model.currencies import USD +from nautilus_trader.model.currencies import USDT from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.enums import AccountType from nautilus_trader.model.enums import OrderSide from nautilus_trader.model.enums import OrderStatus from nautilus_trader.model.enums import TradingState from nautilus_trader.model.enums import TriggerType +from nautilus_trader.model.events import AccountState from nautilus_trader.model.events import OrderDenied from nautilus_trader.model.events import OrderModifyRejected from nautilus_trader.model.identifiers import AccountId @@ -49,6 +52,8 @@ from nautilus_trader.model.identifiers import StrategyId from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.identifiers import VenueOrderId +from nautilus_trader.model.objects import AccountBalance +from nautilus_trader.model.objects import Money from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity from nautilus_trader.model.orders.list import OrderList @@ -63,9 +68,10 @@ from nautilus_trader.trading.strategy import Strategy -AUDUSD_SIM = TestInstrumentProvider.default_fx_ccy("AUD/USD") -GBPUSD_SIM = TestInstrumentProvider.default_fx_ccy("GBP/USD") -XBTUSD_BITMEX = TestInstrumentProvider.xbtusd_bitmex() +_AUDUSD_SIM = TestInstrumentProvider.default_fx_ccy("AUD/USD") +_GBPUSD_SIM = TestInstrumentProvider.default_fx_ccy("GBP/USD") +_XBTUSD_BITMEX = TestInstrumentProvider.xbtusd_bitmex() +_ADAUSDT_BINANCE = TestInstrumentProvider.adausdt_binance() class TestRiskEngineWithCashAccount: @@ -124,7 +130,7 @@ def setup(self): self.exec_engine.register_client(self.exec_client) # Prepare data - self.cache.add_instrument(AUDUSD_SIM) + self.cache.add_instrument(_AUDUSD_SIM) def test_config_risk_engine(self): # Arrange @@ -151,8 +157,8 @@ def test_config_risk_engine(self): assert risk_engine.is_bypassed assert risk_engine.max_order_submit_rate() == (5, timedelta(seconds=1)) assert risk_engine.max_order_modify_rate() == (5, timedelta(seconds=1)) - assert risk_engine.max_notionals_per_order() == {GBPUSD_SIM.id: Decimal("2000000")} - assert risk_engine.max_notional_per_order(GBPUSD_SIM.id) == 2_000_000 + assert risk_engine.max_notionals_per_order() == {_GBPUSD_SIM.id: Decimal("2000000")} + assert risk_engine.max_notional_per_order(_GBPUSD_SIM.id) == 2_000_000 def test_risk_engine_on_stop(self): # Arrange, Act @@ -222,19 +228,19 @@ def test_max_notionals_per_order_when_no_risk_config_returns_empty_dict(self): def test_max_notional_per_order_when_no_risk_config_returns_none(self): # Arrange, Act - result = self.risk_engine.max_notional_per_order(AUDUSD_SIM.id) + result = self.risk_engine.max_notional_per_order(_AUDUSD_SIM.id) assert result is None def test_set_max_notional_per_order_changes_setting(self): # Arrange, Act - self.risk_engine.set_max_notional_per_order(AUDUSD_SIM.id, 1_000_000) + self.risk_engine.set_max_notional_per_order(_AUDUSD_SIM.id, 1_000_000) max_notionals = self.risk_engine.max_notionals_per_order() - max_notional = self.risk_engine.max_notional_per_order(AUDUSD_SIM.id) + max_notional = self.risk_engine.max_notional_per_order(_AUDUSD_SIM.id) # Assert - assert max_notionals == {AUDUSD_SIM.id: Decimal("1000000")} + assert max_notionals == {_AUDUSD_SIM.id: Decimal("1000000")} assert max_notional == Decimal(1_000_000) def test_given_random_command_then_logs_and_continues(self): @@ -243,7 +249,7 @@ def test_given_random_command_then_logs_and_continues(self): client_id=None, trader_id=self.trader_id, strategy_id=StrategyId("SCALPER-001"), - instrument_id=AUDUSD_SIM.id, + instrument_id=_AUDUSD_SIM.id, command_id=UUID4(), ts_init=self.clock.timestamp_ns(), ) @@ -276,7 +282,7 @@ def test_submit_order_with_default_settings_then_sends_to_client(self): ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -311,7 +317,7 @@ def test_submit_order_when_risk_bypassed_sends_to_execution_engine(self): ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -346,20 +352,20 @@ def test_submit_reduce_only_order_when_position_already_closed_then_denies(self) ) order1 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) order2 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), reduce_only=True, ) order3 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), reduce_only=True, @@ -377,7 +383,7 @@ def test_submit_reduce_only_order_when_position_already_closed_then_denies(self) self.risk_engine.execute(submit_order1) self.exec_engine.process(TestEventStubs.order_submitted(order1)) self.exec_engine.process(TestEventStubs.order_accepted(order1)) - self.exec_engine.process(TestEventStubs.order_filled(order1, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(order1, _AUDUSD_SIM)) submit_order2 = SubmitOrder( trader_id=self.trader_id, @@ -391,7 +397,7 @@ def test_submit_reduce_only_order_when_position_already_closed_then_denies(self) self.risk_engine.execute(submit_order2) self.exec_engine.process(TestEventStubs.order_submitted(order2)) self.exec_engine.process(TestEventStubs.order_accepted(order2)) - self.exec_engine.process(TestEventStubs.order_filled(order2, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(order2, _AUDUSD_SIM)) submit_order3 = SubmitOrder( trader_id=self.trader_id, @@ -426,13 +432,13 @@ def test_submit_reduce_only_order_when_position_would_be_increased_then_denies(s ) order1 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) order2 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(200_000), reduce_only=True, @@ -450,7 +456,7 @@ def test_submit_reduce_only_order_when_position_would_be_increased_then_denies(s self.risk_engine.execute(submit_order1) self.exec_engine.process(TestEventStubs.order_submitted(order1)) self.exec_engine.process(TestEventStubs.order_accepted(order1)) - self.exec_engine.process(TestEventStubs.order_filled(order1, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(order1, _AUDUSD_SIM)) submit_order2 = SubmitOrder( trader_id=self.trader_id, @@ -465,7 +471,7 @@ def test_submit_reduce_only_order_when_position_would_be_increased_then_denies(s self.risk_engine.execute(submit_order2) self.exec_engine.process(TestEventStubs.order_submitted(order2)) self.exec_engine.process(TestEventStubs.order_accepted(order2)) - self.exec_engine.process(TestEventStubs.order_filled(order2, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(order2, _AUDUSD_SIM)) # Assert assert order1.status == OrderStatus.FILLED @@ -487,7 +493,7 @@ def test_submit_order_reduce_only_order_with_custom_position_id_not_open_then_de ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), reduce_only=True, @@ -523,7 +529,7 @@ def test_submit_order_when_instrument_not_in_cache_then_denies(self): ) order = strategy.order_factory.market( - GBPUSD_SIM.id, # <-- Not in the cache + _GBPUSD_SIM.id, # <-- Not in the cache OrderSide.BUY, Quantity.from_int(100_000), ) @@ -558,7 +564,7 @@ def test_submit_order_when_invalid_price_precision_then_denies(self): ) order = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("0.999999999"), # <- invalid price @@ -594,7 +600,7 @@ def test_submit_order_when_invalid_negative_price_and_not_option_then_denies(sel ) order = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("-1.0"), # <- invalid price @@ -630,7 +636,7 @@ def test_submit_order_when_invalid_trigger_price_then_denies(self): ) order = strategy.order_factory.stop_limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.00000"), @@ -667,7 +673,7 @@ def test_submit_order_when_invalid_quantity_precision_then_denies(self): ) order = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_str("1.111111111"), # <- invalid quantity Price.from_str("1.00000"), @@ -703,7 +709,7 @@ def test_submit_order_when_invalid_quantity_exceeds_maximum_then_denies(self): ) order = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(1_000_000_000), # <- invalid quantity fat finger! Price.from_str("1.00000"), @@ -739,7 +745,7 @@ def test_submit_order_when_invalid_quantity_less_than_minimum_then_denies(self): ) order = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(1), # <- invalid quantity Price.from_str("1.00000"), @@ -763,7 +769,7 @@ def test_submit_order_when_invalid_quantity_less_than_minimum_then_denies(self): def test_submit_order_when_market_order_and_no_market_then_logs_warning(self): # Arrange - self.risk_engine.set_max_notional_per_order(AUDUSD_SIM.id, 1_000_000) + self.risk_engine.set_max_notional_per_order(_AUDUSD_SIM.id, 1_000_000) self.exec_engine.start() @@ -777,7 +783,7 @@ def test_submit_order_when_market_order_and_no_market_then_logs_warning(self): ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(10_000_000), ) @@ -805,7 +811,7 @@ def test_submit_order_when_less_than_min_notional_for_instrument_then_denies( # Arrange exec_client = MockExecutionClient( client_id=ClientId("BITMEX"), - venue=XBTUSD_BITMEX.id.venue, + venue=_XBTUSD_BITMEX.id.venue, account_type=AccountType.CASH, base_currency=USD, msgbus=self.msgbus, @@ -815,9 +821,9 @@ def test_submit_order_when_less_than_min_notional_for_instrument_then_denies( self.portfolio.update_account(TestEventStubs.cash_account_state(AccountId("BITMEX-001"))) self.exec_engine.register_client(exec_client) - self.cache.add_instrument(XBTUSD_BITMEX) + self.cache.add_instrument(_XBTUSD_BITMEX) quote = TestDataStubs.quote_tick( - instrument=XBTUSD_BITMEX, + instrument=_XBTUSD_BITMEX, bid_price=50_000.00, ask_price=50_001.00, ) @@ -835,7 +841,7 @@ def test_submit_order_when_less_than_min_notional_for_instrument_then_denies( ) order = strategy.order_factory.market( - XBTUSD_BITMEX.id, + _XBTUSD_BITMEX.id, order_side, Quantity.from_str("0.1"), # <-- Less than min notional ($1 USD) ) @@ -864,7 +870,7 @@ def test_submit_order_when_greater_than_max_notional_for_instrument_then_denies( # Arrange exec_client = MockExecutionClient( client_id=ClientId("BITMEX"), - venue=XBTUSD_BITMEX.id.venue, + venue=_XBTUSD_BITMEX.id.venue, account_type=AccountType.CASH, base_currency=USD, msgbus=self.msgbus, @@ -874,9 +880,9 @@ def test_submit_order_when_greater_than_max_notional_for_instrument_then_denies( self.portfolio.update_account(TestEventStubs.cash_account_state(AccountId("BITMEX-001"))) self.exec_engine.register_client(exec_client) - self.cache.add_instrument(XBTUSD_BITMEX) + self.cache.add_instrument(_XBTUSD_BITMEX) quote = TestDataStubs.quote_tick( - instrument=XBTUSD_BITMEX, + instrument=_XBTUSD_BITMEX, bid_price=50_000.00, ask_price=50_001.00, ) @@ -894,7 +900,7 @@ def test_submit_order_when_greater_than_max_notional_for_instrument_then_denies( ) order = strategy.order_factory.market( - XBTUSD_BITMEX.id, + _XBTUSD_BITMEX.id, order_side, Quantity.from_int(11_000_000), # <-- Greater than max notional ($10 million USD) ) @@ -917,10 +923,10 @@ def test_submit_order_when_greater_than_max_notional_for_instrument_then_denies( def test_submit_order_when_buy_market_order_and_over_max_notional_then_denies(self): # Arrange - self.risk_engine.set_max_notional_per_order(AUDUSD_SIM.id, 1_000_000) + self.risk_engine.set_max_notional_per_order(_AUDUSD_SIM.id, 1_000_000) # Initialize market - quote = TestDataStubs.quote_tick(AUDUSD_SIM) + quote = TestDataStubs.quote_tick(_AUDUSD_SIM) self.cache.add_quote_tick(quote) self.exec_engine.start() @@ -935,7 +941,7 @@ def test_submit_order_when_buy_market_order_and_over_max_notional_then_denies(se ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(10_000_000), ) @@ -958,11 +964,11 @@ def test_submit_order_when_buy_market_order_and_over_max_notional_then_denies(se def test_submit_order_when_sell_market_order_and_over_max_notional_then_denies(self): # Arrange - self.risk_engine.set_max_notional_per_order(AUDUSD_SIM.id, 1_000_000) + self.risk_engine.set_max_notional_per_order(_AUDUSD_SIM.id, 1_000_000) # Initialize market quote = QuoteTick( - instrument_id=AUDUSD_SIM.id, + instrument_id=_AUDUSD_SIM.id, bid_price=Price.from_str("0.75000"), ask_price=Price.from_str("0.75005"), bid_size=Quantity.from_int(5_000_000), @@ -984,7 +990,7 @@ def test_submit_order_when_sell_market_order_and_over_max_notional_then_denies(s ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(10_000_000), ) @@ -1007,7 +1013,7 @@ def test_submit_order_when_sell_market_order_and_over_max_notional_then_denies(s def test_submit_order_when_market_order_and_over_free_balance_then_denies(self): # Arrange - Initialize market - quote = TestDataStubs.quote_tick(AUDUSD_SIM) + quote = TestDataStubs.quote_tick(_AUDUSD_SIM) self.cache.add_quote_tick(quote) self.exec_engine.start() @@ -1022,7 +1028,7 @@ def test_submit_order_when_market_order_and_over_free_balance_then_denies(self): ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(10_000_000), ) @@ -1045,7 +1051,7 @@ def test_submit_order_when_market_order_and_over_free_balance_then_denies(self): def test_submit_order_list_buys_when_over_free_balance_then_denies(self): # Arrange - Initialize market - quote = TestDataStubs.quote_tick(AUDUSD_SIM) + quote = TestDataStubs.quote_tick(_AUDUSD_SIM) self.cache.add_quote_tick(quote) self.exec_engine.start() @@ -1060,15 +1066,15 @@ def test_submit_order_list_buys_when_over_free_balance_then_denies(self): ) order1 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(500_000), ) order2 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, - Quantity.from_int(500_000), + Quantity.from_int(600_000), # <--- 100_000 over free balance ) order_list = OrderList( @@ -1094,7 +1100,7 @@ def test_submit_order_list_buys_when_over_free_balance_then_denies(self): def test_submit_order_list_sells_when_over_free_balance_then_denies(self): # Arrange - Initialize market - quote = TestDataStubs.quote_tick(AUDUSD_SIM) + quote = TestDataStubs.quote_tick(_AUDUSD_SIM) self.cache.add_quote_tick(quote) self.exec_engine.start() @@ -1109,15 +1115,15 @@ def test_submit_order_list_sells_when_over_free_balance_then_denies(self): ) order1 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(500_000), ) order2 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, - Quantity.from_int(500_000), + Quantity.from_int(600_000), ) order_list = OrderList( @@ -1158,11 +1164,11 @@ def test_submit_order_list_sells_when_multi_currency_cash_account_over_cumulativ self.exec_engine.deregister_client(self.exec_client) self.exec_engine.register_client(exec_client) self.cache.reset() # Clear accounts - self.cache.add_instrument(AUDUSD_SIM) # Re-add instrument + self.cache.add_instrument(_AUDUSD_SIM) # Re-add instrument self.portfolio.update_account(TestEventStubs.cash_account_state(base_currency=None)) # Prepare market - quote = TestDataStubs.quote_tick(AUDUSD_SIM) + quote = TestDataStubs.quote_tick(_AUDUSD_SIM) self.cache.add_quote_tick(quote) self.exec_engine.start() @@ -1177,15 +1183,15 @@ def test_submit_order_list_sells_when_multi_currency_cash_account_over_cumulativ ) order1 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(5_000), ) order2 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, - Quantity.from_int(5_000), + Quantity.from_int(6_000), ) order_list = OrderList( @@ -1211,10 +1217,10 @@ def test_submit_order_list_sells_when_multi_currency_cash_account_over_cumulativ def test_submit_order_when_reducing_and_buy_order_adds_then_denies(self): # Arrange - self.risk_engine.set_max_notional_per_order(AUDUSD_SIM.id, 1_000_000) + self.risk_engine.set_max_notional_per_order(_AUDUSD_SIM.id, 1_000_000) # Initialize market - quote = TestDataStubs.quote_tick(AUDUSD_SIM) + quote = TestDataStubs.quote_tick(_AUDUSD_SIM) self.cache.add_quote_tick(quote) self.exec_engine.start() @@ -1229,7 +1235,7 @@ def test_submit_order_when_reducing_and_buy_order_adds_then_denies(self): ) order1 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -1247,7 +1253,7 @@ def test_submit_order_when_reducing_and_buy_order_adds_then_denies(self): self.risk_engine.set_trading_state(TradingState.REDUCING) # <-- Allow reducing orders only order2 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -1263,7 +1269,7 @@ def test_submit_order_when_reducing_and_buy_order_adds_then_denies(self): self.exec_engine.process(TestEventStubs.order_submitted(order1)) self.exec_engine.process(TestEventStubs.order_accepted(order1)) - self.exec_engine.process(TestEventStubs.order_filled(order1, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(order1, _AUDUSD_SIM)) # Act self.risk_engine.execute(submit_order2) @@ -1271,15 +1277,15 @@ def test_submit_order_when_reducing_and_buy_order_adds_then_denies(self): # Assert assert order1.status == OrderStatus.FILLED assert order2.status == OrderStatus.DENIED - assert self.portfolio.is_net_long(AUDUSD_SIM.id) + assert self.portfolio.is_net_long(_AUDUSD_SIM.id) assert self.exec_engine.command_count == 1 # <-- Command never reaches engine def test_submit_order_when_reducing_and_sell_order_adds_then_denies(self): # Arrange - self.risk_engine.set_max_notional_per_order(AUDUSD_SIM.id, 1_000_000) + self.risk_engine.set_max_notional_per_order(_AUDUSD_SIM.id, 1_000_000) # Initialize market - quote = TestDataStubs.quote_tick(AUDUSD_SIM) + quote = TestDataStubs.quote_tick(_AUDUSD_SIM) self.cache.add_quote_tick(quote) self.exec_engine.start() @@ -1294,7 +1300,7 @@ def test_submit_order_when_reducing_and_sell_order_adds_then_denies(self): ) order1 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), ) @@ -1312,7 +1318,7 @@ def test_submit_order_when_reducing_and_sell_order_adds_then_denies(self): self.risk_engine.set_trading_state(TradingState.REDUCING) # <-- Allow reducing orders only order2 = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), ) @@ -1328,7 +1334,7 @@ def test_submit_order_when_reducing_and_sell_order_adds_then_denies(self): self.exec_engine.process(TestEventStubs.order_submitted(order1)) self.exec_engine.process(TestEventStubs.order_accepted(order1)) - self.exec_engine.process(TestEventStubs.order_filled(order1, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(order1, _AUDUSD_SIM)) # Act self.risk_engine.execute(submit_order2) @@ -1336,7 +1342,7 @@ def test_submit_order_when_reducing_and_sell_order_adds_then_denies(self): # Assert assert order1.status == OrderStatus.FILLED assert order2.status == OrderStatus.DENIED - assert self.portfolio.is_net_short(AUDUSD_SIM.id) + assert self.portfolio.is_net_short(_AUDUSD_SIM.id) assert self.exec_engine.command_count == 1 # <-- Command never reaches engine def test_submit_order_when_trading_halted_then_denies_order(self): @@ -1353,7 +1359,7 @@ def test_submit_order_when_trading_halted_then_denies_order(self): ) order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -1394,7 +1400,7 @@ def test_submit_order_beyond_rate_limit_then_denies_order(self): order = None for _ in range(101): order = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -1431,20 +1437,20 @@ def test_submit_order_list_when_trading_halted_then_denies_orders(self): ) entry = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) stop_loss = strategy.order_factory.stop_market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.00000"), ) take_profit = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.10000"), @@ -1490,7 +1496,7 @@ def test_submit_order_list_buys_when_trading_reducing_then_denies_orders(self): # Push portfolio LONG long = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) @@ -1508,23 +1514,23 @@ def test_submit_order_list_buys_when_trading_reducing_then_denies_orders(self): self.exec_engine.process(TestEventStubs.order_submitted(long)) self.exec_engine.process(TestEventStubs.order_accepted(long)) - self.exec_engine.process(TestEventStubs.order_filled(long, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(long, _AUDUSD_SIM)) entry = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), ) stop_loss = strategy.order_factory.stop_market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.00000"), ) take_profit = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.10000"), @@ -1570,7 +1576,7 @@ def test_submit_order_list_sells_when_trading_reducing_then_denies_orders(self): # Push portfolio SHORT short = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), ) @@ -1588,23 +1594,23 @@ def test_submit_order_list_sells_when_trading_reducing_then_denies_orders(self): self.exec_engine.process(TestEventStubs.order_submitted(short)) self.exec_engine.process(TestEventStubs.order_accepted(short)) - self.exec_engine.process(TestEventStubs.order_filled(short, AUDUSD_SIM)) + self.exec_engine.process(TestEventStubs.order_filled(short, _AUDUSD_SIM)) entry = strategy.order_factory.market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), ) stop_loss = strategy.order_factory.stop_market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("1.00000"), ) take_profit = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.SELL, Quantity.from_int(100_000), Price.from_str("1.10000"), @@ -1651,7 +1657,7 @@ def test_submit_bracket_with_default_settings_sends_to_client(self): ) bracket = strategy.order_factory.bracket( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), sl_trigger_price=Price.from_str("1.00000"), @@ -1687,7 +1693,7 @@ def test_submit_bracket_with_emulated_orders_sends_to_emulator(self): ) bracket = strategy.order_factory.bracket( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), sl_trigger_price=Price.from_str("1.00000"), @@ -1726,7 +1732,7 @@ def test_submit_bracket_order_when_instrument_not_in_cache_then_denies(self): ) bracket = strategy.order_factory.bracket( - GBPUSD_SIM.id, + _GBPUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), sl_trigger_price=Price.from_str("1.00000"), @@ -1762,7 +1768,7 @@ def test_submit_order_for_emulation_sends_command_to_emulator(self): ) order = strategy.order_factory.limit( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(1_000), Price.from_str("1.00000"), @@ -1793,7 +1799,7 @@ def test_modify_order_when_no_order_found_logs_error(self): modify = ModifyOrder( self.trader_id, strategy.id, - AUDUSD_SIM.id, + _AUDUSD_SIM.id, ClientOrderId("invalid"), VenueOrderId("1"), Quantity.from_int(100_000), @@ -1825,7 +1831,7 @@ def test_modify_order_beyond_rate_limit_then_rejects(self): ) order = strategy.order_factory.stop_market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.00010"), @@ -1838,7 +1844,7 @@ def test_modify_order_beyond_rate_limit_then_rejects(self): modify = ModifyOrder( self.trader_id, strategy.id, - AUDUSD_SIM.id, + _AUDUSD_SIM.id, order.client_order_id, VenueOrderId("1"), Quantity.from_int(100_000), @@ -1869,7 +1875,7 @@ def test_modify_order_with_default_settings_then_sends_to_client(self): ) order = strategy.order_factory.stop_market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.00010"), @@ -1921,7 +1927,7 @@ def test_modify_order_for_emulated_order_then_sends_to_emulator(self): ) order = strategy.order_factory.stop_market( - AUDUSD_SIM.id, + _AUDUSD_SIM.id, OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("1.00020"), @@ -2070,3 +2076,140 @@ def test_submit_order_when_market_order_and_over_free_balance_then_denies( # Assert assert order.status == expected_status + + +class TestRiskEngineWithCryptoCashAccount: + def setup(self): + # Fixture Setup + self.clock = TestClock() + self.trader_id = TestIdStubs.trader_id() + self.account_id = AccountId("BINANCE-001") + self.venue = Venue("BINANCE") + + self.msgbus = MessageBus( + trader_id=self.trader_id, + clock=self.clock, + ) + + self.cache = TestComponentStubs.cache() + + self.portfolio = Portfolio( + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + ) + + self.exec_engine = ExecutionEngine( + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + config=ExecEngineConfig(debug=True), + ) + + self.risk_engine = RiskEngine( + portfolio=self.portfolio, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + config=RiskEngineConfig(debug=True), + ) + + self.emulator = OrderEmulator( + portfolio=self.portfolio, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + ) + + self.exec_client = MockExecutionClient( + client_id=ClientId(self.venue.value), + venue=self.venue, + account_type=AccountType.CASH, + base_currency=USD, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + ) + + balances = [ + AccountBalance( + Money(440, ADA), + Money(0, ADA), + Money(440, ADA), + ), + AccountBalance( + Money(268.84000000, USDT), + Money(0, USDT), + Money(268.84000000, USDT), + ), + ] + + account_state = AccountState( + account_id=self.account_id, + account_type=AccountType.CASH, + base_currency=None, + reported=True, # reported + balances=balances, + margins=[], + info={}, + event_id=UUID4(), + ts_event=0, + ts_init=0, + ) + + self.portfolio.update_account(account_state) + self.exec_engine.register_client(self.exec_client) + + self.risk_engine.start() + self.exec_engine.start() + + @pytest.mark.parametrize( + ("order_side"), + [ + OrderSide.BUY, + OrderSide.SELL, + ], + ) + def test_submit_order_for_less_than_max_cum_transaction_value_adausdt( + self, + order_side: OrderSide, + ) -> None: + # Arrange + self.cache.add_instrument(_ADAUSDT_BINANCE) + quote = TestDataStubs.quote_tick( + instrument=_ADAUSDT_BINANCE, + bid_price=0.6109, + ask_price=0.6110, + ) + self.cache.add_quote_tick(quote) + + strategy = Strategy() + strategy.register( + trader_id=self.trader_id, + portfolio=self.portfolio, + msgbus=self.msgbus, + cache=self.cache, + clock=self.clock, + ) + + order = strategy.order_factory.market( + _ADAUSDT_BINANCE.id, + order_side, + Quantity.from_int(440), + ) + + submit_order = SubmitOrder( + trader_id=self.trader_id, + strategy_id=strategy.id, + position_id=None, + order=order, + command_id=UUID4(), + ts_init=self.clock.timestamp_ns(), + ) + + # Act + self.risk_engine.execute(submit_order) + + # Assert + assert order.status == OrderStatus.INITIALIZED + assert self.exec_engine.command_count == 1 From ac788c7ee81a45c99caff1ab38a59802da613ca6 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Wed, 21 Feb 2024 23:18:07 +1100 Subject: [PATCH 099/104] Fix RiskEngine cash value check when selling --- RELEASES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASES.md b/RELEASES.md index 1f9990cd5aad..a262066b72ab 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -16,6 +16,7 @@ None ### Fixes - Fixed `TradeId` memory leak due assigning unique values to the `Ustr` global string cache (which are never freed for the lifetime of the program) - Fixed `TradeTick` size precision for pyo3 conversion (size precision was incorrectly price precision) +- Fixed `RiskEngine` cash value check when selling (would previously divide quantity by price which is too much), thanks for reporting@AnthonyVince - Fixed FOK time in force behavior (allows fills beyond the top level, will cancel if cannot fill full size) - Fixed IOC time in force behavior (allows fills beyond the top level, will cancel any remaining after all fills are applied) - Fixed `LiveClock` timer behavior for small intervals causing next time to be less than now (timer then would not run) From 11e953f9d619898e84ade95264b23cd42b10c070 Mon Sep 17 00:00:00 2001 From: Pushkar Mishra Date: Thu, 22 Feb 2024 02:48:47 +0530 Subject: [PATCH 100/104] Ported ChandeMomentumOscillator to Rust (#1508) --- nautilus_core/indicators/src/momentum/cmo.rs | 207 ++++++++++++++++++ nautilus_core/indicators/src/momentum/mod.rs | 1 + nautilus_core/indicators/src/python/mod.rs | 1 + .../indicators/src/python/momentum/cmo.rs | 95 ++++++++ .../indicators/src/python/momentum/mod.rs | 1 + nautilus_core/indicators/src/stubs.rs | 7 +- nautilus_trader/core/nautilus_pyo3.pyi | 21 ++ 7 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 nautilus_core/indicators/src/momentum/cmo.rs create mode 100644 nautilus_core/indicators/src/python/momentum/cmo.rs diff --git a/nautilus_core/indicators/src/momentum/cmo.rs b/nautilus_core/indicators/src/momentum/cmo.rs new file mode 100644 index 000000000000..8fff4c3af949 --- /dev/null +++ b/nautilus_core/indicators/src/momentum/cmo.rs @@ -0,0 +1,207 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::fmt::Display; + +use anyhow::Result; +use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; +use pyo3::prelude::*; + +use crate::{ + average::{MovingAverageFactory, MovingAverageType}, + indicator::{Indicator, MovingAverage}, +}; + +#[repr(C)] +#[derive(Debug)] +#[pyclass(module = "nautilus_trader.core.nautilus.pyo3.indicators")] +pub struct ChandeMomentumOscillator { + pub period: usize, + pub ma_type: MovingAverageType, + pub value: f64, + pub count: usize, + pub initialized: bool, + _previous_close: f64, + _average_gain: Box, + _average_loss: Box, + _has_inputs: bool, +} + +impl Display for ChandeMomentumOscillator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}({})", self.name(), self.period) + } +} + +impl Indicator for ChandeMomentumOscillator { + fn name(&self) -> String { + stringify!(ChandeMomentumOscillator).to_string() + } + + fn has_inputs(&self) -> bool { + self._has_inputs + } + + fn initialized(&self) -> bool { + self.initialized + } + + fn handle_quote_tick(&mut self, _tick: &QuoteTick) { + // Function body intentionally left blank. + } + + fn handle_trade_tick(&mut self, _tick: &TradeTick) { + // Function body intentionally left blank. + } + + fn handle_bar(&mut self, bar: &Bar) { + self.update_raw((&bar.close).into()); + } + + fn reset(&mut self) { + self.value = 0.0; + self.count = 0; + self._has_inputs = false; + self.initialized = false; + self._previous_close = 0.0; + } +} + +impl ChandeMomentumOscillator { + pub fn new(period: usize, ma_type: Option) -> Result { + Ok(Self { + period, + ma_type: ma_type.unwrap_or(MovingAverageType::Wilder), + _average_gain: MovingAverageFactory::create(MovingAverageType::Wilder, period), + _average_loss: MovingAverageFactory::create(MovingAverageType::Wilder, period), + _previous_close: 0.0, + value: 0.0, + count: 0, + initialized: false, + _has_inputs: false, + }) + } + + pub fn update_raw(&mut self, close: f64) { + if !self._has_inputs { + self._previous_close = close; + self._has_inputs = true; + } + + let gain: f64 = close - self._previous_close; + if gain > 0.0 { + self._average_gain.update_raw(gain); + self._average_loss.update_raw(0.0); + } else if gain < 0.0 { + self._average_gain.update_raw(0.0); + self._average_loss.update_raw(-gain); + } else { + self._average_gain.update_raw(0.0); + self._average_loss.update_raw(0.0); + } + + if !self.initialized && self._average_gain.initialized() && self._average_loss.initialized() + { + self.initialized = true; + } + if self.initialized { + self.value = 100.0 * (self._average_gain.value() - self._average_loss.value()) + / (self._average_gain.value() + self._average_loss.value()); + } + self._previous_close = close; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////////////// +#[cfg(test)] +mod tests { + use nautilus_model::data::{bar::Bar, quote::QuoteTick}; + use rstest::rstest; + + use crate::{indicator::Indicator, momentum::cmo::ChandeMomentumOscillator, stubs::*}; + + #[rstest] + fn test_cmo_initialized(cmo_10: ChandeMomentumOscillator) { + let display_str = format!("{cmo_10}"); + assert_eq!(display_str, "ChandeMomentumOscillator(10)"); + assert_eq!(cmo_10.period, 10); + assert!(!cmo_10.initialized); + } + + #[rstest] + fn test_initialized_with_required_inputs_returns_true(mut cmo_10: ChandeMomentumOscillator) { + for i in 0..12 { + cmo_10.update_raw(f64::from(i)); + } + assert!(cmo_10.initialized); + } + + #[rstest] + fn test_value_all_higher_inputs_returns_expected_value(mut cmo_10: ChandeMomentumOscillator) { + cmo_10.update_raw(109.93); + cmo_10.update_raw(110.0); + cmo_10.update_raw(109.77); + cmo_10.update_raw(109.96); + cmo_10.update_raw(110.29); + cmo_10.update_raw(110.53); + cmo_10.update_raw(110.27); + cmo_10.update_raw(110.21); + cmo_10.update_raw(110.06); + cmo_10.update_raw(110.19); + cmo_10.update_raw(109.83); + cmo_10.update_raw(109.9); + cmo_10.update_raw(110.0); + cmo_10.update_raw(110.03); + cmo_10.update_raw(110.13); + cmo_10.update_raw(109.95); + cmo_10.update_raw(109.75); + cmo_10.update_raw(110.15); + cmo_10.update_raw(109.9); + cmo_10.update_raw(110.04); + assert_eq!(cmo_10.value, 2.089_629_456_238_705_4); + } + + #[rstest] + fn test_value_with_one_input_returns_expected_value(mut cmo_10: ChandeMomentumOscillator) { + cmo_10.update_raw(1.00000); + assert_eq!(cmo_10.value, 0.0); + } + + #[rstest] + fn test_reset(mut cmo_10: ChandeMomentumOscillator) { + cmo_10.update_raw(1.00020); + cmo_10.update_raw(1.00030); + cmo_10.update_raw(1.00050); + cmo_10.reset(); + assert!(!cmo_10.initialized()); + assert_eq!(cmo_10.count, 0); + } + + #[rstest] + fn test_handle_quote_tick(mut cmo_10: ChandeMomentumOscillator, quote_tick: QuoteTick) { + cmo_10.handle_quote_tick("e_tick); + assert_eq!(cmo_10.count, 0); + assert_eq!(cmo_10.value, 0.0); + } + + #[rstest] + fn test_handle_bar(mut cmo_10: ChandeMomentumOscillator, bar_ethusdt_binance_minute_bid: Bar) { + cmo_10.handle_bar(&bar_ethusdt_binance_minute_bid); + assert_eq!(cmo_10.count, 0); + assert_eq!(cmo_10.value, 0.0); + } +} diff --git a/nautilus_core/indicators/src/momentum/mod.rs b/nautilus_core/indicators/src/momentum/mod.rs index 35fd3d83c086..daf02fc72965 100644 --- a/nautilus_core/indicators/src/momentum/mod.rs +++ b/nautilus_core/indicators/src/momentum/mod.rs @@ -14,4 +14,5 @@ // ------------------------------------------------------------------------------------------------- pub mod aroon; +pub mod cmo; pub mod rsi; diff --git a/nautilus_core/indicators/src/python/mod.rs b/nautilus_core/indicators/src/python/mod.rs index 2efd0a36a0d5..ca7b1da5cbe8 100644 --- a/nautilus_core/indicators/src/python/mod.rs +++ b/nautilus_core/indicators/src/python/mod.rs @@ -37,6 +37,7 @@ pub fn indicators(_: Python<'_>, m: &PyModule) -> PyResult<()> { // momentum m.add_class::()?; m.add_class::()?; + m.add_class::()?; // volatility m.add_class::()?; Ok(()) diff --git a/nautilus_core/indicators/src/python/momentum/cmo.rs b/nautilus_core/indicators/src/python/momentum/cmo.rs new file mode 100644 index 000000000000..37e7915671c1 --- /dev/null +++ b/nautilus_core/indicators/src/python/momentum/cmo.rs @@ -0,0 +1,95 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use nautilus_core::python::to_pyvalue_err; +use nautilus_model::data::{bar::Bar, quote::QuoteTick, trade::TradeTick}; +use pyo3::prelude::*; + +use crate::{ + average::MovingAverageType, indicator::Indicator, momentum::cmo::ChandeMomentumOscillator, +}; + +#[pymethods] +impl ChandeMomentumOscillator { + #[new] + pub fn py_new(period: usize, ma_type: Option) -> PyResult { + Self::new(period, ma_type).map_err(to_pyvalue_err) + } + + #[getter] + #[pyo3(name = "name")] + fn py_name(&self) -> String { + self.name() + } + + #[getter] + #[pyo3(name = "period")] + fn py_period(&self) -> usize { + self.period + } + + #[getter] + #[pyo3(name = "has_inputs")] + fn py_has_inputs(&self) -> bool { + self.has_inputs() + } + + #[getter] + #[pyo3(name = "count")] + fn py_count(&self) -> usize { + self.count + } + + #[getter] + #[pyo3(name = "value")] + fn py_value(&self) -> f64 { + self.value + } + + #[getter] + #[pyo3(name = "initialized")] + fn py_initialized(&self) -> bool { + self.initialized + } + + #[pyo3(name = "update_raw")] + fn py_update_raw(&mut self, close: f64) { + self.update_raw(close); + } + + #[pyo3(name = "handle_quote_tick")] + fn py_handle_quote_tick(&mut self, _tick: &QuoteTick) { + // Function body intentionally left blank. + } + + #[pyo3(name = "handle_trade_tick")] + fn py_handle_trade_tick(&mut self, _tick: &TradeTick) { + // Function body intentionally left blank. + } + + #[pyo3(name = "handle_bar")] + fn py_handle_bar(&mut self, bar: &Bar) { + self.update_raw((&bar.close).into()); + } + + #[pyo3(name = "reset")] + fn py_reset(&mut self) { + self.reset() + } + + fn __repr__(&self) -> String { + format!("ChandeMomentumOscillator({})", self.period) + } +} diff --git a/nautilus_core/indicators/src/python/momentum/mod.rs b/nautilus_core/indicators/src/python/momentum/mod.rs index 35fd3d83c086..daf02fc72965 100644 --- a/nautilus_core/indicators/src/python/momentum/mod.rs +++ b/nautilus_core/indicators/src/python/momentum/mod.rs @@ -14,4 +14,5 @@ // ------------------------------------------------------------------------------------------------- pub mod aroon; +pub mod cmo; pub mod rsi; diff --git a/nautilus_core/indicators/src/stubs.rs b/nautilus_core/indicators/src/stubs.rs index a4450369ebc4..53014cae7566 100644 --- a/nautilus_core/indicators/src/stubs.rs +++ b/nautilus_core/indicators/src/stubs.rs @@ -31,7 +31,7 @@ use crate::{ ema::ExponentialMovingAverage, hma::HullMovingAverage, rma::WilderMovingAverage, sma::SimpleMovingAverage, wma::WeightedMovingAverage, MovingAverageType, }, - momentum::rsi::RelativeStrengthIndex, + momentum::{cmo::ChandeMomentumOscillator, rsi::RelativeStrengthIndex}, ratio::efficiency_ratio::EfficiencyRatio, }; @@ -149,3 +149,8 @@ pub fn efficiency_ratio_10() -> EfficiencyRatio { pub fn rsi_10() -> RelativeStrengthIndex { RelativeStrengthIndex::new(10, Some(MovingAverageType::Exponential)).unwrap() } + +#[fixture] +pub fn cmo_10() -> ChandeMomentumOscillator { + ChandeMomentumOscillator::new(10, Some(MovingAverageType::Wilder)).unwrap() +} diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 57b664ac8741..65bcf09c06ec 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -2049,6 +2049,27 @@ class WilderMovingAverage: def handle_bar(self, bar: Bar) -> None: ... def reset(self) -> None: ... +class ChandeMomentumOscillator: + def __init__( + self, + period: int, + ) -> None: ... + @property + def name(self) -> str: ... + @property + def period(self) -> int: ... + @property + def count(self) -> int: ... + @property + def initialized(self) -> bool: ... + @property + def has_inputs(self) -> bool: ... + @property + def value(self) -> float: ... + def update_raw(self, close: float) -> None: ... + def handle_bar(self, bar: Bar) -> None: ... + def reset(self) -> None: ... + class AroonOscillator: def __init__( self, From db97b95c37f1280ac8cf14e69874aafc84d8f327 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Thu, 22 Feb 2024 18:09:35 +1100 Subject: [PATCH 101/104] Update dependencies including hyper --- nautilus_core/Cargo.lock | 23 ++++++++++++----------- nautilus_core/network/Cargo.toml | 2 +- poetry.lock | 6 +++--- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/nautilus_core/Cargo.lock b/nautilus_core/Cargo.lock index 69046d218c12..1d7e7c542d0d 100644 --- a/nautilus_core/Cargo.lock +++ b/nautilus_core/Cargo.lock @@ -420,7 +420,7 @@ dependencies = [ "http 1.0.0", "http-body 1.0.0", "http-body-util", - "hyper 1.1.0", + "hyper 1.2.0", "hyper-util", "itoa", "matchit", @@ -603,9 +603,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.1" +version = "3.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c764d619ca78fccbf3069b37bd7af92577f044bb15236036662d79b6559f25b7" +checksum = "a3b1be7772ee4501dba05acbe66bb1e8760f6a6c474a36035631638e4415f130" [[package]] name = "bytecheck" @@ -2004,9 +2004,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" dependencies = [ "bytes", "futures-channel", @@ -2018,6 +2018,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", + "smallvec", "tokio", ] @@ -2044,7 +2045,7 @@ dependencies = [ "futures-util", "http 1.0.0", "http-body 1.0.0", - "hyper 1.1.0", + "hyper 1.2.0", "pin-project-lite", "socket2 0.5.5", "tokio", @@ -2591,7 +2592,7 @@ dependencies = [ "futures", "futures-util", "http 1.0.0", - "hyper 1.1.0", + "hyper 1.2.0", "nautilus-core", "nonzero_ext", "pyo3", @@ -2906,9 +2907,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.100" +version = "0.9.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae94056a791d0e1217d18b6cbdccb02c61e3054fc69893607f4067e3bb0b1fd1" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" dependencies = [ "cc", "libc", @@ -4565,9 +4566,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.12.13" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "tempfile" diff --git a/nautilus_core/network/Cargo.toml b/nautilus_core/network/Cargo.toml index d9b3e062c93e..00635600e861 100644 --- a/nautilus_core/network/Cargo.toml +++ b/nautilus_core/network/Cargo.toml @@ -21,7 +21,7 @@ tokio = { workspace = true } dashmap = "5.5.3" futures-util = "0.3.29" http = "1.0.0" -hyper = "1.1.0" +hyper = "1.2.0" nonzero_ext = "0.3.0" reqwest = "0.11.24" tokio-tungstenite = { path = "./tokio-tungstenite", features = ["rustls-tls-native-roots"] } diff --git a/poetry.lock b/poetry.lock index f6914838378a..7b1b59f1680d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2452,13 +2452,13 @@ test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)" [[package]] name = "virtualenv" -version = "20.25.0" +version = "20.25.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.25.0-py3-none-any.whl", hash = "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3"}, - {file = "virtualenv-20.25.0.tar.gz", hash = "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"}, + {file = "virtualenv-20.25.1-py3-none-any.whl", hash = "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a"}, + {file = "virtualenv-20.25.1.tar.gz", hash = "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"}, ] [package.dependencies] From dc7e83768644db90def3174cd6d355fe1e023ef7 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Thu, 22 Feb 2024 18:15:22 +1100 Subject: [PATCH 102/104] Cleanup cash_value base currency --- nautilus_trader/risk/engine.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautilus_trader/risk/engine.pyx b/nautilus_trader/risk/engine.pyx index 002c185f4d70..023ba5b83b98 100644 --- a/nautilus_trader/risk/engine.pyx +++ b/nautilus_trader/risk/engine.pyx @@ -729,7 +729,7 @@ cdef class RiskEngine(Component): ) return False # Denied elif base_currency is not None: - cash_value = Money(order.quantity.as_f64_c(), instrument.base_currency) + cash_value = Money(order.quantity.as_f64_c(), base_currency) free = account.balance_free(base_currency) if cum_notional_sell is None: cum_notional_sell = cash_value From 3d54cecb5bf48ddea47d9135ade7735e6e013df0 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Thu, 22 Feb 2024 20:50:25 +1100 Subject: [PATCH 103/104] Add OrderBookDeltas to_pyo3 conversion --- nautilus_core/model/src/python/data/deltas.rs | 15 ++++- nautilus_trader/common/actor.pxd | 20 +++--- nautilus_trader/common/actor.pyx | 24 +++++-- .../strategies/orderbook_imbalance_rust.py | 10 +-- nautilus_trader/model/data.pxd | 4 +- nautilus_trader/model/data.pyx | 63 +++++++++++-------- .../tracemalloc_orderbook_deltas.py | 16 ++++- tests/unit_tests/model/test_orderbook_data.py | 12 ++++ 8 files changed, 111 insertions(+), 53 deletions(-) diff --git a/nautilus_core/model/src/python/data/deltas.rs b/nautilus_core/model/src/python/data/deltas.rs index cd796c87d96f..21e58a95f117 100644 --- a/nautilus_core/model/src/python/data/deltas.rs +++ b/nautilus_core/model/src/python/data/deltas.rs @@ -16,13 +16,15 @@ use std::{ collections::hash_map::DefaultHasher, hash::{Hash, Hasher}, + ops::Deref, }; use nautilus_core::time::UnixNanos; -use pyo3::{prelude::*, pyclass::CompareOp}; +use pyo3::{prelude::*, pyclass::CompareOp, types::PyCapsule}; use crate::{ data::{delta::OrderBookDelta, deltas::OrderBookDeltas}, + ffi::data::deltas::OrderBookDeltas_API, identifiers::instrument_id::InstrumentId, python::common::PY_MODULE_MODEL, }; @@ -99,6 +101,17 @@ impl OrderBookDeltas { format!("{}:{}", PY_MODULE_MODEL, stringify!(OrderBookDeltas)) } + #[staticmethod] + #[pyo3(name = "from_pycapsule")] + pub fn py_from_pycapsule(capsule: &PyAny) -> OrderBookDeltas { + let capsule: &PyCapsule = capsule + .downcast() + .expect("Error on downcast to `&PyCapsule`"); + let data: &OrderBookDeltas_API = + unsafe { &*(capsule.pointer() as *const OrderBookDeltas_API) }; + data.deref().clone() + } + // /// Creates a `PyCapsule` containing a raw pointer to a `Data::Delta` object. // /// // /// This function takes the current object (assumed to be of a type that can be represented as diff --git a/nautilus_trader/common/actor.pxd b/nautilus_trader/common/actor.pxd index 78246fb6c3dd..270b4e399f7c 100644 --- a/nautilus_trader/common/actor.pxd +++ b/nautilus_trader/common/actor.pxd @@ -51,13 +51,14 @@ from nautilus_trader.portfolio.base cimport PortfolioFacade cdef class Actor(Component): cdef object _executor - cdef set _warning_events - cdef dict _signal_classes - cdef dict _pending_requests - cdef list _indicators - cdef dict _indicators_for_quotes - cdef dict _indicators_for_trades - cdef dict _indicators_for_bars + cdef set[type] _warning_events + cdef dict[str, type] _signal_classes + cdef dict[UUID4, object] _pending_requests + cdef list[Indicator] _indicators + cdef dict[InstrumentId, list[Indicator]] _indicators_for_quotes + cdef dict[InstrumentId, list[Indicator]] _indicators_for_trades + cdef dict[BarType, list[Indicator]] _indicators_for_bars + cdef set[type] _pyo3_conversion_types cdef readonly PortfolioFacade portfolio """The read-only portfolio for the actor.\n\n:returns: `PortfolioFacade`""" @@ -89,7 +90,7 @@ cdef class Actor(Component): cpdef void on_instrument_status(self, InstrumentStatus data) cpdef void on_instrument_close(self, InstrumentClose data) cpdef void on_instrument(self, Instrument instrument) - cpdef void on_order_book_deltas(self, OrderBookDeltas deltas) + cpdef void on_order_book_deltas(self, deltas) cpdef void on_order_book(self, OrderBook order_book) cpdef void on_quote_tick(self, QuoteTick tick) cpdef void on_trade_tick(self, TradeTick tick) @@ -144,6 +145,7 @@ cdef class Actor(Component): dict kwargs=*, ClientId client_id=*, bint managed=*, + bint pyo3_conversion=*, ) cpdef void subscribe_order_book_snapshots( self, @@ -231,7 +233,7 @@ cdef class Actor(Component): cpdef void handle_instrument(self, Instrument instrument) cpdef void handle_instruments(self, list instruments) cpdef void handle_order_book(self, OrderBook order_book) - cpdef void handle_order_book_deltas(self, OrderBookDeltas deltas) + cpdef void handle_order_book_deltas(self, deltas) cpdef void handle_quote_tick(self, QuoteTick tick) cpdef void handle_quote_ticks(self, list ticks) cpdef void handle_trade_tick(self, TradeTick tick) diff --git a/nautilus_trader/common/actor.pyx b/nautilus_trader/common/actor.pyx index 7f43f90e9ba0..40722df2efb1 100644 --- a/nautilus_trader/common/actor.pyx +++ b/nautilus_trader/common/actor.pyx @@ -125,6 +125,8 @@ cdef class Actor(Component): self._indicators_for_trades: dict[InstrumentId, list[Indicator]] = {} self._indicators_for_bars: dict[BarType, list[Indicator]] = {} + self._pyo3_conversion_types = set() + # Configuration self.config = config @@ -382,13 +384,13 @@ cdef class Actor(Component): """ # Optionally override in subclass - cpdef void on_order_book_deltas(self, OrderBookDeltas deltas): + cpdef void on_order_book_deltas(self, deltas): """ Actions to be performed when running and receives order book deltas. Parameters ---------- - deltas : OrderBookDeltas + deltas : OrderBookDeltas or nautilus_pyo3.OrderBookDeltas The order book deltas received. Warnings @@ -1185,6 +1187,7 @@ cdef class Actor(Component): dict kwargs = None, ClientId client_id = None, bint managed = True, + bint pyo3_conversion = False, ): """ Subscribe to the order book data stream, being a snapshot then deltas @@ -1205,11 +1208,17 @@ cdef class Actor(Component): If ``None`` then will be inferred from the venue in the instrument ID. managed : bool, default True If an order book should be managed by the data engine based on the subscribed feed. + pyo3_conversion : bool, default False + If received deltas should be converted to `nautilus_pyo3.OrderBookDeltas` + prior to being passed to the `on_order_book_deltas` handler. """ Condition.not_none(instrument_id, "instrument_id") Condition.true(self.trader_id is not None, "The actor has not been registered") + if pyo3_conversion: + self._pyo3_conversion_types.add(OrderBookDeltas) + self._msgbus.subscribe( topic=f"data.book.deltas" f".{instrument_id.venue}" @@ -2396,15 +2405,17 @@ cdef class Actor(Component): for i in range(length): self.handle_instrument(instruments[i]) - cpdef void handle_order_book_deltas(self, OrderBookDeltas deltas): + cpdef void handle_order_book_deltas(self, deltas): """ Handle the given order book deltas. - Passes to `on_order_book_delta` if state is ``RUNNING``. + Passes to `on_order_book_deltas` if state is ``RUNNING``. + The `deltas` will be `nautilus_pyo3.OrderBookDeltas` if the + pyo3_conversion flag was set for the subscription. Parameters ---------- - deltas : OrderBookDeltas + deltas : OrderBookDeltas or nautilus_pyo3.OrderBookDeltas The order book deltas received. Warnings @@ -2414,6 +2425,9 @@ cdef class Actor(Component): """ Condition.not_none(deltas, "deltas") + if OrderBookDeltas in self._pyo3_conversion_types: + deltas = deltas.to_pyo3() + if self._fsm.state == ComponentState.RUNNING: try: self.on_order_book_deltas(deltas) diff --git a/nautilus_trader/examples/strategies/orderbook_imbalance_rust.py b/nautilus_trader/examples/strategies/orderbook_imbalance_rust.py index 1cdfc5612689..f995cc7b0ceb 100644 --- a/nautilus_trader/examples/strategies/orderbook_imbalance_rust.py +++ b/nautilus_trader/examples/strategies/orderbook_imbalance_rust.py @@ -24,8 +24,6 @@ from nautilus_trader.core.nautilus_pyo3 import OrderBookMbp from nautilus_trader.core.rust.common import LogColor from nautilus_trader.model.book import OrderBook -from nautilus_trader.model.data import OrderBookDelta -from nautilus_trader.model.data import OrderBookDeltas from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.enums import BookType from nautilus_trader.model.enums import OrderSide @@ -138,18 +136,16 @@ def on_start(self) -> None: self.instrument.id, self.book_type, managed=False, # <-- Manually applying deltas to book + pyo3_conversion=True, # <--- Will automatically convert to pyo3 objects ) self._last_trigger_timestamp = self.clock.utc_now() - def on_order_book_deltas(self, deltas: OrderBookDeltas) -> None: + def on_order_book_deltas(self, pyo3_deltas: nautilus_pyo3.OrderBookDeltas) -> None: """ Actions to be performed when order book deltas are received. """ - # Convert to pyo3 objects (the efficiency of this can improve) - pyo3_deltas = OrderBookDelta.to_pyo3_list(deltas.deltas) - for pyo3_delta in pyo3_deltas: - self.book.apply_delta(pyo3_delta) + self.book.apply_deltas(pyo3_deltas) self.imbalance.handle_book_mbp(self.book) self.check_trigger() diff --git a/nautilus_trader/model/data.pxd b/nautilus_trader/model/data.pxd index 21eaf3945cdc..92f0c13dc169 100644 --- a/nautilus_trader/model/data.pxd +++ b/nautilus_trader/model/data.pxd @@ -52,7 +52,7 @@ cpdef list capsule_to_list(capsule) cpdef Data capsule_to_data(capsule) cdef inline void capsule_destructor(object capsule): - cdef CVec* cvec = PyCapsule_GetPointer(capsule, NULL) + cdef CVec *cvec = PyCapsule_GetPointer(capsule, NULL) PyMem_Free(cvec[0].ptr) # de-allocate buffer PyMem_Free(cvec) # de-allocate cvec @@ -244,6 +244,8 @@ cdef class OrderBookDeltas(Data): @staticmethod cdef dict to_dict_c(OrderBookDeltas obj) + cpdef to_pyo3(self) + cdef class OrderBookDepth10(Data): cdef OrderBookDepth10_t _mem diff --git a/nautilus_trader/model/data.pyx b/nautilus_trader/model/data.pyx index 0ec97c7c9355..a24cb7dbdb52 100644 --- a/nautilus_trader/model/data.pyx +++ b/nautilus_trader/model/data.pyx @@ -1888,12 +1888,12 @@ cdef class OrderBookDelta(Data): ) @staticmethod - cdef inline list capsule_to_list_c(object capsule): + cdef list[OrderBookDelta] capsule_to_list_c(object capsule): # SAFETY: Do NOT deallocate the capsule here # It is supposed to be deallocated by the creator cdef CVec* data = PyCapsule_GetPointer(capsule, NULL) cdef OrderBookDelta_t* ptr = data.ptr - cdef list deltas = [] + cdef list[OrderBookDelta] deltas = [] cdef uint64_t i for i in range(0, data.len): @@ -1902,18 +1902,18 @@ cdef class OrderBookDelta(Data): return deltas @staticmethod - cdef inline list_to_capsule_c(list items): + cdef object list_to_capsule_c(list items): # Create a C struct buffer cdef uint64_t len_ = len(items) - cdef OrderBookDelta_t * data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) + cdef OrderBookDelta_t *data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) cdef uint64_t i for i in range(len_): - data[i] = ( items[i])._mem + data[i] = (items[i])._mem if not data: raise MemoryError() # Create CVec - cdef CVec * cvec = PyMem_Malloc(1 * sizeof(CVec)) + cdef CVec *cvec = PyMem_Malloc(1 * sizeof(CVec)) cvec.ptr = data cvec.len = len_ cvec.cap = len_ @@ -2153,7 +2153,7 @@ cdef class OrderBookDeltas(Data): cdef uint64_t len_ = len(deltas) # Create a C OrderBookDeltas_t buffer - cdef OrderBookDelta_t* data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) + cdef OrderBookDelta_t *data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) if not data: raise MemoryError() @@ -2164,7 +2164,7 @@ cdef class OrderBookDeltas(Data): data[i] = delta._mem # Create CVec - cdef CVec* cvec = PyMem_Malloc(1 * sizeof(CVec)) + cdef CVec *cvec = PyMem_Malloc(1 * sizeof(CVec)) if not cvec: raise MemoryError() @@ -2195,7 +2195,7 @@ cdef class OrderBookDeltas(Data): cdef uint64_t len_ = len(deltas) # Create a C OrderBookDeltas_t buffer - cdef OrderBookDelta_t* data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) + cdef OrderBookDelta_t *data = PyMem_Malloc(len_ * sizeof(OrderBookDelta_t)) if not data: raise MemoryError() @@ -2206,7 +2206,7 @@ cdef class OrderBookDeltas(Data): data[i] = delta._mem # Create CVec - cdef CVec* cvec = PyMem_Malloc(1 * sizeof(CVec)) + cdef CVec *cvec = PyMem_Malloc(1 * sizeof(CVec)) if not cvec: raise MemoryError() @@ -2386,6 +2386,15 @@ cdef class OrderBookDeltas(Data): """ return OrderBookDeltas.to_dict_c(obj) + cpdef to_pyo3(self): + cdef OrderBookDeltas_API *data = PyMem_Malloc(sizeof(OrderBookDeltas_API)) + data[0] = self._mem + capsule = PyCapsule_New(data, NULL, NULL) + deltas = nautilus_pyo3.OrderBookDeltas.from_pycapsule(capsule) + PyMem_Free(data) + return deltas + + cdef class OrderBookDepth10(Data): """ @@ -2747,12 +2756,12 @@ cdef class OrderBookDepth10(Data): } @staticmethod - cdef inline list capsule_to_list_c(object capsule): + cdef list[OrderBookDepth10] capsule_to_list_c(object capsule): # SAFETY: Do NOT deallocate the capsule here # It is supposed to be deallocated by the creator cdef CVec* data = PyCapsule_GetPointer(capsule, NULL) cdef OrderBookDepth10_t* ptr = data.ptr - cdef list depths = [] + cdef list[OrderBookDepth10] depths = [] cdef uint64_t i for i in range(0, data.len): @@ -2761,10 +2770,10 @@ cdef class OrderBookDepth10(Data): return depths @staticmethod - cdef inline list_to_capsule_c(list items): + cdef object list_to_capsule_c(list items): # Create a C struct buffer cdef uint64_t len_ = len(items) - cdef OrderBookDepth10_t * data = PyMem_Malloc(len_ * sizeof(OrderBookDepth10_t)) + cdef OrderBookDepth10_t * data = PyMem_Malloc(len_ * sizeof(OrderBookDepth10_t)) cdef uint64_t i for i in range(len_): data[i] = (items[i])._mem @@ -2772,7 +2781,7 @@ cdef class OrderBookDepth10(Data): raise MemoryError() # Create CVec - cdef CVec * cvec = PyMem_Malloc(1 * sizeof(CVec)) + cdef CVec * cvec = PyMem_Malloc(1 * sizeof(CVec)) cvec.ptr = data cvec.len = len_ cvec.cap = len_ @@ -3425,12 +3434,12 @@ cdef class QuoteTick(Data): return quote @staticmethod - cdef inline list capsule_to_list_c(object capsule): + cdef list[QuoteTick] capsule_to_list_c(object capsule): # SAFETY: Do NOT deallocate the capsule here # It is supposed to be deallocated by the creator cdef CVec* data = PyCapsule_GetPointer(capsule, NULL) cdef QuoteTick_t* ptr = data.ptr - cdef list quotes = [] + cdef list[QuoteTick] quotes = [] cdef uint64_t i for i in range(0, data.len): @@ -3439,18 +3448,18 @@ cdef class QuoteTick(Data): return quotes @staticmethod - cdef inline list_to_capsule_c(list items): + cdef object list_to_capsule_c(list items): # Create a C struct buffer cdef uint64_t len_ = len(items) - cdef QuoteTick_t * data = PyMem_Malloc(len_ * sizeof(QuoteTick_t)) + cdef QuoteTick_t * data = PyMem_Malloc(len_ * sizeof(QuoteTick_t)) cdef uint64_t i for i in range(len_): - data[i] = ( items[i])._mem + data[i] = (items[i])._mem if not data: raise MemoryError() # Create CVec - cdef CVec * cvec = PyMem_Malloc(1 * sizeof(CVec)) + cdef CVec *cvec = PyMem_Malloc(1 * sizeof(CVec)) cvec.ptr = data cvec.len = len_ cvec.cap = len_ @@ -3916,12 +3925,12 @@ cdef class TradeTick(Data): return trade @staticmethod - cdef inline list capsule_to_list_c(capsule): + cdef list[TradeTick] capsule_to_list_c(capsule): # SAFETY: Do NOT deallocate the capsule here # It is supposed to be deallocated by the creator cdef CVec* data = PyCapsule_GetPointer(capsule, NULL) cdef TradeTick_t* ptr = data.ptr - cdef list trades = [] + cdef list[TradeTick] trades = [] cdef uint64_t i for i in range(0, data.len): @@ -3930,18 +3939,18 @@ cdef class TradeTick(Data): return trades @staticmethod - cdef inline list_to_capsule_c(list items): + cdef object list_to_capsule_c(list items): # Create a C struct buffer cdef uint64_t len_ = len(items) - cdef TradeTick_t * data = PyMem_Malloc(len_ * sizeof(TradeTick_t)) + cdef TradeTick_t *data = PyMem_Malloc(len_ * sizeof(TradeTick_t)) cdef uint64_t i for i in range(len_): - data[i] = ( items[i])._mem + data[i] = (items[i])._mem if not data: raise MemoryError() # Create CVec - cdef CVec* cvec = PyMem_Malloc(1 * sizeof(CVec)) + cdef CVec *cvec = PyMem_Malloc(1 * sizeof(CVec)) cvec.ptr = data cvec.len = len_ cvec.cap = len_ diff --git a/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py b/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py index b12f2cb3d78e..69538992c693 100644 --- a/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py +++ b/tests/mem_leak_tests/tracemalloc_orderbook_deltas.py @@ -20,13 +20,22 @@ @snapshot_memory(4000) -def run_repr(*args, **kwargs): +def run_to_pyo3(*args, **kwargs): delta = TestDataStubs.order_book_delta() deltas = OrderBookDeltas(delta.instrument_id, deltas=[delta] * 1024) - repr(deltas.deltas) + pyo3_deltas = deltas.to_pyo3() + repr(pyo3_deltas) repr(deltas) +# @snapshot_memory(4000) +# def run_repr(*args, **kwargs): +# delta = TestDataStubs.order_book_delta() +# deltas = OrderBookDeltas(delta.instrument_id, deltas=[delta] * 1024) +# repr(deltas.deltas) +# repr(deltas) + + # @snapshot_memory(4000) # def run_from_pyo3(*args, **kwargs): # pyo3_delta = TestDataProviderPyo3.order_book_delta() @@ -34,5 +43,6 @@ def run_repr(*args, **kwargs): if __name__ == "__main__": - run_repr() + run_to_pyo3() + # run_repr() # run_from_pyo3() diff --git a/tests/unit_tests/model/test_orderbook_data.py b/tests/unit_tests/model/test_orderbook_data.py index 65054520ec95..0ee6814fc351 100644 --- a/tests/unit_tests/model/test_orderbook_data.py +++ b/tests/unit_tests/model/test_orderbook_data.py @@ -395,6 +395,18 @@ def test_deltas_pickle_round_trip() -> None: assert len(deltas.deltas) == len(unpickled.deltas) +def test_deltas_to_pyo3() -> None: + # Arrange + deltas = TestDataStubs.order_book_deltas() + + # Act + pyo3_deltas = deltas.to_pyo3() + + # Assert + assert isinstance(pyo3_deltas, nautilus_pyo3.OrderBookDeltas) + assert len(pyo3_deltas.deltas) == len(deltas.deltas) + + def test_deltas_hash_str_and_repr() -> None: # Arrange order1 = BookOrder( From 5bff0d15e8df77319c31437e8aed0111c1ccb677 Mon Sep 17 00:00:00 2001 From: limx0 Date: Thu, 22 Feb 2024 21:57:21 +1000 Subject: [PATCH 104/104] WIP --- nautilus_trader/adapters/betfair/execution.py | 1 + nautilus_trader/adapters/betfair/parsing/common.py | 6 ++++-- nautilus_trader/adapters/betfair/parsing/requests.py | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/nautilus_trader/adapters/betfair/execution.py b/nautilus_trader/adapters/betfair/execution.py index 22adc95970f2..06f21e1437bc 100644 --- a/nautilus_trader/adapters/betfair/execution.py +++ b/nautilus_trader/adapters/betfair/execution.py @@ -195,6 +195,7 @@ async def connection_account_state(self) -> None: account_detail=account_details, account_funds=account_funds, event_id=UUID4(), + reported=True, ts_event=timestamp, ts_init=timestamp, ) diff --git a/nautilus_trader/adapters/betfair/parsing/common.py b/nautilus_trader/adapters/betfair/parsing/common.py index 97144a674e50..a0d01c2e17cf 100644 --- a/nautilus_trader/adapters/betfair/parsing/common.py +++ b/nautilus_trader/adapters/betfair/parsing/common.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- - +import hashlib from functools import lru_cache +import msgspec from betfair_parser.spec.common import Handicap from betfair_parser.spec.common import MarketId from betfair_parser.spec.common import SelectionId @@ -27,7 +28,8 @@ def hash_market_trade(timestamp: int, price: float, volume: float) -> str: - return f"{str(timestamp)[:-6]}{price}{volume!s}" + data = (timestamp, price, volume) + return hashlib.shake_256(msgspec.json.encode(data)).hexdigest(18) @lru_cache diff --git a/nautilus_trader/adapters/betfair/parsing/requests.py b/nautilus_trader/adapters/betfair/parsing/requests.py index c8d2f5bf09d7..70207df21fb8 100644 --- a/nautilus_trader/adapters/betfair/parsing/requests.py +++ b/nautilus_trader/adapters/betfair/parsing/requests.py @@ -297,6 +297,7 @@ def betfair_account_to_account_state( event_id, ts_event, ts_init, + reported, account_id="001", ) -> AccountState: currency = Currency.from_str(account_detail.currency_code) @@ -307,7 +308,7 @@ def betfair_account_to_account_state( account_id=AccountId(f"{BETFAIR_VENUE.value}-{account_id}"), account_type=AccountType.BETTING, base_currency=currency, - reported=False, + reported=reported, balances=[ AccountBalance( total=Money(balance, currency),