From 37828d90797b1fa644669d89f917028887db5ee6 Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Fri, 5 Jan 2024 14:23:33 +1100 Subject: [PATCH] Refine logging config --- nautilus_core/common/src/ffi/logging.rs | 55 +++------- nautilus_core/common/src/logging.rs | 81 ++++++++++++++- nautilus_core/common/src/python/logging.rs | 113 +++++++++++++++++++++ nautilus_core/common/src/python/mod.rs | 4 + nautilus_trader/common/logging.pxd | 10 +- nautilus_trader/common/logging.pyx | 58 ++++++++++- nautilus_trader/core/includes/common.h | 4 +- nautilus_trader/core/nautilus_pyo3.pyi | 15 +++ nautilus_trader/core/rust/common.pxd | 4 +- nautilus_trader/system/kernel.py | 5 + 10 files changed, 300 insertions(+), 49 deletions(-) create mode 100644 nautilus_core/common/src/python/logging.rs diff --git a/nautilus_core/common/src/ffi/logging.rs b/nautilus_core/common/src/ffi/logging.rs index 6d8d1575d4eb..44ca33c252ff 100644 --- a/nautilus_core/common/src/ffi/logging.rs +++ b/nautilus_core/common/src/ffi/logging.rs @@ -13,26 +13,17 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- -use std::{ - env, - ffi::{c_char, CStr}, -}; +use std::ffi::{c_char, CStr}; -use log::{ - debug, error, info, - kv::{ToValue, Value}, - warn, -}; use nautilus_core::{ ffi::string::{cstr_to_string, cstr_to_ustr, optional_cstr_to_string}, uuid::UUID4, }; use nautilus_model::identifiers::trader_id::TraderId; -use tracing_subscriber::EnvFilter; use crate::{ enums::{LogColor, LogLevel}, - logging::{FileWriterConfig, Logger, LoggerConfig}, + logging::{self}, }; /// Initialize tracing. @@ -47,14 +38,7 @@ use crate::{ /// beginning of the run. #[no_mangle] pub extern "C" fn tracing_init() { - // Skip tracing initialization if `RUST_LOG` is not set - if let Ok(v) = env::var("RUST_LOG") { - tracing_subscriber::fmt() - .with_env_filter(EnvFilter::new(v.clone())) - .try_init() - .unwrap_or_else(|e| eprintln!("Cannot set tracing subscriber because of error: {e}")); - println!("Initialized tracing logs with RUST_LOG={v}"); - } + logging::init_tracing(); } /// Initialize logging. @@ -70,7 +54,7 @@ pub extern "C" fn tracing_init() { /// beginning of the run. /// /// - Assume `config_spec_ptr` is a valid C string pointer. -/// - Assume `file_directory_ptr` is either NULL or a valid C string pointer. +/// - Assume `directory_ptr` is either NULL or a valid C string pointer. /// - Assume `file_name_ptr` is either NULL or a valid C string pointer. /// - Assume `file_format_ptr` is either NULL or a valid C string pointer. #[no_mangle] @@ -78,19 +62,24 @@ pub unsafe extern "C" fn logging_init( trader_id: TraderId, instance_id: UUID4, config_spec_ptr: *const c_char, - file_directory_ptr: *const c_char, + directory_ptr: *const c_char, file_name_ptr: *const c_char, file_format_ptr: *const c_char, ) { let config_spec = cstr_to_string(config_spec_ptr); - let config = LoggerConfig::from_spec(&config_spec); - let directory = optional_cstr_to_string(file_directory_ptr); + let directory = optional_cstr_to_string(directory_ptr); let file_name = optional_cstr_to_string(file_name_ptr); let file_format = optional_cstr_to_string(file_format_ptr); - let file_writer_config = FileWriterConfig::new(directory, file_name, file_format); - Logger::init_with_config(trader_id, instance_id, file_writer_config, config); + logging::init_logging( + trader_id, + instance_id, + config_spec, + directory, + file_name, + file_format, + ); } /// Create a new log event. @@ -108,23 +97,9 @@ pub unsafe extern "C" fn logger_log( message_ptr: *const c_char, ) { let component = cstr_to_ustr(component_ptr); - let color = Value::from(color as u8); let message = CStr::from_ptr(message_ptr).to_string_lossy(); - match level { - LogLevel::Debug => { - debug!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); - } - LogLevel::Info => { - info!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); - } - LogLevel::Warning => { - warn!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); - } - LogLevel::Error => { - error!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); - } - } + logging::log(timestamp_ns, level, color, component, message); } /// Flush logger buffers. diff --git a/nautilus_core/common/src/logging.rs b/nautilus_core/common/src/logging.rs index d614fd09346f..835664cf47fd 100644 --- a/nautilus_core/common/src/logging.rs +++ b/nautilus_core/common/src/logging.rs @@ -14,6 +14,7 @@ // ------------------------------------------------------------------------------------------------- use std::{ + borrow::Cow, collections::HashMap, env, fmt, fs::{create_dir_all, File}, @@ -25,13 +26,18 @@ use std::{ }; use chrono::{prelude::*, Utc}; -use log::{kv::Key, set_boxed_logger, set_max_level, Level, LevelFilter, Log, STATIC_MAX_LEVEL}; +use log::{ + debug, error, info, + kv::{Key, ToValue, Value}, + set_boxed_logger, set_max_level, warn, Level, LevelFilter, Log, STATIC_MAX_LEVEL, +}; use nautilus_core::{datetime::unix_nanos_to_iso8601, time::UnixNanos, uuid::UUID4}; use nautilus_model::identifiers::trader_id::TraderId; use serde::{Deserialize, Serialize}; +use tracing_subscriber::EnvFilter; use ustr::Ustr; -use crate::enums::LogColor; +use crate::enums::{LogColor, LogLevel}; #[cfg_attr( feature = "python", @@ -141,6 +147,52 @@ impl FileWriterConfig { } } +/// Initialize tracing. +/// +/// Tracing is meant to be used to trace/debug async Rust code. It can be +/// configured to filter modules and write up to a specific level only using +/// by passing a configuration using the `RUST_LOG` environment variable. +/// +/// # Safety +/// +/// Should only be called once during an applications run, ideally at the +/// beginning of the run. +pub fn init_tracing() { + // Skip tracing initialization if `RUST_LOG` is not set + if let Ok(v) = env::var("RUST_LOG") { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::new(v.clone())) + .try_init() + .unwrap_or_else(|e| eprintln!("Cannot set tracing subscriber because of error: {e}")); + println!("Initialized tracing logs with RUST_LOG={v}"); + } +} + +/// Initialize logging. +/// +/// Logging should be used for Python and sync Rust logic which is most of +/// the components in the main `nautilus_trader` package. +/// Logging can be configured to filter components and write up to a specific level only +/// by passing a configuration using the `NAUTILUS_LOG` environment variable. +/// +/// # Safety +/// +/// Should only be called once during an applications run, ideally at the +/// beginning of the run. +pub fn init_logging( + trader_id: TraderId, + instance_id: UUID4, + config_spec: String, + directory: Option, + file_name: Option, + file_format: Option, +) { + let config = LoggerConfig::from_spec(&config_spec); + let file_writer_config = FileWriterConfig::new(directory, file_name, file_format); + + Logger::init_with_config(trader_id, instance_id, file_writer_config, config); +} + /// Provides a high-performance logger utilizing a MPSC channel under the hood. /// /// A separate thead is spawned at initialization which receives [`LogEvent`] structs over the @@ -557,6 +609,31 @@ impl Logger { } } +pub fn log( + timestamp_ns: UnixNanos, + level: LogLevel, + color: LogColor, + component: Ustr, + message: Cow<'_, str>, +) { + let color = Value::from(color as u8); + + match level { + LogLevel::Debug => { + debug!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); + } + LogLevel::Info => { + info!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); + } + LogLevel::Warning => { + warn!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); + } + LogLevel::Error => { + error!(timestamp = timestamp_ns, component = component.to_value(), color = color; "{}", message); + } + } +} + //////////////////////////////////////////////////////////////////////////////// // Tests //////////////////////////////////////////////////////////////////////////////// diff --git a/nautilus_core/common/src/python/logging.rs b/nautilus_core/common/src/python/logging.rs new file mode 100644 index 000000000000..43dd08ee10f8 --- /dev/null +++ b/nautilus_core/common/src/python/logging.rs @@ -0,0 +1,113 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (C) 2015-2023 Nautech Systems Pty Ltd. All rights reserved. +// https://nautechsystems.io +// +// Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ------------------------------------------------------------------------------------------------- + +use std::borrow::Cow; + +use nautilus_core::{time::UnixNanos, uuid::UUID4}; +use nautilus_model::identifiers::trader_id::TraderId; +use pyo3::prelude::*; +use ustr::Ustr; + +use crate::{ + enums::{LogColor, LogLevel}, + logging::{self, FileWriterConfig, LoggerConfig}, +}; + +/// Initialize tracing. +/// +/// Tracing is meant to be used to trace/debug async Rust code. It can be +/// configured to filter modules and write up to a specific level only using +/// by passing a configuration using the `RUST_LOG` environment variable. +/// +/// # Safety +/// +/// Should only be called once during an applications run, ideally at the +/// beginning of the run. +#[pyfunction()] +#[pyo3(name = "init_tracing")] +pub fn py_init_tracing() { + logging::init_tracing(); +} + +/// Initialize logging. +/// +/// Logging should be used for Python and sync Rust logic which is most of +/// the components in the main `nautilus_trader` package. +/// Logging can be configured to filter components and write up to a specific level only +/// by passing a configuration using the `NAUTILUS_LOG` environment variable. +/// +/// # Safety +/// +/// Should only be called once during an applications run, ideally at the +/// beginning of the run. +#[pyfunction] +#[pyo3(name = "init_logging")] +pub fn py_init_logging( + trader_id: TraderId, + instance_id: UUID4, + config_spec: String, + directory: Option, + file_name: Option, + file_format: Option, +) { + logging::init_logging( + trader_id, + instance_id, + config_spec, + directory, + file_name, + file_format, + ); +} + +/// Create a new log event. +#[pyfunction] +#[pyo3(name = "logger_log")] +pub fn py_logger_log( + timestamp_ns: UnixNanos, + level: LogLevel, + color: LogColor, + component: String, + message: String, +) { + logging::log( + timestamp_ns, + level, + color, + Ustr::from(&component), + Cow::from(message), + ); +} + +#[pymethods] +impl FileWriterConfig { + #[new] + pub fn py_new( + directory: Option, + file_name: Option, + file_format: Option, + ) -> Self { + Self::new(directory, file_name, file_format) + } +} + +#[pymethods] +impl LoggerConfig { + #[staticmethod] + #[pyo3(name = "from_spec")] + pub fn py_from_spec(spec: String) -> Self { + LoggerConfig::from_spec(&spec) + } +} diff --git a/nautilus_core/common/src/python/mod.rs b/nautilus_core/common/src/python/mod.rs index 1de98b5b2310..db92ea24ce42 100644 --- a/nautilus_core/common/src/python/mod.rs +++ b/nautilus_core/common/src/python/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. // ------------------------------------------------------------------------------------------------- +pub mod logging; pub mod timer; use pyo3::prelude::*; @@ -32,6 +33,9 @@ pub fn common(_: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_function(wrap_pyfunction!(logging::py_init_tracing, m)?)?; + m.add_function(wrap_pyfunction!(logging::py_init_logging, m)?)?; + m.add_function(wrap_pyfunction!(logging::py_logger_log, m)?)?; Ok(()) } diff --git a/nautilus_trader/common/logging.pxd b/nautilus_trader/common/logging.pxd index 6890e0885413..a04eb46348b7 100644 --- a/nautilus_trader/common/logging.pxd +++ b/nautilus_trader/common/logging.pxd @@ -30,7 +30,7 @@ cpdef void init_logging( TraderId trader_id, UUID4 instance_id, str config_spec, - str file_directory, + str directory, str file_name, str file_format, ) @@ -69,6 +69,14 @@ cdef class Logger: str message, ) + # cdef void log( + # self, + # level, + # color, + # str component, + # str message, + # ) + cpdef void change_clock(self, Clock clock) cpdef void flush(self) diff --git a/nautilus_trader/common/logging.pyx b/nautilus_trader/common/logging.pyx index 0b0590905f57..29aba5435a1b 100644 --- a/nautilus_trader/common/logging.pyx +++ b/nautilus_trader/common/logging.pyx @@ -50,6 +50,12 @@ from nautilus_trader.core.uuid cimport UUID4 from nautilus_trader.model.identifiers cimport TraderId +# TODO!: Reimplementing logging config +# from nautilus_trader.core import nautilus_pyo3 +# from nautilus_trader.core.nautilus_pyo3 import logger_log + + + cpdef void init_tracing(): tracing_init() @@ -58,7 +64,7 @@ cpdef void init_logging( TraderId trader_id, UUID4 instance_id, str config_spec, - str file_directory, + str directory, str file_name, str file_format, ): @@ -68,7 +74,7 @@ cpdef void init_logging( trader_id._mem, instance_id._mem, pystr_to_cstr(config_spec), - pystr_to_cstr(file_directory) if file_directory else NULL, + pystr_to_cstr(directory) if directory else NULL, pystr_to_cstr(file_name) if file_name else NULL, pystr_to_cstr(file_format) if file_format else NULL, ) @@ -243,6 +249,22 @@ cdef class Logger: pystr_to_cstr(message), ) + # TODO!: Reimplementing logging config + # cdef void log( + # self, + # level, + # color, + # str component, + # str message, + # ): + # logger_log( + # self._clock.timestamp_ns(), + # level, + # nautilus_pyo3.LogColor.Normal, # In development + # component, + # message, + # ) + cpdef void change_clock(self, Clock clock): """ Change the loggers internal clock to the given clock. @@ -401,6 +423,14 @@ cdef class LoggerAdapter: message, ) + # TODO!: Reimplementing logging config + # self._logger.log( + # nautilus_pyo3.LogLevel.Debug, + # color, + # self._component, + # message, + # ) + cpdef void info( self, str message, LogColor color = LogColor.NORMAL, @@ -431,6 +461,14 @@ cdef class LoggerAdapter: message, ) + # TODO!: Reimplementing logging config + # self._logger.log( + # nautilus_pyo3.LogLevel.Info, + # color, + # self._component, + # message, + # ) + cpdef void warning( self, str message, @@ -462,6 +500,14 @@ cdef class LoggerAdapter: message, ) + # TODO!: Reimplementing logging config + # self._logger.log( + # nautilus_pyo3.LogLevel.Warning, + # color, + # self._component, + # message, + # ) + cpdef void error( self, str message, @@ -493,6 +539,14 @@ cdef class LoggerAdapter: message, ) + # TODO!: Reimplementing logging config + # self._logger.log( + # nautilus_pyo3.LogLevel.Error, + # color, + # self._component, + # message, + # ) + cpdef void exception( self, str message, diff --git a/nautilus_trader/core/includes/common.h b/nautilus_trader/core/includes/common.h index 71e6d215f680..66fcafa61946 100644 --- a/nautilus_trader/core/includes/common.h +++ b/nautilus_trader/core/includes/common.h @@ -460,14 +460,14 @@ void tracing_init(void); * beginning of the run. * * - Assume `config_spec_ptr` is a valid C string pointer. - * - Assume `file_directory_ptr` is either NULL or a valid C string pointer. + * - Assume `directory_ptr` is either NULL or a valid C string pointer. * - Assume `file_name_ptr` is either NULL or a valid C string pointer. * - Assume `file_format_ptr` is either NULL or a valid C string pointer. */ void logging_init(TraderId_t trader_id, UUID4_t instance_id, const char *config_spec_ptr, - const char *file_directory_ptr, + const char *directory_ptr, const char *file_name_ptr, const char *file_format_ptr); diff --git a/nautilus_trader/core/nautilus_pyo3.pyi b/nautilus_trader/core/nautilus_pyo3.pyi index 0c13de876524..7afc1a6dbc84 100644 --- a/nautilus_trader/core/nautilus_pyo3.pyi +++ b/nautilus_trader/core/nautilus_pyo3.pyi @@ -197,6 +197,21 @@ def convert_to_snake_case(s: str) -> str: # Common ################################################################################################### +### Logging + +def init_tracing() -> None: + ... + +def init_logging( + trader_id: TraderId, + instance_id: UUID4, + config_spec: str, + directory: str, + file_name: str, + file_format: str, +) -> None: + ... + ################################################################################################### # Model ################################################################################################### diff --git a/nautilus_trader/core/rust/common.pxd b/nautilus_trader/core/rust/common.pxd index 6c20fc4f4a67..5ac7cae7fe6a 100644 --- a/nautilus_trader/core/rust/common.pxd +++ b/nautilus_trader/core/rust/common.pxd @@ -320,13 +320,13 @@ cdef extern from "../includes/common.h": # beginning of the run. # # - Assume `config_spec_ptr` is a valid C string pointer. - # - Assume `file_directory_ptr` is either NULL or a valid C string pointer. + # - Assume `directory_ptr` is either NULL or a valid C string pointer. # - Assume `file_name_ptr` is either NULL or a valid C string pointer. # - Assume `file_format_ptr` is either NULL or a valid C string pointer. void logging_init(TraderId_t trader_id, UUID4_t instance_id, const char *config_spec_ptr, - const char *file_directory_ptr, + const char *directory_ptr, const char *file_name_ptr, const char *file_format_ptr); diff --git a/nautilus_trader/system/kernel.py b/nautilus_trader/system/kernel.py index f317d0de1ace..29f024f8291c 100644 --- a/nautilus_trader/system/kernel.py +++ b/nautilus_trader/system/kernel.py @@ -25,6 +25,9 @@ import msgspec +# from nautilus_trader.core import nautilus_pyo3 +# from nautilus_trader.core.nautilus_pyo3 import init_logging +# from nautilus_trader.core.nautilus_pyo3 import init_tracing from nautilus_trader.cache.base import CacheFacade from nautilus_trader.cache.cache import Cache from nautilus_trader.cache.database import CacheDatabaseAdapter @@ -158,6 +161,8 @@ def __init__( # noqa (too complex) logging: LoggingConfig = config.logging or LoggingConfig() init_logging( + # nautilus_pyo3.TraderId(self._trader_id.value), # TODO!: Reimplementing logging config + # nautilus_pyo3.UUID4(self._instance_id.value), # TODO!: Reimplementing logging config self._trader_id, self._instance_id, logging.spec_string(),