From d72f50c5daf992d4d0c930981b70092a7e819c4c Mon Sep 17 00:00:00 2001 From: Chris Sellers Date: Sat, 6 Jan 2024 11:04:35 +1100 Subject: [PATCH] Standardize on pytest_benchmark --- nautilus_trader/test_kit/performance.py | 92 ------------------- tests/performance_tests/test_perf_backtest.py | 3 +- .../test_perf_correctness.py | 85 ++++++++--------- tests/performance_tests/test_perf_events.py | 19 ++-- .../test_perf_experiments.py | 10 +- .../performance_tests/test_perf_fill_model.py | 17 ++-- .../test_perf_identifiers.py | 13 ++- .../test_perf_live_execution.py | 20 ++-- tests/performance_tests/test_perf_objects.py | 49 +++++----- tests/performance_tests/test_perf_order.py | 28 ++---- tests/performance_tests/test_perf_queues.py | 53 ----------- .../test_perf_serialization.py | 19 +--- tests/performance_tests/test_perf_stats.py | 24 ++--- .../performance_tests/test_perf_throttler.py | 32 +++---- tests/performance_tests/test_perf_uuid.py | 26 +++--- .../performance_tests/test_perf_wranglers.py | 16 ++-- .../test_perf_xrate_calculator.py | 7 +- 17 files changed, 154 insertions(+), 359 deletions(-) delete mode 100644 nautilus_trader/test_kit/performance.py delete mode 100644 tests/performance_tests/test_perf_queues.py diff --git a/nautilus_trader/test_kit/performance.py b/nautilus_trader/test_kit/performance.py deleted file mode 100644 index 3a4ff3d42fd4..000000000000 --- a/nautilus_trader/test_kit/performance.py +++ /dev/null @@ -1,92 +0,0 @@ -# ------------------------------------------------------------------------------------------------- -# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. -# https://nautechsystems.io -# -# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ------------------------------------------------------------------------------------------------- - -import inspect -import timeit -from collections.abc import Callable - -import pytest - - -class PerformanceHarness: - @pytest.fixture(autouse=True) - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def setup(self, benchmark): - self.benchmark = benchmark - - -class PerformanceBench: - @staticmethod - def profile_function( - target: Callable, - runs: int, - iterations: int, - print_output: bool = True, - ) -> float: - """ - Profile the given function. - - Return the minimum elapsed time in seconds taken to call the given - function iteration times. - - Also prints the elapsed time in milliseconds (ms), microseconds (μs) and - nanoseconds (ns). As a rule of thumb a CPU cycles in 1 nanosecond per - GHz of clock speed. - - Parameters - ---------- - target : Callable - The target to call and profile. - runs : int - The number of runs for the test. - iterations : int - The number of call iterations per run. - print_output : bool - If the output should be printed to the console. - - Raises - ------ - ValueError - If `runs` is not positive (> 1). - ValueError - If `iterations` is not positive (> 1). - - Returns - ------- - float - - """ - if runs < 1: - raise ValueError("runs cannot be less than 1") - if iterations < 1: - raise ValueError("iterations cannot be less than 1") - - results = timeit.Timer(target).repeat(repeat=runs, number=iterations) - minimum = min(results) # In seconds - - if print_output: - result_milli = minimum * 1_000 # 1,000ms in 1 second - result_micro = minimum * 1_000_000 # 1,000,000μs in 1 second - result_nano = minimum * 1_000_000_000 # 1,000,000,000ns in 1 second - print(f"\nPerformance test: {inspect.getmembers(target)[4][1]!s} ") - print( - f"# ~{result_milli:.1f}ms " - f"/ ~{result_micro:.1f}μs " - f"/ {result_nano:.0f}ns " - f"minimum of {runs:,} runs @ {iterations:,} " - f"iteration{'s' if iterations > 1 else ''} each run.", - ) - - return minimum diff --git a/tests/performance_tests/test_perf_backtest.py b/tests/performance_tests/test_perf_backtest.py index 58ad0167ff2b..7d473a58a279 100644 --- a/tests/performance_tests/test_perf_backtest.py +++ b/tests/performance_tests/test_perf_backtest.py @@ -33,7 +33,6 @@ from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.objects import Money from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler -from nautilus_trader.test_kit.performance import PerformanceHarness from nautilus_trader.test_kit.providers import TestDataProvider from nautilus_trader.test_kit.providers import TestInstrumentProvider from nautilus_trader.test_kit.stubs.data import TestDataStubs @@ -44,7 +43,7 @@ USDJPY_SIM = TestInstrumentProvider.default_fx_ccy("USD/JPY") -class TestBacktestEnginePerformance(PerformanceHarness): +class TestBacktestEnginePerformance: @staticmethod def test_run_with_empty_strategy(benchmark): def setup(): diff --git a/tests/performance_tests/test_perf_correctness.py b/tests/performance_tests/test_perf_correctness.py index 2a6e63b9ed52..7a2eef60da8b 100644 --- a/tests/performance_tests/test_perf_correctness.py +++ b/tests/performance_tests/test_perf_correctness.py @@ -14,45 +14,46 @@ # ------------------------------------------------------------------------------------------------- from nautilus_trader.core.correctness import PyCondition -from nautilus_trader.test_kit.performance import PerformanceHarness - - -class TestCorrectnessConditionPerformance(PerformanceHarness): - def test_condition_none(self): - self.benchmark.pedantic( - target=PyCondition.none, - args=(None, "param"), - iterations=100_000, - rounds=1, - ) - # ~0.0ms / ~0.1μs / 142ns minimum of 100,000 runs @ 1 iteration each run. - - def test_condition_true(self): - self.benchmark.pedantic( - target=PyCondition.true, - args=(True, "this should be true"), - iterations=100_000, - rounds=1, - ) - # ~0.0ms / ~0.1μs / 149ns minimum of 100,000 runs @ 1 iteration each run. - - # 100000 iterations @ 12ms with boolean except returning False - # 100000 iterations @ 12ms with void except returning * ! - - def test_condition_valid_string(self): - self.benchmark.pedantic( - target=PyCondition.valid_string, - args=("abc123", "string_param"), - iterations=100_000, - rounds=1, - ) - # ~0.0ms / ~0.2μs / 205ns minimum of 100,000 runs @ 1 iteration each run. - - def test_condition_type_or_none(self): - self.benchmark.pedantic( - target=PyCondition.type_or_none, - args=("hello", str, "world"), - iterations=100_000, - rounds=1, - ) - # ~0.0ms / ~0.2μs / 224ns minimum of 100,000 runs @ 1 iteration each run. + + +def test_condition_none(benchmark): + benchmark.pedantic( + target=PyCondition.none, + args=(None, "param"), + iterations=100_000, + rounds=1, + ) + # ~0.0ms / ~0.1μs / 142ns minimum of 100,000 runs @ 1 iteration each run. + + +def test_condition_true(benchmark): + benchmark.pedantic( + target=PyCondition.true, + args=(True, "this should be true"), + iterations=100_000, + rounds=1, + ) + # ~0.0ms / ~0.1μs / 149ns minimum of 100,000 runs @ 1 iteration each run. + + # 100000 iterations @ 12ms with boolean except returning False + # 100000 iterations @ 12ms with void except returning * ! + + +def test_condition_valid_string(benchmark): + benchmark.pedantic( + target=PyCondition.valid_string, + args=("abc123", "string_param"), + iterations=100_000, + rounds=1, + ) + # ~0.0ms / ~0.2μs / 205ns minimum of 100,000 runs @ 1 iteration each run. + + +def test_condition_type_or_none(benchmark): + benchmark.pedantic( + target=PyCondition.type_or_none, + args=("hello", str, "world"), + iterations=100_000, + rounds=1, + ) + # ~0.0ms / ~0.2μs / 224ns minimum of 100,000 runs @ 1 iteration each run. diff --git a/tests/performance_tests/test_perf_events.py b/tests/performance_tests/test_perf_events.py index 7cfcc901ce82..3b548767d538 100644 --- a/tests/performance_tests/test_perf_events.py +++ b/tests/performance_tests/test_perf_events.py @@ -23,10 +23,9 @@ from nautilus_trader.model.identifiers import Symbol from nautilus_trader.model.identifiers import TraderId from nautilus_trader.model.identifiers import Venue -from nautilus_trader.test_kit.performance import PerformanceBench -STUB_ORDER_DENIED = OrderDenied( +_STUB_ORDER_DENIED = OrderDenied( trader_id=TraderId("TRADER-001"), strategy_id=StrategyId("SCALPER-001"), instrument_id=InstrumentId(Symbol("BTCUSDT"), Venue("BINANCE")), @@ -51,26 +50,26 @@ def stub_order_denied() -> OrderDenied: ) -def test_order_denied_to_dict(): +def test_order_denied_to_dict(benchmark): def call_to_dict() -> None: - OrderDenied.to_dict(STUB_ORDER_DENIED) + OrderDenied.to_dict(_STUB_ORDER_DENIED) - PerformanceBench.profile_function( + benchmark.pedantic( target=call_to_dict, - runs=100_000, + rounds=100_000, iterations=1, ) # ~0.0ms / ~1.8μs / 1841ns minimum of 100,000 runs @ 1 iteration each run. -def test_order_denied_to_dict_then_msgspec_to_json(): +def test_order_denied_to_dict_then_msgspec_to_json(benchmark): def call_to_dict_then_json() -> None: - denied_dict = OrderDenied.to_dict(STUB_ORDER_DENIED) + denied_dict = OrderDenied.to_dict(_STUB_ORDER_DENIED) msgspec.json.encode(denied_dict) - PerformanceBench.profile_function( + benchmark.pedantic( target=call_to_dict_then_json, - runs=100_000, + rounds=100_000, iterations=1, ) # ~0.0ms / ~2.4μs / 2441ns minimum of 100,000 runs @ 1 iteration each run. diff --git a/tests/performance_tests/test_perf_experiments.py b/tests/performance_tests/test_perf_experiments.py index c6a5cd9526ad..c7e6a3d5f454 100644 --- a/tests/performance_tests/test_perf_experiments.py +++ b/tests/performance_tests/test_perf_experiments.py @@ -16,10 +16,6 @@ from nautilus_trader.core.message import Event from nautilus_trader.core.uuid import UUID4 from nautilus_trader.model.events import OrderSubmitted -from nautilus_trader.test_kit.performance import PerformanceHarness - - -EVENT = Event(UUID4(), 0, 0) class Experiments: @@ -34,7 +30,7 @@ def built_in_arithmetic(): return x -class TestPerformanceExperiments(PerformanceHarness): +class TestPerformanceExperiments: @staticmethod def test_builtin_arithmetic(benchmark): benchmark.pedantic( @@ -55,9 +51,11 @@ def test_class_name(benchmark): @staticmethod def test_is_instance(benchmark): + event = Event(UUID4(), 0, 0) + benchmark.pedantic( target=isinstance, - args=(EVENT, OrderSubmitted), + args=(event, OrderSubmitted), iterations=100_000, rounds=1, ) diff --git a/tests/performance_tests/test_perf_fill_model.py b/tests/performance_tests/test_perf_fill_model.py index 87dcdc27fd55..9d9eef4209eb 100644 --- a/tests/performance_tests/test_perf_fill_model.py +++ b/tests/performance_tests/test_perf_fill_model.py @@ -14,28 +14,27 @@ # ------------------------------------------------------------------------------------------------- from nautilus_trader.backtest.models import FillModel -from nautilus_trader.test_kit.performance import PerformanceHarness -model = FillModel( +_FILL_MODEL = FillModel( prob_fill_on_stop=0.95, prob_fill_on_limit=0.5, random_seed=42, ) -class TestFillModelPerformance(PerformanceHarness): - def test_is_limit_filled(self): - self.benchmark.pedantic( - target=model.is_limit_filled, +class TestFillModelPerformance: + def test_is_limit_filled(self, benchmark): + benchmark.pedantic( + target=_FILL_MODEL.is_limit_filled, iterations=100_000, rounds=1, ) # ~0.0ms / ~0.1μs / 106ns minimum of 100,000 runs @ 1 iteration each run. - def test_is_stop_filled(self): - self.benchmark.pedantic( - target=model.is_stop_filled, + def test_is_stop_filled(self, benchmark): + benchmark.pedantic( + target=_FILL_MODEL.is_stop_filled, iterations=100_000, rounds=1, ) diff --git a/tests/performance_tests/test_perf_identifiers.py b/tests/performance_tests/test_perf_identifiers.py index 0af472734a27..08f5efef0668 100644 --- a/tests/performance_tests/test_perf_identifiers.py +++ b/tests/performance_tests/test_perf_identifiers.py @@ -15,30 +15,29 @@ from nautilus_trader.model.identifiers import Symbol from nautilus_trader.model.identifiers import Venue -from nautilus_trader.test_kit.performance import PerformanceBench -def test_symbol_equality(): +def test_symbol_equality(benchmark): symbol = Symbol("AUD/USD") def symbol_equality() -> bool: return symbol == symbol - PerformanceBench.profile_function( + benchmark.pedantic( target=symbol_equality, - runs=1_000_000, + rounds=1_000_000, iterations=1, ) -def test_venue_equality(): +def test_venue_equality(benchmark): venue = Venue("SIM") def venue_equality() -> bool: return venue == venue - PerformanceBench.profile_function( + benchmark.pedantic( target=venue_equality, - runs=1_000_000, + rounds=1_000_000, iterations=1, ) diff --git a/tests/performance_tests/test_perf_live_execution.py b/tests/performance_tests/test_perf_live_execution.py index 6c4cbd785ba3..93ac3ecfd5a6 100644 --- a/tests/performance_tests/test_perf_live_execution.py +++ b/tests/performance_tests/test_perf_live_execution.py @@ -33,7 +33,6 @@ from nautilus_trader.model.objects import Quantity from nautilus_trader.portfolio.portfolio import Portfolio from nautilus_trader.test_kit.mocks.exec_clients import MockExecutionClient -from nautilus_trader.test_kit.performance import PerformanceHarness from nautilus_trader.test_kit.providers import TestInstrumentProvider from nautilus_trader.test_kit.stubs.component import TestComponentStubs from nautilus_trader.test_kit.stubs.events import TestEventStubs @@ -45,7 +44,7 @@ BTCUSDT_BINANCE = TestInstrumentProvider.btcusdt_binance() -class TestLiveExecutionPerformance(PerformanceHarness): +class TestLiveExecutionPerformance: def setup(self): # Fixture Setup self.loop = asyncio.get_event_loop() @@ -120,11 +119,6 @@ def setup(self): logger=self.logger, ) - @pytest.fixture(autouse=True) - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def setup_benchmark(self, benchmark): - self.benchmark = benchmark - def submit_order(self): order = self.strategy.order_factory.market( BTCUSDT_BINANCE.id, @@ -135,7 +129,7 @@ def submit_order(self): self.strategy.submit_order(order) @pytest.mark.asyncio() - def test_execute_command(self): + def test_execute_command(self, benchmark): order = self.strategy.order_factory.market( BTCUSDT_BINANCE.id, OrderSide.BUY, @@ -154,11 +148,11 @@ def test_execute_command(self): def execute_command(): self.exec_engine.execute(command) - self.benchmark.pedantic(execute_command, iterations=100, rounds=100, warmup_rounds=5) + benchmark.pedantic(execute_command, iterations=100, rounds=100, warmup_rounds=5) # ~0.0ms / ~0.2μs / 218ns minimum of 10,000 runs @ 1 iteration each run. @pytest.mark.asyncio() - async def test_submit_order(self): + async def test_submit_order(self, benchmark): self.exec_engine.start() await asyncio.sleep(1) @@ -171,11 +165,11 @@ def submit_order(): self.strategy.submit_order(order) - self.benchmark.pedantic(submit_order, iterations=100, rounds=100, warmup_rounds=5) + benchmark.pedantic(submit_order, iterations=100, rounds=100, warmup_rounds=5) # ~0.0ms / ~25.3μs / 25326ns minimum of 10,000 runs @ 1 iteration each run. @pytest.mark.asyncio() - async def test_submit_order_end_to_end(self): + async def test_submit_order_end_to_end(self, benchmark): self.exec_engine.start() await asyncio.sleep(1) @@ -189,4 +183,4 @@ def run(): self.strategy.submit_order(order) - self.benchmark.pedantic(run, rounds=10, warmup_rounds=5) + benchmark.pedantic(run, rounds=10, warmup_rounds=5) diff --git a/tests/performance_tests/test_perf_objects.py b/tests/performance_tests/test_perf_objects.py index 6c13d5780300..0aeb24990a82 100644 --- a/tests/performance_tests/test_perf_objects.py +++ b/tests/performance_tests/test_perf_objects.py @@ -13,8 +13,6 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -import pytest - from nautilus_trader.model.data import Bar from nautilus_trader.model.data import QuoteTick from nautilus_trader.model.data import TradeTick @@ -25,17 +23,14 @@ from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity -from nautilus_trader.test_kit.performance import PerformanceBench -from nautilus_trader.test_kit.performance import PerformanceHarness from nautilus_trader.test_kit.providers import TestInstrumentProvider from nautilus_trader.test_kit.stubs.data import TestDataStubs from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs -class TestObjectPerformance(PerformanceHarness): - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def test_create_symbol(self): - self.benchmark.pedantic( +class TestObjectPerformance: + def test_create_symbol(self, benchmark): + benchmark.pedantic( target=Symbol, args=("AUD/USD",), iterations=100_000, @@ -43,9 +38,8 @@ def test_create_symbol(self): ) # ~0.0ms / ~0.4μs / 400ns minimum of 100,000 runs @ 1 iteration each run. - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def test_create_instrument_id(self): - self.benchmark.pedantic( + def test_create_instrument_id(self, benchmark): + benchmark.pedantic( target=InstrumentId, args=(Symbol("AUD/USD"), Venue("IDEALPRO")), iterations=100_000, @@ -53,9 +47,8 @@ def test_create_instrument_id(self): ) # ~0.0ms / ~1.3μs / 1251ns minimum of 100,000 runs @ 1 iteration each run. - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def test_instrument_id_to_str(self): - self.benchmark.pedantic( + def test_instrument_id_to_str(self, benchmark): + benchmark.pedantic( target=str, args=(TestIdStubs.audusd_id(),), iterations=100_000, @@ -63,8 +56,8 @@ def test_instrument_id_to_str(self): ) # ~0.0ms / ~0.2μs / 198ns minimum of 100,000 runs @ 1 iteration each run. - def test_create_bar(self): - self.benchmark.pedantic( + def test_create_bar(self, benchmark): + benchmark.pedantic( target=Bar, args=( TestDataStubs.bartype_audusd_1min_bid(), @@ -81,7 +74,7 @@ def test_create_bar(self): ) # ~0.0ms / ~2.7μs / 2717ns minimum of 100,000 runs @ 1 iteration each run. - def test_create_quote_tick(self): + def test_create_quote_tick(self, benchmark): audusd_sim = TestInstrumentProvider.default_fx_ccy("AUD/USD") def create_quote_tick(): @@ -95,14 +88,14 @@ def create_quote_tick(): ts_init=0, ) - PerformanceBench.profile_function( + benchmark.pedantic( target=create_quote_tick, - runs=100000, + rounds=100000, iterations=1, ) # ~0.0ms / ~2.8μs / 2798ns minimum of 100,000 runs @ 1 iteration each run. - def test_create_quote_tick_raw(self): + def test_create_quote_tick_raw(self, benchmark): audusd_sim = TestInstrumentProvider.default_fx_ccy("AUD/USD") def create_quote_tick(): @@ -120,14 +113,14 @@ def create_quote_tick(): 0, ) - PerformanceBench.profile_function( + benchmark.pedantic( target=create_quote_tick, - runs=100000, + rounds=100000, iterations=1, ) # ~0.0ms / ~0.2μs / 218ns minimum of 100,000 runs @ 1 iteration each run. - def test_create_trade_tick(self): + def test_create_trade_tick(self, benchmark): audusd_sim = TestInstrumentProvider.default_fx_ccy("AUD/USD") def create_trade_tick(): @@ -141,14 +134,14 @@ def create_trade_tick(): ts_init=0, ) - PerformanceBench.profile_function( + benchmark.pedantic( target=create_trade_tick, - runs=100000, + rounds=100000, iterations=1, ) # ~0.0ms / ~2.5μs / 2492ns minimum of 100,000 runs @ 1 iteration each run. - def test_create_trade_tick_from_raw(self): + def test_create_trade_tick_from_raw(self, benchmark): audusd_sim = TestInstrumentProvider.default_fx_ccy("AUD/USD") def create_trade_tick(): @@ -164,9 +157,9 @@ def create_trade_tick(): 0, ) - PerformanceBench.profile_function( + benchmark.pedantic( target=create_trade_tick, - runs=100000, + rounds=100000, iterations=1, ) # ~0.0ms / ~0.7μs / 718ns minimum of 100,000 runs @ 1 iteration each run. diff --git a/tests/performance_tests/test_perf_order.py b/tests/performance_tests/test_perf_order.py index fc3529574696..de284efbaecc 100644 --- a/tests/performance_tests/test_perf_order.py +++ b/tests/performance_tests/test_perf_order.py @@ -13,8 +13,6 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -import pytest - from nautilus_trader.common.clock import LiveClock from nautilus_trader.common.clock import TestClock from nautilus_trader.common.factories import OrderFactory @@ -24,14 +22,10 @@ from nautilus_trader.model.identifiers import TraderId from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity -from nautilus_trader.test_kit.performance import PerformanceHarness from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs -AUDUSD_SIM = TestIdStubs.audusd_id() - - -class TestOrderPerformance(PerformanceHarness): +class TestOrderPerformance: def setup(self): # Fixture Setup self.generator = ClientOrderIdGenerator( @@ -46,23 +40,19 @@ def setup(self): clock=TestClock(), ) - @pytest.fixture(autouse=True) - def setup_benchmark(self, benchmark): - self.benchmark = benchmark - - def test_order_id_generator(self): - self.benchmark.pedantic( + def test_order_id_generator(self, benchmark): + benchmark.pedantic( target=self.generator.generate, iterations=100_000, rounds=1, ) # ~0.0ms / ~2.9μs / 2894ns minimum of 100,000 runs @ 1 iteration each run. - def test_market_order_creation(self): - self.benchmark.pedantic( + def test_market_order_creation(self, benchmark): + benchmark.pedantic( target=self.order_factory.market, args=( - AUDUSD_SIM, + TestIdStubs.audusd_id(), OrderSide.BUY, Quantity.from_int(100_000), ), @@ -71,11 +61,11 @@ def test_market_order_creation(self): ) # ~0.0ms / ~10.7μs / 10682ns minimum of 10,000 runs @ 1 iteration each run. - def test_limit_order_creation(self): - self.benchmark.pedantic( + def test_limit_order_creation(self, benchmark): + benchmark.pedantic( target=self.order_factory.limit, args=( - AUDUSD_SIM, + TestIdStubs.audusd_id(), OrderSide.BUY, Quantity.from_int(100_000), Price.from_str("0.80010"), diff --git a/tests/performance_tests/test_perf_queues.py b/tests/performance_tests/test_perf_queues.py deleted file mode 100644 index 1560f9de3dd3..000000000000 --- a/tests/performance_tests/test_perf_queues.py +++ /dev/null @@ -1,53 +0,0 @@ -# ------------------------------------------------------------------------------------------------- -# Copyright (C) 2015-2024 Nautech Systems Pty Ltd. All rights reserved. -# https://nautechsystems.io -# -# Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ------------------------------------------------------------------------------------------------- - -from collections import deque - -import pytest - -from nautilus_trader.test_kit.performance import PerformanceHarness - - -class TestPythonDequePerformance(PerformanceHarness): - def setup(self): - # Fixture Setup - self.deque = deque(maxlen=1000) - self.deque.append(1.0) - - @pytest.fixture(autouse=True) - def setup_benchmark(self, benchmark): - self.benchmark = benchmark - - def peek(self): - return self.deque[0] - - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def test_append(self): - self.benchmark.pedantic( - target=deque(maxlen=1000).append, - args=(1.0,), - iterations=100_000, - rounds=1, - ) - # ~0.0ms / ~0.2μs / 173ns minimum of 100,000 runs @ 1 iteration each run. - - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def test_peek(self): - self.benchmark.pedantic( - target=self.peek, - iterations=100_000, - rounds=1, - ) - # ~0.0ms / ~0.1μs / 144ns minimum of 100,000 runs @ 1 iteration each run. diff --git a/tests/performance_tests/test_perf_serialization.py b/tests/performance_tests/test_perf_serialization.py index 38cf8a942f3b..ce56ac6147f7 100644 --- a/tests/performance_tests/test_perf_serialization.py +++ b/tests/performance_tests/test_perf_serialization.py @@ -14,7 +14,6 @@ # ------------------------------------------------------------------------------------------------- import msgspec -import pytest from nautilus_trader.common.clock import TestClock from nautilus_trader.common.factories import OrderFactory @@ -26,14 +25,10 @@ from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.objects import Quantity from nautilus_trader.serialization.serializer import MsgSpecSerializer -from nautilus_trader.test_kit.performance import PerformanceHarness from nautilus_trader.test_kit.stubs.identifiers import TestIdStubs -AUDUSD = TestIdStubs.audusd_id() - - -class TestSerializationPerformance(PerformanceHarness): +class TestSerializationPerformance: def setup(self): # Fixture Setup self.venue = Venue("SIM") @@ -47,7 +42,7 @@ def setup(self): ) self.order = self.order_factory.market( - AUDUSD, + TestIdStubs.audusd_id(), OrderSide.BUY, Quantity.from_int(100_000), ) @@ -63,14 +58,8 @@ def setup(self): self.serializer = MsgSpecSerializer(encoding=msgspec.msgpack) - @pytest.fixture(autouse=True) - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def setup_benchmark(self, benchmark): - self.benchmark = benchmark - - @pytest.mark.benchmark(disable_gc=True, warmup=True) - def test_serialize_submit_order(self): - self.benchmark.pedantic( + def test_serialize_submit_order(self, benchmark): + benchmark.pedantic( target=self.serializer.serialize, args=(self.command,), iterations=10_000, diff --git a/tests/performance_tests/test_perf_stats.py b/tests/performance_tests/test_perf_stats.py index 394ba6a3a550..5cf968d6d32e 100644 --- a/tests/performance_tests/test_perf_stats.py +++ b/tests/performance_tests/test_perf_stats.py @@ -14,17 +14,14 @@ # ------------------------------------------------------------------------------------------------- import numpy as np -import pytest from nautilus_trader.core.stats import fast_mean from nautilus_trader.core.stats import fast_std -from nautilus_trader.test_kit.performance import PerformanceHarness -class TestFunctionPerformance(PerformanceHarness): - @pytest.mark.benchmark(group="core", disable_gc=True, warmup=True) - def test_np_mean(self): - self.benchmark.pedantic( +class TestFunctionPerformance: + def test_np_mean(self, benchmark): + benchmark.pedantic( target=np.mean, args=(np.random.default_rng(10).random(100),), iterations=10_000, @@ -32,9 +29,8 @@ def test_np_mean(self): ) # ~0ms / ~9μs / 8464ns minimum of 10000 runs @ 1 iterations each run. - @pytest.mark.benchmark(group="core", disable_gc=True, warmup=True) - def test_np_std(self): - self.benchmark.pedantic( + def test_np_std(self, benchmark): + benchmark.pedantic( target=np.std, args=(np.random.default_rng(10).random(100),), iterations=10_000, @@ -42,9 +38,8 @@ def test_np_std(self): ) # ~0ms / ~20μs / 19517ns minimum of 10000 runs @ 1 iterations each run. - @pytest.mark.benchmark(group="core", disable_gc=True, warmup=True) - def test_fast_mean(self): - self.benchmark.pedantic( + def test_fast_mean(self, benchmark): + benchmark.pedantic( target=fast_mean, args=(np.random.default_rng(10).random(100),), iterations=100_000, @@ -52,9 +47,8 @@ def test_fast_mean(self): ) # ~0.0ms / ~0.4μs / 440ns minimum of 100,000 runs @ 1 iteration each run. - @pytest.mark.benchmark(group="core", disable_gc=True, warmup=True) - def test_fast_std(self): - self.benchmark.pedantic( + def test_fast_std(self, benchmark): + benchmark.pedantic( target=fast_std, args=(np.random.default_rng(10).random(100),), iterations=100_000, diff --git a/tests/performance_tests/test_perf_throttler.py b/tests/performance_tests/test_perf_throttler.py index 91caf97e63e3..8490bc0fba01 100644 --- a/tests/performance_tests/test_perf_throttler.py +++ b/tests/performance_tests/test_perf_throttler.py @@ -13,21 +13,19 @@ # limitations under the License. # ------------------------------------------------------------------------------------------------- -from datetime import timedelta - +import pandas as pd import pytest from nautilus_trader.common.component import Throttler -from nautilus_trader.test_kit.performance import PerformanceHarness -@pytest.fixture() -def buffering_throttler(clock, logger): +@pytest.fixture(name="buffering_throttler") +def fixture_buffering_throttler(clock, logger): handler = [] return Throttler( name="Throttler-1", - limit=10000, - interval=timedelta(seconds=1), + limit=10_000, + interval=pd.Timedelta(seconds=1), output_send=handler.append, output_drop=None, clock=clock, @@ -35,19 +33,11 @@ def buffering_throttler(clock, logger): ) -class TestBufferingThrottlerPerformance(PerformanceHarness): - @pytest.mark.skip(reason="intermittent while developing") - def test_send_unlimited(self, buffering_throttler): - def send(): - buffering_throttler.send("MESSAGE") - - self.benchmark.pedantic(send, iterations=100000, rounds=1) - # ~0.0ms / ~0.3μs / 301ns minimum of 10,000 runs @ 1 iteration each run. +def test_send_unlimited(benchmark, buffering_throttler): + benchmark.pedantic(buffering_throttler.send, args=("MESSAGE",), iterations=100_000, rounds=1) + # ~0.0ms / ~0.3μs / 301ns minimum of 10,000 runs @ 1 iteration each run. - @pytest.mark.skip(reason="intermittent while developing") - def test_send_when_limited(self, buffering_throttler): - def send(): - buffering_throttler.send("MESSAGE") - self.benchmark.pedantic(send, iterations=100000, rounds=1) - # ~0.0ms / ~0.2μs / 232ns minimum of 100,000 runs @ 1 iteration each run. +def test_send_when_limited(benchmark, buffering_throttler): + benchmark.pedantic(buffering_throttler.send, args=("MESSAGE",), iterations=100_000, rounds=1) + # ~0.0ms / ~0.2μs / 232ns minimum of 100,000 runs @ 1 iteration each run. diff --git a/tests/performance_tests/test_perf_uuid.py b/tests/performance_tests/test_perf_uuid.py index 33d6230f6000..9030e4579787 100644 --- a/tests/performance_tests/test_perf_uuid.py +++ b/tests/performance_tests/test_perf_uuid.py @@ -18,24 +18,22 @@ import pytest from nautilus_trader.core.uuid import UUID4 -from nautilus_trader.test_kit.performance import PerformanceBench -from nautilus_trader.test_kit.performance import PerformanceHarness -class TestUUIDPerformance(PerformanceHarness): +class TestUUIDPerformance: @pytest.mark.benchmark(group="core", disable_gc=True, warmup=True) @staticmethod def test_make_builtin_uuid(benchmark): benchmark.pedantic( target=uuid.uuid4, - iterations=100000, + iterations=100_000, rounds=1, ) - def test_make_builtin_uuid_bench(self): - PerformanceBench.profile_function( + def test_make_builtin_uuid_bench(self, benchmark): + benchmark.pedantic( target=uuid.uuid4, - runs=100000, + rounds=100_000, iterations=1, ) # ~0.0ms / ~2.1μs / 2067ns minimum of 100,000 runs @ 1 iteration each run. @@ -45,27 +43,27 @@ def test_make_builtin_uuid_bench(self): def test_make_nautilus_uuid(benchmark): benchmark.pedantic( target=UUID4, - iterations=100000, + iterations=100_000, rounds=1, ) - def test_make_nautilus_uuid_bench(self): - PerformanceBench.profile_function( + def test_make_nautilus_uuid_bench(self, benchmark): + benchmark.pedantic( target=UUID4, - runs=100000, + rounds=100_000, iterations=1, ) # ~0.0ms / ~0.8μs / 780ns minimum of 100,000 runs @ 1 iteration each run. - def test_nautilus_uuid_value_bench(self): + def test_nautilus_uuid_value_bench(self, benchmark): uuid = UUID4() def get_uuid_value(): uuid.value - PerformanceBench.profile_function( + benchmark.pedantic( target=get_uuid_value, - runs=100000, + rounds=100_000, iterations=1, ) # ~0.0ms / ~0.2μs / 152ns minimum of 100,000 runs @ 1 iteration each run. (readonly value) diff --git a/tests/performance_tests/test_perf_wranglers.py b/tests/performance_tests/test_perf_wranglers.py index 777d4d7e2675..30d55a897ec8 100644 --- a/tests/performance_tests/test_perf_wranglers.py +++ b/tests/performance_tests/test_perf_wranglers.py @@ -15,14 +15,12 @@ from nautilus_trader.persistence.wranglers import QuoteTickDataWrangler from nautilus_trader.persistence.wranglers import TradeTickDataWrangler -from nautilus_trader.test_kit.performance import PerformanceBench -from nautilus_trader.test_kit.performance import PerformanceHarness from nautilus_trader.test_kit.providers import TestDataProvider from nautilus_trader.test_kit.providers import TestInstrumentProvider -class TestDataWranglersPerformance(PerformanceHarness): - def test_quote_tick_data_wrangler_process_tick_data(self): +class TestDataWranglersPerformance: + def test_quote_tick_data_wrangler_process_tick_data(self, benchmark): usdjpy = TestInstrumentProvider.default_fx_ccy("USD/JPY") wrangler = QuoteTickDataWrangler(instrument=usdjpy) @@ -35,14 +33,14 @@ def wrangler_process(): default_volume=1_000_000, ) - PerformanceBench.profile_function( + benchmark.pedantic( target=wrangler_process, - runs=100, + rounds=100, iterations=1, ) # ~7.8ms / ~7766.6μs / 7766626ns minimum of 100 runs @ 1 iteration each run. - def test_trade_tick_data_wrangler_process(self): + def test_trade_tick_data_wrangler_process(self, benchmark): ethusdt = TestInstrumentProvider.ethusdt_binance() wrangler = TradeTickDataWrangler(instrument=ethusdt) provider = TestDataProvider() @@ -51,9 +49,9 @@ def wrangler_process(): # 69806 ticks in data wrangler.process(data=provider.read_csv_ticks("binance/ethusdt-trades.csv")) - PerformanceBench.profile_function( + benchmark.pedantic( target=wrangler_process, - runs=10, + rounds=10, iterations=1, ) # ~500.2ms / ~500210.6μs / 500210608ns minimum of 10 runs @ 1 iteration each run. diff --git a/tests/performance_tests/test_perf_xrate_calculator.py b/tests/performance_tests/test_perf_xrate_calculator.py index 8060d2f66555..88094bbb1ac1 100644 --- a/tests/performance_tests/test_perf_xrate_calculator.py +++ b/tests/performance_tests/test_perf_xrate_calculator.py @@ -17,7 +17,6 @@ from nautilus_trader.model.currencies import ETH from nautilus_trader.model.currencies import USDT from nautilus_trader.model.enums import PriceType -from nautilus_trader.test_kit.performance import PerformanceBench class TestExchangeRateCalculatorPerformanceTests: @@ -42,10 +41,10 @@ def get_xrate(): ask_quotes=ask_quotes, ) - def test_get_xrate(self): - PerformanceBench.profile_function( + def test_get_xrate(self, benchmark): + benchmark.pedantic( target=self.get_xrate, - runs=100_000, + rounds=100_000, iterations=1, ) # ~0.0ms / ~8.2μs / 8198ns minimum of 100,000 runs @ 1 iteration each run.