From e055aef0e7d28c192e66220776487498a3e00943 Mon Sep 17 00:00:00 2001 From: Calina Cenan Date: Wed, 27 Mar 2024 11:30:06 +0000 Subject: [PATCH] Linter fixes after merge and P/C architecture fix. --- pdr_backend/cli/cli_module.py | 1 - pdr_backend/cli/test/test_cli_module.py | 10 +---- pdr_backend/ppss/sim_ss.py | 10 ----- pdr_backend/ppss/test/test_ppss.py | 1 - pdr_backend/ppss/test/test_sim_ss.py | 29 ++++-------- pdr_backend/sim/multisim_engine.py | 1 - pdr_backend/sim/sim_engine.py | 2 - pdr_backend/sim/sim_plotter.py | 46 ++++++++++++-------- pdr_backend/sim/sim_state.py | 2 +- pdr_backend/sim/test/test_multisim_engine.py | 2 +- pdr_backend/sim/test/test_sim_engine.py | 3 +- ppss.yaml | 1 - sim_plots.py | 36 ++++++++++----- 13 files changed, 67 insertions(+), 77 deletions(-) diff --git a/pdr_backend/cli/cli_module.py b/pdr_backend/cli/cli_module.py index b23bc78b1..88f9852c7 100644 --- a/pdr_backend/cli/cli_module.py +++ b/pdr_backend/cli/cli_module.py @@ -1,6 +1,5 @@ import logging import os -import subprocess import sys from enforce_typing import enforce_types diff --git a/pdr_backend/cli/test/test_cli_module.py b/pdr_backend/cli/test/test_cli_module.py index 52b8c7005..b6a509085 100644 --- a/pdr_backend/cli/test/test_cli_module.py +++ b/pdr_backend/cli/test/test_cli_module.py @@ -212,15 +212,7 @@ def test_do_sim(monkeypatch): mock_f = Mock() monkeypatch.setattr(f"{_CLI_PATH}.SimEngine.run", mock_f) - with patch("pdr_backend.cli.cli_module.subprocess.run") as mock_subprocess: - with patch("sys.argv", ["pdr", "sim", "ppss.yaml"]): - do_sim(MockArgParser_PPSS().parse_args()) - - mock_subprocess.assert_called() - mock_f.assert_not_called() - - with patch("sys.argv", ["streamlit_entrypoint.py", "sim", "ppss.yaml"]): - do_sim(MockArgParser_PPSS().parse_args()) + do_sim(MockArgParser_PPSS().parse_args()) mock_f.assert_called() diff --git a/pdr_backend/ppss/sim_ss.py b/pdr_backend/ppss/sim_ss.py index b25b57bfa..ea77f870d 100644 --- a/pdr_backend/ppss/sim_ss.py +++ b/pdr_backend/ppss/sim_ss.py @@ -20,10 +20,6 @@ class SimSS(StrMixin, CCXTExchangeMixin): def __init__(self, d: dict): self.d = d # yaml_dict["sim_ss"] - # check do_plot - if not isinstance(d["do_plot"], bool): - raise TypeError - # handle log_dir; self.log_dir is supposed to be path-expanded version assert self.log_dir == os.path.abspath(self.log_dir) if not os.path.exists(self.log_dir): @@ -47,10 +43,6 @@ def __init__(self, d: dict): # -------------------------------- # properties direct from yaml dict - @property - def do_plot(self) -> bool: - return self.d["do_plot"] - @property def log_dir(self) -> str: s = self.d["log_dir"] @@ -82,13 +74,11 @@ def is_final_iter(self, iter_i: int) -> bool: @enforce_types def sim_ss_test_dict( - do_plot: bool, log_dir: str, test_n: Optional[int] = None, tradetype: Optional[str] = None, ) -> dict: d = { - "do_plot": do_plot, "log_dir": log_dir, "test_n": test_n or 10, "tradetype": tradetype or "histmock", diff --git a/pdr_backend/ppss/test/test_ppss.py b/pdr_backend/ppss/test/test_ppss.py index 3e7ef1717..2b07a45f1 100644 --- a/pdr_backend/ppss/test/test_ppss.py +++ b/pdr_backend/ppss/test/test_ppss.py @@ -36,7 +36,6 @@ def _test_ppss(yaml_filename=None, yaml_str=None, network=None): assert ppss.predictoor_ss.aimodel_ss.approach == "LinearLogistic" assert ppss.payout_ss.batch_size >= 0 assert 1 <= ppss.predictoor_ss.s_until_epoch_end <= 120 - assert isinstance(ppss.sim_ss.do_plot, bool) assert 0.0 <= ppss.trader_ss.fee_percent <= 0.99 assert "USD" in ppss.trader_ss.buy_amt_str assert ppss.trueval_ss.batch_size >= 0 diff --git a/pdr_backend/ppss/test/test_sim_ss.py b/pdr_backend/ppss/test/test_sim_ss.py index c01728a85..b5d291139 100644 --- a/pdr_backend/ppss/test/test_sim_ss.py +++ b/pdr_backend/ppss/test/test_sim_ss.py @@ -12,11 +12,10 @@ @enforce_types def test_sim_ss_defaults(tmpdir): - d = sim_ss_test_dict(True, _logdir(tmpdir)) + d = sim_ss_test_dict(_logdir(tmpdir)) ss = SimSS(d) # yaml properties - assert isinstance(ss.do_plot, bool) assert isinstance(ss.log_dir, str) assert isinstance(ss.test_n, int) assert 1 <= ss.test_n <= 10000 @@ -29,7 +28,6 @@ def test_sim_ss_defaults(tmpdir): @enforce_types def test_sim_ss_give_values(tmpdir): d = sim_ss_test_dict( - do_plot=False, log_dir=os.path.join(tmpdir, "mylogs"), test_n=13, tradetype="livereal", @@ -37,7 +35,6 @@ def test_sim_ss_give_values(tmpdir): ss = SimSS(d) # yaml properties - assert not ss.do_plot assert ss.log_dir == os.path.join(tmpdir, "mylogs") assert ss.test_n == 13 assert ss.tradetype == "livereal" @@ -46,20 +43,12 @@ def test_sim_ss_give_values(tmpdir): assert "SimSS" in str(ss) -@enforce_types -def test_sim_ss_do_plot_badpaths(tmpdir): - d = sim_ss_test_dict(True, _logdir(tmpdir)) - d["do_plot"] = "not_a_bool" - with pytest.raises(TypeError): - _ = SimSS(d) - - @enforce_types def test_sim_ss_log_dir_relative_path(): # it will work with the relative path expanded_path = os.path.abspath("mylogs") had_before = os.path.exists(expanded_path) - d = sim_ss_test_dict(True, "mylogs") + d = sim_ss_test_dict("mylogs") ss = SimSS(d) assert ss.log_dir == expanded_path if not had_before: @@ -68,16 +57,16 @@ def test_sim_ss_log_dir_relative_path(): @enforce_types def test_sim_ss_test_n_badpaths(tmpdir): - d = sim_ss_test_dict(True, _logdir(tmpdir), test_n=-3) + d = sim_ss_test_dict(_logdir(tmpdir), test_n=-3) with pytest.raises(ValueError): _ = SimSS(d) - d = sim_ss_test_dict(True, _logdir(tmpdir)) + d = sim_ss_test_dict(_logdir(tmpdir)) d["test_n"] = "not_an_int" with pytest.raises(TypeError): _ = SimSS(d) - d = sim_ss_test_dict(True, _logdir(tmpdir)) + d = sim_ss_test_dict(_logdir(tmpdir)) d["test_n"] = 3.2 with pytest.raises(TypeError): _ = SimSS(d) @@ -86,26 +75,26 @@ def test_sim_ss_test_n_badpaths(tmpdir): @enforce_types def test_sim_ss_tradetype_happypaths(tmpdir): for tradetype in TRADETYPE_OPTIONS: - d = sim_ss_test_dict(True, _logdir(tmpdir), tradetype=tradetype) + d = sim_ss_test_dict(_logdir(tmpdir), tradetype=tradetype) ss = SimSS(d) assert ss.tradetype == tradetype @enforce_types def test_sim_ss_tradetype_badpaths(tmpdir): - d = sim_ss_test_dict(True, _logdir(tmpdir)) + d = sim_ss_test_dict(_logdir(tmpdir)) d["tradetype"] = 3.2 with pytest.raises(TypeError): _ = SimSS(d) - d = sim_ss_test_dict(True, _logdir(tmpdir), tradetype="not a tradetype") + d = sim_ss_test_dict(_logdir(tmpdir), tradetype="not a tradetype") with pytest.raises(ValueError): _ = SimSS(d) @enforce_types def test_sim_ss_is_final_iter(tmpdir): - d = sim_ss_test_dict(True, _logdir(tmpdir), test_n=10) + d = sim_ss_test_dict(_logdir(tmpdir), test_n=10) ss = SimSS(d) with pytest.raises(ValueError): _ = ss.is_final_iter(-5) diff --git a/pdr_backend/sim/multisim_engine.py b/pdr_backend/sim/multisim_engine.py index 105c4eb8b..19d59e2a2 100644 --- a/pdr_backend/sim/multisim_engine.py +++ b/pdr_backend/sim/multisim_engine.py @@ -70,7 +70,6 @@ def ppss_from_point(self, point_i: Point) -> PPSS: d = copy.deepcopy(self.d) recursive_update(d, nested_args) ppss = PPSS(d=d, network=self.network) - assert not ppss.sim_ss.do_plot # type: ignore[attr-defined] return ppss @enforce_types diff --git a/pdr_backend/sim/sim_engine.py b/pdr_backend/sim/sim_engine.py index b39f3049b..b9d62f0cf 100644 --- a/pdr_backend/sim/sim_engine.py +++ b/pdr_backend/sim/sim_engine.py @@ -308,8 +308,6 @@ def save_state(self, i: int, N: int): if self.ppss.sim_ss.is_final_iter(i): return True, True - # TODO: remove do_plot in sim_ss -> if the user wants to plot, they start the streamlit app - # don't save first 5 iters -> not interesting # then save the next 5 -> "stuff's happening!" # then save every 5th iter, to balance "stuff's happening" w/ speed diff --git a/pdr_backend/sim/sim_plotter.py b/pdr_backend/sim/sim_plotter.py index 1dd40323c..045fad146 100644 --- a/pdr_backend/sim/sim_plotter.py +++ b/pdr_backend/sim/sim_plotter.py @@ -1,15 +1,15 @@ +import glob +import os +import pickle +import time +from datetime import datetime + import altair as alt import numpy as np import pandas as pd from enforce_typing import enforce_types from pdr_backend.aimodel.aimodel_plotdata import AimodelPlotdata -from datetime import datetime -import pickle -import os -import glob -import time - HEIGHT = 7.5 WIDTH = int(HEIGHT * 3.2) @@ -26,16 +26,17 @@ def __init__( def load_state(self): if not os.path.exists("sim_state"): - raise Exception("sim_state folder does not exist. Please run the simulation first.") + raise Exception( + "sim_state folder does not exist. Please run the simulation first." + ) - all_state_files = glob.glob('sim_state/st_*.pkl') + all_state_files = glob.glob("sim_state/st_*.pkl") if not all_state_files: raise Exception("No state files found. Please run the simulation first.") if not os.path.exists("sim_state/st_final.pkl"): - # TODO logger.info("Simulation still running. Plot will update in real time") # plot previous state to avoid using a pickle that hasn't finished - all_state_files = glob.glob('sim_state/st_*.pkl') + all_state_files = glob.glob("sim_state/st_*.pkl") all_state_files.sort() latest_file = all_state_files[-1] with open(latest_file, "rb") as f: @@ -46,23 +47,32 @@ def load_state(self): return self.st, latest_file.replace("sim_state/st_", "").replace(".pkl", "") - # TODO: logger.info("Simulation finished. Plotting final results") - # TODO: make sure the final state is written to disk! + # make sure the final state is written to disk before unpickling + # avoid race conditions with the pickling itself if file_age_in_seconds("sim_state/st_final.pkl") < 3: time.sleep(3) - self.st = pickle.load(open("sim_state/st_final.pkl", "rb")) - self.aimodel_plotdata = pickle.load(open("sim_state/aimodel_plotdata_final.pkl", "rb")) + with open("sim_state/st_final.pkl", "rb") as f: + self.st = pickle.load(f) + + with open("sim_state/aimodel_plotdata_final.pkl", "rb") as f: + self.aimodel_plotdata = pickle.load(f) + return self.st, "final" def init_state(self): - files = glob.glob('sim_state/*') + files = glob.glob("sim_state/*") for f in files: os.remove(f) - def save_state(self, sim_state, aimodel_plotdata: AimodelPlotdata, is_final: bool = False): - # TODO: for multisim, we could add a unique identifier to the filename and separate states - ts = datetime.now().strftime("%Y%m%d_%H%M%S.%f")[:-3] if not is_final else "final" + def save_state( + self, sim_state, aimodel_plotdata: AimodelPlotdata, is_final: bool = False + ): + ts = ( + datetime.now().strftime("%Y%m%d_%H%M%S.%f")[:-3] + if not is_final + else "final" + ) with open(f"sim_state/st_{ts}.pkl", "wb") as f: pickle.dump(sim_state, f) diff --git a/pdr_backend/sim/sim_state.py b/pdr_backend/sim/sim_state.py index d1a0a7b0a..8bbde5a18 100644 --- a/pdr_backend/sim/sim_state.py +++ b/pdr_backend/sim/sim_state.py @@ -51,7 +51,7 @@ def __init__(self, init_holdings: Dict[str, Eth]): tok: float(amt.amt_eth) for tok, amt in init_holdings.items() } self.init_loop_attributes() - self.iter_number = None + self.iter_number = 0 def init_loop_attributes(self): # 'i' is iteration number i diff --git a/pdr_backend/sim/test/test_multisim_engine.py b/pdr_backend/sim/test/test_multisim_engine.py index ce1fe5017..051bbbfb9 100644 --- a/pdr_backend/sim/test/test_multisim_engine.py +++ b/pdr_backend/sim/test/test_multisim_engine.py @@ -55,7 +55,7 @@ def _constructor_d_with_fast_runtime(tmpdir): # sim ss log_dir = os.path.join(tmpdir, "logs") - d = sim_ss_test_dict(do_plot=False, log_dir=log_dir, test_n=10) + d = sim_ss_test_dict(log_dir=log_dir, test_n=10) constructor_d["sim_ss"] = d return constructor_d diff --git a/pdr_backend/sim/test/test_sim_engine.py b/pdr_backend/sim/test/test_sim_engine.py index ab5b497ff..67f650a4d 100644 --- a/pdr_backend/sim/test/test_sim_engine.py +++ b/pdr_backend/sim/test/test_sim_engine.py @@ -31,9 +31,8 @@ def test_sim_engine(tmpdir): ppss.predictoor_ss = PredictoorSS(d) # sim ss - do_plot = True log_dir = os.path.join(tmpdir, "logs") - d = sim_ss_test_dict(do_plot, log_dir, test_n=5) + d = sim_ss_test_dict(log_dir, test_n=5) ppss.sim_ss = SimSS(d) # go diff --git a/ppss.yaml b/ppss.yaml index 839a4a4bf..ebf556c21 100644 --- a/ppss.yaml +++ b/ppss.yaml @@ -52,7 +52,6 @@ trader_ss: defaultType: spot sim_ss: # sim only - do_plot: True log_dir: logs test_n: 5000 # number of epochs to simulate tradetype: histmock # histmock | livemock | livereal diff --git a/sim_plots.py b/sim_plots.py index 8ee2d13c5..31a24d875 100755 --- a/sim_plots.py +++ b/sim_plots.py @@ -1,10 +1,12 @@ -from pdr_backend.sim.sim_plotter import SimPlotter +import time + +import streamlit + from pdr_backend.aimodel.aimodel_plotter import ( plot_aimodel_response, plot_aimodel_varimps, ) -import streamlit -import time +from pdr_backend.sim.sim_plotter import SimPlotter streamlit.set_page_config(layout="wide") @@ -51,11 +53,15 @@ continue canvas["pdr_profit_vs_time"].altair_chart( - sim_plotter.plot_pdr_profit_vs_time(), use_container_width=True, theme="streamlit" + sim_plotter.plot_pdr_profit_vs_time(), + use_container_width=True, + theme="streamlit", ) canvas["trader_profit_vs_time"].altair_chart( - sim_plotter.plot_trader_profit_vs_time(), use_container_width=True, theme="streamlit" + sim_plotter.plot_trader_profit_vs_time(), + use_container_width=True, + theme="streamlit", ) canvas["accuracy_vs_time"].altair_chart( @@ -63,23 +69,33 @@ ) canvas["f1_precision_recall_vs_time"].altair_chart( - sim_plotter.plot_f1_precision_recall_vs_time(), use_container_width=True, theme="streamlit" + sim_plotter.plot_f1_precision_recall_vs_time(), + use_container_width=True, + theme="streamlit", ) canvas["pdr_profit_vs_ptrue"].altair_chart( - sim_plotter.plot_pdr_profit_vs_ptrue(), use_container_width=True, theme="streamlit" + sim_plotter.plot_pdr_profit_vs_ptrue(), + use_container_width=True, + theme="streamlit", ) canvas["trader_profit_vs_ptrue"].altair_chart( - sim_plotter.plot_trader_profit_vs_ptrue(), use_container_width=True, theme="streamlit" + sim_plotter.plot_trader_profit_vs_ptrue(), + use_container_width=True, + theme="streamlit", ) canvas["aimodel_varimps"].altair_chart( - plot_aimodel_varimps(sim_plotter.aimodel_plotdata), use_container_width=True, theme="streamlit" + plot_aimodel_varimps(sim_plotter.aimodel_plotdata), + use_container_width=True, + theme="streamlit", ) canvas["aimodel_response"].plotly_chart( - plot_aimodel_response(sim_plotter.aimodel_plotdata), use_container_width=True, theme="streamlit" + plot_aimodel_response(sim_plotter.aimodel_plotdata), + use_container_width=True, + theme="streamlit", ) last_ts = new_ts