From 2db2e3b912ccdf633dcc8031a2ef4972ab74dbd8 Mon Sep 17 00:00:00 2001 From: Amichay Oren Date: Sun, 8 Jan 2023 10:12:34 +0200 Subject: [PATCH] adding entry points --- .therapist.yml | 2 +- liualgotrader/scripts/__init__.py | 0 .../scripts/{backtester => backtester.py} | 33 ++--- liualgotrader/scripts/{liu => liu.py} | 122 +++++++++++++----- .../scripts/{market_miner => market_miner.py} | 46 +++---- .../{optimizer => scripts/optimizer.py} | 19 ++- .../{portfolio => scripts/portfolio.py} | 29 +++-- liualgotrader/scripts/{trader => trader.py} | 25 +--- pdm.lock | 20 ++- pyproject.toml | 9 +- 10 files changed, 179 insertions(+), 126 deletions(-) create mode 100644 liualgotrader/scripts/__init__.py rename liualgotrader/scripts/{backtester => backtester.py} (92%) mode change 100755 => 100644 rename liualgotrader/scripts/{liu => liu.py} (81%) mode change 100755 => 100644 rename liualgotrader/scripts/{market_miner => market_miner.py} (79%) mode change 100755 => 100644 rename liualgotrader/{optimizer => scripts/optimizer.py} (96%) rename liualgotrader/{portfolio => scripts/portfolio.py} (91%) rename liualgotrader/scripts/{trader => trader.py} (94%) mode change 100755 => 100644 diff --git a/.therapist.yml b/.therapist.yml index c34d4b717..85afa73a7 100644 --- a/.therapist.yml +++ b/.therapist.yml @@ -3,7 +3,7 @@ actions: run: mypy --namespace-packages --show-error-context --pretty --strict-optional --ignore-missing-imports {files} include: "*.py" bandit: - run: bandit -ii {files} + run: bandit -s B605 -ii {files} include: "*.py" black: run: black -l 79 {files} diff --git a/liualgotrader/scripts/__init__.py b/liualgotrader/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/liualgotrader/scripts/backtester b/liualgotrader/scripts/backtester.py old mode 100755 new mode 100644 similarity index 92% rename from liualgotrader/scripts/backtester rename to liualgotrader/scripts/backtester.py index 02ef67b76..e73cae6c1 --- a/liualgotrader/scripts/backtester +++ b/liualgotrader/scripts/backtester.py @@ -1,36 +1,29 @@ -#!/usr/bin/env python - -import asyncio import getopt import os -import pprint import sys -import traceback -import uuid -import toml +from datetime import date, datetime +from typing import List, Optional + import pandas as pd -import importlib.util -from datetime import datetime, timedelta, date, tzinfo -from typing import List, Dict, Optional +import parsedatetime.parsedatetime as pdt import pygit2 import pytz +import toml from requests.exceptions import HTTPError -import parsedatetime.parsedatetime as pdt +from liualgotrader import enhanced_backtest from liualgotrader.common import config, market_data, trading_data from liualgotrader.common.database import create_db_connection from liualgotrader.common.decorators import timeit from liualgotrader.common.tlog import tlog -from liualgotrader.common.data_loader import TimeScale -from liualgotrader.common.types import AssetType +from liualgotrader.common.types import AssetType, TimeScale from liualgotrader.fincalcs.vwap import add_daily_vwap from liualgotrader.models.algo_run import AlgoRun from liualgotrader.models.new_trades import NewTrade from liualgotrader.models.trending_tickers import TrendingTickers -from liualgotrader.strategies.base import Strategy, StrategyType from liualgotrader.scanners.base import Scanner from liualgotrader.scanners.momentum import Momentum -from liualgotrader import enhanced_backtest +from liualgotrader.strategies.base import Strategy, StrategyType def show_usage(): @@ -77,12 +70,12 @@ def dateFromString(s: str) -> date: dt = result.date() if dt is None: - raise ValueError("Don't understand date '" + s + "'") + raise ValueError(f"Don't understand date '{s}'") return dt -if __name__ == "__main__": +def main_cli() -> None: try: config.build_label = pygit2.Repository("../").describe( describe_strategy=pygit2.GIT_DESCRIBE_TAGS @@ -128,7 +121,7 @@ def dateFromString(s: str) -> date: asset_type = AssetType.US_EQUITIES opts, args = getopt.getopt( sys.argv[3:], - shortopts=[], + shortopts="", longopts=[ "to=", "scale=", @@ -156,7 +149,9 @@ def dateFromString(s: str) -> date: elif arg.lower() == "equity": asset_type = AssetType.US_EQUITIES else: - print(f"ERROR: value {arg} not supported for parameter 'asset'") + print( + f"ERROR: value {arg} not supported for parameter 'asset'" + ) sys.exit(0) elif opt in ("--scale"): if arg in ("day", "minute"): diff --git a/liualgotrader/scripts/liu b/liualgotrader/scripts/liu.py old mode 100755 new mode 100644 similarity index 81% rename from liualgotrader/scripts/liu rename to liualgotrader/scripts/liu.py index fa87fbde9..9c67ad085 --- a/liualgotrader/scripts/liu +++ b/liualgotrader/scripts/liu.py @@ -1,16 +1,18 @@ +import asyncio +import getopt import os -import sys -import pygit2 import pathlib -import requests +import sys import time -import getopt -import asyncio import uuid + +import pygit2 +import requests + from liualgotrader.common import config +from liualgotrader.common.database import create_db_connection from liualgotrader.models.accounts import Accounts from liualgotrader.models.portfolio import Portfolio -from liualgotrader.common.database import create_db_connection def show_version(): @@ -70,7 +72,9 @@ def setup_db( f.write(resolved) f.write("\n") - base_url = "https://raw.github.com/amor71/LiuAlgoTrader/master/database/" + base_url = ( + "https://raw.github.com/amor71/LiuAlgoTrader/master/database/" + ) files = ["schema.sql", "postgres.conf"] if restore_sample_db: files.append("liu_dump.sql") @@ -104,7 +108,9 @@ def setup_db( print("check deployment using `\psql -h localhost -p 5400 -U liu`") -def setup_samples(samples_location: str, user: str, passwd: str, db: str) -> None: +def setup_samples( + samples_location: str, user: str, passwd: str, db: str +) -> None: try: print() print("+--------------------+") @@ -117,10 +123,12 @@ def setup_samples(samples_location: str, user: str, passwd: str, db: str) -> Non if samples_location[-1] != "/": samples_location += "/" - base_url = ( - "https://raw.github.com/amor71/LiuAlgoTrader/master/examples/quickstart/" - ) - files = ["tradeplan.toml", "vwap_short.py", "momentum_long_simplified.py"] + base_url = "https://raw.github.com/amor71/LiuAlgoTrader/master/examples/quickstart/" + files = [ + "tradeplan.toml", + "vwap_short.py", + "momentum_long_simplified.py", + ] for file in files: print(f"Downloading {base_url}{file} to {samples_location}...") @@ -134,7 +142,9 @@ def setup_samples(samples_location: str, user: str, passwd: str, db: str) -> Non if os.name == "nt": with open(f"{samples_location}env_vars.bat", "w") as f: - f.write(f"set DSN=postgresql://{user}:{passwd}@localhost:5400/{db}\n") + f.write( + f"set DSN=postgresql://{user}:{passwd}@localhost:5400/{db}\n" + ) else: with open(f"{samples_location}env_vars.sh", "w") as f: f.write( @@ -158,7 +168,9 @@ def quickstart(): print("| Step #1 - Alpaca & Polygon credentials |") print("+----------------------------------------+") print() - print("To use Liu Algo Trading Framework you need an account with Alpaca Markets,") + print( + "To use Liu Algo Trading Framework you need an account with Alpaca Markets," + ) print("do you already have an account [Y]/n:") i = input() have_funded = len(i) == 0 or (i == "y" or i == "Y" or i.lower() == "yes") @@ -169,13 +181,21 @@ def quickstart(): to_exit = False if not config.alpaca_api_key or not config.alpaca_api_secret: - print("Liu Algo Trading Framework uses Alpaca for both LIVE and PAPER trading.") + print( + "Liu Algo Trading Framework uses Alpaca for both LIVE and PAPER trading." + ) print() print("The Framework expects three environment variables to be set:") - print("`APCA_API_KEY_ID` and `APCA_API_SECRET_KEY`: reflecting the funded") + print( + "`APCA_API_KEY_ID` and `APCA_API_SECRET_KEY`: reflecting the funded" + ) print("account's API key and secret respectively.") - print("And `APCA_API_BASE_URL` reflecting the base URL for your account.") - print("Please set the three environment variables and re-run the wizard.") + print( + "And `APCA_API_BASE_URL` reflecting the base URL for your account." + ) + print( + "Please set the three environment variables and re-run the wizard." + ) to_exit = True if to_exit: @@ -188,7 +208,9 @@ def quickstart(): print() print("Do you already have a PostgreSQL instance configured [N]/y:") i = input() - already_have_db = len(i) > 0 and (i == "y" or i == "Y" or i.lower() == "yes") + already_have_db = len(i) > 0 and ( + i == "y" or i == "Y" or i.lower() == "yes" + ) if already_have_db: print( @@ -200,31 +222,53 @@ def quickstart(): print( "Liu Algo Trading Framework uses `docker-compose` to run a local database." ) - print("The installation will download the docker-compose.yml, database schema") - print("and prepare the database for first time usage. You can stop and re-run") - print("the database using `docker-compose up -d ` and `docker-compose down`") - print("respectively. Your data will not be deleted. For more details RTFM.") + print( + "The installation will download the docker-compose.yml, database schema" + ) + print( + "and prepare the database for first time usage. You can stop and re-run" + ) + print( + "the database using `docker-compose up -d ` and `docker-compose down`" + ) + print( + "respectively. Your data will not be deleted. For more details RTFM." + ) print() print(f"Select location for database files [{pwd}/liu_data/]:") db_location = input() - db_location = f"{pwd}/liu_data/" if not len(db_location) else db_location + db_location = ( + f"{pwd}/liu_data/" if not len(db_location) else db_location + ) print() print("** IMPORTANT NOTE**") - print("The installation wizard, aside from installing PostgreSQL in a docker") - print("may also download a sample database, with existing data to help you") + print( + "The installation wizard, aside from installing PostgreSQL in a docker" + ) + print( + "may also download a sample database, with existing data to help you" + ) print("take your first steps with Liu Algo Trading Framework.") - print("However, in order to restore the sample data, you need to select") + print( + "However, in order to restore the sample data, you need to select" + ) print( "the default username ('liu') password ('liu)', and database name ('liu')." ) - print("after you try out the sample, you can delete data and re-run this") + print( + "after you try out the sample, you can delete data and re-run this" + ) print("wizard to install a fresh copy of the database.") print() print("Would you like to download the sample database? [Y]/n") i = input() - restore_sample_db = len(i) == 0 or (i == "y" or i == "Y" or i.lower() == "yes") - print("Select the database name for keeping track of your trading [liu]:") + restore_sample_db = len(i) == 0 or ( + i == "y" or i == "Y" or i.lower() == "yes" + ) + print( + "Select the database name for keeping track of your trading [liu]:" + ) db_name = input() db_name = "liu" if not len(db_name) else db_name print("Select the database user-name [liu]:") @@ -248,7 +292,9 @@ def quickstart(): print(f"Select location for sample files [{pwd}/liu_samples]:") sample_location = input() sample_location = ( - f"{pwd}/liu_samples" if not len(sample_location) else sample_location + f"{pwd}/liu_samples" + if not len(sample_location) + else sample_location ) print() @@ -267,14 +313,20 @@ def quickstart(): print("+---------------+") print() print("Congratulations, Liu Algo Trader is ready to go!") - print("From here, you can either run the back-testing UI, or read the docs.") - print("The full documentation is available here: `https://liualgotrader.rtfd.io`.") + print( + "From here, you can either run the back-testing UI, or read the docs." + ) + print( + "The full documentation is available here: `https://liualgotrader.rtfd.io`." + ) if restore_sample_db: print() print( "The restored database includes data for the first week and half of Oct 2020." ) - print("You may now run a back-testing sessions for these days, as well as ") + print( + "You may now run a back-testing sessions for these days, as well as " + ) print("analyze an existing trading session.") print() @@ -325,7 +377,7 @@ def create_portfolio(amount: float, credit: float): print(f"Portfolio ID {portfolio_id} created") -def main_cli(): +def main_cli() -> None: config.filename = os.path.basename(__file__) try: diff --git a/liualgotrader/scripts/market_miner b/liualgotrader/scripts/market_miner.py old mode 100755 new mode 100644 similarity index 79% rename from liualgotrader/scripts/market_miner rename to liualgotrader/scripts/market_miner.py index a416be478..9bf1f2267 --- a/liualgotrader/scripts/market_miner +++ b/liualgotrader/scripts/market_miner.py @@ -1,16 +1,17 @@ -#!/usr/bin/env python - """off-hours calculations, and data collections""" import asyncio -import os, sys -import toml -import pygit2 -from typing import Dict, List, Optional +import importlib.util +import os +import sys import traceback -import importlib +from typing import Dict, List, Optional + +import pygit2 +import toml + from liualgotrader.common import config -from liualgotrader.common.tlog import tlog from liualgotrader.common.database import create_db_connection +from liualgotrader.common.tlog import tlog from liualgotrader.miners.base import Miner # rom liualgotrader.miners.stock_cluster import StockCluster @@ -27,7 +28,7 @@ def motd(filename: str, version: str) -> None: async def main(conf_dict: Dict): - task_list: List[Optional[asyncio.tasks]] = [] + task_list: List[Optional[asyncio.Task]] = [] await create_db_connection() for miner in conf_dict["miners"]: @@ -36,10 +37,16 @@ async def main(conf_dict: Dict): spec = importlib.util.spec_from_file_location( "module.name", conf_dict["miners"][miner]["filename"] ) + if not spec: + raise AssertionError( + f"could not load module {conf_dict['miners'][miner]['filename']}" + ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore else: - module = importlib.import_module(f"liualgotrader.miners.{miner}") + module = importlib.import_module( + f"liualgotrader.miners.{miner}" + ) class_name = f"{miner[0].upper()}{miner[1:]}" miner_class = getattr(module, class_name) @@ -48,22 +55,9 @@ async def main(conf_dict: Dict): exit(0) except Exception as e: tlog(f"[ERROR] miner {miner} resulted in exception:`{e}`") - else: - try: - debug = conf_dict["miners"][miner].get("debug", False) - miner = miner_class(debug=debug, data=conf_dict["miners"][miner]) - task_list.append(asyncio.create_task(miner.run())) - - await asyncio.gather(*task_list) - except Exception as e: - tlog(f"[ERROR] aborted w/ exception {e}") - exc_info = sys.exc_info() - traceback.print_exception(*exc_info) - del exc_info - raise - - -if __name__ == "__main__": + + +def main_cli() -> None: """ starting """ diff --git a/liualgotrader/optimizer b/liualgotrader/scripts/optimizer.py similarity index 96% rename from liualgotrader/optimizer rename to liualgotrader/scripts/optimizer.py index b608812fa..64efffafa 100644 --- a/liualgotrader/optimizer +++ b/liualgotrader/scripts/optimizer.py @@ -1,22 +1,19 @@ -#!/usr/bin/env python - +import copy import getopt +import multiprocessing as mp import os import sys import uuid +from datetime import date, datetime +from typing import Dict, List -import copy -from datetime import datetime, date -from typing import List, Dict -import pygit2 -import multiprocessing as mp import parsedatetime.parsedatetime as pdt +import pygit2 - +from liualgotrader.backtesting.optimizer import backtest_iteration from liualgotrader.common import config +from liualgotrader.common.hyperparameter import Hyperparameters, Parameter from liualgotrader.common.tlog import tlog -from liualgotrader.common.hyperparameter import Parameter, Hyperparameters -from liualgotrader.backtesting.optimizer import backtest_iteration def motd(filename: str, version: str) -> None: @@ -145,7 +142,7 @@ def load_configuration(filename: str): return conf_dict -if __name__ == "__main__": +def main_cli() -> None: mp.set_start_method("spawn") if len(sys.argv) > 2: diff --git a/liualgotrader/portfolio b/liualgotrader/scripts/portfolio.py similarity index 91% rename from liualgotrader/portfolio rename to liualgotrader/scripts/portfolio.py index 48992ba3d..cac28a825 100644 --- a/liualgotrader/portfolio +++ b/liualgotrader/scripts/portfolio.py @@ -1,19 +1,19 @@ -#!/usr/bin/env python -import fire -import uuid import asyncio +import uuid + +import fire from tabulate import tabulate from liualgotrader.analytics import analysis -from liualgotrader.reprocess.portfolio import account_transactions from liualgotrader.common.database import create_db_connection -from liualgotrader.models.portfolio import Portfolio from liualgotrader.common.types import AssetType +from liualgotrader.models.portfolio import Portfolio +from liualgotrader.reprocess.portfolio import account_transactions def equity(portfolio_id: str): """Show current equity breakdown""" - portfolio = analysis.get_portfolio_equity(portfolio_id) + portfolio = asyncio.run(analysis.get_portfolio_equity(portfolio_id)) if portfolio.empty: print("Empty portfolio") return @@ -33,7 +33,7 @@ def equity(portfolio_id: str): def account(portfolio_id: str): """Show account transactions for portfolio""" - portfolio = analysis.get_portfolio_cash(portfolio_id) + portfolio = asyncio.run(analysis.get_portfolio_cash(portfolio_id)) if portfolio.empty: print("Empty portfolio transactions") return @@ -107,7 +107,14 @@ def list(): print("Portfolio(s):") data = loop.run_until_complete(Portfolio.list_portfolios()) data = data[ - ["portfolio_id", "tstamp", "last_transaction", "size", "assets", "parameters"] + [ + "portfolio_id", + "tstamp", + "last_transaction", + "size", + "assets", + "parameters", + ] ] print( @@ -129,7 +136,7 @@ def list(): ) -def main(): +def main_cli() -> None: fire.Fire( { "create": create, @@ -141,7 +148,3 @@ def main(): }, name="portfolio", ) - - -if __name__ == "__main__": - main() diff --git a/liualgotrader/scripts/trader b/liualgotrader/scripts/trader.py old mode 100755 new mode 100644 similarity index 94% rename from liualgotrader/scripts/trader rename to liualgotrader/scripts/trader.py index 15a4d7ff3..3d722e6c7 --- a/liualgotrader/scripts/trader +++ b/liualgotrader/scripts/trader.py @@ -1,23 +1,18 @@ -#!/usr/bin/env python +import copy import multiprocessing as mp -from multiprocessing import Queue - import os import sys -import time import uuid -import random -import asyncio -from datetime import datetime -from math import ceil +from multiprocessing import Queue from typing import List + import pygit2 import toml -import copy from pytz import timezone + from liualgotrader.common import config -from liualgotrader.common.tlog import tlog from liualgotrader.common.concurrency import calc_num_consumer_processes +from liualgotrader.common.tlog import tlog from liualgotrader.consumer import consumer_main from liualgotrader.producer import producer_main from liualgotrader.scanners_runner import main @@ -34,15 +29,7 @@ def motd(filename: str, version: str, unique_id: str) -> None: tlog("----------------------------------------------------------") -""" -process main -""" - - -""" -starting -""" -if __name__ == "__main__": +def main_cli(): config.filename = os.path.basename(__file__) mp.set_start_method("spawn") diff --git a/pdm.lock b/pdm.lock index 5f8712ef3..c7ddfd3f6 100644 --- a/pdm.lock +++ b/pdm.lock @@ -880,6 +880,16 @@ dependencies = [ "types-urllib3<1.27", ] +[[package]] +name = "types-tabulate" +version = "0.9.0.0" +summary = "Typing stubs for tabulate" + +[[package]] +name = "types-toml" +version = "0.10.8.1" +summary = "Typing stubs for toml" + [[package]] name = "types-urllib3" version = "1.26.25.4" @@ -987,7 +997,7 @@ summary = "Backport of pathlib-compatible object wrapper for zip files" [metadata] lock_version = "4.1" -content_hash = "sha256:5754c2efd64a5fcd6d64ca7c35739ab650a9bb9a453c4d38787483b64e77a144" +content_hash = "sha256:98a6fc9f235ee7876c8a454e39b64060635e864faacd5d16406445f7d4a24745" [metadata.files] "aiohttp 3.8.3" = [ @@ -2428,6 +2438,14 @@ content_hash = "sha256:5754c2efd64a5fcd6d64ca7c35739ab650a9bb9a453c4d38787483b64 {url = "https://files.pythonhosted.org/packages/9c/95/ad745e140031199bfecb81526189034bcca0b8b71e9db446c86f181ba2ca/types_requests-2.28.11.7-py3-none-any.whl", hash = "sha256:b6a2fca8109f4fdba33052f11ed86102bddb2338519e1827387137fefc66a98b"}, {url = "https://files.pythonhosted.org/packages/f4/2c/72cb8c5bbadc61eee2d28b696700dc8ae6ed70784681d6dad46b518487ba/types-requests-2.28.11.7.tar.gz", hash = "sha256:0ae38633734990d019b80f5463dfa164ebd3581998ac8435f526da6fe4d598c3"}, ] +"types-tabulate 0.9.0.0" = [ + {url = "https://files.pythonhosted.org/packages/bf/7b/3c760cfc4624c18edb72ef0ff4da7914c6f374328722372a0467a38cd7d1/types-tabulate-0.9.0.0.tar.gz", hash = "sha256:4a79474714cea156bcd2185bb9bddd8fb9d3d5227c8d0a7f21bf21caf5f60e67"}, + {url = "https://files.pythonhosted.org/packages/d7/43/34e2ebc7f395a488db53d310b972cdfa62c4bb9cdc31ef59cce7f3a56213/types_tabulate-0.9.0.0-py3-none-any.whl", hash = "sha256:a1cc2aa52d2a9abfe0acbb361ccd49e3400794086a41626ce8784ed2e1ec0b0d"}, +] +"types-toml 0.10.8.1" = [ + {url = "https://files.pythonhosted.org/packages/84/e9/9569cdc5c54b7d36d36b2bacefeb01a1b2feb90c9fd18ee15797ad4fa67b/types_toml-0.10.8.1-py3-none-any.whl", hash = "sha256:b7b5c4977f96ab7b5ac06d8a6590d17c0bf252a96efc03b109c2711fb3e0eafd"}, + {url = "https://files.pythonhosted.org/packages/c6/8b/a8544990966e9b475c1767f21272cbbdc2b470d4e690e04b5510b129b311/types-toml-0.10.8.1.tar.gz", hash = "sha256:171bdb3163d79a520560f24ba916a9fc9bff81659c5448a9fea89240923722be"}, +] "types-urllib3 1.26.25.4" = [ {url = "https://files.pythonhosted.org/packages/37/67/c0751b0f672b0822b11ece210b211bdd1e08d6e21aa62e6d297b054127d7/types-urllib3-1.26.25.4.tar.gz", hash = "sha256:eec5556428eec862b1ac578fb69aab3877995a99ffec9e5a12cf7fbd0cc9daee"}, {url = "https://files.pythonhosted.org/packages/ab/35/7b813668050bfe0820ad84df0ba668adb6abb41ca2bb0f16ad4deabca279/types_urllib3-1.26.25.4-py3-none-any.whl", hash = "sha256:ed6b9e8a8be488796f72306889a06a3fc3cb1aa99af02ab8afb50144d7317e49"}, diff --git a/pyproject.toml b/pyproject.toml index 2505e52b6..5644580ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ license = {text = "MIT"} name = "liualgotrader" readme = "README.md" requires-python = ">=3.10" -version = "0.4.31" +version = "0.4.33" [project.optional-dependencies] dev = [ @@ -51,10 +51,17 @@ dev = [ "isort>=5.11.4", "types-pytz>=2022.7.0.0", "types-requests>=2.28.11.7", + "types-tabulate>=0.9.0.0", + "types-toml>=0.10.8.1", ] [project.scripts] +backtester = "liualgotrader.scripts.backtester:main_cli" liu = "liualgotrader.scripts.liu:main_cli" +market_miner = "liualgotrader.scripts.market_miner:main_cli" +optimizer = "liualgotrader.scripts.optimizer:main_cli" +portfolio = "liualgotrader.scripts.portfolio:main_cli" +trader = "liualgotrader.scripts.trader:main_cli" [tool.pdm.version] source = "scm"