From 3de7bd18cabc6ff988c0bf4732d0ee90a4b47dac Mon Sep 17 00:00:00 2001 From: Youbad Date: Sat, 6 Feb 2021 07:12:46 -0500 Subject: [PATCH 1/7] minor discrepancies --- finrl/misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/finrl/misc.py b/finrl/misc.py index 359d0d0e4..7ccc71b55 100644 --- a/finrl/misc.py +++ b/finrl/misc.py @@ -1,5 +1,5 @@ """ -Various tool function for Freqtrade and scripts +Various tool function and scripts """ import gzip import logging @@ -175,7 +175,7 @@ def render_template(templatefile: str, arguments: dict = {}) -> str: from jinja2 import Environment, PackageLoader, select_autoescape env = Environment( - loader=PackageLoader('freqtrade', 'templates'), + loader=PackageLoader('finrl', 'templates'), autoescape=select_autoescape(['html', 'xml']) ) template = env.get_template(templatefile) From 6c859cfded2d46331b4db01e4206d4c8867d7697 Mon Sep 17 00:00:00 2001 From: Youbad Date: Sat, 6 Feb 2021 08:27:02 -0500 Subject: [PATCH 2/7] docker requirements.txt --- docker/requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/requirements.txt b/docker/requirements.txt index 22288bce3..dfa642608 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -139,3 +139,8 @@ typepy==1.1.2 Werkzeug==1.0.1 yfinance==0.1.55 zipp==3.4.0 +arrow +python-rapidjson +questionary +sqlalchemy +tabulate \ No newline at end of file From aa8bae8fd720c85c8f1ff4e19965fdb4e7b2917e Mon Sep 17 00:00:00 2001 From: Youbad Date: Mon, 8 Feb 2021 04:39:59 -0500 Subject: [PATCH 3/7] Download Data from Crypto Exchange and Fetch --- .gitignore | 5 +- config.json | 7 +- finrl/commands/__init__.py | 7 + finrl/commands/data_commands.py | 126 +++ finrl/commands/deploy_commands.py | 27 + finrl/config/__init__.py | 4 + finrl/config/check_exchange.py | 73 ++ finrl/config/config_setup.py | 25 + finrl/config/config_validation.py | 157 +++ finrl/config/configuration.py | 117 ++- finrl/config/directory_operations.py | 77 ++ finrl/config/timerange.py | 107 ++ finrl/data/__init__.py | 8 + finrl/data/btanalysis.py | 382 +++++++ finrl/data/converter.py | 264 +++++ finrl/data/dataprovider.py | 182 ++++ finrl/data/history/__init__.py | 12 + finrl/data/history/hdf5datahandler.py | 213 ++++ finrl/data/history/history_utils.py | 397 ++++++++ finrl/data/history/idatahandler.py | 255 +++++ finrl/data/history/jsondatahandler.py | 204 ++++ finrl/exchange/__init__.py | 16 + finrl/exchange/bibox.py | 23 + finrl/exchange/binance.py | 89 ++ finrl/exchange/bittrex.py | 23 + finrl/exchange/common.py | 155 +++ finrl/exchange/exchange.py | 1332 +++++++++++++++++++++++++ finrl/exchange/ftx.py | 136 +++ finrl/exchange/kraken.py | 122 +++ finrl/marketdata/yahoodownloader.py | 61 +- finrl/misc.py | 4 +- finrl/resolvers/__init__.py | 4 + finrl/resolvers/exchange_resolver.py | 64 ++ finrl/resolvers/iresolver.py | 178 ++++ finrl/state.py | 10 +- testing_download.py | 40 + 36 files changed, 4888 insertions(+), 18 deletions(-) create mode 100644 finrl/commands/__init__.py create mode 100644 finrl/commands/data_commands.py create mode 100644 finrl/commands/deploy_commands.py create mode 100644 finrl/config/check_exchange.py create mode 100644 finrl/config/config_setup.py create mode 100644 finrl/config/config_validation.py create mode 100644 finrl/config/directory_operations.py create mode 100644 finrl/config/timerange.py create mode 100644 finrl/data/__init__.py create mode 100644 finrl/data/btanalysis.py create mode 100644 finrl/data/converter.py create mode 100644 finrl/data/dataprovider.py create mode 100644 finrl/data/history/__init__.py create mode 100644 finrl/data/history/hdf5datahandler.py create mode 100644 finrl/data/history/history_utils.py create mode 100644 finrl/data/history/idatahandler.py create mode 100644 finrl/data/history/jsondatahandler.py create mode 100644 finrl/exchange/__init__.py create mode 100644 finrl/exchange/bibox.py create mode 100644 finrl/exchange/binance.py create mode 100644 finrl/exchange/bittrex.py create mode 100644 finrl/exchange/common.py create mode 100644 finrl/exchange/exchange.py create mode 100644 finrl/exchange/ftx.py create mode 100644 finrl/exchange/kraken.py create mode 100644 finrl/resolvers/__init__.py create mode 100644 finrl/resolvers/exchange_resolver.py create mode 100644 finrl/resolvers/iresolver.py create mode 100644 testing_download.py diff --git a/.gitignore b/.gitignore index 06c31d4e6..a77e7d0e4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Custom config #results/ - +#USER_DATA +./user_data # remove DS_Store **/.DS_Store @@ -127,6 +128,8 @@ celerybeat.pid # mkdocs documentation /site + + # mypy .mypy_cache/ .dmypy.json diff --git a/config.json b/config.json index 9d4f97a41..287adf47d 100644 --- a/config.json +++ b/config.json @@ -4,7 +4,7 @@ "stake_amount": 0.1, "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "timeframe": "5m", + "timeframe": "1d", "dry_run": true, "exchange": { @@ -47,5 +47,8 @@ "CORS_origins": [], "username": "", "password": "" - } + }, + "dataformat_ohlcv": "json", + "dataformat_trades": "jsongz", + "user_data_dir":"./user_data/" } diff --git a/finrl/commands/__init__.py b/finrl/commands/__init__.py new file mode 100644 index 000000000..6aff3724f --- /dev/null +++ b/finrl/commands/__init__.py @@ -0,0 +1,7 @@ +# flake8: noqa: F401 +""" +Commands module. +Contains all start-commands, subcommands and CLI Interface creation. +""" +from finrl.commands.deploy_commands import start_create_userdir +from finrl.commands.data_commands import start_download_data \ No newline at end of file diff --git a/finrl/commands/data_commands.py b/finrl/commands/data_commands.py new file mode 100644 index 000000000..ada65ac92 --- /dev/null +++ b/finrl/commands/data_commands.py @@ -0,0 +1,126 @@ +import logging +import sys +from collections import defaultdict +from datetime import datetime, timedelta +from typing import Any, Dict, List + +from finrl.config import TimeRange, setup_utils_configuration +from finrl.data.converter import convert_ohlcv_format, convert_trades_format +from finrl.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, + refresh_backtest_trades_data) +from finrl.exceptions import OperationalException +from finrl.exchange import timeframe_to_minutes +from finrl.resolvers import ExchangeResolver +from finrl.state import RunMode + + +logger = logging.getLogger(__name__) + + +def start_download_data(args: Dict[str, Any]) -> None: + """ + Download data (former download_backtest_data.py script) + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + if 'days' in config and 'timerange' in config: + raise OperationalException("--days and --timerange are mutually exclusive. " + "You can only specify one or the other.") + timerange = TimeRange() + if 'days' in config: + time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'timerange' in config: + timerange = timerange.parse_timerange(config['timerange']) + + # Remove stake-currency to skip checks which are not relevant for datadownload + config['stake_currency'] = '' + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + logger.info(f"About to download pairs: {config['pairs']}, " + f"intervals: {config['timeframes']} to {config['datadir']}") + + pairs_not_available: List[str] = [] + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + # Manual validations of relevant settings + exchange.validate_pairs(config['pairs']) + for timeframe in config['timeframes']: + exchange.validate_timeframes(timeframe) + + try: + + if config.get('download_trades'): + pairs_not_available = refresh_backtest_trades_data( + exchange, pairs=config['pairs'], datadir=config['datadir'], + timerange=timerange, erase=bool(config.get('erase')), + data_format=config['dataformat_trades']) + + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=config['pairs'], timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), + data_format_ohlcv=config['dataformat_ohlcv'], + data_format_trades=config['dataformat_trades'], + ) + else: + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config['pairs'], timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), + data_format=config['dataformat_ohlcv']) + + except KeyboardInterrupt: + sys.exit("Interrupt received, aborting ...") + + finally: + if pairs_not_available: + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {exchange.name}.") + + +def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: + """ + Convert data from one format to another + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + if ohlcv: + convert_ohlcv_format(config, + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) + else: + convert_trades_format(config, + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) + + +def start_list_data(args: Dict[str, Any]) -> None: + """ + List available backtest data + """ + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + from tabulate import tabulate + + from freqtrade.data.history.idatahandler import get_datahandler + dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv']) + + paircombs = dhc.ohlcv_get_available_data(config['datadir']) + + if args['pairs']: + paircombs = [comb for comb in paircombs if comb[0] in args['pairs']] + + print(f"Found {len(paircombs)} pair / timeframe combinations.") + groupedpair = defaultdict(list) + for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))): + groupedpair[pair].append(timeframe) + + if groupedpair: + print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()], + headers=("Pair", "Timeframe"), + tablefmt='psql', stralign='right')) diff --git a/finrl/commands/deploy_commands.py b/finrl/commands/deploy_commands.py new file mode 100644 index 000000000..a66018f82 --- /dev/null +++ b/finrl/commands/deploy_commands.py @@ -0,0 +1,27 @@ +import logging +import sys +from pathlib import Path +from typing import Any, Dict + +from finrl.config import setup_utils_configuration +from finrl.config.directory_operations import copy_sample_files, create_userdata_dir +from finrl.exceptions import OperationalException +from finrl.misc import render_template, render_template_with_fallback +from finrl.state import RunMode + + +logger = logging.getLogger(__name__) + + +def start_create_userdir(args: Dict[str, Any]) -> None: + """ + Create "user_data" directory to contain user data strategies, hyperopt, ...) + :param args: Cli args from Arguments() + :return: None + """ + if "user_data_dir" in args and args["user_data_dir"]: + userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) + copy_sample_files(userdir, overwrite=args["reset"]) + else: + logger.warning("`create-userdir` requires --userdir to be set.") + sys.exit(1) diff --git a/finrl/config/__init__.py b/finrl/config/__init__.py index 379c6a8fb..3ac5358b0 100644 --- a/finrl/config/__init__.py +++ b/finrl/config/__init__.py @@ -1 +1,5 @@ +from finrl.config.check_exchange import check_exchange, remove_credentials +from finrl.config.config_setup import setup_utils_configuration +from finrl.config.config_validation import validate_config_consistency from finrl.config.configuration import Configuration +from finrl.config.timerange import TimeRange diff --git a/finrl/config/check_exchange.py b/finrl/config/check_exchange.py new file mode 100644 index 000000000..90553ec9d --- /dev/null +++ b/finrl/config/check_exchange.py @@ -0,0 +1,73 @@ +import logging +from typing import Any, Dict + +from finrl.exceptions import OperationalException +from finrl.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad, + is_exchange_known_ccxt, is_exchange_officially_supported) +from finrl.state import RunMode + + +logger = logging.getLogger(__name__) + + +def remove_credentials(config: Dict[str, Any]) -> None: + """ + Removes exchange keys from the configuration and specifies dry-run + Used for backtesting / hyperopt / edge and utils. + Modifies the input dict! + """ + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['exchange']['password'] = '' + config['exchange']['uid'] = '' + config['dry_run'] = True + + +def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: + """ + Check if the exchange name in the config file is supported by ccxt + :param check_for_bad: if True, check the exchange against the list of known 'bad' + exchanges + :return: False if exchange is 'bad', i.e. is known to work with the bot with + critical issues or does not work at all, crashes, etc. True otherwise. + raises an exception if the exchange if not supported by ccxt. + """ + + if (config['runmode'] in [RunMode.PLOT, RunMode.UTIL_NO_EXCHANGE, RunMode.OTHER] + and not config.get('exchange', {}).get('name')): + # Skip checking exchange in plot mode, since it requires no exchange + return True + logger.info("Checking exchange...") + + exchange = config.get('exchange', {}).get('name').lower() + if not exchange: + raise OperationalException( + f'This command requires a configured exchange. You should either use ' + f'`--exchange ` or specify a configuration file via `--config`.\n' + f'The following exchanges are available for: ' + f'{", ".join(available_exchanges())}' + ) + + if not is_exchange_known_ccxt(exchange): + raise OperationalException( + f'Exchange "{exchange}" is not known to the ccxt library ' + f'and therefore not available for the bot.\n' + f'The following exchanges are available: ' + f'{", ".join(available_exchanges())}' + ) + + if check_for_bad and is_exchange_bad(exchange): + raise OperationalException(f'Exchange "{exchange}" is known to not work with the bot yet. ' + f'Reason: {get_exchange_bad_reason(exchange)}') + + if is_exchange_officially_supported(exchange): + logger.info(f'Exchange "{exchange}" is officially supported ' + f'by FinRL.') + else: + logger.warning(f'Exchange "{exchange}" is known to the the ccxt library, ' + f'available for the bot, but not officially supported ' + f'by FinRL. ' + f'It may work flawlessly (please report back) or have serious issues. ' + f'Use it at your own discretion.') + + return True diff --git a/finrl/config/config_setup.py b/finrl/config/config_setup.py new file mode 100644 index 000000000..66c140977 --- /dev/null +++ b/finrl/config/config_setup.py @@ -0,0 +1,25 @@ +import logging +from typing import Any, Dict + +from finrl.state import RunMode + +from .check_exchange import remove_credentials +from .config_validation import validate_config_consistency +from .configuration import Configuration + +logger = logging.getLogger(__name__) + + +def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for utils subcommands + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args, method) + config = configuration.get_config() + # Ensure we do not use Exchange credentials + remove_credentials(config) + validate_config_consistency(config) + + return config diff --git a/finrl/config/config_validation.py b/finrl/config/config_validation.py new file mode 100644 index 000000000..4e74aba47 --- /dev/null +++ b/finrl/config/config_validation.py @@ -0,0 +1,157 @@ +import logging +from copy import deepcopy +from typing import Any, Dict + +from jsonschema import Draft4Validator, validators +from jsonschema.exceptions import ValidationError, best_match + +from finrl import constants +from finrl.exceptions import OperationalException +from finrl.state import RunMode + + +logger = logging.getLogger(__name__) + + +def _extend_validator(validator_class): + """ + Extended validator for the configuration JSON Schema. + Currently it only handles defaults for subschemas. + """ + validate_properties = validator_class.VALIDATORS['properties'] + + def set_defaults(validator, properties, instance, schema): + for prop, subschema in properties.items(): + if 'default' in subschema: + instance.setdefault(prop, subschema['default']) + + for error in validate_properties( + validator, properties, instance, schema, + ): + yield error + + return validators.extend( + validator_class, {'properties': set_defaults} + ) + + +FinrlValidator = _extend_validator(Draft4Validator) + + +def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: + """ + Validate the configuration follow the Config Schema + :param conf: Config in JSON format + :return: Returns the config if valid, otherwise throw an exception + """ + conf_schema = deepcopy(constants.CONF_SCHEMA) + if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE): + conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED + else: + conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED + try: + FinrlValidator(conf_schema).validate(conf) + return conf + except ValidationError as e: + logger.critical( + f"Invalid configuration. See config.json.example. Reason: {e}" + ) + raise ValidationError( + best_match(Draft4Validator(conf_schema).iter_errors(conf)).message + ) + + +def validate_config_consistency(conf: Dict[str, Any]) -> None: + """ + Validate the configuration consistency. + Should be ran after loading both configuration and strategy, + since strategies can set certain configuration settings too. + :param conf: Config in JSON format + :return: Returns None if everything is ok, otherwise throw an OperationalException + """ + + # validating trailing stoploss + _validate_trailing_stoploss(conf) + _validate_edge(conf) + _validate_whitelist(conf) + _validate_unlimited_amount(conf) + + # validate configuration before returning + logger.info('Validating configuration ...') + validate_config_schema(conf) + + +def _validate_unlimited_amount(conf: Dict[str, Any]) -> None: + """ + If edge is disabled, either max_open_trades or stake_amount need to be set. + :raise: OperationalException if config validation failed + """ + if (not conf.get('edge', {}).get('enabled') + and conf.get('max_open_trades') == float('inf') + and conf.get('stake_amount') == constants.UNLIMITED_STAKE_AMOUNT): + raise OperationalException("`max_open_trades` and `stake_amount` cannot both be unlimited.") + + +def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: + + if conf.get('stoploss') == 0.0: + raise OperationalException( + 'The config stoploss needs to be different from 0 to avoid problems with sell orders.' + ) + # Skip if trailing stoploss is not activated + if not conf.get('trailing_stop', False): + return + + tsl_positive = float(conf.get('trailing_stop_positive', 0)) + tsl_offset = float(conf.get('trailing_stop_positive_offset', 0)) + tsl_only_offset = conf.get('trailing_only_offset_is_reached', False) + + if tsl_only_offset: + if tsl_positive == 0.0: + raise OperationalException( + 'The config trailing_only_offset_is_reached needs ' + 'trailing_stop_positive_offset to be more than 0 in your config.') + if tsl_positive > 0 and 0 < tsl_offset <= tsl_positive: + raise OperationalException( + 'The config trailing_stop_positive_offset needs ' + 'to be greater than trailing_stop_positive in your config.') + + # Fetch again without default + if 'trailing_stop_positive' in conf and float(conf['trailing_stop_positive']) == 0.0: + raise OperationalException( + 'The config trailing_stop_positive needs to be different from 0 ' + 'to avoid problems with sell orders.' + ) + + +def _validate_edge(conf: Dict[str, Any]) -> None: + """ + Edge and Dynamic whitelist should not both be enabled, since edge overrides dynamic whitelists. + """ + + if not conf.get('edge', {}).get('enabled'): + return + + if conf.get('pairlist', {}).get('method') == 'VolumePairList': + raise OperationalException( + "Edge and VolumePairList are incompatible, " + "Edge will override whatever pairs VolumePairlist selects." + ) + if not conf.get('ask_strategy', {}).get('use_sell_signal', True): + raise OperationalException( + "Edge requires `use_sell_signal` to be True, otherwise no sells will happen." + ) + + +def _validate_whitelist(conf: Dict[str, Any]) -> None: + """ + Dynamic whitelist does not require pair_whitelist to be set - however StaticWhitelist does. + """ + if conf.get('runmode', RunMode.OTHER) in [RunMode.OTHER, RunMode.PLOT, + RunMode.UTIL_NO_EXCHANGE, RunMode.UTIL_EXCHANGE]: + return + + for pl in conf.get('pairlists', [{'method': 'StaticPairList'}]): + if (pl.get('method') == 'StaticPairList' + and not conf.get('exchange', {}).get('pair_whitelist')): + raise OperationalException("StaticPairList requires pair_whitelist to be set.") diff --git a/finrl/config/configuration.py b/finrl/config/configuration.py index d9144c485..6bf2e7fba 100644 --- a/finrl/config/configuration.py +++ b/finrl/config/configuration.py @@ -8,7 +8,10 @@ from typing import Any, Callable, Dict, List, Optional from finrl import constants +from finrl.config.check_exchange import check_exchange +from finrl.config.directory_operations import create_datadir, create_userdata_dir from finrl.config.load_config import load_config_file +from finrl.exceptions import OperationalException from finrl.loggers import setup_logging from finrl.misc import deep_merge_dicts, json_load from finrl.state import NON_UTIL_MODES, TRADING_MODES, RunMode @@ -16,6 +19,7 @@ logger = logging.getLogger(__name__) + class Configuration: """ Class to read and init the bot configuration @@ -91,10 +95,121 @@ def load_config(self) -> Dict[str, Any]: # Keep a copy of the original configuration file config['original_config'] = deepcopy(config) + self._process_logging_options(config) + + self._process_runmode(config) + + self._process_optimize_options(config) + + + # Check if the exchange set by the user is supported + check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) + self._resolve_pairs_list(config) return config + def _process_logging_options(self, config: Dict[str, Any]) -> None: + """ + Extract information for sys.argv and load logging configuration: + the -v/--verbose, --logfile options + """ + # Log level + config.update({'verbosity': self.args.get('verbosity', 0)}) + + if 'logfile' in self.args and self.args['logfile']: + config.update({'logfile': self.args['logfile']}) + + setup_logging(config) + + + def _process_datadir_options(self, config: Dict[str, Any]) -> None: + """ + Extract information for sys.argv and load directory configurations + --user-data, --datadir + """ + # Check exchange parameter here - otherwise `datadir` might be wrong. + if 'exchange' in self.args and self.args['exchange']: + config['exchange']['name'] = self.args['exchange'] + logger.info(f"Using exchange {config['exchange']['name']}") + + if 'pair_whitelist' not in config['exchange']: + config['exchange']['pair_whitelist'] = [] + + if 'user_data_dir' in self.args and self.args['user_data_dir']: + config.update({'user_data_dir': self.args['user_data_dir']}) + elif 'user_data_dir' not in config: + # Default to cwd/user_data (legacy option ...) + config.update({'user_data_dir': str(Path.cwd() / 'user_data')}) + + # reset to user_data_dir so this contains the absolute path. + config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False) + logger.info('Using user-data directory: %s ...', config['user_data_dir']) + + config.update({'datadir': create_datadir(config, self.args.get('datadir', None))}) + logger.info('Using data directory: %s ...', config.get('datadir')) + + if self.args.get('exportfilename'): + self._args_to_config(config, argname='exportfilename', + logstring='Storing backtest results to {} ...') + config['exportfilename'] = Path(config['exportfilename']) + else: + config['exportfilename'] = (config['user_data_dir'] + / 'backtest_results') + + def _process_optimize_options(self, config: Dict[str, Any]) -> None: + + # This will override the strategy configuration + self._args_to_config(config, argname='timeframes', + logstring='Parameter -i/--timeframes detected ... ' + 'Using timeframes: {} ...') + + self._args_to_config(config, argname='position_stacking', + logstring='Parameter --enable-position-stacking detected ...') + + self._args_to_config(config, argname='timerange', + logstring='Parameter --timerange detected: {} ...') + + self._process_datadir_options(config) + + self._args_to_config(config, argname='timeframes', + logstring='Overriding timeframe with Command line argument') + + def _process_runmode(self, config: Dict[str, Any]) -> None: + + self._args_to_config(config, argname='dry_run', + logstring='Parameter --dry-run detected, ' + 'overriding dry_run to: {} ...') + if not self.runmode: + # Handle real mode, infer dry/live from config + self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE + logger.info(f"Runmode set to {self.runmode.value}.") + + config.update({'runmode': self.runmode}) + + def _args_to_config(self, config: Dict[str, Any], argname: str, + logstring: str, logfun: Optional[Callable] = None, + deprecated_msg: Optional[str] = None) -> None: + """ + :param config: Configuration dictionary + :param argname: Argumentname in self.args - will be copied to config dict. + :param logstring: Logging String + :param logfun: logfun is applied to the configuration entry before passing + that entry to the log string using .format(). + sample: logfun=len (prints the length of the found + configuration instead of the content) + """ + if (argname in self.args and self.args[argname] is not None + and self.args[argname] is not False): + + config.update({argname: self.args[argname]}) + if logfun: + logger.info(logstring.format(logfun(config[argname]))) + else: + logger.info(logstring.format(config[argname])) + if deprecated_msg: + warnings.warn(f"DEPRECATED: {deprecated_msg}", DeprecationWarning) + def _resolve_pairs_list(self, config: Dict[str, Any]) -> None: """ Helper for download script. @@ -129,4 +244,4 @@ def _resolve_pairs_list(self, config: Dict[str, Any]) -> None: with pairs_file.open('r') as f: config['pairs'] = json_load(f) if 'pairs' in config: - config['pairs'].sort() \ No newline at end of file + config['pairs'].sort() diff --git a/finrl/config/directory_operations.py b/finrl/config/directory_operations.py new file mode 100644 index 000000000..f855259b3 --- /dev/null +++ b/finrl/config/directory_operations.py @@ -0,0 +1,77 @@ +import logging +import shutil +from pathlib import Path +from typing import Any, Dict, Optional + +from finrl.constants import USER_DATA_FILES +from finrl.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Path: + + folder = Path(datadir) if datadir else Path(f"{config['user_data_dir']}/data") + if not datadir: + # set datadir + exchange_name = config.get('exchange', {}).get('name').lower() + folder = folder.joinpath(exchange_name) + + if not folder.is_dir(): + folder.mkdir(parents=True) + logger.info(f'Created data directory: {datadir}') + return folder + + +def create_userdata_dir(directory: str, create_dir: bool = False) -> Path: + """ + Create userdata directory structure. + if create_dir is True, then the parent-directory will be created if it does not exist. + Sub-directories will always be created if the parent directory exists. + Raises OperationalException if given a non-existing directory. + :param directory: Directory to check + :param create_dir: Create directory if it does not exist. + :return: Path object containing the directory + """ + sub_dirs = ["backtest_results", "data", "logs", + "notebooks", "plot", "agents_trained", ] + folder = Path(directory) + if not folder.is_dir(): + if create_dir: + folder.mkdir(parents=True) + logger.info(f'Created user-data directory: {folder}') + else: + raise OperationalException( + f"Directory `{folder}` does not exist. " + "Please use `finrl create-userdir` to create a user directory") + + # Create required subdirectories + for f in sub_dirs: + subfolder = folder / f + if not subfolder.is_dir(): + subfolder.mkdir(parents=False) + return folder + + +def copy_sample_files(directory: Path, overwrite: bool = False) -> None: + """ + Copy files from templates to User data directory. + :param directory: Directory to copy data to + :param overwrite: Overwrite existing sample files + """ + if not directory.is_dir(): + raise OperationalException(f"Directory `{directory}` does not exist.") + sourcedir = Path(__file__).parents[1] / "templates" + for source, target in USER_DATA_FILES.items(): + targetdir = directory / target + if not targetdir.is_dir(): + raise OperationalException(f"Directory `{targetdir}` does not exist.") + targetfile = targetdir / source + if targetfile.exists(): + if not overwrite: + logger.warning(f"File `{targetfile}` exists already, not deploying sample file.") + continue + else: + logger.warning(f"File `{targetfile}` exists already, overwriting.") + shutil.copy(str(sourcedir / source), str(targetfile)) diff --git a/finrl/config/timerange.py b/finrl/config/timerange.py new file mode 100644 index 000000000..32bbd02a0 --- /dev/null +++ b/finrl/config/timerange.py @@ -0,0 +1,107 @@ +""" +This module contains the argument manager class +""" +import logging +import re +from typing import Optional + +import arrow + + +logger = logging.getLogger(__name__) + + +class TimeRange: + """ + object defining timerange inputs. + [start/stop]type defines if [start/stop]ts shall be used. + if *type is None, don't use corresponding startvalue. + """ + + def __init__(self, starttype: Optional[str] = None, stoptype: Optional[str] = None, + startts: int = 0, stopts: int = 0): + + self.starttype: Optional[str] = starttype + self.stoptype: Optional[str] = stoptype + self.startts: int = startts + self.stopts: int = stopts + + def __eq__(self, other): + """Override the default Equals behavior""" + return (self.starttype == other.starttype and self.stoptype == other.stoptype + and self.startts == other.startts and self.stopts == other.stopts) + + def subtract_start(self, seconds: int) -> None: + """ + Subtracts from startts if startts is set. + :param seconds: Seconds to subtract from starttime + :return: None (Modifies the object in place) + """ + if self.startts: + self.startts = self.startts - seconds + + def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int, + min_date: arrow.Arrow) -> None: + """ + Adjust startts by candles. + Applies only if no startup-candles have been available. + :param timeframe_secs: Timeframe in seconds e.g. `timeframe_to_seconds('5m')` + :param startup_candles: Number of candles to move start-date forward + :param min_date: Minimum data date loaded. Key kriterium to decide if start-time + has to be moved + :return: None (Modifies the object in place) + """ + if (not self.starttype or (startup_candles + and min_date.int_timestamp >= self.startts)): + # If no startts was defined, or backtest-data starts at the defined backtest-date + logger.warning("Moving start-date by %s candles to account for startup time.", + startup_candles) + self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles) + self.starttype = 'date' + + @staticmethod + def parse_timerange(text: Optional[str]) -> 'TimeRange': + """ + Parse the value of the argument --timerange to determine what is the range desired + :param text: value from --timerange + :return: Start and End range period + """ + if text is None: + return TimeRange(None, None, 0, 0) + syntax = [(r'^-(\d{8})$', (None, 'date')), + (r'^(\d{8})-$', ('date', None)), + (r'^(\d{8})-(\d{8})$', ('date', 'date')), + (r'^-(\d{10})$', (None, 'date')), + (r'^(\d{10})-$', ('date', None)), + (r'^(\d{10})-(\d{10})$', ('date', 'date')), + (r'^-(\d{13})$', (None, 'date')), + (r'^(\d{13})-$', ('date', None)), + (r'^(\d{13})-(\d{13})$', ('date', 'date')), + ] + for rex, stype in syntax: + # Apply the regular expression to text + match = re.match(rex, text) + if match: # Regex has matched + rvals = match.groups() + index = 0 + start: int = 0 + stop: int = 0 + if stype[0]: + starts = rvals[index] + if stype[0] == 'date' and len(starts) == 8: + start = arrow.get(starts, 'YYYYMMDD').int_timestamp + elif len(starts) == 13: + start = int(starts) // 1000 + else: + start = int(starts) + index += 1 + if stype[1]: + stops = rvals[index] + if stype[1] == 'date' and len(stops) == 8: + stop = arrow.get(stops, 'YYYYMMDD').int_timestamp + elif len(stops) == 13: + stop = int(stops) // 1000 + else: + stop = int(stops) + return TimeRange(stype[0], stype[1], start, stop) + raise Exception('Incorrect syntax for timerange "%s"' % text) diff --git a/finrl/data/__init__.py b/finrl/data/__init__.py new file mode 100644 index 000000000..b41193cba --- /dev/null +++ b/finrl/data/__init__.py @@ -0,0 +1,8 @@ +""" +Module to handle data operations +""" + +# limit what's imported when using `from finrl.data import *` +__all__ = [ + 'converter' +] diff --git a/finrl/data/btanalysis.py b/finrl/data/btanalysis.py new file mode 100644 index 000000000..86e9a980d --- /dev/null +++ b/finrl/data/btanalysis.py @@ -0,0 +1,382 @@ +""" +Helpers when analyzing backtest data +""" +import logging +from datetime import timezone +from pathlib import Path +from typing import Any, Dict, Optional, Tuple, Union + +import numpy as np +import pandas as pd + +from finrl.constants import LAST_BT_RESULT_FN +from finrl.misc import json_load +from finrl.persistence import Trade, init_db + + +logger = logging.getLogger(__name__) + +# must align with columns in backtest.py +BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration", + "open_rate", "close_rate", "open_at_end", "sell_reason"] + + +def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str: + """ + Get latest backtest export based on '.last_result.json'. + :param directory: Directory to search for last result + :param variant: 'backtest' or 'hyperopt' - the method to return + :return: string containing the filename of the latest backtest result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + if isinstance(directory, str): + directory = Path(directory) + if not directory.is_dir(): + raise ValueError(f"Directory '{directory}' does not exist.") + filename = directory / LAST_BT_RESULT_FN + + if not filename.is_file(): + raise ValueError( + f"Directory '{directory}' does not seem to contain backtest statistics yet.") + + with filename.open() as file: + data = json_load(file) + + if f'latest_{variant}' not in data: + raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.") + + return data[f'latest_{variant}'] + + +def get_latest_backtest_filename(directory: Union[Path, str]) -> str: + """ + Get latest backtest export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest backtest result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + return get_latest_optimize_filename(directory, 'backtest') + + +def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str: + """ + Get latest hyperopt export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest hyperopt result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + try: + return get_latest_optimize_filename(directory, 'hyperopt') + except ValueError: + # Return default (legacy) pickle filename + return 'hyperopt_results.pickle' + + +def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str = None) -> Path: + """ + Get latest hyperopt export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest hyperopt result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + if isinstance(directory, str): + directory = Path(directory) + if predef_filename: + return directory / predef_filename + return directory / get_latest_hyperopt_filename(directory) + + +def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: + """ + Load backtest statistics file. + :param filename: pathlib.Path object, or string pointing to the file. + :return: a dictionary containing the resulting file. + """ + if isinstance(filename, str): + filename = Path(filename) + if filename.is_dir(): + filename = filename / get_latest_backtest_filename(filename) + if not filename.is_file(): + raise ValueError(f"File {filename} does not exist.") + logger.info(f"Loading backtest result from {filename}") + with filename.open() as file: + data = json_load(file) + + return data + + +def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame: + """ + Load backtest data file. + :param filename: pathlib.Path object, or string pointing to a file or directory + :param strategy: Strategy to load - mainly relevant for multi-strategy backtests + Can also serve as protection to load the correct result. + :return: a dataframe with the analysis results + :raise: ValueError if loading goes wrong. + """ + data = load_backtest_stats(filename) + if not isinstance(data, list): + # new, nested format + if 'strategy' not in data: + raise ValueError("Unknown dataformat.") + + if not strategy: + if len(data['strategy']) == 1: + strategy = list(data['strategy'].keys())[0] + else: + raise ValueError("Detected backtest result with more than one strategy. " + "Please specify a strategy.") + + if strategy not in data['strategy']: + raise ValueError(f"Strategy {strategy} not available in the backtest result.") + + data = data['strategy'][strategy]['trades'] + df = pd.DataFrame(data) + df['open_date'] = pd.to_datetime(df['open_date'], + utc=True, + infer_datetime_format=True + ) + df['close_date'] = pd.to_datetime(df['close_date'], + utc=True, + infer_datetime_format=True + ) + else: + # old format - only with lists. + df = pd.DataFrame(data, columns=BT_DATA_COLUMNS) + + df['open_date'] = pd.to_datetime(df['open_date'], + unit='s', + utc=True, + infer_datetime_format=True + ) + df['close_date'] = pd.to_datetime(df['close_date'], + unit='s', + utc=True, + infer_datetime_format=True + ) + df['profit_abs'] = df['close_rate'] - df['open_rate'] + df = df.sort_values("open_date").reset_index(drop=True) + return df + + +def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataFrame: + """ + Find overlapping trades by expanding each trade once per period it was open + and then counting overlaps. + :param results: Results Dataframe - can be loaded + :param timeframe: Timeframe used for backtest + :return: dataframe with open-counts per time-period in timeframe + """ + from freqtrade.exchange import timeframe_to_minutes + timeframe_min = timeframe_to_minutes(timeframe) + dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'], + freq=f"{timeframe_min}min")) + for row in results[['open_date', 'close_date']].iterrows()] + deltas = [len(x) for x in dates] + dates = pd.Series(pd.concat(dates).values, name='date') + df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns) + + df2 = pd.concat([dates, df2], axis=1) + df2 = df2.set_index('date') + df_final = df2.resample(f"{timeframe_min}min")[['pair']].count() + df_final = df_final.rename({'pair': 'open_trades'}, axis=1) + return df_final + + +def evaluate_result_multi(results: pd.DataFrame, timeframe: str, + max_open_trades: int) -> pd.DataFrame: + """ + Find overlapping trades by expanding each trade once per period it was open + and then counting overlaps + :param results: Results Dataframe - can be loaded + :param timeframe: Frequency used for the backtest + :param max_open_trades: parameter max_open_trades used during backtest run + :return: dataframe with open-counts per time-period in freq + """ + df_final = analyze_trade_parallelism(results, timeframe) + return df_final[df_final['open_trades'] > max_open_trades] + + +def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame: + """ + Load trades from a DB (using dburl) + :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite) + :param strategy: Strategy to load - mainly relevant for multi-strategy backtests + Can also serve as protection to load the correct result. + :return: Dataframe containing Trades + """ + init_db(db_url, clean_open_orders=False) + + columns = ["pair", "open_date", "close_date", "profit", "profit_percent", + "open_rate", "close_rate", "amount", "trade_duration", "sell_reason", + "fee_open", "fee_close", "open_rate_requested", "close_rate_requested", + "stake_amount", "max_rate", "min_rate", "id", "exchange", + "stop_loss", "initial_stop_loss", "strategy", "timeframe"] + + filters = [] + if strategy: + filters.append(Trade.strategy == strategy) + + trades = pd.DataFrame([(t.pair, + t.open_date.replace(tzinfo=timezone.utc), + t.close_date.replace(tzinfo=timezone.utc) if t.close_date else None, + t.calc_profit(), t.calc_profit_ratio(), + t.open_rate, t.close_rate, t.amount, + (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2) + if t.close_date else None), + t.sell_reason, + t.fee_open, t.fee_close, + t.open_rate_requested, + t.close_rate_requested, + t.stake_amount, + t.max_rate, + t.min_rate, + t.id, t.exchange, + t.stop_loss, t.initial_stop_loss, + t.strategy, t.timeframe + ) + for t in Trade.get_trades(filters).all()], + columns=columns) + + return trades + + +def load_trades(source: str, db_url: str, exportfilename: Path, + no_trades: bool = False, strategy: Optional[str] = None) -> pd.DataFrame: + """ + Based on configuration option 'trade_source': + * loads data from DB (using `db_url`) + * loads data from backtestfile (using `exportfilename`) + :param source: "DB" or "file" - specify source to load from + :param db_url: sqlalchemy formatted url to a database + :param exportfilename: Json file generated by backtesting + :param no_trades: Skip using trades, only return backtesting data columns + :return: DataFrame containing trades + """ + if no_trades: + df = pd.DataFrame(columns=BT_DATA_COLUMNS) + return df + + if source == "DB": + return load_trades_from_db(db_url) + elif source == "file": + return load_backtest_data(exportfilename, strategy) + + +def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame, + date_index=False) -> pd.DataFrame: + """ + Compare trades and backtested pair DataFrames to get trades performed on backtested period + :return: the DataFrame of a trades of period + """ + if date_index: + trades_start = dataframe.index[0] + trades_stop = dataframe.index[-1] + else: + trades_start = dataframe.iloc[0]['date'] + trades_stop = dataframe.iloc[-1]['date'] + trades = trades.loc[(trades['open_date'] >= trades_start) & + (trades['close_date'] <= trades_stop)] + return trades + + +def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: + """ + Calculate market change based on "column". + Calculation is done by taking the first non-null and the last non-null element of each column + and calculating the pctchange as "(last - first) / first". + Then the results per pair are combined as mean. + + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: + """ + tmp_means = [] + for pair, df in data.items(): + start = df[column].dropna().iloc[0] + end = df[column].dropna().iloc[-1] + tmp_means.append((end - start) / start) + + return np.mean(tmp_means) + + +def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], + column: str = "close") -> pd.DataFrame: + """ + Combine multiple dataframes "column" + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: DataFrame with the column renamed to the dict key, and a column + named mean, containing the mean of all pairs. + """ + df_comb = pd.concat([data[pair].set_index('date').rename( + {column: pair}, axis=1)[pair] for pair in data], axis=1) + + df_comb['mean'] = df_comb.mean(axis=1) + + return df_comb + + +def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, + timeframe: str) -> pd.DataFrame: + """ + Adds a column `col_name` with the cumulative profit for the given trades array. + :param df: DataFrame with date index + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :param col_name: Column name that will be assigned the results + :param timeframe: Timeframe used during the operations + :return: Returns df with one additional column, col_name, containing the cumulative profit. + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + from freqtrade.exchange import timeframe_to_minutes + timeframe_minutes = timeframe_to_minutes(timeframe) + # Resample to timeframe to make sure trades match candles + _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' + )[['profit_percent']].sum() + df.loc[:, col_name] = _trades_sum.cumsum() + # Set first value to 0 + df.loc[df.iloc[0].name, col_name] = 0 + # FFill to get continuous + df[col_name] = df[col_name].ffill() + return df + + +def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', + value_col: str = 'profit_percent' + ) -> Tuple[float, pd.Timestamp, pd.Timestamp]: + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') + :param value_col: Column in DataFrame to use for values (defaults to 'profit_percent') + :return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = pd.DataFrame() + max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() + max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() + max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + + idxmin = max_drawdown_df['drawdown'].idxmin() + if idxmin == 0: + raise ValueError("No losing trade, therefore no drawdown.") + high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] + low_date = profit_results.loc[idxmin, date_col] + return abs(min(max_drawdown_df['drawdown'])), high_date, low_date diff --git a/finrl/data/converter.py b/finrl/data/converter.py new file mode 100644 index 000000000..fed9d248a --- /dev/null +++ b/finrl/data/converter.py @@ -0,0 +1,264 @@ +""" +Functions to convert data from one format to another +""" +import itertools +import logging +from datetime import datetime, timezone +from operator import itemgetter +from typing import Any, Dict, List + +import pandas as pd +from pandas import DataFrame, to_datetime + +from finrl.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList + + +logger = logging.getLogger(__name__) + + +def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *, + fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame: + """ + Converts a list with candle (OHLCV) data (in format returned by ccxt.fetch_ohlcv) + to a Dataframe + :param ohlcv: list with candle (OHLCV) data, as returned by exchange.async_get_candle_history + :param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data + :param pair: Pair this data is for (used to warn if fillup was necessary) + :param fill_missing: fill up missing candles with 0 candles + (see ohlcv_fill_up_missing_data for details) + :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete + :return: DataFrame + """ + logger.debug(f"Converting candle (OHLCV) data to dataframe for pair {pair}.") + cols = DEFAULT_DATAFRAME_COLUMNS + df = DataFrame(ohlcv, columns=cols) + + df['date'] = to_datetime(df['date'], unit='ms', utc=True, infer_datetime_format=True) + + # Some exchanges return int values for Volume and even for OHLC. + # Convert them since TA-LIB indicators used in the strategy assume floats + # and fail with exception... + df = df.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float', + 'volume': 'float'}) + return clean_ohlcv_dataframe(df, timeframe, pair, + fill_missing=fill_missing, + drop_incomplete=drop_incomplete) + + +def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, + fill_missing: bool = True, + drop_incomplete: bool = True) -> DataFrame: + """ + Clense a OHLCV dataframe by + * Grouping it by date (removes duplicate tics) + * dropping last candles if requested + * Filling up missing data (if requested) + :param data: DataFrame containing candle (OHLCV) data. + :param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data + :param pair: Pair this data is for (used to warn if fillup was necessary) + :param fill_missing: fill up missing candles with 0 candles + (see ohlcv_fill_up_missing_data for details) + :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete + :return: DataFrame + """ + # group by index and aggregate results to eliminate duplicate ticks + data = data.groupby(by='date', as_index=False, sort=True).agg({ + 'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'max', + }) + # eliminate partial candle + if drop_incomplete: + data.drop(data.tail(1).index, inplace=True) + logger.debug('Dropping last candle') + + if fill_missing: + return ohlcv_fill_up_missing_data(data, timeframe, pair) + else: + return data + + +def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) -> DataFrame: + """ + Fills up missing data with 0 volume rows, + using the previous close as price for "open", "high" "low" and "close", volume is set to 0 + + """ + from freqtrade.exchange import timeframe_to_minutes + + ohlcv_dict = { + 'open': 'first', + 'high': 'max', + 'low': 'min', + 'close': 'last', + 'volume': 'sum' + } + timeframe_minutes = timeframe_to_minutes(timeframe) + # Resample to create "NAN" values + df = dataframe.resample(f'{timeframe_minutes}min', on='date').agg(ohlcv_dict) + + # Forwardfill close for missing columns + df['close'] = df['close'].fillna(method='ffill') + # Use close for "open, high, low" + df.loc[:, ['open', 'high', 'low']] = df[['open', 'high', 'low']].fillna( + value={'open': df['close'], + 'high': df['close'], + 'low': df['close'], + }) + df.reset_index(inplace=True) + len_before = len(dataframe) + len_after = len(df) + if len_before != len_after: + logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}") + return df + + +def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date') -> DataFrame: + """ + Trim dataframe based on given timerange + :param df: Dataframe to trim + :param timerange: timerange (use start and end date if available) + :param: df_date_col: Column in the dataframe to use as Date column + :return: trimmed dataframe + """ + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + df = df.loc[df[df_date_col] >= start, :] + if timerange.stoptype == 'date': + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) + df = df.loc[df[df_date_col] <= stop, :] + return df + + +def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: + """ + TODO: This should get a dedicated test + Gets order book list, returns dataframe with below format per suggested by creslin + ------------------------------------------------------------------- + b_sum b_size bids asks a_size a_sum + ------------------------------------------------------------------- + """ + cols = ['bids', 'b_size'] + + bids_frame = DataFrame(bids, columns=cols) + # add cumulative sum column + bids_frame['b_sum'] = bids_frame['b_size'].cumsum() + cols2 = ['asks', 'a_size'] + asks_frame = DataFrame(asks, columns=cols2) + # add cumulative sum column + asks_frame['a_sum'] = asks_frame['a_size'].cumsum() + + frame = pd.concat([bids_frame['b_sum'], bids_frame['b_size'], bids_frame['bids'], + asks_frame['asks'], asks_frame['a_size'], asks_frame['a_sum']], axis=1, + keys=['b_sum', 'b_size', 'bids', 'asks', 'a_size', 'a_sum']) + # logger.info('order book %s', frame ) + return frame + + +def trades_remove_duplicates(trades: List[List]) -> List[List]: + """ + Removes duplicates from the trades list. + Uses itertools.groupby to avoid converting to pandas. + Tests show it as being pretty efficient on lists of 4M Lists. + :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns + :return: same format as above, but with duplicates removed + """ + return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))] + + +def trades_dict_to_list(trades: List[Dict]) -> TradeList: + """ + Convert fetch_trades result into a List (to be more memory efficient). + :param trades: List of trades, as returned by ccxt.fetch_trades. + :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns + """ + return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] + + +def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame: + """ + Converts trades list to OHLCV list + :param trades: List of trades, as returned by ccxt.fetch_trades. + :param timeframe: Timeframe to resample data to + :return: OHLCV Dataframe. + :raises: ValueError if no trades are provided + """ + from freqtrade.exchange import timeframe_to_minutes + timeframe_minutes = timeframe_to_minutes(timeframe) + if not trades: + raise ValueError('Trade-list empty.') + df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', + utc=True,) + df = df.set_index('timestamp') + + df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc() + df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum() + df_new['date'] = df_new.index + # Drop 0 volume rows + df_new = df_new.dropna() + return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS] + + +def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert trades from one format to another format. + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + + if 'pairs' not in config: + config['pairs'] = src.trades_get_pairs(config['datadir']) + logger.info(f"Converting trades for {config['pairs']}") + + for pair in config['pairs']: + data = src.trades_load(pair=pair) + logger.info(f"Converting {len(data)} trades for {pair}") + trg.trades_store(pair, data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source Trade data for {pair}.") + src.trades_purge(pair=pair) + + +def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert OHLCV from one format to another + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + timeframes = config.get('timeframes', [config.get('timeframe')]) + logger.info(f"Converting candle (OHLCV) for timeframe {timeframes}") + + if 'pairs' not in config: + config['pairs'] = [] + # Check timeframes or fall back to timeframe. + for timeframe in timeframes: + config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'], + timeframe)) + logger.info(f"Converting candle (OHLCV) data for {config['pairs']}") + + for timeframe in timeframes: + for pair in config['pairs']: + data = src.ohlcv_load(pair=pair, timeframe=timeframe, + timerange=None, + fill_missing=False, + drop_incomplete=False, + startup_candles=0) + logger.info(f"Converting {len(data)} candles for {pair}") + if len(data) > 0: + trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source data for {pair} / {timeframe}") + src.ohlcv_purge(pair=pair, timeframe=timeframe) diff --git a/finrl/data/dataprovider.py b/finrl/data/dataprovider.py new file mode 100644 index 000000000..9b545c246 --- /dev/null +++ b/finrl/data/dataprovider.py @@ -0,0 +1,182 @@ +""" +Dataprovider +Responsible to provide data to the bot +including ticker and orderbook data, live and historical candle (OHLCV) data +Common Interface for bot and strategy to access data. +""" +import logging +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple + +from pandas import DataFrame + +from finrl.constants import ListPairsWithTimeframes, PairWithTimeframe +from finrl.data.history import load_pair_history +from finrl.exceptions import ExchangeError, OperationalException +from finrl.exchange import Exchange +from finrl.state import RunMode + + +logger = logging.getLogger(__name__) + + +class DataProvider: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: + self._config = config + self._exchange = exchange + self._pairlists = pairlists + self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} + + def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None: + """ + Store cached Dataframe. + Using private method as this should never be used by a user + (but the class is exposed via `self.dp` to the strategy) + :param pair: pair to get the data for + :param timeframe: Timeframe to get data for + :param dataframe: analyzed dataframe + """ + self.__cached_pairs[(pair, timeframe)] = (dataframe, datetime.now(timezone.utc)) + + def add_pairlisthandler(self, pairlists) -> None: + """ + Allow adding pairlisthandler after initialization + """ + self._pairlists = pairlists + + def refresh( + self, + pairlist: ListPairsWithTimeframes, + helping_pairs: ListPairsWithTimeframes = None, + ) -> None: + """ + Refresh data, called with each cycle + """ + if helping_pairs: + self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) + else: + self._exchange.refresh_latest_ohlcv(pairlist) + + @property + def available_pairs(self) -> ListPairsWithTimeframes: + """ + Return a list of tuples containing (pair, timeframe) for which data is currently cached. + Should be whitelist + open trades. + """ + return list(self._exchange._klines.keys()) + + def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: + """ + Get candle (OHLCV) data for the given pair as DataFrame + Please use the `available_pairs` method to verify which pairs are currently cached. + :param pair: pair to get the data for + :param timeframe: Timeframe to get data for + :param copy: copy dataframe before returning if True. + Use False only for read-only operations (where the dataframe is not modified) + """ + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + return self._exchange.klines( + (pair, timeframe or self._config["timeframe"]), copy=copy + ) + else: + return DataFrame() + + def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame: + """ + Get stored historical candle (OHLCV) data + :param pair: pair to get the data for + :param timeframe: timeframe to get data for + """ + return load_pair_history( + pair=pair, + timeframe=timeframe or self._config["timeframe"], + datadir=self._config["datadir"], + data_format=self._config.get("dataformat_ohlcv", "json"), + ) + + def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame: + """ + Return pair candle (OHLCV) data, either live or cached historical -- depending + on the runmode. + :param pair: pair to get the data for + :param timeframe: timeframe to get data for + :return: Dataframe for this pair + """ + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + # Get live OHLCV data. + data = self.ohlcv(pair=pair, timeframe=timeframe) + else: + # Get historical OHLCV data (cached on disk). + data = self.historic_ohlcv(pair=pair, timeframe=timeframe) + if len(data) == 0: + logger.warning(f"No data found for ({pair}, {timeframe}).") + return data + + def get_analyzed_dataframe( + self, pair: str, timeframe: str + ) -> Tuple[DataFrame, datetime]: + """ + :param pair: pair to get the data for + :param timeframe: timeframe to get data for + :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe + combination. + Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. + """ + if (pair, timeframe) in self.__cached_pairs: + return self.__cached_pairs[(pair, timeframe)] + else: + + return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) + + def market(self, pair: str) -> Optional[Dict[str, Any]]: + """ + Return market data for the pair + :param pair: Pair to get the data for + :return: Market data dict from ccxt or None if market info is not available for the pair + """ + return self._exchange.markets.get(pair) + + def ticker(self, pair: str): + """ + Return last ticker data from exchange + :param pair: Pair to get the data for + :return: Ticker dict from exchange or empty dict if ticker is not available for the pair + """ + try: + return self._exchange.fetch_ticker(pair) + except ExchangeError: + return {} + + def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: + """ + Fetch latest l2 orderbook data + Warning: Does a network request - so use with common sense. + :param pair: pair to get the data for + :param maximum: Maximum number of orderbook entries to query + :return: dict including bids/asks with a total of `maximum` entries. + """ + return self._exchange.fetch_l2_order_book(pair, maximum) + + @property + def runmode(self) -> RunMode: + """ + Get runmode of the bot + can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". + """ + return RunMode(self._config.get("runmode", RunMode.OTHER)) + + def current_whitelist(self) -> List[str]: + """ + fetch latest available whitelist. + + Useful when you have a large whitelist and need to call each pair as an informative pair. + As available pairs does not show whitelist until after informative pairs have been cached. + :return: list of pairs in whitelist + """ + + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException( + "Dataprovider was not initialized with a pairlist provider." + ) diff --git a/finrl/data/history/__init__.py b/finrl/data/history/__init__.py new file mode 100644 index 000000000..107f9c401 --- /dev/null +++ b/finrl/data/history/__init__.py @@ -0,0 +1,12 @@ +""" +Handle historic data (ohlcv). + +Includes: +* load data for a pair (or a list of pairs) from disk +* download data from exchange and store to disk +""" +# flake8: noqa: F401 +from .history_utils import (convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, + refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, + validate_backtest_data) +from .idatahandler import get_datahandler diff --git a/finrl/data/history/hdf5datahandler.py b/finrl/data/history/hdf5datahandler.py new file mode 100644 index 000000000..c2432b79f --- /dev/null +++ b/finrl/data/history/hdf5datahandler.py @@ -0,0 +1,213 @@ +import logging +import re +from pathlib import Path +from typing import List, Optional + +import numpy as np +import pandas as pd + +from finrl import misc +from finrl.config import TimeRange +from finrl.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, + ListPairsWithTimeframes, TradeList) + +from .idatahandler import IDataHandler + + +logger = logging.getLogger(__name__) + + +class HDF5DataHandler(IDataHandler): + + _columns = DEFAULT_DATAFRAME_COLUMNS + + @classmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Tuples of (pair, timeframe) + """ + _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.h5)', p.name) + for p in datadir.glob("*.h5")] + return [(match[1].replace('_', '/'), match[2]) for match in _tmp + if match and len(match.groups()) > 1] + + @classmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.h5)', p.name) + for p in datadir.glob(f"*{timeframe}.h5")] + # Check if regex found something and only return these results + return [match[0].replace('_', '/') for match in _tmp if match] + + def ohlcv_store(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + """ + Store data in hdf5 file. + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + key = self._pair_ohlcv_key(pair, timeframe) + _data = data.copy() + + filename = self._pair_data_filename(self._datadir, pair, timeframe) + + ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc') + ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date']) + + ds.close() + + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None) -> pd.DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversion to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + key = self._pair_ohlcv_key(pair, timeframe) + filename = self._pair_data_filename(self._datadir, pair, timeframe) + + if not filename.exists(): + return pd.DataFrame(columns=self._columns) + where = [] + if timerange: + if timerange.starttype == 'date': + where.append(f"date >= Timestamp({timerange.startts * 1e9})") + if timerange.stoptype == 'date': + where.append(f"date < Timestamp({timerange.stopts * 1e9})") + + pairdata = pd.read_hdf(filename, key=key, mode="r", where=where) + + if list(pairdata.columns) != self._columns: + raise ValueError("Wrong dataframe format") + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) + return pairdata + + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.exists(): + filename.unlink() + return True + return False + + def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + raise NotImplementedError() + + @classmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + _tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name) + for p in datadir.glob("*trades.h5")] + # Check if regex found something and only return these results to avoid exceptions. + return [match[0].replace('_', '/') for match in _tmp if match] + + def trades_store(self, pair: str, data: TradeList) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + key = self._pair_trades_key(pair) + + ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair), + mode='a', complevel=9, complib='blosc') + ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS), + format='table', data_columns=['timestamp']) + ds.close() + + def trades_append(self, pair: str, data: TradeList): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + raise NotImplementedError() + + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from h5 file. + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + key = self._pair_trades_key(pair) + filename = self._pair_trades_filename(self._datadir, pair) + + if not filename.exists(): + return [] + where = [] + if timerange: + if timerange.starttype == 'date': + where.append(f"timestamp >= {timerange.startts * 1e3}") + if timerange.stoptype == 'date': + where.append(f"timestamp < {timerange.stopts * 1e3}") + + trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where) + trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None}) + return trades.values.tolist() + + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.exists(): + filename.unlink() + return True + return False + + @classmethod + def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str: + return f"{pair}/ohlcv/tf_{timeframe}" + + @classmethod + def _pair_trades_key(cls, pair: str) -> str: + return f"{pair}/trades" + + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5') + return filename + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-trades.h5') + return filename diff --git a/finrl/data/history/history_utils.py b/finrl/data/history/history_utils.py new file mode 100644 index 000000000..b8a9b2d38 --- /dev/null +++ b/finrl/data/history/history_utils.py @@ -0,0 +1,397 @@ +import logging +import operator +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import arrow +from pandas import DataFrame + +from finrl.config import TimeRange +from finrl.constants import DEFAULT_DATAFRAME_COLUMNS +from finrl.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, + trades_remove_duplicates, trades_to_ohlcv) +from finrl.data.history.idatahandler import IDataHandler, get_datahandler +from finrl.exceptions import OperationalException +from finrl.exchange import Exchange +from finrl.misc import format_ms_time + + +logger = logging.getLogger(__name__) + + +def load_pair_history(pair: str, + timeframe: str, + datadir: Path, *, + timerange: Optional[TimeRange] = None, + fill_up_missing: bool = True, + drop_incomplete: bool = True, + startup_candles: int = 0, + data_format: str = None, + data_handler: IDataHandler = None, + ) -> DataFrame: + """ + Load cached ohlcv history for the given pair. + + :param pair: Pair to load data for + :param timeframe: Timeframe (e.g. "5m") + :param datadir: Path to the data storage location. + :param data_format: Format of the data. Ignored if data_handler is set. + :param timerange: Limit data to be loaded to this timerange + :param fill_up_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period + :param data_handler: Initialized data-handler to use. + Will be initialized from data_format if not set + :return: DataFrame with ohlcv data, or empty DataFrame + """ + data_handler = get_datahandler(datadir, data_format, data_handler) + + return data_handler.ohlcv_load(pair=pair, + timeframe=timeframe, + timerange=timerange, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete, + startup_candles=startup_candles, + ) + + +def load_data(datadir: Path, + timeframe: str, + pairs: List[str], *, + timerange: Optional[TimeRange] = None, + fill_up_missing: bool = True, + startup_candles: int = 0, + fail_without_data: bool = False, + data_format: str = 'json', + ) -> Dict[str, DataFrame]: + """ + Load ohlcv history data for a list of pairs. + + :param datadir: Path to the data storage location. + :param timeframe: Timeframe (e.g. "5m") + :param pairs: List of pairs to load + :param timerange: Limit data to be loaded to this timerange + :param fill_up_missing: Fill missing values with "No action"-candles + :param startup_candles: Additional candles to load at the start of the period + :param fail_without_data: Raise OperationalException if no data is found. + :param data_format: Data format which should be used. Defaults to json + :return: dict(:) + """ + result: Dict[str, DataFrame] = {} + if startup_candles > 0 and timerange: + logger.info(f'Using indicator startup period: {startup_candles} ...') + + data_handler = get_datahandler(datadir, data_format) + + for pair in pairs: + hist = load_pair_history(pair=pair, timeframe=timeframe, + datadir=datadir, timerange=timerange, + fill_up_missing=fill_up_missing, + startup_candles=startup_candles, + data_handler=data_handler + ) + if not hist.empty: + result[pair] = hist + + if fail_without_data and not result: + raise OperationalException("No data found. Terminating.") + return result + + +def refresh_data(datadir: Path, + timeframe: str, + pairs: List[str], + exchange: Exchange, + data_format: str = None, + timerange: Optional[TimeRange] = None, + ) -> None: + """ + Refresh ohlcv history data for a list of pairs. + + :param datadir: Path to the data storage location. + :param timeframe: Timeframe (e.g. "5m") + :param pairs: List of pairs to load + :param exchange: Exchange object + :param timerange: Limit data to be loaded to this timerange + """ + data_handler = get_datahandler(datadir, data_format) + for pair in pairs: + _download_pair_history(pair=pair, timeframe=timeframe, + datadir=datadir, timerange=timerange, + exchange=exchange, data_handler=data_handler) + + +def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], + data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: + """ + Load cached data to download more data. + If timerange is passed in, checks whether data from an before the stored data will be + downloaded. + If that's the case then what's available should be completely overwritten. + Otherwise downloads always start at the end of the available data to avoid data gaps. + Note: Only used by download_pair_history(). + """ + start = None + if timerange: + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + + # Intentionally don't pass timerange in - since we need to load the full dataset. + data = data_handler.ohlcv_load(pair, timeframe=timeframe, + timerange=None, fill_missing=False, + drop_incomplete=True, warn_no_data=False) + if not data.empty: + if start and start < data.iloc[0]['date']: + # Earlier data than existing data requested, redownload all + data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) + else: + start = data.iloc[-1]['date'] + + start_ms = int(start.timestamp() * 1000) if start else None + return data, start_ms + + +def _download_pair_history(datadir: Path, + exchange: Exchange, + pair: str, *, + timeframe: str = '5m', + timerange: Optional[TimeRange] = None, + data_handler: IDataHandler = None) -> bool: + """ + Download latest candles from the exchange for the pair and timeframe passed in parameters + The data is downloaded starting from the last correct data that + exists in a cache. If timerange starts earlier than the data in the cache, + the full data will be redownloaded + + Based on @Rybolov work: https://github.com/rybolov/freqtrade-data + + :param pair: pair to download + :param timeframe: Timeframe (e.g "5m") + :param timerange: range of time to download + :return: bool with success state + """ + data_handler = get_datahandler(datadir, data_handler=data_handler) + + try: + logger.info( + f'Download history data for pair: "{pair}", timeframe: {timeframe} ' + f'and store in {datadir}.' + ) + + # data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) + data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, + data_handler=data_handler) + + logger.debug("Current Start: %s", + f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + logger.debug("Current End: %s", + f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + + # Default since_ms to 30 days if nothing is given + new_data = exchange.get_historic_ohlcv(pair=pair, + timeframe=timeframe, + since_ms=since_ms if since_ms else + int(arrow.utcnow().shift( + days=-30).float_timestamp) * 1000 + ) + # TODO: Maybe move parsing to exchange class (?) + new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, + fill_missing=False, drop_incomplete=True) + if data.empty: + data = new_dataframe + else: + # Run cleaning again to ensure there were no duplicate candles + # Especially between existing and new data. + data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair, + fill_missing=False, drop_incomplete=False) + + logger.debug("New Start: %s", + f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + logger.debug("New End: %s", + f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + + data_handler.ohlcv_store(pair, timeframe, data=data) + return True + + except Exception: + logger.exception( + f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}.' + ) + return False + + +def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], + datadir: Path, timerange: Optional[TimeRange] = None, + erase: bool = False, data_format: str = None) -> List[str]: + """ + Refresh stored ohlcv data for backtesting and hyperopt operations. + :return: List of pairs that are not available. + """ + pairs_not_available = [] + data_handler = get_datahandler(datadir, data_format) + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for timeframe in timeframes: + + if erase: + if data_handler.ohlcv_purge(pair, timeframe): + logger.info( + f'Deleting existing data for pair {pair}, interval {timeframe}.') + + logger.info(f'Downloading pair {pair}, interval {timeframe}.') + _download_pair_history(datadir=datadir, exchange=exchange, + pair=pair, timeframe=str(timeframe), + timerange=timerange, data_handler=data_handler) + return pairs_not_available + + +def _download_trades_history(exchange: Exchange, + pair: str, *, + timerange: Optional[TimeRange] = None, + data_handler: IDataHandler + ) -> bool: + """ + Download trade history from the exchange. + Appends to previously downloaded trades data. + """ + try: + + since = timerange.startts * 1000 if \ + (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift( + days=-30).float_timestamp) * 1000 + + trades = data_handler.trades_load(pair) + + # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + + if trades and since < trades[0][0]: + # since is before the first trade + logger.info(f"Start earlier than available data. Redownloading trades for {pair}...") + trades = [] + + from_id = trades[-1][1] if trades else None + if trades and since < trades[-1][0]: + # Reset since to the last available point + # - 5 seconds (to ensure we're getting all trades) + since = trades[-1][0] - (5 * 1000) + logger.info(f"Using last trade date -5s - Downloading trades for {pair} " + f"since: {format_ms_time(since)}.") + + logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}") + logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}") + logger.info(f"Current Amount of trades: {len(trades)}") + + # Default since_ms to 30 days if nothing is given + new_trades = exchange.get_historic_trades(pair=pair, + since=since, + from_id=from_id, + ) + trades.extend(new_trades[1]) + # Remove duplicates to make sure we're not storing data we don't need + trades = trades_remove_duplicates(trades) + data_handler.trades_store(pair, data=trades) + + logger.debug(f"New Start: {format_ms_time(trades[0][0])}") + logger.debug(f"New End: {format_ms_time(trades[-1][0])}") + logger.info(f"New Amount of trades: {len(trades)}") + return True + + except Exception: + logger.exception( + f'Failed to download historic trades for pair: "{pair}". ' + ) + return False + + +def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, + timerange: TimeRange, erase: bool = False, + data_format: str = 'jsongz') -> List[str]: + """ + Refresh stored trades data for backtesting and hyperopt operations. + :return: List of pairs that are not available. + """ + pairs_not_available = [] + data_handler = get_datahandler(datadir, data_format=data_format) + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + + if erase: + if data_handler.trades_purge(pair): + logger.info(f'Deleting existing data for pair {pair}.') + + logger.info(f'Downloading trades for pair {pair}.') + _download_trades_history(exchange=exchange, + pair=pair, + timerange=timerange, + data_handler=data_handler) + return pairs_not_available + + +def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], + datadir: Path, timerange: TimeRange, erase: bool = False, + data_format_ohlcv: str = 'json', + data_format_trades: str = 'jsongz') -> None: + """ + Convert stored trades data to ohlcv data + """ + data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) + data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) + + for pair in pairs: + trades = data_handler_trades.trades_load(pair) + for timeframe in timeframes: + if erase: + if data_handler_ohlcv.ohlcv_purge(pair, timeframe): + logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') + try: + ohlcv = trades_to_ohlcv(trades, timeframe) + # Store ohlcv + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) + except ValueError: + logger.exception(f'Could not convert {pair} to OHLCV.') + + +def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum common timerange for the given backtest data. + + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeranges = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return (min(timeranges, key=operator.itemgetter(0))[0], + max(timeranges, key=operator.itemgetter(1))[1]) + + +def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime, + max_date: datetime, timeframe_min: int) -> bool: + """ + Validates preprocessed backtesting data for missing values and shows warnings about it that. + + :param data: preprocessed backtesting data (as DataFrame) + :param pair: pair used for log output. + :param min_date: start-date of the data + :param max_date: end-date of the data + :param timeframe_min: Timeframe in minutes + """ + # total difference in minutes / timeframe-minutes + expected_frames = int((max_date - min_date).total_seconds() // 60 // timeframe_min) + found_missing = False + dflen = len(data) + if dflen < expected_frames: + found_missing = True + logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", + pair, expected_frames, dflen, expected_frames - dflen) + return found_missing diff --git a/finrl/data/history/idatahandler.py b/finrl/data/history/idatahandler.py new file mode 100644 index 000000000..3028fb871 --- /dev/null +++ b/finrl/data/history/idatahandler.py @@ -0,0 +1,255 @@ +""" +Abstract datahandler interface. +It's subclasses handle and storing data from disk. + +""" +import logging +from abc import ABC, abstractclassmethod, abstractmethod +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from typing import List, Optional, Type + +from pandas import DataFrame + +from finrl.config import TimeRange +from finrl.constants import ListPairsWithTimeframes, TradeList +from finrl.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe +from finrl.exchange import timeframe_to_seconds + + +logger = logging.getLogger(__name__) + + +class IDataHandler(ABC): + + def __init__(self, datadir: Path) -> None: + self._datadir = datadir + + @abstractclassmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Tuples of (pair, timeframe) + """ + + @abstractclassmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + @abstractmethod + def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Store ohlcv data. + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + + @abstractmethod + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None, + ) -> DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversion to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + + @abstractmethod + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + + @abstractmethod + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + + @abstractclassmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + + @abstractmethod + def trades_store(self, pair: str, data: TradeList) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + + @abstractmethod + def trades_append(self, pair: str, data: TradeList): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + + @abstractmethod + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + + @abstractmethod + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + + def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + Removes duplicates in the process. + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + + def ohlcv_load(self, pair, timeframe: str, + timerange: Optional[TimeRange] = None, + fill_missing: bool = True, + drop_incomplete: bool = True, + startup_candles: int = 0, + warn_no_data: bool = True + ) -> DataFrame: + """ + Load cached candle (OHLCV) data for the given pair. + + :param pair: Pair to load data for + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange + :param fill_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period + :param warn_no_data: Log a warning message when no data is found + :return: DataFrame with ohlcv data, or empty DataFrame + """ + # Fix startup period + timerange_startup = deepcopy(timerange) + if startup_candles > 0 and timerange_startup: + timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) + + pairdf = self._ohlcv_load(pair, timeframe, + timerange=timerange_startup) + if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): + return pairdf + else: + enddate = pairdf.iloc[-1]['date'] + + if timerange_startup: + self._validate_pairdata(pair, pairdf, timerange_startup) + pairdf = trim_dataframe(pairdf, timerange_startup) + if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): + return pairdf + + # incomplete candles should only be dropped if we didn't trim the end beforehand. + pairdf = clean_ohlcv_dataframe(pairdf, timeframe, + pair=pair, + fill_missing=fill_missing, + drop_incomplete=(drop_incomplete and + enddate == pairdf.iloc[-1]['date'])) + self._check_empty_df(pairdf, pair, timeframe, warn_no_data) + return pairdf + + def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str, warn_no_data: bool): + """ + Warn on empty dataframe + """ + if pairdf.empty: + if warn_no_data: + logger.warning( + f'No history data for pair: "{pair}", timeframe: {timeframe}. ' + 'Use fetchData Module to download the data' + ) + return True + return False + + def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): + """ + Validates pairdata for missing data at start end end and logs warnings. + :param pairdata: Dataframe to validate + :param timerange: Timerange specified for start and end dates + """ + + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + if pairdata.iloc[0]['date'] > start: + logger.warning(f"Missing data at start for pair {pair}, " + f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}") + if timerange.stoptype == 'date': + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) + if pairdata.iloc[-1]['date'] < stop: + logger.warning(f"Missing data at end for pair {pair}, " + f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}") + + +def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: + """ + Get datahandler class. + Could be done using Resolvers, but since this may be called often and resolvers + are rather expensive, doing this directly should improve performance. + :param datatype: datatype to use. + :return: Datahandler class + """ + + if datatype == 'json': + from .jsondatahandler import JsonDataHandler + return JsonDataHandler + elif datatype == 'jsongz': + from .jsondatahandler import JsonGzDataHandler + return JsonGzDataHandler + elif datatype == 'hdf5': + from .hdf5datahandler import HDF5DataHandler + return HDF5DataHandler + else: + raise ValueError(f"No datahandler for datatype {datatype} available.") + + +def get_datahandler(datadir: Path, data_format: str = None, + data_handler: IDataHandler = None) -> IDataHandler: + """ + :param datadir: Folder to save data + :data_format: dataformat to use + :data_handler: returns this datahandler if it exists or initializes a new one + """ + + if not data_handler: + HandlerClass = get_datahandlerclass(data_format or 'json') + data_handler = HandlerClass(datadir) + return data_handler diff --git a/finrl/data/history/jsondatahandler.py b/finrl/data/history/jsondatahandler.py new file mode 100644 index 000000000..4e2373811 --- /dev/null +++ b/finrl/data/history/jsondatahandler.py @@ -0,0 +1,204 @@ +import logging +import re +from pathlib import Path +from typing import List, Optional + +import numpy as np +from pandas import DataFrame, read_json, to_datetime + +from finrl import misc +from finrl.config import TimeRange +from finrl.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList +from finrl.data.converter import trades_dict_to_list + +from .idatahandler import IDataHandler + + +logger = logging.getLogger(__name__) + + +class JsonDataHandler(IDataHandler): + + _use_zip = False + _columns = DEFAULT_DATAFRAME_COLUMNS + + @classmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Tuples of (pair, timeframe) + """ + _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name) + for p in datadir.glob(f"*.{cls._get_file_extension()}")] + return [(match[1].replace('_', '/'), match[2]) for match in _tmp + if match and len(match.groups()) > 1] + + @classmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) + for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] + # Check if regex found something and only return these results + return [match[0].replace('_', '/') for match in _tmp if match] + + def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Store data in json format "values". + format looks as follows: + [[,,,,]] + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + _data = data.copy() + # Convert date to int + _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000 + + # Reset index, select only appropriate columns and save as json + _data.reset_index(drop=True).loc[:, self._columns].to_json( + filename, orient="values", + compression='gzip' if self._use_zip else None) + + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None, + ) -> DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversion to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if not filename.exists(): + return DataFrame(columns=self._columns) + pairdata = read_json(filename, orient='values') + pairdata.columns = self._columns + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) + pairdata['date'] = to_datetime(pairdata['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + return pairdata + + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.exists(): + filename.unlink() + return True + return False + + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + raise NotImplementedError() + + @classmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) + for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] + # Check if regex found something and only return these results to avoid exceptions. + return [match[0].replace('_', '/') for match in _tmp if match] + + def trades_store(self, pair: str, data: TradeList) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + filename = self._pair_trades_filename(self._datadir, pair) + misc.file_dump_json(filename, data, is_zip=self._use_zip) + + def trades_append(self, pair: str, data: TradeList): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + raise NotImplementedError() + + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + # TODO: respect timerange ... + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + filename = self._pair_trades_filename(self._datadir, pair) + tradesdata = misc.file_load_json(filename) + + if not tradesdata: + return [] + + if isinstance(tradesdata[0], dict): + # Convert trades dict to list + logger.info("Old trades format detected - converting") + tradesdata = trades_dict_to_list(tradesdata) + pass + return tradesdata + + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.exists(): + filename.unlink() + return True + return False + + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') + return filename + + @classmethod + def _get_file_extension(cls): + return "json.gz" if cls._use_zip else "json" + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') + return filename + + +class JsonGzDataHandler(JsonDataHandler): + + _use_zip = True diff --git a/finrl/exchange/__init__.py b/finrl/exchange/__init__.py new file mode 100644 index 000000000..616ba45a8 --- /dev/null +++ b/finrl/exchange/__init__.py @@ -0,0 +1,16 @@ +# flake8: noqa: F401 +# isort: off +from finrl.exchange.common import MAP_EXCHANGE_CHILDCLASS +from finrl.exchange.exchange import Exchange +# isort: on +from finrl.exchange.bibox import Bibox +from finrl.exchange.binance import Binance +from finrl.exchange.bittrex import Bittrex +from finrl.exchange.exchange import (available_exchanges, ccxt_exchanges, + get_exchange_bad_reason, is_exchange_bad, + is_exchange_known_ccxt, is_exchange_officially_supported, + market_is_active, timeframe_to_minutes, timeframe_to_msecs, + timeframe_to_next_date, timeframe_to_prev_date, + timeframe_to_seconds) +from finrl.exchange.ftx import Ftx +from finrl.exchange.kraken import Kraken diff --git a/finrl/exchange/bibox.py b/finrl/exchange/bibox.py new file mode 100644 index 000000000..ae5458922 --- /dev/null +++ b/finrl/exchange/bibox.py @@ -0,0 +1,23 @@ +""" Bibox exchange subclass """ +import logging +from typing import Dict + +from finrl.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bibox(Exchange): + """ + Bibox exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + # fetchCurrencies API point requires authentication for Bibox, + # so switch it off for Freqtrade load_markets() + _ccxt_config: Dict = {"has": {"fetchCurrencies": False}} diff --git a/finrl/exchange/binance.py b/finrl/exchange/binance.py new file mode 100644 index 000000000..5220f6ea4 --- /dev/null +++ b/finrl/exchange/binance.py @@ -0,0 +1,89 @@ +""" Binance exchange subclass """ +import logging +from typing import Dict + +import ccxt + +from finrl.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, + OperationalException, TemporaryError) +from finrl.exchange import Exchange +from finrl.exchange.common import retrier + + +logger = logging.getLogger(__name__) + + +class Binance(Exchange): + + _ft_has: Dict = { + "stoploss_on_exchange": True, + "order_time_in_force": ['gtc', 'fok', 'ioc'], + "trades_pagination": "id", + "trades_pagination_arg": "fromId", + "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], + } + + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return order['type'] == 'stop_loss_limit' and stop_loss > float(order['info']['stopPrice']) + + @retrier(retries=0) + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + creates a stoploss limit order. + this stoploss-limit is binance-specific. + It may work with a limited number of other exchanges, but this has not been tested yet. + """ + # Limit price threshold: As limit price should always be below stop-price + limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + rate = stop_price * limit_price_pct + + ordertype = "stop_loss_limit" + + stop_price = self.price_to_precision(pair, stop_price) + + # Ensure rate is less than stop price + if stop_price <= rate: + raise OperationalException( + 'In stoploss limit order, stop price should be more than limit price') + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + params = self._params.copy() + params.update({'stopPrice': stop_price}) + + amount = self.amount_to_precision(pair, amount) + + rate = self.price_to_precision(pair, rate) + + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=rate, params=params) + logger.info('stoploss limit order added for %s. ' + 'stop price: %s. limit: %s', pair, stop_price, rate) + return order + except ccxt.InsufficientFunds as e: + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} sell order on market {pair}. ' + f'Tried to sell amount {amount} at rate {rate}. ' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + # Errors: + # `binance Order would trigger immediately.` + raise InvalidOrderException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to sell amount {amount} at rate {rate}. ' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/finrl/exchange/bittrex.py b/finrl/exchange/bittrex.py new file mode 100644 index 000000000..30dc48d08 --- /dev/null +++ b/finrl/exchange/bittrex.py @@ -0,0 +1,23 @@ +""" Bittrex exchange subclass """ +import logging +from typing import Dict + +from finrl.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bittrex(Exchange): + """ + Bittrex exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + _ft_has: Dict = { + "l2_limit_range": [1, 25, 500], + } diff --git a/finrl/exchange/common.py b/finrl/exchange/common.py new file mode 100644 index 000000000..e9ff3e016 --- /dev/null +++ b/finrl/exchange/common.py @@ -0,0 +1,155 @@ +import asyncio +import logging +import time +from functools import wraps + +from finrl.exceptions import DDosProtection, RetryableOrderError, TemporaryError + + +logger = logging.getLogger(__name__) + + +# Maximum default retry count. +# Functions are always called RETRY_COUNT + 1 times (for the original call) +API_RETRY_COUNT = 4 +API_FETCH_ORDER_RETRY_COUNT = 5 + +BAD_EXCHANGES = { + "bitmex": "Various reasons.", + "bitstamp": "Does not provide history. ", + "hitbtc": "This API cannot be used. " + "Use `hitbtc2` exchange id to access this exchange.", + "phemex": "Does not provide history. ", + **dict.fromkeys([ + 'adara', + 'anxpro', + 'bigone', + 'coinbase', + 'coinexchange', + 'coinmarketcap', + 'lykke', + 'xbtce', + ], "Does not provide timeframes. ccxt fetchOHLCV: False"), + **dict.fromkeys([ + 'bcex', + 'bit2c', + 'bitbay', + 'bitflyer', + 'bitforex', + 'bithumb', + 'bitso', + 'bitstamp1', + 'bl3p', + 'braziliex', + 'btcbox', + 'btcchina', + 'btctradeim', + 'btctradeua', + 'bxinth', + 'chilebit', + 'coincheck', + 'coinegg', + 'coinfalcon', + 'coinfloor', + 'coingi', + 'coinmate', + 'coinone', + 'coinspot', + 'coolcoin', + 'crypton', + 'deribit', + 'exmo', + 'exx', + 'flowbtc', + 'foxbit', + 'fybse', + # 'hitbtc', + 'ice3x', + 'independentreserve', + 'indodax', + 'itbit', + 'lakebtc', + 'latoken', + 'liquid', + 'livecoin', + 'luno', + 'mixcoins', + 'negociecoins', + 'nova', + 'paymium', + 'southxchange', + 'stronghold', + 'surbitcoin', + 'therock', + 'tidex', + 'vaultoro', + 'vbtc', + 'virwox', + 'yobit', + 'zaif', + ], "Does not provide timeframes. ccxt fetchOHLCV: emulated"), +} + +MAP_EXCHANGE_CHILDCLASS = { + 'binanceus': 'binance', + 'binanceje': 'binance', +} + + +def calculate_backoff(retrycount, max_retries): + """ + Calculate backoff + """ + return (max_retries - retrycount) ** 2 + 1 + + +def retrier_async(f): + async def wrapper(*args, **kwargs): + count = kwargs.pop('count', API_RETRY_COUNT) + try: + return await f(*args, **kwargs) + except TemporaryError as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + logger.warning('retrying %s() still for %s times', f.__name__, count) + count -= 1 + kwargs.update({'count': count}) + if isinstance(ex, DDosProtection): + backoff_delay = calculate_backoff(count + 1, API_RETRY_COUNT) + logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}") + await asyncio.sleep(backoff_delay) + return await wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + + +def retrier(_func=None, retries=API_RETRY_COUNT): + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + count = kwargs.pop('count', retries) + try: + return f(*args, **kwargs) + except (TemporaryError, RetryableOrderError) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + logger.warning('retrying %s() still for %s times', f.__name__, count) + count -= 1 + kwargs.update({'count': count}) + if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError): + # increasing backoff + backoff_delay = calculate_backoff(count + 1, retries) + logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}") + time.sleep(backoff_delay) + return wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + # Support both @retrier and @retrier(retries=2) syntax + if _func is None: + return decorator + else: + return decorator(_func) diff --git a/finrl/exchange/exchange.py b/finrl/exchange/exchange.py new file mode 100644 index 000000000..b196c076c --- /dev/null +++ b/finrl/exchange/exchange.py @@ -0,0 +1,1332 @@ +# pragma pylint: disable=W0603 +""" +Cryptocurrency Exchanges support +""" +import asyncio +import inspect +import logging +from copy import deepcopy +from datetime import datetime, timezone +from math import ceil +from typing import Any, Dict, List, Optional, Tuple + +import arrow +import ccxt +import ccxt.async_support as ccxt_async +from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, + decimal_to_precision) +from pandas import DataFrame + +from finrl.constants import ListPairsWithTimeframes +from finrl.data.converter import ohlcv_to_dataframe, trades_dict_to_list +from finrl.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, + InvalidOrderException, OperationalException, RetryableOrderError, + TemporaryError) +from finrl.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier, + retrier_async) +from finrl.misc import deep_merge_dicts, safe_value_fallback2 + + +CcxtModuleType = Any + + +logger = logging.getLogger(__name__) + + +class Exchange: + + _config: Dict = {} + + # Parameters to add directly to ccxt sync/async initialization. + _ccxt_config: Dict = {} + + # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) + _params: Dict = {} + + # Dict to specify which options each exchange implements + # This defines defaults, which can be selectively overridden by subclasses using _ft_has + # or by specifying them in the configuration. + _ft_has_default: Dict = { + "stoploss_on_exchange": False, + "order_time_in_force": ["gtc"], + "ohlcv_candle_limit": 500, + "ohlcv_partial_candle": True, + "trades_pagination": "time", # Possible are "time" or "id" + "trades_pagination_arg": "since", + "l2_limit_range": None, + } + _ft_has: Dict = {} + + def __init__(self, config: Dict[str, Any], validate: bool = True) -> None: + """ + Initializes this module with the given config, + it does basic validation whether the specified exchange and pairs are valid. + :return: None + """ + self._api: ccxt.Exchange = None + self._api_async: ccxt_async.Exchange = None + + self._config.update(config) + + # Holds last candle refreshed time of each pair + self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} + # Timestamp of last markets refresh + self._last_markets_refresh: int = 0 + + # Holds candles + self._klines: Dict[Tuple[str, str], DataFrame] = {} + + # Holds all open sell orders for dry_run + self._dry_run_open_orders: Dict[str, Any] = {} + + if config['dry_run']: + logger.info('Instance is running with dry_run enabled') + logger.info(f"Using CCXT {ccxt.__version__}") + exchange_config = config['exchange'] + + # Deep merge ft_has with default ft_has options + self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default)) + if exchange_config.get('_ft_has_params'): + self._ft_has = deep_merge_dicts(exchange_config.get('_ft_has_params'), + self._ft_has) + logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) + + # Assign this directly for easy access + self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit'] + self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle'] + + self._trades_pagination = self._ft_has['trades_pagination'] + self._trades_pagination_arg = self._ft_has['trades_pagination_arg'] + + # Initialize ccxt objects + ccxt_config = self._ccxt_config.copy() + ccxt_config = deep_merge_dicts(exchange_config.get('ccxt_config', {}), ccxt_config) + ccxt_config = deep_merge_dicts(exchange_config.get('ccxt_sync_config', {}), ccxt_config) + + self._api = self._init_ccxt(exchange_config, ccxt_kwargs=ccxt_config) + + ccxt_async_config = self._ccxt_config.copy() + ccxt_async_config = deep_merge_dicts(exchange_config.get('ccxt_config', {}), + ccxt_async_config) + ccxt_async_config = deep_merge_dicts(exchange_config.get('ccxt_async_config', {}), + ccxt_async_config) + self._api_async = self._init_ccxt( + exchange_config, ccxt_async, ccxt_kwargs=ccxt_async_config) + + logger.info('Using Exchange "%s"', self.name) + + if validate: + # Check if timeframe is available + self.validate_timeframes(config.get('timeframe')) + + # Initial markets load + self._load_markets() + + # Check if all pairs are available + self.validate_stakecurrency(config['stake_currency']) + if not exchange_config.get('skip_pair_validation'): + self.validate_pairs(config['exchange']['pair_whitelist']) + self.validate_ordertypes(config.get('order_types', {})) + self.validate_order_time_in_force(config.get('order_time_in_force', {})) + self.validate_required_startup_candles(config.get('startup_candle_count', 0)) + + # Converts the interval provided in minutes in config to seconds + self.markets_refresh_interval: int = exchange_config.get( + "markets_refresh_interval", 60) * 60 + + def __del__(self): + """ + Destructor - clean up async stuff + """ + logger.debug("Exchange object destroyed, closing async loop") + if self._api_async and inspect.iscoroutinefunction(self._api_async.close): + asyncio.get_event_loop().run_until_complete(self._api_async.close()) + + def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, + ccxt_kwargs: dict = None) -> ccxt.Exchange: + """ + Initialize ccxt with given config and return valid + ccxt instance. + """ + # Find matching class for the given exchange name + name = exchange_config['name'] + + if not is_exchange_known_ccxt(name, ccxt_module): + raise OperationalException(f'Exchange {name} is not supported by ccxt') + + ex_config = { + 'apiKey': exchange_config.get('key'), + 'secret': exchange_config.get('secret'), + 'password': exchange_config.get('password'), + 'uid': exchange_config.get('uid', ''), + } + if ccxt_kwargs: + logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + ex_config.update(ccxt_kwargs) + try: + + api = getattr(ccxt_module, name.lower())(ex_config) + except (KeyError, AttributeError) as e: + raise OperationalException(f'Exchange {name} is not supported') from e + except ccxt.BaseError as e: + raise OperationalException(f"Initialization of ccxt failed. Reason: {e}") from e + + self.set_sandbox(api, exchange_config, name) + + return api + + @property + def name(self) -> str: + """exchange Name (from ccxt)""" + return self._api.name + + @property + def id(self) -> str: + """exchange ccxt id""" + return self._api.id + + @property + def timeframes(self) -> List[str]: + return list((self._api.timeframes or {}).keys()) + + @property + def ohlcv_candle_limit(self) -> int: + """exchange ohlcv candle limit""" + return int(self._ohlcv_candle_limit) + + @property + def markets(self) -> Dict: + """exchange ccxt markets""" + if not self._api.markets: + logger.info("Markets were not loaded. Loading them now..") + self._load_markets() + return self._api.markets + + @property + def precisionMode(self) -> str: + """exchange ccxt precisionMode""" + return self._api.precisionMode + + def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None, + pairs_only: bool = False, active_only: bool = False) -> Dict: + """ + Return exchange ccxt markets, filtered out by base currency and quote currency + if this was requested in parameters. + + TODO: consider moving it to the Dataprovider + """ + markets = self.markets + if not markets: + raise OperationalException("Markets were not loaded.") + + if base_currencies: + markets = {k: v for k, v in markets.items() if v['base'] in base_currencies} + if quote_currencies: + markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies} + if pairs_only: + markets = {k: v for k, v in markets.items() if self.market_is_tradable(v)} + if active_only: + markets = {k: v for k, v in markets.items() if market_is_active(v)} + return markets + + def get_quote_currencies(self) -> List[str]: + """ + Return a list of supported quote currencies + """ + markets = self.markets + return sorted(set([x['quote'] for _, x in markets.items()])) + + def get_pair_quote_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets.get(pair, {}).get('quote', '') + + def get_pair_base_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets.get(pair, {}).get('base', '') + + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + By default, checks if it's splittable by `/` and both sides correspond to base / quote + """ + symbol_parts = market['symbol'].split('/') + return (len(symbol_parts) == 2 and + len(symbol_parts[0]) > 0 and + len(symbol_parts[1]) > 0 and + symbol_parts[0] == market.get('base') and + symbol_parts[1] == market.get('quote') + ) + + def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: + if pair_interval in self._klines: + return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] + else: + return DataFrame() + + def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None: + if exchange_config.get('sandbox'): + if api.urls.get('test'): + api.urls['api'] = api.urls['test'] + logger.info("Enabled Sandbox API on %s", name) + else: + logger.warning( + f"No Sandbox URL in CCXT for {name}, exiting. Please check your config.json") + raise OperationalException(f'Exchange {name} does not provide a sandbox api') + + def _load_async_markets(self, reload: bool = False) -> None: + try: + if self._api_async: + asyncio.get_event_loop().run_until_complete( + self._api_async.load_markets(reload=reload)) + + except (asyncio.TimeoutError, ccxt.BaseError) as e: + logger.warning('Could not load async markets. Reason: %s', e) + return + + def _load_markets(self) -> None: + """ Initialize markets both sync and async """ + try: + self._api.load_markets() + self._load_async_markets() + self._last_markets_refresh = arrow.utcnow().int_timestamp + except ccxt.BaseError as e: + logger.warning('Unable to initialize markets. Reason: %s', e) + + def reload_markets(self) -> None: + """Reload markets both sync and async if refresh interval has passed """ + # Check whether markets have to be reloaded + if (self._last_markets_refresh > 0) and ( + self._last_markets_refresh + self.markets_refresh_interval + > arrow.utcnow().int_timestamp): + return None + logger.debug("Performing scheduled market reload..") + try: + self._api.load_markets(reload=True) + # Also reload async markets to avoid issues with newly listed pairs + self._load_async_markets(reload=True) + self._last_markets_refresh = arrow.utcnow().int_timestamp + except ccxt.BaseError: + logger.exception("Could not reload markets.") + + def validate_stakecurrency(self, stake_currency: str) -> None: + """ + Checks stake-currency against available currencies on the exchange. + :param stake_currency: Stake-currency to validate + :raise: OperationalException if stake-currency is not available. + """ + quote_currencies = self.get_quote_currencies() + if stake_currency not in quote_currencies: + raise OperationalException( + f"{stake_currency} is not available as stake on {self.name}. " + f"Available currencies are: {', '.join(quote_currencies)}") + + def validate_pairs(self, pairs: List[str]) -> None: + """ + Checks if all given pairs are tradable on the current exchange. + :param pairs: list of pairs + :raise: OperationalException if one pair is not available + :return: None + """ + + if not self.markets: + logger.warning('Unable to validate pairs (assuming they are correct).') + return + invalid_pairs = [] + for pair in pairs: + # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs + # TODO: add a support for having coins in BTC/USDT format + if self.markets and pair not in self.markets: + raise OperationalException( + f'Pair {pair} is not available on {self.name}. ' + f'Please remove {pair} from your whitelist.') + + # From ccxt Documentation: + # markets.info: An associative array of non-common market properties, + # including fees, rates, limits and other general market information. + # The internal info array is different for each particular market, + # its contents depend on the exchange. + # It can also be a string or similar ... so we need to verify that first. + elif (isinstance(self.markets[pair].get('info', None), dict) + and self.markets[pair].get('info', {}).get('IsRestricted', False)): + # Warn users about restricted pairs in whitelist. + # We cannot determine reliably if Users are affected. + logger.warning(f"Pair {pair} is restricted for some users on this exchange." + f"Please check if you are impacted by this restriction " + f"on the exchange and eventually remove {pair} from your whitelist.") + if (self._config['stake_currency'] and + self.get_pair_quote_currency(pair) != self._config['stake_currency']): + invalid_pairs.append(pair) + if invalid_pairs: + raise OperationalException( + f"Stake-currency '{self._config['stake_currency']}' not compatible with " + f"pair-whitelist. Please remove the following pairs: {invalid_pairs}") + + def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str: + """ + Get valid pair combination of curr_1 and curr_2 by trying both combinations. + """ + for pair in [f"{curr_1}/{curr_2}", f"{curr_2}/{curr_1}"]: + if pair in self.markets and self.markets[pair].get('active'): + return pair + raise ExchangeError(f"Could not combine {curr_1} and {curr_2} to get a valid pair.") + + def validate_timeframes(self, timeframe: Optional[str]) -> None: + """ + Check if timeframe from config is a supported timeframe on the exchange + """ + if not hasattr(self._api, "timeframes") or self._api.timeframes is None: + # If timeframes attribute is missing (or is None), the exchange probably + # has no fetchOHLCV method. + # Therefore we also show that. + raise OperationalException( + f"The ccxt library does not provide the list of timeframes " + f"for the exchange \"{self.name}\" and this exchange " + f"is therefore not supported. ccxt fetchOHLCV: {self.exchange_has('fetchOHLCV')}") + + if timeframe and (timeframe not in self.timeframes): + raise OperationalException( + f"Invalid timeframe '{timeframe}'. This exchange supports: {self.timeframes}") + + if timeframe and timeframe_to_minutes(timeframe) < 1: + raise OperationalException("Timeframes < 1m are currently not supported by Freqtrade.") + + def validate_ordertypes(self, order_types: Dict) -> None: + """ + Checks if order-types configured in strategy/config are supported + """ + if any(v == 'market' for k, v in order_types.items()): + if not self.exchange_has('createMarketOrder'): + raise OperationalException( + f'Exchange {self.name} does not support market orders.') + + if (order_types.get("stoploss_on_exchange") + and not self._ft_has.get("stoploss_on_exchange", False)): + raise OperationalException( + f'On exchange stoploss is not supported for {self.name}.' + ) + + def validate_order_time_in_force(self, order_time_in_force: Dict) -> None: + """ + Checks if order time in force configured in strategy/config are supported + """ + if any(v not in self._ft_has["order_time_in_force"] + for k, v in order_time_in_force.items()): + raise OperationalException( + f'Time in force policies are not supported for {self.name} yet.') + + def validate_required_startup_candles(self, startup_candles: int) -> None: + """ + Checks if required startup_candles is more than ohlcv_candle_limit. + Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. + """ + if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']: + raise OperationalException( + f"This strategy requires {startup_candles} candles to start. " + f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.") + + def exchange_has(self, endpoint: str) -> bool: + """ + Checks if exchange implements a specific API endpoint. + Wrapper around ccxt 'has' attribute + :param endpoint: Name of endpoint (e.g. 'fetchOHLCV', 'fetchTickers') + :return: bool + """ + return endpoint in self._api.has and self._api.has[endpoint] + + def amount_to_precision(self, pair: str, amount: float) -> float: + ''' + Returns the amount to buy or sell to a precision the Exchange accepts + Reimplementation of ccxt internal methods - ensuring we can test the result is correct + based on our definitions. + ''' + if self.markets[pair]['precision']['amount']: + amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE, + precision=self.markets[pair]['precision']['amount'], + counting_mode=self.precisionMode, + )) + + return amount + + def price_to_precision(self, pair: str, price: float) -> float: + ''' + Returns the price rounded up to the precision the Exchange accepts. + Partial Reimplementation of ccxt internal method decimal_to_precision(), + which does not support rounding up + TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and + align with amount_to_precision(). + Rounds up + ''' + if self.markets[pair]['precision']['price']: + # price = float(decimal_to_precision(price, rounding_mode=ROUND, + # precision=self.markets[pair]['precision']['price'], + # counting_mode=self.precisionMode, + # )) + if self.precisionMode == TICK_SIZE: + precision = self.markets[pair]['precision']['price'] + missing = price % precision + if missing != 0: + price = price - missing + precision + else: + symbol_prec = self.markets[pair]['precision']['price'] + big_price = price * pow(10, symbol_prec) + price = ceil(big_price) / pow(10, symbol_prec) + return price + + def price_get_one_pip(self, pair: str, price: float) -> float: + """ + Get's the "1 pip" value for this pair. + Used in PriceFilter to calculate the 1pip movements. + """ + precision = self.markets[pair]['precision']['price'] + if self.precisionMode == TICK_SIZE: + return precision + else: + return 1 / pow(10, precision) + + def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, + rate: float, params: Dict = {}) -> Dict[str, Any]: + order_id = f'dry_run_{side}_{datetime.now().timestamp()}' + _amount = self.amount_to_precision(pair, amount) + dry_order = { + 'id': order_id, + 'symbol': pair, + 'price': rate, + 'average': rate, + 'amount': _amount, + 'cost': _amount * rate, + 'type': ordertype, + 'side': side, + 'remaining': _amount, + 'datetime': arrow.utcnow().isoformat(), + 'timestamp': int(arrow.utcnow().int_timestamp * 1000), + 'status': "closed" if ordertype == "market" else "open", + 'fee': None, + 'info': {} + } + self._store_dry_order(dry_order, pair) + # Copy order and close it - so the returned order is open unless it's a market order + return dry_order + + def _store_dry_order(self, dry_order: Dict, pair: str) -> None: + closed_order = dry_order.copy() + if closed_order['type'] in ["market", "limit"]: + closed_order.update({ + 'status': 'closed', + 'filled': closed_order['amount'], + 'remaining': 0, + 'fee': { + 'currency': self.get_pair_quote_currency(pair), + 'cost': dry_order['cost'] * self.get_fee(pair), + 'rate': self.get_fee(pair) + } + }) + if closed_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: + closed_order["info"].update({"stopPrice": closed_order["price"]}) + self._dry_run_open_orders[closed_order["id"]] = closed_order + + def create_order(self, pair: str, ordertype: str, side: str, amount: float, + rate: float, params: Dict = {}) -> Dict: + try: + # Set the precision for amount and price(rate) as accepted by the exchange + amount = self.amount_to_precision(pair, amount) + needs_price = (ordertype != 'market' + or self._api.options.get("createMarketBuyOrderRequiresPrice", False)) + rate_for_order = self.price_to_precision(pair, rate) if needs_price else None + + return self._api.create_order(pair, ordertype, side, + amount, rate_for_order, params) + + except ccxt.InsufficientFunds as e: + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' + f'Tried to {side} amount {amount} at rate {rate}.' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise ExchangeError( + f'Could not create {ordertype} {side} order on market {pair}. ' + f'Tried to {side} amount {amount} at rate {rate}. ' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + def buy(self, pair: str, ordertype: str, amount: float, + rate: float, time_in_force: str) -> Dict: + + if self._config['dry_run']: + dry_order = self.dry_run_order(pair, ordertype, "buy", amount, rate) + return dry_order + + params = self._params.copy() + if time_in_force != 'gtc' and ordertype != 'market': + params.update({'timeInForce': time_in_force}) + + return self.create_order(pair, ordertype, 'buy', amount, rate, params) + + def sell(self, pair: str, ordertype: str, amount: float, + rate: float, time_in_force: str = 'gtc') -> Dict: + + if self._config['dry_run']: + dry_order = self.dry_run_order(pair, ordertype, "sell", amount, rate) + return dry_order + + params = self._params.copy() + if time_in_force != 'gtc' and ordertype != 'market': + params.update({'timeInForce': time_in_force}) + + return self.create_order(pair, ordertype, 'sell', amount, rate, params) + + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + raise OperationalException(f"stoploss is not implemented for {self.name}.") + + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + creates a stoploss order. + The precise ordertype is determined by the order_types dict or exchange default. + Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each + exchange's subclass. + The exception below should never raise, since we disallow + starting the bot in validate_ordertypes() + Note: Changes to this interface need to be applied to all sub-classes too. + """ + + raise OperationalException(f"stoploss is not implemented for {self.name}.") + + @retrier + def get_balance(self, currency: str) -> float: + if self._config['dry_run']: + return self._config['dry_run_wallet'] + + # ccxt exception is already handled by get_balances + balances = self.get_balances() + balance = balances.get(currency) + if balance is None: + raise TemporaryError( + f'Could not get {currency} balance due to malformed exchange response: {balances}') + return balance['free'] + + @retrier + def get_balances(self) -> dict: + if self._config['dry_run']: + return {} + + try: + balances = self._api.fetch_balance() + # Remove additional info from ccxt results + balances.pop("info", None) + balances.pop("free", None) + balances.pop("total", None) + balances.pop("used", None) + + return balances + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier + def get_tickers(self) -> Dict: + try: + return self._api.fetch_tickers() + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching tickers in batch. ' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier + def fetch_ticker(self, pair: str) -> dict: + try: + if pair not in self._api.markets or not self._api.markets[pair].get('active'): + raise ExchangeError(f"Pair {pair} not available") + data = self._api.fetch_ticker(pair) + return data + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + def get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int) -> List: + """ + Get candle history using asyncio and returns the list of candles. + Handles all async work for this. + Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. + :param pair: Pair to download + :param timeframe: Timeframe to get data for + :param since_ms: Timestamp in milliseconds to get history from + :return: List with candle (OHLCV) data + """ + return asyncio.get_event_loop().run_until_complete( + self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, + since_ms=since_ms)) + + def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, + since_ms: int) -> DataFrame: + """ + Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe + :param pair: Pair to download + :param timeframe: Timeframe to get data for + :param since_ms: Timestamp in milliseconds to get history from + :return: OHLCV DataFrame + """ + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) + return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + + async def _async_get_historic_ohlcv(self, pair: str, + timeframe: str, + since_ms: int) -> List: + """ + Download historic ohlcv + """ + + one_call = timeframe_to_msecs(timeframe) * self._ohlcv_candle_limit + logger.debug( + "one_call: %s msecs (%s)", + one_call, + arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) + ) + input_coroutines = [self._async_get_candle_history( + pair, timeframe, since) for since in + range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] + + results = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # Combine gathered results + data: List = [] + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) + # Sort data again after extending the result - above calls return in "async order" + data = sorted(data, key=lambda x: x[0]) + logger.info("Downloaded data for %s with length %s.", pair, len(data)) + return data + + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes) -> List[Tuple[str, List]]: + """ + Refresh in-memory OHLCV asynchronously and set `_klines` with the result + Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). + Only used in the dataprovider.refresh() method. + :param pair_list: List of 2 element tuples containing pair, interval to refresh + :return: TODO: return value is only used in the tests, get rid of it + """ + logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) + + input_coroutines = [] + + # Gather coroutines to run + for pair, timeframe in set(pair_list): + if (not ((pair, timeframe) in self._klines) + or self._now_is_time_to_refresh(pair, timeframe)): + input_coroutines.append(self._async_get_candle_history(pair, timeframe)) + else: + logger.debug( + "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", + pair, timeframe + ) + + results = asyncio.get_event_loop().run_until_complete( + asyncio.gather(*input_coroutines, return_exceptions=True)) + + # handle caching + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple (has 3 elements) + pair, timeframe, ticks = res + # keeping last candle time as last refreshed time of the pair + if ticks: + self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 + # keeping parsed dataframe in cache + self._klines[(pair, timeframe)] = ohlcv_to_dataframe( + ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + + return results + + def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool: + # Timeframe in seconds + interval_in_sec = timeframe_to_seconds(timeframe) + + return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0) + + interval_in_sec) >= arrow.utcnow().int_timestamp) + + @retrier_async + async def _async_get_candle_history(self, pair: str, timeframe: str, + since_ms: Optional[int] = None) -> Tuple[str, str, List]: + """ + Asynchronously get candle history data using fetch_ohlcv + returns tuple: (pair, timeframe, ohlcv_list) + """ + try: + # Fetch OHLCV asynchronously + s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else '' + logger.debug( + "Fetching pair %s, interval %s, since %s %s...", + pair, timeframe, since_ms, s + ) + + data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, + since=since_ms) + + # Some exchanges sort OHLCV in ASC order and others in DESC. + # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) + # while GDAX returns the list of OHLCV in DESC order (newest first, oldest last) + # Only sort if necessary to save computing time + try: + if data and data[0][0] > data[-1][0]: + data = sorted(data, key=lambda x: x[0]) + except IndexError: + logger.exception("Error loading %s. Result was %s.", pair, data) + return pair, timeframe, [] + logger.debug("Done fetching pair %s, interval %s ...", pair, timeframe) + return pair, timeframe, data + + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical ' + f'candle (OHLCV) data. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError(f'Could not fetch historical candle (OHLCV) data ' + f'for pair {pair} due to {e.__class__.__name__}. ' + f'Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch historical candle (OHLCV) data ' + f'for pair {pair}. Message: {e}') from e + + @retrier_async + async def _async_fetch_trades(self, pair: str, + since: Optional[int] = None, + params: Optional[dict] = None) -> List[List]: + """ + Asyncronously gets trade history using fetch_trades. + Handles exchange errors, does one call to the exchange. + :param pair: Pair to fetch trade data for + :param since: Since as integer timestamp in milliseconds + returns: List of dicts containing trades + """ + try: + # fetch trades asynchronously + if params: + logger.debug("Fetching trades for pair %s, params: %s ", pair, params) + trades = await self._api_async.fetch_trades(pair, params=params, limit=1000) + else: + logger.debug( + "Fetching trades for pair %s, since %s %s...", + pair, since, + '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' + ) + trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) + return trades_dict_to_list(trades) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical trade data.' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. ' + f'Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e + + async def _async_get_trade_history_id(self, pair: str, + until: int, + since: Optional[int] = None, + from_id: Optional[str] = None) -> Tuple[str, List[List]]: + """ + Asyncronously gets trade history using fetch_trades + use this when exchange uses id-based iteration (check `self._trades_pagination`) + :param pair: Pair to fetch trade data for + :param since: Since as integer timestamp in milliseconds + :param until: Until as integer timestamp in milliseconds + :param from_id: Download data starting with ID (if id is known). Ignores "since" if set. + returns tuple: (pair, trades-list) + """ + + trades: List[List] = [] + + if not from_id: + # Fetch first elements using timebased method to get an ID to paginate on + # Depending on the Exchange, this can introduce a drift at the start of the interval + # of up to an hour. + # e.g. Binance returns the "last 1000" candles within a 1h time interval + # - so we will miss the first trades. + t = await self._async_fetch_trades(pair, since=since) + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + from_id = t[-1][1] + trades.extend(t[:-1]) + while True: + t = await self._async_fetch_trades(pair, + params={self._trades_pagination_arg: from_id}) + if len(t): + # Skip last id since its the key for the next call + trades.extend(t[:-1]) + if from_id == t[-1][1] or t[-1][0] > until: + logger.debug(f"Stopping because from_id did not change. " + f"Reached {t[-1][0]} > {until}") + # Reached the end of the defined-download period - add last trade as well. + trades.extend(t[-1:]) + break + + from_id = t[-1][1] + else: + break + + return (pair, trades) + + async def _async_get_trade_history_time(self, pair: str, until: int, + since: Optional[int] = None) -> Tuple[str, List[List]]: + """ + Asyncronously gets trade history using fetch_trades, + when the exchange uses time-based iteration (check `self._trades_pagination`) + :param pair: Pair to fetch trade data for + :param since: Since as integer timestamp in milliseconds + :param until: Until as integer timestamp in milliseconds + returns tuple: (pair, trades-list) + """ + + trades: List[List] = [] + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + while True: + t = await self._async_fetch_trades(pair, since=since) + if len(t): + since = t[-1][1] + trades.extend(t) + # Reached the end of the defined-download period + if until and t[-1][0] > until: + logger.debug( + f"Stopping because until was reached. {t[-1][0]} > {until}") + break + else: + break + + return (pair, trades) + + async def _async_get_trade_history(self, pair: str, + since: Optional[int] = None, + until: Optional[int] = None, + from_id: Optional[str] = None) -> Tuple[str, List[List]]: + """ + Async wrapper handling downloading trades using either time or id based methods. + """ + + logger.debug(f"_async_get_trade_history(), pair: {pair}, " + f"since: {since}, until: {until}, from_id: {from_id}") + + if until is None: + until = ccxt.Exchange.milliseconds() + logger.debug(f"Exchange milliseconds: {until}") + + if self._trades_pagination == 'time': + return await self._async_get_trade_history_time( + pair=pair, since=since, until=until) + elif self._trades_pagination == 'id': + return await self._async_get_trade_history_id( + pair=pair, since=since, until=until, from_id=from_id + ) + else: + raise OperationalException(f"Exchange {self.name} does use neither time, " + f"nor id based pagination") + + def get_historic_trades(self, pair: str, + since: Optional[int] = None, + until: Optional[int] = None, + from_id: Optional[str] = None) -> Tuple[str, List]: + """ + Get trade history data using asyncio. + Handles all async work and returns the list of candles. + Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. + :param pair: Pair to download + :param since: Timestamp in milliseconds to get history from + :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined. + :param from_id: Download data starting with ID (if id is known) + :returns List of trade data + """ + if not self.exchange_has("fetchTrades"): + raise OperationalException("This exchange does not suport downloading Trades.") + + return asyncio.get_event_loop().run_until_complete( + self._async_get_trade_history(pair=pair, since=since, + until=until, from_id=from_id)) + + def check_order_canceled_empty(self, order: Dict) -> bool: + """ + Verify if an order has been cancelled without being partially filled + :param order: Order dict as returned from fetch_order() + :return: True if order has been cancelled without being filled, False otherwise. + """ + return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0 + + @retrier + def cancel_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + order = self._dry_run_open_orders.get(order_id) + if order: + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) + return order + else: + return {} + + try: + return self._api.cancel_order(order_id, pair) + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not cancel order. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to cancel_stoploss_order to allow easy overriding in other classes + cancel_stoploss_order = cancel_order + + def is_cancel_order_result_suitable(self, corder) -> bool: + if not isinstance(corder, dict): + return False + + required = ('fee', 'status', 'amount') + return all(k in corder for k in required) + + def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: Orderid to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + try: + corder = self.cancel_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + except InvalidOrderException: + logger.warning(f"Could not cancel order {order_id} for {pair}.") + try: + order = self.fetch_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + try: + return self._api.fetch_order(order_id, pair) + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to fetch_stoploss_order to allow easy overriding in other classes + fetch_stoploss_order = fetch_order + + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: + """ + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. + """ + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) + + @staticmethod + def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]): + """ + Get next greater value in the list. + Used by fetch_l2_order_book if the api only supports a limited range + """ + if not limit_range: + return limit + return min([x for x in limit_range if limit <= x] + [max(limit_range)]) + + @retrier + def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: + """ + Get L2 order book from exchange. + Can be limited to a certain amount (if supported). + Returns a dict in the format + {'asks': [price, volume], 'bids': [price, volume]} + """ + limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range']) + try: + + return self._api.fetch_l2_order_book(pair, limit1) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier + def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + """ + Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + + :param order_id order_id: Order-id as given when creating the order + :param pair: Pair the order is for + :param since: datetime object of the order creation time. Assumes object is in UTC. + """ + if self._config['dry_run']: + return [] + if not self.exchange_has('fetchMyTrades'): + return [] + try: + # Allow 5s offset to catch slight time offsets (discovered in #1185) + # since needs to be int in milliseconds + my_trades = self._api.fetch_my_trades( + pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) + matched_trades = [trade for trade in my_trades if trade['order'] == order_id] + + return matched_trades + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier + def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, + price: float = 1, taker_or_maker: str = 'maker') -> float: + try: + # validate that markets are loaded before trying to get fee + if self._api.markets is None or len(self._api.markets) == 0: + self._api.load_markets() + + return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + price=price, takerOrMaker=taker_or_maker)['rate'] + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @staticmethod + def order_has_fee(order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) + + def calculate_fee_rate(self, order: Dict) -> Optional[float]: + """ + Calculate fee rate if it's not given by the exchange. + :param order: Order or trade (one trade) dict + """ + if order['fee'].get('rate') is not None: + return order['fee'].get('rate') + fee_curr = order['fee']['currency'] + # Calculate fee based on order details + if fee_curr in self.get_pair_base_currency(order['symbol']): + # Base currency - divide by amount + return round( + order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) + elif fee_curr in self.get_pair_quote_currency(order['symbol']): + # Quote currency - divide by cost + return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None + else: + # If Fee currency is a different currency + if not order['cost']: + # If cost is None or 0.0 -> falsy, return None + return None + try: + comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) + tick = self.fetch_ticker(comb) + + fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) + except ExchangeError: + return None + + def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: + """ + Extract tuple of cost, currency, rate. + Requires order_has_fee to run first! + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + self.calculate_fee_rate(order)) + + +def is_exchange_bad(exchange_name: str) -> bool: + return exchange_name in BAD_EXCHANGES + + +def get_exchange_bad_reason(exchange_name: str) -> str: + return BAD_EXCHANGES.get(exchange_name, "") + + +def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: + return exchange_name in ccxt_exchanges(ccxt_module) + + +def is_exchange_officially_supported(exchange_name: str) -> bool: + return exchange_name in ['bittrex', 'binance', 'kraken'] + + +def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: + """ + Return the list of all exchanges known to ccxt + """ + return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges + + +def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: + """ + Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list + """ + exchanges = ccxt_exchanges(ccxt_module) + return [x for x in exchanges if not is_exchange_bad(x)] + + +def timeframe_to_seconds(timeframe: str) -> int: + """ + Translates the timeframe interval value written in the human readable + form ('1m', '5m', '1h', '1d', '1w', etc.) to the number + of seconds for one timeframe interval. + """ + return ccxt.Exchange.parse_timeframe(timeframe) + + +def timeframe_to_minutes(timeframe: str) -> int: + """ + Same as timeframe_to_seconds, but returns minutes. + """ + return ccxt.Exchange.parse_timeframe(timeframe) // 60 + + +def timeframe_to_msecs(timeframe: str) -> int: + """ + Same as timeframe_to_seconds, but returns milliseconds. + """ + return ccxt.Exchange.parse_timeframe(timeframe) * 1000 + + +def timeframe_to_prev_date(timeframe: str, date: datetime = None) -> datetime: + """ + Use Timeframe and determine last possible candle. + :param timeframe: timeframe in string format (e.g. "5m") + :param date: date to use. Defaults to utcnow() + :returns: date of previous candle (with utc timezone) + """ + if not date: + date = datetime.now(timezone.utc) + + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, + ROUND_DOWN) // 1000 + return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) + + +def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: + """ + Use Timeframe and determine next candle. + :param timeframe: timeframe in string format (e.g. "5m") + :param date: date to use. Defaults to utcnow() + :returns: date of next candle (with utc timezone) + """ + if not date: + date = datetime.now(timezone.utc) + new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000, + ROUND_UP) // 1000 + return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) + + +def market_is_active(market: Dict) -> bool: + """ + Return True if the market is active. + """ + # "It's active, if the active flag isn't explicitly set to false. If it's missing or + # true then it's true. If it's undefined, then it's most likely true, but not 100% )" + # See https://github.com/ccxt/ccxt/issues/4874, + # https://github.com/ccxt/ccxt/issues/4075#issuecomment-434760520 + return market.get('active', True) is not False diff --git a/finrl/exchange/ftx.py b/finrl/exchange/ftx.py new file mode 100644 index 000000000..48348262e --- /dev/null +++ b/finrl/exchange/ftx.py @@ -0,0 +1,136 @@ +""" FTX exchange subclass """ +import logging +from typing import Any, Dict + +import ccxt + +from finrl.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, + OperationalException, TemporaryError) +from finrl.exchange import Exchange +from finrl.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier + + +logger = logging.getLogger(__name__) + + +class Ftx(Exchange): + + _ft_has: Dict = { + "stoploss_on_exchange": True, + "ohlcv_candle_limit": 1500, + } + + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + Default checks + check if pair is spot pair (no futures trading yet). + """ + parent_check = super().market_is_tradable(market) + + return (parent_check and + market.get('spot', False) is True) + + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return order['type'] == 'stop' and stop_loss > float(order['price']) + + @retrier(retries=0) + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + Creates a stoploss order. + depending on order_types.stoploss configuration, uses 'market' or limit order. + + Limit orders are defined by having orderPrice set, otherwise a market order is used. + """ + limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + limit_rate = stop_price * limit_price_pct + + ordertype = "stop" + + stop_price = self.price_to_precision(pair, stop_price) + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + params = self._params.copy() + if order_types.get('stoploss', 'market') == 'limit': + # set orderPrice to place limit order, otherwise it's a market order + params['orderPrice'] = limit_rate + + amount = self.amount_to_precision(pair, amount) + + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=stop_price, params=params) + logger.info('stoploss order added for %s. ' + 'stop price: %s.', pair, stop_price) + return order + except ccxt.InsufficientFunds as e: + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + try: + orders = self._api.fetch_orders(pair, None, params={'type': 'stop'}) + + order = [order for order in orders if order['id'] == order_id] + if len(order) == 1: + return order[0] + else: + raise InvalidOrderException(f"Could not get stoploss order for id {order_id}") + + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier + def cancel_stoploss_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + return {} + try: + return self._api.cancel_order(order_id, pair, params={'type': 'stop'}) + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not cancel order. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/finrl/exchange/kraken.py b/finrl/exchange/kraken.py new file mode 100644 index 000000000..8d34e9a02 --- /dev/null +++ b/finrl/exchange/kraken.py @@ -0,0 +1,122 @@ +""" Kraken exchange subclass """ +import logging +from typing import Any, Dict + +import ccxt + +from finrl.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, + OperationalException, TemporaryError) +from finrl.exchange import Exchange +from finrl.exchange.common import retrier + + +logger = logging.getLogger(__name__) + + +class Kraken(Exchange): + + _params: Dict = {"trading_agreement": "agree"} + _ft_has: Dict = { + "stoploss_on_exchange": True, + "trades_pagination": "id", + "trades_pagination_arg": "since", + } + + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + Default checks + check if pair is darkpool pair. + """ + parent_check = super().market_is_tradable(market) + + return (parent_check and + market.get('darkpool', False) is False) + + @retrier + def get_balances(self) -> dict: + if self._config['dry_run']: + return {} + + try: + balances = self._api.fetch_balance() + # Remove additional info from ccxt results + balances.pop("info", None) + balances.pop("free", None) + balances.pop("total", None) + balances.pop("used", None) + + orders = self._api.fetch_open_orders() + order_list = [(x["symbol"].split("/")[0 if x["side"] == "sell" else 1], + x["remaining"], + # Don't remove the below comment, this can be important for debuggung + # x["side"], x["amount"], + ) for x in orders] + for bal in balances: + balances[bal]['used'] = sum(order[1] for order in order_list if order[0] == bal) + balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used'] + + return balances + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return (order['type'] in ('stop-loss', 'stop-loss-limit') + and stop_loss > float(order['price'])) + + @retrier(retries=0) + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + Creates a stoploss market order. + Stoploss market orders is the only stoploss type supported by kraken. + """ + params = self._params.copy() + + if order_types.get('stoploss', 'market') == 'limit': + ordertype = "stop-loss-limit" + limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + limit_rate = stop_price * limit_price_pct + params['price2'] = self.price_to_precision(pair, limit_rate) + else: + ordertype = "stop-loss" + + stop_price = self.price_to_precision(pair, stop_price) + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + amount = self.amount_to_precision(pair, amount) + + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=stop_price, params=params) + logger.info('stoploss order added for %s. ' + 'stop price: %s.', pair, stop_price) + return order + except ccxt.InsufficientFunds as e: + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/finrl/marketdata/yahoodownloader.py b/finrl/marketdata/yahoodownloader.py index 813a01163..c979289cc 100644 --- a/finrl/marketdata/yahoodownloader.py +++ b/finrl/marketdata/yahoodownloader.py @@ -4,8 +4,11 @@ import pandas as pd import yfinance as yf - - +from finrl.exceptions import * +import os +import sys +import glob +from datetime import datetime class YahooDownloader: """Provides methods for retrieving daily stock data from Yahoo Finance API @@ -26,13 +29,12 @@ class YahooDownloader: """ - def __init__(self, start_date: str, end_date: str, ticker_list: list): + def __init__(self, config: dict): - self.start_date = start_date - self.end_date = end_date - self.ticker_list = ticker_list - def fetch_data(self) -> pd.DataFrame: + self.config = config + + def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> pd.DataFrame: """Fetches data from Yahoo API Parameters ---------- @@ -43,6 +45,9 @@ def fetch_data(self) -> pd.DataFrame: 7 columns: A date, open, high, low, close, volume and tick symbol for the specified stock ticker """ + self.start_date = start_date + self.end_date = end_date + self.ticker_list = ticker_list # Download and save the data in a pandas DataFrame: data_df = pd.DataFrame() for tic in self.ticker_list: @@ -83,7 +88,45 @@ def fetch_data(self) -> pd.DataFrame: return data_df - def select_equal_rows_stock(self, df): + def fetch_data_crypto(self) -> pd.DataFrame: + """ + Fetches data from local history directory (default= user_data/data/exchange) + + Parameters + ---------- + config.json ---> Exchange, Whitelist, timeframe + + Returns + ------- + `pd.DataFrame` + 7 columns: A date, open, high, low, close, volume and tick symbol + for the specified stock ticker + """ + + datadir = self.config['datadir'] + exchange = self.config["exchange"]["name"] + timeframe = self.config["timeframe"] + # Check if regex found something and only return these results + df = pd.DataFrame() + for i in self.config["pairs"]: + i = i.replace("/","_") + try: + i_df = pd.read_json(f'{os.getcwd()}/{datadir}/{i}-{timeframe}.json') + i_df["tic"] = i + i_df.columns = ["date", "open","high", "low", "close", "volume", "tic"] + i_df.date = i_df.date.apply(lambda d: datetime.fromtimestamp(d/1000)) + df = df.append(i_df) + print(f"coin {i} completed...") + except: + print(f'coin {i} not available') + pass + print(df.shape) + return df + + def select_equal_rows_stock(self, df,start_date: str, end_date: str, ticker_list: list): + self.start_date = start_date + self.end_date = end_date + self.ticker_list = ticker_list df_check = df.tic.value_counts() df_check = pd.DataFrame(df_check).reset_index() df_check.columns = ["tic", "counts"] @@ -93,3 +136,5 @@ def select_equal_rows_stock(self, df): select_stocks_list = list(names[equal_list]) df = df[df.tic.isin(select_stocks_list)] return df + + \ No newline at end of file diff --git a/finrl/misc.py b/finrl/misc.py index 7ccc71b55..5b991c619 100644 --- a/finrl/misc.py +++ b/finrl/misc.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Any from typing.io import IO - import numpy as np import rapidjson @@ -192,3 +191,6 @@ def render_template_with_fallback(templatefile: str, templatefallbackfile: str, return render_template(templatefile, arguments) except TemplateNotFound: return render_template(templatefallbackfile, arguments) + + + diff --git a/finrl/resolvers/__init__.py b/finrl/resolvers/__init__.py new file mode 100644 index 000000000..acfae0367 --- /dev/null +++ b/finrl/resolvers/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa: F401 +# isort: off +from finrl.resolvers.iresolver import IResolver +from finrl.resolvers.exchange_resolver import ExchangeResolver \ No newline at end of file diff --git a/finrl/resolvers/exchange_resolver.py b/finrl/resolvers/exchange_resolver.py new file mode 100644 index 000000000..c2d04c93e --- /dev/null +++ b/finrl/resolvers/exchange_resolver.py @@ -0,0 +1,64 @@ +""" +This module loads custom exchanges +""" +import logging + +import finrl.exchange as exchanges +from finrl.exchange import MAP_EXCHANGE_CHILDCLASS, Exchange +from finrl.resolvers import IResolver + + +logger = logging.getLogger(__name__) + + +class ExchangeResolver(IResolver): + """ + This class contains all the logic to load a custom exchange class + """ + object_type = Exchange + + @staticmethod + def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: + """ + Load the custom class from config parameter + :param config: configuration dictionary + """ + # Map exchange name to avoid duplicate classes for identical exchanges + exchange_name = MAP_EXCHANGE_CHILDCLASS.get(exchange_name, exchange_name) + exchange_name = exchange_name.title() + exchange = None + try: + exchange = ExchangeResolver._load_exchange(exchange_name, + kwargs={'config': config, + 'validate': validate}) + except ImportError: + logger.info( + f"No {exchange_name} specific subclass found. Using the generic class instead.") + if not exchange: + exchange = Exchange(config, validate=validate) + return exchange + + @staticmethod + def _load_exchange(exchange_name: str, kwargs: dict) -> Exchange: + """ + Loads the specified exchange. + Only checks for exchanges exported in freqtrade.exchanges + :param exchange_name: name of the module to import + :return: Exchange instance or None + """ + + try: + ex_class = getattr(exchanges, exchange_name) + + exchange = ex_class(**kwargs) + if exchange: + logger.info(f"Using resolved exchange '{exchange_name}'...") + return exchange + except AttributeError: + # Pass and raise ImportError instead + pass + + raise ImportError( + f"Impossible to load Exchange '{exchange_name}'. This class does not exist " + "or contains Python code errors." + ) diff --git a/finrl/resolvers/iresolver.py b/finrl/resolvers/iresolver.py new file mode 100644 index 000000000..b3b78fe48 --- /dev/null +++ b/finrl/resolvers/iresolver.py @@ -0,0 +1,178 @@ +# pragma pylint: disable=attribute-defined-outside-init + +""" +This module load custom objects +""" +import importlib.util +import inspect +import logging +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union + +from finrl.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +class IResolver: + """ + This class contains all the logic to load custom classes + """ + # Childclasses need to override this + object_type: Type[Any] + object_type_str: str + user_subdir: Optional[str] = None + initial_search_path: Optional[Path] + + @classmethod + def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None, + extra_dir: Optional[str] = None) -> List[Path]: + + abs_paths: List[Path] = [] + if cls.initial_search_path: + abs_paths.append(cls.initial_search_path) + + if user_subdir: + abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) + + if extra_dir: + # Add extra directory to the top of the search paths + abs_paths.insert(0, Path(extra_dir).resolve()) + + return abs_paths + + @classmethod + def _get_valid_object(cls, module_path: Path, object_name: Optional[str], + enum_failed: bool = False) -> Iterator[Any]: + """ + Generator returning objects with matching object_type and object_name in the path given. + :param module_path: absolute path to the module + :param object_name: Class name of the object + :param enum_failed: If True, will return None for modules which fail. + Otherwise, failing modules are skipped. + :return: generator containing tuple of matching objects + Tuple format: [Object, source] + """ + + # Generate spec based on absolute path + # Pass object_name as first argument to have logging print a reasonable name. + spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) # type: ignore # importlib does not use typehints + except (ModuleNotFoundError, SyntaxError, ImportError) as err: + # Catch errors in case a specific module is not installed + logger.warning(f"Could not import {module_path} due to '{err}'") + if enum_failed: + return iter([None]) + + valid_objects_gen = ( + (obj, inspect.getsource(module)) for + name, obj in inspect.getmembers( + module, inspect.isclass) if ((object_name is None or object_name == name) + and issubclass(obj, cls.object_type) + and obj is not cls.object_type) + ) + return valid_objects_gen + + @classmethod + def _search_object(cls, directory: Path, *, object_name: str, add_source: bool = False + ) -> Union[Tuple[Any, Path], Tuple[None, None]]: + """ + Search for the objectname in the given directory + :param directory: relative or absolute directory path + :param object_name: ClassName of the object to load + :return: object class + """ + logger.debug(f"Searching for {cls.object_type.__name__} {object_name} in '{directory}'") + for entry in directory.iterdir(): + # Only consider python files + if not str(entry).endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + module_path = entry.resolve() + + obj = next(cls._get_valid_object(module_path, object_name), None) + + if obj: + obj[0].__file__ = str(entry) + if add_source: + obj[0].__source__ = obj[1] + return (obj[0], module_path) + return (None, None) + + @classmethod + def _load_object(cls, paths: List[Path], *, object_name: str, add_source: bool = False, + kwargs: dict = {}) -> Optional[Any]: + """ + Try to load object from path list. + """ + + for _path in paths: + try: + (module, module_path) = cls._search_object(directory=_path, + object_name=object_name, + add_source=add_source) + if module: + logger.info( + f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} " + f"from '{module_path}'...") + return module(**kwargs) + except FileNotFoundError: + logger.warning('Path "%s" does not exist.', _path.resolve()) + + return None + + @classmethod + def load_object(cls, object_name: str, config: dict, *, kwargs: dict, + extra_dir: Optional[str] = None) -> Any: + """ + Search and loads the specified object as configured in hte child class. + :param objectname: name of the module to import + :param config: configuration dictionary + :param extra_dir: additional directory to search for the given pairlist + :raises: OperationalException if the class is invalid or does not exist. + :return: Object instance or None + """ + + abs_paths = cls.build_search_paths(config, + user_subdir=cls.user_subdir, + extra_dir=extra_dir) + + found_object = cls._load_object(paths=abs_paths, object_name=object_name, + kwargs=kwargs) + if found_object: + return found_object + raise OperationalException( + f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist " + "or contains Python code errors." + ) + + @classmethod + def search_all_objects(cls, directory: Path, + enum_failed: bool) -> List[Dict[str, Any]]: + """ + Searches a directory for valid objects + :param directory: Path to search + :param enum_failed: If True, will return None for modules which fail. + Otherwise, failing modules are skipped. + :return: List of dicts containing 'name', 'class' and 'location' entires + """ + logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") + objects = [] + for entry in directory.iterdir(): + # Only consider python files + if not str(entry).endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + module_path = entry.resolve() + logger.debug(f"Path {module_path}") + for obj in cls._get_valid_object(module_path, object_name=None, + enum_failed=enum_failed): + objects.append( + {'name': obj[0].__name__ if obj is not None else '', + 'class': obj[0] if obj is not None else None, + 'location': entry, + }) + return objects diff --git a/finrl/state.py b/finrl/state.py index 8ddff71d9..aed66a2e8 100644 --- a/finrl/state.py +++ b/finrl/state.py @@ -20,14 +20,14 @@ def __str__(self): class RunMode(Enum): """ - Bot running mode (backtest, hyperopt, ...) - can be "live", "dry-run", "backtest", "edge", "hyperopt". + Bot running mode (training, backtest, Predictions ...) + can be "live", "dry-run", "backtest", "train", "prediction" """ LIVE = "live" DRY_RUN = "dry_run" BACKTEST = "backtest" - EDGE = "edge" - HYPEROPT = "hyperopt" + PREDICTION = "predicition" + TRAIN = "train" UTIL_EXCHANGE = "util_exchange" UTIL_NO_EXCHANGE = "util_no_exchange" PLOT = "plot" @@ -35,5 +35,5 @@ class RunMode(Enum): TRADING_MODES = [RunMode.LIVE, RunMode.DRY_RUN] -OPTIMIZE_MODES = [RunMode.BACKTEST, RunMode.EDGE, RunMode.HYPEROPT] +OPTIMIZE_MODES = [RunMode.BACKTEST, RunMode.PREDICTION, RunMode.TRAIN] NON_UTIL_MODES = TRADING_MODES + OPTIMIZE_MODES diff --git a/testing_download.py b/testing_download.py new file mode 100644 index 000000000..c9685d51c --- /dev/null +++ b/testing_download.py @@ -0,0 +1,40 @@ +from finrl.config.configuration import Configuration +from finrl.config.directory_operations import create_userdata_dir +from finrl.commands import start_download_data +from pathlib import Path +from finrl.marketdata.yahoodownloader import YahooDownloader +import pandas as pd + + +#### CREATE USER DATA DIRECTORY IN DESIGNATED PATH, IF NO NAME INDICATED DEFAULT TO user_data +####### create dir to false if only to check existence of directory + +create_userdata_dir("./user_data",create_dir=True) + + +###### Pull Configuration File (using finrl/config/configuration.py) +config = Configuration.from_files(["config.json"]) + +##### EXAMPLE +##### if directory path is kept none, default = user_data +# create_userdata_dir("./finrl_testing", create_dir=True) + +##### args are the different options that could overide config options + +ARGS_DOWNLOAD_DATA = {'config': ['config.json'], 'datadir': None, + 'user_data_dir': None, 'pairs': None, 'pairs_file': None, + 'days': 160, 'timerange': None, + 'download_trades': False, 'exchange': 'binance', + 'timeframes': ['1d'], 'erase': False, + 'dataformat_ohlcv': None, 'dataformat_trades': None} + +######## downloads data to our local data repository as dictated by our config, or we could overide it using 'datadir' +start_download_data(ARGS_DOWNLOAD_DATA) + +################# fetches all our local data and outputs a df with the normal format (index:date, open, high, low, close, volume and tick symbol) +################ can be modified to get its own ARGS and overide config info, using config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + +df = YahooDownloader(config).fetch_data_crypto() + +print(df.head()) \ No newline at end of file From 742f5f3f91f81120983ba3b5dcdba66344943222 Mon Sep 17 00:00:00 2001 From: Youbad Date: Mon, 8 Feb 2021 04:49:54 -0500 Subject: [PATCH 4/7] Organize Data and FetchData --- finrl/data/fetchdata.py | 139 ++++++++++++++++++++++++ finrl/marketdata/yahoodownloader.py | 61 ++--------- testing_download.py | 4 +- user_data/data/binance/BNB_BTC-1d.json | 1 + user_data/data/binance/ETH_BTC-1d.json | 1 + user_data/data/binance/INJ_BTC-1d.json | 1 + user_data/data/binance/LINK_BTC-1d.json | 1 + user_data/data/binance/LTC_BTC-1d.json | 1 + user_data/data/binance/XMR_BTC-1d.json | 1 + user_data/data/binance/XRP_BTC-1d.json | 1 + user_data/data/binance/YFI_BTC-1d.json | 1 + 11 files changed, 157 insertions(+), 55 deletions(-) create mode 100644 finrl/data/fetchdata.py create mode 100644 user_data/data/binance/BNB_BTC-1d.json create mode 100644 user_data/data/binance/ETH_BTC-1d.json create mode 100644 user_data/data/binance/INJ_BTC-1d.json create mode 100644 user_data/data/binance/LINK_BTC-1d.json create mode 100644 user_data/data/binance/LTC_BTC-1d.json create mode 100644 user_data/data/binance/XMR_BTC-1d.json create mode 100644 user_data/data/binance/XRP_BTC-1d.json create mode 100644 user_data/data/binance/YFI_BTC-1d.json diff --git a/finrl/data/fetchdata.py b/finrl/data/fetchdata.py new file mode 100644 index 000000000..97a0ea782 --- /dev/null +++ b/finrl/data/fetchdata.py @@ -0,0 +1,139 @@ +"""Contains methods and classes to collect data from +Yahoo Finance API +""" + +import pandas as pd +import yfinance as yf +from finrl.exceptions import * +import os +import sys +import glob +from datetime import datetime + +class FetchData: + """Provides methods for retrieving daily stock data from + Yahoo Finance API + + Attributes + ---------- + start_date : str + start date of the data (modified from config.py) + end_date : str + end date of the data (modified from config.py) + ticker_list : list + a list of stock tickers (modified from config.py) + + Methods + ------- + fetch_data() + Fetches data from yahoo API + + """ + + def __init__(self, config: dict): + self.config = config + + def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> pd.DataFrame: + """Fetches data from Yahoo API + Parameters + ---------- + + Returns + ------- + `pd.DataFrame` + 7 columns: A date, open, high, low, close, volume and tick symbol + for the specified stock ticker + """ + self.start_date = start_date + self.end_date = end_date + self.ticker_list = ticker_list + # Download and save the data in a pandas DataFrame: + data_df = pd.DataFrame() + for tic in self.ticker_list: + temp_df = yf.download(tic, start=self.start_date, end=self.end_date) + temp_df["tic"] = tic + data_df = data_df.append(temp_df) + # reset the index, we want to use numbers as index instead of dates + data_df = data_df.reset_index() + try: + # convert the column names to standardized names + data_df.columns = [ + "date", + "open", + "high", + "low", + "close", + "adjcp", + "volume", + "tic", + ] + # use adjusted close price instead of close price + data_df["close"] = data_df["adjcp"] + # drop the adjusted close price column + data_df = data_df.drop("adjcp", 1) + except NotImplementedError: + print("the features are not supported currently") + # create day of the week column (monday = 0) + data_df["day"] = data_df["date"].dt.dayofweek + # convert date to standard string format, easy to filter + data_df["date"] = data_df.date.apply(lambda x: x.strftime("%Y-%m-%d")) + # drop missing data + data_df = data_df.dropna() + data_df = data_df.reset_index(drop=True) + print("Shape of DataFrame: ", data_df.shape) + # print("Display DataFrame: ", data_df.head()) + + data_df = data_df.sort_values(by=['date','tic']).reset_index(drop=True) + + return data_df + + def fetch_data_crypto(self) -> pd.DataFrame: + """ + Fetches data from local history directory (default= user_data/data/exchange) + + Parameters + ---------- + config.json ---> Exchange, Whitelist, timeframe + + Returns + ------- + `pd.DataFrame` + 7 columns: A date, open, high, low, close, volume and tick symbol + for the specified stock ticker + """ + + datadir = self.config['datadir'] + exchange = self.config["exchange"]["name"] + timeframe = self.config["timeframe"] + # Check if regex found something and only return these results + df = pd.DataFrame() + for i in self.config["pairs"]: + i = i.replace("/","_") + try: + i_df = pd.read_json(f'{os.getcwd()}/{datadir}/{i}-{timeframe}.json') + i_df["tic"] = i + i_df.columns = ["date", "open","high", "low", "close", "volume", "tic"] + i_df.date = i_df.date.apply(lambda d: datetime.fromtimestamp(d/1000)) + df = df.append(i_df) + print(f"coin {i} completed...") + except: + print(f'coin {i} not available') + pass + print(df.shape) + return df + + def select_equal_rows_stock(self, df,start_date: str, end_date: str, ticker_list: list): + self.start_date = start_date + self.end_date = end_date + self.ticker_list = ticker_list + df_check = df.tic.value_counts() + df_check = pd.DataFrame(df_check).reset_index() + df_check.columns = ["tic", "counts"] + mean_df = df_check.counts.mean() + equal_list = list(df.tic.value_counts() >= mean_df) + names = df.tic.value_counts().index + select_stocks_list = list(names[equal_list]) + df = df[df.tic.isin(select_stocks_list)] + return df + + \ No newline at end of file diff --git a/finrl/marketdata/yahoodownloader.py b/finrl/marketdata/yahoodownloader.py index c979289cc..813a01163 100644 --- a/finrl/marketdata/yahoodownloader.py +++ b/finrl/marketdata/yahoodownloader.py @@ -4,11 +4,8 @@ import pandas as pd import yfinance as yf -from finrl.exceptions import * -import os -import sys -import glob -from datetime import datetime + + class YahooDownloader: """Provides methods for retrieving daily stock data from Yahoo Finance API @@ -29,12 +26,13 @@ class YahooDownloader: """ - def __init__(self, config: dict): + def __init__(self, start_date: str, end_date: str, ticker_list: list): + self.start_date = start_date + self.end_date = end_date + self.ticker_list = ticker_list - self.config = config - - def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> pd.DataFrame: + def fetch_data(self) -> pd.DataFrame: """Fetches data from Yahoo API Parameters ---------- @@ -45,9 +43,6 @@ def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> 7 columns: A date, open, high, low, close, volume and tick symbol for the specified stock ticker """ - self.start_date = start_date - self.end_date = end_date - self.ticker_list = ticker_list # Download and save the data in a pandas DataFrame: data_df = pd.DataFrame() for tic in self.ticker_list: @@ -88,45 +83,7 @@ def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> return data_df - def fetch_data_crypto(self) -> pd.DataFrame: - """ - Fetches data from local history directory (default= user_data/data/exchange) - - Parameters - ---------- - config.json ---> Exchange, Whitelist, timeframe - - Returns - ------- - `pd.DataFrame` - 7 columns: A date, open, high, low, close, volume and tick symbol - for the specified stock ticker - """ - - datadir = self.config['datadir'] - exchange = self.config["exchange"]["name"] - timeframe = self.config["timeframe"] - # Check if regex found something and only return these results - df = pd.DataFrame() - for i in self.config["pairs"]: - i = i.replace("/","_") - try: - i_df = pd.read_json(f'{os.getcwd()}/{datadir}/{i}-{timeframe}.json') - i_df["tic"] = i - i_df.columns = ["date", "open","high", "low", "close", "volume", "tic"] - i_df.date = i_df.date.apply(lambda d: datetime.fromtimestamp(d/1000)) - df = df.append(i_df) - print(f"coin {i} completed...") - except: - print(f'coin {i} not available') - pass - print(df.shape) - return df - - def select_equal_rows_stock(self, df,start_date: str, end_date: str, ticker_list: list): - self.start_date = start_date - self.end_date = end_date - self.ticker_list = ticker_list + def select_equal_rows_stock(self, df): df_check = df.tic.value_counts() df_check = pd.DataFrame(df_check).reset_index() df_check.columns = ["tic", "counts"] @@ -136,5 +93,3 @@ def select_equal_rows_stock(self, df,start_date: str, end_date: str, ticker_list select_stocks_list = list(names[equal_list]) df = df[df.tic.isin(select_stocks_list)] return df - - \ No newline at end of file diff --git a/testing_download.py b/testing_download.py index c9685d51c..c794ae6db 100644 --- a/testing_download.py +++ b/testing_download.py @@ -2,7 +2,7 @@ from finrl.config.directory_operations import create_userdata_dir from finrl.commands import start_download_data from pathlib import Path -from finrl.marketdata.yahoodownloader import YahooDownloader +from finrl.data.fetchdata import FetchData import pandas as pd @@ -35,6 +35,6 @@ ################ can be modified to get its own ARGS and overide config info, using config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) -df = YahooDownloader(config).fetch_data_crypto() +df = FetchData(config).fetch_data_crypto() print(df.head()) \ No newline at end of file diff --git a/user_data/data/binance/BNB_BTC-1d.json b/user_data/data/binance/BNB_BTC-1d.json new file mode 100644 index 000000000..c9befa953 --- /dev/null +++ b/user_data/data/binance/BNB_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.0010939,0.0011439,0.0010631,0.0011122,896161.05],[1610323200000,0.0011122,0.0011686,0.0010716,0.0010777,1316064.5800000001],[1610409600000,0.0010777,0.001141,0.0010707,0.001124,811689.88],[1610496000000,0.001124,0.0011488,0.0010633,0.001073,930646.88],[1610582400000,0.001073,0.0010964,0.0010383,0.0010692,771390.78],[1610668800000,0.0010692,0.001145,0.0010564,0.0011136,1081973.6799999999],[1610755200000,0.0011132,0.001197,0.0010983,0.001197,1071038.24],[1610841600000,0.0011971,0.001302,0.0011829,0.0012798,1341107.8899999999],[1610928000000,0.00128,0.0013078,0.0011984,0.0012377,1031635.76],[1611014400000,0.001238,0.0012739,0.0011479,0.0011854,1205618.0700000001],[1611100800000,0.0011845,0.0012245,0.0011622,0.0012003,757916.72],[1611187200000,0.0012003,0.001285,0.0011939,0.0012521,1120086.4199999999],[1611273600000,0.0012531,0.0012847,0.0012044,0.00124,899755.99],[1611360000000,0.0012409,0.001281,0.0012281,0.0012749,710774.11],[1611446400000,0.0012749,0.001306,0.0012624,0.001297,767213.14],[1611532800000,0.0012969,0.0013119,0.0012342,0.0012958,845797.58],[1611619200000,0.001296,0.0013173,0.001261,0.0012883,861243.0600000001],[1611705600000,0.0012883,0.0013556,0.0012804,0.0013468,837454.52],[1611792000000,0.0013476,0.0013643,0.0012636,0.0012746,934701.88],[1611878400000,0.0012745,0.0013233,0.0011367,0.0012513,1765674.05],[1611964800000,0.0012516,0.001332,0.0012351,0.0013066,940466.33],[1612051200000,0.0013068,0.0013572,0.0013001,0.0013397,739923.13],[1612137600000,0.0013401,0.001556,0.0013123,0.0015347,2030911.78],[1612224000000,0.0015347,0.0015441,0.0013945,0.0014374,1389414.5800000001],[1612310400000,0.0014376,0.0014495,0.0013721,0.0013863,878126.74],[1612396800000,0.0013863,0.0015234,0.0013635,0.0015158,1206646.3799999999],[1612483200000,0.001515,0.001819,0.0014001,0.0017671,2087775.6599999999],[1612569600000,0.0017665,0.0018998,0.001584,0.0018587,2561813.7200000002],[1612656000000,0.0018577,0.0019002,0.0017188,0.0017635,1362648.25]] \ No newline at end of file diff --git a/user_data/data/binance/ETH_BTC-1d.json b/user_data/data/binance/ETH_BTC-1d.json new file mode 100644 index 000000000..01c2738db --- /dev/null +++ b/user_data/data/binance/ETH_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.03183,0.033873,0.031565,0.032878,562292.176],[1610323200000,0.032877,0.033061,0.02971,0.0307,768405.683],[1610409600000,0.030692,0.032154,0.030275,0.030851,404213.128],[1610496000000,0.030852,0.031371,0.030027,0.030232,314834.428],[1610582400000,0.030234,0.031522,0.02944,0.031451,398031.019],[1610668800000,0.031455,0.032577,0.031114,0.031812,435872.083],[1610755200000,0.031805,0.034829,0.031647,0.034114,371173.997],[1610841600000,0.03411,0.034877,0.033758,0.0344,311123.371],[1610928000000,0.0344,0.034643,0.033286,0.034357,296037.996],[1611014400000,0.034363,0.038877,0.034241,0.038079,652730.699],[1611100800000,0.038079,0.038993,0.036272,0.038819,504868.539],[1611187200000,0.03882,0.03905,0.036017,0.036036,473041.336],[1611273600000,0.036056,0.03845,0.0355,0.037402,383799.818],[1611360000000,0.037404,0.039159,0.037189,0.03845,293801.404],[1611446400000,0.038449,0.043364,0.038449,0.043163,466092.492],[1611532800000,0.04316,0.045,0.040438,0.040854,608404.258],[1611619200000,0.040853,0.0423,0.040074,0.042065,422323.023],[1611705600000,0.042065,0.042636,0.040375,0.040841,443591.086],[1611792000000,0.040833,0.0424,0.039764,0.039843,383916.529],[1611878400000,0.039838,0.040902,0.036233,0.040281,843388.973],[1611964800000,0.040282,0.040579,0.039202,0.04022,351100.102],[1612051200000,0.04022,0.040553,0.039564,0.039662,281806.768],[1612137600000,0.039664,0.041153,0.038477,0.040996,359752.565],[1612224000000,0.040999,0.043329,0.040506,0.042623,621840.635],[1612310400000,0.042623,0.044466,0.04173,0.044242,493335.917],[1612396800000,0.044245,0.044734,0.0428,0.043194,468671.189],[1612483200000,0.0432,0.04618,0.0431,0.044888,420896.429],[1612569600000,0.044892,0.045236,0.040611,0.04276,626839.154],[1612656000000,0.04276,0.043175,0.039926,0.041537,400986.455]] \ No newline at end of file diff --git a/user_data/data/binance/INJ_BTC-1d.json b/user_data/data/binance/INJ_BTC-1d.json new file mode 100644 index 000000000..5d78ddf14 --- /dev/null +++ b/user_data/data/binance/INJ_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.00011875,0.00012284,0.0001128,0.00011543,240810.1],[1610323200000,0.00011498,0.00013332,0.00011245,0.00012484,567336.6],[1610409600000,0.00012485,0.000149,0.00012349,0.00014377,598175.3],[1610496000000,0.00014377,0.00017788,0.00014364,0.00014726,1497484.6000000001],[1610582400000,0.0001472,0.00017777,0.00014444,0.0001721,1028796.1],[1610668800000,0.0001721,0.00017999,0.00014984,0.00017496,735693.5],[1610755200000,0.00017498,0.000179,0.00016751,0.00017282,416839.1],[1610841600000,0.00017276,0.00017871,0.00016661,0.00017264,335967.2],[1610928000000,0.00017263,0.00020879,0.00016751,0.000198,690592.5],[1611014400000,0.00019735,0.0002999,0.00018901,0.00024112,2175355.8999999999],[1611100800000,0.00024113,0.0002615,0.0002294,0.00024005,597787.7],[1611187200000,0.00024005,0.00025316,0.00022601,0.00022746,381727.2],[1611273600000,0.00022792,0.0003,0.0002265,0.00026785,540868.1],[1611360000000,0.00026788,0.00028024,0.00026027,0.0002627,286818.1],[1611446400000,0.0002631,0.00028012,0.0002578,0.00027269,224141.7],[1611532800000,0.00027239,0.00028648,0.00024275,0.00026073,330640.1],[1611619200000,0.00026167,0.00028046,0.00024244,0.00025634,400538.3],[1611705600000,0.00025631,0.0002688,0.000242,0.00024668,324124.2],[1611792000000,0.00024565,0.0002752,0.00023618,0.00024931,407217.9],[1611878400000,0.00024932,0.00025053,0.00020099,0.00023539,601954.8],[1611964800000,0.00023489,0.000295,0.00022622,0.00027495,764537.5],[1612051200000,0.0002766,0.00029376,0.00024628,0.00028742,487478.0],[1612137600000,0.00028717,0.00031965,0.00025525,0.00030957,552714.1],[1612224000000,0.00030991,0.0003369,0.00027949,0.00032479,449930.4],[1612310400000,0.00032478,0.00033168,0.00029476,0.00030109,347558.7],[1612396800000,0.00030156,0.00036611,0.000292,0.00034589,444930.1],[1612483200000,0.00034481,0.0003644,0.0003229,0.00032361,393771.8],[1612569600000,0.0003229,0.00032684,0.00027747,0.00028651,754613.5],[1612656000000,0.00028635,0.00030575,0.00027177,0.00029718,382123.3]] \ No newline at end of file diff --git a/user_data/data/binance/LINK_BTC-1d.json b/user_data/data/binance/LINK_BTC-1d.json new file mode 100644 index 000000000..87725a590 --- /dev/null +++ b/user_data/data/binance/LINK_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.00043793,0.00045424,0.00042,0.00042458,4111541.2999999998],[1610323200000,0.00042462,0.0004323,0.00040271,0.00041321,3755678.2000000002],[1610409600000,0.00041313,0.00042983,0.000405,0.00041017,2176875.6000000001],[1610496000000,0.00041018,0.00046093,0.00040654,0.00042669,3192078.7999999998],[1610582400000,0.00042671,0.00046016,0.00040655,0.00045918,3482017.5],[1610668800000,0.00045906,0.0005829,0.00045541,0.00056458,9806976.0999999996],[1610755200000,0.00056443,0.00062175,0.000551,0.00055913,7475094.0],[1610841600000,0.00055867,0.0006624,0.00054335,0.00064887,5821751.2999999998],[1610928000000,0.00064957,0.0006595,0.00057817,0.00060157,4610946.2999999998],[1611014400000,0.00060161,0.00061434,0.00056473,0.000573,3384645.1000000001],[1611100800000,0.00057311,0.00061751,0.00055934,0.00061616,2874333.5],[1611187200000,0.00061599,0.000638,0.00059221,0.00059713,3156647.1000000001],[1611273600000,0.00059678,0.00069077,0.00058323,0.00065398,4689245.2999999998],[1611360000000,0.00065452,0.00079499,0.00065098,0.0007732,4884734.5999999996],[1611446400000,0.00077325,0.000785,0.00072538,0.00076773,3290066.7000000002],[1611532800000,0.00076867,0.00079245,0.0006896,0.00072392,3742641.1000000001],[1611619200000,0.00072395,0.00073516,0.00069512,0.0007113,2328659.1000000001],[1611705600000,0.00071144,0.00071412,0.00068045,0.00069091,1862080.2],[1611792000000,0.00069047,0.00077777,0.00068594,0.00069004,3153516.7999999998],[1611878400000,0.00068881,0.00070483,0.00060235,0.00066449,5693779.0999999996],[1611964800000,0.00066415,0.00070376,0.00064612,0.00069,2723362.8999999999],[1612051200000,0.00069,0.00072136,0.00067405,0.00068143,2208869.2999999998],[1612137600000,0.00068197,0.00068986,0.00064378,0.0006826,2405125.2999999998],[1612224000000,0.00068262,0.00069646,0.00064398,0.00066615,2283612.7999999998],[1612310400000,0.00066668,0.00070195,0.00064747,0.00066687,2879923.8999999999],[1612396800000,0.0006668,0.00069117,0.000616,0.00066317,3496642.2999999998],[1612483200000,0.00066339,0.00072,0.00051,0.00068631,5817233.5],[1612569600000,0.00068647,0.00069681,0.00060361,0.00063858,4177615.3999999999],[1612656000000,0.00063877,0.00064555,0.0006115,0.00063734,1982891.5]] \ No newline at end of file diff --git a/user_data/data/binance/LTC_BTC-1d.json b/user_data/data/binance/LTC_BTC-1d.json new file mode 100644 index 000000000..cb03ba8d1 --- /dev/null +++ b/user_data/data/binance/LTC_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.004423,0.004614,0.004294,0.004458,585515.91],[1610323200000,0.004459,0.004535,0.003663,0.003922,1232906.8400000001],[1610409600000,0.003922,0.004078,0.003841,0.003918,623934.42],[1610496000000,0.003919,0.004039,0.003853,0.003943,381866.28],[1610582400000,0.003946,0.003973,0.003807,0.003894,380993.45],[1610668800000,0.003894,0.004005,0.00379,0.00391,518554.59],[1610755200000,0.00391,0.004062,0.003876,0.003988,422530.14],[1610841600000,0.003987,0.004102,0.003944,0.003985,400756.51],[1610928000000,0.003986,0.004192,0.00388,0.004151,387144.2],[1611014400000,0.004149,0.004469,0.00411,0.004231,513199.2],[1611100800000,0.00423,0.004304,0.004094,0.00422,430819.77],[1611187200000,0.004222,0.004264,0.004082,0.004207,415626.97],[1611273600000,0.004207,0.004462,0.004133,0.004178,428668.54],[1611360000000,0.004176,0.004363,0.004162,0.004291,229290.02],[1611446400000,0.004291,0.0044,0.004269,0.004373,222412.35],[1611532800000,0.004373,0.0045,0.004164,0.004249,353144.02],[1611619200000,0.00425,0.004288,0.004125,0.004148,256807.35],[1611705600000,0.004148,0.004173,0.003995,0.004034,295778.66],[1611792000000,0.004034,0.004171,0.003986,0.003994,264330.74],[1611878400000,0.003995,0.004145,0.00373,0.00393,689379.3],[1611964800000,0.003933,0.003965,0.003798,0.003891,362106.12],[1612051200000,0.003892,0.003956,0.003825,0.003909,235610.49],[1612137600000,0.00391,0.003989,0.003811,0.003939,313288.55],[1612224000000,0.003937,0.004194,0.003925,0.004,467082.57],[1612310400000,0.004003,0.004278,0.004002,0.004146,465504.1],[1612396800000,0.004144,0.004175,0.003901,0.003927,498771.28],[1612483200000,0.003927,0.004172,0.003896,0.004049,428951.12],[1612569600000,0.004047,0.00416,0.003881,0.003968,502336.83],[1612656000000,0.00397,0.004007,0.003874,0.003886,323744.65]] \ No newline at end of file diff --git a/user_data/data/binance/XMR_BTC-1d.json b/user_data/data/binance/XMR_BTC-1d.json new file mode 100644 index 000000000..a3c390526 --- /dev/null +++ b/user_data/data/binance/XMR_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.003676,0.004953,0.00361,0.004842,277649.261],[1610323200000,0.004842,0.004986,0.004127,0.004457,295979.862],[1610409600000,0.004453,0.005013,0.004394,0.004673,176951.933],[1610496000000,0.004673,0.00484,0.004482,0.004584,162449.807],[1610582400000,0.004582,0.004582,0.004155,0.004184,109990.821],[1610668800000,0.004187,0.00445,0.00407,0.00427,82153.955],[1610755200000,0.00427,0.004386,0.0042,0.004318,87965.191],[1610841600000,0.004322,0.004512,0.004295,0.004458,90445.351],[1610928000000,0.004456,0.004483,0.00411,0.004284,92342.316],[1611014400000,0.004285,0.00455,0.004097,0.004376,108822.158],[1611100800000,0.004376,0.004471,0.004255,0.004307,78985.694],[1611187200000,0.004306,0.004431,0.004217,0.004228,102287.846],[1611273600000,0.00423,0.004309,0.004086,0.004119,89374.453],[1611360000000,0.004119,0.004397,0.004119,0.004331,61608.443],[1611446400000,0.004332,0.004384,0.004222,0.004281,58057.823],[1611532800000,0.004279,0.004305,0.004095,0.004256,72089.882],[1611619200000,0.004249,0.004391,0.004202,0.004244,90071.256],[1611705600000,0.004243,0.004381,0.00411,0.004131,129022.66],[1611792000000,0.004131,0.004459,0.004022,0.004059,125795.657],[1611878400000,0.004063,0.00417,0.003701,0.004098,147179.245],[1611964800000,0.004098,0.004232,0.003991,0.004069,75166.715],[1612051200000,0.004067,0.004178,0.004029,0.004163,80151.255],[1612137600000,0.004163,0.004358,0.004117,0.004275,126957.75],[1612224000000,0.004275,0.004313,0.004087,0.004259,109853.541],[1612310400000,0.004263,0.00433,0.004047,0.00411,99533.306],[1612396800000,0.004111,0.004119,0.003948,0.004006,100918.95],[1612483200000,0.00401,0.004102,0.003956,0.004059,79707.062],[1612569600000,0.004061,0.004061,0.003797,0.003881,101098.22],[1612656000000,0.003882,0.004042,0.003847,0.003878,77980.144]] \ No newline at end of file diff --git a/user_data/data/binance/XRP_BTC-1d.json b/user_data/data/binance/XRP_BTC-1d.json new file mode 100644 index 000000000..a32da8084 --- /dev/null +++ b/user_data/data/binance/XRP_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.00000808,0.00000909,0.00000801,0.00000824,335849348.0],[1610323200000,0.00000825,0.00000886,0.0000075,0.00000812,385366406.0],[1610409600000,0.00000811,0.0000088,0.0000076,0.0000086,240835208.0],[1610496000000,0.00000858,0.00000891,0.0000081,0.00000818,161816300.0],[1610582400000,0.00000818,0.00000819,0.00000741,0.00000757,186772052.0],[1610668800000,0.00000758,0.00000783,0.00000749,0.00000761,165925182.0],[1610755200000,0.00000761,0.00000784,0.00000752,0.00000774,127505493.0],[1610841600000,0.00000776,0.00000794,0.00000763,0.00000774,96972319.0],[1610928000000,0.00000775,0.0000079,0.00000763,0.00000778,90361168.0],[1611014400000,0.0000078,0.00000906,0.00000774,0.00000819,257865675.0],[1611100800000,0.0000082,0.00000874,0.00000804,0.00000832,123240635.0],[1611187200000,0.00000833,0.00000889,0.00000831,0.00000869,187295800.0],[1611273600000,0.00000868,0.00000876,0.00000816,0.00000826,169780586.0],[1611360000000,0.00000826,0.00000868,0.00000826,0.00000847,88809560.0],[1611446400000,0.00000847,0.00000869,0.00000833,0.00000848,69863280.0],[1611532800000,0.00000848,0.00000863,0.00000795,0.00000829,105054239.0],[1611619200000,0.00000828,0.00000849,0.00000817,0.00000827,89264084.0],[1611705600000,0.00000826,0.00000838,0.00000809,0.00000825,81365867.0],[1611792000000,0.00000825,0.00000836,0.00000785,0.00000792,114924389.0],[1611878400000,0.00000791,0.00000965,0.00000742,0.00000827,545934729.0],[1611964800000,0.00000826,0.00001532,0.00000817,0.00001295,1898105958.0],[1612051200000,0.00001294,0.00001515,0.00001138,0.00001497,1374584167.0],[1612137600000,0.00001497,0.00002219,0.00001067,0.00001108,2844400378.0],[1612224000000,0.00001108,0.00001177,0.00000993,0.00001051,674118953.0],[1612310400000,0.0000105,0.00001142,0.00001019,0.00001055,350972515.0],[1612396800000,0.00001055,0.00001286,0.0000101,0.00001206,558336591.0],[1612483200000,0.00001206,0.00001271,0.00001136,0.00001181,368880235.0],[1612569600000,0.00001181,0.00001195,0.00001067,0.00001131,220671819.0],[1612656000000,0.00001131,0.00001161,0.0000104,0.00001079,269747506.0]] \ No newline at end of file diff --git a/user_data/data/binance/YFI_BTC-1d.json b/user_data/data/binance/YFI_BTC-1d.json new file mode 100644 index 000000000..b17b76b07 --- /dev/null +++ b/user_data/data/binance/YFI_BTC-1d.json @@ -0,0 +1 @@ +[[1610236800000,0.89371,0.93487,0.86159,0.88936,743.03],[1610323200000,0.89063,0.92459,0.81746,0.84953,901.1919],[1610409600000,0.85019,0.91285,0.82892,0.87312,525.107],[1610496000000,0.87328,0.978,0.85656,0.89861,860.7165],[1610582400000,0.89932,0.902,0.80834,0.84143,509.677],[1610668800000,0.84161,0.86225,0.81283,0.82127,603.0692],[1610755200000,0.82128,0.95939,0.815,0.93814,881.8335],[1610841600000,0.93968,1.0233,0.91314,0.96503,1027.5019],[1610928000000,0.96549,1.11025,0.96549,1.03605,1017.5169],[1611014400000,1.03669,1.052,0.93546,0.95772,605.4637],[1611100800000,0.95771,0.9847,0.92384,0.95884,535.9971],[1611187200000,0.95873,0.96234,0.88353,0.90402,497.4276],[1611273600000,0.90485,0.98949,0.8847,0.93339,469.3682],[1611360000000,0.93429,0.97153,0.91333,0.95172,475.7279],[1611446400000,0.95204,1.04008,0.95186,0.99885,507.6008],[1611532800000,0.99858,1.01542,0.90294,0.90757,549.392],[1611619200000,0.90858,0.947,0.90634,0.93144,361.1257],[1611705600000,0.9308,0.93948,0.89925,0.9102,286.3793],[1611792000000,0.91021,0.93958,0.88848,0.88959,229.9818],[1611878400000,0.88967,0.90028,0.77584,0.87374,525.8227],[1611964800000,0.87458,0.94499,0.84338,0.91051,351.2875],[1612051200000,0.9106,0.98564,0.90589,0.91179,437.0897],[1612137600000,0.91155,0.9299,0.873,0.92987,265.3799],[1612224000000,0.92987,0.93632,0.87401,0.89404,344.821],[1612310400000,0.89474,0.95006,0.88758,0.90559,465.0828],[1612396800000,0.9078,0.93542,0.80111,0.84125,883.6873],[1612483200000,0.84221,0.9,0.82554,0.84795,596.6591],[1612569600000,0.84834,0.85761,0.75801,0.8054,457.9554],[1612656000000,0.80535,0.815,0.77947,0.79512,151.571]] \ No newline at end of file From d6ef8266f90bca80f88eddfcc96db05d27a72f1c Mon Sep 17 00:00:00 2001 From: Youbad Date: Mon, 8 Feb 2021 05:02:13 -0500 Subject: [PATCH 5/7] Minor Discrepancy in Docker requirements --- docker/requirements.txt | 3 ++- finrl/commands/__init__.py | 2 +- finrl/commands/data_commands.py | 2 +- testing_download.py | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker/requirements.txt b/docker/requirements.txt index dfa642608..d852501f3 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -143,4 +143,5 @@ arrow python-rapidjson questionary sqlalchemy -tabulate \ No newline at end of file +tabulate +ccxt \ No newline at end of file diff --git a/finrl/commands/__init__.py b/finrl/commands/__init__.py index 6aff3724f..3fbce3a44 100644 --- a/finrl/commands/__init__.py +++ b/finrl/commands/__init__.py @@ -4,4 +4,4 @@ Contains all start-commands, subcommands and CLI Interface creation. """ from finrl.commands.deploy_commands import start_create_userdir -from finrl.commands.data_commands import start_download_data \ No newline at end of file +from finrl.commands.data_commands import start_download_cryptodata \ No newline at end of file diff --git a/finrl/commands/data_commands.py b/finrl/commands/data_commands.py index ada65ac92..f8202121e 100644 --- a/finrl/commands/data_commands.py +++ b/finrl/commands/data_commands.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -def start_download_data(args: Dict[str, Any]) -> None: +def start_download_cryptodata(args: Dict[str, Any]) -> None: """ Download data (former download_backtest_data.py script) """ diff --git a/testing_download.py b/testing_download.py index c794ae6db..91abbe486 100644 --- a/testing_download.py +++ b/testing_download.py @@ -1,6 +1,6 @@ from finrl.config.configuration import Configuration from finrl.config.directory_operations import create_userdata_dir -from finrl.commands import start_download_data +from finrl.commands import start_download_cryptodata from pathlib import Path from finrl.data.fetchdata import FetchData import pandas as pd @@ -29,7 +29,7 @@ 'dataformat_ohlcv': None, 'dataformat_trades': None} ######## downloads data to our local data repository as dictated by our config, or we could overide it using 'datadir' -start_download_data(ARGS_DOWNLOAD_DATA) +start_download_cryptodata(ARGS_DOWNLOAD_DATA) ################# fetches all our local data and outputs a df with the normal format (index:date, open, high, low, close, volume and tick symbol) ################ can be modified to get its own ARGS and overide config info, using config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) From a8195261052001b28bed24128dd944a70c79c75f Mon Sep 17 00:00:00 2001 From: Youbad Date: Mon, 8 Feb 2021 08:49:55 -0500 Subject: [PATCH 6/7] Same Method for Stock Market using Ticker List. --- config.json | 32 ++++++++++++++++ finrl/commands/__init__.py | 2 +- finrl/commands/data_commands.py | 68 ++++++++++++++++++++++++++++++++- finrl/config/configuration.py | 3 +- finrl/data/fetchdata.py | 23 +++++------ finrl/exchange/exchange.py | 1 + testing_download.py | 44 ++++++++++++++------- 7 files changed, 143 insertions(+), 30 deletions(-) diff --git a/config.json b/config.json index 287adf47d..2286b8342 100644 --- a/config.json +++ b/config.json @@ -30,6 +30,38 @@ ] }, + "ticker_list": [ + "AAPL", + "MSFT", + "JPM", + "V", + "RTX", + "PG", + "GS", + "NKE", + "DIS", + "AXP", + "HD", + "INTC", + "WMT", + "IBM", + "MRK", + "UNH", + "KO", + "CAT", + "TRV", + "JNJ", + "CVX", + "MCD", + "VZ", + "CSCO", + "XOM", + "BA", + "MMM", + "PFE", + "WBA", + "DD" + ], "pairlists": [ {"method": "StaticPairList"} ], diff --git a/finrl/commands/__init__.py b/finrl/commands/__init__.py index 3fbce3a44..b50c484ba 100644 --- a/finrl/commands/__init__.py +++ b/finrl/commands/__init__.py @@ -4,4 +4,4 @@ Contains all start-commands, subcommands and CLI Interface creation. """ from finrl.commands.deploy_commands import start_create_userdir -from finrl.commands.data_commands import start_download_cryptodata \ No newline at end of file +from finrl.commands.data_commands import start_download_cryptodata, start_download_stockdata \ No newline at end of file diff --git a/finrl/commands/data_commands.py b/finrl/commands/data_commands.py index f8202121e..f74445b9c 100644 --- a/finrl/commands/data_commands.py +++ b/finrl/commands/data_commands.py @@ -1,9 +1,15 @@ import logging import sys +import yfinance +import pandas as pd +import yfinance as yf +import os + from collections import defaultdict from datetime import datetime, timedelta from typing import Any, Dict, List + from finrl.config import TimeRange, setup_utils_configuration from finrl.data.converter import convert_ohlcv_format, convert_trades_format from finrl.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, @@ -19,7 +25,16 @@ def start_download_cryptodata(args: Dict[str, Any]) -> None: """ - Download data (former download_backtest_data.py script) + Parameters: + ARGS_DOWNLOAD_DATA = {'config': ['config.json'], 'datadir': None, + 'user_data_dir': None, 'pairs': None, 'pairs_file': None, + 'days': 160, 'timerange': None, + 'download_trades': False, 'exchange': 'binance', + 'timeframes': ['1d'], 'erase': False, + 'dataformat_ohlcv': None, 'dataformat_trades': None} + + Returns: + Json files in user_data/data/exchange/*.json """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) if 'days' in config and 'timerange' in config: @@ -82,6 +97,57 @@ def start_download_cryptodata(args: Dict[str, Any]) -> None: logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " f"on exchange {exchange.name}.") +def start_download_stockdata(args: Dict[str, Any]) -> None: + """Fetches data from Yahoo API + Parameters + ---------- + ticker_list, timerange, + Returns + ------- + Json of data + """ + args["exchange"] = "yahoo" + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + + if 'days' in config and 'timerange' in config: + raise OperationalException("--days and --timerange are mutually exclusive. " + "You can only specify one or the other.") + + config["datadir"] = "user_data/data/yahoo" + + timerange = TimeRange() + if 'days' in config: + time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + start = datetime.fromtimestamp(timerange.startts).strftime("%Y-%m-%d") + end = datetime.now().strftime("%Y-%m-%d") + + if 'timerange' in config: + timerange = timerange.parse_timerange(config['timerange']) + start = datetime.fromtimestamp(timerange.startts).strftime("%Y-%m-%d") + end = datetime.fromtimestamp(timerange.stopts).strftime("%Y-%m-%d") + try: + data_df = pd.DataFrame() + for tic in config['ticker_list']: + temp_df = yf.download(tic, start=start, end=end) + temp_df.columns = [ + "open", + "high", + "low", + "close", + "adjcp", + "volume", + ] + temp_df["close"] = temp_df["adjcp"] + temp_df = temp_df.drop(["adjcp"], axis=1) + temp_df.to_json(f'{os.getcwd()}/{config["datadir"]}/{tic}.json') + except KeyboardInterrupt: + sys.exit("Interrupt received, aborting ...") + + + + def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ diff --git a/finrl/config/configuration.py b/finrl/config/configuration.py index 6bf2e7fba..eb5b95829 100644 --- a/finrl/config/configuration.py +++ b/finrl/config/configuration.py @@ -101,7 +101,6 @@ def load_config(self) -> Dict[str, Any]: self._process_optimize_options(config) - # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -169,6 +168,8 @@ def _process_optimize_options(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='timerange', logstring='Parameter --timerange detected: {} ...') + self._args_to_config(config, argname='days', + logstring='Parameter --days detected: {} ...') self._process_datadir_options(config) diff --git a/finrl/data/fetchdata.py b/finrl/data/fetchdata.py index 97a0ea782..c4b626ead 100644 --- a/finrl/data/fetchdata.py +++ b/finrl/data/fetchdata.py @@ -33,7 +33,7 @@ class FetchData: def __init__(self, config: dict): self.config = config - def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> pd.DataFrame: + def fetch_data_stock(self) -> pd.DataFrame: """Fetches data from Yahoo API Parameters ---------- @@ -44,14 +44,16 @@ def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> 7 columns: A date, open, high, low, close, volume and tick symbol for the specified stock ticker """ - self.start_date = start_date - self.end_date = end_date - self.ticker_list = ticker_list + exchange = "yahoo" + datadir = f'{self.config["user_data_dir"]}/data/{exchange}' + print(datadir) + timeframe = self.config["timeframe"] + ticker_list = self.config["ticker_list"] # Download and save the data in a pandas DataFrame: data_df = pd.DataFrame() - for tic in self.ticker_list: - temp_df = yf.download(tic, start=self.start_date, end=self.end_date) - temp_df["tic"] = tic + for i in ticker_list: + temp_df = pd.read_json(f'{os.getcwd()}/{datadir}/{i}.json') + temp_df["tic"] = i data_df = data_df.append(temp_df) # reset the index, we want to use numbers as index instead of dates data_df = data_df.reset_index() @@ -63,14 +65,9 @@ def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> "high", "low", "close", - "adjcp", "volume", "tic", ] - # use adjusted close price instead of close price - data_df["close"] = data_df["adjcp"] - # drop the adjusted close price column - data_df = data_df.drop("adjcp", 1) except NotImplementedError: print("the features are not supported currently") # create day of the week column (monday = 0) @@ -84,7 +81,7 @@ def fetch_data_stock(self,start_date: str, end_date: str, ticker_list: list) -> # print("Display DataFrame: ", data_df.head()) data_df = data_df.sort_values(by=['date','tic']).reset_index(drop=True) - + print(data_df.head()) return data_df def fetch_data_crypto(self) -> pd.DataFrame: diff --git a/finrl/exchange/exchange.py b/finrl/exchange/exchange.py index b196c076c..1c87ef921 100644 --- a/finrl/exchange/exchange.py +++ b/finrl/exchange/exchange.py @@ -1258,6 +1258,7 @@ def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: """ Return the list of all exchanges known to ccxt """ + ccxt.exchanges.append("yahoo") return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges diff --git a/testing_download.py b/testing_download.py index 91abbe486..058454e17 100644 --- a/testing_download.py +++ b/testing_download.py @@ -1,18 +1,20 @@ from finrl.config.configuration import Configuration from finrl.config.directory_operations import create_userdata_dir -from finrl.commands import start_download_cryptodata +from finrl.commands import start_download_cryptodata, start_download_stockdata from pathlib import Path from finrl.data.fetchdata import FetchData import pandas as pd - +from finrl.config import TimeRange +from datetime import datetime, timedelta +import arrow #### CREATE USER DATA DIRECTORY IN DESIGNATED PATH, IF NO NAME INDICATED DEFAULT TO user_data ####### create dir to false if only to check existence of directory -create_userdata_dir("./user_data",create_dir=True) +# create_userdata_dir("./user_data",create_dir=True) -###### Pull Configuration File (using finrl/config/configuration.py) +# ###### Pull Configuration File (using finrl/config/configuration.py) config = Configuration.from_files(["config.json"]) ##### EXAMPLE @@ -21,20 +23,34 @@ ##### args are the different options that could overide config options +# ARGS_DOWNLOAD_DATA = {'config': ['config.json'], 'datadir': None, +# 'user_data_dir': None, 'pairs': None, 'pairs_file': None, +# 'days': 160, 'timerange': None, +# 'download_trades': False, 'exchange': 'binance', +# 'timeframes': ['1d'], 'erase': False, +# 'dataformat_ohlcv': None, 'dataformat_trades': None} + +# ######## downloads data to our local data repository as dictated by our config, or we could overide it using 'datadir' +# start_download_cryptodata(ARGS_DOWNLOAD_DATA) + +# ################# fetches all our local data and outputs a df with the normal format (index:date, open, high, low, close, volume and tick symbol) +# ################ can be modified to get its own ARGS and overide config info, using config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + +# df = FetchData(config).fetch_data_crypto() + +# print(df.head()) + +################## Either input timerange or days for period of download ARGS_DOWNLOAD_DATA = {'config': ['config.json'], 'datadir': None, - 'user_data_dir': None, 'pairs': None, 'pairs_file': None, - 'days': 160, 'timerange': None, - 'download_trades': False, 'exchange': 'binance', - 'timeframes': ['1d'], 'erase': False, - 'dataformat_ohlcv': None, 'dataformat_trades': None} + 'user_data_dir': None, 'days': None, 'timerange': "20200101-20210101", + 'timeframes': ['1d'], 'erase': False} + -######## downloads data to our local data repository as dictated by our config, or we could overide it using 'datadir' -start_download_cryptodata(ARGS_DOWNLOAD_DATA) +start_download_stockdata(ARGS_DOWNLOAD_DATA) -################# fetches all our local data and outputs a df with the normal format (index:date, open, high, low, close, volume and tick symbol) -################ can be modified to get its own ARGS and overide config info, using config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) -df = FetchData(config).fetch_data_crypto() +df = FetchData(config).fetch_data_stock() print(df.head()) \ No newline at end of file From 0b35c42fa5bf536f1c2befb7b47983f32ea4888a Mon Sep 17 00:00:00 2001 From: Youbad Date: Wed, 10 Feb 2021 04:34:07 -0500 Subject: [PATCH 7/7] gitignore data --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a77e7d0e4..c47922ecf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Custom config #results/ #USER_DATA -./user_data +user_data/ # remove DS_Store **/.DS_Store @@ -35,6 +35,7 @@ share/python-wheels/ *.egg MANIFEST + # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it.