Merge pull request #1 from Youbadawy/GENERAL_DATA_REPO

General data repo

.gitignore

@@ -1,6 +1,7 @@
 # Custom config
 #results/
-
+#USER_DATA
+user_data/
 
 # remove DS_Store
 **/.DS_Store
@@ -34,6 +35,7 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -127,6 +129,8 @@ celerybeat.pid
 # mkdocs documentation
 /site
 
+
+
 # mypy
 .mypy_cache/
 .dmypy.json

config.json

@@ -4,7 +4,7 @@
     "stake_amount": 0.1,
     "tradable_balance_ratio": 0.99,
     "fiat_display_currency": "USD",
-    "timeframe": "5m",
+    "timeframe": "1d",
     "dry_run": true,
 
     "exchange": {
@@ -30,6 +30,38 @@
 
         ]
     },
+    "ticker_list": [
+        "AAPL",
+        "MSFT",
+        "JPM",
+        "V",
+        "RTX",
+        "PG",
+        "GS",
+        "NKE",
+        "DIS",
+        "AXP",
+        "HD",
+        "INTC",
+        "WMT",
+        "IBM",
+        "MRK",
+        "UNH",
+        "KO",
+        "CAT",
+        "TRV",
+        "JNJ",
+        "CVX",
+        "MCD",
+        "VZ",
+        "CSCO",
+        "XOM",
+        "BA",
+        "MMM",
+        "PFE",
+        "WBA",
+        "DD"
+    ],
     "pairlists": [
         {"method": "StaticPairList"}
     ],
@@ -47,5 +79,8 @@
         "CORS_origins": [],
         "username": "",
         "password": ""
-    }
+    },
+    "dataformat_ohlcv": "json",
+    "dataformat_trades": "jsongz",
+    "user_data_dir":"./user_data/"
 }

docker/requirements.txt

@@ -139,3 +139,9 @@ typepy==1.1.2
 Werkzeug==1.0.1
 yfinance==0.1.55
 zipp==3.4.0
+arrow
+python-rapidjson
+questionary
+sqlalchemy
+tabulate
+ccxt

finrl/commands/__init__.py

@@ -0,0 +1,7 @@
+# flake8: noqa: F401
+"""
+Commands module.
+Contains all start-commands, subcommands and CLI Interface creation.
+"""
+from finrl.commands.deploy_commands import start_create_userdir
+from finrl.commands.data_commands import start_download_cryptodata, start_download_stockdata

finrl/commands/data_commands.py

@@ -0,0 +1,192 @@
+import logging
+import sys
+import yfinance
+import pandas as pd
+import yfinance as yf
+import os
+
+from collections import defaultdict
+from datetime import datetime, timedelta
+from typing import Any, Dict, List
+
+
+from finrl.config import TimeRange, setup_utils_configuration
+from finrl.data.converter import convert_ohlcv_format, convert_trades_format
+from finrl.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
+                                  refresh_backtest_trades_data)
+from finrl.exceptions import OperationalException
+from finrl.exchange import timeframe_to_minutes
+from finrl.resolvers import ExchangeResolver
+from finrl.state import RunMode
+
+
+logger = logging.getLogger(__name__)
+
+
+def start_download_cryptodata(args: Dict[str, Any]) -> None:
+    """
+    Parameters:
+    ARGS_DOWNLOAD_DATA = {'config': ['config.json'], 'datadir': None, 
+                      'user_data_dir': None, 'pairs': None, 'pairs_file': None, 
+                      'days': 160, 'timerange': None, 
+                      'download_trades': False, 'exchange': 'binance', 
+                      'timeframes': ['1d'], 'erase': False, 
+                      'dataformat_ohlcv': None, 'dataformat_trades': None}
+    
+    Returns:
+    Json files in user_data/data/exchange/*.json
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+    if 'days' in config and 'timerange' in config:
+        raise OperationalException("--days and --timerange are mutually exclusive. "
+                                   "You can only specify one or the other.")
+    timerange = TimeRange()
+    if 'days' in config:
+        time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
+        timerange = TimeRange.parse_timerange(f'{time_since}-')
+
+    if 'timerange' in config:
+        timerange = timerange.parse_timerange(config['timerange'])
+
+    # Remove stake-currency to skip checks which are not relevant for datadownload
+    config['stake_currency'] = ''
+
+    if 'pairs' not in config:
+        raise OperationalException(
+            "Downloading data requires a list of pairs. "
+            "Please check the documentation on how to configure this.")
+
+    logger.info(f"About to download pairs: {config['pairs']}, "
+                f"intervals: {config['timeframes']} to {config['datadir']}")
+
+    pairs_not_available: List[str] = []
+
+    # Init exchange
+    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
+    # Manual validations of relevant settings
+    exchange.validate_pairs(config['pairs'])
+    for timeframe in config['timeframes']:
+        exchange.validate_timeframes(timeframe)
+
+    try:
+
+        if config.get('download_trades'):
+            pairs_not_available = refresh_backtest_trades_data(
+                exchange, pairs=config['pairs'], datadir=config['datadir'],
+                timerange=timerange, erase=bool(config.get('erase')),
+                data_format=config['dataformat_trades'])
+
+            # Convert downloaded trade data to different timeframes
+            convert_trades_to_ohlcv(
+                pairs=config['pairs'], timeframes=config['timeframes'],
+                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
+                data_format_ohlcv=config['dataformat_ohlcv'],
+                data_format_trades=config['dataformat_trades'],
+                )
+        else:
+            pairs_not_available = refresh_backtest_ohlcv_data(
+                exchange, pairs=config['pairs'], timeframes=config['timeframes'],
+                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
+                data_format=config['dataformat_ohlcv'])
+
+    except KeyboardInterrupt:
+        sys.exit("Interrupt received, aborting ...")
+
+    finally:
+        if pairs_not_available:
+            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
+                        f"on exchange {exchange.name}.")
+
+def start_download_stockdata(args: Dict[str, Any]) -> None:
+    """Fetches data from Yahoo API
+    Parameters
+    ----------
+    ticker_list, timerange, 
+    Returns
+    -------
+    Json of data
+    """
+    args["exchange"] = "yahoo"
+    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
+
+
+    if 'days' in config and 'timerange' in config:
+        raise OperationalException("--days and --timerange are mutually exclusive. "
+                                    "You can only specify one or the other.")
+
+    config["datadir"] = "user_data/data/yahoo"
+
+    timerange = TimeRange()
+    if 'days' in config:
+        time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
+        timerange = TimeRange.parse_timerange(f'{time_since}-')
+        start = datetime.fromtimestamp(timerange.startts).strftime("%Y-%m-%d")
+        end = datetime.now().strftime("%Y-%m-%d")
+
+    if 'timerange' in config:
+        timerange = timerange.parse_timerange(config['timerange'])
+        start = datetime.fromtimestamp(timerange.startts).strftime("%Y-%m-%d")
+        end = datetime.fromtimestamp(timerange.stopts).strftime("%Y-%m-%d")
+    try:
+        data_df = pd.DataFrame()
+        for tic in config['ticker_list']:
+            temp_df = yf.download(tic, start=start, end=end)
+            temp_df.columns = [
+                "open",
+                "high",
+                "low",
+                "close",
+                "adjcp",
+                "volume",
+            ]
+            temp_df["close"] = temp_df["adjcp"]
+            temp_df = temp_df.drop(["adjcp"], axis=1)
+            temp_df.to_json(f'{os.getcwd()}/{config["datadir"]}/{tic}.json')
+    except KeyboardInterrupt:
+        sys.exit("Interrupt received, aborting ...")
+
+
+
+
+
+def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
+    """
+    Convert data from one format to another
+    """
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+    if ohlcv:
+        convert_ohlcv_format(config,
+                             convert_from=args['format_from'], convert_to=args['format_to'],
+                             erase=args['erase'])
+    else:
+        convert_trades_format(config,
+                              convert_from=args['format_from'], convert_to=args['format_to'],
+                              erase=args['erase'])
+
+
+def start_list_data(args: Dict[str, Any]) -> None:
+    """
+    List available backtest data
+    """
+
+    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
+
+    from tabulate import tabulate
+
+    from freqtrade.data.history.idatahandler import get_datahandler
+    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
+
+    paircombs = dhc.ohlcv_get_available_data(config['datadir'])
+
+    if args['pairs']:
+        paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
+
+    print(f"Found {len(paircombs)} pair / timeframe combinations.")
+    groupedpair = defaultdict(list)
+    for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))):
+        groupedpair[pair].append(timeframe)
+
+    if groupedpair:
+        print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()],
+                       headers=("Pair", "Timeframe"),
+                       tablefmt='psql', stralign='right'))

finrl/commands/deploy_commands.py

@@ -0,0 +1,27 @@
+import logging
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+from finrl.config import setup_utils_configuration
+from finrl.config.directory_operations import copy_sample_files, create_userdata_dir
+from finrl.exceptions import OperationalException
+from finrl.misc import render_template, render_template_with_fallback
+from finrl.state import RunMode
+
+
+logger = logging.getLogger(__name__)
+
+
+def start_create_userdir(args: Dict[str, Any]) -> None:
+    """
+    Create "user_data" directory to contain user data strategies, hyperopt, ...)
+    :param args: Cli args from Arguments()
+    :return: None
+    """
+    if "user_data_dir" in args and args["user_data_dir"]:
+        userdir = create_userdata_dir(args["user_data_dir"], create_dir=True)
+        copy_sample_files(userdir, overwrite=args["reset"])
+    else:
+        logger.warning("`create-userdir` requires --userdir to be set.")
+        sys.exit(1)

finrl/config/__init__.py

@@ -1 +1,5 @@
+from finrl.config.check_exchange import check_exchange, remove_credentials
+from finrl.config.config_setup import setup_utils_configuration
+from finrl.config.config_validation import validate_config_consistency
 from finrl.config.configuration import Configuration
+from finrl.config.timerange import TimeRange