Automatically update eurusd.csv if required + introduction of unit tests

.gitignore

@@ -1,3 +1,6 @@
 # Ingore Virtual Environment
 .venv/
 venv/
+# Innore other files
+*.pyc
+.DS_Store

.vscode/settings.json

@@ -0,0 +1,11 @@
+{
+	"python.testing.unittestArgs": [
+		"-v",
+		"-s",
+		"./tests",
+		"-p",
+		"*_test.py"
+	],
+	"python.testing.unittestEnabled": true,
+	"python.testing.pytestEnabled": false
+}

README.md

@@ -116,17 +116,10 @@ The csv file has the following first line:
 Date/Time,Transaction Code,Transaction Subcode,Symbol,Buy/Sell,Open/Close,Quantity,Expiration Date,Strike,Call/Put,Price,Fees,Amount,Description,Account Reference
 </code></p>
 
-If you delete the __eurusd.csv__ file, a current version is downloaded directly
-from <https://www.bundesbank.de/de/statistiken/wechselkurse>.
+If you delete the __eurusd.csv__ file or your Tastytrade transaction history contains more recent transactions than the available data in this file, a current version is downloaded automatically directly from <https://www.bundesbank.de/de/statistiken/wechselkurse>.
 (Link to the data: [eurusd.csv](https://www.bundesbank.de/statistic-rmi/StatisticDownload?tsId=BBEX3.D.USD.EUR.BB.AC.000&its_csvFormat=en&its_fileFormat=csv&mode=its&its_from=2010))
-You can also download a new eurusd.csv file in the current directory with
-<pre>
-python3 tw-pnl.py --download-eurusd
-</pre>
-If you invoke the tw-pnl.py script from another directory, you can also place
-an updated eurusd.csv file into the current directory.
 
-The option __--usd__ can be used to not translate pnl data into Euro.
+The option __--usd__ can be used to prevent converting pnl data into Euro.
 
 Per default the script stops on unknown trading symbols (underlyings) and you have
 to hardcode into the source code if it is an individual stock or some ETF/fond.

tastytradehelper.py

@@ -0,0 +1,223 @@
+import csv
+from datetime import datetime, timedelta
+from io import StringIO
+import os
+import pandas
+from pandas.api.types import union_categoricals
+import urllib.request
+
+class TastytradeHelper:
+    _eurusd_rates = {}
+    _eurusd_filename = 'eurusd.csv'
+
+    @staticmethod
+    def price_from_description(description: str) -> float:
+        """
+        Extract the price from the transaction description string and return it.
+        """
+        price = 0
+        if len(description) == 0 or not description.startswith('Bought') and not description.startswith('Sold'):
+            return price
+
+        parts = description.split('@')
+        assert len(parts) == 2, f"Expected 2 parts, got {len(parts)}: {parts}"
+
+        price = float(parts[-1].strip().replace(',', '').replace('"', ''))
+        return price
+
+    @staticmethod
+    def consolidate_and_sort_transactions(transactions) -> pandas.DataFrame:
+        """
+        Consolidate the array with DataFrames and sort them descending by date.
+        Keep the category columns consistent during concatenation of the dataframes.
+        """
+        if len(transactions) == 1:
+            consolidated_transactions = transactions[0]
+        else:
+            for col in set.intersection(
+                *[
+                    set(df.select_dtypes(include='category').columns)
+                    for df in transactions
+                ]
+            ):
+                # Generate the union category across dfs for this column
+                uc = union_categoricals([df[col] for df in transactions])
+                # Change to union category for all dataframes
+                for df in transactions:
+                     df[col] = pandas.Categorical(df[col].values, categories=uc.categories)
+            consolidated_transactions = pandas.concat(transactions, ignore_index=True)
+            consolidated_transactions = consolidated_transactions.sort_values(by=['Date/Time',], ascending=False)
+            consolidated_transactions = consolidated_transactions.reset_index(drop=True)
+        return consolidated_transactions
+
+    @staticmethod
+    def get_eurusd(date: str) -> float:
+        """
+        Get the EURUSD exchange rate for the given date.
+        """
+        real_date = datetime.strptime(date, '%Y-%m-%d')
+        while True:
+            try:
+                rate = TastytradeHelper._eurusd_rates[real_date.strftime('%Y-%m-%d')]
+            except KeyError:
+                raise ValueError(f'No EUR/USD exchange rate available for date {date}')
+
+            if rate is not None:
+                return rate
+
+            # Try the previous day
+            real_date -= timedelta(days=1)
+
+    @staticmethod
+    def is_legacy_csv(csv_filename: str) -> bool:
+        """ Checks the first line of the csv data file if the header fits the legacy or the current format.
+        """
+        header_legacy = 'Date/Time,Transaction Code,' + \
+    			'Transaction Subcode,Symbol,Buy/Sell,Open/Close,Quantity,' + \
+				'Expiration Date,Strike,Call/Put,Price,Fees,Amount,Description,' + \
+				'Account Reference\n'
+        header = 'Date,Type,Sub Type,Action,Symbol,Instrument Type,Description,Value,Quantity,' + \
+        		'Average Price,Commissions,Fees,Multiplier,Root Symbol,Underlying Symbol,Expiration Date,' + \
+                'Strike Price,Call or Put,Order #,Currency\n'
+        with open(csv_filename, encoding='UTF8') as f:
+            content = f.readlines(1)
+            if content[0] == header_legacy:
+                legacy_format = True
+            elif content[0] == header:
+                legacy_format = False
+            else:
+                raise ValueError('Wrong first line in csv file. Please download trade history from the Tastytrade app!')
+        return legacy_format
+
+    @staticmethod
+    def read_transaction_history(csv_filename: str) -> pandas.DataFrame:
+        """
+        Read the Tastytrade transaction history from a CSV file and return it as a DataFrame.
+        """
+        csv_string = csv_filename
+        if not TastytradeHelper.is_legacy_csv(csv_filename):
+            csv_string = StringIO(TastytradeHelper.transform_csv(csv_filename))
+        transaction_history = pandas.read_csv(csv_string, parse_dates=['Date/Time'])
+
+        transaction_history['Expiration Date'] = transaction_history['Expiration Date'].astype('object')
+        for i in ('Open/Close', 'Buy/Sell', 'Call/Put'):
+            #print(wk[i].value_counts(dropna=False))
+            transaction_history[i] = transaction_history[i].fillna('').astype('category')
+            #print(wk[i].value_counts(dropna=False))
+        for i in ('Account Reference', 'Transaction Subcode', 'Transaction Code'):
+            #print(wk[i].value_counts(dropna=False))
+            transaction_history[i] = transaction_history[i].astype('category')
+
+        return transaction_history
+
+    @staticmethod
+    def transform_csv(csv_filename: str) -> str:
+        """
+        Transform the CSV file data from new data format back to the old data format.
+        """
+        transformed_data = 'Date/Time,Transaction Code,Transaction Subcode,Symbol,Buy/Sell,Open/Close,Quantity,Expiration Date,Strike,Call/Put,Price,Fees,Amount,Description,Account Reference'
+        with open(csv_filename, encoding='UTF8') as f:
+            reader = csv.reader(f, delimiter=',')
+            for row in reader:
+                if row[0] == 'Date':
+                    continue
+                date = datetime.fromisoformat(row[0][:19]).strftime('%m/%d/%Y %H:%M') # Convert ISO date to old date format
+
+                transaction_code = row[1]
+                transaction_subcode = row[2]
+                action = row[3]
+                symbol = row[4]
+                if symbol.startswith('.'):  # Remove leading dot from symbol
+                    symbol = symbol[1:]
+
+                # Extract buy/sell and open/close from action
+                buy_sell = ''
+                open_close = ''
+                if action.startswith('BUY'):
+                    buy_sell = 'Buy'
+                elif action.startswith('SELL'):
+                    buy_sell = 'Sell'
+                if action.endswith('TO_OPEN'):
+                    open_close = 'Open'
+                elif action.endswith('TO_CLOSE'):
+                    open_close = 'Close'
+
+                quantity = row[8]
+
+                # Transform the expiration date
+                if row[15] != '':
+                    expiration_date = datetime.strptime(row[15], '%m/%d/%y').strftime('%m/%d/%Y')
+                else:
+                    expiration_date = 'n/a'
+
+                strike = row[16]
+
+                if len(row[17]) > 0:
+                    call_put = row[17][0]
+                else:
+                    call_put = ''
+
+                description = row[6]
+                price = TastytradeHelper.price_from_description(description)
+
+                fees = float(row[11])
+                try:
+                    commission = float(row[10])
+                    fees += commission
+                except ValueError:
+                    pass
+                fees = abs(fees) # Fees are always positive in the old format
+
+                amount = row[7].replace(',', '')
+
+                account_refrerence = 'account'
+
+                transformed_data += f'\n{date},{transaction_code},{transaction_subcode},{symbol},{buy_sell},{open_close},{quantity},{expiration_date},{strike},{call_put},{price},{fees},{amount},{description},{account_refrerence}'
+
+            return transformed_data
+
+    @staticmethod
+    def _read_eurusd_rates():
+        """
+        Read the EURUSD exchange rates from the CSV file eurusd.csv.
+        """
+        TastytradeHelper._eurusd_rates = {}
+        if os.path.exists(TastytradeHelper._eurusd_filename):
+            with open(TastytradeHelper._eurusd_filename, encoding='UTF8') as csv_file:
+                reader = csv.reader(csv_file)
+                for _ in range(5):
+                    next(reader)
+                for (date, usd, _) in reader:
+                    if date != '' and usd != '.':
+                        TastytradeHelper._eurusd_rates[date] = float(usd)
+                    else:
+                        TastytradeHelper._eurusd_rates[date] = None
+
+    @staticmethod
+    def update_eurusd(recent_date: datetime.date) -> bool:
+        """
+        Checks if the EURUSD exchange rates csv file is up to date and updates the data if necessary.
+        """
+        eurusd_url: str = 'https://www.bundesbank.de/statistic-rmi/StatisticDownload?tsId=BBEX3.D.USD.EUR.BB.AC.000&its_csvFormat=en&its_fileFormat=csv&mode=its&its_from=2010'
+
+        # Try to open the local eurusd.csv file and check the last date in the file.
+        TastytradeHelper._read_eurusd_rates()
+        # access the last entry in the dictionary
+        read_date = ''
+        index = 0
+        while read_date == '':
+            index -= 1
+            read_date = list(TastytradeHelper._eurusd_rates.keys())[index]
+
+        last_rate_date = datetime.strptime(read_date, '%Y-%m-%d').date()
+        if last_rate_date < recent_date:
+            # Download the latest EURUSD exchange rates from the Bundesbank
+            # and append them to the local eurusd.csv file.
+            try:
+                urllib.request.urlretrieve(eurusd_url, TastytradeHelper._eurusd_filename)
+            except urllib.error.URLError as exc:
+                return False
+            # Read the updated exchange rates again
+            TastytradeHelper._read_eurusd_rates()
+        return True
+

tests/data/CL Future.csv

@@ -0,0 +1,3 @@
+Date,Type,Sub Type,Action,Symbol,Instrument Type,Description,Value,Quantity,Average Price,Commissions,Fees,Multiplier,Root Symbol,Underlying Symbol,Expiration Date,Strike Price,Call or Put,Order #,Currency
+2023-03-04T14:16:27+0100,Trade,Sell,SELL,/CLJ4,Future,Sold 1 /CLJ4 @ 79.53,-160.00,1,-160.00,-1.25,-1.82,,,,,,,310910163,USD
+2023-03-04T14:12:32+0100,Trade,Buy,BUY,/CLJ4,Future,Bought 1 /CLJ4 @ 79.69,0.00,1,0.00,-1.25,-1.82,,,,,,,310909959,USD

tests/data/Legacy Format.csv

@@ -0,0 +1,2 @@
+Date/Time,Transaction Code,Transaction Subcode,Symbol,Buy/Sell,Open/Close,Quantity,Expiration Date,Strike,Call/Put,Price,Fees,Amount,Description,Account Reference
+02/19/2020 11:00 PM,Money Movement,Withdrawal,,,,,,,,,0.00,1234.5,Wire Funds Received a/o 2/18,John Doe...06

tests/data/MES LT112.csv

@@ -0,0 +1,8 @@
+Date,Type,Sub Type,Action,Symbol,Instrument Type,Description,Value,Quantity,Average Price,Commissions,Fees,Multiplier,Root Symbol,Underlying Symbol,Expiration Date,Strike Price,Call or Put,Order #,Currency
+2024-09-20T23:00:00+0200,Receive Deliver,Expiration,BUY_TO_CLOSE,./MESU4MESU4 240920P5050,Future Option,Removal of option due to expiration,0.00,1,0.00,--,0.00,1,"./MESU4MESU4 ",/MESU4,9/20/24,5050.0,PUT,,USD
+2024-09-20T23:00:00+0200,Receive Deliver,Expiration,SELL_TO_CLOSE,./MESU4MESU4 240920P5100,Future Option,Removal of option due to expiration,0.00,1,0.00,--,0.00,1,"./MESU4MESU4 ",/MESU4,9/20/24,5100.0,PUT,,USD
+2024-09-10T15:32:26+0200,Trade,Buy to Close,BUY_TO_CLOSE,./MESU4MESU4 240920P4650,Future Option,Bought 2 /MESU4 MESU4 09/20/24 Put 4650.00 @ 1.1,-11.00,2,-5.50,0.00,-1.04,1,"./MESU4MESU4 ",/MESU4,9/20/24,4650.0,PUT,327129782,USD
+2024-05-23T17:05:16+0200,Trade,Sell to Open,SELL_TO_OPEN,./MESU4MESU4 240920P4650,Future Option,Sold 2 /MESU4 MESU4 09/20/24 Put 4650.00 @ 22.0,220.00,2,110.00,-3.00,-1.04,1,"./MESU4MESU4 ",/MESU4,9/20/24,4650.0,PUT,323968393,USD
+2024-05-23T17:01:37+0200,Trade,Sell to Open,SELL_TO_OPEN,./MESU4MESU4 240920P5050,Future Option,Sold 1 /MESU4 MESU4 09/20/24 Put 5050.00 @ 54.0,270.00,1,270.00,-1.50,-0.46,1,"./MESU4MESU4 ",/MESU4,9/20/24,5050.0,PUT,323965426,USD
+2024-05-23T17:01:37+0200,Trade,Buy to Open,BUY_TO_OPEN,./MESU4MESU4 240920P5100,Future Option,Bought 1 /MESU4 MESU4 09/20/24 Put 5100.00 @ 61.5,-307.50,1,-307.50,-1.50,-0.46,1,"./MESU4MESU4 ",/MESU4,9/20/24,5100.0,PUT,323965426,USD
+2024-01-03T14:00:00+0200,Money Movement,Deposit,,,,Wire Funds Received,"1,000.00",0,,--,0.00,,,,,,,,USD

tests/data/test_consolidate_and_sort_transactions_result.csv

@@ -0,0 +1,10 @@
+Date,Type,Sub Type,Action,Symbol,Instrument Type,Description,Value,Quantity,Average Price,Commissions,Fees,Multiplier,Root Symbol,Underlying Symbol,Expiration Date,Strike Price,Call or Put,Order #,Currency
+2024-09-20T23:00:00+0200,Receive Deliver,Expiration,BUY_TO_CLOSE,./MESU4MESU4 240920P5050,Future Option,Removal of option due to expiration,0.00,1,0.00,--,0.00,1,"./MESU4MESU4 ",/MESU4,9/20/24,5050.0,PUT,,USD
+2024-09-20T23:00:00+0200,Receive Deliver,Expiration,SELL_TO_CLOSE,./MESU4MESU4 240920P5100,Future Option,Removal of option due to expiration,0.00,1,0.00,--,0.00,1,"./MESU4MESU4 ",/MESU4,9/20/24,5100.0,PUT,,USD
+2024-09-10T15:32:26+0200,Trade,Buy to Close,BUY_TO_CLOSE,./MESU4MESU4 240920P4650,Future Option,Bought 2 /MESU4 MESU4 09/20/24 Put 4650.00 @ 1.1,-11.00,2,-5.50,0.00,-1.04,1,"./MESU4MESU4 ",/MESU4,9/20/24,4650.0,PUT,327129782,USD
+2024-05-23T17:05:16+0200,Trade,Sell to Open,SELL_TO_OPEN,./MESU4MESU4 240920P4650,Future Option,Sold 2 /MESU4 MESU4 09/20/24 Put 4650.00 @ 22.0,220.00,2,110.00,-3.00,-1.04,1,"./MESU4MESU4 ",/MESU4,9/20/24,4650.0,PUT,323968393,USD
+2024-05-23T17:01:37+0200,Trade,Sell to Open,SELL_TO_OPEN,./MESU4MESU4 240920P5050,Future Option,Sold 1 /MESU4 MESU4 09/20/24 Put 5050.00 @ 54.0,270.00,1,270.00,-1.50,-0.46,1,"./MESU4MESU4 ",/MESU4,9/20/24,5050.0,PUT,323965426,USD
+2024-05-23T17:01:37+0200,Trade,Buy to Open,BUY_TO_OPEN,./MESU4MESU4 240920P5100,Future Option,Bought 1 /MESU4 MESU4 09/20/24 Put 5100.00 @ 61.5,-307.50,1,-307.50,-1.50,-0.46,1,"./MESU4MESU4 ",/MESU4,9/20/24,5100.0,PUT,323965426,USD
+2024-01-03T14:00:00+0200,Money Movement,Deposit,,,,Wire Funds Received,"1,000.00",0,,--,0.00,,,,,,,,USD
+2023-03-04T14:16:27+0100,Trade,Sell,SELL,/CLJ4,Future,Sold 1 /CLJ4 @ 79.53,-160.00,1,-160.00,-1.25,-1.82,,,,,,,310910163,USD
+2023-03-04T14:12:32+0100,Trade,Buy,BUY,/CLJ4,Future,Bought 1 /CLJ4 @ 79.69,0.00,1,0.00,-1.25,-1.82,,,,,,,310909959,USD

tests/tastytradehelper_test.py

@@ -0,0 +1,61 @@
+import os
+import unittest
+
+import pandas
+
+from tastytradehelper import TastytradeHelper
+from pandas.testing import assert_frame_equal
+
+class TastytradeHelperTests(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def _get_test_data_directory(self):
+        """
+        Return the directory where the test data is stored.
+        """
+        return os.path.join(os.path.dirname(__file__), 'data')
+
+    def test_is_legacy_csv(self):
+        """
+        Test the detection of the CSV file format.
+        """
+        testdata_directory = self._get_test_data_directory()
+        self.assertTrue(TastytradeHelper.is_legacy_csv(testdata_directory + os.sep + 'Legacy Format.csv'))
+        self.assertFalse(TastytradeHelper.is_legacy_csv(testdata_directory + os.sep + 'MES LT112.csv'))
+        #TODO: Add test for wrong first line in csv file (raise ValueError)
+
+    def test_price_from_description(self):
+        extracted_price = TastytradeHelper.price_from_description('Bought 1 SPY 100 16 OCT 20 340 Call @ 1.23')
+        self.assertEqual(extracted_price, 1.23)
+        extracted_price = TastytradeHelper.price_from_description('Sold 1 SPY 100 16 OCT 20 300 Call @ "2,323.23"')
+        self.assertEqual(extracted_price, 2323.23)
+
+    def test_consolidate_and_sort_transactions(self):
+        """
+        Test the consolidation and sorting of transactions from multiple csv files.
+        Transactions are expected to be sorted descending by date.
+        """
+        testdata_directory = self._get_test_data_directory()
+        transactions = []
+        # Read first older data file and then the newer one
+        transactions.append(TastytradeHelper.read_transaction_history(testdata_directory + os.sep + 'CL Future.csv'))
+        transactions.append(TastytradeHelper.read_transaction_history(testdata_directory + os.sep + 'MES LT112.csv'))
+        consolidated_data = TastytradeHelper.consolidate_and_sort_transactions(transactions)
+        # Expected data from the two files sorted descending by date
+        expected_data = TastytradeHelper.read_transaction_history(testdata_directory + os.sep + 'test_consolidate_and_sort_transactions_result.csv')
+        pandas.testing.assert_frame_equal(consolidated_data, expected_data, check_categorical=False)
+
+    def test_get_eurusd(self):
+        """
+        Test the retrieval of the EURUSD exchange rate for a given date.
+        """
+        TastytradeHelper._eurusd_rates = {'2020-10-15': 1.17, '2020-10-16': None,'2020-10-17': 1.16}
+        self.assertEqual(TastytradeHelper.get_eurusd('2020-10-15'), 1.17)
+        self.assertEqual(TastytradeHelper.get_eurusd('2020-10-16'), 1.17)
+        self.assertEqual(TastytradeHelper.get_eurusd('2020-10-17'), 1.16)
+
+        # Expect Exception if there is no exchange rate available for the given date
+        with self.assertRaises(ValueError):
+            TastytradeHelper.get_eurusd('2020-10-14')
+