Enable Defining A Custom Thousand Separator Format

assets/config/example.yaml

@@ -29,6 +29,8 @@ parsers:
     delimiter: ","
     columns: [skip, skip, memo, date, inflow, skip]
     date_format: M/d/yyyy
+    decimal_separator: "."
+    thousand_separator: ","
 
 upload_to_ynab:
   # This is the default. If you've set the 'upload' option on

docs/how-to-configure.md

@@ -205,6 +205,26 @@ date_format: dd/MM/yyyy # e.g. 20/04/2020
 
 Date format used in your bank's csv file. For all possibilities, visit [table of possible tokens](https://moment.github.io/luxon/#/parsing?id=table-of-tokens)
 
+### `thousand_separator`
+
+```yaml
+thousand_separator: "," # e.g. 1,234.00 (comma)
+thousand_separator: "." # e.g. 1.234,00 (dot)
+thousand_separator: ""  # e.g. 12345.00 (no separator)
+```
+
+The thousands separator used by your bank. Default: _no separator_.
+
+### `decimal_separator`
+
+```yaml
+decimal_separator: "," # e.g. 420,69 (comma)
+decimal_separator: "." # e.g. 420.69 (dot)
+decimal_separator: ""  # Auto-detect separator. Not recommended.
+```
+
+The decimals separator used by your bank. The tool tries to auto-detect the separator if you don't set one **(not recommended)**.
+
 ## Upload to YNAB
 
 ### `upload_transactions`

src/lib/parser.spec.fixtures.ts

@@ -45,6 +45,16 @@ invalid footer row
   `1; 9/27/2020 ;871.13;Devpoint;IL95 2010 7730 7319 4618 209
 2; 11/26/2019 ;908.31;Realcube;GE78 WG91 9644 2111 5080 45
 3; 3/6/2020 ;152.13;Oyoloo;GR11 2705 328W VAZB OZUD NLWB DJT`,
+
+  // Thousand separator
+  `1;9/27/2020;8,711.13;Devpoint;IL95 2010 7730 7319 4618 209
+2;11/26/2019;9,081.31;Realcube;GE78 WG91 9644 2111 5080 45
+3;3/6/2020;212.13;Oyoloo;GR11 2705 328W VAZB OZUD NLWB DJT`,
+
+  // Dot thousand separator, comma decimal separator
+  `1;9/27/2020;8.711,13;Devpoint;IL95 2010 7730 7319 4618 209
+2;11/26/2019;9.081,31;Realcube;GE78 WG91 9644 2111 5080 45
+3;3/6/2020;212,13;Oyoloo;GR11 2705 328W VAZB OZUD NLWB DJT`,
 ];
 
 export const defaultParser: Parser =
@@ -56,6 +66,7 @@ export const defaultParser: Parser =
     footer_rows: 0,
     header_rows: 0,
     name: "TEST",
+    decimal_separator: ",",
   };
 
 export const bankFile: BankFile = {

src/lib/parser.spec.ts

@@ -3,10 +3,11 @@ import * as fixtures from "./parser.spec.fixtures";
 
 // Patch readFileSync so it produces a different CSV for every test
 let csvFixtureBeingTested: number;
+let customCsv: string | undefined;
 jest.mock("fs", () => {
   return {
     readFileSync: () => {
-      return fixtures.csv[csvFixtureBeingTested];
+      return customCsv || fixtures.csv[csvFixtureBeingTested];
     },
   };
 });
@@ -80,6 +81,86 @@ describe("parser", () => {
     const result = runParser(csvFixtures.dateSurroundedBySpaces, parseCfg);
     expect(result.transactions).toHaveLength(3);
   });
+
+  /**
+   * This tests only uses a single line of CSV instead of a full fixture.
+   * The text "AMOUNT" gets replaced with an amount in a number of different formats.
+   * Every amount should parse to the same value (1234.56).
+   */
+  it("parses amounts in different formats", () => {
+    const expectedAmount = 1234.56;
+
+    // Create a simple parser for just 2 columns: date and amount
+    const { parseBankFile } = require("./parser");
+    const parser = { ...fixtures.defaultParser, columns: ["date", "amount"] };
+
+    // CSV text with a number of different amounts and separators
+    const customCsvTemplate = "9/27/2020;AMOUNT";
+    const testData: string[][] = [
+      /* [amount, decimal_separator, thousand_separator] */
+      ["1234.56"],
+      ["1234,56"],
+      ["1234.56", "."],
+      ["1234,56", ","],
+      ["1234,56 EUR", ","],
+      ["1234.56 EUR", "."],
+      ["1,234.56", ".", ","],
+      ["1.234,56", ",", "."],
+      ["$1,234.56", ".", ","],
+      ["$1.234,56", ",", "."],
+      ["€ 1234.56", "."],
+      ["1,234.56 USD", ".", ","],
+    ];
+
+    const errors: string[][] = [];
+    testData.forEach((amount) => {
+      // Set customCSV that will be returned by the mocked readFileSync
+      customCsv = customCsvTemplate.replace("AMOUNT", amount[0]);
+
+      // Configure the parser to use separators
+      parser.decimal_separator = amount[1] || undefined;
+      parser.thousand_separator = amount[2] || undefined;
+
+      // Parse the CSV and compare the result to the expected amount
+      const result = parseBankFile(fixtures.bankFile, [parser]);
+      const actualAmount = result.transactions[0].amount;
+      if (actualAmount !== expectedAmount) errors.push([actualAmount, amount]);
+    });
+
+    // Unset customCsv so the mock goes back using parser.spec.fixtures.ts
+    customCsv = undefined;
+    if (errors.length > 0) {
+      throw new Error(
+        `Amounts did not parse correctly: ${JSON.stringify(errors)}`
+      );
+    }
+  });
+
+  it("can parse thousand separators in amounts field", () => {
+    const parseCfg = {
+      columns: ["", "date", "amount", "payee", "memo"],
+      thousand_separator: ",",
+    };
+    const result = runParser(csvFixtures.thousandSeparators, parseCfg);
+    expect(result.transactions[0].amount).toEqual(8711.13);
+    expect(result.transactions[1].amount).toEqual(9081.31);
+    expect(result.transactions[2].amount).toEqual(212.13);
+  });
+
+  it("can parse localized separators in amounts field", () => {
+    const parseCfg = {
+      columns: ["", "date", "amount", "payee", "memo"],
+      thousand_separator: ".",
+      decimal_separator: ",",
+    };
+    const result = runParser(
+      csvFixtures.dotThousandSeparatorsCommaDecimalSeparator,
+      parseCfg
+    );
+    expect(result.transactions[0].amount).toEqual(8711.13);
+    expect(result.transactions[1].amount).toEqual(9081.31);
+    expect(result.transactions[2].amount).toEqual(212.13);
+  });
 });
 
 const runParser = (fixtureId: number, parseCfg?: Partial<Parser>) => {
@@ -98,6 +179,8 @@ enum csvFixtures {
   inOutIndicator = 4,
   payeeField = 5,
   dateSurroundedBySpaces = 6,
+  thousandSeparators = 7,
+  dotThousandSeparatorsCommaDecimalSeparator = 8,
 }
 
 const validateTransaction = (tx: Transaction) => {

src/lib/parser.ts

@@ -33,7 +33,7 @@ export function parseBankFile(source: BankFile, parsers: Parser[]) {
 
 export function buildTransaction(record: any, parser: Parser): Transaction {
   const tx: Transaction = {
-    amount: parseAmount(record, parser.outflow_indicator),
+    amount: parseAmount(record, parser),
     date: parseDate(record, parser.date_format),
     memo: mergeMemoFields(record),
   };
@@ -62,19 +62,35 @@ function parseDate(record: any, dateFormat: string) {
   throw "PARSING ERROR";
 }
 
-function parseAmount(record: any, outflowIndicator?: string): number {
+function parseAmount(record: any, parser: Parser): number {
+  const { thousand_separator, decimal_separator, outflow_indicator } = parser;
   const { inflow, outflow, amount, in_out_flag } = record;
   let value = inflow || outflow || amount;
 
   if (typeof value === "string") {
-    value = value.replace(",", "."); // "420,69" ==> "420.69"
+    if (thousand_separator) {
+      value = value.replace(thousand_separator, ""); // 69.420,00 -> 69420.00
+    }
+
+    if (decimal_separator) {
+      value = value.replace(decimal_separator, "."); // 69420,00 -> 69420.00
+    }
+
+    if (!decimal_separator && !thousand_separator) {
+      // Backwards compatibility: if value has a ',' convert it to a '.'
+      value = value.replace(",", ".");
+    }
+
+    // Remove non digit, non decimal separator, non minus characters
+    value = value.replace(/[^0-9-.]/g, ""); // $420.69 -> 420.69
+
     value = parseFloat(value); // "420.69" ==> 420.69
   }
 
   // If the outflow column exists, OR
   // If the in_out_flag column exists AND it contains the outflow indicator
   // invert the value of the amount
-  if (outflow !== undefined || in_out_flag?.startsWith(outflowIndicator)) {
+  if (outflow !== undefined || in_out_flag?.startsWith(outflow_indicator)) {
     value = -value; // 420.69 ==> -420.69
   }
 

src/types.ts

@@ -54,4 +54,6 @@ export type Parser = {
   footer_rows: number;
   header_rows: number;
   outflow_indicator?: string;
+  thousand_separator?: string;
+  decimal_separator?: string;
 };