Setup and scrape tasks which define a set of scrapers and relevant scraping options

.eslintrc

@@ -3,6 +3,8 @@
     "arrow-body-style": 0,
     "no-shadow": 0,
     "no-console": 0,
+    "no-underscore-dangle": ["error", { "allow": ["_private"] }],
+    "class-methods-use-this": 0,
     "no-await-in-loop": 0
   },
   "globals": {

.gitignore

@@ -62,3 +62,6 @@ lib/
 
 # OS related
 .DS_Store
+
+# IDEs
+.idea

package.json

@@ -36,6 +36,7 @@
   },
   "dependencies": {
     "babel-polyfill": "^6.26.0",
+    "colors": "^1.2.1",
     "inquirer": "^4.0.1",
     "israeli-bank-scrapers": "^0.5.5",
     "json2csv": "^3.11.5",

src/constants.js

@@ -1,2 +1,3 @@
 const PASSWORD_FIELD = 'password';
-export default PASSWORD_FIELD;
+const DATE_AND_TIME_MOMENT_FORMAT = 'DD-MM-YYYY_HH-mm-ss';
+export { PASSWORD_FIELD, DATE_AND_TIME_MOMENT_FORMAT };

src/definitions.js

@@ -2,4 +2,5 @@ import os from 'os';
 
 export const CONFIG_FOLDER = `${os.homedir()}/.ynab-updater`;
 export const SETTINGS_FILE = `${CONFIG_FOLDER}/settings.json`;
+export const TASKS_FOLDER = `${CONFIG_FOLDER}/tasks`;
 export const DOWNLOAD_FOLDER = `${os.homedir()}/Downloads/Transactions`;

src/helpers/files.js

@@ -6,19 +6,42 @@ import jsonfile from 'jsonfile';
 const writeFileAsync = util.promisify(fs.writeFile);
 const existsAsync = util.promisify(fs.exists);
 const makeDirAsync = util.promisify(fs.mkdir);
+const readdirAsync = util.promisify(fs.readdir);
+const deleteFileAsync = util.promisify(fs.unlink);
 const readJsonFileAsync = util.promisify(jsonfile.readFile);
 const writeJsonFileAsync = util.promisify(jsonfile.writeFile);
 
-async function verifyFolder(filePath) {
-  const folder = path.dirname(filePath);
-  const exists = await existsAsync(folder);
-  if (!exists) {
-    await makeDirAsync(folder);
+async function verifyFolder(folderPath) {
+  const pathTokens = folderPath.split(path.sep);
+  let currentPath = '';
+  for (let i = 0; i < pathTokens.length; i += 1) {
+    const folder = pathTokens[i];
+    currentPath += folder + path.sep;
+    if (!await existsAsync(currentPath)) {
+      await makeDirAsync(currentPath);
+    }
+  }
+}
+
+export async function getFolderFiles(folderPath, suffix) {
+  await verifyFolder(folderPath);
+  const files = await readdirAsync(folderPath);
+  if (suffix) {
+    return files.filter(filePath => (path.extname(filePath) || '').toLowerCase() === suffix.toLowerCase());
+  }
+  return files;
+}
+
+export async function deleteFile(filePath) {
+  if (await existsAsync(filePath)) {
+    return deleteFileAsync(filePath);
   }
+  return null;
 }
 
 export async function writeFile(filePath, data, options) {
-  await verifyFolder(filePath);
+  const folderPath = path.dirname(filePath);
+  await verifyFolder(folderPath);
   return writeFileAsync(filePath, data, options);
 }
 
@@ -31,6 +54,7 @@ export async function readJsonFile(filePath, options) {
 }
 
 export async function writeJsonFile(filePath, obj, options) {
-  await verifyFolder(filePath);
+  const folderPath = path.dirname(filePath);
+  await verifyFolder(folderPath);
   await writeJsonFileAsync(filePath, obj, options);
 }

src/helpers/tasks-manager.js

@@ -0,0 +1,81 @@
+import path from 'path';
+import colors from 'colors/safe';
+import moment from 'moment';
+import { writeJsonFile, readJsonFile, getFolderFiles, deleteFile } from '../helpers/files';
+import { TASKS_FOLDER } from '../definitions';
+import { SCRAPERS } from '../helpers/scrapers';
+
+function writeSummaryLine(key, value) {
+  console.log(`- ${colors.bold(key)}: ${value}`);
+}
+
+class TaskManager {
+  async getTasksList() {
+    const files = await getFolderFiles(TASKS_FOLDER, '.json');
+    const result = files.map(file => path.basename(file, '.json'));
+    return result;
+  }
+
+  async hasTask(taskName) {
+    const tasksList = await this.getTasksList();
+    return taskName && !!tasksList.find(taskListName =>
+      taskListName.toLowerCase() === taskName.toLowerCase());
+  }
+
+  isValidTaskName(taskName) {
+    return taskName && taskName.match(/^[a-zA-Z0-9_-]+$/);
+  }
+
+  async loadTask(taskName) {
+    if (taskName && this.hasTask(taskName)) {
+      return readJsonFile(`${TASKS_FOLDER}/${taskName}.json`);
+    }
+
+    throw new Error(`failed to find a task named ${taskName}`);
+  }
+
+  async saveTask(taskName, taskData) {
+    if (taskName && this.isValidTaskName(taskName)) {
+      return writeJsonFile(`${TASKS_FOLDER}/${taskName}.json`, taskData);
+    }
+
+    throw new Error(`invalid task name provided ${taskName}`);
+  }
+
+  async deleteTask(taskName) {
+    if (taskName && this.hasTask(taskName)) {
+      await deleteFile(`${TASKS_FOLDER}/${taskName}.json`);
+    } else {
+      throw new Error(`invalid task name provided ${taskName}`);
+    }
+  }
+
+  async printTaskSummary(taskName) {
+    const taskData = await this.loadTask(taskName);
+
+    if (taskData) {
+      const scrapers = taskData.scrapers || [];
+      const {
+        dateDiffByMonth,
+        combineInstallments,
+      } = taskData.options;
+      const {
+        combineReport,
+        saveLocation,
+      } = taskData.output;
+      const substractValue = dateDiffByMonth - 1;
+      const startMoment = moment().subtract(substractValue, 'month').startOf('month');
+
+      console.log(colors.underline.bold(`Task ${taskName} Summary`));
+      writeSummaryLine('Scrapers', scrapers.map(scraper => SCRAPERS[scraper.id].name).join(', '));
+      writeSummaryLine('Start scraping from', startMoment.format('ll'));
+      writeSummaryLine('Combine installments', combineInstallments ? 'yes' : 'no');
+      writeSummaryLine('Save to location', saveLocation);
+      writeSummaryLine('Create single report', combineReport ? 'yes' : 'no');
+    }
+  }
+}
+
+const taskManager = new TaskManager();
+
+export default taskManager;

src/index.js

@@ -1,6 +1,13 @@
 import yargs from 'yargs';
-import scrape from './scrape/scrape-individual';
+import colors from 'colors/safe';
 import setupMainMenu from './setup/setup-main-menu';
+import scrapingMainMenu from './scrape/scraping-main-menu';
+
+// set theme
+colors.setTheme({
+  title: 'bgCyan',
+  notify: 'magenta',
+});
 
 const args = yargs.options({
   mode: {
@@ -16,7 +23,7 @@ const args = yargs.options({
 }).help().argv;
 
 if (!args.mode || args.mode === 'scrape') {
-  scrape(args.show);
+  scrapingMainMenu(args.show);
 } else if (args.mode === 'setup') {
   setupMainMenu();
 }

src/scrape/generate-reports.js

@@ -0,0 +1,39 @@
+import json2csv from 'json2csv';
+import colors from 'colors/safe';
+import moment from 'moment';
+import { DATE_AND_TIME_MOMENT_FORMAT } from '../constants';
+import { writeFile } from '../helpers/files';
+
+async function exportAccountData(account, saveLocation) {
+  const fields = ['Date', 'Payee', 'Inflow', 'Installment', 'Total'];
+  const csv = json2csv({ data: account.txns, fields, withBOM: true });
+  await writeFile(`${saveLocation}/${account.scraperName} (${account.accountNumber}).csv`, csv);
+}
+
+export async function generateSeparatedReports(scrapedAccounts, saveLocation) {
+  let numFiles = 0;
+  for (let i = 0; i < scrapedAccounts.length; i += 1) {
+    const account = scrapedAccounts[i];
+    if (account.txns.length) {
+      console.log(colors.notify(`exporting ${account.txns.length} transactions for account # ${account.accountNumber}`));
+      await exportAccountData(account, saveLocation);
+      numFiles += 1;
+    } else {
+      console.log(`no transactions for account # ${account.accountNumber}`);
+    }
+  }
+
+  console.log(colors.notify(`${numFiles} csv files saved under ${saveLocation}`));
+}
+
+export async function generateSingleReport(scrapedAccounts, saveLocation) {
+  const fileTransactions = scrapedAccounts.reduce((acc, account) => {
+    acc.push(...account.txns);
+    return acc;
+  }, []);
+  const filePath = `${saveLocation}/${moment().format(DATE_AND_TIME_MOMENT_FORMAT)}.csv`;
+  const fileFields = ['Institute', 'Account', 'Date', 'Payee', 'Inflow', 'Installment', 'Total'];
+  const fileContent = json2csv({ data: fileTransactions, fileFields, withBOM: true });
+  await writeFile(filePath, fileContent);
+  console.log(colors.notify(`created file ${filePath}`));
+}

src/scrape/scrape-base.js

@@ -0,0 +1,71 @@
+import moment from 'moment';
+import { SCRAPERS, createScraper } from '../helpers/scrapers';
+
+async function prepareResults(scraperId, scraperName, scraperResult, combineInstallments) {
+  return scraperResult.accounts.map((account) => {
+    console.log(`${scraperName}: scraped ${account.txns.length} transactions from account ${account.accountNumber}`);
+
+    const txns = account.txns.map((txn) => {
+      return {
+        Company: scraperName,
+        AccountNumber: account.accountNumber,
+        Date: moment(txn.date).format('DD/MM/YYYY'),
+        Payee: txn.description,
+        Inflow: txn.type !== 'installments' || !combineInstallments ? txn.chargedAmount : txn.originalAmount,
+        Installment: txn.installments ? txn.installments.number : null,
+        Total: txn.installments ? txn.installments.total : null,
+      };
+    });
+
+    return {
+      scraperId,
+      scraperName,
+      accountNumber: account.accountNumber,
+      txns,
+    };
+  });
+}
+
+export default async function (scraperId, credentials, options) {
+  const {
+    combineInstallments,
+    startDate,
+    showBrowser,
+  } = options;
+
+  const scraperOptions = {
+    companyId: scraperId,
+    startDate,
+    combineInstallments,
+    showBrowser,
+    verbose: false,
+  };
+  const scraperName = SCRAPERS[scraperId] ?
+    SCRAPERS[scraperId].name : null;
+
+  if (!scraperName) {
+    throw new Error(`unknown scraper with id ${scraperId}`);
+  }
+  console.log(`scraping ${scraperName}`);
+
+  let scraperResult;
+  try {
+    const scraper = createScraper(scraperOptions);
+    scraper.onProgress((companyId, payload) => {
+      console.log(`${scraperName}: ${payload.type}`);
+    });
+    scraperResult = await scraper.scrape(credentials);
+  } catch (e) {
+    console.error(e.message);
+    throw e;
+  }
+
+  console.log(`success: ${scraperResult.success}`);
+  if (!scraperResult.success) {
+    console.log(`error type: ${scraperResult.errorType}`);
+    console.log('error:', scraperResult.errorMessage);
+    throw new Error(scraperResult.errorMessage);
+  }
+
+  return prepareResults(scraperId, scraperName, scraperResult, combineInstallments);
+}