diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6a0d4e56..8ca11bc7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,8 +71,9 @@ You can override this async function however way you want, as long as your retur installments: { number: int, // the current installment number total: int, // the total number of installments - } - }], + }, + status: string //can either be 'completed' or 'pending' + }], }], errorType: "invalidPassword"|"changePassword"|"timeout"|"generic", // only on success=false errorMessage: string, // only on success=false diff --git a/README.md b/README.md index f604bcae..aae14cae 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,9 @@ The structure of the result object is as follows: installments: { number: int, // the current installment number total: int, // the total number of installments - } - }], + }, + status: string //can either be 'completed' or 'pending' + }], }], errorType: "invalidPassword"|"changePassword"|"timeout"|"generic", // only on success=false errorMessage: string, // only on success=false diff --git a/src/constants.js b/src/constants.js index 9b3800f8..3c706aad 100644 --- a/src/constants.js +++ b/src/constants.js @@ -33,3 +33,8 @@ export const NAVIGATION_ERRORS = { }; export const GENERAL_ERROR = 'GENERAL_ERROR'; + +export const TRANSACTION_STATUS = { + COMPLETED: 'COMPLETED', + PENDING: 'PENDING', +}; diff --git a/src/scrapers/base-isracard-amex.js b/src/scrapers/base-isracard-amex.js index 27bb0c37..1911504a 100644 --- a/src/scrapers/base-isracard-amex.js +++ b/src/scrapers/base-isracard-amex.js @@ -4,7 +4,7 @@ import moment from 'moment'; import { BaseScraperWithBrowser, LOGIN_RESULT } from './base-scraper-with-browser'; import { fetchGetWithinPage, fetchPostWithinPage } from '../helpers/fetch'; -import { SCRAPE_PROGRESS_TYPES, NORMAL_TXN_TYPE, INSTALLMENTS_TXN_TYPE, SHEKEL_CURRENCY_KEYWORD, SHEKEL_CURRENCY, ALT_SHEKEL_CURRENCY } from '../constants'; +import { SCRAPE_PROGRESS_TYPES, NORMAL_TXN_TYPE, INSTALLMENTS_TXN_TYPE, SHEKEL_CURRENCY_KEYWORD, SHEKEL_CURRENCY, ALT_SHEKEL_CURRENCY, TRANSACTION_STATUS } from '../constants'; import getAllMonthMoments from '../helpers/dates'; import { fixInstallments, filterOldTransactions } from '../helpers/transactions'; @@ -104,6 +104,7 @@ function convertTransactions(txns, processedDate) { chargedAmount: isOutbound ? -txn.paymentSumOutbound : -txn.paymentSum, description: isOutbound ? txn.fullSupplierNameOutbound : txn.fullSupplierNameHeb, installments: getInstallmentsInfo(txn), + status: TRANSACTION_STATUS.COMPLETED, }; }); } diff --git a/src/scrapers/discount.js b/src/scrapers/discount.js index d020178b..e168eade 100644 --- a/src/scrapers/discount.js +++ b/src/scrapers/discount.js @@ -1,15 +1,19 @@ +import _ from 'lodash'; import moment from 'moment'; import { BaseScraperWithBrowser, LOGIN_RESULT } from './base-scraper-with-browser'; import { waitUntilElementFound } from '../helpers/elements-interactions'; import { waitForNavigation } from '../helpers/navigation'; import { fetchGetWithinPage } from '../helpers/fetch'; -import { NORMAL_TXN_TYPE } from '../constants'; +import { NORMAL_TXN_TYPE, TRANSACTION_STATUS } from '../constants'; const BASE_URL = 'https://start.telebank.co.il'; const DATE_FORMAT = 'YYYYMMDD'; -function convertTransactions(txns) { +function convertTransactions(txns, txnStatus) { + if (!txns) { + return []; + } return txns.map((txn) => { return { type: NORMAL_TXN_TYPE, @@ -20,6 +24,7 @@ function convertTransactions(txns) { originalCurrency: 'ILS', chargedAmount: txn.OperationAmount, description: txn.OperationDescriptionToDisplay, + status: txnStatus, }; }); } @@ -36,7 +41,7 @@ async function fetchAccountData(page, options) { const startMoment = moment.max(defaultStartMoment, moment(startDate)); const startDateStr = startMoment.format(DATE_FORMAT); - const txnsUrl = `${apiSiteUrl}/lastTransactions/${accountNumber}/Date?IsCategoryDescCode=True&IsTransactionDetails=True&IsEventNames=True&FromDate=${startDateStr}`; + const txnsUrl = `${apiSiteUrl}/lastTransactions/${accountNumber}/Date?IsCategoryDescCode=True&IsTransactionDetails=True&IsEventNames=True&IsFutureTransactionFlag=True&FromDate=${startDateStr}`; const txnsResult = await fetchGetWithinPage(page, txnsUrl); if (txnsResult.Error) { return { @@ -45,13 +50,19 @@ async function fetchAccountData(page, options) { errorMessage: txnsResult.Error.MsgText, }; } - const txns = convertTransactions(txnsResult.CurrentAccountLastTransactions.OperationEntry); + + const completedTxns = convertTransactions( + txnsResult.CurrentAccountLastTransactions.OperationEntry, + TRANSACTION_STATUS.COMPLETED, + ); + const rawFutureTxns = _.get(txnsResult, 'CurrentAccountLastTransactions.FutureTransactionsBlock.FutureTransactionEntry'); + const pendingTxns = convertTransactions(rawFutureTxns, TRANSACTION_STATUS.PENDING); const accountData = { success: true, accounts: [{ accountNumber, - txns, + txns: [...completedTxns, ...pendingTxns], }], }; diff --git a/src/scrapers/hapoalim.js b/src/scrapers/hapoalim.js index 5cfd7eb8..e48bd685 100644 --- a/src/scrapers/hapoalim.js +++ b/src/scrapers/hapoalim.js @@ -2,7 +2,7 @@ import moment from 'moment'; import { BaseScraperWithBrowser, LOGIN_RESULT } from './base-scraper-with-browser'; import { waitForRedirect, getCurrentUrl } from '../helpers/navigation'; -import { NORMAL_TXN_TYPE } from '../constants'; +import { NORMAL_TXN_TYPE, TRANSACTION_STATUS } from '../constants'; import { fetchGetWithinPage } from '../helpers/fetch'; const BASE_URL = 'https://login.bankhapoalim.co.il'; @@ -20,6 +20,7 @@ function convertTransactions(txns) { originalCurrency: 'ILS', chargedAmount: isOutbound ? -txn.eventAmount : txn.eventAmount, description: txn.activityDescription, + status: txn.serialNumber === 0 ? TRANSACTION_STATUS.PENDING : TRANSACTION_STATUS.COMPLETED, }; }); } diff --git a/src/scrapers/leumi-card.js b/src/scrapers/leumi-card.js index 5c3cea76..0be53a8f 100644 --- a/src/scrapers/leumi-card.js +++ b/src/scrapers/leumi-card.js @@ -4,7 +4,7 @@ import moment from 'moment'; import { BaseScraperWithBrowser, LOGIN_RESULT } from './base-scraper-with-browser'; import { waitForNavigationAndDomLoad, waitForRedirect } from '../helpers/navigation'; import { waitUntilElementFound } from '../helpers/elements-interactions'; -import { NORMAL_TXN_TYPE, INSTALLMENTS_TXN_TYPE, SHEKEL_CURRENCY_SYMBOL, SHEKEL_CURRENCY } from '../constants'; +import { NORMAL_TXN_TYPE, INSTALLMENTS_TXN_TYPE, SHEKEL_CURRENCY_SYMBOL, SHEKEL_CURRENCY, TRANSACTION_STATUS } from '../constants'; import getAllMonthMoments from '../helpers/dates'; import { fixInstallments, sortTransactionsByDate, filterOldTransactions } from '../helpers/transactions'; @@ -108,6 +108,7 @@ function convertTransactions(rawTxns) { chargedAmount: -chargedAmountData.amount, description: txn.description.trim(), installments: getInstallmentsInfo(txn.comments), + status: TRANSACTION_STATUS.COMPLETED, }; }); } diff --git a/src/scrapers/leumi.js b/src/scrapers/leumi.js index b333fc33..90b31e3d 100644 --- a/src/scrapers/leumi.js +++ b/src/scrapers/leumi.js @@ -52,30 +52,15 @@ function convertTransactions(txns) { originalAmount: amount, originalCurrency: SHEKEL_CURRENCY, chargedAmount: amount, + status: txn.status, description: txn.description, /* memo: txn.memo, TODO add this line to export transaction memo */ }; }); } -async function fetchTransactionsForAccount(page, startDate) { - await dropdownSelect(page, 'select#ddlTransactionPeriod', '004'); - await waitUntilElementFound(page, 'select#ddlTransactionPeriod'); - await fillInput( - page, - 'input#dtFromDate_textBox', - startDate.format('DD/MM/YY'), - ); - await clickButton(page, 'input#btnDisplayDates'); - await waitForNavigation(page); - await waitUntilElementFound(page, 'table#WorkSpaceBox table#ctlActivityTable'); - await clickButton(page, 'a#lnkCtlExpandAll'); - - const selectedSnifAccount = await page.$eval('#ddlAccounts_m_ddl option[selected="selected"]', (option) => { - return (option.innerText || '').trim().replace(/‎|\u200E/gi, ''); - }); - - const accountNumber = selectedSnifAccount.replace('/', '_'); +async function extractCompletedTransactionsFromPage(page) { + const txns = []; const tdsValues = await page.$$eval('#WorkSpaceBox #ctlActivityTable tr td', (tds) => { return tds.map(td => @@ -85,14 +70,13 @@ async function fetchTransactionsForAccount(page, startDate) { })); }); - const txns = []; for (const element of tdsValues) { if (element.classList.includes('ExtendedActivityColumnDate')) { - const newTransaction = {}; + const newTransaction = { status: 'completed' }; newTransaction.date = (element.innerText || '').trim(); txns.push(newTransaction); } else if (element.classList.includes('ActivityTableColumn1LTR') - || element.classList.includes('ActivityTableColumn1')) { + || element.classList.includes('ActivityTableColumn1')) { const changedTransaction = txns.pop(); changedTransaction.description = element.innerText; txns.push(changedTransaction); @@ -119,6 +103,77 @@ async function fetchTransactionsForAccount(page, startDate) { } } + return txns; +} + +async function extractPendingTransactionsFromPage(page) { + const txns = []; + + const tdsValues = await page.$$eval('#WorkSpaceBox #trTodayActivityNapaTableUpper tr td', (tds) => { + return tds.map(td => + ({ + classList: td.getAttribute('class'), + innerText: td.innerText, + })); + }); + + for (const element of tdsValues) { + if (element.classList.includes('Colume1Width')) { + const newTransaction = { status: 'pending' }; + newTransaction.date = (element.innerText || '').trim(); + txns.push(newTransaction); + } else if (element.classList.includes('Colume2Width')) { + const changedTransaction = txns.pop(); + changedTransaction.description = element.innerText; + txns.push(changedTransaction); + } else if (element.classList.includes('Colume3Width')) { + const changedTransaction = txns.pop(); + changedTransaction.reference = element.innerText; + txns.push(changedTransaction); + } else if (element.classList.includes('Colume4Width')) { + const changedTransaction = txns.pop(); + changedTransaction.debit = element.innerText; + txns.push(changedTransaction); + } else if (element.classList.includes('Colume5Width')) { + const changedTransaction = txns.pop(); + changedTransaction.credit = element.innerText; + txns.push(changedTransaction); + } else if (element.classList.includes('Colume6Width')) { + const changedTransaction = txns.pop(); + changedTransaction.balance = element.innerText; + txns.push(changedTransaction); + } + } + + return txns; +} + +async function fetchTransactionsForAccount(page, startDate) { + await dropdownSelect(page, 'select#ddlTransactionPeriod', '004'); + await waitUntilElementFound(page, 'select#ddlTransactionPeriod'); + await fillInput( + page, + 'input#dtFromDate_textBox', + startDate.format(DATE_FORMAT), + ); + await clickButton(page, 'input#btnDisplayDates'); + await waitForNavigation(page); + await waitUntilElementFound(page, 'table#WorkSpaceBox table#ctlActivityTable'); + await clickButton(page, 'a#lnkCtlExpandAll'); + + const selectedSnifAccount = await page.$eval('#ddlAccounts_m_ddl option[selected="selected"]', (option) => { + return option.innerText; + }); + + const accountNumber = selectedSnifAccount.replace('/', '_'); + + const pendingTxns = await extractPendingTransactionsFromPage(page); + const completedTxns = await extractCompletedTransactionsFromPage(page); + const txns = [ + ...pendingTxns, + ...completedTxns, + ]; + return { accountNumber, txns: convertTransactions(txns), diff --git a/src/scrapers/visa-cal.js b/src/scrapers/visa-cal.js index b5f80121..7965e7a3 100644 --- a/src/scrapers/visa-cal.js +++ b/src/scrapers/visa-cal.js @@ -11,6 +11,7 @@ import { SHEKEL_CURRENCY, DOLLAR_CURRENCY_SYMBOL, DOLLAR_CURRENCY, + TRANSACTION_STATUS, } from '../constants'; import { fetchGet, fetchPost } from '../helpers/fetch'; import { fixInstallments, sortTransactionsByDate, filterOldTransactions } from '../helpers/transactions'; @@ -107,6 +108,7 @@ function convertTransactions(txns) { chargedAmount: -txn.DebitAmount.Value, description: txn.MerchantDetails.Name, installments: getInstallmentsInfo(txn), + status: TRANSACTION_STATUS.COMPLETED, }; }); }