New: handle oneZero scrape

packages/main/src/backend/configManager/configManager.ts

@@ -4,6 +4,7 @@ import { decrypt, encrypt } from '@/backend/configManager/encryption/crypto';
 import { existsSync, promises as fs } from 'fs';
 import configExample from './defaultConfig';
 import logger from '/@/logging/logger';
+import type { CompanyTypes } from 'israeli-bank-scrapers-core';
 
 export async function getConfig(configPath: string = configFilePath): Promise<Config> {
   const configFromFile = await getConfigFromFile(configPath);
@@ -35,6 +36,23 @@ export async function updateConfig(configPath: string, configToUpdate: Config):
   await fs.writeFile(configPath, encryptedConfigStr);
 }
 
+export async function updateOtpLongTermToken(
+  key: CompanyTypes,
+  otpLongTermToken: string,
+  configPath: string = configFilePath,
+): Promise<void> {
+  const config = await getConfig(configPath);
+  const account = config.scraping.accountsToScrape.find((acc) => acc.key === key) as {
+    loginFields: { otpLongTermToken: string };
+  };
+  if (account) {
+    account.loginFields.otpLongTermToken = otpLongTermToken;
+    await updateConfig(configPath, config);
+  } else {
+    throw new Error(`Account with key ${key} not found`);
+  }
+}
+
 async function getConfigFromFile(configPath: string) {
   if (existsSync(configPath)) {
     return fs.readFile(configPath, {

packages/main/src/backend/import/bankScraper.ts

@@ -1,13 +1,12 @@
 import { type AccountToScrapeConfig } from '@/backend/commonTypes';
-import { CompanyTypes, createScraper, SCRAPERS, type ScraperOptions } from 'israeli-bank-scrapers-core';
+import { CompanyTypes, createScraper, type ScraperOptions, SCRAPERS } from 'israeli-bank-scrapers-core';
+import { ipcMain } from 'electron';
+import { EventNames, type EventPublisher } from '@/backend/eventEmitters/EventEmitter';
+import { updateOtpLongTermToken } from '@/backend/configManager/configManager';
 
 export const inputVendors = Object.keys(SCRAPERS)
-  // Deprecated. see https://github.com/eshaham/israeli-bank-scrapers/blob/07ecd3de0c4aa051f119aa943493f0cda943158c/src/definitions.ts#L26-L29
   .filter((key) => key !== CompanyTypes.hapoalimBeOnline)
-  .map((key) => ({
-    key,
-    ...SCRAPERS[key as CompanyTypes],
-  }));
+  .map((key) => ({ key, ...SCRAPERS[key as CompanyTypes] }));
 
 interface ScrapeParameters {
   companyId: AccountToScrapeConfig['key'];
@@ -23,20 +22,38 @@ export async function scrape(
   { companyId, credentials, startDate, timeout, showBrowser = false }: ScrapeParameters,
   emitProgressEvent: EmitProgressEventFunction,
   chromePath: string,
+  eventPublisher: EventPublisher,
 ) {
   const options: ScraperOptions = {
-    companyId, // mandatory; one of 'hapoalim', 'discount', 'otsarHahayal', 'leumiCard', 'isracard', 'amex'
-    startDate, // the date to fetch transactions from (can't be before the minimum allowed time difference for the scraper)
-    combineInstallments: false, // if set to true, all installment transactions will be combine into the first one
-    showBrowser, // shows the browser while scraping, good for debugging (default false)
-    verbose: false, // include more debug info about in the output
+    companyId,
+    startDate,
+    combineInstallments: false,
+    showBrowser,
+    verbose: false,
     executablePath: chromePath,
     defaultTimeout: timeout,
   };
+
   const scraper = createScraper(options);
-  scraper.onProgress((eventCompanyId: string, payload: { type: string }) => {
-    emitProgressEvent(companyId, payload.type);
-  });
-  const scrapeResult = await scraper.scrape(credentials);
-  return scrapeResult;
+  scraper.onProgress((eventCompanyId, payload) => emitProgressEvent(companyId, payload.type));
+
+  if (companyId === CompanyTypes.oneZero) {
+    const creds = credentials as typeof credentials & { otpLongTermToken: string; phoneNumber: string };
+    if (!creds.otpLongTermToken) {
+      await scraper.triggerTwoFactorAuth(creds.phoneNumber);
+      await eventPublisher.emit(EventNames.GET_OTP);
+      const otpCode = await new Promise<string>((resolve) => {
+        ipcMain.once('get-otp-response', (event, input) => resolve(input));
+      });
+      const result = await scraper.getLongTermTwoFactorToken(otpCode);
+      if ('longTermTwoFactorAuthToken' in result) {
+        await updateOtpLongTermToken(companyId, result.longTermTwoFactorAuthToken);
+        creds.otpLongTermToken = result.longTermTwoFactorAuthToken;
+      } else {
+        throw new Error('Failed to get long-term two-factor auth token');
+      }
+    }
+  }
+
+  return await scraper.scrape(credentials);
 }

packages/main/src/backend/import/importTransactions.ts

@@ -1,25 +1,20 @@
-import { configFilePath, userDataPath } from '@/app-globals';
+import { userDataPath } from '@/app-globals';
 import {
   type AccountToScrapeConfig,
   type Config,
   type EnrichedTransaction,
-  type FinancialAccountDetails,
   type ScraperScrapingResult,
 } from '@/backend/commonTypes';
-import { getConfig } from '@/backend/configManager/configManager';
 import * as bankScraper from '@/backend/import/bankScraper';
 import Bottleneck from 'bottleneck';
 import { type Transaction } from 'israeli-bank-scrapers-core/lib/transactions';
-import _ from 'lodash';
 import moment from 'moment';
-// import * as categoryCalculation from '@/backend/import/categoryCalculationScript';
 import {
   AccountStatus,
-  BudgetTrackingEventEmitter,
   DownalodChromeEvent,
   EventNames,
-  ImporterEvent,
   type EventPublisher,
+  ImporterEvent,
 } from '../eventEmitters/EventEmitter';
 import { calculateTransactionHash } from '../transactions/transactions';
 import getChrome from './downloadChromium';
@@ -97,26 +92,6 @@ function emitChromeDownload(eventPublisher: EventPublisher, percent: number) {
   eventPublisher.emit(EventNames.DOWNLOAD_CHROME, new DownalodChromeEvent(percent));
 }
 
-export async function getFinancialAccountDetails(): Promise<FinancialAccountDetails[]> {
-  const config = await getConfig(configFilePath);
-  const eventEmitter = new BudgetTrackingEventEmitter();
-
-  const startDate = moment().subtract(30, 'days').startOf('day').toDate();
-
-  const companyIdToTransactions = await scrapeFinancialAccountsAndFetchTransactions(
-    config.scraping,
-    startDate,
-    eventEmitter,
-  );
-  const financialAccountDetails: { name: string; accountNumber: string }[] = [];
-  Object.keys(companyIdToTransactions).forEach((companyId) => {
-    let accountNumbers = companyIdToTransactions[companyId].map((transaction) => transaction.accountNumber);
-    accountNumbers = _.uniq(accountNumbers);
-    accountNumbers.forEach((accountNumber) => financialAccountDetails.push({ name: companyId, accountNumber }));
-  });
-  return financialAccountDetails;
-}
-
 async function fetchTransactions(
   account: AccountToScrapeConfig,
   startDate: Date,
@@ -142,6 +117,7 @@ async function fetchTransactions(
       },
       emitImporterProgressEvent,
       chromePath,
+      eventPublisher,
     );
     if (!scrapeResult.success) {
       throw new Error(`${scrapeResult.errorType}: ${scrapeResult.errorMessage}`);
@@ -193,14 +169,12 @@ async function postProcessTransactions(
 
 function enrichTransaction(transaction: Transaction, companyId: string, accountNumber: string): EnrichedTransaction {
   const hash = calculateTransactionHash(transaction, companyId, accountNumber);
-  // const category = categoryCalculation.getCategoryNameByTransactionDescription(transaction.description);
-  const enrichedTransaction: EnrichedTransaction = {
+  return {
     ...transaction,
     accountNumber,
     // category,
     hash,
   };
-  return enrichedTransaction;
 }
 
 function transactionsDateComparator(t1: Transaction, t2: Transaction) {

packages/main/src/manual/setupHelpers.ts

@@ -2,7 +2,6 @@
 import { FETCH_YNAB_ACCOUNT_DATA_STATUS, type YnabAccountDataType, type YnabConfig } from '@/backend/commonTypes';
 import { getConfig } from '@/backend/configManager/configManager';
 import { getYnabAccountDetails, isAccessTokenValid } from '@/backend/export/outputVendors/ynab/ynab';
-// import { getFinancialAccountDetails } from '@/backend/import/importTransactions';
 
 export async function getYnabAccountData(_: unknown, ynabOptions: YnabConfig['options']): Promise<YnabAccountDataType> {
   const config = await getConfig();
@@ -13,21 +12,16 @@ export async function getYnabAccountData(_: unknown, ynabOptions: YnabConfig['op
       status: FETCH_YNAB_ACCOUNT_DATA_STATUS.INVALID_ACCESS_TOKEN,
     };
   }
-  // const ynabAccountDataPromise = getYnabAccountDetails(config.outputVendors);
-  // const financialAccountDetailsPromise = getFinancialAccountDetails();
-  // const [ynabAccountData, financialAccountDetails] = await Promise.all([ynabAccountDataPromise, financialAccountDetailsPromise]);
   const ynabAccountData = await getYnabAccountDetails(
     config.outputVendors,
     ynabOptions.budgetId,
     ynabOptions.accessToken,
   );
-  // const financialAccountDetails = [];
 
   return {
     status: ynabAccountData.categories
       ? FETCH_YNAB_ACCOUNT_DATA_STATUS.SUCCESS
       : FETCH_YNAB_ACCOUNT_DATA_STATUS.INVALID_BUDGET_ID,
     ynabAccountData,
-    // financialAccountDetails
   };
 }

packages/renderer/src/components/App.tsx

@@ -1,6 +1,7 @@
 import { StoresProvider } from '../store';
 import './App.css';
 import Body from './Body';
+import GetOtp from './GetOtp';
 import TopBar from './topBar/TopBar';
 
 function App() {
@@ -9,6 +10,7 @@ function App() {
       <div className="App">
         <TopBar />
         <Body />
+        <GetOtp />
       </div>
     </StoresProvider>
   );

packages/renderer/src/components/GetOtp.tsx

@@ -1,7 +1,7 @@
 import React, { useEffect, useState } from 'react';
 import { Button, FormControl, Modal } from 'react-bootstrap';
 import { observer } from 'mobx-react-lite';
-import { useConfigStore } from '/@/store/ConfigStore';
+import { useConfigStore } from '../store/ConfigStore';
 import { sendOTPResponse } from '#preload';
 
 const GetOtp = () => {