From aec5ba73519c187efc550e3f6074eba6f25c368e Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 27 May 2024 07:16:46 +0300 Subject: [PATCH 1/6] Speed up back iteration --- workers/loc.api/sync/helpers/get-back-iterable.js | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/workers/loc.api/sync/helpers/get-back-iterable.js b/workers/loc.api/sync/helpers/get-back-iterable.js index 18eee8180..2e31f278b 100644 --- a/workers/loc.api/sync/helpers/get-back-iterable.js +++ b/workers/loc.api/sync/helpers/get-back-iterable.js @@ -5,15 +5,18 @@ module.exports = (array) => { [Symbol.iterator] (areEntriesReturned) { return { index: array.length, + res: { + done: false, + value: undefined + }, next () { this.index -= 1 + this.res.done = this.index < 0 + this.res.value = areEntriesReturned + ? [this.index, array[this.index]] + : array[this.index] - return { - done: this.index < 0, - value: areEntriesReturned - ? [this.index, array[this.index]] - : array[this.index] - } + return this.res } } }, From 4a53ea37094adad039bc5863a5232d1de0418498 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 27 May 2024 07:22:29 +0300 Subject: [PATCH 2/6] Add utility to push large array into array --- workers/loc.api/helpers/index.js | 6 ++++-- workers/loc.api/helpers/utils.js | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/workers/loc.api/helpers/index.js b/workers/loc.api/helpers/index.js index 677615245..528f54f6f 100644 --- a/workers/loc.api/helpers/index.js +++ b/workers/loc.api/helpers/index.js @@ -11,7 +11,8 @@ const { pickLowerObjectsNumbers, sumAllObjectsNumbers, pickAllLowerObjectsNumbers, - sumArrayVolumes + sumArrayVolumes, + pushLargeArr } = require('./utils') const { isSubAccountApiKeys, @@ -33,5 +34,6 @@ module.exports = { pickLowerObjectsNumbers, sumAllObjectsNumbers, pickAllLowerObjectsNumbers, - sumArrayVolumes + sumArrayVolumes, + pushLargeArr } diff --git a/workers/loc.api/helpers/utils.js b/workers/loc.api/helpers/utils.js index 541bb2591..75a8a374e 100644 --- a/workers/loc.api/helpers/utils.js +++ b/workers/loc.api/helpers/utils.js @@ -266,6 +266,12 @@ const sumArrayVolumes = (propName, objects = []) => { }, []) } +const pushLargeArr = (dest, src) => { + for (const item of src) { + dest.push(item) + } +} + module.exports = { checkParamsAuth, tryParseJSON, @@ -276,5 +282,6 @@ module.exports = { pickLowerObjectsNumbers, sumAllObjectsNumbers, pickAllLowerObjectsNumbers, - sumArrayVolumes + sumArrayVolumes, + pushLargeArr } From 4ee6760901970bdc45bd16c82d75f2e125890b63 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 27 May 2024 07:24:27 +0300 Subject: [PATCH 3/6] Add CurrencyConversionError class --- workers/loc.api/errors/index.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/errors/index.js b/workers/loc.api/errors/index.js index 6cd45c546..2f5915376 100644 --- a/workers/loc.api/errors/index.js +++ b/workers/loc.api/errors/index.js @@ -248,6 +248,12 @@ class AuthTokenTTLSettingError extends ArgsParamsError { } } +class CurrencyConversionError extends BaseError { + constructor (data, message = 'ERR_CURRENCY_HAS_NOT_BEEN_CONVERTED_TO_USD') { + super({ data, message }) + } +} + module.exports = { BaseError, CollSyncPermissionError, @@ -284,5 +290,6 @@ module.exports = { LastSyncedInfoGettingError, SyncInfoUpdatingError, AuthTokenGenerationError, - AuthTokenTTLSettingError + AuthTokenTTLSettingError, + CurrencyConversionError } From 604c3419d2694ca270a1d36217eafc8aab81a9de Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 27 May 2024 07:25:49 +0300 Subject: [PATCH 4/6] Add CurrencyPairSeparationError class --- workers/loc.api/errors/index.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/workers/loc.api/errors/index.js b/workers/loc.api/errors/index.js index 2f5915376..da5d3ef0e 100644 --- a/workers/loc.api/errors/index.js +++ b/workers/loc.api/errors/index.js @@ -254,6 +254,12 @@ class CurrencyConversionError extends BaseError { } } +class CurrencyPairSeparationError extends BaseError { + constructor (data, message = 'ERR_CURRENCY_PAIR_HAS_NOT_BEEN_SEPARATED_CORRECTLY') { + super({ data, message }) + } +} + module.exports = { BaseError, CollSyncPermissionError, @@ -291,5 +297,6 @@ module.exports = { SyncInfoUpdatingError, AuthTokenGenerationError, AuthTokenTTLSettingError, - CurrencyConversionError + CurrencyConversionError, + CurrencyPairSeparationError } From 7f12b5b211170e84e5d606e6abad61812c4a52bb Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 27 May 2024 07:30:06 +0300 Subject: [PATCH 5/6] Add core fn for lookup trx with realized or unrealized profit --- .../transaction.tax.report/helpers/index.js | 4 +- .../helpers/look-up-trades.js | 331 ++++++++++++++++++ 2 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 workers/loc.api/sync/transaction.tax.report/helpers/look-up-trades.js diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/index.js b/workers/loc.api/sync/transaction.tax.report/helpers/index.js index 7761b3c11..903a38c7e 100644 --- a/workers/loc.api/sync/transaction.tax.report/helpers/index.js +++ b/workers/loc.api/sync/transaction.tax.report/helpers/index.js @@ -3,9 +3,11 @@ const TRX_TAX_STRATEGIES = require('./trx.tax.strategies') const remapTrades = require('./remap-trades') const remapMovements = require('./remap-movements') +const lookUpTrades = require('./look-up-trades') module.exports = { TRX_TAX_STRATEGIES, remapTrades, - remapMovements + remapMovements, + lookUpTrades } diff --git a/workers/loc.api/sync/transaction.tax.report/helpers/look-up-trades.js b/workers/loc.api/sync/transaction.tax.report/helpers/look-up-trades.js new file mode 100644 index 000000000..a1b4b002d --- /dev/null +++ b/workers/loc.api/sync/transaction.tax.report/helpers/look-up-trades.js @@ -0,0 +1,331 @@ +'use strict' + +const { setImmediate } = require('node:timers/promises') +const splitSymbolPairs = require( + 'bfx-report/workers/loc.api/helpers/split-symbol-pairs' +) + +const { + isForexSymb, + getBackIterable +} = require('../../helpers') + +const { + CurrencyConversionError, + CurrencyPairSeparationError +} = require('../../../errors') + +module.exports = async (trades, opts) => { + const { + isBackIterativeSaleLookUp = false, + isBackIterativeBuyLookUp = false, + isBuyTradesWithUnrealizedProfitRequired = false, + isNotGainOrLossRequired = false + } = opts ?? {} + + const saleTradesWithRealizedProfit = [] + const buyTradesWithUnrealizedProfit = [] + + if ( + !Array.isArray(trades) || + trades.length === 0 + ) { + return { + saleTradesWithRealizedProfit, + buyTradesWithUnrealizedProfit + } + } + + let lastLoopUnlockMts = Date.now() + const tradeIterator = isBackIterativeSaleLookUp + ? getBackIterable(trades) + : trades + + for (const [i, trade] of tradeIterator.entries()) { + const currentLoopUnlockMts = Date.now() + + /* + * Trx hist restoring is a hard sync operation, + * to prevent EventLoop locking more than 1sec + * it needs to resolve async queue + */ + if ((currentLoopUnlockMts - lastLoopUnlockMts) > 1000) { + await setImmediate() + + lastLoopUnlockMts = currentLoopUnlockMts + } + + trade.isAdditionalTrxMovements = trade.isAdditionalTrxMovements ?? false + + trade.isSaleTrx = trade.isSaleTrx ?? false + trade.isSaleTrxHistFilled = trade.isSaleTrxHistFilled ?? false + trade.saleFilledAmount = trade.saleFilledAmount ?? 0 + trade.costForSaleTrxUsd = trade.costForSaleTrxUsd ?? 0 + trade.buyTrxsForRealizedProfit = trade + .buyTrxsForRealizedProfit ?? [] + + trade.isBuyTrx = trade.isBuyTrx ?? false + trade.isBuyTrxHistFilled = trade.isBuyTrxHistFilled ?? false + trade.buyFilledAmount = trade.buyFilledAmount ?? 0 + trade.proceedsForBuyTrxUsd = trade.proceedsForBuyTrxUsd ?? 0 + trade.saleTrxsForRealizedProfit = trade + .saleTrxsForRealizedProfit ?? [] + + if ( + !trade?.symbol || + !Number.isFinite(trade?.execPrice) || + ( + !isBuyTradesWithUnrealizedProfitRequired && + trade.execPrice === 0 + ) || + !Number.isFinite(trade?.execAmount) || + trade.execAmount === 0 + ) { + continue + } + + const [firstSymb, lastSymb] = ( + trade?.firstSymb && + trade?.lastSymb + ) + ? [trade?.firstSymb, trade?.lastSymb] + : splitSymbolPairs(trade.symbol) + trade.firstSymb = firstSymb + trade.lastSymb = lastSymb + + /* + * Exapmle of considered trxs as sale: + * - buy ETC:BTC -> amount 5, price 0.5 (here needs to be considered as 2 trxs: buy ETC and sale BTC) + * - sale ETC:BTC -> amount -2, price 0.6 (here needs to be considered as 2 trxs: sale ETC and buy BTC) + * - sale ETC:USD -> amount -3, price 4000 + * - sale UST:EUR - > amount -3, price 0.9 (here needs to be considered EUR price and converted to USD) + */ + const isLastSymbForex = isForexSymb(lastSymb) + const isDistinctSale = trade.execAmount < 0 + const isSaleBetweenCrypto = ( + trade.execAmount > 0 && + !isLastSymbForex + ) + trade.isSaleTrx = isDistinctSale || isSaleBetweenCrypto + trade.isBuyTrx = ( + trade.execAmount > 0 || + !isLastSymbForex + ) + + if ( + !trade.isSaleTrx || + trade.isBuyTradesWithUnrealizedProfitForPrevPeriod + ) { + continue + } + if ( + !firstSymb || + !lastSymb + ) { + throw new CurrencyPairSeparationError({ + symbol: trade.symbol, + firstSymb, + lastSymb + }) + } + + const saleAmount = trade.execAmount < 0 + ? Math.abs(trade.execAmount) + : Math.abs(trade.execAmount * trade.execPrice) + const _salePriceUsd = isDistinctSale + ? trade.firstSymbPriceUsd + : trade.lastSymbPriceUsd + const salePriceUsd = isNotGainOrLossRequired ? 0 : _salePriceUsd + const saleAsset = isDistinctSale + ? firstSymb + : lastSymb + + if (!Number.isFinite(salePriceUsd)) { + throw new CurrencyConversionError({ + symbol: saleAsset, + priceUsd: salePriceUsd + }) + } + + const startPoint = isBackIterativeBuyLookUp + ? trades.length - 1 + : i + 1 + const checkPoint = (j) => ( + isBackIterativeBuyLookUp + ? i < j + : trades.length > j + ) + const shiftPoint = (j) => ( + isBackIterativeBuyLookUp + ? j - 1 + : j + 1 + ) + + for (let j = startPoint; checkPoint(j); j = shiftPoint(j)) { + if (trade.isSaleTrxHistFilled) { + break + } + + const tradeForLookup = trades[j] + + if ( + tradeForLookup?.isBuyTrxHistFilled || + !tradeForLookup?.symbol || + !Number.isFinite(tradeForLookup?.execAmount) || + tradeForLookup.execAmount === 0 || + !Number.isFinite(tradeForLookup?.execPrice) || + ( + !isBuyTradesWithUnrealizedProfitRequired && + tradeForLookup.execPrice === 0 + ) + ) { + continue + } + + tradeForLookup.isBuyTrx = tradeForLookup.isBuyTrx ?? false + tradeForLookup.isBuyTrxHistFilled = tradeForLookup + .isBuyTrxHistFilled ?? false + tradeForLookup.buyFilledAmount = tradeForLookup + .buyFilledAmount ?? 0 + tradeForLookup.proceedsForBuyTrxUsd = tradeForLookup.proceedsForBuyTrxUsd ?? 0 + tradeForLookup.saleTrxsForRealizedProfit = tradeForLookup + .saleTrxsForRealizedProfit ?? [] + + const [firstSymbForLookup, lastSymbForLookup] = ( + tradeForLookup?.firstSymb && + tradeForLookup?.lastSymb + ) + ? [tradeForLookup?.firstSymb, tradeForLookup?.lastSymb] + : splitSymbolPairs(tradeForLookup.symbol) + tradeForLookup.firstSymb = firstSymbForLookup + tradeForLookup.lastSymb = lastSymbForLookup + + if ( + !firstSymbForLookup || + !lastSymbForLookup + ) { + throw new CurrencyPairSeparationError({ + symbol: tradeForLookup.symbol, + firstSymb: firstSymbForLookup, + lastSymb: lastSymbForLookup + }) + } + + if ( + tradeForLookup.execAmount < 0 && + isForexSymb(lastSymbForLookup) + ) { + continue + } + + tradeForLookup.isBuyTrx = true + + const buyAsset = tradeForLookup.execAmount > 0 + ? firstSymbForLookup + : lastSymbForLookup + + if (saleAsset !== buyAsset) { + continue + } + + tradeForLookup.saleTrxsForRealizedProfit.push(trade) + trade.buyTrxsForRealizedProfit.push(tradeForLookup) + + const buyAmount = tradeForLookup.execAmount > 0 + ? Math.abs(tradeForLookup.execAmount) + : Math.abs(tradeForLookup.execAmount * tradeForLookup.execPrice) + const _buyPriceUsd = tradeForLookup.execAmount > 0 + ? tradeForLookup.firstSymbPriceUsd + : tradeForLookup.lastSymbPriceUsd + const buyPriceUsd = isNotGainOrLossRequired ? 0 : _buyPriceUsd + const buyRestAmount = buyAmount - tradeForLookup.buyFilledAmount + const saleRestAmount = saleAmount - trade.saleFilledAmount + + if (!Number.isFinite(buyPriceUsd)) { + throw new CurrencyConversionError({ + symbol: buyAsset, + priceUsd: buyPriceUsd + }) + } + + if (buyRestAmount < saleRestAmount) { + tradeForLookup.buyFilledAmount = buyAmount + trade.saleFilledAmount += buyRestAmount + tradeForLookup.proceedsForBuyTrxUsd += buyRestAmount * salePriceUsd + trade.costForSaleTrxUsd += buyRestAmount * buyPriceUsd + tradeForLookup.isBuyTrxHistFilled = true + } + if (buyRestAmount > saleRestAmount) { + tradeForLookup.buyFilledAmount += saleRestAmount + trade.saleFilledAmount = saleAmount + tradeForLookup.proceedsForBuyTrxUsd += saleRestAmount * salePriceUsd + trade.costForSaleTrxUsd += saleRestAmount * buyPriceUsd + trade.isSaleTrxHistFilled = true + } + if (buyRestAmount === saleRestAmount) { + tradeForLookup.buyFilledAmount = buyAmount + trade.saleFilledAmount = saleAmount + tradeForLookup.proceedsForBuyTrxUsd += buyRestAmount * salePriceUsd + trade.costForSaleTrxUsd += buyRestAmount * buyPriceUsd + tradeForLookup.isBuyTrxHistFilled = true + trade.isSaleTrxHistFilled = true + } + + if (tradeForLookup.isBuyTrxHistFilled) { + tradeForLookup.buyAsset = buyAsset + tradeForLookup.buyAmount = buyAmount + tradeForLookup.mtsAcquiredForBuyTrx = tradeForLookup.mtsCreate + tradeForLookup.mtsSoldForBuyTrx = trade.mtsCreate + tradeForLookup.costForBuyTrxUsd = buyAmount * buyPriceUsd + tradeForLookup.gainOrLossForBuyTrxUsd = tradeForLookup.proceedsForBuyTrxUsd - tradeForLookup.costForBuyTrxUsd + } + } + + trade.saleAsset = saleAsset + trade.saleAmount = saleAmount + trade.mtsAcquiredForSaleTrx = ( + trade.buyTrxsForRealizedProfit[0]?.mtsCreate > + trade.buyTrxsForRealizedProfit[trade.buyTrxsForRealizedProfit.length - 1]?.mtsCreate + ) + ? trade.buyTrxsForRealizedProfit[trade.buyTrxsForRealizedProfit.length - 1]?.mtsCreate + : trade.buyTrxsForRealizedProfit[0]?.mtsCreate + trade.mtsSoldForSaleTrx = trade.mtsCreate + trade.proceedsForSaleTrxUsd = saleAmount * salePriceUsd + trade.gainOrLossUsd = trade.proceedsForSaleTrxUsd - trade.costForSaleTrxUsd + } + + for (const trade of trades) { + if ( + isBuyTradesWithUnrealizedProfitRequired && + trade?.isBuyTrx && + !trade?.isBuyTrxHistFilled + ) { + trade.isBuyTradesWithUnrealizedProfitForPrevPeriod = true + buyTradesWithUnrealizedProfit.push(trade) + } + + if ( + isBuyTradesWithUnrealizedProfitRequired || + trade?.isBuyTradesWithUnrealizedProfitForPrevPeriod || + !trade?.isSaleTrx || + trade?.isAdditionalTrxMovements + ) { + continue + } + + saleTradesWithRealizedProfit.push({ + asset: trade.saleAsset, + amount: trade.saleAmount, + mtsAcquired: trade.mtsAcquiredForSaleTrx, + mtsSold: trade.mtsSoldForSaleTrx, + proceeds: trade.proceedsForSaleTrxUsd, + cost: trade.costForSaleTrxUsd, + gainOrLoss: trade.gainOrLossUsd + }) + } + + return { + saleTradesWithRealizedProfit, + buyTradesWithUnrealizedProfit + } +} From b3bdb097095c1d43fa426ae71c078c443e6dc4a4 Mon Sep 17 00:00:00 2001 From: Vladimir Voronkov Date: Mon, 27 May 2024 07:34:53 +0300 Subject: [PATCH 6/6] Implement main flow for lookup trx with required start-end --- .../sync/transaction.tax.report/index.js | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/workers/loc.api/sync/transaction.tax.report/index.js b/workers/loc.api/sync/transaction.tax.report/index.js index 93ddde252..e6b862684 100644 --- a/workers/loc.api/sync/transaction.tax.report/index.js +++ b/workers/loc.api/sync/transaction.tax.report/index.js @@ -1,9 +1,12 @@ 'use strict' +const { pushLargeArr } = require('../../helpers/utils') + const { TRX_TAX_STRATEGIES, remapTrades, - remapMovements + remapMovements, + lookUpTrades } = require('./helpers') const { decorateInjectable } = require('../../di/utils') @@ -102,8 +105,39 @@ class TransactionTaxReport { }) : { trxs: [] } - // TODO: - return [] + const isBackIterativeSaleLookUp = isFIFO && !isLIFO + const isBackIterativeBuyLookUp = isFIFO && !isLIFO + + const { buyTradesWithUnrealizedProfit } = await lookUpTrades( + trxsForPrevPeriod, + { + isBackIterativeSaleLookUp, + isBackIterativeBuyLookUp, + isBuyTradesWithUnrealizedProfitRequired: true, + isNotGainOrLossRequired: true + } + ) + + pushLargeArr(trxsForCurrPeriod, buyTradesWithUnrealizedProfit) + pushLargeArr( + trxsForConvToUsd, + buyTradesWithUnrealizedProfit + .filter((trx) => ( + !Number.isFinite(trx?.firstSymbPriceUsd) || + !Number.isFinite(trx?.lastSymbPriceUsd) + )) + ) + await this.#convertCurrencies(trxsForConvToUsd) + + const { saleTradesWithRealizedProfit } = await lookUpTrades( + trxsForCurrPeriod, + { + isBackIterativeSaleLookUp, + isBackIterativeBuyLookUp + } + ) + + return saleTradesWithRealizedProfit } async #getTrxs (params) { @@ -163,6 +197,9 @@ class TransactionTaxReport { } } + // TODO: + async #convertCurrencies (trxs, opts) {} + async #getTrades ({ user, start,