diff --git a/workers/loc.api/sync/dao/dao.better.sqlite.js b/workers/loc.api/sync/dao/dao.better.sqlite.js index b2b735c60..6c2a2a9cc 100644 --- a/workers/loc.api/sync/dao/dao.better.sqlite.js +++ b/workers/loc.api/sync/dao/dao.better.sqlite.js @@ -988,6 +988,7 @@ class BetterSqliteDAO extends DAO { filter = {}, sort = [], subQuery = { + filter: {}, sort: [] }, groupFns = [], @@ -1003,7 +1004,10 @@ class BetterSqliteDAO extends DAO { group, groupProj } = getGroupQuery({ groupFns, groupResBy }) - const _subQuery = getSubQuery({ name: collName, subQuery }) + const { + subQuery: _subQuery, + subQueryValues + } = getSubQuery({ name: collName, subQuery }) const _sort = getOrderQuery(sort) const { where, @@ -1035,7 +1039,7 @@ class BetterSqliteDAO extends DAO { return this.query({ action: MAIN_DB_WORKER_ACTIONS.ALL, sql, - params: { ...values, ...limitVal } + params: { ...values, ...subQueryValues, ...limitVal } }, { withWorkerThreads: true }) } diff --git a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js index 7d95bc35d..32316f81a 100644 --- a/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js +++ b/workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js @@ -24,6 +24,24 @@ class MigrationV41 extends AbstractMigration { 'ALTER TABLE trades ADD COLUMN exactUsdValue DECIMAL(22,12)', 'ALTER TABLE movements ADD COLUMN exactUsdValue DECIMAL(22,12)', + 'ALTER TABLE ledgers ADD COLUMN _isInvoicePayOrder INT', + `UPDATE ledgers SET _isInvoicePayOrder = ( + SELECT 1 FROM ( + SELECT * FROM ledgers AS l + WHERE l.description COLLATE NOCASE LIKE '%InvoicePay Order%' + AND l._id = ledgers._id + ) + )`, + + 'ALTER TABLE ledgers ADD COLUMN _isAirdropOnWallet INT', + `UPDATE ledgers SET _isAirdropOnWallet = ( + SELECT 1 FROM ( + SELECT * FROM ledgers AS l + WHERE l.description COLLATE NOCASE LIKE '%Airdrop on wallet%' + AND l._id = ledgers._id + ) + )`, + 'ALTER TABLE trades ADD COLUMN _isExchange INT', `UPDATE trades SET _isExchange = ( SELECT 1 FROM ( diff --git a/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js b/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js index 6c6745764..08d11a503 100644 --- a/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js +++ b/workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js @@ -44,7 +44,7 @@ module.exports = (args, methodColl, opts) => { group, groupProj } = getGroupQuery(methodColl) - const subQuery = getSubQuery(methodColl) + const { subQuery, subQueryValues } = getSubQuery(methodColl) const projection = getProjectionQuery( _model, exclude, @@ -65,6 +65,6 @@ module.exports = (args, methodColl, opts) => { return { sql, - sqlParams: { ...values, ...limitVal } + sqlParams: { ...values, ...subQueryValues, ...limitVal } } } diff --git a/workers/loc.api/sync/dao/helpers/get-sub-query.js b/workers/loc.api/sync/dao/helpers/get-sub-query.js index 43f69cd9a..b536f5024 100644 --- a/workers/loc.api/sync/dao/helpers/get-sub-query.js +++ b/workers/loc.api/sync/dao/helpers/get-sub-query.js @@ -1,16 +1,39 @@ 'use strict' +const getWhereQuery = require('./get-where-query') const getOrderQuery = require('./get-order-query') -module.exports = ({ - name, - subQuery: { sort = [] } = {} -} = {}) => { +module.exports = (params) => { + const { + name + } = params ?? {} + const filter = params?.subQuery?.filter ?? {} + const sort = params?.subQuery?.sort ?? [] + + const alias = `${name}_sub_q` + const { + where, + values + } = getWhereQuery( + filter, + { alias } + ) const _sort = getOrderQuery(sort) - if (!_sort) { - return name + if ( + !_sort && + !where + ) { + return { + subQuery: name, + subQueryValues: {} + } } - return `(SELECT * FROM ${name} ${_sort})` + const subQuery = `(SELECT * FROM ${name} AS ${alias} ${where} ${_sort})` + + return { + subQuery, + subQueryValues: values + } } diff --git a/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js b/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js index e34febd4c..26d52e079 100644 --- a/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js +++ b/workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js @@ -16,7 +16,10 @@ module.exports = ( '_isMarginFundingPayment', '_isAffiliateRebate', '_isStakingPayments', - '_isSubAccountsTransfer' + '_isSubAccountsTransfer', + '_isInvoicePayOrder', + '_isAirdropOnWallet', + '_isExchange' ] ) => { if ( diff --git a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js index 71f0472c8..57830ae32 100644 --- a/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js +++ b/workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js @@ -133,6 +133,14 @@ class ApiMiddlewareHandlerAfter { { fieldName: '_isSubAccountsTransfer', pattern: '^transfer.+sa[(].+[)]' + }, + { + fieldName: '_isInvoicePayOrder', + pattern: 'InvoicePay Order' + }, + { + fieldName: '_isAirdropOnWallet', + pattern: 'Airdrop on wallet' } ] ), diff --git a/workers/loc.api/sync/movements/index.js b/workers/loc.api/sync/movements/index.js index 52b575a03..6f9ca590d 100644 --- a/workers/loc.api/sync/movements/index.js +++ b/workers/loc.api/sync/movements/index.js @@ -3,6 +3,8 @@ const { orderBy } = require('lodash') const { merge } = require('lib-js-util-base') +const { pushLargeArr } = require('../../helpers/utils') + const { decorateInjectable } = require('../../di/utils') const depsTypes = (TYPES) => [ @@ -41,7 +43,8 @@ class Movements { isExcludePrivate = true, isWithdrawals = false, isDeposits = false, - isMovementsWithoutSATransferLedgers = false + isMovementsWithoutSATransferLedgers = false, + areExtraPaymentsIncluded = false } = params ?? {} const user = await this.authenticator @@ -86,11 +89,44 @@ class Movements { } ) + const ledgersOrder = this._getLedgersOrder(sort) + const extraMovementsPromise = this.getExtraMovements({ + auth: user, + start, + end, + sort: ledgersOrder, + isWithdrawals, + isDeposits, + isExcludePrivate, + areExtraPaymentsIncluded + }) + if (isMovementsWithoutSATransferLedgers) { - return movementsPromise + const [ + movements, + extraMovements + ] = await Promise.all([ + movementsPromise, + extraMovementsPromise + ]) + + const remapedLedgers = this._remapLedgersToMovements( + extraMovements + ) + pushLargeArr(movements, remapedLedgers) + const { + propNames, + orders + } = this._getLodashOrder(sort) + const orderedRes = orderBy( + movements, + propNames, + orders + ) + + return orderedRes } - const ledgersOrder = this._getLedgersOrder(sort) const ledgersPromise = this.getSubAccountsTransferLedgers({ auth: user, start, @@ -103,16 +139,19 @@ class Movements { const [ movements, + extraMovements, ledgers ] = await Promise.all([ movementsPromise, + extraMovementsPromise, ledgersPromise ]) + pushLargeArr(extraMovements, ledgers) const remapedLedgers = this._remapLedgersToMovements( - ledgers + extraMovements ) - movements.push(...remapedLedgers) + pushLargeArr(movements, remapedLedgers) const { propNames, @@ -176,6 +215,72 @@ class Movements { ) } + /* + * Considers the following additional movements from ledgers: + * - `InvoicePay Order` + * - `Airdrop on wallet` + */ + getExtraMovements (params = {}) { + const { + auth = {}, + start = 0, + end = Date.now(), + filter: _filter, + sort = [['mts', -1], ['id', -1]], + projection = this.ledgersModel, + exclude = ['user_id'], + isExcludePrivate = true, + isWithdrawals = false, + isDeposits = false, + areExtraPaymentsIncluded = false + } = params ?? {} + + const withdrawalsFilter = isWithdrawals + ? { $lt: { amount: 0 } } + : {} + const depositsFilter = isDeposits + ? { $gt: { amount: 0 } } + : {} + const filter = merge( + {}, + withdrawalsFilter, + depositsFilter, + _filter + ) + const extraPaymentsFilter = areExtraPaymentsIncluded + ? { + $or: { + $eq: { + _isInvoicePayOrder: 1, + _isAirdropOnWallet: 1, + _isMarginFundingPayment: 1, + _isAffiliateRebate: 1, + _isStakingPayments: 1 + } + } + } + : { $or: { $eq: { _isInvoicePayOrder: 1 } } } + + return this.dao.getElemsInCollBy( + this.ALLOWED_COLLS.LEDGERS, + { + subQuery: { + filter: extraPaymentsFilter + }, + filter: { + $lte: { mts: end }, + $gte: { mts: start }, + user_id: auth._id, + ...filter + }, + sort, + projection, + exclude, + isExcludePrivate + } + ) + } + _remapLedgersToMovements (ledgers) { return ledgers.map((ledger) => { const { diff --git a/workers/loc.api/sync/schema/models.js b/workers/loc.api/sync/schema/models.js index 0b4da88a4..4a6cf2ec1 100644 --- a/workers/loc.api/sync/schema/models.js +++ b/workers/loc.api/sync/schema/models.js @@ -119,6 +119,8 @@ const _models = new Map([ _isStakingPayments: 'INT', _isSubAccountsTransfer: 'INT', _isBalanceRecalced: 'INT', + _isInvoicePayOrder: 'INT', + _isAirdropOnWallet: 'INT', subUserId: 'INT', user_id: 'INT NOT NULL', @@ -134,6 +136,8 @@ const _models = new Map([ ['user_id', '_category', 'mts'], ['user_id', 'mts'], ['currency', 'mts'], + ['_isInvoicePayOrder'], + ['_isAirdropOnWallet'], ['user_id', 'subUserId', 'mts', 'WHERE subUserId IS NOT NULL'], ['subUserId', 'mts', '_id', diff --git a/workers/loc.api/sync/transaction.tax.report/index.js b/workers/loc.api/sync/transaction.tax.report/index.js index a45d74950..ebc1db968 100644 --- a/workers/loc.api/sync/transaction.tax.report/index.js +++ b/workers/loc.api/sync/transaction.tax.report/index.js @@ -183,14 +183,16 @@ class TransactionTaxReport { start, end, isWithdrawals: true, - isExcludePrivate: false + isExcludePrivate: false, + areExtraPaymentsIncluded: true }) const depositsPromise = this.movements.getMovements({ auth: user, start, end, isDeposits: true, - isExcludePrivate: false + isExcludePrivate: false, + areExtraPaymentsIncluded: true }) const [