Add InvoicePayOrder and AirdropOnWallet ledgers to movements

workers/loc.api/sync/dao/dao.better.sqlite.js

@@ -988,6 +988,7 @@ class BetterSqliteDAO extends DAO {
       filter = {},
       sort = [],
       subQuery = {
+        filter: {},
         sort: []
       },
       groupFns = [],
@@ -1003,7 +1004,10 @@ class BetterSqliteDAO extends DAO {
       group,
       groupProj
     } = getGroupQuery({ groupFns, groupResBy })
-    const _subQuery = getSubQuery({ name: collName, subQuery })
+    const {
+      subQuery: _subQuery,
+      subQueryValues
+    } = getSubQuery({ name: collName, subQuery })
     const _sort = getOrderQuery(sort)
     const {
       where,
@@ -1035,7 +1039,7 @@ class BetterSqliteDAO extends DAO {
     return this.query({
       action: MAIN_DB_WORKER_ACTIONS.ALL,
       sql,
-      params: { ...values, ...limitVal }
+      params: { ...values, ...subQueryValues, ...limitVal }
     }, { withWorkerThreads: true })
   }
 

workers/loc.api/sync/dao/db-migrations/sqlite-migrations/migration.v41.1716385152034.js

@@ -24,6 +24,24 @@ class MigrationV41 extends AbstractMigration {
       'ALTER TABLE trades ADD COLUMN exactUsdValue DECIMAL(22,12)',
       'ALTER TABLE movements ADD COLUMN exactUsdValue DECIMAL(22,12)',
 
+      'ALTER TABLE ledgers ADD COLUMN _isInvoicePayOrder INT',
+      `UPDATE ledgers SET _isInvoicePayOrder = (
+        SELECT 1 FROM (
+          SELECT * FROM ledgers AS l
+          WHERE l.description COLLATE NOCASE LIKE '%InvoicePay Order%'
+            AND l._id = ledgers._id
+        )
+      )`,
+
+      'ALTER TABLE ledgers ADD COLUMN _isAirdropOnWallet INT',
+      `UPDATE ledgers SET _isAirdropOnWallet = (
+        SELECT 1 FROM (
+          SELECT * FROM ledgers AS l
+          WHERE l.description COLLATE NOCASE LIKE '%Airdrop on wallet%'
+            AND l._id = ledgers._id
+        )
+      )`,
+
       'ALTER TABLE trades ADD COLUMN _isExchange INT',
       `UPDATE trades SET _isExchange = (
         SELECT 1 FROM (

workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js

@@ -44,7 +44,7 @@ module.exports = (args, methodColl, opts) => {
     group,
     groupProj
   } = getGroupQuery(methodColl)
-  const subQuery = getSubQuery(methodColl)
+  const { subQuery, subQueryValues } = getSubQuery(methodColl)
   const projection = getProjectionQuery(
     _model,
     exclude,
@@ -65,6 +65,6 @@ module.exports = (args, methodColl, opts) => {
 
   return {
     sql,
-    sqlParams: { ...values, ...limitVal }
+    sqlParams: { ...values, ...subQueryValues, ...limitVal }
   }
 }

workers/loc.api/sync/dao/helpers/get-sub-query.js

@@ -1,16 +1,39 @@
 'use strict'
 
+const getWhereQuery = require('./get-where-query')
 const getOrderQuery = require('./get-order-query')
 
-module.exports = ({
-  name,
-  subQuery: { sort = [] } = {}
-} = {}) => {
+module.exports = (params) => {
+  const {
+    name
+  } = params ?? {}
+  const filter = params?.subQuery?.filter ?? {}
+  const sort = params?.subQuery?.sort ?? []
+
+  const alias = `${name}_sub_q`
+  const {
+    where,
+    values
+  } = getWhereQuery(
+    filter,
+    { alias }
+  )
   const _sort = getOrderQuery(sort)
 
-  if (!_sort) {
-    return name
+  if (
+    !_sort &&
+    !where
+  ) {
+    return {
+      subQuery: name,
+      subQueryValues: {}
+    }
   }
 
-  return `(SELECT * FROM ${name} ${_sort})`
+  const subQuery = `(SELECT * FROM ${name} AS ${alias} ${where} ${_sort})`
+
+  return {
+    subQuery,
+    subQueryValues: values
+  }
 }

workers/loc.api/sync/dao/helpers/serialization/deserialize-val.js

@@ -16,7 +16,10 @@ module.exports = (
     '_isMarginFundingPayment',
     '_isAffiliateRebate',
     '_isStakingPayments',
-    '_isSubAccountsTransfer'
+    '_isSubAccountsTransfer',
+    '_isInvoicePayOrder',
+    '_isAirdropOnWallet',
+    '_isExchange'
   ]
 ) => {
   if (

workers/loc.api/sync/data.inserter/api.middleware/api.middleware.handler.after.js

@@ -133,6 +133,14 @@ class ApiMiddlewareHandlerAfter {
             {
               fieldName: '_isSubAccountsTransfer',
               pattern: '^transfer.+sa[(].+[)]'
+            },
+            {
+              fieldName: '_isInvoicePayOrder',
+              pattern: 'InvoicePay Order'
+            },
+            {
+              fieldName: '_isAirdropOnWallet',
+              pattern: 'Airdrop on wallet'
             }
           ]
         ),

workers/loc.api/sync/movements/index.js

@@ -3,6 +3,8 @@
 const { orderBy } = require('lodash')
 const { merge } = require('lib-js-util-base')
 
+const { pushLargeArr } = require('../../helpers/utils')
+
 const { decorateInjectable } = require('../../di/utils')
 
 const depsTypes = (TYPES) => [
@@ -41,7 +43,8 @@ class Movements {
       isExcludePrivate = true,
       isWithdrawals = false,
       isDeposits = false,
-      isMovementsWithoutSATransferLedgers = false
+      isMovementsWithoutSATransferLedgers = false,
+      areExtraPaymentsIncluded = false
     } = params ?? {}
 
     const user = await this.authenticator
@@ -86,11 +89,44 @@ class Movements {
       }
     )
 
+    const ledgersOrder = this._getLedgersOrder(sort)
+    const extraMovementsPromise = this.getExtraMovements({
+      auth: user,
+      start,
+      end,
+      sort: ledgersOrder,
+      isWithdrawals,
+      isDeposits,
+      isExcludePrivate,
+      areExtraPaymentsIncluded
+    })
+
     if (isMovementsWithoutSATransferLedgers) {
-      return movementsPromise
+      const [
+        movements,
+        extraMovements
+      ] = await Promise.all([
+        movementsPromise,
+        extraMovementsPromise
+      ])
+
+      const remapedLedgers = this._remapLedgersToMovements(
+        extraMovements
+      )
+      pushLargeArr(movements, remapedLedgers)
+      const {
+        propNames,
+        orders
+      } = this._getLodashOrder(sort)
+      const orderedRes = orderBy(
+        movements,
+        propNames,
+        orders
+      )
+
+      return orderedRes
     }
 
-    const ledgersOrder = this._getLedgersOrder(sort)
     const ledgersPromise = this.getSubAccountsTransferLedgers({
       auth: user,
       start,
@@ -103,16 +139,19 @@ class Movements {
 
     const [
       movements,
+      extraMovements,
       ledgers
     ] = await Promise.all([
       movementsPromise,
+      extraMovementsPromise,
       ledgersPromise
     ])
 
+    pushLargeArr(extraMovements, ledgers)
     const remapedLedgers = this._remapLedgersToMovements(
-      ledgers
+      extraMovements
     )
-    movements.push(...remapedLedgers)
+    pushLargeArr(movements, remapedLedgers)
 
     const {
       propNames,
@@ -176,6 +215,72 @@ class Movements {
     )
   }
 
+  /*
+   * Considers the following additional movements from ledgers:
+   *   - `InvoicePay Order`
+   *   - `Airdrop on wallet`
+   */
+  getExtraMovements (params = {}) {
+    const {
+      auth = {},
+      start = 0,
+      end = Date.now(),
+      filter: _filter,
+      sort = [['mts', -1], ['id', -1]],
+      projection = this.ledgersModel,
+      exclude = ['user_id'],
+      isExcludePrivate = true,
+      isWithdrawals = false,
+      isDeposits = false,
+      areExtraPaymentsIncluded = false
+    } = params ?? {}
+
+    const withdrawalsFilter = isWithdrawals
+      ? { $lt: { amount: 0 } }
+      : {}
+    const depositsFilter = isDeposits
+      ? { $gt: { amount: 0 } }
+      : {}
+    const filter = merge(
+      {},
+      withdrawalsFilter,
+      depositsFilter,
+      _filter
+    )
+    const extraPaymentsFilter = areExtraPaymentsIncluded
+      ? {
+          $or: {
+            $eq: {
+              _isInvoicePayOrder: 1,
+              _isAirdropOnWallet: 1,
+              _isMarginFundingPayment: 1,
+              _isAffiliateRebate: 1,
+              _isStakingPayments: 1
+            }
+          }
+        }
+      : { $or: { $eq: { _isInvoicePayOrder: 1 } } }
+
+    return this.dao.getElemsInCollBy(
+      this.ALLOWED_COLLS.LEDGERS,
+      {
+        subQuery: {
+          filter: extraPaymentsFilter
+        },
+        filter: {
+          $lte: { mts: end },
+          $gte: { mts: start },
+          user_id: auth._id,
+          ...filter
+        },
+        sort,
+        projection,
+        exclude,
+        isExcludePrivate
+      }
+    )
+  }
+
   _remapLedgersToMovements (ledgers) {
     return ledgers.map((ledger) => {
       const {

workers/loc.api/sync/schema/models.js

@@ -119,6 +119,8 @@ const _models = new Map([
       _isStakingPayments: 'INT',
       _isSubAccountsTransfer: 'INT',
       _isBalanceRecalced: 'INT',
+      _isInvoicePayOrder: 'INT',
+      _isAirdropOnWallet: 'INT',
       subUserId: 'INT',
       user_id: 'INT NOT NULL',
 
@@ -134,6 +136,8 @@ const _models = new Map([
         ['user_id', '_category', 'mts'],
         ['user_id', 'mts'],
         ['currency', 'mts'],
+        ['_isInvoicePayOrder'],
+        ['_isAirdropOnWallet'],
         ['user_id', 'subUserId', 'mts',
           'WHERE subUserId IS NOT NULL'],
         ['subUserId', 'mts', '_id',

workers/loc.api/sync/transaction.tax.report/index.js

@@ -183,14 +183,16 @@ class TransactionTaxReport {
       start,
       end,
       isWithdrawals: true,
-      isExcludePrivate: false
+      isExcludePrivate: false,
+      areExtraPaymentsIncluded: true
     })
     const depositsPromise = this.movements.getMovements({
       auth: user,
       start,
       end,
       isDeposits: true,
-      isExcludePrivate: false
+      isExcludePrivate: false,
+      areExtraPaymentsIncluded: true
     })
 
     const [