Merge pull request #375 from ZIMkaRU/bugfix/enhance-sub-account-sync

Enhance sub account sync
bitfinexcom · May 6, 2024 · 01937e7 · 01937e7
2 parents 33e7e51 + f605371
commit 01937e7
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 5 deletions.
diff --git a/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js b/workers/loc.api/sync/data.inserter/hooks/recalc.sub.account.ledgers.balances.hook.js
@@ -82,7 +82,8 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook {
     ) {
       return {
         balance: _nativeBalance,
-        balanceUsd: _nativeBalanceUsd
+        balanceUsd: _nativeBalanceUsd,
+        _isBalanceRecalced: null
       }
     }
 
@@ -143,7 +144,8 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook {
 
     return {
       balance,
-      balanceUsd
+      balanceUsd,
+      _isBalanceRecalced: 1
     }
   }
 
@@ -316,11 +318,17 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook {
       return
     }
 
+    const authIds = [...auth]
+      .map(([key, auth]) => auth?._id)
+      .filter((id) => Number.isFinite(id))
+    const requiredUserIds = [...new Set(authIds)]
+
     const firstNotRecalcedElem = await this.dao.getElemInCollBy(
       tableName,
       {
         $isNotNull: 'subUserId',
-        $isNull: '_isBalanceRecalced'
+        $isNull: '_isBalanceRecalced',
+        $in: { user_id: requiredUserIds }
       },
       [['mts', 1], ['id', 1]]
     )
@@ -346,6 +354,7 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook {
           filter: {
             $gte: { mts },
             $nin: { _id: skipedIds },
+            $in: { user_id: requiredUserIds },
             $isNotNull: 'subUserId'
           },
           sort: [['mts', 1], ['id', 1]],
@@ -377,7 +386,8 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook {
         const _elem = elem ?? {}
         const {
           balance,
-          balanceUsd
+          balanceUsd,
+          _isBalanceRecalced
         } = await this._getRecalcBalanceAsync(
           auth,
           recordsToGetBalances,
@@ -387,7 +397,7 @@ class RecalcSubAccountLedgersBalancesHook extends DataInserterHook {
         recalcElems.push(Object.assign(_elem, {
           balance,
           balanceUsd,
-          _isBalanceRecalced: 1
+          _isBalanceRecalced
         }))
       }
 

diff --git a/workers/loc.api/sync/data.inserter/index.js b/workers/loc.api/sync/data.inserter/index.js
@@ -552,6 +552,7 @@ class DataInserter extends EventEmitter {
 
     let count = 0
     let serialRequestsCount = 0
+    let serialRequestsCountWithEndLessThanStart = 0
     let prevRes = []
 
     while (true) {
@@ -617,6 +618,15 @@ class DataInserter extends EventEmitter {
         isAllData = true
       }
 
+      // Prevent very rare issue: end <= start
+      if (currIterationArgs.params.start >= currIterationArgs.params.end) {
+        currIterationArgs.params.end = currIterationArgs.params.start + 1
+        serialRequestsCountWithEndLessThanStart += 1
+      }
+      if (serialRequestsCountWithEndLessThanStart > 1) {
+        isAllData = true
+      }
+
       const normalizedApiData = normalizeApiData(res, model)
 
       /*