Feat: Added an option to fast forward the ledger state (#1196)

* Feat: added an option to fast forward the ledger state

* Docs: fixed a typo in the docs

* Refactor: renamed test methods to comply with codacy

* Refactor: refactored the tests and new methods

* Docs: fixed a typo in the docs

* Refactor: refactored the mock classes and the test

* Refactor: renamed setup method

* Refactor: refactored some code

* Refactor: refactor according to requests

* Refactor: removed unused import

* Refactor: refactored some more code

src/main/java/com/iota/iri/service/ledger/LedgerService.java

@@ -14,6 +14,20 @@
  * This class is stateless and does not hold any domain specific models.<br />
  */
 public interface LedgerService {
+    /**
+     * Restores the ledger state after a restart of IRI, which allows us to fast forward to the point where we
+     * stopped before the restart.<br />
+     * <br />
+     * It looks for the last solid milestone that was applied to the ledger in the database and then replays all
+     * milestones leading up to this point by applying them to the latest snapshot. We do not check every single
+     * milestone again but assume that the data in the database is correct. If the database would have any
+     * inconsistencies and the application fails, the latest solid milestone tracker will check and apply the milestones
+     * one by one and repair the corresponding inconsistencies.<br />
+     *
+     * @throws LedgerException if anything unexpected happens while trying to restore the ledger state
+     */
+    void restoreLedgerState() throws LedgerException;
+
     /**
      * Applies the given milestone to the ledger state.<br />
      * <br />

src/main/java/com/iota/iri/service/ledger/impl/LedgerServiceImpl.java

@@ -71,6 +71,18 @@ public LedgerServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider,
         return this;
     }
 
+    @Override
+    public void restoreLedgerState() throws LedgerException {
+        try {
+            Optional<MilestoneViewModel> milestone = milestoneService.findLatestProcessedSolidMilestoneInDatabase();
+            if (milestone.isPresent()) {
+                snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.get().index());
+            }
+        } catch (Exception e) {
+            throw new LedgerException("unexpected error while restoring the ledger state", e);
+        }
+    }
+
     @Override
     public boolean applyMilestoneToLedger(MilestoneViewModel milestone) throws LedgerException {
         if(generateStateDiff(milestone)) {

src/main/java/com/iota/iri/service/milestone/LatestSolidMilestoneTracker.java

@@ -21,10 +21,10 @@ public interface LatestSolidMilestoneTracker {
      *
      * @throws MilestoneException if anything unexpected happens while updating the latest solid milestone
      */
-    void checkForNewLatestSolidMilestones() throws MilestoneException;
+    void trackLatestSolidMilestone() throws MilestoneException;
 
     /**
-     * This method starts the background worker that automatically calls {@link #checkForNewLatestSolidMilestones()}
+     * This method starts the background worker that automatically calls {@link #trackLatestSolidMilestone()}
      * periodically to keep the latest solid milestone up to date.<br />
      */
     void start();

src/main/java/com/iota/iri/service/milestone/MilestoneService.java

@@ -1,15 +1,31 @@
 package com.iota.iri.service.milestone;
 
+import com.iota.iri.controllers.MilestoneViewModel;
 import com.iota.iri.controllers.TransactionViewModel;
 import com.iota.iri.crypto.SpongeFactory;
 import com.iota.iri.model.Hash;
 
+import java.util.Optional;
+
 /**
  * Represents the service that contains all the relevant business logic for interacting with milestones.<br />
  * <br />
  * This class is stateless and does not hold any domain specific models.<br />
  */
 public interface MilestoneService {
+    /**
+     * Finds the latest solid milestone that was previously processed by IRI (before a restart) by performing a search
+     * in the database.<br />
+     * <br />
+     * It determines if the milestones were processed by checking the {@code snapshotIndex} value of their corresponding
+     * transactions.<br />
+     *
+     * @return the latest solid milestone that was previously processed by IRI or an empty value if no previously
+     *         processed solid milestone can be found
+     * @throws MilestoneException if anything unexpected happend while performing the search
+     */
+    Optional<MilestoneViewModel> findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException;
+
     /**
      * Analyzes the given transaction to determine if it is a valid milestone.<br />
      * <br />

src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java

@@ -29,7 +29,7 @@
  */
 public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTracker {
     /**
-     * Holds the interval (in milliseconds) in which the {@link #checkForNewLatestSolidMilestones()} method gets
+     * Holds the interval (in milliseconds) in which the {@link #trackLatestSolidMilestone()} method gets
      * called by the background worker.<br />
      */
     private static final int RESCAN_INTERVAL = 5000;
@@ -76,6 +76,11 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac
     private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService(
             "Latest Solid Milestone Tracker", log.delegate());
 
+    /**
+     * Boolean flag that is used to identify the first iteration of the background worker.<br />
+     */
+    private boolean firstRun = true;
+
     /**
      * Holds the milestone index of the milestone that caused the repair logic to get started.<br />
      */
@@ -138,7 +143,7 @@ public void shutdown() {
      * {@link LatestMilestoneTracker} in sync (if we happen to process a new latest milestone faster).<br />
      */
     @Override
-    public void checkForNewLatestSolidMilestones() throws MilestoneException {
+    public void trackLatestSolidMilestone() throws MilestoneException {
         try {
             int currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex();
             if (currentSolidMilestoneIndex < latestMilestoneTracker.getLatestMilestoneIndex()) {
@@ -147,28 +152,38 @@ public void checkForNewLatestSolidMilestones() throws MilestoneException {
                         (nextMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null &&
                         TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) {
 
-                    syncLatestMilestoneTracker(nextMilestone);
+                    syncLatestMilestoneTracker(nextMilestone.getHash(), nextMilestone.index());
                     applySolidMilestoneToLedger(nextMilestone);
                     logChange(currentSolidMilestoneIndex);
 
                     currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex();
                 }
+            } else {
+                syncLatestMilestoneTracker(snapshotProvider.getLatestSnapshot().getHash(),
+                        currentSolidMilestoneIndex);
             }
         } catch (Exception e) {
-            throw new MilestoneException(e);
+            throw new MilestoneException("unexpected error while checking for new latest solid milestones", e);
         }
     }
 
     /**
      * Contains the logic for the background worker.<br />
      * <br />
-     * It simply calls {@link #checkForNewLatestSolidMilestones()} and wraps with a log handler that prevents the {@link
+     * It simply calls {@link #trackLatestSolidMilestone()} and wraps with a log handler that prevents the {@link
      * MilestoneException} to crash the worker.<br />
      */
     private void latestSolidMilestoneTrackerThread() {
         try {
-            checkForNewLatestSolidMilestones();
-        } catch (MilestoneException e) {
+            if (firstRun) {
+                firstRun = false;
+
+                ledgerService.restoreLedgerState();
+                logChange(snapshotProvider.getInitialSnapshot().getIndex());
+            }
+
+            trackLatestSolidMilestone();
+        } catch (Exception e) {
             log.error("error while updating the solid milestone", e);
         }
     }
@@ -178,7 +193,7 @@ private void latestSolidMilestoneTrackerThread() {
      * <br />
      * If the application of the milestone fails, we start a repair routine which will revert the milestones preceding
      * our current milestone and consequently try to reapply them in the next iteration of the {@link
-     * #checkForNewLatestSolidMilestones()} method (until the problem is solved).<br />
+     * #trackLatestSolidMilestone()} method (until the problem is solved).<br />
      *
      * @param milestone the milestone that shall be applied to the ledger state
      * @throws Exception if anything unexpected goes wrong while applying the milestone to the ledger
@@ -237,11 +252,12 @@ private void stopRepair() {
      * Note: This method ensures that the latest milestone index is always bigger or equals the latest solid milestone
      *       index.
      *
-     * @param processedMilestone the milestone that currently gets processed
+     * @param milestoneHash transaction hash of the milestone
+     * @param milestoneIndex milestone index
      */
-    private void syncLatestMilestoneTracker(MilestoneViewModel processedMilestone) {
-        if(processedMilestone.index() > latestMilestoneTracker.getLatestMilestoneIndex()) {
-            latestMilestoneTracker.setLatestMilestone(processedMilestone.getHash(), processedMilestone.index());
+    private void syncLatestMilestoneTracker(Hash milestoneHash, int milestoneIndex) {
+        if(milestoneIndex > latestMilestoneTracker.getLatestMilestoneIndex()) {
+            latestMilestoneTracker.setLatestMilestone(milestoneHash, milestoneIndex);
         }
     }
 

src/main/java/com/iota/iri/service/milestone/impl/MilestoneServiceImpl.java

@@ -101,6 +101,42 @@ public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvide
 
     //region {PUBLIC METHODS] //////////////////////////////////////////////////////////////////////////////////////////
 
+    /**
+     * {@inheritDoc}
+     * <br />
+     * We first check the trivial case where the node was fully synced. If no processed solid milestone could be found
+     * within the last two milestones of the node, we perform a binary search from present to past, which reduces the
+     * amount of database requests to a minimum (even with a huge amount of milestones in the database).<br />
+     */
+    @Override
+    public Optional<MilestoneViewModel> findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException {
+        try {
+            // if we have no milestone in our database -> abort
+            MilestoneViewModel latestMilestone = MilestoneViewModel.latest(tangle);
+            if (latestMilestone == null) {
+                return Optional.empty();
+            }
+
+             // trivial case #1: the node was fully synced
+            if (wasMilestoneAppliedToLedger(latestMilestone)) {
+                return Optional.of(latestMilestone);
+            }
+
+             // trivial case #2: the node was fully synced but the last milestone was not processed, yet
+            MilestoneViewModel latestMilestonePredecessor = MilestoneViewModel.findClosestPrevMilestone(tangle,
+                    latestMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex());
+            if (latestMilestonePredecessor != null && wasMilestoneAppliedToLedger(latestMilestonePredecessor)) {
+                return Optional.of(latestMilestonePredecessor);
+            }
+
+             // non-trivial case: do a binary search in the database
+            return binarySearchLatestProcessedSolidMilestoneInDatabase(latestMilestone);
+        } catch (Exception e) {
+            throw new MilestoneException(
+                    "unexpected error while trying to find the latest processed solid milestone in the database", e);
+        }
+    }
+
     @Override
     public void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int index) throws MilestoneException {
         if (index <= 0) {
@@ -224,6 +260,93 @@ public boolean isTransactionConfirmed(TransactionViewModel transaction) {
 
     //region [PRIVATE UTILITY METHODS] /////////////////////////////////////////////////////////////////////////////////
 
+    /**
+     * Performs a binary search for the latest solid milestone which was already processed by the node and applied to
+     * the ledger state at some point in the past (i.e. before IRI got restarted).<br />
+     * <br />
+     * It searches from present to past using a binary search algorithm which quickly narrows down the amount of
+     * candidates even for big databases.<br />
+     *
+     * @param latestMilestone the latest milestone in the database (used to define the search range)
+     * @return the latest solid milestone that was previously processed by IRI or an empty value if no previously
+     *         processed solid milestone can be found
+     * @throws Exception if anything unexpected happens while performing the search
+     */
+    private Optional<MilestoneViewModel> binarySearchLatestProcessedSolidMilestoneInDatabase(
+            MilestoneViewModel latestMilestone) throws Exception {
+
+        Optional<MilestoneViewModel> lastAppliedCandidate = Optional.empty();
+
+        int rangeEnd = latestMilestone.index();
+        int rangeStart = snapshotProvider.getInitialSnapshot().getIndex() + 1;
+        while (rangeEnd - rangeStart >= 0) {
+            // if no candidate found in range -> return last candidate
+            MilestoneViewModel currentCandidate = getMilestoneInMiddleOfRange(rangeStart, rangeEnd);
+            if (currentCandidate == null) {
+                return lastAppliedCandidate;
+            }
+
+             // if the milestone was applied -> continue to search for "later" ones that might have also been applied
+            if (wasMilestoneAppliedToLedger(currentCandidate)) {
+                rangeStart = currentCandidate.index() + 1;
+
+                lastAppliedCandidate = Optional.of(currentCandidate);
+            }
+
+             // if the milestone was not applied -> continue to search for "earlier" ones
+            else {
+                rangeEnd = currentCandidate.index() - 1;
+            }
+        }
+
+        return lastAppliedCandidate;
+    }
+
+    /**
+     * Determines the milestone in the middle of the range defined by {@code rangeStart} and {@code rangeEnd}.<br />
+     * <br />
+     * It is used by the binary search algorithm of {@link #findLatestProcessedSolidMilestoneInDatabase()}. It first
+     * calculates the index that represents the middle of the range and then tries to find the milestone that is closest
+     * to this index.<br/>
+     * <br />
+     * Note: We start looking for younger milestones first, because most of the times the latest processed solid
+     *       milestone is close to the end.<br />
+     *
+     * @param rangeStart the milestone index representing the start of our search range
+     * @param rangeEnd the milestone index representing the end of our search range
+     * @return the milestone that is closest to the middle of the given range or {@code null} if no milestone can be
+     *         found
+     * @throws Exception if anything unexpected happens while trying to get the milestone
+     */
+    private MilestoneViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) throws Exception {
+        int range = rangeEnd - rangeStart;
+        int middleOfRange = rangeEnd - range / 2;
+
+        MilestoneViewModel milestone = MilestoneViewModel.findClosestNextMilestone(tangle, middleOfRange - 1, rangeEnd);
+        if (milestone == null) {
+            milestone = MilestoneViewModel.findClosestPrevMilestone(tangle, middleOfRange, rangeStart);
+        }
+
+        return milestone;
+    }
+
+    /**
+     * Checks if the milestone was applied to the ledger at some point in the past (before a restart of IRI).<br />
+     * <br />
+     * Since the {@code snapshotIndex} value is used as a flag to determine if the milestone was already applied to the
+     * ledger, we can use it to determine if it was processed by IRI in the past. If this value is set we should also
+     * have a corresponding {@link StateDiff} entry in the database.<br />
+     *
+     * @param milestone the milestone that shall be checked
+     * @return {@code true} if the milestone has been processed by IRI before and {@code false} otherwise
+     * @throws Exception if anything unexpected happens while checking the milestone
+     */
+    private boolean wasMilestoneAppliedToLedger(MilestoneViewModel milestone) throws Exception {
+        TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash());
+        return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT &&
+                milestoneTransaction.snapshotIndex() != 0;
+    }
+
     /**
      * This method implements the logic described by {@link #updateMilestoneIndexOfMilestoneTransactions(Hash, int)} but
      * accepts some additional parameters that allow it to be reused by different parts of this service.<br />