Feat: Added an option to fast forward the ledger state

src/main/java/com/iota/iri/service/ledger/LedgerService.java

@@ -14,6 +14,20 @@
  * This class is stateless and does not hold any domain specific models.<br />
  */
 public interface LedgerService {
+    /**
+     * Restores the ledger state after a restart of IRI, which allows us to fast forward to the point where we
+     * stopped before the restart.<br />
+     * <br />
+     * It looks for the last solid milestone that was applied to the ledger in the database and then replays all
+     * milestones leading up to this point by applying them to the latest snapshot. We do not check every single
+     * milestone again but assume that the data in the database is correct. If the database would have any
+     * inconsistencies and the application fails, the latest solid milestone tracker will check and apply the milestones
+     * one by one and repair the corresponding inconsistencies.<br />
+     *
+     * @throws LedgerException if anything unexpected happens while trying to restore the ledger state
+     */
+    void restoreLedgerState() throws LedgerException;
+
     /**
      * Applies the given milestone to the ledger state.<br />
      * <br />

src/main/java/com/iota/iri/service/ledger/impl/LedgerServiceImpl.java

@@ -71,6 +71,18 @@ public LedgerServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider,
         return this;
     }
 
+    @Override
+    public void restoreLedgerState() throws LedgerException {
+        try {
+            Optional<MilestoneViewModel> milestone = milestoneService.findLatestProcessedSolidMilestoneInDatabase();
+            if (milestone.isPresent()) {
+                snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), milestone.get().index());
+            }
+        } catch (Exception e) {
+            throw new LedgerException("unexpected error while restoring the ledger state", e);
+        }
+    }
+
     @Override
     public boolean applyMilestoneToLedger(MilestoneViewModel milestone) throws LedgerException {
         if(generateStateDiff(milestone)) {

src/main/java/com/iota/iri/service/milestone/MilestoneService.java

@@ -1,15 +1,31 @@
 package com.iota.iri.service.milestone;
 
+import com.iota.iri.controllers.MilestoneViewModel;
 import com.iota.iri.controllers.TransactionViewModel;
 import com.iota.iri.crypto.SpongeFactory;
 import com.iota.iri.model.Hash;
 
+import java.util.Optional;
+
 /**
  * Represents the service that contains all the relevant business logic for interacting with milestones.<br />
  * <br />
  * This class is stateless and does not hold any domain specific models.<br />
  */
 public interface MilestoneService {
+    /**
+     * Finds the latest solid milestone that was previously processed by IRI (before a restart) by performing a search
+     * in the database.<br />
+     * <br />
+     * It determines if the milestones were processed by checking the {@code snapshotIndex} value of their corresponding
+     * transactions.<br />
+     *
+     * @return the latest solid milestone that was previously processed by IRI or an empty value if no previously
+     *         processed solid milestone can be found
+     * @throws MilestoneException if anything unexpected happend while performing the search
+     */
+    Optional<MilestoneViewModel> findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException;
+
     /**
      * Analyzes the given transaction to determine if it is a valid milestone.<br />
      * <br />

src/main/java/com/iota/iri/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java

@@ -76,6 +76,11 @@ public class LatestSolidMilestoneTrackerImpl implements LatestSolidMilestoneTrac
     private final SilentScheduledExecutorService executorService = new DedicatedScheduledExecutorService(
             "Latest Solid Milestone Tracker", log.delegate());
 
+    /**
+     * Boolean flag that is used to identify the first iteration of the background worker.<br />
+     */
+    private boolean firstRun = true;
+
     /**
      * Holds the milestone index of the milestone that caused the repair logic to get started.<br />
      */
@@ -140,22 +145,32 @@ public void shutdown() {
     @Override
     public void checkForNewLatestSolidMilestones() throws MilestoneException {
         try {
+            if (firstRun) {
+                firstRun = false;
+
+                ledgerService.restoreLedgerState();
+                logChange(snapshotProvider.getInitialSnapshot().getIndex());
+            }
+
             int currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex();
             if (currentSolidMilestoneIndex < latestMilestoneTracker.getLatestMilestoneIndex()) {
                 MilestoneViewModel nextMilestone;
                 while (!Thread.currentThread().isInterrupted() &&
                         (nextMilestone = MilestoneViewModel.get(tangle, currentSolidMilestoneIndex + 1)) != null &&
                         TransactionViewModel.fromHash(tangle, nextMilestone.getHash()).isSolid()) {
 
-                    syncLatestMilestoneTracker(nextMilestone);
+                    syncLatestMilestoneTracker(nextMilestone.getHash(), nextMilestone.index());
                     applySolidMilestoneToLedger(nextMilestone);
                     logChange(currentSolidMilestoneIndex);
 
                     currentSolidMilestoneIndex = snapshotProvider.getLatestSnapshot().getIndex();
                 }
+            } else {
+                syncLatestMilestoneTracker(snapshotProvider.getLatestSnapshot().getHash(),
+                        snapshotProvider.getLatestSnapshot().getIndex());
             }
         } catch (Exception e) {
-            throw new MilestoneException(e);
+            throw new MilestoneException("unexpected error while checking for new latest solid milestones", e);
         }
     }
 
@@ -237,11 +252,12 @@ private void stopRepair() {
      * Note: This method ensures that the latest milestone index is always bigger or equals the latest solid milestone
      *       index.
      *
-     * @param processedMilestone the milestone that currently gets processed
+     * @param milestoneHash transaction hash of the milestone
+     * @param milestoneIndex milestone index
      */
-    private void syncLatestMilestoneTracker(MilestoneViewModel processedMilestone) {
-        if(processedMilestone.index() > latestMilestoneTracker.getLatestMilestoneIndex()) {
-            latestMilestoneTracker.setLatestMilestone(processedMilestone.getHash(), processedMilestone.index());
+    private void syncLatestMilestoneTracker(Hash milestoneHash, int milestoneIndex) {
+        if(milestoneIndex > latestMilestoneTracker.getLatestMilestoneIndex()) {
+            latestMilestoneTracker.setLatestMilestone(milestoneHash, milestoneIndex);
         }
     }
 

src/main/java/com/iota/iri/service/milestone/impl/MilestoneServiceImpl.java

@@ -101,6 +101,42 @@ public MilestoneServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvide
 
     //region {PUBLIC METHODS] //////////////////////////////////////////////////////////////////////////////////////////
 
+    /**
+     * {@inheritDoc}
+     * <br />
+     * We first check the trivial case where the node was fully synced. If no processed solid milestone could be found
+     * within the last two milestones of the node, we perform a binary search from present to past, which reduces the
+     * amount of database requests to a minimum (even with a huge amount of milestones in the database).<br />
+     */
+    @Override
+    public Optional<MilestoneViewModel> findLatestProcessedSolidMilestoneInDatabase() throws MilestoneException {
+        try {
+            // if we have no milestone in our database -> abort
+            MilestoneViewModel latestMilestone = MilestoneViewModel.latest(tangle);
+            if (latestMilestone == null) {
+                return Optional.empty();
+            }
+
+             // trivial case #1: the node was fully synced
+            if (wasMilestoneAppliedToLedger(latestMilestone)) {
+                return Optional.of(latestMilestone);
+            }
+
+             // trivial case #2: the node was fully synced but the last milestone was not processed, yet
+            MilestoneViewModel latestMilestonePredecessor = MilestoneViewModel.findClosestPrevMilestone(tangle,
+                    latestMilestone.index(), snapshotProvider.getInitialSnapshot().getIndex());
+            if (latestMilestonePredecessor != null && wasMilestoneAppliedToLedger(latestMilestonePredecessor)) {
+                return Optional.of(latestMilestonePredecessor);
+            }
+
+             // non-trivial case: do a binary search in the database
+            return binarySearchLatestProcessedSolidMilestoneInDatabase(latestMilestone);
+        } catch (Exception e) {
+            throw new MilestoneException(
+                    "unexpected error while trying to find the latest processed solid milestone in the database", e);
+        }
+    }
+
     @Override
     public void updateMilestoneIndexOfMilestoneTransactions(Hash milestoneHash, int index) throws MilestoneException {
         if (index <= 0) {
@@ -224,6 +260,93 @@ public boolean isTransactionConfirmed(TransactionViewModel transaction) {
 
     //region [PRIVATE UTILITY METHODS] /////////////////////////////////////////////////////////////////////////////////
 
+    /**
+     * Performs a binary search for the latest solid milestone which was already processed by the node and applied to
+     * the ledger state at some point in the past (i.e. before IRI got restarted).<br />
+     * <br />
+     * It searches from present to past using a binary search algorithm which quickly narrows down the amount of
+     * candidates even for big databases.<br />
+     *
+     * @param latestMilestone the latest milestone in the database (used to define the search range)
+     * @return the latest solid milestone that was previously processed by IRI or an empty value if no previously
+     *         processed solid milestone can be found
+     * @throws Exception if anything unexpected happens while performing the search
+     */
+    private Optional<MilestoneViewModel> binarySearchLatestProcessedSolidMilestoneInDatabase(
+            MilestoneViewModel latestMilestone) throws Exception {
+
+        Optional<MilestoneViewModel> lastAppliedCandidate = Optional.empty();
+
+        int rangeEnd = latestMilestone.index();
+        int rangeStart = snapshotProvider.getInitialSnapshot().getIndex() + 1;
+        while (rangeEnd - rangeStart >= 0) {
+            // if no candidate found in range -> return last candidate
+            MilestoneViewModel currentCandidate = getMilestoneInMiddleOfRange(rangeStart, rangeEnd);
+            if (currentCandidate == null) {
+                return lastAppliedCandidate;
+            }
+
+             // if the milestone was applied -> continue to search for "later" ones that might have also been applied
+            if (wasMilestoneAppliedToLedger(currentCandidate)) {
+                rangeStart = currentCandidate.index() + 1;
+
+                lastAppliedCandidate = Optional.of(currentCandidate);
+            }
+
+             // if the milestone was not applied -> continue to search for "earlier" ones
+            else {
+                rangeEnd = currentCandidate.index() - 1;
+            }
+        }
+
+        return lastAppliedCandidate;
+    }
+
+    /**
+     * Determines the milestone in the middle of the range defined by {@code rangeStart} and {@code rangeEnd}.<br />
+     * <br />
+     * It is used by the binary search algorithm of {@link #findLatestProcessedSolidMilestoneInDatabase()}. It first
+     * calculates the index that represents the middle of the range and then tries to find the milestone that is closest
+     * to this index.<br/>
+     * <br />
+     * Note: We start looking for younger milestones first, because most of the times the latest processed solid
+     *       milestone is close to the end.<br />
+     *
+     * @param rangeStart the milestone index representing the start of our search range
+     * @param rangeEnd the milestone index representing the end of our search range
+     * @return the milestone that is closest to the middle of the given range or {@code null} if no milestone can be
+     *         found
+     * @throws Exception if anything unexpected happens while trying to get the milestone
+     */
+    private MilestoneViewModel getMilestoneInMiddleOfRange(int rangeStart, int rangeEnd) throws Exception {
+        int range = rangeEnd - rangeStart;
+        int middleOfRange = rangeEnd - range / 2;
+
+        MilestoneViewModel milestone = MilestoneViewModel.findClosestNextMilestone(tangle, middleOfRange - 1, rangeEnd);
+        if (milestone == null) {
+            milestone = MilestoneViewModel.findClosestPrevMilestone(tangle, middleOfRange, rangeStart);
+        }
+
+        return milestone;
+    }
+
+    /**
+     * Checks if the milestone was applied to the ledger at some point in the past (before a restart of IRI).<br />
+     * <br />
+     * Since the {@code snapshotIndex} value is used as a flag to determine if the milestone was already applied to the
+     * ledger, we can use it to determine if it was processed by IRI in the past. If this value is set we should also
+     * have a corresponding {@link StateDiff} entry in the database.<br />
+     *
+     * @param milestone the milestone that shall be checked
+     * @return {@code true} if the milestone has been processed by IRI before and {@code false} otherwise
+     * @throws Exception if anything unexpected happens while checking the milestone
+     */
+    private boolean wasMilestoneAppliedToLedger(MilestoneViewModel milestone) throws Exception {
+        TransactionViewModel milestoneTransaction = TransactionViewModel.fromHash(tangle, milestone.getHash());
+        return milestoneTransaction.getType() != TransactionViewModel.PREFILLED_SLOT &&
+                milestoneTransaction.snapshotIndex() != 0;
+    }
+
     /**
      * This method implements the logic described by {@link #updateMilestoneIndexOfMilestoneTransactions(Hash, int)} but
      * accepts some additional parameters that allow it to be reused by different parts of this service.<br />

src/test/java/com/iota/iri/TangleMockUtils.java

@@ -0,0 +1,96 @@
+package com.iota.iri;
+
+import com.iota.iri.controllers.TransactionViewModel;
+import com.iota.iri.model.Hash;
+import com.iota.iri.model.IntegerIndex;
+import com.iota.iri.model.StateDiff;
+import com.iota.iri.model.persistables.Milestone;
+import com.iota.iri.model.persistables.Transaction;
+import com.iota.iri.storage.Tangle;
+import com.iota.iri.utils.Pair;
+import org.mockito.Mockito;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Contains utilities that help to mock the retrieval of database entries from the tangle.<br />
+ * <br />
+ * Mocking the tangle allows us to write unit tests that perform much faster than spinning up a new database for every
+ * test.<br />
+ */
+public class TangleMockUtils {
+    //region [mockMilestone] ///////////////////////////////////////////////////////////////////////////////////////////
+
+    /**
+     * Registers a {@link Milestone} in the mocked tangle that can consequently be accessed by the tested classes.<br />
+     * <br />
+     * It first creates the {@link Milestone} with the given details and then mocks the retrieval methods of the tangle
+     * to return this object. In addition to mocking the specific retrieval method for the given hash, we also mock the
+     * retrieval method for the "latest" entity so the mocked tangle returns the elements in the order that they were
+     * mocked / created (which allows the mocked tangle to behave just like a normal one).<br />
+     * <br />
+     * Note: We return the mocked object which allows us to set additional fields or modify it after "injecting" it into
+     *       the mocked tangle.<br />
+     *
+     * @param tangle mocked tangle object that shall retrieve a milestone object when being queried for it
+     * @param hash transaction hash of the milestone
+     * @param index milestone index of the milestone
+     * @return the Milestone object that be returned by the mocked tangle upon request
+     */
+    public static Milestone mockMilestone(Tangle tangle, Hash hash, int index) {
+        Milestone milestone = new Milestone();
+        milestone.hash = hash;
+        milestone.index = new IntegerIndex(index);
+
+        try {
+            Mockito.when(tangle.load(Milestone.class, new IntegerIndex(index))).thenReturn(milestone);
+            Mockito.when(tangle.getLatest(Milestone.class, IntegerIndex.class)).thenReturn(new Pair<>(milestone.index,
+                    milestone));
+        } catch (Exception e) {
+            // the exception can not be raised since we mock
+        }
+
+        return milestone;
+    }
+
+    //endregion ////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+    //region [mockTransaction] /////////////////////////////////////////////////////////////////////////////////////////
+
+    public static Transaction mockTransaction(Tangle tangle, Hash hash) {
+        Transaction transaction = new Transaction();
+        transaction.bytes = new byte[0];
+        transaction.type = TransactionViewModel.FILLED_SLOT;
+        transaction.parsed = true;
+
+        try {
+            Mockito.when(tangle.load(Transaction.class, hash)).thenReturn(transaction);
+            Mockito.when(tangle.getLatest(Transaction.class, Hash.class)).thenReturn(new Pair<>(hash, transaction));
+        } catch (Exception e) {
+            // the exception can not be raised since we mock
+        }
+
+        return transaction;
+    }
+
+    //endregion ////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+    //region [mockStateDiff] ///////////////////////////////////////////////////////////////////////////////////////////
+
+    public static StateDiff mockStateDiff(Tangle tangle, Hash hash, Map<Hash, Long> balanceDiff) {
+        StateDiff stateDiff = new StateDiff();
+        stateDiff.state = balanceDiff;
+
+        try {
+            Mockito.when(tangle.load(StateDiff.class, hash)).thenReturn(stateDiff);
+            Mockito.when(tangle.getLatest(StateDiff.class, Hash.class)).thenReturn(new Pair<>(hash, stateDiff));
+        } catch (Exception e) {
+            // the exception can not be raised since we mock
+        }
+
+        return stateDiff;
+    }
+
+    //endregion ////////////////////////////////////////////////////////////////////////////////////////////////////////
+}