Merge pull request #487 from input-output-hk/erikd/corrupt-ledger-sta…

…te-file Handle case where latest ledger state file cannot be parsed
IntersectMBO · Jan 18, 2021 · 2409a65 · 2409a65
2 parents 316f939 + 0db9898
commit 2409a65
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 26 deletions.
diff --git a/cardano-db-sync/src/Cardano/DbSync.hs b/cardano-db-sync/src/Cardano/DbSync.hs
@@ -144,10 +144,10 @@ runDbSyncNode plugin enp =
       liftIO $ do
         -- Must run plugin startup after the genesis distribution has been inserted/validate.
         runDbStartup trce plugin
-
         case genCfg of
           GenesisCardano _ bCfg _sCfg -> do
             ledgerVar <- initLedgerStateVar genCfg
+            loadLatestLedgerState (envLedgerStateDir genesisEnv) ledgerVar
             runDbSyncNodeNodeClient genesisEnv ledgerVar
                 iomgr trce plugin (cardanoCodecConfig bCfg) (enpSocketPath enp)
   where
@@ -176,17 +176,19 @@ runDbSyncNodeNodeClient env ledgerVar iomgr trce plugin codecConfig (SocketPath
     clientSubscriptionParams
     (dbSyncProtocols trce env plugin queryVar ledgerVar)
   where
-    clientSubscriptionParams = ClientSubscriptionParams {
-        cspAddress = Snocket.localAddressFromPath socketPath,
-        cspConnectionAttemptDelay = Nothing,
-        cspErrorPolicies = networkErrorPolicies <> consensusErrorPolicy (Proxy @CardanoBlock)
+    clientSubscriptionParams =
+      ClientSubscriptionParams
+        { cspAddress = Snocket.localAddressFromPath socketPath
+        , cspConnectionAttemptDelay = Nothing
+        , cspErrorPolicies = networkErrorPolicies <> consensusErrorPolicy (Proxy @CardanoBlock)
         }
 
-    networkSubscriptionTracers = NetworkSubscriptionTracers {
-        nsMuxTracer = muxTracer,
-        nsHandshakeTracer = handshakeTracer,
-        nsErrorPolicyTracer = errorPolicyTracer,
-        nsSubscriptionTracer = subscriptionTracer
+    networkSubscriptionTracers =
+      NetworkSubscriptionTracers
+        { nsMuxTracer = muxTracer
+        , nsHandshakeTracer = handshakeTracer
+        , nsErrorPolicyTracer = errorPolicyTracer
+        , nsSubscriptionTracer = subscriptionTracer
         }
 
     errorPolicyTracer :: Tracer IO (WithAddr LocalAddress ErrorPolicyTrace)
@@ -214,11 +216,11 @@ dbSyncProtocols
     -> ConnectionId LocalAddress
     -> NodeToClientProtocols 'InitiatorMode BSL.ByteString IO () Void
 dbSyncProtocols trce env plugin queryVar ledgerVar _version codecs _connectionId =
-    NodeToClientProtocols {
-          localChainSyncProtocol = localChainSyncPtcl
-        , localTxSubmissionProtocol = dummylocalTxSubmit
-        , localStateQueryProtocol = localStateQuery
-        }
+    NodeToClientProtocols
+      { localChainSyncProtocol = localChainSyncPtcl
+      , localTxSubmissionProtocol = dummylocalTxSubmit
+      , localStateQueryProtocol = localStateQuery
+      }
   where
     localChainSyncTracer :: Tracer IO (TraceSendRecv (ChainSync CardanoBlock(Point CardanoBlock) (Tip CardanoBlock)))
     localChainSyncTracer = toLogObject $ appendName "ChainSync" trce
@@ -284,13 +286,13 @@ logDbState trce = do
 
 getLatestPoints :: LedgerStateDir -> IO [Point CardanoBlock]
 getLatestPoints ledgerStateDir = do
-    xs <- listLedgerStateSlotNos ledgerStateDir
+    xs <- lsfSlotNo <<$>> listLedgerStateFilesOrdered ledgerStateDir
     ys <- catMaybes <$> DB.runDbNoLogging (mapM DB.querySlotHash xs)
     pure $ mapMaybe convert ys
   where
     convert :: (SlotNo, ByteString) -> Maybe (Point CardanoBlock)
     convert (slot, hashBlob) =
-      fmap (Point . Point.block slot) (convertHashBlob hashBlob)
+      Point . Point.block slot <$> convertHashBlob hashBlob
 
     convertHashBlob :: ByteString -> Maybe (HeaderHash CardanoBlock)
     convertHashBlob = Just . fromRawHash (Proxy @CardanoBlock)

diff --git a/cardano-db-sync/src/Cardano/DbSync/Database.hs b/cardano-db-sync/src/Cardano/DbSync/Database.hs
@@ -95,7 +95,7 @@ runActions trce env plugin ledgerState actions = do
             pure Done
         ([], DbRollBackToPoint sn:ys) -> do
             runRollbacks trce plugin sn
-            liftIO $ loadLedgerState (envLedgerStateDir env) ledgerState sn
+            liftIO $ loadLedgerStateAtSlot (envLedgerStateDir env) ledgerState sn
             dbAction Continue ys
         (ys, zs) -> do
           insertBlockList trce env ledgerState plugin ys

diff --git a/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs b/cardano-db-sync/src/Cardano/DbSync/LedgerState.hs
@@ -12,8 +12,9 @@ module Cardano.DbSync.LedgerState
   , LedgerStateFile (..)
   , applyBlock
   , initLedgerStateVar
-  , listLedgerStateSlotNos
-  , loadLedgerState
+  , ledgerStateTipSlot
+  , loadLatestLedgerState
+  , loadLedgerStateAtSlot
   , readLedgerState
   , saveLedgerState
   , listLedgerStateFilesOrdered
@@ -143,6 +144,11 @@ applyBlock env (LedgerStateVar stateVar) blk =
         Left err -> panic err
         Right result -> result
 
+ledgerStateTipSlot :: LedgerStateVar -> IO SlotNo
+ledgerStateTipSlot (LedgerStateVar stateVar) = do
+  lstate <- readTVarIO stateVar
+  pure $ fromWithOrigin (SlotNo 0) . ledgerTipSlot $ ledgerState (clsState lstate)
+
 saveLedgerState :: LedgerStateDir -> LedgerStateVar -> LedgerStateSnapshot -> SyncState -> IO ()
 saveLedgerState lsd@(LedgerStateDir stateDir) (LedgerStateVar stateVar) snapshot synced = do
   atomically $ writeTVar stateVar ledger
@@ -181,8 +187,17 @@ saveLedgerState lsd@(LedgerStateDir stateDir) (LedgerStateVar stateVar) snapshot
     codecConfig :: CodecConfig (CardanoBlock StandardCrypto)
     codecConfig = configCodec (clsConfig ledger)
 
-loadLedgerState :: LedgerStateDir -> LedgerStateVar -> SlotNo -> IO ()
-loadLedgerState stateDir (LedgerStateVar stateVar) slotNo = do
+loadLatestLedgerState :: LedgerStateDir -> LedgerStateVar -> IO ()
+loadLatestLedgerState stateDir ledgerVar = do
+  files <- listLedgerStateFilesOrdered stateDir
+  ledger <- readLedgerState ledgerVar
+  mcs <- firstJustM (loadFile ledger) files
+  case mcs of
+    Just cs -> atomically $ writeTVar (unLedgerStateVar ledgerVar) cs
+    Nothing -> pure ()
+
+loadLedgerStateAtSlot :: LedgerStateDir -> LedgerStateVar -> SlotNo -> IO ()
+loadLedgerStateAtSlot stateDir (LedgerStateVar stateVar) slotNo = do
   -- Read current state to get the LedgerConfig and CodecConfig.
   lstate <- readLedgerState (LedgerStateVar stateVar)
   -- Load the state
@@ -260,7 +275,9 @@ loadFile ledger lsf = do
         Left (_ :: IOException) -> pure Nothing
         Right bs ->
           case decode bs of
-            Left err -> panic (textShow err)
+            Left _err -> do
+              safeRemoveFile fp
+              pure Nothing
             Right ls -> pure $ Just ls
 
 
@@ -295,9 +312,6 @@ listLedgerStateFilesOrdered (LedgerStateDir stateDir) = do
     revSlotNoOrder :: LedgerStateFile -> LedgerStateFile -> Ordering
     revSlotNoOrder a b = compare (lsfSlotNo b) (lsfSlotNo a)
 
-listLedgerStateSlotNos :: LedgerStateDir -> IO [SlotNo]
-listLedgerStateSlotNos = fmap3 lsfSlotNo listLedgerStateFilesOrdered
-
 readLedgerState :: LedgerStateVar -> IO CardanoLedgerState
 readLedgerState (LedgerStateVar stateVar) = readTVarIO stateVar