feat: Retroactive prune very old blocks from store

store/rootmulti/store.go

@@ -46,14 +46,18 @@ type Store struct {
 	stores         map[types.StoreKey]types.CommitKVStore
 	keysByName     map[string]types.StoreKey
 	lazyLoading    bool
-	pruneHeights   []int64
+	pruneHeights   []int64 // CVCH: A running list of heights to prune at the next pruningOpts.Interval
 	initialVersion int64
 	removalMap     map[types.StoreKey]bool
 
 	traceWriter       io.Writer
 	traceContext      types.TraceContext
 	traceContextMutex sync.Mutex
 
+	histPruneHeight   int64 // CVCH: -1 Unstarted; 0 Finished; +# Current Height we are pruning down from
+	histPruneAmount   int64 // CVCH: Positive Integer of blocks to prune @ Each pruning-interval
+	histPruneFinished bool  // CVCH: true at start, then falsewhen histPrune is complete
+
 	interBlockCache types.MultiStorePersistentCache
 
 	listeners map[types.StoreKey][]types.WriteListener
@@ -79,6 +83,10 @@ func NewStore(db dbm.DB) *Store {
 		pruneHeights:  make([]int64, 0),
 		listeners:     make(map[types.StoreKey][]types.WriteListener),
 		removalMap:    make(map[types.StoreKey]bool),
+
+		histPruneHeight:   -1,    // CVCH
+		histPruneAmount:   10,    // CVCH
+		histPruneFinished: false, // CVCH
 	}
 }
 
@@ -384,6 +392,68 @@ func (rs *Store) LastCommitID() types.CommitID {
 	return rs.lastCommitInfo.CommitID()
 }
 
+// CVCH: Chill Validation - After Pruning 'strategy' changes, previous heights may be left around.
+// Example 1: KeepRecent = 100 is changed to KeepRecent = 50, leaving the oldest 50 blocks unpruned after restart.
+// Example 2: KeepEvery = 100 is changed to KeepEvery = 35, leaving the old "Every" heights in the DB.
+func (rs *Store) addPruneHistorical(previousHeight int64) {
+	debuginfo := false // show debug info
+
+	// Clean Historical is already finished
+	if rs.histPruneFinished {
+		return
+	}
+
+	// Avoid historical pruning too early in the blockchain lifecycle
+	// Only consider cleaning historical heights if current height > KeepRecent)
+	if previousHeight < int64(rs.pruningOpts.KeepRecent) {
+		if debuginfo {
+			fmt.Printf("**CVCH Too Early: %d !> %d\n", previousHeight, rs.pruningOpts.KeepRecent)
+			fmt.Printf("**CVCH Early Finished!!\n")
+		}
+		rs.histPruneFinished = true // Since we have started below the KeepRecent height, histPrune is unnecessary
+		return
+	}
+
+	// Debug
+	if debuginfo {
+		fmt.Printf("**CVCH Run : %d > %d\n", previousHeight, rs.pruningOpts.KeepRecent)
+		fmt.Printf("**CVCH cHH : %d\n", rs.histPruneHeight)
+	}
+
+	//       Start from current height, then move towards 0 since more recent blocks are more likely to be present)
+	//       Remove all previous blocks older than KeepRecent
+	//       NOTE: SNAPSHOTS MUST be within KeepRecent (Must KeepRecent > snapshot-keep-recent * snapshot-interval (a multiple of pruning-keep-every in older cosmos-sdk versions, or within keep-recent for newer cosmos-sdk versions)
+
+	// Clean Historical hasn't been set up yet
+	if rs.histPruneHeight == -1 {
+		rs.histPruneHeight = previousHeight - int64(rs.pruningOpts.KeepRecent) // We are not concerned about duplicate height removals
+		if debuginfo {
+			fmt.Printf("**CVCH Init histPruneHeight = %d\n", rs.histPruneHeight)
+		}
+	}
+
+	if debuginfo {
+		fmt.Printf("**PH CVCH Starting: histPruneHeight %d\n", rs.histPruneHeight)
+	}
+	// Determine starting and ending heights to prune
+	var histCtr int64
+	for histCtr = 0; histCtr < rs.histPruneAmount; histCtr++ {
+		if debuginfo {
+			fmt.Printf("**PH CVCH Add %d\n", rs.histPruneHeight)
+		}
+		rs.pruneHeights = append(rs.pruneHeights, rs.histPruneHeight)
+
+		rs.histPruneHeight-- // Reduce histPruneHeight for next time through the loop
+		if rs.histPruneHeight < 1 {
+			if debuginfo {
+				fmt.Printf("**CVCH Finished!!\n")
+			}
+			rs.histPruneFinished = true
+			break // do not remove heights below 1
+		}
+	}
+}
+
 // Commit implements Committer/CommitStore.
 func (rs *Store) Commit() types.CommitID {
 	var previousHeight, version int64
@@ -420,10 +490,17 @@ func (rs *Store) Commit() types.CommitID {
 	// be pruned, where pruneHeight = (commitHeight - 1) - KeepRecent.
 	if rs.pruningOpts.Interval > 0 && int64(rs.pruningOpts.KeepRecent) < previousHeight {
 		pruneHeight := previousHeight - int64(rs.pruningOpts.KeepRecent)
+		//fmt.Printf("**CVCH PH.Future Add pruneHeight %d (previousHeight %d)\n", pruneHeight, previousHeight)
 		rs.pruneHeights = append(rs.pruneHeights, pruneHeight)
 	}
 
-	// batch prune if the current height is a pruning interval height
+	// If the next version will be a pruning interval, pre-append any historical prunes.
+	// This early addtion makes testing in the existing framework easy
+	if rs.pruningOpts.Interval > 0 && (version+1)%int64(rs.pruningOpts.Interval) == 0 {
+		rs.addPruneHistorical(previousHeight) // CVCH: Add additional block heights for historical pruning
+	}
+
+	// Execute batch prune if the current height is a pruning interval height
 	if rs.pruningOpts.Interval > 0 && version%int64(rs.pruningOpts.Interval) == 0 {
 		rs.pruneStores()
 	}
@@ -439,6 +516,7 @@ func (rs *Store) Commit() types.CommitID {
 // pruneStores will batch delete a list of heights from each mounted sub-store.
 // Afterwards, pruneHeights is reset.
 func (rs *Store) pruneStores() {
+	debuginfo := false // show debug info - Helpful to see which heights are pruned
 	if len(rs.pruneHeights) == 0 {
 		return
 	}
@@ -449,6 +527,9 @@ func (rs *Store) pruneStores() {
 			// it to get the underlying IAVL store.
 			store = rs.GetCommitKVStore(key)
 
+			if debuginfo {
+				fmt.Printf("**pruneStores Key: %s  Heights: %v\n", key, rs.pruneHeights)
+			}
 			if err := store.(*iavl.Store).DeleteVersions(rs.pruneHeights...); err != nil {
 				if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist {
 					panic(err)

store/rootmulti/store_test.go

@@ -525,13 +525,17 @@ func TestMultiStore_PruningRestart(t *testing.T) {
 	// ensure we've persisted the current batch of heights to prune to the store's DB
 	ph, err := getPruningHeights(ms.db)
 	require.NoError(t, err)
-	require.Equal(t, []int64{1, 2, 3, 4, 5, 6, 7}, ph)
+	for _, h := range []int64{1, 2, 3, 4, 5, 6, 7} {
+		require.Contains(t, ph, h) // Expect each height to be ready for prune (may have duplicates at first due to histPrune)
+	}
 
 	// "restart"
 	ms = newMultiStoreWithMounts(db, types.NewPruningOptions(2, 11))
 	err = ms.LoadLatestVersion()
 	require.NoError(t, err)
-	require.Equal(t, []int64{1, 2, 3, 4, 5, 6, 7}, ms.pruneHeights)
+	for _, h := range []int64{1, 2, 3, 4, 5, 6, 7} {
+		require.Contains(t, ph, h) // Expect each height to be ready for prune (may have duplicates at first due to histPrune)
+	}
 
 	// commit one more block and ensure the heights have been pruned
 	ms.Commit()
@@ -543,6 +547,172 @@ func TestMultiStore_PruningRestart(t *testing.T) {
 	}
 }
 
+func TestMultiStore_PruningHistoricalEarlyFinish(t *testing.T) {
+	// For brand new databases, historical pruning should immidiately be considered complete
+	db := dbm.NewMemDB()
+	ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 20, Interval: 5})
+	require.NoError(t, ms.LoadLatestVersion())
+
+	// Commit more than prune interval
+	for i := int64(0); i < 10; i++ {
+		ms.Commit()
+	}
+
+	// Should already be flagged complete since we started at zero height
+	require.Equal(t, ms.histPruneFinished, true)
+}
+
+// Confirm that decreasing the KeepRecent setting from 100->10 prunes the new offset
+func TestMultiStore_PruningHistoricalDecrease(t *testing.T) {
+	debuginfo := false // Show debug logs?
+
+	if debuginfo {
+		fmt.Println("TestMultiStore_PruningHistoricalReduce")
+	}
+
+	// Create DB
+	db := dbm.NewMemDB()
+	ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 100, Interval: 5})
+	require.NoError(t, ms.LoadLatestVersion())
+
+	// Add Heights 1-57
+	for i := int64(0); i < 57; i++ {
+		ms.Commit()
+	}
+	_, err := getPruningHeights(ms.db)
+	require.Error(t, err) // Should Error w/o height to prune
+
+	// Next pruneHeight 60, historicalPrune queue height = 59, previousHeight = 58, previousHeight(58) - keepRecent(10) = 48
+	historicMarker := int64(48)
+
+	// Simulate Reconfig + "Restart" + CHANGE PruningOptions (to a lower KeepRecent), Check for historical prune
+	if debuginfo {
+		fmt.Printf("Restart with new PruneingOptions\n")
+	}
+	ms = newMultiStoreWithMounts(db, types.NewPruningOptions(10, 5))
+	err = ms.LoadLatestVersion()
+	require.NoError(t, err)
+	ms.Commit() // Boring commit
+
+	// commit until all historic heights have been pruned
+	// Loop 0: Prune Heights: 39-48
+	// Loop 1: Prune Heights: 29-38
+	// Loop 2: Prune Heights: 19-28
+	// Loop 3: Prune Heights:  9-18
+	// Loop 4: Prune Heights:  1- 8
+	for i := 0; i < 5; i++ {
+		if debuginfo {
+			fmt.Printf("Historic Prune Loop %d\n", i)
+		}
+		require.Equal(t, false, ms.histPruneFinished) // Should not be done pruning at the beginning of this loop
+
+		// This commit should queue a lot of historical prunes
+		ms.Commit()
+		for h := historicMarker; h > historicMarker-10 && h > 0; h-- {
+			require.Contains(t, ms.pruneHeights, h) // Expect each historicalHeight to be queued for prune
+		}
+		historicMarker -= 10
+
+		// Run another series of dummy commits until the next historical height queue
+		for dummy := int64(0); dummy < 4; dummy++ {
+			if debuginfo {
+				fmt.Printf("\tBoring Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1)
+			}
+			ms.Commit()
+		}
+	}
+
+	// Historic Prune - Should be complete
+	require.Equal(t, true, ms.histPruneFinished)
+
+	// Run some extra commits to see if the program dies
+	for i := int64(0); i < 10; i++ {
+		if debuginfo {
+			fmt.Printf("\tExtra Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1)
+		}
+		ms.Commit()
+	}
+	targetHeights := []int64{80, 81, 82}
+	require.Equal(t, targetHeights, ms.pruneHeights)
+}
+
+// Confirm that increasing the KeepRecent setting from 10->30 prunes the new offset
+func TestMultiStore_PruningHistoricalIncrease(t *testing.T) {
+	debuginfo := false // Show debug logs?
+
+	if debuginfo {
+		fmt.Println("TestMultiStore_PruningHistoricalIncrease")
+	}
+
+	// Create DB
+	db := dbm.NewMemDB()
+	ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 10, Interval: 5})
+	require.NoError(t, ms.LoadLatestVersion())
+
+	// Add enough heights to start pruning after increase
+	targetHeights := []int64{45, 46}
+	for i := int64(0); i < 57; i++ {
+		ms.Commit()
+	}
+	require.Equal(t, true, ms.histPruneFinished) // Should not be done pruning at the beginning of this loop
+	ph, err := getPruningHeights(ms.db)
+	require.Equal(t, targetHeights, ph) // Should have some pruneHeights due to low KeepRecent
+
+	// Next pruneHeight 60, historicalPrune queue height = 59, previousHeight = 58, previousHeight(58) - keepRecent(30) = 28
+	historicMarker := int64(28)
+
+	// Simulate Reconfig + "Restart" + CHANGE PruningOptions (to a lower KeepRecent), Check for historical prune
+	if debuginfo {
+		fmt.Printf("Restart with new PruneingOptions\n")
+	}
+	ms = newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 30, Interval: 10})
+	err = ms.LoadLatestVersion()
+	require.NoError(t, err)
+	ph, err = getPruningHeights(ms.db)
+	require.Equal(t, targetHeights, ph) // Should have some pruneHeights due to low KeepRecent
+	ms.Commit()                         // Boring commit
+
+	// commit until all historic heights have been pruned
+	// Loop 0: Prune Heights: 19-28
+	// Loop 1: Prune Heights:  9-18
+	// Loop 2: Prune Heights:  1- 8
+	for i := 0; i < 3; i++ {
+		if debuginfo {
+			fmt.Printf("Historic Prune Loop %d\n", i)
+		}
+		require.Equal(t, false, ms.histPruneFinished) // Should not be done pruning at the beginning of this loop
+
+		// This commit should queue a lot of historical prunes
+		ms.Commit()
+		for h := historicMarker; h > historicMarker-10 && h > 0; h-- {
+			require.Contains(t, ms.pruneHeights, h) // Expect each historicalHeight to be queued for prune
+		}
+		historicMarker -= 10
+
+		// Run another series of dummy commits until the next historical height queue
+		for dummy := int64(0); dummy < 9; dummy++ {
+			if debuginfo {
+				fmt.Printf("\tBoring Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1)
+			}
+			ms.Commit()
+		}
+	}
+
+	// Historic Prune - Should be complete
+	require.Equal(t, true, ms.histPruneFinished)
+
+	// Run some extra commits to see if the program dies
+	for i := int64(0); i < 10; i++ {
+		if debuginfo {
+			fmt.Printf("\tExtra Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1)
+		}
+		ms.Commit()
+	}
+
+	targetHeights = []int64{60, 61, 62, 63, 64, 65, 66, 67}
+	require.Equal(t, targetHeights, ms.pruneHeights)
+}
+
 func TestSetInitialVersion(t *testing.T) {
 	db := dbm.NewMemDB()
 	multi := newMultiStoreWithMounts(db, types.PruneNothing)