From 2f7b56cac82be1548450b47f90d42dc44b8b2442 Mon Sep 17 00:00:00 2001 From: Chll Validation Date: Wed, 20 Apr 2022 23:19:55 +0900 Subject: [PATCH 1/5] CVCH WIP extended test ok --- store/rootmulti/store.go | 70 +++++++++++++++++++++++++++++- store/rootmulti/store_test.go | 81 +++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 2 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 0e52d1f288db..d56aaecb2225 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -46,7 +46,7 @@ type Store struct { stores map[types.StoreKey]types.CommitKVStore keysByName map[string]types.StoreKey lazyLoading bool - pruneHeights []int64 + pruneHeights []int64 // CVCH: A running list of heights to prune at the next pruningOpts.Interval initialVersion int64 removalMap map[types.StoreKey]bool @@ -54,6 +54,10 @@ type Store struct { traceContext types.TraceContext traceContextMutex sync.Mutex + cleanHouseHeight int64 // CVCH: -1 Unstarted; 0 Finished; +# Current Height we are pruning down from + cleanHouseAmount int64 // CVCH: Positive Integer of blocks to prune @ Each pruning-interval + cleanHouseFinished bool // CVCH: true at false, then turn off when cleanHouse is complete + interBlockCache types.MultiStorePersistentCache listeners map[types.StoreKey][]types.WriteListener @@ -79,6 +83,10 @@ func NewStore(db dbm.DB) *Store { pruneHeights: make([]int64, 0), listeners: make(map[types.StoreKey][]types.WriteListener), removalMap: make(map[types.StoreKey]bool), + + cleanHouseHeight: -1, // CVCH + cleanHouseAmount: 10, // CVCH + cleanHouseFinished: false, // CVCH } } @@ -384,6 +392,56 @@ func (rs *Store) LastCommitID() types.CommitID { return rs.lastCommitInfo.CommitID() } +// CVCH: Chill Validation - After Pruning 'strategy' changes, previous heights may be left around. +// Example 1: KeepRecent = 100 is changed to KeepRecent = 50, leaving the oldest 50 blocks unpruned after restart. +// Example 2: KeepEvery = 100 is changed to KeepRecent = 50, leaving the old "Every" heights in the DB. +func (rs *Store) addPruneHistorical(previousHeight int64) { + // Clean Historical is already finished + if rs.cleanHouseFinished { + return + } + + // Avoid cleaning historical too early in the blockchain lifecycle + // Only consider cleaning historical heights if current height > KeepRecent) + if previousHeight < int64(rs.pruningOpts.KeepRecent) { + fmt.Printf("**CVCH Too Early: %d !> %d\n", previousHeight, rs.pruningOpts.KeepRecent) + fmt.Printf("**CVCH Early Finished!!\n") + rs.cleanHouseFinished = true // Since we have started below the KeepRecent height, cleanHouse is unnecessary + return + } + + // Debug + fmt.Printf("**CVCH Run : %d > %d\n", previousHeight, rs.pruningOpts.KeepRecent) + fmt.Printf("**CVCH cHH : %d\n", rs.cleanHouseHeight) + + // Start from current height, then move towards 0 since more recent blocks are more likely to be present) + // Remove all previous blocks older than KeepRecent + // NOTE: SNAPSHOTS MUST be within KeepRecent (Must KeepRecent > snapshot-keep-recent * snapshot-interval (a multiple of pruning-keep-every in older cosmos-sdk versions, or within keep-recent for newer cosmos-sdk versions) + + // Clean Historical hasn't been set up yet + if rs.cleanHouseHeight == -1 { + rs.cleanHouseHeight = previousHeight - int64(rs.pruningOpts.KeepRecent) // We are not concerned about duplicate height removals + fmt.Printf("**CVCH Init cleanHouseHeight = %d\n", rs.cleanHouseHeight) + } + + fmt.Printf("**PH CVCH Starting: cleanHouseHeight %d\n", rs.cleanHouseHeight) + // Determine starting and ending heights to clean + var cleanOffset int64 + for cleanOffset = 0; cleanOffset < rs.cleanHouseAmount; cleanOffset++ { + fmt.Printf("**PH CVCH Add %d\n", rs.cleanHouseHeight) + rs.pruneHeights = append(rs.pruneHeights, rs.cleanHouseHeight) + + rs.cleanHouseHeight-- // Reduce cleanHouseHeight for next time through the loop + if rs.cleanHouseHeight < 1 { + fmt.Printf("\t\t**PH CVCH **COMPLETE** cHH:%d\n", rs.cleanHouseHeight) + fmt.Printf("**CVCH Finished!!\n") + rs.cleanHouseFinished = true + break // do not remove heights below 1 + } + } + fmt.Printf("**PH CVCH Resulting: cleanHouseHeight %d\n", rs.cleanHouseHeight) +} + // Commit implements Committer/CommitStore. func (rs *Store) Commit() types.CommitID { var previousHeight, version int64 @@ -420,10 +478,17 @@ func (rs *Store) Commit() types.CommitID { // be pruned, where pruneHeight = (commitHeight - 1) - KeepRecent. if rs.pruningOpts.Interval > 0 && int64(rs.pruningOpts.KeepRecent) < previousHeight { pruneHeight := previousHeight - int64(rs.pruningOpts.KeepRecent) + fmt.Printf("**PH.Future Add pruneHeight %d (previousHeight %d)\n", pruneHeight, previousHeight) rs.pruneHeights = append(rs.pruneHeights, pruneHeight) } - // batch prune if the current height is a pruning interval height + // If the next version will be a pruning interval, pre-append any historical prunes. + // This early addtion makes testing in the existing framework easy + if rs.pruningOpts.Interval > 0 && (version+1)%int64(rs.pruningOpts.Interval) == 0 { + rs.addPruneHistorical(previousHeight) // CVCH: Add additional block heights for clean house pruning + } + + // Execute batch prune if the current height is a pruning interval height if rs.pruningOpts.Interval > 0 && version%int64(rs.pruningOpts.Interval) == 0 { rs.pruneStores() } @@ -449,6 +514,7 @@ func (rs *Store) pruneStores() { // it to get the underlying IAVL store. store = rs.GetCommitKVStore(key) + fmt.Printf("**PH DV %s => %v\n", key, rs.pruneHeights) if err := store.(*iavl.Store).DeleteVersions(rs.pruneHeights...); err != nil { if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist { panic(err) diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index 0a50f11e5c56..71cb98aedcc9 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -543,6 +543,87 @@ func TestMultiStore_PruningRestart(t *testing.T) { } } +func TestMultiStore_PruningHistoricalEarlyFinish(t *testing.T) { + fmt.Println("TestMultiStore_PruningHistoricalEarlyFinish") + db := dbm.NewMemDB() + ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 20, Interval: 5}) + require.NoError(t, ms.LoadLatestVersion()) + + // Commit enough to build up heights to prune, where on the next block we should + // batch delete. + for i := int64(0); i < 10; i++ { + ms.Commit() + } + require.Equal(t, ms.cleanHouseFinished, true) // Should already be flagged complete since we started at zero height + fmt.Println("") + fmt.Println("") +} + +func TestMultiStore_PruningHistorical(t *testing.T) { + fmt.Println("TestMultiStore_PruningHistorical") + db := dbm.NewMemDB() + ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 100, Interval: 5}) + require.NoError(t, ms.LoadLatestVersion()) + + // Add Heights 1-50 + for i := int64(0); i < 50; i++ { + ms.Commit() + } + _, err := getPruningHeights(ms.db) + require.Error(t, err) // Should Error w/o height to prune + + // commit more (new heights 51 - 157, 1-54 pruned, 55, 56 unpruned until height = 160) + for i := int64(0); i < 107; i++ { + ms.Commit() + } + // require.NoError(t, err) + ph, err := getPruningHeights(ms.db) + fmt.Printf("gPH: %v\n", ph) + require.Equal(t, []int64{55, 56}, ph) + + // next pruneHeight 160, historicalPrune queue height = 159, previousHeight = 158, previousHeight(158) - keepRecent(10) = 148 + historicMarker := int64(148) + + // "restart" + CHANGE PruningOptions (to a lower KeepRecent), Check for historical prune + fmt.Printf("Restart with new PruneingOptions\n") + + targetPruneHeights := []int64{55, 56} + + ms = newMultiStoreWithMounts(db, types.NewPruningOptions(10, 5)) + err = ms.LoadLatestVersion() + require.NoError(t, err) + require.Equal(t, targetPruneHeights, ms.pruneHeights) + + // 158 + fmt.Printf("Commit Height 158\n") + ms.Commit() + targetPruneHeights = append(targetPruneHeights, 147) // expect to prune the previousHeight - keepRecent + require.Equal(t, targetPruneHeights, ms.pruneHeights) // Check that db reloaded oldPruneHeights + + // 159 + fmt.Printf("Commit Height 159\n") + ms.Commit() + // Expect each historical Height to be queued for prune + for h := historicMarker; h > historicMarker-10; h-- { + require.Contains(t, ms.pruneHeights, h) + } + historicMarker -= 10 + + // 160-163 + for ms.cleanHouseFinished == false { + for i := int64(0); i < 5; i++ { + fmt.Printf("\tCommit Height %d\n", ms.lastCommitInfo.GetVersion()+1) + ms.Commit() + } + // Expect each historical Height to be queued for prune + for h := historicMarker; h > historicMarker-10 && h > 0; h-- { + require.Contains(t, ms.pruneHeights, h) + } + historicMarker -= 10 + } + +} + func TestSetInitialVersion(t *testing.T) { db := dbm.NewMemDB() multi := newMultiStoreWithMounts(db, types.PruneNothing) From 92e8214a5df50706b6ba52e679f37887bfdae450 Mon Sep 17 00:00:00 2001 From: Chll Validation Date: Thu, 21 Apr 2022 01:37:11 +0900 Subject: [PATCH 2/5] CVCH Clean Basic Tests --- store/rootmulti/store.go | 2 +- store/rootmulti/store_test.go | 104 ++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index d56aaecb2225..9192d1cab186 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -478,7 +478,7 @@ func (rs *Store) Commit() types.CommitID { // be pruned, where pruneHeight = (commitHeight - 1) - KeepRecent. if rs.pruningOpts.Interval > 0 && int64(rs.pruningOpts.KeepRecent) < previousHeight { pruneHeight := previousHeight - int64(rs.pruningOpts.KeepRecent) - fmt.Printf("**PH.Future Add pruneHeight %d (previousHeight %d)\n", pruneHeight, previousHeight) + fmt.Printf("**CVCH PH.Future Add pruneHeight %d (previousHeight %d)\n", pruneHeight, previousHeight) rs.pruneHeights = append(rs.pruneHeights, pruneHeight) } diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index 71cb98aedcc9..fcbd73a81b5e 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -544,84 +544,92 @@ func TestMultiStore_PruningRestart(t *testing.T) { } func TestMultiStore_PruningHistoricalEarlyFinish(t *testing.T) { - fmt.Println("TestMultiStore_PruningHistoricalEarlyFinish") + // For brand new databases, historical pruning should immidiately be considered complete db := dbm.NewMemDB() ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 20, Interval: 5}) require.NoError(t, ms.LoadLatestVersion()) - // Commit enough to build up heights to prune, where on the next block we should - // batch delete. + // Commit more than prune interval for i := int64(0); i < 10; i++ { ms.Commit() } - require.Equal(t, ms.cleanHouseFinished, true) // Should already be flagged complete since we started at zero height - fmt.Println("") - fmt.Println("") + + // Should already be flagged complete since we started at zero height + require.Equal(t, ms.cleanHouseFinished, true) } +// Confirm that historical pruning starts and completes func TestMultiStore_PruningHistorical(t *testing.T) { - fmt.Println("TestMultiStore_PruningHistorical") + debuginfo := false // Show debug logs? + + if debuginfo { + fmt.Println("TestMultiStore_PruningHistorical") + } + + // Create DB db := dbm.NewMemDB() ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 100, Interval: 5}) require.NoError(t, ms.LoadLatestVersion()) - // Add Heights 1-50 - for i := int64(0); i < 50; i++ { + // Add Heights 1-57 + for i := int64(0); i < 57; i++ { ms.Commit() } _, err := getPruningHeights(ms.db) require.Error(t, err) // Should Error w/o height to prune - // commit more (new heights 51 - 157, 1-54 pruned, 55, 56 unpruned until height = 160) - for i := int64(0); i < 107; i++ { - ms.Commit() - } - // require.NoError(t, err) - ph, err := getPruningHeights(ms.db) - fmt.Printf("gPH: %v\n", ph) - require.Equal(t, []int64{55, 56}, ph) - - // next pruneHeight 160, historicalPrune queue height = 159, previousHeight = 158, previousHeight(158) - keepRecent(10) = 148 - historicMarker := int64(148) + // Next pruneHeight 60, historicalPrune queue height = 59, previousHeight = 58, previousHeight(58) - keepRecent(10) = 48 + historicMarker := int64(48) // "restart" + CHANGE PruningOptions (to a lower KeepRecent), Check for historical prune - fmt.Printf("Restart with new PruneingOptions\n") - - targetPruneHeights := []int64{55, 56} - + if debuginfo { + fmt.Printf("Restart with new PruneingOptions\n") + } ms = newMultiStoreWithMounts(db, types.NewPruningOptions(10, 5)) err = ms.LoadLatestVersion() require.NoError(t, err) - require.Equal(t, targetPruneHeights, ms.pruneHeights) - - // 158 - fmt.Printf("Commit Height 158\n") - ms.Commit() - targetPruneHeights = append(targetPruneHeights, 147) // expect to prune the previousHeight - keepRecent - require.Equal(t, targetPruneHeights, ms.pruneHeights) // Check that db reloaded oldPruneHeights - - // 159 - fmt.Printf("Commit Height 159\n") - ms.Commit() - // Expect each historical Height to be queued for prune - for h := historicMarker; h > historicMarker-10; h-- { - require.Contains(t, ms.pruneHeights, h) - } - historicMarker -= 10 - - // 160-163 - for ms.cleanHouseFinished == false { - for i := int64(0); i < 5; i++ { - fmt.Printf("\tCommit Height %d\n", ms.lastCommitInfo.GetVersion()+1) - ms.Commit() + ms.Commit() // Boring commit + + // commit until all historic heights have been pruned + // Loop 0: Prune Heights: 39-48 + // Loop 1: Prune Heights: 29-38 + // Loop 2: Prune Heights: 19-28 + // Loop 3: Prune Heights: 9-18 + // Loop 4: Prune Heights: 1- 8 + for i := 0; i < 5; i++ { + if debuginfo { + fmt.Printf("Historic Prune Loop %d\n", i) } - // Expect each historical Height to be queued for prune + require.Equal(t, false, ms.cleanHouseFinished) // Should not be done pruning at the beginning of this loop + + // This commit should queue a lot of historical prunes + ms.Commit() for h := historicMarker; h > historicMarker-10 && h > 0; h-- { - require.Contains(t, ms.pruneHeights, h) + require.Contains(t, ms.pruneHeights, h) // Expect each historicalHeight to be queued for prune } historicMarker -= 10 + + // Run another series of dummy commits until the next historical height queue + for dummy := int64(0); dummy < 4; dummy++ { + if debuginfo { + fmt.Printf("\tBoring Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1) + } + ms.Commit() + } } + // Historic Prune - Should be complete + require.Equal(t, true, ms.cleanHouseFinished) + + // Run some extra commits to see if the program dies + for i := int64(0); i < 10; i++ { + if debuginfo { + fmt.Printf("\tExtra Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1) + } + ms.Commit() + } + targetHeights := []int64{80, 81, 82} + require.Equal(t, targetHeights, ms.pruneHeights) } func TestSetInitialVersion(t *testing.T) { From a3d7b9d99937a17660e1740302e6448eaca4f674 Mon Sep 17 00:00:00 2001 From: Chll Validation Date: Thu, 21 Apr 2022 02:19:42 +0900 Subject: [PATCH 3/5] CVCH Implement KeepRecent Tests for increase and decrease --- store/rootmulti/store.go | 83 +++++++++++++++++++------------- store/rootmulti/store_test.go | 91 ++++++++++++++++++++++++++++++++--- 2 files changed, 133 insertions(+), 41 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 9192d1cab186..af5f4486c489 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -54,9 +54,9 @@ type Store struct { traceContext types.TraceContext traceContextMutex sync.Mutex - cleanHouseHeight int64 // CVCH: -1 Unstarted; 0 Finished; +# Current Height we are pruning down from - cleanHouseAmount int64 // CVCH: Positive Integer of blocks to prune @ Each pruning-interval - cleanHouseFinished bool // CVCH: true at false, then turn off when cleanHouse is complete + histPruneHeight int64 // CVCH: -1 Unstarted; 0 Finished; +# Current Height we are pruning down from + histPruneAmount int64 // CVCH: Positive Integer of blocks to prune @ Each pruning-interval + histPruneFinished bool // CVCH: true at start, then falsewhen histPrune is complete interBlockCache types.MultiStorePersistentCache @@ -84,9 +84,9 @@ func NewStore(db dbm.DB) *Store { listeners: make(map[types.StoreKey][]types.WriteListener), removalMap: make(map[types.StoreKey]bool), - cleanHouseHeight: -1, // CVCH - cleanHouseAmount: 10, // CVCH - cleanHouseFinished: false, // CVCH + histPruneHeight: -1, // CVCH + histPruneAmount: 10, // CVCH + histPruneFinished: false, // CVCH } } @@ -396,50 +396,62 @@ func (rs *Store) LastCommitID() types.CommitID { // Example 1: KeepRecent = 100 is changed to KeepRecent = 50, leaving the oldest 50 blocks unpruned after restart. // Example 2: KeepEvery = 100 is changed to KeepRecent = 50, leaving the old "Every" heights in the DB. func (rs *Store) addPruneHistorical(previousHeight int64) { + debuginfo := false // show debug info + // Clean Historical is already finished - if rs.cleanHouseFinished { + if rs.histPruneFinished { return } - // Avoid cleaning historical too early in the blockchain lifecycle + // Avoid historical pruning too early in the blockchain lifecycle // Only consider cleaning historical heights if current height > KeepRecent) if previousHeight < int64(rs.pruningOpts.KeepRecent) { - fmt.Printf("**CVCH Too Early: %d !> %d\n", previousHeight, rs.pruningOpts.KeepRecent) - fmt.Printf("**CVCH Early Finished!!\n") - rs.cleanHouseFinished = true // Since we have started below the KeepRecent height, cleanHouse is unnecessary + if debuginfo { + fmt.Printf("**CVCH Too Early: %d !> %d\n", previousHeight, rs.pruningOpts.KeepRecent) + fmt.Printf("**CVCH Early Finished!!\n") + } + rs.histPruneFinished = true // Since we have started below the KeepRecent height, histPrune is unnecessary return } // Debug - fmt.Printf("**CVCH Run : %d > %d\n", previousHeight, rs.pruningOpts.KeepRecent) - fmt.Printf("**CVCH cHH : %d\n", rs.cleanHouseHeight) + if debuginfo { + fmt.Printf("**CVCH Run : %d > %d\n", previousHeight, rs.pruningOpts.KeepRecent) + fmt.Printf("**CVCH cHH : %d\n", rs.histPruneHeight) + } // Start from current height, then move towards 0 since more recent blocks are more likely to be present) // Remove all previous blocks older than KeepRecent // NOTE: SNAPSHOTS MUST be within KeepRecent (Must KeepRecent > snapshot-keep-recent * snapshot-interval (a multiple of pruning-keep-every in older cosmos-sdk versions, or within keep-recent for newer cosmos-sdk versions) // Clean Historical hasn't been set up yet - if rs.cleanHouseHeight == -1 { - rs.cleanHouseHeight = previousHeight - int64(rs.pruningOpts.KeepRecent) // We are not concerned about duplicate height removals - fmt.Printf("**CVCH Init cleanHouseHeight = %d\n", rs.cleanHouseHeight) - } - - fmt.Printf("**PH CVCH Starting: cleanHouseHeight %d\n", rs.cleanHouseHeight) - // Determine starting and ending heights to clean - var cleanOffset int64 - for cleanOffset = 0; cleanOffset < rs.cleanHouseAmount; cleanOffset++ { - fmt.Printf("**PH CVCH Add %d\n", rs.cleanHouseHeight) - rs.pruneHeights = append(rs.pruneHeights, rs.cleanHouseHeight) - - rs.cleanHouseHeight-- // Reduce cleanHouseHeight for next time through the loop - if rs.cleanHouseHeight < 1 { - fmt.Printf("\t\t**PH CVCH **COMPLETE** cHH:%d\n", rs.cleanHouseHeight) - fmt.Printf("**CVCH Finished!!\n") - rs.cleanHouseFinished = true + if rs.histPruneHeight == -1 { + rs.histPruneHeight = previousHeight - int64(rs.pruningOpts.KeepRecent) // We are not concerned about duplicate height removals + if debuginfo { + fmt.Printf("**CVCH Init histPruneHeight = %d\n", rs.histPruneHeight) + } + } + + if debuginfo { + fmt.Printf("**PH CVCH Starting: histPruneHeight %d\n", rs.histPruneHeight) + } + // Determine starting and ending heights to prune + var histCtr int64 + for histCtr = 0; histCtr < rs.histPruneAmount; histCtr++ { + if debuginfo { + fmt.Printf("**PH CVCH Add %d\n", rs.histPruneHeight) + } + rs.pruneHeights = append(rs.pruneHeights, rs.histPruneHeight) + + rs.histPruneHeight-- // Reduce histPruneHeight for next time through the loop + if rs.histPruneHeight < 1 { + if debuginfo { + fmt.Printf("**CVCH Finished!!\n") + } + rs.histPruneFinished = true break // do not remove heights below 1 } } - fmt.Printf("**PH CVCH Resulting: cleanHouseHeight %d\n", rs.cleanHouseHeight) } // Commit implements Committer/CommitStore. @@ -478,14 +490,14 @@ func (rs *Store) Commit() types.CommitID { // be pruned, where pruneHeight = (commitHeight - 1) - KeepRecent. if rs.pruningOpts.Interval > 0 && int64(rs.pruningOpts.KeepRecent) < previousHeight { pruneHeight := previousHeight - int64(rs.pruningOpts.KeepRecent) - fmt.Printf("**CVCH PH.Future Add pruneHeight %d (previousHeight %d)\n", pruneHeight, previousHeight) + //fmt.Printf("**CVCH PH.Future Add pruneHeight %d (previousHeight %d)\n", pruneHeight, previousHeight) rs.pruneHeights = append(rs.pruneHeights, pruneHeight) } // If the next version will be a pruning interval, pre-append any historical prunes. // This early addtion makes testing in the existing framework easy if rs.pruningOpts.Interval > 0 && (version+1)%int64(rs.pruningOpts.Interval) == 0 { - rs.addPruneHistorical(previousHeight) // CVCH: Add additional block heights for clean house pruning + rs.addPruneHistorical(previousHeight) // CVCH: Add additional block heights for historical pruning } // Execute batch prune if the current height is a pruning interval height @@ -504,6 +516,7 @@ func (rs *Store) Commit() types.CommitID { // pruneStores will batch delete a list of heights from each mounted sub-store. // Afterwards, pruneHeights is reset. func (rs *Store) pruneStores() { + debuginfo := false // show debug info - Helpful to see which heights are pruned if len(rs.pruneHeights) == 0 { return } @@ -514,7 +527,9 @@ func (rs *Store) pruneStores() { // it to get the underlying IAVL store. store = rs.GetCommitKVStore(key) - fmt.Printf("**PH DV %s => %v\n", key, rs.pruneHeights) + if debuginfo { + fmt.Printf("**pruneStores Key: %s Heights: %v\n", key, rs.pruneHeights) + } if err := store.(*iavl.Store).DeleteVersions(rs.pruneHeights...); err != nil { if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist { panic(err) diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index fcbd73a81b5e..e282fe690c83 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -555,15 +555,15 @@ func TestMultiStore_PruningHistoricalEarlyFinish(t *testing.T) { } // Should already be flagged complete since we started at zero height - require.Equal(t, ms.cleanHouseFinished, true) + require.Equal(t, ms.histPruneFinished, true) } -// Confirm that historical pruning starts and completes -func TestMultiStore_PruningHistorical(t *testing.T) { +// Confirm that decreasing the KeepRecent setting from 100->10 prunes the new offset +func TestMultiStore_PruningHistoricalDecrease(t *testing.T) { debuginfo := false // Show debug logs? if debuginfo { - fmt.Println("TestMultiStore_PruningHistorical") + fmt.Println("TestMultiStore_PruningHistoricalReduce") } // Create DB @@ -581,7 +581,7 @@ func TestMultiStore_PruningHistorical(t *testing.T) { // Next pruneHeight 60, historicalPrune queue height = 59, previousHeight = 58, previousHeight(58) - keepRecent(10) = 48 historicMarker := int64(48) - // "restart" + CHANGE PruningOptions (to a lower KeepRecent), Check for historical prune + // Simulate Reconfig + "Restart" + CHANGE PruningOptions (to a lower KeepRecent), Check for historical prune if debuginfo { fmt.Printf("Restart with new PruneingOptions\n") } @@ -600,7 +600,7 @@ func TestMultiStore_PruningHistorical(t *testing.T) { if debuginfo { fmt.Printf("Historic Prune Loop %d\n", i) } - require.Equal(t, false, ms.cleanHouseFinished) // Should not be done pruning at the beginning of this loop + require.Equal(t, false, ms.histPruneFinished) // Should not be done pruning at the beginning of this loop // This commit should queue a lot of historical prunes ms.Commit() @@ -619,7 +619,7 @@ func TestMultiStore_PruningHistorical(t *testing.T) { } // Historic Prune - Should be complete - require.Equal(t, true, ms.cleanHouseFinished) + require.Equal(t, true, ms.histPruneFinished) // Run some extra commits to see if the program dies for i := int64(0); i < 10; i++ { @@ -632,6 +632,83 @@ func TestMultiStore_PruningHistorical(t *testing.T) { require.Equal(t, targetHeights, ms.pruneHeights) } +// Confirm that increasing the KeepRecent setting from 10->30 prunes the new offset +func TestMultiStore_PruningHistoricalIncrease(t *testing.T) { + debuginfo := false // Show debug logs? + + if debuginfo { + fmt.Println("TestMultiStore_PruningHistoricalIncrease") + } + + // Create DB + db := dbm.NewMemDB() + ms := newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 10, Interval: 5}) + require.NoError(t, ms.LoadLatestVersion()) + + // Add enough heights to start pruning after increase + targetHeights := []int64{45, 46} + for i := int64(0); i < 57; i++ { + ms.Commit() + } + require.Equal(t, true, ms.histPruneFinished) // Should not be done pruning at the beginning of this loop + ph, err := getPruningHeights(ms.db) + require.Equal(t, targetHeights, ph) // Should have some pruneHeights due to low KeepRecent + + // Next pruneHeight 60, historicalPrune queue height = 59, previousHeight = 58, previousHeight(58) - keepRecent(30) = 28 + historicMarker := int64(28) + + // Simulate Reconfig + "Restart" + CHANGE PruningOptions (to a lower KeepRecent), Check for historical prune + if debuginfo { + fmt.Printf("Restart with new PruneingOptions\n") + } + ms = newMultiStoreWithMounts(db, types.PruningOptions{KeepRecent: 30, Interval: 10}) + err = ms.LoadLatestVersion() + require.NoError(t, err) + ph, err = getPruningHeights(ms.db) + require.Equal(t, targetHeights, ph) // Should have some pruneHeights due to low KeepRecent + ms.Commit() // Boring commit + + // commit until all historic heights have been pruned + // Loop 0: Prune Heights: 19-28 + // Loop 1: Prune Heights: 9-18 + // Loop 2: Prune Heights: 1- 8 + for i := 0; i < 3; i++ { + if debuginfo { + fmt.Printf("Historic Prune Loop %d\n", i) + } + require.Equal(t, false, ms.histPruneFinished) // Should not be done pruning at the beginning of this loop + + // This commit should queue a lot of historical prunes + ms.Commit() + for h := historicMarker; h > historicMarker-10 && h > 0; h-- { + require.Contains(t, ms.pruneHeights, h) // Expect each historicalHeight to be queued for prune + } + historicMarker -= 10 + + // Run another series of dummy commits until the next historical height queue + for dummy := int64(0); dummy < 9; dummy++ { + if debuginfo { + fmt.Printf("\tBoring Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1) + } + ms.Commit() + } + } + + // Historic Prune - Should be complete + require.Equal(t, true, ms.histPruneFinished) + + // Run some extra commits to see if the program dies + for i := int64(0); i < 10; i++ { + if debuginfo { + fmt.Printf("\tExtra Commit Height %d\n", ms.lastCommitInfo.GetVersion()+1) + } + ms.Commit() + } + + targetHeights = []int64{60, 61, 62, 63, 64, 65, 66, 67} + require.Equal(t, targetHeights, ms.pruneHeights) +} + func TestSetInitialVersion(t *testing.T) { db := dbm.NewMemDB() multi := newMultiStoreWithMounts(db, types.PruneNothing) From 9b3dd8ccb9a0bfc15146430887ba2c425491b0cc Mon Sep 17 00:00:00 2001 From: Chll Validation Date: Thu, 21 Apr 2022 02:25:12 +0900 Subject: [PATCH 4/5] CVCH: Adjust original tests for PruningRestart to accomodate duplicate heights from pruneHist --- store/rootmulti/store_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/store/rootmulti/store_test.go b/store/rootmulti/store_test.go index e282fe690c83..ce9cf8f7462c 100644 --- a/store/rootmulti/store_test.go +++ b/store/rootmulti/store_test.go @@ -525,13 +525,17 @@ func TestMultiStore_PruningRestart(t *testing.T) { // ensure we've persisted the current batch of heights to prune to the store's DB ph, err := getPruningHeights(ms.db) require.NoError(t, err) - require.Equal(t, []int64{1, 2, 3, 4, 5, 6, 7}, ph) + for _, h := range []int64{1, 2, 3, 4, 5, 6, 7} { + require.Contains(t, ph, h) // Expect each height to be ready for prune (may have duplicates at first due to histPrune) + } // "restart" ms = newMultiStoreWithMounts(db, types.NewPruningOptions(2, 11)) err = ms.LoadLatestVersion() require.NoError(t, err) - require.Equal(t, []int64{1, 2, 3, 4, 5, 6, 7}, ms.pruneHeights) + for _, h := range []int64{1, 2, 3, 4, 5, 6, 7} { + require.Contains(t, ph, h) // Expect each height to be ready for prune (may have duplicates at first due to histPrune) + } // commit one more block and ensure the heights have been pruned ms.Commit() From b54cfde9bd4ebed343bd15b49896485b9a9d2db1 Mon Sep 17 00:00:00 2001 From: Chll Validation Date: Thu, 21 Apr 2022 02:41:44 +0900 Subject: [PATCH 5/5] CVCH - fix example that only applies to earlier cosmos-sdk, but keep it for now --- store/rootmulti/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index af5f4486c489..185f564ee93f 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -394,7 +394,7 @@ func (rs *Store) LastCommitID() types.CommitID { // CVCH: Chill Validation - After Pruning 'strategy' changes, previous heights may be left around. // Example 1: KeepRecent = 100 is changed to KeepRecent = 50, leaving the oldest 50 blocks unpruned after restart. -// Example 2: KeepEvery = 100 is changed to KeepRecent = 50, leaving the old "Every" heights in the DB. +// Example 2: KeepEvery = 100 is changed to KeepEvery = 35, leaving the old "Every" heights in the DB. func (rs *Store) addPruneHistorical(previousHeight int64) { debuginfo := false // show debug info