fix(snapshots): raise the per snapshot item limit (#304)

Increases the snapshot per item size limit when restoring from state-sync

store/snapshots/manager.go

@@ -16,6 +16,29 @@ import (
 	storetypes "cosmossdk.io/store/types"
 )
 
+const (
+	opNone     operation = ""
+	opSnapshot operation = "snapshot"
+	opPrune    operation = "prune"
+	opRestore  operation = "restore"
+
+	chunkBufferSize = 4
+
+	// snapshotMaxItemSize limits the size of both KVStore entries and snapshot
+	// extension payloads during a state-sync restore.
+	// Unexported so copied in manager_test.go for testing
+	snapshotMaxItemSize = int(512e6)
+)
+
+// operation represents a Manager operation. Only one operation can be in progress at a time.
+type operation string
+
+// restoreDone represents the result of a restore operation.
+type restoreDone struct {
+	complete bool  // if true, restore completed successfully (not prematurely)
+	err      error // if non-nil, restore errored
+}
+
 // Manager manages snapshot and restore operations for an app, making sure only a single
 // long-running operation is in progress at any given time, and provides convenience methods
 // mirroring the ABCI interface.

store/snapshots/manager_test.go

@@ -2,6 +2,7 @@ package snapshots_test
 
 import (
 	"errors"
+	"io"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -237,12 +238,88 @@ func TestManager_Restore(t *testing.T) {
 	require.NoError(t, err)
 }
 
-func TestManager_TakeError(t *testing.T) {
-	snapshotter := &mockErrorSnapshotter{}
-	store, err := snapshots.NewStore(coretesting.NewMemDB(), GetTempDir(t))
+const snapshotMaxItemSize = int(512e6) // Copied from github.com/cosmos/cosmos-sdk/snapshots
+
+func TestManager_RestoreLargeItem(t *testing.T) {
+	store := setupStore(t)
+	target := &mockSnapshotter{}
+	extSnapshotter := newExtSnapshotter(0)
+	manager := snapshots.NewManager(store, target)
+	err := manager.RegisterExtensions(extSnapshotter)
 	require.NoError(t, err)
-	manager := snapshots.NewManager(store, opts, snapshotter, nil, log.NewNopLogger())
 
-	_, err = manager.Create(1)
+	largeItem := make([]byte, snapshotMaxItemSize)
+
+	// The protobuf wrapper introduces extra bytes
+	adjustedSize := 2*snapshotMaxItemSize - (&types.SnapshotItem{
+		Item: &types.SnapshotItem_ExtensionPayload{
+			ExtensionPayload: &types.SnapshotExtensionPayload{
+				Payload: largeItem,
+			},
+		},
+	}).Size()
+	largeItem = largeItem[:adjustedSize]
+	expectItems := [][]byte{largeItem}
+
+	chunks := snapshotItems(expectItems, newExtSnapshotter(1))
+
+	// Starting a restore works
+	err = manager.Restore(types.Snapshot{
+		Height:   3,
+		Format:   2,
+		Hash:     []byte{1, 2, 3},
+		Chunks:   1,
+		Metadata: types.Metadata{ChunkHashes: checksums(chunks)},
+	})
+	require.NoError(t, err)
+
+	// Feeding the chunks should work
+	for i, chunk := range chunks {
+		done, err := manager.RestoreChunk(chunk)
+		require.NoError(t, err)
+		if i == len(chunks)-1 {
+			assert.True(t, done)
+		} else {
+			assert.False(t, done)
+		}
+	}
+
+	assert.Equal(t, expectItems, target.items)
+	assert.Equal(t, 1, len(extSnapshotter.state))
+}
+
+func TestManager_CannotRestoreTooLargeItem(t *testing.T) {
+	store := setupStore(t)
+	target := &mockSnapshotter{}
+	extSnapshotter := newExtSnapshotter(0)
+	manager := snapshots.NewManager(store, target)
+	err := manager.RegisterExtensions(extSnapshotter)
+	require.NoError(t, err)
+
+	// The protobuf wrapper introduces extra bytes
+	largeItem := make([]byte, snapshotMaxItemSize)
+	expectItems := [][]byte{largeItem}
+
+	chunks := snapshotItems(expectItems, newExtSnapshotter(1))
+
+	// Starting a restore works
+	err = manager.Restore(types.Snapshot{
+		Height:   3,
+		Format:   2,
+		Hash:     []byte{1, 2, 3},
+		Chunks:   1,
+		Metadata: types.Metadata{ChunkHashes: checksums(chunks)},
+	})
+	require.NoError(t, err)
+
+	// Feeding the chunks fails
+	for _, chunk := range chunks {
+		_, err = manager.RestoreChunk(chunk)
+
+		if err != nil {
+			break
+		}
+	}
 	require.Error(t, err)
+	require.True(t, errors.Is(err, io.ErrShortBuffer))
 }