Allow manually remove invalid snapshots on restore

script/demo/config.stargz.toml

@@ -5,4 +5,6 @@ ipfs = true
 host = "registry2:5000"
 insecure = true
 [directory_cache]
-direct = true
+direct = true
+[snapshotter]
+allow_invalid_mounts_on_restart = true

service/config.go

@@ -32,6 +32,9 @@ type Config struct {
 
 	// ResolverConfig is config for resolving registries.
 	ResolverConfig `toml:"resolver"`
+
+	// SnapshotterConfig is snapshotter-related config.
+	SnapshotterConfig `toml:"snapshotter"`
 }
 
 // KubeconfigKeychainConfig is config for kubeconfig-based keychain.
@@ -55,3 +58,12 @@ type CRIKeychainConfig struct {
 
 // ResolverConfig is config for resolving registries.
 type ResolverConfig resolver.Config
+
+// SnapshotterConfig is snapshotter-related config.
+type SnapshotterConfig struct {
+	// AllowInvalidMountsOnRestart allows that there are snapshot mounts that cannot access to the
+	// data source when restarting the snapshotter.
+	// NOTE: User needs to manually remove the snapshots from containerd's metadata store using
+	//       ctr (e.g. `ctr snapshot rm`).
+	AllowInvalidMountsOnRestart bool `toml:"allow_invalid_mounts_on_restart"`
+}

service/service.go

@@ -93,7 +93,12 @@ func NewStargzSnapshotterService(ctx context.Context, root string, config *Confi
 
 	var snapshotter snapshots.Snapshotter
 
-	snapshotter, err = snbase.NewSnapshotter(ctx, snapshotterRoot(root), fs, snbase.AsynchronousRemove)
+	snOpts := []snbase.Opt{snbase.AsynchronousRemove}
+	if config.SnapshotterConfig.AllowInvalidMountsOnRestart {
+		snOpts = append(snOpts, snbase.AllowInvalidMountsOnRestart)
+	}
+
+	snapshotter, err = snbase.NewSnapshotter(ctx, snapshotterRoot(root), fs, snOpts...)
 	if err != nil {
 		log.G(ctx).WithError(err).Fatalf("failed to create new snapshotter")
 	}

snapshot/snapshot.go

@@ -71,8 +71,9 @@ type FileSystem interface {
 
 // SnapshotterConfig is used to configure the remote snapshotter instance
 type SnapshotterConfig struct {
-	asyncRemove bool
-	noRestore   bool
+	asyncRemove                 bool
+	noRestore                   bool
+	allowInvalidMountsOnRestart bool
 }
 
 // Opt is an option to configure the remote snapshotter
@@ -92,15 +93,21 @@ func NoRestore(config *SnapshotterConfig) error {
 	return nil
 }
 
+func AllowInvalidMountsOnRestart(config *SnapshotterConfig) error {
+	config.allowInvalidMountsOnRestart = true
+	return nil
+}
+
 type snapshotter struct {
 	root        string
 	ms          *storage.MetaStore
 	asyncRemove bool
 
 	// fs is a filesystem that this snapshotter recognizes.
-	fs        FileSystem
-	userxattr bool // whether to enable "userxattr" mount option
-	noRestore bool
+	fs                          FileSystem
+	userxattr                   bool // whether to enable "userxattr" mount option
+	noRestore                   bool
+	allowInvalidMountsOnRestart bool
 }
 
 // NewSnapshotter returns a Snapshotter which can use unpacked remote layers
@@ -144,12 +151,13 @@ func NewSnapshotter(ctx context.Context, root string, targetFs FileSystem, opts
 	}
 
 	o := &snapshotter{
-		root:        root,
-		ms:          ms,
-		asyncRemove: config.asyncRemove,
-		fs:          targetFs,
-		userxattr:   userxattr,
-		noRestore:   config.noRestore,
+		root:                        root,
+		ms:                          ms,
+		asyncRemove:                 config.asyncRemove,
+		fs:                          targetFs,
+		userxattr:                   userxattr,
+		noRestore:                   config.noRestore,
+		allowInvalidMountsOnRestart: config.allowInvalidMountsOnRestart,
 	}
 
 	if err := o.restoreRemoteSnapshot(ctx); err != nil {
@@ -741,6 +749,14 @@ func (o *snapshotter) restoreRemoteSnapshot(ctx context.Context) error {
 	}
 	for _, info := range task {
 		if err := o.prepareRemoteSnapshot(ctx, info.Name, info.Labels); err != nil {
+			if o.allowInvalidMountsOnRestart {
+				logrus.WithError(err).Warnf("failed to restore remote snapshot %s; remove this snapshot manually", info.Name)
+				// This snapshot mount is invalid but allow this.
+				// NOTE: snapshotter.Mount() will fail to return the mountpoint of these invalid snapshots so
+				//       containerd cannot use them anymore. User needs to manually remove the snapshots from
+				//       containerd's metadata store using ctr (e.g. `ctr snapshot rm`).
+				continue
+			}
 			return fmt.Errorf("failed to prepare remote snapshot: %s: %w", info.Name, err)
 		}
 	}