implement L2L backup shipping

Fill in the missing implementation for backup shipping to another LINSTOR cluster. Because we can't "wish" for a specific snapshot name, we store a mapping of requested snapshot name to actually taken snapshot name in a special LINSTOR KV. Other than that, it is simply a matter of wiring up the necessary plumbing. This currently cannot restore a backup once it is shipped and the local snapshot deleted, and it is unlikely that such a thing could be implemented, as LINSTOR does not support downloading a snapshot once shipped. Signed-off-by: Moritz Wanzenböck <[email protected]>
piraeusdatastore · Apr 19, 2024 · fe28885 · fe28885
1 parent 2ad3efc
commit fe28885
Show file tree

Hide file tree

Showing 6 changed files with 146 additions and 27 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Support for creating backups with LINSTOR-to-LINSTOR shipping.
+
 ## [1.5.0] - 2024-03-19
 
 ### Added

diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.21
 toolchain go1.21.5
 
 require (
-	github.com/LINBIT/golinstor v0.50.0
+	github.com/LINBIT/golinstor v0.51.0
 	github.com/container-storage-interface/spec v1.9.0
 	github.com/haySwim/data v0.2.0
 	github.com/kubernetes-csi/csi-test/v5 v5.2.0

diff --git a/go.sum b/go.sum
@@ -1,5 +1,5 @@
-github.com/LINBIT/golinstor v0.50.0 h1:WLdk+Jca/6BSOgmGaFqYrGEZlh0D7kjps4ak20Ntj80=
-github.com/LINBIT/golinstor v0.50.0/go.mod h1:MCkHNdHxoGw4mnt8DGsSqWNF5ZGhYFy6Lr4tQLyVBs0=
+github.com/LINBIT/golinstor v0.51.0 h1:XvfcQN3jg7b/s79wrpTe/jzfFDGSNZy2pjGG8SDtFRM=
+github.com/LINBIT/golinstor v0.51.0/go.mod h1:MCkHNdHxoGw4mnt8DGsSqWNF5ZGhYFy6Lr4tQLyVBs0=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=

diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
@@ -854,7 +854,44 @@ func (s *Linstor) reconcileBackup(ctx context.Context, id, sourceVolId string, p
 
 		return &snap, nil
 	case volume.SnapshotTypeLinstor:
-		return nil, fmt.Errorf("linstor-to-linstor snapshots not implemented")
+		kv, err := s.client.KeyValueStore.Get(ctx, linstor.LinstorBackupKVName)
+		if nil404(err) != nil {
+			return nil, fmt.Errorf("error checking for existing LINSTOR backup: %w", err)
+		}
+
+		var snapName string
+
+		if kv == nil || kv.Props[id] == "" {
+			yes := true
+
+			snapName, err = s.client.Backup.Ship(ctx, params.RemoteName, lapi.BackupShipRequest{
+				SrcRscName:   sourceVolId,
+				DstRscName:   "backup-" + sourceVolId,
+				DstStorPool:  params.LinstorTargetStoragePool,
+				DownloadOnly: &yes,
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error creating LINSTOR backup: %w", err)
+			}
+
+			err = s.client.KeyValueStore.CreateOrModify(ctx, linstor.LinstorBackupKVName, lapi.GenericPropsModify{
+				OverrideProps: map[string]string{
+					id: fmt.Sprintf("%s/%s", sourceVolId, snapName),
+				},
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error creating LINSTOR backup: %w", err)
+			}
+		} else {
+			snapName = kv.Props[id]
+		}
+
+		snap, err := s.client.Resources.GetSnapshot(ctx, sourceVolId, snapName)
+		if err != nil {
+			return nil, fmt.Errorf("error fetching snapshot for backup: %w", err)
+		}
+
+		return &snap, nil
 	default:
 		return nil, fmt.Errorf("unsupported snapshot type '%s', don't know how to create a backup", params.Type)
 	}
@@ -896,7 +933,32 @@ func (s *Linstor) reconcileRemote(ctx context.Context, params *volume.SnapshotPa
 
 		return nil
 	case volume.SnapshotTypeLinstor:
-		return fmt.Errorf("Linstor-to-Linstor snapshots not implemented")
+		log.Debug("search for LINSTOR remote with matching name")
+
+		remotes, err := s.client.Remote.GetAllLinstor(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to list existing remotes: %w", err)
+		}
+
+		for _, r := range remotes {
+			if r.RemoteName == params.RemoteName {
+				log.WithField("remote", r).Debug("found existing LINSTOR remote with matching name")
+				return nil
+			}
+		}
+
+		log.Debug("No existing remote found, creating a new one")
+
+		err = s.client.Remote.CreateLinstor(ctx, lapi.LinstorRemote{
+			RemoteName: params.RemoteName,
+			Url:        params.LinstorTargetUrl,
+			ClusterId:  params.LinstorTargetClusterID,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to create new LINSTOR remote: %w", err)
+		}
+
+		return nil
 	default:
 		return fmt.Errorf("unsupported snapshot type '%s', don't know how to configure remote", params.Type)
 	}
@@ -906,7 +968,13 @@ func (s *Linstor) reconcileRemote(ctx context.Context, params *volume.SnapshotPa
 func (s *Linstor) SnapDelete(ctx context.Context, snap *volume.Snapshot) error {
 	log := s.log.WithField("snapshot", snap)
 
-	if snap.Remote != "" {
+	kv, err := s.client.KeyValueStore.Get(ctx, linstor.LinstorBackupKVName)
+	if nil404(err) != nil {
+		return fmt.Errorf("error checking for existing LINSTOR backup: %w", err)
+	}
+
+	// Only try this if we are sure this is an S3 backup
+	if snap.Remote != "" && (kv == nil || kv.Props[snap.SnapshotId] == "") {
 		log.WithField("remote", snap.Remote).Debug("deleting backup from remote")
 
 		backups, err := s.client.Backup.GetAll(ctx, snap.Remote, snap.GetSourceVolumeId(), snap.GetSnapshotId())
@@ -934,13 +1002,22 @@ func (s *Linstor) SnapDelete(ctx context.Context, snap *volume.Snapshot) error {
 
 	log.Debug("deleting local snapshot")
 
-	err := s.client.Resources.DeleteSnapshot(ctx, snap.GetSourceVolumeId(), snap.SnapshotId)
+	err = s.client.Resources.DeleteSnapshot(ctx, snap.GetSourceVolumeId(), snap.SnapshotId)
 	if nil404(err) != nil {
 		return fmt.Errorf("failed to remove snapshot: %w", err)
 	}
 
 	err = s.deleteResourceDefinitionAndGroupIfUnused(ctx, snap.GetSourceVolumeId())
 
+	if kv != nil && kv.Props[snap.SnapshotId] != "" {
+		err := s.client.KeyValueStore.CreateOrModify(ctx, linstor.LinstorBackupKVName, lapi.GenericPropsModify{
+			DeleteProps: []string{snap.SnapshotId},
+		})
+		if err != nil {
+			return fmt.Errorf("error deleting LINSTOR backup property: %w", err)
+		}
+	}
+
 	return nil
 }
 
@@ -1483,7 +1560,30 @@ func (s *Linstor) snapOrBackupById(ctx context.Context, id string) (*lapi.Snapsh
 		}
 	}
 
-	log.Debug("no snapshot matching id found, trying backups")
+	log.Debug("no snapshot matching id found, trying LINSTOR backups")
+
+	kv, err := s.client.KeyValueStore.Get(ctx, linstor.LinstorBackupKVName)
+	if nil404(err) != nil {
+		return nil, nil, fmt.Errorf("failed to list snapshot properties: %w", err)
+	}
+
+	if kv != nil && kv.Props[id] != "" {
+		log.WithField("snapshot property", kv.Props[id]).Debug("found snapshot property")
+
+		parts := strings.SplitN(kv.Props[id], "/", 2)
+		if len(parts) != 2 {
+			return nil, nil, fmt.Errorf("failed to parse snapshot property: %s", kv.Props[id])
+		}
+
+		snap, err := s.client.Resources.GetSnapshot(ctx, parts[0], parts[1])
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to find snapshot: %w", err)
+		}
+
+		return &snap, nil, nil
+	}
+
+	log.Debug("no snapshot matching id found, trying S3 backups")
 
 	s3remotes, err := s.client.Remote.GetAllS3(ctx)
 	if err != nil {

diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go
@@ -26,6 +26,9 @@ const (
 	// DriverName is the name used in CSI calls for this driver.
 	DriverName = "linstor.csi.linbit.com"
 
+	// LinstorBackupKVName is the name of the KV store used to map L2L backups to local snapshot names
+	LinstorBackupKVName = "csi-backup-mapping"
+
 	// LegacyParameterPassKey is the Aux props key in linstor where serialized CSI parameters
 	// are stored.
 	LegacyParameterPassKey = lc.NamespcAuxiliary + "/csi-volume-annotations"

diff --git a/pkg/volume/snapshot_params.go b/pkg/volume/snapshot_params.go
@@ -21,16 +21,19 @@ const (
 )
 
 type SnapshotParameters struct {
-	Type             SnapshotType `json:"type,omitempty"`
-	AllowIncremental bool         `json:"allow-incremental"`
-	RemoteName       string       `json:"remote-name,omitempty"`
-	DeleteLocal      bool         `json:"delete-local,omitempty"`
-	S3Endpoint       string       `json:"s3-endpoint,omitempty"`
-	S3Bucket         string       `json:"s3-bucket,omitempty"`
-	S3SigningRegion  string       `json:"s3-signing-region,omitempty"`
-	S3UsePathStyle   bool         `json:"s3-use-path-style"`
-	S3AccessKey      string       `json:"-"`
-	S3SecretKey      string       `json:"-"`
+	Type                     SnapshotType `json:"type,omitempty"`
+	AllowIncremental         bool         `json:"allow-incremental"`
+	RemoteName               string       `json:"remote-name,omitempty"`
+	DeleteLocal              bool         `json:"delete-local,omitempty"`
+	S3Endpoint               string       `json:"s3-endpoint,omitempty"`
+	S3Bucket                 string       `json:"s3-bucket,omitempty"`
+	S3SigningRegion          string       `json:"s3-signing-region,omitempty"`
+	S3UsePathStyle           bool         `json:"s3-use-path-style"`
+	S3AccessKey              string       `json:"-"`
+	S3SecretKey              string       `json:"-"`
+	LinstorTargetUrl         string       `json:"linstor-target-url,omitempty"`
+	LinstorTargetClusterID   string       `json:"linstor-target-cluster-id,omitempty"`
+	LinstorTargetStoragePool string       `json:"linstor-target-storage-pool,omitempty"`
 }
 
 func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParameters, error) {
@@ -81,6 +84,12 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete
 			}
 
 			p.S3UsePathStyle = b
+		case "/linstor-target-url":
+			p.LinstorTargetUrl = v
+		case "/linstor-target-cluster-id":
+			p.LinstorTargetClusterID = v
+		case "/linstor-target-storage-pool":
+			p.LinstorTargetStoragePool = v
 		default:
 			log.WithField("key", k).Warn("ignoring unknown snapshot parameter key")
 		}
@@ -103,14 +112,17 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete
 func (s *SnapshotParameters) String() string {
 	// NB: we use a value here instead of a pointer, so we don't recurse endlessly.
 	return fmt.Sprint(SnapshotParameters{
-		Type:             s.Type,
-		AllowIncremental: s.AllowIncremental,
-		RemoteName:       s.RemoteName,
-		S3Endpoint:       s.S3Endpoint,
-		S3Bucket:         s.S3Bucket,
-		S3SigningRegion:  s.S3SigningRegion,
-		S3UsePathStyle:   s.S3UsePathStyle,
-		S3AccessKey:      "***",
-		S3SecretKey:      "***",
+		Type:                     s.Type,
+		AllowIncremental:         s.AllowIncremental,
+		RemoteName:               s.RemoteName,
+		S3Endpoint:               s.S3Endpoint,
+		S3Bucket:                 s.S3Bucket,
+		S3SigningRegion:          s.S3SigningRegion,
+		S3UsePathStyle:           s.S3UsePathStyle,
+		S3AccessKey:              "***",
+		S3SecretKey:              "***",
+		LinstorTargetUrl:         s.LinstorTargetUrl,
+		LinstorTargetClusterID:   s.LinstorTargetClusterID,
+		LinstorTargetStoragePool: s.LinstorTargetStoragePool,
 	})
 }