implement L2L backup shipping

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Support for creating backups with LINSTOR-to-LINSTOR shipping.
+
 ## [1.5.0] - 2024-03-19
 
 ### Added

go.mod

@@ -5,7 +5,7 @@ go 1.21
 toolchain go1.21.5
 
 require (
-	github.com/LINBIT/golinstor v0.50.0
+	github.com/LINBIT/golinstor v0.51.0
 	github.com/container-storage-interface/spec v1.9.0
 	github.com/haySwim/data v0.2.0
 	github.com/kubernetes-csi/csi-test/v5 v5.2.0

go.sum

@@ -1,5 +1,5 @@
-github.com/LINBIT/golinstor v0.50.0 h1:WLdk+Jca/6BSOgmGaFqYrGEZlh0D7kjps4ak20Ntj80=
-github.com/LINBIT/golinstor v0.50.0/go.mod h1:MCkHNdHxoGw4mnt8DGsSqWNF5ZGhYFy6Lr4tQLyVBs0=
+github.com/LINBIT/golinstor v0.51.0 h1:XvfcQN3jg7b/s79wrpTe/jzfFDGSNZy2pjGG8SDtFRM=
+github.com/LINBIT/golinstor v0.51.0/go.mod h1:MCkHNdHxoGw4mnt8DGsSqWNF5ZGhYFy6Lr4tQLyVBs0=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=

pkg/client/linstor.go

@@ -854,7 +854,48 @@ func (s *Linstor) reconcileBackup(ctx context.Context, id, sourceVolId string, p
 
 		return &snap, nil
 	case volume.SnapshotTypeLinstor:
-		return nil, fmt.Errorf("linstor-to-linstor snapshots not implemented")
+		kv, err := s.client.KeyValueStore.Get(ctx, linstor.LinstorBackupKVName)
+		if nil404(err) != nil {
+			return nil, fmt.Errorf("error checking for existing LINSTOR backup: %w", err)
+		}
+
+		var snapName string
+
+		if kv == nil || kv.Props[id] == "" {
+			yes := true
+
+			snapName, err = s.client.Backup.Ship(ctx, params.RemoteName, lapi.BackupShipRequest{
+				SrcRscName:   sourceVolId,
+				DstRscName:   "backup-" + sourceVolId,
+				DstStorPool:  params.LinstorTargetStoragePool,
+				DownloadOnly: &yes,
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error creating LINSTOR backup: %w", err)
+			}
+
+			// Store the name of the created snapshot in the LINSTOR KV. We need this because if some later stage fails,
+			// for example because the snapshot did not get ready in time, we do not want to create a second backup.
+			// Instead, we check the KV for the provided snapshot ID, which maps to the hopefully existing local
+			// snapshot name.
+			err = s.client.KeyValueStore.CreateOrModify(ctx, linstor.LinstorBackupKVName, lapi.GenericPropsModify{
+				OverrideProps: map[string]string{
+					id: fmt.Sprintf("%s/%s", sourceVolId, snapName),
+				},
+			})
+			if err != nil {
+				return nil, fmt.Errorf("error storing LINSTOR backup name property: %w", err)
+			}
+		} else {
+			snapName = kv.Props[id]
+		}
+
+		snap, err := s.client.Resources.GetSnapshot(ctx, sourceVolId, snapName)
+		if err != nil {
+			return nil, fmt.Errorf("error fetching snapshot for backup: %w", err)
+		}
+
+		return &snap, nil
 	default:
 		return nil, fmt.Errorf("unsupported snapshot type '%s', don't know how to create a backup", params.Type)
 	}
@@ -896,7 +937,32 @@ func (s *Linstor) reconcileRemote(ctx context.Context, params *volume.SnapshotPa
 
 		return nil
 	case volume.SnapshotTypeLinstor:
-		return fmt.Errorf("Linstor-to-Linstor snapshots not implemented")
+		log.Debug("search for LINSTOR remote with matching name")
+
+		remotes, err := s.client.Remote.GetAllLinstor(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to list existing remotes: %w", err)
+		}
+
+		for _, r := range remotes {
+			if r.RemoteName == params.RemoteName {
+				log.WithField("remote", r).Debug("found existing LINSTOR remote with matching name")
+				return nil
+			}
+		}
+
+		log.Debug("No existing remote found, creating a new one")
+
+		err = s.client.Remote.CreateLinstor(ctx, lapi.LinstorRemote{
+			RemoteName: params.RemoteName,
+			Url:        params.LinstorTargetUrl,
+			ClusterId:  params.LinstorTargetClusterID,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to create new LINSTOR remote: %w", err)
+		}
+
+		return nil
 	default:
 		return fmt.Errorf("unsupported snapshot type '%s', don't know how to configure remote", params.Type)
 	}
@@ -910,8 +976,12 @@ func (s *Linstor) SnapDelete(ctx context.Context, snap *volume.Snapshot) error {
 		log.WithField("remote", snap.Remote).Debug("deleting backup from remote")
 
 		backups, err := s.client.Backup.GetAll(ctx, snap.Remote, snap.GetSourceVolumeId(), snap.GetSnapshotId())
-		if nil404(err) != nil {
-			return fmt.Errorf("failed to list backups for snapshot %s: %w", snap.GetSnapshotId(), err)
+		if err != nil {
+			var apiErrors lapi.ApiCallError
+			// Make sure this is actually an S3 backup
+			if !errors.As(err, &apiErrors) || !apiErrors.Is(lapiconsts.FailInvldRemoteName) {
+				return fmt.Errorf("failed to list backups for snapshot %s: %w", snap.GetSnapshotId(), err)
+			}
 		}
 
 		if backups != nil {
@@ -940,10 +1010,21 @@ func (s *Linstor) SnapDelete(ctx context.Context, snap *volume.Snapshot) error {
 	}
 
 	err = s.deleteResourceDefinitionAndGroupIfUnused(ctx, snap.GetSourceVolumeId())
+	if nil404(err) != nil {
+		return fmt.Errorf("failed to remove resource definition: %w", err)
+	}
 
 	return nil
 }
 
+func (s *Linstor) DeleteTemporarySnapshotID(ctx context.Context, id string) error {
+	err := s.client.KeyValueStore.CreateOrModify(ctx, linstor.LinstorBackupKVName, lapi.GenericPropsModify{
+		DeleteProps: []string{id},
+	})
+
+	return nil404(err)
+}
+
 // VolFromSnap creates the volume using the data contained within the snapshot.
 func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *volume.Info, params *volume.Parameters, snapParams *volume.SnapshotParameters, topologies *csi.TopologyRequirement) error {
 	logger := s.log.WithFields(logrus.Fields{
@@ -1483,7 +1564,30 @@ func (s *Linstor) snapOrBackupById(ctx context.Context, id string) (*lapi.Snapsh
 		}
 	}
 
-	log.Debug("no snapshot matching id found, trying backups")
+	log.Debug("no snapshot matching id found, trying LINSTOR backups")
+
+	kv, err := s.client.KeyValueStore.Get(ctx, linstor.LinstorBackupKVName)
+	if nil404(err) != nil {
+		return nil, nil, fmt.Errorf("failed to list snapshot properties: %w", err)
+	}
+
+	if kv != nil && kv.Props[id] != "" {
+		log.WithField("snapshot property", kv.Props[id]).Debug("found snapshot property")
+
+		parts := strings.SplitN(kv.Props[id], "/", 2)
+		if len(parts) != 2 {
+			return nil, nil, fmt.Errorf("failed to parse snapshot property: %s", kv.Props[id])
+		}
+
+		snap, err := s.client.Resources.GetSnapshot(ctx, parts[0], parts[1])
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to find snapshot: %w", err)
+		}
+
+		return &snap, nil, nil
+	}
+
+	log.Debug("no snapshot matching id found, trying S3 backups")
 
 	s3remotes, err := s.client.Remote.GetAllS3(ctx)
 	if err != nil {

pkg/client/mock.go

@@ -148,6 +148,10 @@ func (s *MockStorage) SnapDelete(ctx context.Context, snap *volume.Snapshot) err
 	return nil
 }
 
+func (s *MockStorage) DeleteTemporarySnapshotID(ctx context.Context, id string) error {
+	return nil
+}
+
 func (s *MockStorage) ListSnaps(ctx context.Context, start, limit int) ([]*volume.Snapshot, error) {
 	if limit == 0 {
 		limit = len(s.snapshots) - start

pkg/driver/driver.go

@@ -967,6 +967,15 @@ func (d Driver) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotReque
 				return nil, status.Errorf(codes.Internal, "failed to delete local snapshot: %s", err)
 			}
 
+			if existingSnap.ReadyToUse {
+				d.log.WithField("snapshot id", id).Debug("snapshot ready, delete temporary ID mapping if it exists")
+
+				err := d.Snapshots.DeleteTemporarySnapshotID(ctx, id)
+				if err != nil {
+					return nil, status.Errorf(codes.Internal, "failed to delete temporary snapshot ID: %v", err)
+				}
+			}
+
 			return &csi.CreateSnapshotResponse{Snapshot: &existingSnap.Snapshot}, nil
 		}
 
@@ -984,6 +993,15 @@ func (d Driver) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotReque
 		return nil, status.Errorf(codes.Internal, "failed to delete local snapshot: %s", err)
 	}
 
+	if snap.ReadyToUse {
+		d.log.WithField("snapshot id", id).Debug("snapshot ready, delete temporary ID mapping if it exists")
+
+		err := d.Snapshots.DeleteTemporarySnapshotID(ctx, id)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "failed to delete temporary snapshot ID: %v", err)
+		}
+	}
+
 	return &csi.CreateSnapshotResponse{Snapshot: &snap.Snapshot}, nil
 }
 

pkg/linstor/const.go

@@ -26,6 +26,9 @@ const (
 	// DriverName is the name used in CSI calls for this driver.
 	DriverName = "linstor.csi.linbit.com"
 
+	// LinstorBackupKVName is the name of the KV store used to map L2L backups to local snapshot names
+	LinstorBackupKVName = "csi-backup-mapping"
+
 	// LegacyParameterPassKey is the Aux props key in linstor where serialized CSI parameters
 	// are stored.
 	LegacyParameterPassKey = lc.NamespcAuxiliary + "/csi-volume-annotations"

pkg/volume/snapshot_params.go

@@ -21,16 +21,19 @@ const (
 )
 
 type SnapshotParameters struct {
-	Type             SnapshotType `json:"type,omitempty"`
-	AllowIncremental bool         `json:"allow-incremental"`
-	RemoteName       string       `json:"remote-name,omitempty"`
-	DeleteLocal      bool         `json:"delete-local,omitempty"`
-	S3Endpoint       string       `json:"s3-endpoint,omitempty"`
-	S3Bucket         string       `json:"s3-bucket,omitempty"`
-	S3SigningRegion  string       `json:"s3-signing-region,omitempty"`
-	S3UsePathStyle   bool         `json:"s3-use-path-style"`
-	S3AccessKey      string       `json:"-"`
-	S3SecretKey      string       `json:"-"`
+	Type                     SnapshotType `json:"type,omitempty"`
+	AllowIncremental         bool         `json:"allow-incremental"`
+	RemoteName               string       `json:"remote-name,omitempty"`
+	DeleteLocal              bool         `json:"delete-local,omitempty"`
+	S3Endpoint               string       `json:"s3-endpoint,omitempty"`
+	S3Bucket                 string       `json:"s3-bucket,omitempty"`
+	S3SigningRegion          string       `json:"s3-signing-region,omitempty"`
+	S3UsePathStyle           bool         `json:"s3-use-path-style"`
+	S3AccessKey              string       `json:"-"`
+	S3SecretKey              string       `json:"-"`
+	LinstorTargetUrl         string       `json:"linstor-target-url,omitempty"`
+	LinstorTargetClusterID   string       `json:"linstor-target-cluster-id,omitempty"`
+	LinstorTargetStoragePool string       `json:"linstor-target-storage-pool,omitempty"`
 }
 
 func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParameters, error) {
@@ -81,6 +84,12 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete
 			}
 
 			p.S3UsePathStyle = b
+		case "/linstor-target-url":
+			p.LinstorTargetUrl = v
+		case "/linstor-target-cluster-id":
+			p.LinstorTargetClusterID = v
+		case "/linstor-target-storage-pool":
+			p.LinstorTargetStoragePool = v
 		default:
 			log.WithField("key", k).Warn("ignoring unknown snapshot parameter key")
 		}
@@ -103,14 +112,17 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete
 func (s *SnapshotParameters) String() string {
 	// NB: we use a value here instead of a pointer, so we don't recurse endlessly.
 	return fmt.Sprint(SnapshotParameters{
-		Type:             s.Type,
-		AllowIncremental: s.AllowIncremental,
-		RemoteName:       s.RemoteName,
-		S3Endpoint:       s.S3Endpoint,
-		S3Bucket:         s.S3Bucket,
-		S3SigningRegion:  s.S3SigningRegion,
-		S3UsePathStyle:   s.S3UsePathStyle,
-		S3AccessKey:      "***",
-		S3SecretKey:      "***",
+		Type:                     s.Type,
+		AllowIncremental:         s.AllowIncremental,
+		RemoteName:               s.RemoteName,
+		S3Endpoint:               s.S3Endpoint,
+		S3Bucket:                 s.S3Bucket,
+		S3SigningRegion:          s.S3SigningRegion,
+		S3UsePathStyle:           s.S3UsePathStyle,
+		S3AccessKey:              "***",
+		S3SecretKey:              "***",
+		LinstorTargetUrl:         s.LinstorTargetUrl,
+		LinstorTargetClusterID:   s.LinstorTargetClusterID,
+		LinstorTargetStoragePool: s.LinstorTargetStoragePool,
 	})
 }

pkg/volume/volume.go

@@ -81,6 +81,8 @@ type SnapshotCreateDeleter interface {
 	ListSnaps(ctx context.Context, start, limit int) ([]*Snapshot, error)
 	// VolFromSnap creates a new volume based on the provided snapshot.
 	VolFromSnap(ctx context.Context, snap *Snapshot, vol *Info, params *Parameters, snapParams *SnapshotParameters, topologies *csi.TopologyRequirement) error
+	// DeleteTemporarySnapshotID deletes the temporary snapshot ID.
+	DeleteTemporarySnapshotID(ctx context.Context, id string) error
 }
 
 // AttacherDettacher handles operations relating to volume accessiblity on nodes.