diff --git a/pkg/hostpath/controllerserver.go b/pkg/hostpath/controllerserver.go index 4c007659a..8ff0025a0 100644 --- a/pkg/hostpath/controllerserver.go +++ b/pkg/hostpath/controllerserver.go @@ -34,7 +34,6 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "k8s.io/klog/v2" - utilexec "k8s.io/utils/exec" "github.com/kubernetes-csi/csi-driver-host-path/pkg/state" ) @@ -543,21 +542,10 @@ func (hp *hostPath) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotR snapshotID := uuid.NewUUID().String() creationTime := ptypes.TimestampNow() - volPath := hostPathVolume.VolPath file := hp.getSnapshotPath(snapshotID) - var cmd []string - if hostPathVolume.VolAccessType == state.BlockAccess { - glog.V(4).Infof("Creating snapshot of Raw Block Mode Volume") - cmd = []string{"cp", volPath, file} - } else { - glog.V(4).Infof("Creating snapshot of Filsystem Mode Volume") - cmd = []string{"tar", "czf", file, "-C", volPath, "."} - } - executor := utilexec.New() - out, err := executor.Command(cmd[0], cmd[1:]...).CombinedOutput() - if err != nil { - return nil, fmt.Errorf("failed create snapshot: %w: %s", err, out) + if err := hp.createSnapshotFromVolume(hostPathVolume, file); err != nil { + return nil, err } glog.V(4).Infof("create volume snapshot %s", file) @@ -601,6 +589,11 @@ func (hp *hostPath) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotR hp.mutex.Lock() defer hp.mutex.Unlock() + // If the snapshot has a GroupSnapshotID, deletion is not allowed and should return InvalidArgument. + if snapshot, err := hp.state.GetSnapshotByID(snapshotID); err != nil && snapshot.GroupSnapshotID != "" { + return nil, status.Errorf(codes.InvalidArgument, "Snapshot with ID %s is part of groupsnapshot %s", snapshotID, snapshot.GroupSnapshotID) + } + glog.V(4).Infof("deleting snapshot %s", snapshotID) path := hp.getSnapshotPath(snapshotID) os.RemoveAll(path) @@ -651,11 +644,12 @@ func (hp *hostPath) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsReq for _, snap := range hpSnapshots { snapshot := csi.Snapshot{ - SnapshotId: snap.Id, - SourceVolumeId: snap.VolID, - CreationTime: snap.CreationTime, - SizeBytes: snap.SizeBytes, - ReadyToUse: snap.ReadyToUse, + SnapshotId: snap.Id, + SourceVolumeId: snap.VolID, + CreationTime: snap.CreationTime, + SizeBytes: snap.SizeBytes, + ReadyToUse: snap.ReadyToUse, + GroupSnapshotId: snap.GroupSnapshotID, } snapshots = append(snapshots, snapshot) } @@ -767,11 +761,12 @@ func convertSnapshot(snap state.Snapshot) *csi.ListSnapshotsResponse { entries := []*csi.ListSnapshotsResponse_Entry{ { Snapshot: &csi.Snapshot{ - SnapshotId: snap.Id, - SourceVolumeId: snap.VolID, - CreationTime: snap.CreationTime, - SizeBytes: snap.SizeBytes, - ReadyToUse: snap.ReadyToUse, + SnapshotId: snap.Id, + SourceVolumeId: snap.VolID, + CreationTime: snap.CreationTime, + SizeBytes: snap.SizeBytes, + ReadyToUse: snap.ReadyToUse, + GroupSnapshotId: snap.GroupSnapshotID, }, }, } diff --git a/pkg/hostpath/groupcontrollerserver.go b/pkg/hostpath/groupcontrollerserver.go new file mode 100644 index 000000000..a6905bca7 --- /dev/null +++ b/pkg/hostpath/groupcontrollerserver.go @@ -0,0 +1,281 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hostpath + +import ( + "os" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" + "github.com/golang/protobuf/ptypes" + "github.com/pborman/uuid" + "golang.org/x/net/context" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/kubernetes-csi/csi-driver-host-path/pkg/state" +) + +func (hp *hostPath) GroupControllerGetCapabilities(context.Context, *csi.GroupControllerGetCapabilitiesRequest) (*csi.GroupControllerGetCapabilitiesResponse, error) { + return &csi.GroupControllerGetCapabilitiesResponse{ + Capabilities: []*csi.GroupControllerServiceCapability{{ + Type: &csi.GroupControllerServiceCapability_Rpc{ + Rpc: &csi.GroupControllerServiceCapability_RPC{ + Type: csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT, + }, + }, + }}, + }, nil +} + +func (hp *hostPath) CreateVolumeGroupSnapshot(ctx context.Context, req *csi.CreateVolumeGroupSnapshotRequest) (*csi.CreateVolumeGroupSnapshotResponse, error) { + if err := hp.validateGroupControllerServiceRequest(csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT); err != nil { + glog.V(3).Infof("invalid create volume group snapshot req: %v", req) + return nil, err + } + + if len(req.GetName()) == 0 { + return nil, status.Error(codes.InvalidArgument, "Name missing in request") + } + // Check arguments + if len(req.GetSourceVolumeIds()) == 0 { + return nil, status.Error(codes.InvalidArgument, "SourceVolumeIds missing in request") + } + + // Lock before acting on global state. A production-quality + // driver might use more fine-grained locking. + hp.mutex.Lock() + defer hp.mutex.Unlock() + + // Need to check for already existing groupsnapshot name, and if found check for the + // requested sourceVolumeIds and sourceVolumeIds of groupsnapshot that has been created. + if exGS, err := hp.state.GetGroupSnapshotByName(req.GetName()); err == nil { + // Since err is nil, it means the groupsnapshot with the same name already exists. Need + // to check if the sourceVolumeIds of existing groupsnapshot is the same as in new request. + + if !exGS.MatchesSourceVolumeIDs(req.GetSourceVolumeIds()) { + return nil, status.Errorf(codes.AlreadyExists, "group snapshot with the same name: %s but with different SourceVolumeIds already exist", req.GetName()) + } + + // same groupsnapshot has been created. + snapshots := make([]*csi.Snapshot, len(exGS.SnapshotIDs)) + readyToUse := true + + for i, snapshotID := range exGS.SnapshotIDs { + snapshot, err := hp.state.GetSnapshotByID(snapshotID) + if err != nil { + return nil, err + } + + snapshots[i] = &csi.Snapshot{ + SizeBytes: snapshot.SizeBytes, + CreationTime: snapshot.CreationTime, + ReadyToUse: snapshot.ReadyToUse, + GroupSnapshotId: snapshot.GroupSnapshotID, + } + + readyToUse = readyToUse && snapshot.ReadyToUse + } + + return &csi.CreateVolumeGroupSnapshotResponse{ + GroupSnapshot: &csi.VolumeGroupSnapshot{ + GroupSnapshotId: exGS.Id, + Snapshots: snapshots, + CreationTime: exGS.CreationTime, + ReadyToUse: readyToUse, + }, + }, nil + } + + groupSnapshot := state.GroupSnapshot{ + Name: req.GetName(), + Id: uuid.NewUUID().String(), + CreationTime: ptypes.TimestampNow(), + SnapshotIDs: make([]string, len(req.GetSourceVolumeIds())), + SourceVolumeIDs: make([]string, len(req.GetSourceVolumeIds())), + ReadyToUse: true, + } + + copy(groupSnapshot.SourceVolumeIDs, req.GetSourceVolumeIds()) + + snapshots := make([]*csi.Snapshot, len(req.GetSourceVolumeIds())) + + // TODO: defer a cleanup function to remove snapshots in case of a failure + + for i, volumeID := range req.GetSourceVolumeIds() { + hostPathVolume, err := hp.state.GetVolumeByID(volumeID) + if err != nil { + return nil, err + } + + snapshotID := uuid.NewUUID().String() + file := hp.getSnapshotPath(snapshotID) + + if err := hp.createSnapshotFromVolume(hostPathVolume, file); err != nil { + return nil, err + } + + glog.V(4).Infof("create volume snapshot %s", file) + snapshot := state.Snapshot{} + snapshot.Name = req.GetName() + "-" + volumeID + snapshot.Id = snapshotID + snapshot.VolID = volumeID + snapshot.Path = file + snapshot.CreationTime = groupSnapshot.CreationTime + snapshot.SizeBytes = hostPathVolume.VolSize + snapshot.ReadyToUse = true + snapshot.GroupSnapshotID = groupSnapshot.Id + + hp.state.UpdateSnapshot(snapshot) + + groupSnapshot.SnapshotIDs[i] = snapshotID + + snapshots[i] = &csi.Snapshot{ + SizeBytes: hostPathVolume.VolSize, + SnapshotId: snapshotID, + SourceVolumeId: volumeID, + CreationTime: groupSnapshot.CreationTime, + ReadyToUse: true, + GroupSnapshotId: groupSnapshot.Id, + } + } + + if err := hp.state.UpdateGroupSnapshot(groupSnapshot); err != nil { + return nil, err + } + + return &csi.CreateVolumeGroupSnapshotResponse{ + GroupSnapshot: &csi.VolumeGroupSnapshot{ + GroupSnapshotId: groupSnapshot.Id, + Snapshots: snapshots, + CreationTime: groupSnapshot.CreationTime, + ReadyToUse: groupSnapshot.ReadyToUse, + }, + }, nil +} + +func (hp *hostPath) DeleteVolumeGroupSnapshot(ctx context.Context, req *csi.DeleteVolumeGroupSnapshotRequest) (*csi.DeleteVolumeGroupSnapshotResponse, error) { + if err := hp.validateGroupControllerServiceRequest(csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT); err != nil { + glog.V(3).Infof("invalid delete volume group snapshot req: %v", req) + return nil, err + } + + // Check arguments + if len(req.GetGroupSnapshotId()) == 0 { + return nil, status.Error(codes.InvalidArgument, "GroupSnapshot ID missing in request") + } + + groupSnapshotID := req.GetGroupSnapshotId() + + // Lock before acting on global state. A production-quality + // driver might use more fine-grained locking. + hp.mutex.Lock() + defer hp.mutex.Unlock() + + groupSnapshot, err := hp.state.GetGroupSnapshotByID(groupSnapshotID) + if err != nil { + // ok if NotFound, the VolumeGroupSnapshot was deleted already + if status.Code(err) == codes.NotFound { + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil + } + + return nil, err + } + + for _, snapshotID := range groupSnapshot.SnapshotIDs { + glog.V(4).Infof("deleting snapshot %s", snapshotID) + path := hp.getSnapshotPath(snapshotID) + os.RemoveAll(path) + + if err := hp.state.DeleteSnapshot(snapshotID); err != nil { + return nil, err + } + } + + glog.V(4).Infof("deleting groupsnapshot %s", groupSnapshotID) + if err := hp.state.DeleteGroupSnapshot(groupSnapshotID); err != nil { + return nil, err + } + + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil +} + +func (hp *hostPath) GetVolumeGroupSnapshot(ctx context.Context, req *csi.GetVolumeGroupSnapshotRequest) (*csi.GetVolumeGroupSnapshotResponse, error) { + if err := hp.validateGroupControllerServiceRequest(csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT); err != nil { + glog.V(3).Infof("invalid get volume group snapshot req: %v", req) + return nil, err + } + + groupSnapshotID := req.GetGroupSnapshotId() + + // Check arguments + if len(groupSnapshotID) == 0 { + return nil, status.Error(codes.InvalidArgument, "GroupSnapshot ID missing in request") + } + + // Lock before acting on global state. A production-quality + // driver might use more fine-grained locking. + hp.mutex.Lock() + defer hp.mutex.Unlock() + + groupSnapshot, err := hp.state.GetGroupSnapshotByID(groupSnapshotID) + if err != nil { + return nil, err + } + + if !groupSnapshot.MatchesSourceVolumeIDs(req.GetSnapshotIds()) { + return nil, status.Error(codes.InvalidArgument, "Snapshot IDs do not match the GroupSnapshot IDs") + } + + snapshots := make([]*csi.Snapshot, len(groupSnapshot.SnapshotIDs)) + for i, snapshotID := range groupSnapshot.SnapshotIDs { + snapshot, err := hp.state.GetSnapshotByID(snapshotID) + if err != nil { + return nil, err + } + + snapshots[i] = &csi.Snapshot{ + SizeBytes: snapshot.SizeBytes, + SnapshotId: snapshotID, + SourceVolumeId: snapshot.VolID, + CreationTime: snapshot.CreationTime, + ReadyToUse: snapshot.ReadyToUse, + GroupSnapshotId: snapshot.GroupSnapshotID, + } + } + + return &csi.GetVolumeGroupSnapshotResponse{ + GroupSnapshot: &csi.VolumeGroupSnapshot{ + GroupSnapshotId: groupSnapshotID, + Snapshots: snapshots, + CreationTime: groupSnapshot.CreationTime, + ReadyToUse: groupSnapshot.ReadyToUse, + }, + }, nil +} + +func (hp *hostPath) validateGroupControllerServiceRequest(c csi.GroupControllerServiceCapability_RPC_Type) error { + if c == csi.GroupControllerServiceCapability_RPC_UNKNOWN { + return nil + } + + if c == csi.GroupControllerServiceCapability_RPC_CREATE_DELETE_GET_VOLUME_GROUP_SNAPSHOT { + return nil + } + + return status.Errorf(codes.InvalidArgument, "unsupported capability %s", c) +} diff --git a/pkg/hostpath/hostpath.go b/pkg/hostpath/hostpath.go index 6958e9631..1488094ff 100644 --- a/pkg/hostpath/hostpath.go +++ b/pkg/hostpath/hostpath.go @@ -123,7 +123,7 @@ func NewHostPathDriver(cfg Config) (*hostPath, error) { func (hp *hostPath) Run() error { s := NewNonBlockingGRPCServer() // hp itself implements ControllerServer, NodeServer, and IdentityServer. - s.Start(hp.config.Endpoint, hp, hp, hp) + s.Start(hp.config.Endpoint, hp, hp, hp, hp) s.Wait() return nil @@ -377,3 +377,21 @@ func (hp *hostPath) getAttachCount() int64 { } return count } + +func (hp *hostPath) createSnapshotFromVolume(vol state.Volume, file string) error { + var cmd []string + if vol.VolAccessType == state.BlockAccess { + glog.V(4).Infof("Creating snapshot of Raw Block Mode Volume") + cmd = []string{"cp", vol.VolPath, file} + } else { + glog.V(4).Infof("Creating snapshot of Filsystem Mode Volume") + cmd = []string{"tar", "czf", file, "-C", vol.VolPath, "."} + } + executor := utilexec.New() + out, err := executor.Command(cmd[0], cmd[1:]...).CombinedOutput() + if err != nil { + return fmt.Errorf("failed create snapshot: %w: %s", err, out) + } + + return nil +} diff --git a/pkg/hostpath/identityserver.go b/pkg/hostpath/identityserver.go index 05225d800..8bf69edc9 100644 --- a/pkg/hostpath/identityserver.go +++ b/pkg/hostpath/identityserver.go @@ -55,6 +55,13 @@ func (hp *hostPath) GetPluginCapabilities(ctx context.Context, req *csi.GetPlugi }, }, }, + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_GROUP_CONTROLLER_SERVICE, + }, + }, + }, } if hp.config.EnableTopology { caps = append(caps, &csi.PluginCapability{ diff --git a/pkg/hostpath/server.go b/pkg/hostpath/server.go index 8687fe191..01b5f911d 100644 --- a/pkg/hostpath/server.go +++ b/pkg/hostpath/server.go @@ -40,11 +40,11 @@ type nonBlockingGRPCServer struct { cleanup func() } -func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { +func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, gcs csi.GroupControllerServer) { s.wg.Add(1) - go s.serve(endpoint, ids, cs, ns) + go s.serve(endpoint, ids, cs, ns, gcs) return } @@ -63,7 +63,7 @@ func (s *nonBlockingGRPCServer) ForceStop() { s.cleanup() } -func (s *nonBlockingGRPCServer) serve(ep string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { +func (s *nonBlockingGRPCServer) serve(ep string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, gcs csi.GroupControllerServer) { listener, cleanup, err := endpoint.Listen(ep) if err != nil { glog.Fatalf("Failed to listen: %v", err) @@ -85,6 +85,9 @@ func (s *nonBlockingGRPCServer) serve(ep string, ids csi.IdentityServer, cs csi. if ns != nil { csi.RegisterNodeServer(server, ns) } + if gcs != nil { + csi.RegisterGroupControllerServer(server, gcs) + } glog.Infof("Listening for connections on address: %#v", listener.Addr()) diff --git a/pkg/state/state.go b/pkg/state/state.go index 0068ca3f5..a1429530f 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "os" + "sort" timestamp "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/grpc/codes" @@ -58,13 +59,23 @@ type Volume struct { } type Snapshot struct { - Name string - Id string - VolID string - Path string - CreationTime *timestamp.Timestamp - SizeBytes int64 - ReadyToUse bool + Name string + Id string + VolID string + Path string + CreationTime *timestamp.Timestamp + SizeBytes int64 + ReadyToUse bool + GroupSnapshotID string +} + +type GroupSnapshot struct { + Name string + Id string + SnapshotIDs []string + SourceVolumeIDs []string + CreationTime *timestamp.Timestamp + ReadyToUse bool } // State is the interface that the rest of the code has to use to @@ -112,11 +123,32 @@ type State interface { // snapshot ID. It is not an error when such a snapshot // does not exist. DeleteSnapshot(snapshotID string) error + + // GetGroupSnapshotByID retrieves a groupsnapshot by its unique ID or + // returns an error including that ID when not found. + GetGroupSnapshotByID(vgsID string) (GroupSnapshot, error) + + // GetGroupSnapshotByName retrieves a groupsnapshot by its name or + // returns an error including that name when not found. + GetGroupSnapshotByName(volName string) (GroupSnapshot, error) + + // GetGroupSnapshots returns all currently existing groupsnapshots. + GetGroupSnapshots() []GroupSnapshot + + // UpdateGroupSnapshot updates the existing hostpath groupsnapshot, + // identified by its snapshot ID, or adds it if it does not exist yet. + UpdateGroupSnapshot(snapshot GroupSnapshot) error + + // DeleteGroupSnapshot deletes the groupsnapshot with the given + // groupsnapshot ID. It is not an error when such a groupsnapshot does + // not exist. + DeleteGroupSnapshot(groupSnapshotID string) error } type resources struct { - Volumes []Volume - Snapshots []Snapshot + Volumes []Volume + Snapshots []Snapshot + GroupSnapshots []GroupSnapshot } type state struct { @@ -257,3 +289,70 @@ func (s *state) DeleteSnapshot(snapshotID string) error { } return nil } + +func (s *state) GetGroupSnapshotByID(groupSnapshotID string) (GroupSnapshot, error) { + for _, groupSnapshot := range s.GroupSnapshots { + if groupSnapshot.Id == groupSnapshotID { + return groupSnapshot, nil + } + } + return GroupSnapshot{}, status.Errorf(codes.NotFound, "groupsnapshot id %s does not exist in the groupsnapshots list", groupSnapshotID) +} + +func (s *state) GetGroupSnapshotByName(name string) (GroupSnapshot, error) { + for _, groupSnapshot := range s.GroupSnapshots { + if groupSnapshot.Name == name { + return groupSnapshot, nil + } + } + return GroupSnapshot{}, status.Errorf(codes.NotFound, "groupsnapshot name %s does not exist in the groupsnapshots list", name) +} + +func (s *state) GetGroupSnapshots() []GroupSnapshot { + groupSnapshots := make([]GroupSnapshot, len(s.GroupSnapshots)) + for i, groupSnapshot := range s.GroupSnapshots { + groupSnapshots[i] = groupSnapshot + } + return groupSnapshots +} + +func (s *state) UpdateGroupSnapshot(update GroupSnapshot) error { + for i, groupSnapshot := range s.GroupSnapshots { + if groupSnapshot.Id == update.Id { + s.GroupSnapshots[i] = update + return s.dump() + } + } + s.GroupSnapshots = append(s.GroupSnapshots, update) + return s.dump() +} + +func (s *state) DeleteGroupSnapshot(groupSnapshotID string) error { + for i, groupSnapshot := range s.GroupSnapshots { + if groupSnapshot.Id == groupSnapshotID { + s.GroupSnapshots = append(s.GroupSnapshots[:i], s.GroupSnapshots[i+1:]...) + return s.dump() + } + } + return nil +} + +func (gs *GroupSnapshot) MatchesSourceVolumeIDs(sourceVolumeIDs []string) bool { + stateSourceVolumeIDs := gs.SourceVolumeIDs + + if len(stateSourceVolumeIDs) != len(sourceVolumeIDs) { + return false + } + + // sort slices so that values are at the same location + sort.Strings(stateSourceVolumeIDs) + sort.Strings(sourceVolumeIDs) + + for i, v := range stateSourceVolumeIDs { + if v != sourceVolumeIDs[i] { + return false + } + } + + return true +} diff --git a/pkg/state/state_test.go b/pkg/state/state_test.go index f18ca7223..259f54d93 100644 --- a/pkg/state/state_test.go +++ b/pkg/state/state_test.go @@ -124,3 +124,34 @@ func TestSnapshotsFromSameSource(t *testing.T) { _, err = s.GetSnapshotByName("bar-name") require.NoError(t, err, "get existing snapshot by name 'bar-name'") } + +func TestVolumeGroupSnapshots(t *testing.T) { + tmp := t.TempDir() + statefileName := path.Join(tmp, "state.json") + + s, err := New(statefileName) + require.NoError(t, err, "construct state") + require.Empty(t, s.GetGroupSnapshots(), "initial groupsnapshots") + + _, err = s.GetGroupSnapshotByID("foo") + require.Equal(t, codes.NotFound, status.Convert(err).Code(), "GetGroupSnapshotByID of non-existent groupsnapshot") + require.Contains(t, status.Convert(err).Message(), "foo") + + err = s.UpdateGroupSnapshot(GroupSnapshot{Id: "foo", Name: "bar"}) + require.NoError(t, err, "add groupsnapshot") + + s, err = New(statefileName) + require.NoError(t, err, "reconstruct state") + _, err = s.GetGroupSnapshotByID("foo") + require.NoError(t, err, "get existing groupsnapshot by ID") + _, err = s.GetGroupSnapshotByName("bar") + require.NoError(t, err, "get existing groupsnapshot by name") + + err = s.DeleteGroupSnapshot("foo") + require.NoError(t, err, "delete existing groupsnapshot") + + err = s.DeleteGroupSnapshot("foo") + require.NoError(t, err, "delete non-existent groupsnapshot") + + require.Empty(t, s.GetGroupSnapshots(), "final groupsnapshots") +}