From 17f50fb4bf45fc1f6a92b3f8526a1af3ab5600dd Mon Sep 17 00:00:00 2001 From: Radostin Stoyanov Date: Fri, 18 Dec 2020 19:31:16 +0000 Subject: [PATCH 1/4] Use Options as exportCheckpoint() argument Instead of individual values from ContainerCheckpointOptions, provide the options object. This is a preparation for the next patch where one more value of the options object is required in exportCheckpoint(). Signed-off-by: Radostin Stoyanov --- libpod/container_internal_linux.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index cefe122099..9dd39ac8cb 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -790,11 +790,11 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr return nil } -func (c *Container) exportCheckpoint(dest string, ignoreRootfs bool) error { +func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error { if (len(c.config.NamedVolumes) > 0) || (len(c.Dependencies()) > 0) { return errors.Errorf("Cannot export checkpoints of containers with named volumes or dependencies") } - logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), dest) + logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), options.TargetFile) includeFiles := []string{ "checkpoint", @@ -807,7 +807,7 @@ func (c *Container) exportCheckpoint(dest string, ignoreRootfs bool) error { // Get root file-system changes included in the checkpoint archive rootfsDiffPath := filepath.Join(c.bundlePath(), "rootfs-diff.tar") deleteFilesList := filepath.Join(c.bundlePath(), "deleted.files") - if !ignoreRootfs { + if !options.IgnoreRootfs { // To correctly track deleted files, let's go through the output of 'podman diff' tarFiles, err := c.runtime.GetDiff("", c.ID()) if err != nil { @@ -880,13 +880,13 @@ func (c *Container) exportCheckpoint(dest string, ignoreRootfs bool) error { return errors.Wrapf(err, "error reading checkpoint directory %q", c.ID()) } - outFile, err := os.Create(dest) + outFile, err := os.Create(options.TargetFile) if err != nil { - return errors.Wrapf(err, "error creating checkpoint export file %q", dest) + return errors.Wrapf(err, "error creating checkpoint export file %q", options.TargetFile) } defer outFile.Close() - if err := os.Chmod(dest, 0600); err != nil { + if err := os.Chmod(options.TargetFile, 0600); err != nil { return err } @@ -963,7 +963,7 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO defer c.newContainerEvent(events.Checkpoint) if options.TargetFile != "" { - if err = c.exportCheckpoint(options.TargetFile, options.IgnoreRootfs); err != nil { + if err = c.exportCheckpoint(options); err != nil { return err } } From 2b35876c8d03ddc12e8becd2364c39cd621e191d Mon Sep 17 00:00:00 2001 From: Radostin Stoyanov Date: Fri, 25 Dec 2020 14:15:22 +0000 Subject: [PATCH 2/4] Use Options as CRImportCheckpoint() argument Instead of specifying restore option arguments individually from RestoreOptions, provide the 'options' object to the CRImportCheckpoint method. This change makes the code in CRImportCheckpoint easier to extend as it doesn't require excessive number of function parameters. Signed-off-by: Radostin Stoyanov --- pkg/checkpoint/checkpoint_restore.go | 11 ++++++----- pkg/domain/infra/abi/containers.go | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/checkpoint/checkpoint_restore.go b/pkg/checkpoint/checkpoint_restore.go index 9de04266f6..90f629354e 100644 --- a/pkg/checkpoint/checkpoint_restore.go +++ b/pkg/checkpoint/checkpoint_restore.go @@ -8,6 +8,7 @@ import ( "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/libpod/image" + "github.com/containers/podman/v2/pkg/domain/entities" "github.com/containers/podman/v2/pkg/errorhandling" "github.com/containers/podman/v2/pkg/util" "github.com/containers/storage/pkg/archive" @@ -36,10 +37,10 @@ func crImportFromJSON(filePath string, v interface{}) error { // CRImportCheckpoint it the function which imports the information // from checkpoint tarball and re-creates the container from that information -func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input string, name string) ([]*libpod.Container, error) { +func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOptions entities.RestoreOptions) ([]*libpod.Container, error) { // First get the container definition from the // tarball to a temporary directory - archiveFile, err := os.Open(input) + archiveFile, err := os.Open(restoreOptions.Import) if err != nil { return nil, errors.Wrap(err, "failed to open checkpoint archive for import") } @@ -66,7 +67,7 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input stri }() err = archive.Untar(archiveFile, dir, options) if err != nil { - return nil, errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input) + return nil, errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", restoreOptions.Import) } // Load spec.dump from temporary directory @@ -90,9 +91,9 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input stri newName := false // Check if the restored container gets a new name - if name != "" { + if restoreOptions.Name != "" { config.ID = "" - config.Name = name + config.Name = restoreOptions.Name newName = true } diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 9d7c2daea5..721a8c3ab8 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -538,7 +538,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st switch { case options.Import != "": - cons, err = checkpoint.CRImportCheckpoint(ctx, ic.Libpod, options.Import, options.Name) + cons, err = checkpoint.CRImportCheckpoint(ctx, ic.Libpod, options) case options.All: cons, err = ic.Libpod.GetContainers(filterFuncs...) default: From 288ccc4c84e917df0ff0ee659f47e3ecbc87796a Mon Sep 17 00:00:00 2001 From: Radostin Stoyanov Date: Fri, 18 Dec 2020 20:07:08 +0000 Subject: [PATCH 3/4] Include named volumes in container migration When migrating a container with associated volumes, the content of these volumes should be made available on the destination machine. This patch enables container checkpoint/restore with named volumes by including the content of volumes in checkpoint file. On restore, volumes associated with container are created and their content is restored. The --ignore-volumes option is introduced to disable this feature. Example: # podman container checkpoint --export checkpoint.tar.gz The content of all volumes associated with the container are included in `checkpoint.tar.gz` # podman container checkpoint --export checkpoint.tar.gz --ignore-volumes The content of volumes is not included in `checkpoint.tar.gz`. This is useful, for example, when the checkpoint/restore is performed on the same machine. # podman container restore --import checkpoint.tar.gz The associated volumes will be created and their content will be restored. Podman will exit with an error if volumes with the same name already exist on the system or the content of volumes is not included in checkpoint.tar.gz # podman container restore --ignore-volumes --import checkpoint.tar.gz Volumes associated with container must already exist. Podman will not create them or restore their content. Signed-off-by: Radostin Stoyanov --- cmd/podman/containers/checkpoint.go | 4 + cmd/podman/containers/restore.go | 4 + .../markdown/podman-container-checkpoint.1.md | 6 ++ .../markdown/podman-container-restore.1.md | 7 ++ libpod/container_api.go | 3 + libpod/container_internal_linux.go | 73 ++++++++++++++++++- pkg/api/handlers/libpod/containers.go | 1 + pkg/checkpoint/checkpoint_restore.go | 18 ++++- pkg/domain/entities/containers.go | 2 + pkg/domain/infra/abi/containers.go | 2 + 10 files changed, 116 insertions(+), 4 deletions(-) diff --git a/cmd/podman/containers/checkpoint.go b/cmd/podman/containers/checkpoint.go index b6dc213487..4a477eb10f 100644 --- a/cmd/podman/containers/checkpoint.go +++ b/cmd/podman/containers/checkpoint.go @@ -57,6 +57,7 @@ func init() { _ = checkpointCommand.RegisterFlagCompletionFunc(exportFlagName, completion.AutocompleteDefault) flags.BoolVar(&checkpointOptions.IgnoreRootFS, "ignore-rootfs", false, "Do not include root file-system changes when exporting") + flags.BoolVar(&checkpointOptions.IgnoreVolumes, "ignore-volumes", false, "Do not export volumes associated with container") validate.AddLatestFlag(checkpointCommand, &checkpointOptions.Latest) } @@ -68,6 +69,9 @@ func checkpoint(cmd *cobra.Command, args []string) error { if checkpointOptions.Export == "" && checkpointOptions.IgnoreRootFS { return errors.Errorf("--ignore-rootfs can only be used with --export") } + if checkpointOptions.Export == "" && checkpointOptions.IgnoreVolumes { + return errors.Errorf("--ignore-volumes can only be used with --export") + } responses, err := registry.ContainerEngine().ContainerCheckpoint(context.Background(), args, checkpointOptions) if err != nil { return err diff --git a/cmd/podman/containers/restore.go b/cmd/podman/containers/restore.go index 6a1d2b3190..5245a68fa2 100644 --- a/cmd/podman/containers/restore.go +++ b/cmd/podman/containers/restore.go @@ -62,6 +62,7 @@ func init() { flags.BoolVar(&restoreOptions.IgnoreRootFS, "ignore-rootfs", false, "Do not apply root file-system changes when importing from exported checkpoint") flags.BoolVar(&restoreOptions.IgnoreStaticIP, "ignore-static-ip", false, "Ignore IP address set via --static-ip") flags.BoolVar(&restoreOptions.IgnoreStaticMAC, "ignore-static-mac", false, "Ignore MAC address set via --mac-address") + flags.BoolVar(&restoreOptions.IgnoreVolumes, "ignore-volumes", false, "Do not export volumes associated with container") validate.AddLatestFlag(restoreCommand, &restoreOptions.Latest) } @@ -73,6 +74,9 @@ func restore(_ *cobra.Command, args []string) error { if restoreOptions.Import == "" && restoreOptions.IgnoreRootFS { return errors.Errorf("--ignore-rootfs can only be used with --import") } + if restoreOptions.Import == "" && restoreOptions.IgnoreVolumes { + return errors.Errorf("--ignore-volumes can only be used with --import") + } if restoreOptions.Import == "" && restoreOptions.Name != "" { return errors.Errorf("--name can only be used with --import") } diff --git a/docs/source/markdown/podman-container-checkpoint.1.md b/docs/source/markdown/podman-container-checkpoint.1.md index bfda782c50..6a94691562 100644 --- a/docs/source/markdown/podman-container-checkpoint.1.md +++ b/docs/source/markdown/podman-container-checkpoint.1.md @@ -52,6 +52,12 @@ exported to a tar.gz file it is possible with the help of **--ignore-rootfs** to explicitly disable including changes to the root file-system into the checkpoint archive file. +#### **--ignore-volumes** + +This option must be used in combination with the **--export, -e** option. +When this option is specified, the content of volumes associated with +the container will not be included into the checkpoint tar.gz file. + ## EXAMPLE podman container checkpoint mywebserver diff --git a/docs/source/markdown/podman-container-restore.1.md b/docs/source/markdown/podman-container-restore.1.md index 494e7db1ec..0593e6fe9f 100644 --- a/docs/source/markdown/podman-container-restore.1.md +++ b/docs/source/markdown/podman-container-restore.1.md @@ -85,6 +85,13 @@ exported checkpoint with **--name, -n**. Using **--ignore-static-mac** tells Podman to ignore the MAC address if it was configured with **--mac-address** during container creation. + +#### **--ignore-volumes** + +This option must be used in combination with the **--import, -i** option. +When restoring containers from a checkpoint tar.gz file with this option, +the content of associated volumes will not be restored. + ## EXAMPLE podman container restore mywebserver diff --git a/libpod/container_api.go b/libpod/container_api.go index c3e1a23d23..2c7dc79c9c 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -703,6 +703,9 @@ type ContainerCheckpointOptions struct { // important to be able to restore a container multiple // times with '--import --name'. IgnoreStaticMAC bool + // IgnoreVolumes tells the API to not export or not to import + // the content of volumes associated with the container + IgnoreVolumes bool } // Checkpoint checkpoints a container diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 9dd39ac8cb..f947d1ed98 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -791,8 +791,8 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr } func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error { - if (len(c.config.NamedVolumes) > 0) || (len(c.Dependencies()) > 0) { - return errors.Errorf("Cannot export checkpoints of containers with named volumes or dependencies") + if len(c.Dependencies()) > 0 { + return errors.Errorf("Cannot export checkpoints of containers with dependencies") } logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), options.TargetFile) @@ -870,6 +870,47 @@ func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error { } } + // Folder containing archived volumes that will be included in the export + expVolDir := filepath.Join(c.bundlePath(), "volumes") + + // Create an archive for each volume associated with the container + if !options.IgnoreVolumes { + if err := os.MkdirAll(expVolDir, 0700); err != nil { + return errors.Wrapf(err, "error creating volumes export directory %q", expVolDir) + } + + for _, v := range c.config.NamedVolumes { + volumeTarFilePath := filepath.Join("volumes", v.Name+".tar") + volumeTarFileFullPath := filepath.Join(c.bundlePath(), volumeTarFilePath) + + volumeTarFile, err := os.Create(volumeTarFileFullPath) + if err != nil { + return errors.Wrapf(err, "error creating %q", volumeTarFileFullPath) + } + + volume, err := c.runtime.GetVolume(v.Name) + if err != nil { + return err + } + + input, err := archive.TarWithOptions(volume.MountPoint(), &archive.TarOptions{ + Compression: archive.Uncompressed, + IncludeSourceDir: true, + }) + if err != nil { + return errors.Wrapf(err, "error reading volume directory %q", v.Dest) + } + + _, err = io.Copy(volumeTarFile, input) + if err != nil { + return err + } + volumeTarFile.Close() + + includeFiles = append(includeFiles, volumeTarFilePath) + } + } + input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{ Compression: archive.Gzip, IncludeSourceDir: true, @@ -898,6 +939,10 @@ func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error { os.Remove(rootfsDiffPath) os.Remove(deleteFilesList) + if !options.IgnoreVolumes { + os.RemoveAll(expVolDir) + } + return nil } @@ -1193,6 +1238,30 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti return err } + // When restoring from an imported archive, allow restoring the content of volumes. + // Volumes are created in setupContainer() + if options.TargetFile != "" && !options.IgnoreVolumes { + for _, v := range c.config.NamedVolumes { + volumeFilePath := filepath.Join(c.bundlePath(), "volumes", v.Name+".tar") + + volumeFile, err := os.Open(volumeFilePath) + if err != nil { + return errors.Wrapf(err, "Failed to open volume file %s", volumeFilePath) + } + defer volumeFile.Close() + + volume, err := c.runtime.GetVolume(v.Name) + if err != nil { + return errors.Wrapf(err, "Failed to retrieve volume %s", v.Name) + } + + mountPoint := volume.MountPoint() + if err := archive.UntarUncompressed(volumeFile, mountPoint, nil); err != nil { + return errors.Wrapf(err, "Failed to extract volume %s to %s", volumeFilePath, mountPoint) + } + } + } + // Before actually restarting the container, apply the root file-system changes if !options.IgnoreRootfs { rootfsDiffPath := filepath.Join(c.bundlePath(), "rootfs-diff.tar") diff --git a/pkg/api/handlers/libpod/containers.go b/pkg/api/handlers/libpod/containers.go index 14eb44831e..6b07b1cc5d 100644 --- a/pkg/api/handlers/libpod/containers.go +++ b/pkg/api/handlers/libpod/containers.go @@ -275,6 +275,7 @@ func Restore(w http.ResponseWriter, r *http.Request) { Import bool `schema:"import"` Name string `schema:"name"` IgnoreRootFS bool `schema:"ignoreRootFS"` + IgnoreVolumes bool `schema:"ignoreVolumes"` IgnoreStaticIP bool `schema:"ignoreStaticIP"` IgnoreStaticMAC bool `schema:"ignoreStaticMAC"` }{ diff --git a/pkg/checkpoint/checkpoint_restore.go b/pkg/checkpoint/checkpoint_restore.go index 90f629354e..f6cd3b38f5 100644 --- a/pkg/checkpoint/checkpoint_restore.go +++ b/pkg/checkpoint/checkpoint_restore.go @@ -54,6 +54,7 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOpt "rootfs-diff.tar", "network.status", "deleted.files", + "volumes", }, } dir, err := ioutil.TempDir("", "checkpoint") @@ -83,8 +84,21 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOpt } // This should not happen as checkpoints with these options are not exported. - if (len(config.Dependencies) > 0) || (len(config.NamedVolumes) > 0) { - return nil, errors.Errorf("Cannot import checkpoints of containers with named volumes or dependencies") + if len(config.Dependencies) > 0 { + return nil, errors.Errorf("Cannot import checkpoints of containers with dependencies") + } + + // Volumes included in the checkpoint should not exist + if !restoreOptions.IgnoreVolumes { + for _, vol := range config.NamedVolumes { + exists, err := runtime.HasVolume(vol.Name) + if err != nil { + return nil, err + } + if exists { + return nil, errors.Errorf("volume with name %s already exists. Use --ignore-volumes to not restore content of volumes", vol.Name) + } + } } ctrID := config.ID diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index 05b9b774eb..a67ecebd56 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -173,6 +173,7 @@ type CheckpointOptions struct { All bool Export string IgnoreRootFS bool + IgnoreVolumes bool Keep bool Latest bool LeaveRunning bool @@ -187,6 +188,7 @@ type CheckpointReport struct { type RestoreOptions struct { All bool IgnoreRootFS bool + IgnoreVolumes bool IgnoreStaticIP bool IgnoreStaticMAC bool Import string diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 721a8c3ab8..f7a5389340 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -487,6 +487,7 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [ TCPEstablished: options.TCPEstablished, TargetFile: options.Export, IgnoreRootfs: options.IgnoreRootFS, + IgnoreVolumes: options.IgnoreVolumes, KeepRunning: options.LeaveRunning, } @@ -525,6 +526,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st TargetFile: options.Import, Name: options.Name, IgnoreRootfs: options.IgnoreRootFS, + IgnoreVolumes: options.IgnoreVolumes, IgnoreStaticIP: options.IgnoreStaticIP, IgnoreStaticMAC: options.IgnoreStaticMAC, } From 1215bd9ffd9ac068f30212a68f36c7d77542f92d Mon Sep 17 00:00:00 2001 From: Radostin Stoyanov Date: Tue, 29 Dec 2020 09:03:08 +0000 Subject: [PATCH 4/4] test: Add checkpoint/restore with volumes Signed-off-by: Radostin Stoyanov --- .../build/basicalpine/Containerfile.volume | 2 + test/e2e/checkpoint_test.go | 95 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 test/e2e/build/basicalpine/Containerfile.volume diff --git a/test/e2e/build/basicalpine/Containerfile.volume b/test/e2e/build/basicalpine/Containerfile.volume new file mode 100644 index 0000000000..6a4fc82420 --- /dev/null +++ b/test/e2e/build/basicalpine/Containerfile.volume @@ -0,0 +1,2 @@ +FROM alpine +VOLUME "/volume0" diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go index 75310b9610..4c6b2cf5c6 100644 --- a/test/e2e/checkpoint_test.go +++ b/test/e2e/checkpoint_test.go @@ -652,4 +652,99 @@ var _ = Describe("Podman checkpoint", func() { // Remove exported checkpoint os.Remove(fileName) }) + + It("podman checkpoint a container with volumes", func() { + session := podmanTest.Podman([]string{ + "build", "-f", "build/basicalpine/Containerfile.volume", "-t", "test-cr-volume", + }) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Start the container + localRunString := getRunString([]string{ + "--rm", + "-v", "/volume1", + "-v", "my-test-vol:/volume2", + "test-cr-volume", + "top", + }) + session = podmanTest.Podman(localRunString) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + + cid := session.OutputToString() + + // Add file in volume0 + result := podmanTest.Podman([]string{ + "exec", "-l", "/bin/sh", "-c", "echo " + cid + " > /volume0/test.output", + }) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + + // Add file in volume1 + result = podmanTest.Podman([]string{ + "exec", "-l", "/bin/sh", "-c", "echo " + cid + " > /volume1/test.output", + }) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + + // Add file in volume2 + result = podmanTest.Podman([]string{ + "exec", "-l", "/bin/sh", "-c", "echo " + cid + " > /volume2/test.output", + }) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + + checkpointFileName := "/tmp/checkpoint-" + cid + ".tar.gz" + + // Checkpoint the container + result = podmanTest.Podman([]string{"container", "checkpoint", "-l", "-e", checkpointFileName}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + Expect(podmanTest.NumberOfContainers()).To(Equal(0)) + + // Restore container should fail because named volume still exists + result = podmanTest.Podman([]string{"container", "restore", "-i", checkpointFileName}) + result.WaitWithDefaultTimeout() + Expect(result).To(ExitWithError()) + Expect(result.ErrorToString()).To(ContainSubstring( + "volume with name my-test-vol already exists. Use --ignore-volumes to not restore content of volumes", + )) + + // Remove named volume + session = podmanTest.Podman([]string{"volume", "rm", "my-test-vol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Restoring container + result = podmanTest.Podman([]string{"container", "restore", "-i", checkpointFileName}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + Expect(podmanTest.NumberOfContainers()).To(Equal(1)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + + // Validate volume0 content + result = podmanTest.Podman([]string{"exec", "-l", "cat", "/volume0/test.output"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(result.OutputToString()).To(ContainSubstring(cid)) + + // Validate volume1 content + result = podmanTest.Podman([]string{"exec", "-l", "cat", "/volume1/test.output"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(result.OutputToString()).To(ContainSubstring(cid)) + + // Validate volume2 content + result = podmanTest.Podman([]string{"exec", "-l", "cat", "/volume2/test.output"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(result.OutputToString()).To(ContainSubstring(cid)) + + // Remove exported checkpoint + os.Remove(checkpointFileName) + }) })