fuse: automatically use squashfuse for images, deprecate --sif-fuse

Deprecate the explicit `--sif-fuse` flag and `sif fuse` directive for
`singularity.conf`. These were previously used to enable experimental
FUSE mount of SIF/SquashFS containers.

Modify image handling so that we now try squashfuse mounts
automatically, with fall back to temporary sandbox extraction, when:

* squashfs kernel mounts have been disabled in `singularity.conf`
* we are running in a non-setuid / user namespace flow.

Add a `--tmp-sandbox` flag to allow forcing extraction to a temporary
sandbox when a kernel mount or FUSE mount would otherwise be used.

Fixes sylabs#2216

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## Changes Since Last Release
 
+### Changed defaults / behaviours
+
+- In native mode, SIF/SquashFS container images will now be mounted with
+  squashfuse when kernel mounts are disabled in `singularity.conf`, or cannot be
+  used (non-setuid / user namespace workflow). If the FUSE mount fails,
+  Singularity will fall back to extracting the container to a temporary sandbox
+  in order to run it.
+
 ### New Features & Functionality
 
 - The `registry login` and `registry logout` commands now support a `--authfile
@@ -27,10 +35,18 @@
   executable, then the `run` / `exec` / `shell` commands in `--oci` mode can be
   given the `--app <appname>` flag, and will automatically invoke the relevant
   SCIF command.
-- SIF/SquashFS container images can now be mounted using FUSE in all native mode
-  flows, including setuid mode. To enable, use the `--sif-fuse` flag, or set
-  `sif fuse = yes` in `singularity.conf`. Overlay partitions and extfs images
-  are not yet supported.
+- A new `--tmp-sandbox` flag has been added to the `run / shell / exec /
+  instance start` commands. This will force Singularity to extract a container
+  to a temporary sandbox before running it, when it would otherwise perform a
+  kernel or FUSE mount.
+
+### Deprecated Functionality
+
+- The experimental `--sif-fuse` flag, and `sif fuse` directive in
+  `singularity.conf` are deprecated. The flag and directive were used to enable
+  experimental mounting of SIF/SquashFS container images with FUSE in prior
+  versions of Singularity. From 4.1, FUSE mounts are used automatically when
+  kernel mounts are disabled / not available.
 
 ### Bug Fixes
 

cmd/internal/cli/action_flags.go

@@ -766,6 +766,7 @@ var actionSIFFUSEFlag = cmdline.Flag{
 	Name:         "sif-fuse",
 	Usage:        "attempt FUSE mount of SIF",
 	EnvKeys:      []string{"SIF_FUSE"},
+	Deprecated:   "FUSE mounts are now used automatically when kernel mounts are disabled / unavailable.",
 }
 
 // --proot (hidden)
@@ -891,6 +892,7 @@ func init() {
 		cmdManager.RegisterFlagForCmd(&commonOCIFlag, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&commonNoOCIFlag, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&commonKeepLayersFlag, actionsInstanceCmd...)
+		cmdManager.RegisterFlagForCmd(&actionTmpSandbox, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&actionNoTmpSandbox, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&commonAuthFileFlag, actionsInstanceCmd...)
 		cmdManager.RegisterFlagForCmd(&actionDevice, actionsCmd...)

cmd/internal/cli/actions.go

@@ -387,6 +387,7 @@ func launchContainer(cmd *cobra.Command, ep launcher.ExecParams) error {
 		launcher.OptDevice(device),
 		launcher.OptCdiDirs(cdiDirs),
 		launcher.OptNoCompat(noCompat),
+		launcher.OptTmpSandbox(tmpSandbox),
 		launcher.OptNoTmpSandbox(noTmpSandbox),
 	}
 

cmd/internal/cli/singularity.go

@@ -84,6 +84,7 @@ var (
 
 	// Options controlling the unpacking of images to temporary sandboxes
 	canUseTmpSandbox bool
+	tmpSandbox       bool
 	noTmpSandbox     bool
 
 	// Use OCI runtime and OCI SIF?
@@ -305,6 +306,16 @@ var commonKeepLayersFlag = cmdline.Flag{
 	EnvKeys:      []string{"KEEP_LAYERS"},
 }
 
+// --tmp-sandbox
+var actionTmpSandbox = cmdline.Flag{
+	ID:           "actionTmpSandbox",
+	Value:        &tmpSandbox,
+	DefaultValue: false,
+	Name:         "tmp-sandbox",
+	Usage:        "Forces unpacking of images into temporary sandbox dirs when a kernel or FUSE mount would otherwise be used.",
+	EnvKeys:      []string{"TMP_SANDBOX"},
+}
+
 // --no-tmp-sandbox
 var actionNoTmpSandbox = cmdline.Flag{
 	ID:           "actionNoTmpSandbox",

e2e/actions/actions.go

@@ -2507,8 +2507,10 @@ func (c actionTests) actionCompat(t *testing.T) {
 	}
 }
 
-// actionSquashfuse tests that squashfuse SIF mount works.
-func (c actionTests) actionSIFFUSE(t *testing.T) {
+// actionFUSEImage tests that squashfuse SIF mount works. Currently forced here
+// via deprecated `--sif-fuse` flag as this is convenient to include non-userns
+// profiles without changing global config.
+func (c actionTests) actionFUSEImage(t *testing.T) {
 	require.Command(t, "squashfuse")
 	require.Command(t, "fusermount")
 	e2e.EnsureImage(t, c.env)
@@ -2521,11 +2523,14 @@ func (c actionTests) actionSIFFUSE(t *testing.T) {
 			e2e.AsSubtest(p.String()),
 			e2e.WithProfile(e2e.UserNamespaceProfile),
 			e2e.WithCommand("exec"),
+			e2e.WithGlobalOptions("-d"),
 			e2e.WithArgs("--sif-fuse", c.env.ImagePath, "ps"),
 			e2e.ExpectExit(
 				0,
 				e2e.ExpectOutput(e2e.ContainMatch, "squashfuse"),
 				e2e.ExpectError(e2e.ContainMatch, "Mounting image with FUSE"),
+				e2e.ExpectError(e2e.ContainMatch, "PostStartHost()"),
+				e2e.ExpectError(e2e.ContainMatch, "CleanupHost()"),
 			),
 		)
 
@@ -2536,45 +2541,59 @@ func (c actionTests) actionSIFFUSE(t *testing.T) {
 	}
 }
 
-// Verify that the FUSE mounts, and the CleanupHost() process are not seen when
-// --sif-fuse should not be in effect.
-func (c actionTests) actionNoSIFFUSE(t *testing.T) {
+// Verify that the FUSE mounts, and the PostStartHost/CleanupHost() processes are not seen when
+// FUSE mounts of a SIF image should not be in effect.
+func (c actionTests) actionNoFUSEImage(t *testing.T) {
 	e2e.EnsureImage(t, c.env)
 
-	for _, p := range e2e.NativeProfiles {
+	for _, p := range []e2e.Profile{e2e.RootProfile, e2e.UserProfile} {
 		c.env.RunSingularity(
 			t,
 			e2e.AsSubtest(p.String()),
 			e2e.WithProfile(p),
 			e2e.WithCommand("exec"),
 			e2e.WithGlobalOptions("-d"),
-			e2e.WithArgs(c.env.ImagePath, "mount"),
+			e2e.WithArgs(c.env.ImagePath, "ps"),
 			e2e.ExpectExit(
 				0,
 				e2e.ExpectError(e2e.UnwantedContainMatch, "squashfuse"),
+				e2e.ExpectError(e2e.UnwantedContainMatch, "PostStartHost()"),
 				e2e.ExpectError(e2e.UnwantedContainMatch, "CleanupHost()"),
 			),
 		)
 	}
 }
 
-// actionTmpSandboxFlag tests the command-line option prohibiting unpacking of image
+// actionTmpSandboxFlag tests the command-line options forcing / prohibiting unpacking of image
 // files into temporary sandbox dirs.
 func (c actionTests) actionTmpSandboxFlag(t *testing.T) {
 	e2e.EnsureImage(t, c.env)
 
-	profiles := []e2e.Profile{e2e.UserProfile, e2e.RootProfile, e2e.FakerootProfile, e2e.UserNamespaceProfile}
-
-	for _, p := range profiles {
-		c.env.RunSingularity(
-			t,
-			e2e.AsSubtest(p.String()),
-			e2e.WithProfile(p),
-			e2e.WithCommand("exec"),
-			e2e.WithArgs("--sif-fuse=false", "--no-tmp-sandbox", "-u", c.env.ImagePath, "/bin/true"),
-			e2e.ExpectExit(255),
-		)
+	// --tmp-sandbox should override kernel mount (setuid profiles) and squashfuse mount (userns profiles).
+	for _, p := range e2e.NativeProfiles {
+		t.Run(p.String(), func(t *testing.T) {
+			c.env.RunSingularity(
+				t,
+				e2e.AsSubtest("tmp-sandbox"),
+				e2e.WithProfile(p),
+				e2e.WithCommand("exec"),
+				e2e.WithArgs("--tmp-sandbox", c.env.ImagePath, "/bin/sh", "-c", "echo $SINGULARITY_CONTAINER"),
+				e2e.ExpectExit(0,
+					e2e.ExpectOutput(e2e.RegexMatch, `/rootfs-\d+/root$`), // <tmpdir>/rootfs-xxxxxxxxx/root
+					e2e.ExpectError(e2e.ContainMatch, "Converting SIF file to temporary sandbox"),
+				),
+			)
+		})
 	}
+
+	c.env.RunSingularity(
+		t,
+		e2e.AsSubtest("no-tmp-sandbox"),
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("exec"),
+		e2e.WithArgs("--tmp-sandbox", "--no-tmp-sandbox", c.env.ImagePath, "/bin/sh", "-c", "echo $SINGULARITY_CONTAINER"),
+		e2e.ExpectExit(255),
+	)
 }
 
 // Make sure --workdir and --scratch work together nicely even when workdir is a
@@ -2831,9 +2850,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 		"compat":                       np(c.actionCompat),               // test --compat
 		"umask":                        np(c.actionUmask),                // test umask propagation
 		"invalidRemote":                np(c.invalidRemote),              // GHSA-5mv9-q7fq-9394
-		"SIFFUSE":                      np(c.actionSIFFUSE),              // test --sif-fuse
-		"NoSIFFUSE":                    np(c.actionNoSIFFUSE),            // test absence of squashfs and CleanupHost()
-		"TmpSandboxFlag":               c.actionTmpSandboxFlag,           // test --no-tmp-sandbox flag
+		"FUSEImage":                    np(c.actionFUSEImage),            // test explicit FUSE image mount
+		"NoFUSEImage":                  np(c.actionNoFUSEImage),          // test absence of squashfuse and CleanupHost()
+		"TmpSandboxFlag":               c.actionTmpSandboxFlag,           // test --tmp-sandbox / --no-tmp-sandbox flag
 		"relWorkdirScratch":            np(c.relWorkdirScratch),          // test relative --workdir with --scratch
 		"ociRelWorkdirScratch":         np(c.actionOciRelWorkdirScratch), // test relative --workdir with --scratch in OCI mode
 		"auth":                         np(c.actionAuth),                 // tests action cmds w/authenticated pulls from OCI registries

e2e/config/config.go

@@ -533,7 +533,8 @@ func (c configTests) configGlobal(t *testing.T) {
 			profile:        e2e.UserProfile,
 			directive:      "allow kernel squashfs",
 			directiveValue: "no",
-			exit:           255,
+			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.ContainMatch, "Mounting image with FUSE"),
 		},
 		{
 			name:           "AllowKernelSquashfsYes_Container",
@@ -542,6 +543,7 @@ func (c configTests) configGlobal(t *testing.T) {
 			directive:      "allow kernel squashfs",
 			directiveValue: "yes",
 			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.UnwantedContainMatch, "Mounting image with FUSE"),
 		},
 		// Standalone ext3 rootfs
 		{
@@ -635,7 +637,8 @@ func (c configTests) configGlobal(t *testing.T) {
 			profile:        e2e.UserProfile,
 			directive:      "allow kernel squashfs",
 			directiveValue: "no",
-			exit:           255,
+			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.ContainMatch, "Mounting image with FUSE"),
 		},
 		{
 			name:           "AllowKernelSquashfsYes_SIF",
@@ -644,6 +647,7 @@ func (c configTests) configGlobal(t *testing.T) {
 			directive:      "allow kernel squashfs",
 			directiveValue: "yes",
 			exit:           0,
+			resultOp:       e2e.ExpectError(e2e.UnwantedContainMatch, "Mounting image with FUSE"),
 		},
 		// Encrypted squashFS rootfs in SIF
 		{

internal/pkg/runtime/launcher/native/launcher_linux.go

@@ -977,16 +977,17 @@ func (l *Launcher) setCgroups(instanceName string) error {
 	return nil
 }
 
-// PrepareImage perfoms any image preparation required before execution.
-// This is currently limited to extraction or FUSE mount when using the user namespace,
-// and activating any image driver plugins that might handle the image mount.
+// PrepareImage performs any image preparation required before execution.
+// This is currently limited to extraction or FUSE mount.
 func (l *Launcher) prepareImage(c context.Context, image string) error {
 	if strings.HasPrefix(image, "instance://") {
 		return nil
 	}
 
 	insideUserNs, _ := namespaces.IsInsideUserNamespace(os.Getpid())
 	isUserNs := insideUserNs || l.cfg.Namespaces.User
+	noKernelMount := l.cfg.TmpSandbox || isUserNs || l.cfg.SIFFUSE
+	tryFuse := !l.cfg.TmpSandbox
 
 	img, err := imgutil.Init(image, false)
 	if err != nil {
@@ -1002,20 +1003,20 @@ func (l *Launcher) prepareImage(c context.Context, image string) error {
 	case imgutil.SANDBOX:
 		return nil
 	case imgutil.SQUASHFS:
-		if isUserNs || l.cfg.SIFFUSE {
-			return l.prepareSquashfs(c, img, l.cfg.SIFFUSE)
+		if !l.engineConfig.File.AllowKernelSquashfs || noKernelMount {
+			return l.prepareSquashfs(c, img, tryFuse)
 		}
 		// setuid, kernel squashfs permitted, fuse not requested - no action needed
 		return nil
 	case imgutil.ENCRYPTSQUASHFS:
-		if isUserNs || l.cfg.SIFFUSE {
+		if !l.engineConfig.File.AllowKernelSquashfs || noKernelMount {
 			return fmt.Errorf("encrypted SIF files are only supported in setuid mode, with kernel mounts")
 		}
 		// setuid, kernel squashfs permitted, fuse not requested - no action needed
 		return nil
 	case imgutil.EXT3:
 		if isUserNs || l.cfg.SIFFUSE {
-			return l.prepareExtfs(c, img, l.cfg.SIFFUSE)
+			return l.prepareExtfs(c, img)
 		}
 		// setuid, kernel extfs permitted, fuse not requested - no action needed
 		return nil
@@ -1069,7 +1070,7 @@ func (l *Launcher) prepareSquashfs(ctx context.Context, img *imgutil.Image, tryF
 	return fmt.Errorf("extraction failed: %v", err)
 }
 
-func (l *Launcher) prepareExtfs(_ context.Context, _ *imgutil.Image, _ bool) error {
+func (l *Launcher) prepareExtfs(_ context.Context, _ *imgutil.Image) error {
 	// TODO - Enable fuse2fs handling
 	return fmt.Errorf("extfs images can only be run in setuid mode with kernel extfs mounts enabled")
 }

internal/pkg/runtime/launcher/options.go

@@ -149,6 +149,10 @@ type Options struct {
 	// This will be used by a launcher handling OCI images directly.
 	TransportOptions *ociimage.TransportOptions
 
+	// TmpSandbox forces unpacking of images into temporary sandbox dirs when a
+	// kernel or FUSE mount would otherwise be used.
+	TmpSandbox bool
+
 	// NoTmpSandbox prohibits unpacking of images into temporary sandbox dirs.
 	NoTmpSandbox bool
 
@@ -512,6 +516,15 @@ func OptSIFFuse(b bool) Option {
 	}
 }
 
+// TmpSandbox forces unpacking of images into temporary sandbox dirs when a
+// kernel or FUSE mount would otherwise be used.
+func OptTmpSandbox(b bool) Option {
+	return func(lo *Options) error {
+		lo.TmpSandbox = b
+		return nil
+	}
+}
+
 // OptNoTmpSandbox prohibits unpacking of images into temporary sandbox dirs.
 func OptNoTmpSandbox(b bool) Option {
 	return func(lo *Options) error {

pkg/util/singularityconf/config.go

@@ -329,7 +329,8 @@ allow container dir = {{ if eq .AllowContainerDir true }}yes{{ else }}no{{ end }
 # ALLOW KERNEL SQUASHFS: [BOOL]
 # DEFAULT: yes
 # If set to no, Singularity will not perform any kernel mounts of squashfs filesystems.
-# This affects both stand-alone image files and filesystems embedded in a SIF file.
+# Instead, for SIF / SquashFS containers, a squashfuse mount will be attempted, with
+# extraction to a temporary sandbox directory if this fails.
 # Applicable to setuid mode only.
 allow kernel squashfs = {{ if eq .AllowKernelSquashfs true }}yes{{ else }}no{{ end }}
 
@@ -523,7 +524,7 @@ systemd cgroups = {{ if eq .SystemdCgroups true }}yes{{ else }}no{{ end }}
 
 # SIF FUSE: [BOOL]
 # DEFAULT: no
-# EXPERIMENTAL
+# DEPRECATED - FUSE mounts are now used automatically when kernel mounts are disabled / unavailable.
 # Whether to try mounting SIF images with Squashfuse by default.
 # Applies only to unprivileged / user namespace flows. Requires squashfuse and
 # fusermount on PATH. Will fall back to extracting the SIF on failure.