Merge pull request #1378 from dfr/freebsd-chflags

Add support for exporting and importing file flags on FreeBSD

pkg/archive/archive.go

@@ -527,6 +527,9 @@ func (ta *tarAppender) addTarFile(path, name string) error {
 	if err := ReadUserXattrToTarHeader(path, hdr); err != nil {
 		return err
 	}
+	if err := ReadFileFlagsToTarHeader(path, hdr); err != nil {
+		return err
+	}
 	if ta.CopyPass {
 		copyPassHeader(hdr)
 	}
@@ -770,6 +773,15 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L
 
 	}
 
+	// We defer setting flags on directories until the end of
+	// Unpack or UnpackLayer in case setting them makes the
+	// directory immutable.
+	if hdr.Typeflag != tar.TypeDir {
+		if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil {
+			return err
+		}
+	}
+
 	if len(errs) > 0 {
 		logrus.WithFields(logrus.Fields{
 			"errors": errs,
@@ -1101,6 +1113,9 @@ loop:
 		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
 			return err
 		}
+		if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil {
+			return err
+		}
 	}
 	return nil
 }

pkg/archive/changes_bsd_test.go

@@ -0,0 +1,98 @@
+//go:build freebsd
+// +build freebsd
+
+package archive
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/system"
+	"github.com/stretchr/testify/require"
+)
+
+// Verify that file flag changes are reported
+func TestChangeFileFlags(t *testing.T) {
+	src := t.TempDir()
+	createSampleDir(t, src)
+
+	dst := src + "-copy"
+	err := copyDir(src, dst)
+	require.NoError(t, err)
+	file1 := path.Join(dst, "dir1/file1-1")
+	err = system.Lchflags(file1, system.UF_READONLY)
+	require.NoError(t, err)
+
+	changes, err := ChangesDirs(dst, &idtools.IDMappings{}, src, &idtools.IDMappings{})
+	require.NoError(t, err)
+
+	expectedChanges := []Change{
+		{"/dir1", ChangeModify},
+		{"/dir1/file1-1", ChangeModify},
+	}
+	checkChanges(t, expectedChanges, changes)
+}
+
+// Verify that file flag changes are copied
+func TestCopyFileFlags(t *testing.T) {
+	src := t.TempDir()
+	createSampleDir(t, src)
+	file1 := path.Join(src, "dir1/file1-1")
+	err := system.Lchflags(file1, system.UF_READONLY)
+	require.NoError(t, err)
+
+	dst := src + "-copy"
+	err = copyDir(src, dst)
+	require.NoError(t, err)
+
+	changes, err := ChangesDirs(dst, &idtools.IDMappings{}, src, &idtools.IDMappings{})
+	require.NoError(t, err)
+
+	if len(changes) != 0 {
+		t.Fatalf("Changes with no difference should have detect no changes, but detected %d", len(changes))
+	}
+}
+
+// Make sure we can apply changes to an immutable file, including deleting
+func TestApplyToImmutable(t *testing.T) {
+	// Make a directory with an immutable file
+	src := t.TempDir()
+	createSampleDir(t, src)
+	file1 := path.Join(src, "dir1/file1-1")
+	file2 := path.Join(src, "dir1/file1-2")
+	require.NoError(t, os.Chmod(file1, 0777))
+	require.NoError(t, system.Lchflags(file1, system.SF_IMMUTABLE))
+	require.NoError(t, system.Lchflags(file2, system.SF_IMMUTABLE))
+
+	// Copy it, and change file1, delete file2
+	dst := src + "-copy"
+	err := copyDir(src, dst)
+	require.NoError(t, err)
+	file1 = path.Join(dst, "dir1/file1-1")
+	file2 = path.Join(dst, "dir1/file1-2")
+	require.NoError(t, system.Lchflags(file1, 0))
+	require.NoError(t, os.Chmod(file1, 0666))
+	require.NoError(t, system.Lchflags(file2, 0))
+	require.NoError(t, os.RemoveAll(file2))
+
+	changes, err := ChangesDirs(dst, &idtools.IDMappings{}, src, &idtools.IDMappings{})
+	require.NoError(t, err)
+
+	layer, err := ExportChanges(dst, changes, nil, nil)
+	require.NoError(t, err)
+
+	layerCopy, err := NewTempArchive(layer, "")
+	require.NoError(t, err)
+
+	_, err = ApplyLayer(src, layerCopy)
+	require.NoError(t, err)
+
+	changes2, err := ChangesDirs(src, &idtools.IDMappings{}, dst, &idtools.IDMappings{})
+	require.NoError(t, err)
+
+	if len(changes2) != 0 {
+		t.Fatalf("Unexpected differences after reapplying mutation: %v", changes2)
+	}
+}

pkg/archive/changes_unix.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package archive
@@ -29,6 +30,7 @@ func statDifferent(oldStat *system.StatT, oldInfo *FileInfo, newStat *system.Sta
 	if oldStat.Mode() != newStat.Mode() ||
 		ownerChanged ||
 		oldStat.Rdev() != newStat.Rdev() ||
+		oldStat.Flags() != newStat.Flags() ||
 		// Don't look at size for dirs, its not a good measure of change
 		(oldStat.Mode()&unix.S_IFDIR != unix.S_IFDIR &&
 			(!sameFsTimeSpec(oldStat.Mtim(), newStat.Mtim()) || (oldStat.Size() != newStat.Size()))) {

pkg/archive/diff.go

@@ -145,6 +145,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
 						return nil
 					}
 					if _, exists := unpackedPaths[path]; !exists {
+						if err := resetImmutable(path, nil); err != nil {
+							return err
+						}
 						err := os.RemoveAll(path)
 						return err
 					}
@@ -156,6 +159,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
 			} else {
 				originalBase := base[len(WhiteoutPrefix):]
 				originalPath := filepath.Join(dir, originalBase)
+				if err := resetImmutable(originalPath, nil); err != nil {
+					return 0, err
+				}
 				if err := os.RemoveAll(originalPath); err != nil {
 					return 0, err
 				}
@@ -165,7 +171,15 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
 			// The only exception is when it is a directory *and* the file from
 			// the layer is also a directory. Then we want to merge them (i.e.
 			// just apply the metadata from the layer).
+			//
+			// We always reset the immutable flag (if present) to allow metadata
+			// changes and to allow directory modification. The flag will be
+			// re-applied based on the contents of hdr either at the end for
+			// directories or in createTarFile otherwise.
 			if fi, err := os.Lstat(path); err == nil {
+				if err := resetImmutable(path, &fi); err != nil {
+					return 0, err
+				}
 				if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) {
 					if err := os.RemoveAll(path); err != nil {
 						return 0, err
@@ -215,6 +229,9 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
 		if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil {
 			return 0, err
 		}
+		if err := WriteFileFlagsFromTarHeader(path, hdr); err != nil {
+			return 0, err
+		}
 	}
 
 	return size, nil

pkg/archive/fflags_bsd.go

@@ -0,0 +1,167 @@
+//go:build freebsd
+// +build freebsd
+
+package archive
+
+import (
+	"archive/tar"
+	"fmt"
+	"math/bits"
+	"os"
+	"strings"
+	"syscall"
+
+	"github.com/containers/storage/pkg/system"
+)
+
+const (
+	paxSCHILYFflags = "SCHILY.fflags"
+)
+
+var (
+	flagNameToValue = map[string]uint32{
+		"sappnd":     system.SF_APPEND,
+		"sappend":    system.SF_APPEND,
+		"arch":       system.SF_ARCHIVED,
+		"archived":   system.SF_ARCHIVED,
+		"schg":       system.SF_IMMUTABLE,
+		"schange":    system.SF_IMMUTABLE,
+		"simmutable": system.SF_IMMUTABLE,
+		"sunlnk":     system.SF_NOUNLINK,
+		"sunlink":    system.SF_NOUNLINK,
+		"snapshot":   system.SF_SNAPSHOT,
+		"uappnd":     system.UF_APPEND,
+		"uappend":    system.UF_APPEND,
+		"uarch":      system.UF_ARCHIVE,
+		"uarchive":   system.UF_ARCHIVE,
+		"hidden":     system.UF_HIDDEN,
+		"uhidden":    system.UF_HIDDEN,
+		"uchg":       system.UF_IMMUTABLE,
+		"uchange":    system.UF_IMMUTABLE,
+		"uimmutable": system.UF_IMMUTABLE,
+		"uunlnk":     system.UF_NOUNLINK,
+		"uunlink":    system.UF_NOUNLINK,
+		"offline":    system.UF_OFFLINE,
+		"uoffline":   system.UF_OFFLINE,
+		"opaque":     system.UF_OPAQUE,
+		"rdonly":     system.UF_READONLY,
+		"urdonly":    system.UF_READONLY,
+		"readonly":   system.UF_READONLY,
+		"ureadonly":  system.UF_READONLY,
+		"reparse":    system.UF_REPARSE,
+		"ureparse":   system.UF_REPARSE,
+		"sparse":     system.UF_SPARSE,
+		"usparse":    system.UF_SPARSE,
+		"system":     system.UF_SYSTEM,
+		"usystem":    system.UF_SYSTEM,
+	}
+	// Only include the short names for the reverse map
+	flagValueToName = map[uint32]string{
+		system.SF_APPEND:    "sappnd",
+		system.SF_ARCHIVED:  "arch",
+		system.SF_IMMUTABLE: "schg",
+		system.SF_NOUNLINK:  "sunlnk",
+		system.SF_SNAPSHOT:  "snapshot",
+		system.UF_APPEND:    "uappnd",
+		system.UF_ARCHIVE:   "uarch",
+		system.UF_HIDDEN:    "hidden",
+		system.UF_IMMUTABLE: "uchg",
+		system.UF_NOUNLINK:  "uunlnk",
+		system.UF_OFFLINE:   "offline",
+		system.UF_OPAQUE:    "opaque",
+		system.UF_READONLY:  "rdonly",
+		system.UF_REPARSE:   "reparse",
+		system.UF_SPARSE:    "sparse",
+		system.UF_SYSTEM:    "system",
+	}
+)
+
+func parseFileFlags(fflags string) (uint32, uint32, error) {
+	var set, clear uint32 = 0, 0
+	for _, fflag := range strings.Split(fflags, ",") {
+		isClear := false
+		if strings.HasPrefix(fflag, "no") {
+			isClear = true
+			fflag = strings.TrimPrefix(fflag, "no")
+		}
+		if value, ok := flagNameToValue[fflag]; ok {
+			if isClear {
+				clear |= value
+			} else {
+				set |= value
+			}
+		} else {
+			return 0, 0, fmt.Errorf("parsing file flags, unrecognised token: %s", fflag)
+		}
+	}
+	return set, clear, nil
+}
+
+func formatFileFlags(fflags uint32) (string, error) {
+	var res = []string{}
+	for fflags != 0 {
+		// Extract lowest set bit
+		fflag := uint32(1) << bits.TrailingZeros32(fflags)
+		if name, ok := flagValueToName[fflag]; ok {
+			res = append(res, name)
+		} else {
+			return "", fmt.Errorf("formatting file flags, unrecognised flag: %x", fflag)
+		}
+		fflags &= ^fflag
+	}
+	return strings.Join(res, ","), nil
+}
+
+func ReadFileFlagsToTarHeader(path string, hdr *tar.Header) error {
+	st, err := system.Lstat(path)
+	if err != nil {
+		return err
+	}
+	fflags, err := formatFileFlags(st.Flags())
+	if err != nil {
+		return err
+	}
+	if fflags != "" {
+		if hdr.PAXRecords == nil {
+			hdr.PAXRecords = map[string]string{}
+		}
+		hdr.PAXRecords[paxSCHILYFflags] = fflags
+	}
+	return nil
+}
+
+func WriteFileFlagsFromTarHeader(path string, hdr *tar.Header) error {
+	if fflags, ok := hdr.PAXRecords[paxSCHILYFflags]; ok {
+		var set, clear uint32
+		set, clear, err := parseFileFlags(fflags)
+		if err != nil {
+			return err
+		}
+
+		// Apply the delta to the existing file flags
+		st, err := system.Lstat(path)
+		if err != nil {
+			return err
+		}
+		return system.Lchflags(path, (st.Flags() & ^clear)|set)
+	}
+	return nil
+}
+
+func resetImmutable(path string, fi *os.FileInfo) error {
+	var flags uint32
+	if fi != nil {
+		flags = (*fi).Sys().(*syscall.Stat_t).Flags
+	} else {
+		st, err := system.Lstat(path)
+		if err != nil {
+			return err
+		}
+		flags = st.Flags()
+	}
+	if flags&(system.SF_IMMUTABLE|system.UF_IMMUTABLE) != 0 {
+		flags &= ^(system.SF_IMMUTABLE | system.UF_IMMUTABLE)
+		return system.Lchflags(path, flags)
+	}
+	return nil
+}

pkg/archive/fflags_unsupported.go

@@ -0,0 +1,21 @@
+//go:build !freebsd
+// +build !freebsd
+
+package archive
+
+import (
+	"archive/tar"
+	"os"
+)
+
+func ReadFileFlagsToTarHeader(path string, hdr *tar.Header) error {
+	return nil
+}
+
+func WriteFileFlagsFromTarHeader(path string, hdr *tar.Header) error {
+	return nil
+}
+
+func resetImmutable(path string, fi *os.FileInfo) error {
+	return nil
+}