Skip to content

Commit

Permalink
Merge pull request #101 from aaronlehmann/copy-include-exclude
Browse files Browse the repository at this point in the history 
Add IncludePatterns and ExcludePatterns options for Copy
  • Loading branch information
@aaronlehmann
aaronlehmann authored May 3, 2021
2 parents 8599091 + 10412d7 commit 5c8be85
Show file tree
Hide file tree
Showing 6 changed files with 432 additions and 106 deletions.
247 changes: 220 additions & 27 deletions copy/copy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ import (
"time"

"github.com/containerd/continuity/fs"
"github.com/docker/docker/pkg/fileutils"
"github.com/pkg/errors"
"github.com/tonistiigi/fsutil/prefix"
)

var bufferPool = &sync.Pool{
Expand Down Expand Up @@ -86,7 +88,10 @@ func Copy(ctx context.Context, srcRoot, src, dstRoot, dst string, opts ...Opt) e
return err
}

c := newCopier(ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler)
c, err := newCopier(ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler, ci.IncludePatterns, ci.ExcludePatterns)
if err != nil {
return err
}
srcs := []string{src}

if ci.AllowWildcards {
Expand All @@ -109,7 +114,8 @@ func Copy(ctx context.Context, srcRoot, src, dstRoot, dst string, opts ...Opt) e
if err != nil {
return err
}
if err := c.copy(ctx, srcFollowed, dst, false); err != nil {
skipIncludePatterns := len(c.includePatterns) == 0
if err := c.copy(ctx, srcFollowed, "", dst, false, skipIncludePatterns); err != nil {
return err
}
}
Expand Down Expand Up @@ -162,6 +168,10 @@ type CopyInfo struct {
XAttrErrorHandler XAttrErrorHandler
CopyDirContents bool
FollowLinks bool
// Include only files/dirs matching at least one of these patterns
IncludePatterns []string
// Exclude files/dir matching any of these patterns (even if they match an include pattern)
ExcludePatterns []string
}

type Opt func(*CopyInfo)
Expand Down Expand Up @@ -197,48 +207,117 @@ func AllowXAttrErrors(ci *CopyInfo) {
WithXAttrErrorHandler(h)(ci)
}

func WithIncludePattern(includePattern string) Opt {
return func(ci *CopyInfo) {
ci.IncludePatterns = append(ci.IncludePatterns, includePattern)
}
}

func WithExcludePattern(excludePattern string) Opt {
return func(ci *CopyInfo) {
ci.ExcludePatterns = append(ci.ExcludePatterns, excludePattern)
}
}

type copier struct {
chown Chowner
utime *time.Time
mode *int
inodes map[uint64]string
xattrErrorHandler XAttrErrorHandler
chown Chowner
utime *time.Time
mode *int
inodes map[uint64]string
xattrErrorHandler XAttrErrorHandler
includePatterns []string
excludePatternMatcher *fileutils.PatternMatcher
parentDirs []parentDir
}

func newCopier(chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler) *copier {
type parentDir struct {
srcPath string
dstPath string
copied bool
}

func newCopier(chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler, includePatterns, excludePatterns []string) (*copier, error) {
if xeh == nil {
xeh = func(dst, src, key string, err error) error {
return err
}
}
return &copier{inodes: map[uint64]string{}, chown: chown, utime: tm, xattrErrorHandler: xeh, mode: mode}

var pm *fileutils.PatternMatcher
if len(excludePatterns) != 0 {
var err error
pm, err = fileutils.NewPatternMatcher(excludePatterns)
if err != nil {
return nil, errors.Wrapf(err, "invalid excludepatterns: %s", excludePatterns)
}
}

return &copier{
inodes: map[uint64]string{},
chown: chown,
utime: tm,
xattrErrorHandler: xeh,
mode: mode,
includePatterns: includePatterns,
excludePatternMatcher: pm,
}, nil
}

// dest is always clean
func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMetadata bool) error {
func (c *copier) copy(ctx context.Context, src, srcComponents, target string, overwriteTargetMetadata, skipIncludePatterns bool) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}

fi, err := os.Lstat(src)
if err != nil {
return errors.Wrapf(err, "failed to stat %s", src)
}

var include bool
if srcComponents != "" {
if !skipIncludePatterns {
include, skipIncludePatterns, err = c.include(srcComponents, fi)
if err != nil {
return err
}
if !include {
return nil
}
}
exclude, err := c.exclude(srcComponents, fi)
if err != nil {
return err
}
if exclude {
return nil
}
}

if !fi.IsDir() {
if include {
if err := c.createParentDirs(src, srcComponents, target, overwriteTargetMetadata); err != nil {
return err
}
}
if err := ensureEmptyFileTarget(target); err != nil {
return err
}
} else if skipIncludePatterns {
if err := c.createParentDirs(src, srcComponents, target, overwriteTargetMetadata); err != nil {
return err
}
}

copyFileInfo := true

switch {
case fi.IsDir():
if created, err := c.copyDirectory(ctx, src, target, fi, overwriteTargetMetadata); err != nil {
if created, err := c.copyDirectory(ctx, src, srcComponents, target, fi, overwriteTargetMetadata, skipIncludePatterns); err != nil {
return err
} else if !overwriteTargetMetadata {
} else if !overwriteTargetMetadata || !skipIncludePatterns {
copyFileInfo = created
}
case (fi.Mode() & os.ModeType) == 0:
Expand Down Expand Up @@ -282,43 +361,157 @@ func (c *copier) copy(ctx context.Context, src, target string, overwriteTargetMe
return nil
}

func (c *copier) copyDirectory(ctx context.Context, src, dst string, stat os.FileInfo, overwriteTargetMetadata bool) (bool, error) {
func (c *copier) include(path string, fi os.FileInfo) (bool, bool, error) {
matched := false
partial := true
for _, pattern := range c.includePatterns {
if fi.IsDir() {
pattern = strings.TrimSuffix(pattern, string(filepath.Separator))
}

if ok, p := prefix.Match(pattern, path, false); ok {
matched = true
if !p {
partial = false
break
}
}
}

if !matched {
return false, false, nil
}
if fi.IsDir() {
return true, !partial, nil
}
return !partial, !partial, nil
}

func (c *copier) exclude(path string, fi os.FileInfo) (bool, error) {
if c.excludePatternMatcher == nil {
return false, nil
}

m, err := c.excludePatternMatcher.Matches(path)
if err != nil {
return false, errors.Wrap(err, "failed to match excludepatterns")
}
if !m {
return false, nil
}

if fi.IsDir() && c.excludePatternMatcher.Exclusions() {
dirSlash := path + string(filepath.Separator)
for _, pat := range c.excludePatternMatcher.Patterns() {
if !pat.Exclusion() {
continue
}
patStr := pat.String() + string(filepath.Separator)
if strings.HasPrefix(patStr, dirSlash) {
return false, nil
}
}
}
return true, nil
}

// Delayed creation of parent directories when a file or dir matches an include
// pattern.
func (c *copier) createParentDirs(src, srcComponents, target string, overwriteTargetMetadata bool) error {
for i, parentDir := range c.parentDirs {
if parentDir.copied {
continue
}

fi, err := os.Stat(parentDir.srcPath)
if err != nil {
return errors.Wrapf(err, "failed to stat %s", src)
}
if !fi.IsDir() {
return errors.Errorf("%s is not a directory", parentDir.srcPath)
}

created, err := copyDirectoryOnly(parentDir.srcPath, parentDir.dstPath, fi, overwriteTargetMetadata)
if err != nil {
return err
}
if created {
if err := c.copyFileInfo(fi, parentDir.dstPath); err != nil {
return errors.Wrap(err, "failed to copy file info")
}

if err := copyXAttrs(parentDir.dstPath, parentDir.srcPath, c.xattrErrorHandler); err != nil {
return errors.Wrap(err, "failed to copy xattrs")
}
}

c.parentDirs[i].copied = true
}
return nil
}

func (c *copier) copyDirectory(ctx context.Context, src, srcComponents, dst string, stat os.FileInfo, overwriteTargetMetadata, skipIncludePatterns bool) (bool, error) {
if !stat.IsDir() {
return false, errors.Errorf("source is not directory")
}

created := false

if st, err := os.Lstat(dst); err != nil {
if !os.IsNotExist(err) {
return false, err
}
created = true
if err := os.Mkdir(dst, stat.Mode()); err != nil {
return created, errors.Wrapf(err, "failed to mkdir %s", dst)
}
} else if !st.IsDir() {
return false, errors.Errorf("cannot copy to non-directory: %s", dst)
} else if overwriteTargetMetadata {
if err := os.Chmod(dst, stat.Mode()); err != nil {
return false, errors.Wrapf(err, "failed to chmod on %s", dst)
// If there are no include patterns or this directory matched an include
// pattern exactly, go ahead and create the directory. Otherwise, delay to
// handle include patterns like a/*/c where we do not want to create a/b
// until we encounter a/b/c.
if skipIncludePatterns {
var err error
created, err = copyDirectoryOnly(src, dst, stat, overwriteTargetMetadata)
if err != nil {
return created, err
}
}

c.parentDirs = append(c.parentDirs, parentDir{
srcPath: src,
dstPath: dst,
copied: skipIncludePatterns,
})

defer func() {
c.parentDirs = c.parentDirs[:len(c.parentDirs)-1]
}()

fis, err := ioutil.ReadDir(src)
if err != nil {
return false, errors.Wrapf(err, "failed to read %s", src)
}

for _, fi := range fis {
if err := c.copy(ctx, filepath.Join(src, fi.Name()), filepath.Join(dst, fi.Name()), true); err != nil {
if err := c.copy(ctx, filepath.Join(src, fi.Name()), filepath.Join(srcComponents, fi.Name()), filepath.Join(dst, fi.Name()), true, skipIncludePatterns); err != nil {
return false, err
}
}

return created, nil
}

func copyDirectoryOnly(src, dst string, stat os.FileInfo, overwriteTargetMetadata bool) (bool, error) {
if st, err := os.Lstat(dst); err != nil {
if !os.IsNotExist(err) {
return false, err
}
if err := os.Mkdir(dst, stat.Mode()); err != nil {
return false, errors.Wrapf(err, "failed to mkdir %s", dst)
}
return true, nil
} else if !st.IsDir() {
return false, errors.Errorf("cannot copy to non-directory: %s", dst)
} else if overwriteTargetMetadata {
if err := os.Chmod(dst, stat.Mode()); err != nil {
return false, errors.Wrapf(err, "failed to chmod on %s", dst)
}
}
return false, nil
}

func ensureEmptyFileTarget(dst string) error {
fi, err := os.Lstat(dst)
if err != nil {
Expand Down
Loading

0 comments on commit 5c8be85

Please sign in to comment.