From c2aae9a082205d3ba72d54dceddcbd9b57b94408 Mon Sep 17 00:00:00 2001 From: arrebole Date: Tue, 16 May 2023 17:34:09 +0800 Subject: [PATCH] =?UTF-8?q?Feature:=20=E4=B8=8A=E4=BC=A0=E6=97=B6=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=BF=BD=E7=95=A5=E9=9A=90=E8=97=8F=E7=9A=84=E6=96=87?= =?UTF-8?q?=E4=BB=B6=20(#92)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Feature: 上传时默认忽略隐藏的文件 - 用户上传目录时，目录中包含的 . 开头的文件默认会被忽略 - 用户可以使用 --all 参数 强制上传所以文件 * 忽略windows下的隐藏文件 - 增加对windows系统下隐藏文件忽略的支持 * 上传隐藏目录触发提示 如果上传的是目录，并且是隐藏的目录，没有使用 `-all` 参数则触发提示 * chore: 增加忽略文件的测试用例 --- README.md | 3 +- commands.go | 4 ++ fsutil/ignore.go | 10 ++++ fsutil/ignore_unix.go | 13 ++++++ fsutil/ignore_windows.go | 23 ++++++++++ putiginore_test.go | 99 ++++++++++++++++++++++++++++++++++++++++ session.go | 50 +++++++++++++++----- utils.go | 15 ------ 8 files changed, 189 insertions(+), 28 deletions(-) create mode 100644 fsutil/ignore.go create mode 100644 fsutil/ignore_unix.go create mode 100644 fsutil/ignore_windows.go create mode 100644 putiginore_test.go diff --git a/README.md b/README.md index 0fc19e8..413d88a 100644 --- a/README.md +++ b/README.md @@ -363,7 +363,7 @@ upx get -c /baima_text_auditer.tar | options | 说明 | |---------|-----------------------------| | -w | 多线程下载 (1-10) (default: 5) | - +| -all | 上传包含目录下隐藏的文件和文件夹 | #### 语法 ```bash upx put [remote-file] @@ -391,6 +391,7 @@ upx put ./video /myfiles | options | 说明 | |---------|-----------------------------| | -w | 多线程下载 (1-10) (default: 5) | +| -all | 上传包含目录下隐藏的文件和文件夹 | | --remote | 远程路径 | #### 语法 diff --git a/commands.go b/commands.go index 9fb9e1d..acb271c 100644 --- a/commands.go +++ b/commands.go @@ -347,11 +347,13 @@ func NewPutCommand() cli.Command { localPath, upPath, c.Int("w"), + c.Bool("all"), ) return nil }, Flags: []cli.Flag{ cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5}, + cli.BoolFlag{Name: "all", Usage: "upload all files including hidden files"}, }, } } @@ -370,10 +372,12 @@ func NewUploadCommand() cli.Command { c.Args(), c.String("remote"), c.Int("w"), + c.Bool("all"), ) return nil }, Flags: []cli.Flag{ + cli.BoolFlag{Name: "all", Usage: "upload all files including hidden files"}, cli.IntFlag{Name: "w", Usage: "max concurrent threads", Value: 5}, cli.StringFlag{Name: "remote", Usage: "remote path", Value: "./"}, }, diff --git a/fsutil/ignore.go b/fsutil/ignore.go new file mode 100644 index 0000000..2e1c437 --- /dev/null +++ b/fsutil/ignore.go @@ -0,0 +1,10 @@ +package fsutil + +import ( + "strings" +) + +// 判断文件是否是 . 开头的 +func hasDotPrefix(filename string) bool { + return strings.HasPrefix(filename, ".") +} diff --git a/fsutil/ignore_unix.go b/fsutil/ignore_unix.go new file mode 100644 index 0000000..a7e5379 --- /dev/null +++ b/fsutil/ignore_unix.go @@ -0,0 +1,13 @@ +//go:build linux || darwin + +package fsutil + +import ( + "io/fs" + "path/filepath" +) + +// 判断文件是否是需要忽略的文件 +func IsIgnoreFile(path string, fileInfo fs.FileInfo) bool { + return hasDotPrefix(filepath.Base(path)) +} diff --git a/fsutil/ignore_windows.go b/fsutil/ignore_windows.go new file mode 100644 index 0000000..5d328b2 --- /dev/null +++ b/fsutil/ignore_windows.go @@ -0,0 +1,23 @@ +//go:build windows + +package fsutil + +import ( + "io/fs" + "path/filepath" + "syscall" +) + +// 判断文件是否是需要忽略的文件 +func IsIgnoreFile(path string, fileInfo fs.FileInfo) bool { + for hasDotPrefix(filepath.Base(path)) { + return true + } + + underlyingData := fileInfo.Sys().(*syscall.Win32FileAttributeData) + if underlyingData != nil { + return underlyingData.FileAttributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0 + } + + return false +} diff --git a/putiginore_test.go b/putiginore_test.go new file mode 100644 index 0000000..512c4c1 --- /dev/null +++ b/putiginore_test.go @@ -0,0 +1,99 @@ +package upx + +import ( + "io/ioutil" + "path" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Ls(up string) ([]string, error) { + b, err := Upx("ls", up) + if err != nil { + return nil, err + } + + var ups = make([]string, 0) + output := strings.TrimRight(string(b), "\n") + for _, line := range strings.Split(output, "\n") { + items := strings.Split(line, " ") + ups = append(ups, items[len(items)-1]) + } + return ups, nil +} + +func TestPutIgnore(t *testing.T) { + SetUp() + defer TearDown() + + upRootPath := path.Join(ROOT, "iginore") + Upx("mkdir", upRootPath) + + localRootPath, err := ioutil.TempDir("", "test") + assert.NoError(t, err) + localRootName := filepath.Base(localRootPath) + + CreateFile(path.Join(localRootPath, "FILE1")) + CreateFile(path.Join(localRootPath, "FILE2")) + CreateFile(path.Join(localRootPath, ".FILE3")) + CreateFile(path.Join(localRootPath, ".FILES/FILE")) + + // 上传文件夹 + // 不包含隐藏的文件，所以只有FILE1和FILE2 + Upx("put", localRootPath, upRootPath) + files, err := Ls(path.Join(upRootPath, localRootName)) + + assert.NoError(t, err) + assert.Len(t, files, 2) + assert.ElementsMatch( + t, + files, + []string{"FILE1", "FILE2"}, + ) + + // 上传隐藏的文件夹, 无all，上传失效 + Upx( + "put", + path.Join(localRootPath, ".FILES"), + path.Join(upRootPath, localRootName, ".FILES"), + ) + files, err = Ls(path.Join(upRootPath, localRootName)) + assert.NoError(t, err) + + assert.Len(t, files, 2) + assert.ElementsMatch( + t, + files, + []string{"FILE1", "FILE2"}, + ) + + // 上传隐藏的文件夹, 有all，上传成功 + Upx( + "put", + "-all", + path.Join(localRootPath, ".FILES"), + path.Join(upRootPath, localRootName, ".FILES"), + ) + files, err = Ls(path.Join(upRootPath, localRootName)) + assert.NoError(t, err) + assert.Len(t, files, 3) + assert.ElementsMatch( + t, + files, + []string{"FILE1", "FILE2", ".FILES"}, + ) + + // 上传所有文件 + Upx("put", "-all", localRootPath, upRootPath) + files, err = Ls(path.Join(upRootPath, localRootName)) + assert.NoError(t, err) + assert.Len(t, files, 4) + assert.ElementsMatch( + t, + files, + []string{"FILE1", "FILE2", ".FILE3", ".FILES"}, + ) +} diff --git a/session.go b/session.go index 13762a2..bb64dfd 100644 --- a/session.go +++ b/session.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io/fs" "io/ioutil" "log" "math/rand" @@ -20,6 +21,7 @@ import ( "github.com/arrebole/progressbar" "github.com/fatih/color" "github.com/upyun/go-sdk/v3/upyun" + "github.com/upyun/upx/fsutil" "github.com/upyun/upx/partial" ) @@ -496,7 +498,6 @@ func (sess *Session) putFileWithProgress(localPath, upPath string, localInfo os. return err } defer fd.Close() - cfg := &upyun.PutObjectConfig{ Path: upPath, Headers: map[string]string{ @@ -601,20 +602,32 @@ func (sess *Session) putFilesWitchProgress(localFiles []*UploadedFile, workers i wg.Wait() } -func (sess *Session) putDir(localPath, upPath string, workers int) { +func (sess *Session) putDir(localPath, upPath string, workers int, withIgnore bool) { + localAbsPath, err := filepath.Abs(localPath) + if err != nil { + PrintErrorAndExit(err.Error()) + } + // 如果上传的是目录，并且是隐藏的目录，则触发提示 + rootDirInfo, err := os.Stat(localAbsPath) + if err != nil { + PrintErrorAndExit(err.Error()) + } + if !withIgnore && fsutil.IsIgnoreFile(localAbsPath, rootDirInfo) { + PrintErrorAndExit("%s is a ignore dir, use `-all` to force put all files", localAbsPath) + } + type FileInfo struct { fpath string fInfo os.FileInfo } localFiles := make(chan *FileInfo, workers*2) var wg sync.WaitGroup - var err error wg.Add(workers) for w := 0; w < workers; w++ { go func() { defer wg.Done() for info := range localFiles { - rel, _ := filepath.Rel(localPath, info.fpath) + rel, _ := filepath.Rel(localAbsPath, info.fpath) desPath := path.Join(upPath, filepath.ToSlash(rel)) if fInfo, err := os.Stat(info.fpath); err == nil && fInfo.IsDir() { err = sess.updriver.Mkdir(desPath) @@ -628,13 +641,21 @@ func (sess *Session) putDir(localPath, upPath string, workers int) { }() } - walk(localPath, func(fpath string, fInfo os.FileInfo, err error) { - if err == nil { + filepath.Walk(localAbsPath, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !withIgnore && fsutil.IsIgnoreFile(path, info) { + if info.IsDir() { + return filepath.SkipDir + } + } else { localFiles <- &FileInfo{ - fpath: fpath, - fInfo: fInfo, + fpath: path, + fInfo: info, } } + return nil }) close(localFiles) @@ -642,7 +663,7 @@ func (sess *Session) putDir(localPath, upPath string, workers int) { } // / Put 上传单文件或单目录 -func (sess *Session) Put(localPath, upPath string, workers int) { +func (sess *Session) Put(localPath, upPath string, workers int, withIgnore bool) { upPath = sess.AbsPath(upPath) exist, isDir := false, false @@ -693,7 +714,7 @@ func (sess *Session) Put(localPath, upPath string, workers int) { upPath = path.Join(upPath, filepath.Base(localPath)) } } - sess.putDir(localPath, upPath, workers) + sess.putDir(localPath, upPath, workers, withIgnore) } else { if isDir { upPath = path.Join(upPath, filepath.Base(localPath)) @@ -703,7 +724,7 @@ func (sess *Session) Put(localPath, upPath string, workers int) { } // put 的升级版命令, 支持多文件上传 -func (sess *Session) Upload(filenames []string, upPath string, workers int) { +func (sess *Session) Upload(filenames []string, upPath string, workers int, withIgnore bool) { upPath = sess.AbsPath(upPath) // 检测云端的目的地目录 @@ -741,7 +762,12 @@ func (sess *Session) Upload(filenames []string, upPath string, workers int) { // 上传目录 for _, localPath := range dirs { - sess.putDir(localPath, path.Join(upPath, filepath.Base(localPath)), workers) + sess.putDir( + localPath, + path.Join(upPath, filepath.Base(localPath)), + workers, + withIgnore, + ) } // 上传文件 diff --git a/utils.go b/utils.go index 7a574cb..28d9906 100644 --- a/utils.go +++ b/utils.go @@ -4,9 +4,7 @@ import ( "crypto/md5" "fmt" "io" - "io/ioutil" "os" - "path/filepath" "strconv" "strings" "time" @@ -134,19 +132,6 @@ func md5File(fpath string) (string, error) { return fmt.Sprintf("%x", hash.Sum(nil)), nil } -func walk(root string, f func(string, os.FileInfo, error)) { - fi, err := os.Stat(root) - if err == nil && fi != nil && fi.IsDir() { - fInfos, err := ioutil.ReadDir(root) - f(root, fi, err) - for _, fInfo := range fInfos { - walk(filepath.Join(root, fInfo.Name()), f) - } - } else { - f(root, fi, err) - } -} - func contains(slice []string, item string) bool { for _, s := range slice { if s == item {