From 869f8c8f2996ffd64c5edca46cc0c2c3576d7995 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Fri, 15 Sep 2023 12:12:03 +0800 Subject: [PATCH 01/13] refactor: support bypass seal flag --- cmd/cmd_object.go | 23 +++++++++++++++++++---- cmd/utils.go | 1 + 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index cb50805..ad227f2 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -74,6 +74,11 @@ $ gnfd-cmd object put --recursive folderName gnfd://bucket-name`, Value: false, Usage: "performed on all files or objects under the specified directory or prefix in a recursive way", }, + &cli.BoolFlag{ + Name: bypassSealFlag, + Value: false, + Usage: "if set this flag as true, it will not wait for the file to be sealed after the uploading is completed.", + }, }, } } @@ -400,6 +405,7 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte partSize := ctx.Uint64(partSizeFlag) resumableUpload := ctx.Bool(resumableFlag) + bypassSeal := ctx.Bool(bypassSealFlag) opts := sdktypes.CreateObjectOptions{} if contentType != "" { opts.ContentType = contentType @@ -480,20 +486,29 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte return toCmdErr(err) } - // Check if object is sealed - timeout := time.After(15 * time.Second) - ticker := time.NewTicker(2 * time.Second) + if bypassSeal { + fmt.Printf("upload %s to %s \n", objectName, urlInfo) + return nil + } + // Check if object is sealed + timeout := time.After(1 * time.Hour) + ticker := time.NewTicker(3 * time.Second) + count := 0 + fmt.Println("sealing...") for { select { case <-timeout: return toCmdErr(errors.New("object not sealed after 15 seconds")) case <-ticker.C: + count++ headObjOutput, queryErr := gnfdClient.HeadObject(c, bucketName, objectName) if queryErr != nil { return queryErr } - + if count%10 == 0 { + fmt.Println("sealing...") + } if headObjOutput.ObjectInfo.GetObjectStatus().String() == "OBJECT_STATUS_SEALED" { ticker.Stop() fmt.Printf("upload %s to %s \n", objectName, urlInfo) diff --git a/cmd/utils.go b/cmd/utils.go index 8e613b4..57228a1 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -44,6 +44,7 @@ const ( startOffsetFlag = "start" endOffsetFlag = "end" recursiveFlag = "recursive" + bypassSealFlag = "bypassSeal" addMemberFlag = "addMembers" removeMemberFlag = "removeMembers" renewMemberFlag = "renewMembers" From 6ca2e8a7c91f8e9dba41273c06ed13b6a6fa8184 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Fri, 15 Sep 2023 12:40:21 +0800 Subject: [PATCH 02/13] feat: support print upload progress --- cmd/cmd_object.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index ad227f2..8d09ab5 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -399,7 +399,7 @@ func uploadFolder(urlInfo string, ctx *cli.Context, func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Context, gnfdClient client.Client, uploadSigleFolder, printTxnHash bool, objectSize int64) error { - + var file *os.File contentType := ctx.String(contentTypeFlag) secondarySPAccs := ctx.String(secondarySPFlag) partSize := ctx.Uint64(partSizeFlag) @@ -444,7 +444,7 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte } } else { // Open the referenced file. - file, err := os.Open(filePath) + file, err = os.Open(filePath) if err != nil { return err } @@ -481,8 +481,13 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte } defer reader.Close() + progressReader := &ProgressReader{ + Reader: reader, + Total: objectSize, + } + if err = gnfdClient.PutObject(c, bucketName, objectName, - objectSize, reader, opt); err != nil { + objectSize, progressReader, opt); err != nil { return toCmdErr(err) } @@ -495,6 +500,7 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte timeout := time.After(1 * time.Hour) ticker := time.NewTicker(3 * time.Second) count := 0 + fmt.Println() fmt.Println("sealing...") for { select { @@ -604,6 +610,24 @@ func getObject(ctx *cli.Context) error { return nil } +type ProgressReader struct { + io.Reader + Total int64 + Current int64 +} + +func (pr *ProgressReader) Read(p []byte) (int, error) { + n, err := pr.Reader.Read(p) + pr.Current += int64(n) + pr.printProgress() + return n, err +} + +func (pr *ProgressReader) printProgress() { + progress := float64(pr.Current) / float64(pr.Total) * 100 + fmt.Printf("\ruploading progress:%.2f%%", progress) +} + // cancelCreateObject cancel the created object on chain func cancelCreateObject(ctx *cli.Context) error { if ctx.NArg() != 1 { From ba40f9e671bcbaa0c44744904c659dd24e49be38 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Fri, 15 Sep 2023 12:47:15 +0800 Subject: [PATCH 03/13] fix: fix file max size --- cmd/cmd_object.go | 5 ++++ cmd/utils.go | 59 ++++++++++++++++++++++++----------------------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index 8d09ab5..33582e9 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -486,6 +486,11 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte Total: objectSize, } + // if the file is more than 2G , it needs to force use resume uploading + if objectSize > maxPutWithoutResumeSize { + opt.DisableResumable = false + } + if err = gnfdClient.PutObject(c, bucketName, objectName, objectSize, progressReader, opt); err != nil { return toCmdErr(err) diff --git a/cmd/utils.go b/cmd/utils.go index 57228a1..41d7878 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -28,35 +28,36 @@ import ( ) const ( - Version = "v0.1.0" - maxFileSize = 10 * 1024 * 1024 * 1024 - publicReadType = "public-read" - privateType = "private" - inheritType = "inherit" - effectAllow = "allow" - effectDeny = "deny" - primarySPFlag = "primarySP" - chargeQuotaFlag = "chargedQuota" - visibilityFlag = "visibility" - paymentFlag = "paymentAddress" - secondarySPFlag = "secondarySPs" - contentTypeFlag = "contentType" - startOffsetFlag = "start" - endOffsetFlag = "end" - recursiveFlag = "recursive" - bypassSealFlag = "bypassSeal" - addMemberFlag = "addMembers" - removeMemberFlag = "removeMembers" - renewMemberFlag = "renewMembers" - groupOwnerFlag = "groupOwner" - groupMemberExpireFlag = "expireTime" - groupIDFlag = "groupId" - granteeFlag = "grantee" - actionsFlag = "actions" - effectFlag = "effect" - expireTimeFlag = "expire" - IdFlag = "id" - DestChainIdFlag = "destChainId" + Version = "v0.1.0" + maxFileSize = 64 * 1024 * 1024 * 1024 + maxPutWithoutResumeSize = 2 * 1024 * 1024 * 1024 + publicReadType = "public-read" + privateType = "private" + inheritType = "inherit" + effectAllow = "allow" + effectDeny = "deny" + primarySPFlag = "primarySP" + chargeQuotaFlag = "chargedQuota" + visibilityFlag = "visibility" + paymentFlag = "paymentAddress" + secondarySPFlag = "secondarySPs" + contentTypeFlag = "contentType" + startOffsetFlag = "start" + endOffsetFlag = "end" + recursiveFlag = "recursive" + bypassSealFlag = "bypassSeal" + addMemberFlag = "addMembers" + removeMemberFlag = "removeMembers" + renewMemberFlag = "renewMembers" + groupOwnerFlag = "groupOwner" + groupMemberExpireFlag = "expireTime" + groupIDFlag = "groupId" + granteeFlag = "grantee" + actionsFlag = "actions" + effectFlag = "effect" + expireTimeFlag = "expire" + IdFlag = "id" + DestChainIdFlag = "destChainId" ownerAddressFlag = "owner" addressFlag = "address" From 0c7a47574edbe1a82491ec50e8e84b77942b507e Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Fri, 15 Sep 2023 13:32:29 +0800 Subject: [PATCH 04/13] feat: support print download progress --- cmd/cmd_object.go | 55 ++++++++++++++++++++++++++--------------------- cmd/utils.go | 36 +++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 25 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index 33582e9..332abd8 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -549,7 +549,7 @@ func getObject(ctx *cli.Context) error { c, cancelGetObject := context.WithCancel(globalContext) defer cancelGetObject() - _, err = gnfdClient.HeadObject(c, bucketName, objectName) + chainInfo, err := gnfdClient.HeadObject(c, bucketName, objectName) if err != nil { return toCmdErr(ErrObjectNotExist) } @@ -592,45 +592,50 @@ func getObject(ctx *cli.Context) error { } fmt.Printf("resumable download object %s, the file path is %s \n", objectName, filePath) } else { - // If file exist, open it in append mode - fd, err := os.OpenFile(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0660) + var fd *os.File + dir := filepath.Dir(filePath) + fileName := "." + filepath.Base(filePath) + ".tmp" + tempFilePath := filepath.Join(dir, fileName) + + tempFilePath, err = checkIfDownloadFileExist(tempFilePath, objectName) + if err != nil { + return toCmdErr(err) + } + // download to the temp file firstly + fd, err = os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY, 0660) if err != nil { return err } defer fd.Close() - body, info, err := gnfdClient.GetObject(c, bucketName, objectName, opt) - if err != nil { + pw := &ProgressWriter{ + Writer: fd, + Total: int64(chainInfo.ObjectInfo.PayloadSize), + StartTime: time.Now(), + LastPrinted: time.Now(), + } + + body, info, getErr := gnfdClient.GetObject(c, bucketName, objectName, opt) + if getErr != nil { return toCmdErr(err) } - _, err = io.Copy(fd, body) + _, err = io.Copy(pw, body) if err != nil { return toCmdErr(err) } - fmt.Printf("download object %s, the file path is %s, content length:%d \n", objectName, filePath, uint64(info.Size)) - } - - return nil -} -type ProgressReader struct { - io.Reader - Total int64 - Current int64 -} + err = os.Rename(tempFilePath, filePath) + if err != nil { + fmt.Printf("failed to rename %s to %s \n", tempFilePath, filePath) + return nil + } + } -func (pr *ProgressReader) Read(p []byte) (int, error) { - n, err := pr.Reader.Read(p) - pr.Current += int64(n) - pr.printProgress() - return n, err -} + fmt.Printf("\ndownload object %s, the file path is %s, content length:%d \n", objectName, filePath, uint64(info.Size)) -func (pr *ProgressReader) printProgress() { - progress := float64(pr.Current) / float64(pr.Total) * 100 - fmt.Printf("\ruploading progress:%.2f%%", progress) + return nil } // cancelCreateObject cancel the created object on chain diff --git a/cmd/utils.go b/cmd/utils.go index 41d7878..be6ab8c 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -538,3 +538,39 @@ func parseFileByArg(ctx *cli.Context, argIndex int) (int64, error) { } return objectSize, nil } + +type ProgressReader struct { + io.Reader + Total int64 + Current int64 +} + +func (pr *ProgressReader) Read(p []byte) (int, error) { + n, err := pr.Reader.Read(p) + pr.Current += int64(n) + pr.printProgress() + return n, err +} + +func (pr *ProgressReader) printProgress() { + progress := float64(pr.Current) / float64(pr.Total) * 100 + fmt.Printf("\ruploading progress:%.2f%%", progress) +} + +type ProgressWriter struct { + io.Writer + Total int64 + Current int64 +} + +func (pw *ProgressWriter) Write(p []byte) (int, error) { + n, err := pw.Writer.Write(p) + pw.Current += int64(n) + pw.printProgress() + return n, err +} + +func (pw *ProgressWriter) printProgress() { + progress := float64(pw.Current) / float64(pw.Total) * 100 + fmt.Printf("\rdownloading progress:%.2f%%", progress) +} From 362b65ce48c023b0321170b1a27b6a1d86af11b5 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Fri, 15 Sep 2023 13:40:22 +0800 Subject: [PATCH 05/13] fix: support download to temp file --- cmd/cmd_object.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index 332abd8..e92c1d3 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -633,6 +633,12 @@ func getObject(ctx *cli.Context) error { } } + err = os.Rename(tempFilePath, filePath) + if err != nil { + fmt.Printf("failed to rename %s to %s \n", tempFilePath, filePath) + return nil + } + fmt.Printf("\ndownload object %s, the file path is %s, content length:%d \n", objectName, filePath, uint64(info.Size)) return nil From c3173079416ef808b8204cfa34651e35eed9fd31 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Fri, 15 Sep 2023 14:09:58 +0800 Subject: [PATCH 06/13] fix: fix recursive parse object name error --- cmd/cmd_object.go | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index e92c1d3..c916cbd 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path" "path/filepath" "strings" "time" @@ -387,8 +388,7 @@ func uploadFolder(urlInfo string, ctx *cli.Context, } // upload folder for id, info := range fileInfos { - // pathList := strings.Split(info.Name(), "/") - objectName := filePaths[id] + objectName := path.Base(filePaths[id]) if uploadErr := uploadFile(bucketName, objectName, filePaths[id], urlInfo, ctx, gnfdClient, false, false, info.Size()); uploadErr != nil { fmt.Printf("failed to upload object: %s, error:%v \n", objectName, uploadErr) } @@ -497,7 +497,7 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte } if bypassSeal { - fmt.Printf("upload %s to %s \n", objectName, urlInfo) + fmt.Printf("\nupload %s to %s \n", objectName, urlInfo) return nil } @@ -571,6 +571,30 @@ func getObject(ctx *cli.Context) error { } } +<<<<<<< HEAD +======= + st, err := os.Stat(filePath) + if err == nil { + // If the destination exists and is a directory. + if st.IsDir() { + fmt.Printf("file:%s is a directory\n", filePath) + filePath = filePath + "/" + objectName + } + fmt.Printf("download file:%s already exist\n", filePath) + return nil + } + + dir := filepath.Dir(filePath) + fileName := "." + filepath.Base(filePath) + ".tmp" + tempFilePath := filepath.Join(dir, fileName) + // download to the temp file firstly + fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY, 0660) + if err != nil { + return err + } + + defer fd.Close() +>>>>>>> db6e1d5 (fix: fix recursive parse object name error) opt := sdktypes.GetObjectOptions{} startOffset := ctx.Int64(startOffsetFlag) endOffset := ctx.Int64(endOffsetFlag) From 280e7bb51d3668bb333ab1a4f4c48919d11a756f Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Fri, 15 Sep 2023 15:12:24 +0800 Subject: [PATCH 07/13] feat: support print rate of download and uploding --- cmd/cmd_object.go | 33 +----------------------- cmd/utils.go | 64 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 38 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index c916cbd..caec0fe 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -571,30 +571,6 @@ func getObject(ctx *cli.Context) error { } } -<<<<<<< HEAD -======= - st, err := os.Stat(filePath) - if err == nil { - // If the destination exists and is a directory. - if st.IsDir() { - fmt.Printf("file:%s is a directory\n", filePath) - filePath = filePath + "/" + objectName - } - fmt.Printf("download file:%s already exist\n", filePath) - return nil - } - - dir := filepath.Dir(filePath) - fileName := "." + filepath.Base(filePath) + ".tmp" - tempFilePath := filepath.Join(dir, fileName) - // download to the temp file firstly - fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY, 0660) - if err != nil { - return err - } - - defer fd.Close() ->>>>>>> db6e1d5 (fix: fix recursive parse object name error) opt := sdktypes.GetObjectOptions{} startOffset := ctx.Int64(startOffsetFlag) endOffset := ctx.Int64(endOffsetFlag) @@ -655,16 +631,9 @@ func getObject(ctx *cli.Context) error { fmt.Printf("failed to rename %s to %s \n", tempFilePath, filePath) return nil } + fmt.Printf("\ndownload object %s, the file path is %s, content length:%d \n", objectName, filePath, uint64(info.Size)) } - err = os.Rename(tempFilePath, filePath) - if err != nil { - fmt.Printf("failed to rename %s to %s \n", tempFilePath, filePath) - return nil - } - - fmt.Printf("\ndownload object %s, the file path is %s, content length:%d \n", objectName, filePath, uint64(info.Size)) - return nil } diff --git a/cmd/utils.go b/cmd/utils.go index be6ab8c..049e672 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -541,8 +541,10 @@ func parseFileByArg(ctx *cli.Context, argIndex int) (int64, error) { type ProgressReader struct { io.Reader - Total int64 - Current int64 + Total int64 + Current int64 + StartTime time.Time + LastPrinted time.Time } func (pr *ProgressReader) Read(p []byte) (int, error) { @@ -554,13 +556,23 @@ func (pr *ProgressReader) Read(p []byte) (int, error) { func (pr *ProgressReader) printProgress() { progress := float64(pr.Current) / float64(pr.Total) * 100 - fmt.Printf("\ruploading progress:%.2f%%", progress) + now := time.Now() + elapsed := now.Sub(pr.StartTime) + uploadSpeed := float64(pr.Current) / elapsed.Seconds() + + if now.Sub(pr.LastPrinted) >= time.Second { // print rate every second + fmt.Printf("\ruploading progress: %.2f%% [ %s / %s ], : %s", + progress, getConvertSize(pr.Current), getConvertSize(pr.Total), getConvertRate(uploadSpeed)) + pr.LastPrinted = now + } } type ProgressWriter struct { io.Writer - Total int64 - Current int64 + Total int64 + Current int64 + StartTime time.Time + LastPrinted time.Time } func (pw *ProgressWriter) Write(p []byte) (int, error) { @@ -572,5 +584,45 @@ func (pw *ProgressWriter) Write(p []byte) (int, error) { func (pw *ProgressWriter) printProgress() { progress := float64(pw.Current) / float64(pw.Total) * 100 - fmt.Printf("\rdownloading progress:%.2f%%", progress) + now := time.Now() + + elapsed := now.Sub(pw.StartTime) + downloadedBytes := pw.Current + downloadSpeed := float64(downloadedBytes) / elapsed.Seconds() + + if now.Sub(pw.LastPrinted) >= time.Second { // print rate every second + fmt.Printf("\rdownloding progress: %.2f%% [ %s / %s ], : %s", + progress, getConvertSize(pw.Current), getConvertSize(pw.Total), getConvertRate(downloadSpeed)) + pw.LastPrinted = now + } +} + +func getConvertSize(fileSize int64) string { + var convertedSize string + if fileSize > 1<<30 { + convertedSize = fmt.Sprintf("%.2fG", float64(fileSize)/(1<<30)) + } else if fileSize > 1<<20 { + convertedSize = fmt.Sprintf("%.2fM", float64(fileSize)/(1<<20)) + } else if fileSize > 1<<10 { + convertedSize = fmt.Sprintf("%.2fK", float64(fileSize)/(1<<10)) + } else { + convertedSize = fmt.Sprintf("%dB", fileSize) + } + return convertedSize +} + +func getConvertRate(rate float64) string { + const ( + KB = 1024 + MB = 1024 * KB + ) + + switch { + case rate >= MB: + return fmt.Sprintf("%.2f MB/s", rate/MB) + case rate >= KB: + return fmt.Sprintf("%.2f KB/s", rate/KB) + default: + return fmt.Sprintf("%.2f Byte /s", rate) + } } From bb81e2d4a50e5b94bc80b74b04bfc1e3abee4469 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Mon, 18 Sep 2023 12:32:27 +0800 Subject: [PATCH 08/13] fix: fix download check exist file --- cmd/cmd_object.go | 23 +++++++++++++++++++++++ cmd/utils.go | 19 ++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index caec0fe..35a3c55 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -571,6 +571,29 @@ func getObject(ctx *cli.Context) error { } } +<<<<<<< HEAD +======= + filePath, err = checkIfDownloadFileExist(filePath, objectName) + if err != nil { + return toCmdErr(err) + } + dir := filepath.Dir(filePath) + fileName := "." + filepath.Base(filePath) + ".tmp" + tempFilePath := filepath.Join(dir, fileName) + + tempFilePath, err = checkIfDownloadFileExist(tempFilePath, objectName) + if err != nil { + return toCmdErr(err) + } + + // download to the temp file firstly + fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY, 0660) + if err != nil { + return err + } + + defer fd.Close() +>>>>>>> 0adb378 (fix: fix download check exist file) opt := sdktypes.GetObjectOptions{} startOffset := ctx.Int64(startOffsetFlag) endOffset := ctx.Int64(endOffsetFlag) diff --git a/cmd/utils.go b/cmd/utils.go index 049e672..3ed2508 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -561,7 +561,7 @@ func (pr *ProgressReader) printProgress() { uploadSpeed := float64(pr.Current) / elapsed.Seconds() if now.Sub(pr.LastPrinted) >= time.Second { // print rate every second - fmt.Printf("\ruploading progress: %.2f%% [ %s / %s ], : %s", + fmt.Printf("\ruploading progress: %.2f%% [ %s / %s ], rate: %10s", progress, getConvertSize(pr.Current), getConvertSize(pr.Total), getConvertRate(uploadSpeed)) pr.LastPrinted = now } @@ -591,7 +591,7 @@ func (pw *ProgressWriter) printProgress() { downloadSpeed := float64(downloadedBytes) / elapsed.Seconds() if now.Sub(pw.LastPrinted) >= time.Second { // print rate every second - fmt.Printf("\rdownloding progress: %.2f%% [ %s / %s ], : %s", + fmt.Printf("\rdownloding progress: %.2f%% [ %s / %s ], rate: %s ", progress, getConvertSize(pw.Current), getConvertSize(pw.Total), getConvertRate(downloadSpeed)) pw.LastPrinted = now } @@ -623,6 +623,19 @@ func getConvertRate(rate float64) string { case rate >= KB: return fmt.Sprintf("%.2f KB/s", rate/KB) default: - return fmt.Sprintf("%.2f Byte /s", rate) + return fmt.Sprintf("%.2f Byte/s", rate) } } + +func checkIfDownloadFileExist(filePath, objectName string) (string, error) { + st, err := os.Stat(filePath) + if err == nil { + // If the destination exists and is a directory. + if st.IsDir() { + filePath = filePath + "/" + objectName + return filePath, nil + } + return filePath, fmt.Errorf("download file:%s already exist\n", filePath) + } + return filePath, nil +} From e8d25c6824300da63c6c63f0a4224aad3fa0edaf Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Mon, 18 Sep 2023 17:52:32 +0800 Subject: [PATCH 09/13] fix: fix upload progress --- cmd/cmd_object.go | 23 ----------------------- cmd/utils.go | 19 ++++++++++++++----- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index 35a3c55..caec0fe 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -571,29 +571,6 @@ func getObject(ctx *cli.Context) error { } } -<<<<<<< HEAD -======= - filePath, err = checkIfDownloadFileExist(filePath, objectName) - if err != nil { - return toCmdErr(err) - } - dir := filepath.Dir(filePath) - fileName := "." + filepath.Base(filePath) + ".tmp" - tempFilePath := filepath.Join(dir, fileName) - - tempFilePath, err = checkIfDownloadFileExist(tempFilePath, objectName) - if err != nil { - return toCmdErr(err) - } - - // download to the temp file firstly - fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_WRONLY, 0660) - if err != nil { - return err - } - - defer fd.Close() ->>>>>>> 0adb378 (fix: fix download check exist file) opt := sdktypes.GetObjectOptions{} startOffset := ctx.Int64(startOffsetFlag) endOffset := ctx.Int64(endOffsetFlag) diff --git a/cmd/utils.go b/cmd/utils.go index 3ed2508..8a72758 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -105,6 +105,7 @@ const ( noBalanceErr = "key not found" maxListMemberNum = 1000 + objectLargerSize = 10 * 1024 * 1024 ) var ( @@ -541,10 +542,11 @@ func parseFileByArg(ctx *cli.Context, argIndex int) (int64, error) { type ProgressReader struct { io.Reader - Total int64 - Current int64 - StartTime time.Time - LastPrinted time.Time + Total int64 + Current int64 + StartTime time.Time + LastPrinted time.Time + LastPrintedStr string } func (pr *ProgressReader) Read(p []byte) (int, error) { @@ -561,8 +563,15 @@ func (pr *ProgressReader) printProgress() { uploadSpeed := float64(pr.Current) / elapsed.Seconds() if now.Sub(pr.LastPrinted) >= time.Second { // print rate every second - fmt.Printf("\ruploading progress: %.2f%% [ %s / %s ], rate: %10s", + progressStr := fmt.Sprintf("uploading progress: %.2f%% [ %s / %s ], rate: %s", progress, getConvertSize(pr.Current), getConvertSize(pr.Total), getConvertRate(uploadSpeed)) + + // Clear current line + fmt.Print("\r", strings.Repeat(" ", len(pr.LastPrintedStr)), "\r") + + // Print new progress + fmt.Print(progressStr) + pr.LastPrinted = now } } From fd66db944798b7f307b2784d2736e283cc882436 Mon Sep 17 00:00:00 2001 From: Chris Li Date: Mon, 18 Sep 2023 17:49:54 +0800 Subject: [PATCH 10/13] feat: support resumable download (#89) * feat: support resumable download * feat: fix some comment --- cmd/cmd_object.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index caec0fe..b1e3164 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -403,9 +403,14 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte contentType := ctx.String(contentTypeFlag) secondarySPAccs := ctx.String(secondarySPFlag) partSize := ctx.Uint64(partSizeFlag) +<<<<<<< HEAD resumableUpload := ctx.Bool(resumableFlag) +======= +>>>>>>> 83af1a5 (feat: support resumable download (#89)) bypassSeal := ctx.Bool(bypassSealFlag) + resumableUpload := ctx.Bool(resumableFlag) + opts := sdktypes.CreateObjectOptions{} if contentType != "" { opts.ContentType = contentType @@ -531,6 +536,7 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte // getObject download the object payload from sp func getObject(ctx *cli.Context) error { + var err error if ctx.NArg() < 1 { return toCmdErr(fmt.Errorf("args number less than one")) } @@ -571,6 +577,11 @@ func getObject(ctx *cli.Context) error { } } + filePath, err = checkIfDownloadFileExist(filePath, objectName) + if err != nil { + return toCmdErr(err) + } + opt := sdktypes.GetObjectOptions{} startOffset := ctx.Int64(startOffsetFlag) endOffset := ctx.Int64(endOffsetFlag) From 9ced606031212222894acbf5fcb46bcebc5dd95f Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Mon, 18 Sep 2023 18:56:21 +0800 Subject: [PATCH 11/13] fix: fix confict --- cmd/cmd_object.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index b1e3164..85f1b7d 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -403,13 +403,8 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte contentType := ctx.String(contentTypeFlag) secondarySPAccs := ctx.String(secondarySPFlag) partSize := ctx.Uint64(partSizeFlag) -<<<<<<< HEAD resumableUpload := ctx.Bool(resumableFlag) - -======= ->>>>>>> 83af1a5 (feat: support resumable download (#89)) bypassSeal := ctx.Bool(bypassSealFlag) - resumableUpload := ctx.Bool(resumableFlag) opts := sdktypes.CreateObjectOptions{} if contentType != "" { From 7eb10c5b93576c4fd9388986b5c247d7d1960a54 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Mon, 18 Sep 2023 19:25:14 +0800 Subject: [PATCH 12/13] fix: fix confict --- cmd/cmd_object.go | 29 +++++++++++++++++++++-------- cmd/utils.go | 2 -- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index 85f1b7d..84a14aa 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -481,19 +481,32 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte } defer reader.Close() - progressReader := &ProgressReader{ - Reader: reader, - Total: objectSize, - } - // if the file is more than 2G , it needs to force use resume uploading if objectSize > maxPutWithoutResumeSize { opt.DisableResumable = false } - if err = gnfdClient.PutObject(c, bucketName, objectName, - objectSize, progressReader, opt); err != nil { - return toCmdErr(err) + if opt.DisableResumable { + progressReader := &ProgressReader{ + Reader: reader, + Total: objectSize, + StartTime: time.Now(), + LastPrinted: time.Now(), + } + + if objectSize > objectLargerSize { + progressReader.LastPrinted = time.Now().Add(2 * time.Second) + } + + if err = gnfdClient.PutObject(c, bucketName, objectName, + objectSize, progressReader, opt); err != nil { + return toCmdErr(err) + } + } else { + if err = gnfdClient.PutObject(c, bucketName, objectName, + objectSize, reader, opt); err != nil { + return toCmdErr(err) + } } if bypassSeal { diff --git a/cmd/utils.go b/cmd/utils.go index 8a72758..40f8b2f 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -565,10 +565,8 @@ func (pr *ProgressReader) printProgress() { if now.Sub(pr.LastPrinted) >= time.Second { // print rate every second progressStr := fmt.Sprintf("uploading progress: %.2f%% [ %s / %s ], rate: %s", progress, getConvertSize(pr.Current), getConvertSize(pr.Total), getConvertRate(uploadSpeed)) - // Clear current line fmt.Print("\r", strings.Repeat(" ", len(pr.LastPrintedStr)), "\r") - // Print new progress fmt.Print(progressStr) From 5905819755be44748ba8af3361458bdc1114f9b6 Mon Sep 17 00:00:00 2001 From: flywukong <2229306838@qq.com> Date: Tue, 19 Sep 2023 10:39:51 +0800 Subject: [PATCH 13/13] fix: fix comment --- cmd/cmd_object.go | 5 +++-- cmd/utils.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/cmd_object.go b/cmd/cmd_object.go index 84a14aa..69ed671 100644 --- a/cmd/cmd_object.go +++ b/cmd/cmd_object.go @@ -494,7 +494,8 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte LastPrinted: time.Now(), } - if objectSize > objectLargerSize { + // if print big file progress, the printing progress should be delayed to obtain a more accurate display. + if objectSize > progressDelayPrintSize { progressReader.LastPrinted = time.Now().Add(2 * time.Second) } @@ -523,7 +524,7 @@ func uploadFile(bucketName, objectName, filePath, urlInfo string, ctx *cli.Conte for { select { case <-timeout: - return toCmdErr(errors.New("object not sealed after 15 seconds")) + return toCmdErr(errors.New("object not sealed after 1 hour")) case <-ticker.C: count++ headObjOutput, queryErr := gnfdClient.HeadObject(c, bucketName, objectName) diff --git a/cmd/utils.go b/cmd/utils.go index 40f8b2f..96da141 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -103,9 +103,9 @@ const ( StatusSPrefix = "STATUS_" defaultMaxKey = 500 - noBalanceErr = "key not found" - maxListMemberNum = 1000 - objectLargerSize = 10 * 1024 * 1024 + noBalanceErr = "key not found" + maxListMemberNum = 1000 + progressDelayPrintSize = 10 * 1024 * 1024 ) var (