From a055d0c8b25d9f308f9c3e1b024c36e2f50a1208 Mon Sep 17 00:00:00 2001 From: Javad Rajabzadeh Date: Fri, 19 Jul 2024 20:10:09 +0330 Subject: [PATCH] feat(daemon): add import command to download pruned snapshots (#1424) --- cmd/cmd.go | 40 ++++++ cmd/daemon/import.go | 149 ++++++++++++++++++++++ cmd/daemon/main.go | 1 + cmd/daemon/prune.go | 16 +-- cmd/downlaod_mgr.go | 225 ++++++++++++++++++++++++++++++++++ go.mod | 7 +- go.sum | 14 +++ scripts/snapshot.py | 2 +- util/downloader/downloader.go | 7 +- util/utils.go | 30 +++++ util/utils_test.go | 21 ++++ 11 files changed, 493 insertions(+), 19 deletions(-) create mode 100644 cmd/daemon/import.go create mode 100644 cmd/downlaod_mgr.go diff --git a/cmd/cmd.go b/cmd/cmd.go index b59cc6bf5..9e873a1c1 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + "github.com/k0kubun/go-ansi" "github.com/manifoldco/promptui" "github.com/pactus-project/pactus/config" "github.com/pactus-project/pactus/crypto" @@ -29,6 +30,7 @@ import ( "github.com/pactus-project/pactus/wallet" "github.com/pactus-project/pactus/wallet/addresspath" "github.com/pactus-project/pactus/wallet/vault" + "github.com/schollz/progressbar/v3" ) const ( @@ -121,6 +123,20 @@ func PromptInput(label string) string { return result } +// PromptSelect prompts create choice menu for select by user. +func PromptSelect(label string, items []string) int { + prompt := promptui.Select{ + Label: label, + Items: items, + Pointer: promptui.PipeCursor, + } + + choice, _, err := prompt.Run() + FatalErrorCheck(err) + + return choice +} + // PromptInputWithSuggestion prompts the user for an input string with a suggestion. func PromptInputWithSuggestion(label, suggestion string) string { prompt := promptui.Prompt{ @@ -617,3 +633,27 @@ func MakeValidatorKey(walletInstance *wallet.Wallet, valAddrsInfo []vault.Addres return valKeys, nil } + +func TerminalProgressBar(totalSize, barWidth int, showBytes bool) *progressbar.ProgressBar { + if barWidth < 15 { + barWidth = 15 + } + + opts := []progressbar.Option{ + progressbar.OptionSetWriter(ansi.NewAnsiStdout()), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionShowBytes(showBytes), + progressbar.OptionSetWidth(barWidth), + progressbar.OptionSetElapsedTime(false), + progressbar.OptionSetPredictTime(false), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + } + + return progressbar.NewOptions(totalSize, opts...) +} diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go new file mode 100644 index 000000000..b9eebe7f1 --- /dev/null +++ b/cmd/daemon/import.go @@ -0,0 +1,149 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/gofrs/flock" + "github.com/pactus-project/pactus/cmd" + "github.com/pactus-project/pactus/genesis" + "github.com/pactus-project/pactus/util" + "github.com/spf13/cobra" +) + +func buildImportCmd(parentCmd *cobra.Command) { + importCmd := &cobra.Command{ + Use: "import", + Short: "download and import pruned data", + } + parentCmd.AddCommand(importCmd) + + workingDirOpt := addWorkingDirOption(importCmd) + serverAddrOpt := importCmd.Flags().String("server-addr", "https://download.pactus.org", + "import server address") + + importCmd.Run = func(c *cobra.Command, _ []string) { + workingDir, err := filepath.Abs(*workingDirOpt) + cmd.FatalErrorCheck(err) + + err = os.Chdir(workingDir) + cmd.FatalErrorCheck(err) + + conf, gen, err := cmd.MakeConfig(workingDir) + cmd.FatalErrorCheck(err) + + lockFilePath := filepath.Join(workingDir, ".pactus.lock") + fileLock := flock.New(lockFilePath) + + locked, err := fileLock.TryLock() + cmd.FatalErrorCheck(err) + + if !locked { + cmd.PrintWarnMsgf("Could not lock '%s', another instance is running?", lockFilePath) + + return + } + + storeDir, _ := filepath.Abs(conf.Store.StorePath()) + if !util.IsDirNotExistsOrEmpty(storeDir) { + cmd.PrintErrorMsgf("The data directory is not empty: %s", conf.Store.StorePath()) + + return + } + + snapshotURL := *serverAddrOpt + + switch gen.ChainType() { + case genesis.Mainnet: + snapshotURL += "/mainnet/" + case genesis.Testnet: + snapshotURL += "/testnet/" + case genesis.Localnet: + cmd.PrintErrorMsgf("Unsupported chain type: %s", gen.ChainType()) + + return + } + + metadata, err := cmd.GetSnapshotMetadata(c.Context(), snapshotURL) + if err != nil { + cmd.PrintErrorMsgf("Failed to get snapshot metadata: %s", err) + + return + } + + snapshots := make([]string, 0, len(metadata)) + + for _, m := range metadata { + item := fmt.Sprintf("snapshot %s (%s)", + parseTime(m.CreatedAt).Format("2006-01-02"), + util.FormatBytesToHumanReadable(m.TotalSize), + ) + + snapshots = append(snapshots, item) + } + + cmd.PrintLine() + + choice := cmd.PromptSelect("Please select a snapshot", snapshots) + + selected := metadata[choice] + tmpDir := util.TempDirPath() + extractPath := fmt.Sprintf("%s/data", tmpDir) + + err = os.MkdirAll(extractPath, 0o750) + cmd.FatalErrorCheck(err) + + cmd.PrintLine() + + zipFileList := cmd.DownloadManager( + c.Context(), + &selected, + snapshotURL, + tmpDir, + downloadProgressBar, + ) + + for _, zFile := range zipFileList { + err := cmd.ExtractAndStoreFile(zFile, extractPath) + cmd.FatalErrorCheck(err) + } + + err = os.MkdirAll(filepath.Dir(conf.Store.StorePath()), 0o750) + cmd.FatalErrorCheck(err) + + err = cmd.CopyAllFiles(extractPath, conf.Store.StorePath()) + cmd.FatalErrorCheck(err) + + err = os.RemoveAll(tmpDir) + cmd.FatalErrorCheck(err) + + _ = fileLock.Unlock() + + cmd.PrintLine() + cmd.PrintLine() + cmd.PrintInfoMsgf("✅ Your node successfully imported prune data.") + cmd.PrintLine() + cmd.PrintInfoMsgf("You can start the node by running this command:") + cmd.PrintInfoMsgf("./pactus-daemon start -w %v", workingDir) + } +} + +func downloadProgressBar(fileName string, totalSize, downloaded int64, _ float64) { + bar := cmd.TerminalProgressBar(int(totalSize), 30, true) + bar.Describe(fileName) + err := bar.Add(int(downloaded)) + cmd.FatalErrorCheck(err) +} + +func parseTime(dateString string) time.Time { + const layout = "2006-01-02T15:04:05.000000" + + parsedTime, err := time.Parse(layout, dateString) + if err != nil { + return time.Time{} + } + + return parsedTime +} diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index f877ef29d..424f42945 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -24,6 +24,7 @@ func main() { buildInitCmd(rootCmd) buildStartCmd(rootCmd) buildPruneCmd(rootCmd) + buildImportCmd(rootCmd) err := rootCmd.Execute() if err != nil { diff --git a/cmd/daemon/prune.go b/cmd/daemon/prune.go index 80b2978c4..3fd49ade9 100644 --- a/cmd/daemon/prune.go +++ b/cmd/daemon/prune.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "github.com/gofrs/flock" "github.com/pactus-project/pactus/cmd" @@ -115,15 +114,8 @@ func buildPruneCmd(parentCmd *cobra.Command) { } func pruningProgressBar(prunedCount, skippedCount, totalCount uint32) { - percentage := float64(prunedCount+skippedCount) / float64(totalCount) * 100 - if percentage > 100 { - percentage = 100 - } - - barLength := 40 - filledLength := int(float64(barLength) * percentage / 100) - - bar := strings.Repeat("=", filledLength) + strings.Repeat(" ", barLength-filledLength) - fmt.Printf("\r [%s] %.0f%% Pruned: %d | Skipped: %d", //nolint - bar, percentage, prunedCount, skippedCount) + bar := cmd.TerminalProgressBar(int(totalCount), 30, false) + bar.Describe(fmt.Sprintf("Pruned: %d | Skipped: %d", prunedCount, skippedCount)) + err := bar.Add(int(prunedCount + skippedCount)) + cmd.FatalErrorCheck(err) } diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go new file mode 100644 index 000000000..ed00b6bba --- /dev/null +++ b/cmd/downlaod_mgr.go @@ -0,0 +1,225 @@ +package cmd + +import ( + "archive/zip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + + "github.com/pactus-project/pactus/util/downloader" +) + +const maxDecompressedSize = 10 << 20 // 10 MB + +type Metadata struct { + Name string `json:"name"` + CreatedAt string `json:"created_at"` + Compress string `json:"compress"` + TotalSize uint64 `json:"total_size"` + Data []*SnapshotData `json:"data"` +} + +type SnapshotData struct { + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + Size uint64 `json:"size"` +} + +func GetSnapshotMetadata(ctx context.Context, snapshotURL string) ([]Metadata, error) { + cli := http.DefaultClient + metaURL, err := url.JoinPath(snapshotURL, "metadata.json") + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, http.NoBody) + if err != nil { + return nil, err + } + + resp, err := cli.Do(req) + if err != nil { + return nil, err + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New(resp.Status) + } + + metadata := make([]Metadata, 0) + + dec := json.NewDecoder(resp.Body) + + if err := dec.Decode(&metadata); err != nil { + return nil, err + } + + return metadata, nil +} + +func DownloadManager( + ctx context.Context, + metadata *Metadata, + baseURL, tempPath string, + stateFunc func(fileName string, totalSize, downloaded int64, percentage float64), +) []string { + zipFileListPath := make([]string, 0) + + for _, data := range metadata.Data { + done := make(chan struct{}) + dlLink, err := url.JoinPath(baseURL, data.Path) + FatalErrorCheck(err) + + fileName := filepath.Base(dlLink) + + filePath := fmt.Sprintf("%s/%s", tempPath, fileName) + + dl := downloader.New( + dlLink, + filePath, + data.Sha, + ) + + dl.Start(ctx) + + go func() { + err := <-dl.Errors() + FatalErrorCheck(err) + }() + + go func() { + for state := range dl.Stats() { + stateFunc(fileName, state.TotalSize, state.Downloaded, state.Percent) + if state.Completed { + done <- struct{}{} + close(done) + + return + } + } + }() + + <-done + zipFileListPath = append(zipFileListPath, filePath) + } + + return zipFileListPath +} + +func ExtractAndStoreFile(zipFilePath, extractPath string) error { + r, err := zip.OpenReader(zipFilePath) + if err != nil { + return fmt.Errorf("failed to open zip file: %w", err) + } + defer func() { + _ = r.Close() + }() + + for _, f := range r.File { + rc, err := f.Open() + if err != nil { + return fmt.Errorf("failed to open file in zip archive: %w", err) + } + + fpath := fmt.Sprintf("%s/%s", extractPath, f.Name) + + outFile, err := os.Create(fpath) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + + // fixed potential DoS vulnerability via decompression bomb + lr := io.LimitedReader{R: rc, N: maxDecompressedSize} + _, err = io.Copy(outFile, &lr) + if err != nil { + return fmt.Errorf("failed to copy file contents: %w", err) + } + + // check if the file size exceeds the limit + if lr.N <= 0 { + return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fpath) + } + + _ = rc.Close() + _ = outFile.Close() + } + + return nil +} + +// CopyAllFiles copies all files from srcDir to dstDir. +func CopyAllFiles(srcDir, dstDir string) error { + err := os.MkdirAll(dstDir, 0o750) + if err != nil { + return fmt.Errorf("failed to create destination directory: %w", err) + } + + return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil // Skip directories + } + + relativePath, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + + dstPath := filepath.Join(dstDir, relativePath) + + err = os.MkdirAll(filepath.Dir(dstPath), 0o750) + if err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + err = copyFile(path, dstPath) + if err != nil { + return fmt.Errorf("failed to copy file from %s to %s: %w", path, dstPath, err) + } + + return nil + }) +} + +func copyFile(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open source file: %w", err) + } + defer func() { + _ = sourceFile.Close() + }() + + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("failed to create destination file: %w", err) + } + defer func() { + _ = destinationFile.Close() + }() + + _, err = io.Copy(destinationFile, sourceFile) + if err != nil { + return fmt.Errorf("failed to copy file contents: %w", err) + } + + err = destinationFile.Sync() + if err != nil { + return fmt.Errorf("failed to sync destination file: %w", err) + } + + return nil +} diff --git a/go.mod b/go.mod index c625a42ec..c6549f1c6 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect + github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect @@ -103,6 +104,7 @@ require ( github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -144,6 +146,8 @@ require ( github.com/quic-go/quic-go v0.45.1 // indirect github.com/quic-go/webtransport-go v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/schollz/progressbar/v3 v3.14.4 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect @@ -160,7 +164,8 @@ require ( golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.22.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect diff --git a/go.sum b/go.sum index 87f67f0e2..0e88d2230 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -311,6 +313,8 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -445,6 +449,8 @@ github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= @@ -454,6 +460,8 @@ github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.14.4 h1:W9ZrDSJk7eqmQhd3uxFNNcTr0QL+xuGNI9dEMrw0r74= +github.com/schollz/progressbar/v3 v3.14.4/go.mod h1:aT3UQ7yGm+2ZjeXPqsjTenwL3ddUiuZ0kfQ/2tHlyNI= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -693,8 +701,11 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -705,6 +716,9 @@ golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/scripts/snapshot.py b/scripts/snapshot.py index d97f6818c..cfcb70cd0 100644 --- a/scripts/snapshot.py +++ b/scripts/snapshot.py @@ -168,7 +168,7 @@ def create_snapshot(self): for file in files: file_path = os.path.join(root, file) file_name, file_ext = os.path.splitext(file) - compressed_file_name = f"{file_name}.{self.args.compress}" + compressed_file_name = f"{file_name}{file_ext}.{self.args.compress}" compressed_file_path = os.path.join(snapshot_dir, compressed_file_name) rel_path = os.path.relpath(compressed_file_path, self.args.snapshot_path) diff --git a/util/downloader/downloader.go b/util/downloader/downloader.go index bae58fe09..ee842bd89 100644 --- a/util/downloader/downloader.go +++ b/util/downloader/downloader.go @@ -276,9 +276,6 @@ func (d *Downloader) stop() { } func (d *Downloader) handleError(err error) { - select { - case d.errCh <- err: - default: - d.stop() - } + d.errCh <- err + d.stop() } diff --git a/util/utils.go b/util/utils.go index 19f29711e..c84c5bc88 100644 --- a/util/utils.go +++ b/util/utils.go @@ -2,6 +2,7 @@ package util import ( crand "crypto/rand" + "fmt" "math/big" "math/bits" @@ -130,3 +131,32 @@ func LogScale(val int) int { return 1 << bitlen } + +func FormatBytesToHumanReadable(bytes uint64) string { + const ( + _ = iota + KB = 1 << (10 * iota) + MB + GB + TB + ) + unit := "Bytes" + value := float64(bytes) + + switch { + case bytes >= TB: + unit = "TB" + value /= TB + case bytes >= GB: + unit = "GB" + value /= GB + case bytes >= MB: + unit = "MB" + value /= MB + case bytes >= KB: + unit = "KB" + value /= KB + } + + return fmt.Sprintf("%.2f %s", value, unit) +} diff --git a/util/utils_test.go b/util/utils_test.go index 1d125275a..0b248535d 100644 --- a/util/utils_test.go +++ b/util/utils_test.go @@ -118,3 +118,24 @@ func TestLogScale(t *testing.T) { assert.Equal(t, testCase.expected, result, "LogScale(%d) failed", testCase.input) } } + +func TestFormatBytesToHumanReadable(t *testing.T) { + tests := []struct { + bytes uint64 + expected string + }{ + {1048576, "1.00 MB"}, + {3145728, "3.00 MB"}, + {1024, "1.00 KB"}, + {512, "512.00 Bytes"}, + {1_073_741_824, "1.00 GB"}, + {1_099_511_627_776, "1.00 TB"}, + } + + for _, test := range tests { + result := FormatBytesToHumanReadable(test.bytes) + if result != test.expected { + t.Errorf("FormatBytesToHumanReadable(%d) returned %s, expected %s", test.bytes, result, test.expected) + } + } +}