From 151923a44cedf3f5ed131c4047d56d2dccae7c42 Mon Sep 17 00:00:00 2001 From: Javad Date: Thu, 18 Jul 2024 12:09:06 +0330 Subject: [PATCH 01/11] feat: add import command to download snapshots --- cmd/daemon/import.go | 162 +++++++++++++++++++++++++++++++++++ cmd/daemon/main.go | 1 + cmd/downlaod_mgr.go | 198 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+) create mode 100644 cmd/daemon/import.go create mode 100644 cmd/downlaod_mgr.go diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go new file mode 100644 index 000000000..30d8b2c9e --- /dev/null +++ b/cmd/daemon/import.go @@ -0,0 +1,162 @@ +package main + +import ( + "fmt" + "github.com/gofrs/flock" + "github.com/pactus-project/pactus/cmd" + "github.com/pactus-project/pactus/genesis" + "github.com/pactus-project/pactus/util" + "github.com/spf13/cobra" + "os" + "path/filepath" + "sort" + "strings" +) + +func buildImportCmd(parentCmd *cobra.Command) { + importCmd := &cobra.Command{ + Use: "import", + Short: "download and import pruned data", + } + parentCmd.AddCommand(importCmd) + + workingDirOpt := addWorkingDirOption(importCmd) + + importCmd.Run = func(c *cobra.Command, args []string) { + workingDir, err := filepath.Abs(*workingDirOpt) + cmd.FatalErrorCheck(err) + + err = os.Chdir(workingDir) + cmd.FatalErrorCheck(err) + + conf, gen, err := cmd.MakeConfig(workingDir) + cmd.FatalErrorCheck(err) + + lockFilePath := filepath.Join(workingDir, ".pactus.lock") + fileLock := flock.New(lockFilePath) + + locked, err := fileLock.TryLock() + if err != nil { + // handle unable to attempt to acquire lock + cmd.FatalErrorCheck(err) + } + + if !locked { + cmd.PrintWarnMsgf("Could not lock '%s', another instance is running?", lockFilePath) + + return + } + + cmd.PrintInfoMsgf("Checking data in %s exists...", workingDir) + storeDir, _ := filepath.Abs(conf.Store.StorePath()) + if !util.IsDirNotExistsOrEmpty(storeDir) { + cmd.PrintErrorMsgf("The data directory is not empty: %s", conf.Store.StorePath()) + + return + } + + snapshotUrl := "" + + switch gen.ChainType() { + case genesis.Mainnet: + snapshotUrl = cmd.SnapshotBaseUrl + "mainnet/" + case genesis.Testnet: + snapshotUrl = cmd.SnapshotBaseUrl + "testnet/" + default: + cmd.PrintErrorMsgf("Unsupported chain type: %s", gen.ChainType()) + + return + } + + cmd.PrintInfoMsgf("Getting snapshots metadata...") + metadata, err := cmd.SnapshotMetadata(c.Context(), snapshotUrl) + if err != nil { + cmd.PrintErrorMsgf("Failed to get snapshot metadata: %s", err) + + return + } + + sort.Slice(metadata, func(i, j int) bool { + return i > j + }) + + cmd.PrintLine() + + for i, m := range metadata { + fmt.Printf("%d. snapshot %s (%.2f MB)\n", i+1, + m.CreatedAt, + float64(m.TotalSize)/1024/1024) + } + + cmd.PrintLine() + + var choice int + fmt.Printf("Please select a snapshot [1-%d]: ", len(metadata)) + _, err = fmt.Scanf("%d", &choice) + if err != nil { + cmd.PrintErrorMsgf("invalid input: %s", err) + + return + } + + if choice < 1 || choice > len(metadata) { + cmd.PrintErrorMsgf("Invalid choice.") + + return + } + + selected := metadata[choice-1] + tmpDir := util.TempDirPath() + extractPath := fmt.Sprintf("%s/extracted", tmpDir) + + err = os.MkdirAll(extractPath, os.ModePerm) + cmd.FatalErrorCheck(err) + + cmd.PrintLine() + + zipFileList := make([]string, 0) + + cmd.DownloadManager( + c.Context(), + selected, + snapshotUrl, + tmpDir, + zipFileList, + downloadProgressBar, + ) + + for _, zFile := range zipFileList { + err := cmd.ExtractAndStoreFile(zFile, extractPath) + cmd.FatalErrorCheck(err) + } + + err = os.MkdirAll(filepath.Dir(conf.Store.StorePath()), os.ModePerm) + cmd.FatalErrorCheck(err) + + err = cmd.CopyAllFiles(extractPath, conf.Store.StorePath()) + cmd.FatalErrorCheck(err) + + err = os.RemoveAll(tmpDir) + cmd.FatalErrorCheck(err) + + _ = fileLock.Unlock() + + } +} + +func downloadProgressBar(fileName string, totalSize, downloaded int64, percentage float64) { + barWidth := 30 + completedWidth := int(float64(barWidth) * (percentage / 100)) + + progressBar := fmt.Sprintf( + "\r[%-*s] %3.0f%% %s (%.2f MB/ %.2f MB)", + barWidth, + strings.Repeat("=", completedWidth), + percentage, + fileName, + float64(downloaded)/1024/1024, + float64(totalSize)/1024/1024, + ) + + fmt.Print(progressBar) +} diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index f877ef29d..424f42945 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -24,6 +24,7 @@ func main() { buildInitCmd(rootCmd) buildStartCmd(rootCmd) buildPruneCmd(rootCmd) + buildImportCmd(rootCmd) err := rootCmd.Execute() if err != nil { diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go new file mode 100644 index 000000000..4ab662aea --- /dev/null +++ b/cmd/downlaod_mgr.go @@ -0,0 +1,198 @@ +package cmd + +import ( + "archive/zip" + "context" + "encoding/json" + "errors" + "fmt" + "github.com/pactus-project/pactus/util/downloader" + "io" + "net/http" + "net/url" + "os" + "path/filepath" +) + +const ( + SnapshotBaseUrl = "https://data.pacviewer.com/files/" +) + +type Metadata struct { + Name string `json:"name"` + CreatedAt string `json:"created_at"` + Compress string `json:"compress"` + TotalSize int `json:"total_size"` + Data []struct { + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + Size int `json:"size"` + } `json:"data"` +} + +func SnapshotMetadata(ctx context.Context, snapshotUrl string) ([]Metadata, error) { + cli := http.DefaultClient + metaUrl, err := url.JoinPath(snapshotUrl, "metadata.json") + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaUrl, http.NoBody) + if err != nil { + return nil, err + } + + resp, err := cli.Do(req) + if err != nil { + return nil, err + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New(resp.Status) + } + + metadata := make([]Metadata, 0) + return metadata, json.NewDecoder(resp.Body).Decode(&metadata) +} + +func DownloadManager( + ctx context.Context, + metadata Metadata, + baseUrl, tempPath string, + zipFileListPath []string, + stateFunc func(fileName string, totalSize, downloaded int64, percentage float64), +) { + + for _, data := range metadata.Data { + done := make(chan struct{}) + dlLink, err := url.JoinPath(baseUrl, data.Path) + FatalErrorCheck(err) + + fileName := filepath.Base(dlLink) + + filePath := fmt.Sprintf("%s/%s", tempPath, fileName) + + dl := downloader.New( + dlLink, + filePath, + data.Sha, + ) + + dl.Start(ctx) + + go func() { + for state := range dl.Stats() { + stateFunc(fileName, state.TotalSize, state.Downloaded, state.Percent) + if state.Completed { + done <- struct{}{} + close(done) + return + } + } + }() + + <-done + zipFileListPath = append(zipFileListPath, filePath) + } +} + +func ExtractAndStoreFile(zipFilePath, extractPath string) error { + r, err := zip.OpenReader(zipFilePath) + if err != nil { + return fmt.Errorf("failed to open zip file: %v", err) + } + defer func() { + _ = r.Close() + }() + + for _, f := range r.File { + rc, err := f.Open() + if err != nil { + return fmt.Errorf("failed to open file in zip archive: %v", err) + } + + fpath := filepath.Join(extractPath, f.Name) + + outFile, err := os.Create(fpath) + if err != nil { + return fmt.Errorf("failed to create file: %v", err) + } + + _, err = io.Copy(outFile, rc) + if err != nil { + return fmt.Errorf("failed to copy file contents: %v", err) + } + + _ = rc.Close() + _ = outFile.Close() + } + + return nil +} + +// CopyAllFiles copies all files from srcDir to dstDir +func CopyAllFiles(srcDir, dstDir string) error { + err := os.MkdirAll(dstDir, os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create destination directory: %v", err) + } + + return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil // Skip directories + } + + relativePath, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + + dstPath := filepath.Join(dstDir, relativePath) + + err = os.MkdirAll(filepath.Dir(dstPath), os.ModePerm) + if err != nil { + return fmt.Errorf("failed to create directory: %v", err) + } + + err = copyFile(path, dstPath) + if err != nil { + return fmt.Errorf("failed to copy file from %s to %s: %v", path, dstPath, err) + } + + return nil + }) +} + +func copyFile(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open source file: %v", err) + } + defer sourceFile.Close() + + destinationFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("failed to create destination file: %v", err) + } + defer destinationFile.Close() + + _, err = io.Copy(destinationFile, sourceFile) + if err != nil { + return fmt.Errorf("failed to copy file contents: %v", err) + } + + err = destinationFile.Sync() + if err != nil { + return fmt.Errorf("failed to sync destination file: %v", err) + } + + return nil +} From ac39a5bd90b51eb98e97e1ab2d5d52e090755dd9 Mon Sep 17 00:00:00 2001 From: Javad Date: Thu, 18 Jul 2024 21:58:31 +0330 Subject: [PATCH 02/11] fix: add extension file to compressed file name for prevent duplicate --- scripts/snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/snapshot.py b/scripts/snapshot.py index d97f6818c..cfcb70cd0 100644 --- a/scripts/snapshot.py +++ b/scripts/snapshot.py @@ -168,7 +168,7 @@ def create_snapshot(self): for file in files: file_path = os.path.join(root, file) file_name, file_ext = os.path.splitext(file) - compressed_file_name = f"{file_name}.{self.args.compress}" + compressed_file_name = f"{file_name}{file_ext}.{self.args.compress}" compressed_file_path = os.path.join(snapshot_dir, compressed_file_name) rel_path = os.path.relpath(compressed_file_path, self.args.snapshot_path) From cc1eee3306d391cf325b6a69b78c7bc5de96cffd Mon Sep 17 00:00:00 2001 From: Javad Date: Thu, 18 Jul 2024 21:59:09 +0330 Subject: [PATCH 03/11] feat: format bytest to human readable size --- util/downloader/downloader.go | 7 ++----- util/utils.go | 30 ++++++++++++++++++++++++++++++ util/utils_test.go | 19 +++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/util/downloader/downloader.go b/util/downloader/downloader.go index bae58fe09..ee842bd89 100644 --- a/util/downloader/downloader.go +++ b/util/downloader/downloader.go @@ -276,9 +276,6 @@ func (d *Downloader) stop() { } func (d *Downloader) handleError(err error) { - select { - case d.errCh <- err: - default: - d.stop() - } + d.errCh <- err + d.stop() } diff --git a/util/utils.go b/util/utils.go index 19f29711e..c84c5bc88 100644 --- a/util/utils.go +++ b/util/utils.go @@ -2,6 +2,7 @@ package util import ( crand "crypto/rand" + "fmt" "math/big" "math/bits" @@ -130,3 +131,32 @@ func LogScale(val int) int { return 1 << bitlen } + +func FormatBytesToHumanReadable(bytes uint64) string { + const ( + _ = iota + KB = 1 << (10 * iota) + MB + GB + TB + ) + unit := "Bytes" + value := float64(bytes) + + switch { + case bytes >= TB: + unit = "TB" + value /= TB + case bytes >= GB: + unit = "GB" + value /= GB + case bytes >= MB: + unit = "MB" + value /= MB + case bytes >= KB: + unit = "KB" + value /= KB + } + + return fmt.Sprintf("%.2f %s", value, unit) +} diff --git a/util/utils_test.go b/util/utils_test.go index 1d125275a..2c4a4763d 100644 --- a/util/utils_test.go +++ b/util/utils_test.go @@ -118,3 +118,22 @@ func TestLogScale(t *testing.T) { assert.Equal(t, testCase.expected, result, "LogScale(%d) failed", testCase.input) } } + +func TestFormatBytesToHumanReadable(t *testing.T) { + tests := []struct { + bytes uint64 + expected string + }{ + {1048576, "1.00 MB"}, + {3145728, "3.00 MB"}, + {1024, "1.00 KB"}, + {512, "512.00 Bytes"}, + } + + for _, test := range tests { + result := FormatBytesToHumanReadable(test.bytes) + if result != test.expected { + t.Errorf("FormatBytesToHumanReadable(%d) returned %s, expected %s", test.bytes, result, test.expected) + } + } +} From eab634fbe8ae4201a2d9d1ebc04838056e59ab25 Mon Sep 17 00:00:00 2001 From: Javad Date: Thu, 18 Jul 2024 21:59:22 +0330 Subject: [PATCH 04/11] fix: linter errors --- cmd/downlaod_mgr.go | 116 +++++++++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 40 deletions(-) diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index 4ab662aea..bde76a608 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -6,39 +6,45 @@ import ( "encoding/json" "errors" "fmt" - "github.com/pactus-project/pactus/util/downloader" "io" "net/http" "net/url" "os" "path/filepath" -) -const ( - SnapshotBaseUrl = "https://data.pacviewer.com/files/" + "github.com/pactus-project/pactus/util/downloader" ) +const maxDecompressedSize = 10 << 20 // 10 MB + +var snapshotBaseUrls = []string{ + "https://download.pactus.org/files/", + "https://data.pacviewer.com/files/", +} + type Metadata struct { - Name string `json:"name"` - CreatedAt string `json:"created_at"` - Compress string `json:"compress"` - TotalSize int `json:"total_size"` - Data []struct { - Name string `json:"name"` - Path string `json:"path"` - Sha string `json:"sha"` - Size int `json:"size"` - } `json:"data"` + Name string `json:"name"` + CreatedAt string `json:"created_at"` + Compress string `json:"compress"` + TotalSize int `json:"total_size"` + Data []*SnapshotData `json:"data"` } -func SnapshotMetadata(ctx context.Context, snapshotUrl string) ([]Metadata, error) { +type SnapshotData struct { + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + Size int `json:"size"` +} + +func GetSnapshotMetadata(ctx context.Context, snapshotURL string) ([]Metadata, error) { cli := http.DefaultClient - metaUrl, err := url.JoinPath(snapshotUrl, "metadata.json") + metaURL, err := url.JoinPath(snapshotURL, "metadata.json") if err != nil { return nil, err } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaUrl, http.NoBody) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, metaURL, http.NoBody) if err != nil { return nil, err } @@ -56,20 +62,27 @@ func SnapshotMetadata(ctx context.Context, snapshotUrl string) ([]Metadata, erro } metadata := make([]Metadata, 0) - return metadata, json.NewDecoder(resp.Body).Decode(&metadata) + + dec := json.NewDecoder(resp.Body) + + if err := dec.Decode(&metadata); err != nil { + return nil, err + } + + return metadata, nil } func DownloadManager( ctx context.Context, - metadata Metadata, - baseUrl, tempPath string, - zipFileListPath []string, + metadata *Metadata, + baseURL, tempPath string, stateFunc func(fileName string, totalSize, downloaded int64, percentage float64), -) { +) []string { + zipFileListPath := make([]string, 0) for _, data := range metadata.Data { done := make(chan struct{}) - dlLink, err := url.JoinPath(baseUrl, data.Path) + dlLink, err := url.JoinPath(baseURL, data.Path) FatalErrorCheck(err) fileName := filepath.Base(dlLink) @@ -84,12 +97,18 @@ func DownloadManager( dl.Start(ctx) + go func() { + err := <-dl.Errors() + FatalErrorCheck(err) + }() + go func() { for state := range dl.Stats() { stateFunc(fileName, state.TotalSize, state.Downloaded, state.Percent) if state.Completed { done <- struct{}{} close(done) + return } } @@ -98,12 +117,14 @@ func DownloadManager( <-done zipFileListPath = append(zipFileListPath, filePath) } + + return zipFileListPath } func ExtractAndStoreFile(zipFilePath, extractPath string) error { r, err := zip.OpenReader(zipFilePath) if err != nil { - return fmt.Errorf("failed to open zip file: %v", err) + return fmt.Errorf("failed to open zip file: %w", err) } defer func() { _ = r.Close() @@ -112,19 +133,26 @@ func ExtractAndStoreFile(zipFilePath, extractPath string) error { for _, f := range r.File { rc, err := f.Open() if err != nil { - return fmt.Errorf("failed to open file in zip archive: %v", err) + return fmt.Errorf("failed to open file in zip archive: %w", err) } fpath := filepath.Join(extractPath, f.Name) outFile, err := os.Create(fpath) if err != nil { - return fmt.Errorf("failed to create file: %v", err) + return fmt.Errorf("failed to create file: %w", err) } - _, err = io.Copy(outFile, rc) + // fixed potential DoS vulnerability via decompression bomb + lr := io.LimitedReader{R: rc, N: maxDecompressedSize} + _, err = io.Copy(outFile, &lr) if err != nil { - return fmt.Errorf("failed to copy file contents: %v", err) + return fmt.Errorf("failed to copy file contents: %w", err) + } + + // check if the file size exceeds the limit + if lr.N <= 0 { + return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fpath) } _ = rc.Close() @@ -134,11 +162,15 @@ func ExtractAndStoreFile(zipFilePath, extractPath string) error { return nil } -// CopyAllFiles copies all files from srcDir to dstDir +func SnapshotServer() string { + return snapshotBaseUrls[0] +} + +// CopyAllFiles copies all files from srcDir to dstDir. func CopyAllFiles(srcDir, dstDir string) error { - err := os.MkdirAll(dstDir, os.ModePerm) + err := os.MkdirAll(dstDir, 0o750) if err != nil { - return fmt.Errorf("failed to create destination directory: %v", err) + return fmt.Errorf("failed to create destination directory: %w", err) } return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { @@ -157,14 +189,14 @@ func CopyAllFiles(srcDir, dstDir string) error { dstPath := filepath.Join(dstDir, relativePath) - err = os.MkdirAll(filepath.Dir(dstPath), os.ModePerm) + err = os.MkdirAll(filepath.Dir(dstPath), 0o750) if err != nil { - return fmt.Errorf("failed to create directory: %v", err) + return fmt.Errorf("failed to create directory: %w", err) } err = copyFile(path, dstPath) if err != nil { - return fmt.Errorf("failed to copy file from %s to %s: %v", path, dstPath, err) + return fmt.Errorf("failed to copy file from %s to %s: %w", path, dstPath, err) } return nil @@ -174,24 +206,28 @@ func CopyAllFiles(srcDir, dstDir string) error { func copyFile(src, dst string) error { sourceFile, err := os.Open(src) if err != nil { - return fmt.Errorf("failed to open source file: %v", err) + return fmt.Errorf("failed to open source file: %w", err) } - defer sourceFile.Close() + defer func() { + _ = sourceFile.Close() + }() destinationFile, err := os.Create(dst) if err != nil { - return fmt.Errorf("failed to create destination file: %v", err) + return fmt.Errorf("failed to create destination file: %w", err) } - defer destinationFile.Close() + defer func() { + _ = destinationFile.Close() + }() _, err = io.Copy(destinationFile, sourceFile) if err != nil { - return fmt.Errorf("failed to copy file contents: %v", err) + return fmt.Errorf("failed to copy file contents: %w", err) } err = destinationFile.Sync() if err != nil { - return fmt.Errorf("failed to sync destination file: %v", err) + return fmt.Errorf("failed to sync destination file: %w", err) } return nil From cb0ab9bb555f6434a53ab4829f685fde84121a74 Mon Sep 17 00:00:00 2001 From: Javad Date: Thu, 18 Jul 2024 21:59:39 +0330 Subject: [PATCH 05/11] fix: remove list zip files path --- cmd/daemon/import.go | 86 +++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index 30d8b2c9e..4d77fac83 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -2,15 +2,17 @@ package main import ( "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + "github.com/gofrs/flock" "github.com/pactus-project/pactus/cmd" "github.com/pactus-project/pactus/genesis" "github.com/pactus-project/pactus/util" "github.com/spf13/cobra" - "os" - "path/filepath" - "sort" - "strings" ) func buildImportCmd(parentCmd *cobra.Command) { @@ -21,8 +23,9 @@ func buildImportCmd(parentCmd *cobra.Command) { parentCmd.AddCommand(importCmd) workingDirOpt := addWorkingDirOption(importCmd) + serverAddrOpt := importCmd.Flags().String("server-addr", "", "custom import server") - importCmd.Run = func(c *cobra.Command, args []string) { + importCmd.Run = func(c *cobra.Command, _ []string) { workingDir, err := filepath.Abs(*workingDirOpt) cmd.FatalErrorCheck(err) @@ -36,10 +39,7 @@ func buildImportCmd(parentCmd *cobra.Command) { fileLock := flock.New(lockFilePath) locked, err := fileLock.TryLock() - if err != nil { - // handle unable to attempt to acquire lock - cmd.FatalErrorCheck(err) - } + cmd.FatalErrorCheck(err) if !locked { cmd.PrintWarnMsgf("Could not lock '%s', another instance is running?", lockFilePath) @@ -47,7 +47,6 @@ func buildImportCmd(parentCmd *cobra.Command) { return } - cmd.PrintInfoMsgf("Checking data in %s exists...", workingDir) storeDir, _ := filepath.Abs(conf.Store.StorePath()) if !util.IsDirNotExistsOrEmpty(storeDir) { cmd.PrintErrorMsgf("The data directory is not empty: %s", conf.Store.StorePath()) @@ -55,21 +54,26 @@ func buildImportCmd(parentCmd *cobra.Command) { return } - snapshotUrl := "" + serverAddr := cmd.SnapshotServer() + + if *serverAddrOpt != "" { + serverAddr = *serverAddrOpt + } + + snapshotURL := "mainnet/" switch gen.ChainType() { case genesis.Mainnet: - snapshotUrl = cmd.SnapshotBaseUrl + "mainnet/" + snapshotURL = serverAddr + "mainnet/" case genesis.Testnet: - snapshotUrl = cmd.SnapshotBaseUrl + "testnet/" - default: + snapshotURL = serverAddr + "testnet/" + case genesis.Localnet: cmd.PrintErrorMsgf("Unsupported chain type: %s", gen.ChainType()) return } - cmd.PrintInfoMsgf("Getting snapshots metadata...") - metadata, err := cmd.SnapshotMetadata(c.Context(), snapshotUrl) + metadata, err := cmd.GetSnapshotMetadata(c.Context(), snapshotURL) if err != nil { cmd.PrintErrorMsgf("Failed to get snapshot metadata: %s", err) @@ -83,9 +87,9 @@ func buildImportCmd(parentCmd *cobra.Command) { cmd.PrintLine() for i, m := range metadata { - fmt.Printf("%d. snapshot %s (%.2f MB)\n", i+1, - m.CreatedAt, - float64(m.TotalSize)/1024/1024) + fmt.Printf("%d. snapshot %s (%s)\n", i+1, + parseDate(m.CreatedAt), + util.FormatBytesToHumanReadable(uint64(m.TotalSize))) } cmd.PrintLine() @@ -93,11 +97,7 @@ func buildImportCmd(parentCmd *cobra.Command) { var choice int fmt.Printf("Please select a snapshot [1-%d]: ", len(metadata)) _, err = fmt.Scanf("%d", &choice) - if err != nil { - cmd.PrintErrorMsgf("invalid input: %s", err) - - return - } + cmd.FatalErrorCheck(err) if choice < 1 || choice > len(metadata) { cmd.PrintErrorMsgf("Invalid choice.") @@ -107,21 +107,18 @@ func buildImportCmd(parentCmd *cobra.Command) { selected := metadata[choice-1] tmpDir := util.TempDirPath() - extractPath := fmt.Sprintf("%s/extracted", tmpDir) + extractPath := fmt.Sprintf("%s/data", tmpDir) - err = os.MkdirAll(extractPath, os.ModePerm) + err = os.MkdirAll(extractPath, 0o750) cmd.FatalErrorCheck(err) cmd.PrintLine() - zipFileList := make([]string, 0) - - cmd.DownloadManager( + zipFileList := cmd.DownloadManager( c.Context(), - selected, - snapshotUrl, + &selected, + snapshotURL, tmpDir, - zipFileList, downloadProgressBar, ) @@ -130,7 +127,7 @@ func buildImportCmd(parentCmd *cobra.Command) { cmd.FatalErrorCheck(err) } - err = os.MkdirAll(filepath.Dir(conf.Store.StorePath()), os.ModePerm) + err = os.MkdirAll(filepath.Dir(conf.Store.StorePath()), 0o750) cmd.FatalErrorCheck(err) err = cmd.CopyAllFiles(extractPath, conf.Store.StorePath()) @@ -141,6 +138,12 @@ func buildImportCmd(parentCmd *cobra.Command) { _ = fileLock.Unlock() + cmd.PrintLine() + cmd.PrintLine() + cmd.PrintInfoMsgf("✅ Your node successfully imported prune data.") + cmd.PrintLine() + cmd.PrintInfoMsgf("You can start the node by running this command:") + cmd.PrintInfoMsgf("./pactus-daemon start -w %v", workingDir) } } @@ -149,14 +152,25 @@ func downloadProgressBar(fileName string, totalSize, downloaded int64, percentag completedWidth := int(float64(barWidth) * (percentage / 100)) progressBar := fmt.Sprintf( - "\r[%-*s] %3.0f%% %s (%.2f MB/ %.2f MB)", + "\r[%-*s] %3.0f%% %s (%s/ %s)", barWidth, strings.Repeat("=", completedWidth), percentage, fileName, - float64(downloaded)/1024/1024, - float64(totalSize)/1024/1024, + util.FormatBytesToHumanReadable(uint64(downloaded)), + util.FormatBytesToHumanReadable(uint64(totalSize)), ) fmt.Print(progressBar) } + +func parseDate(dateString string) string { + const layout = "2006-01-02T15:04:05.000000" + + parsedTime, err := time.Parse(layout, dateString) + if err != nil { + return "" + } + + return parsedTime.Format("2006-01-02") +} From 358c3cbdc6ebd93ef73d89219e34b485a17ce525 Mon Sep 17 00:00:00 2001 From: Javad Date: Thu, 18 Jul 2024 22:07:43 +0330 Subject: [PATCH 06/11] fix: linter errors --- cmd/daemon/import.go | 9 +++++---- cmd/downlaod_mgr.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index 4d77fac83..61bf76f33 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -81,13 +81,14 @@ func buildImportCmd(parentCmd *cobra.Command) { } sort.Slice(metadata, func(i, j int) bool { - return i > j + return i > j //nolint }) cmd.PrintLine() for i, m := range metadata { - fmt.Printf("%d. snapshot %s (%s)\n", i+1, + fmt.Printf("%d. snapshot %s (%s)\n", //nolint + i+1, parseDate(m.CreatedAt), util.FormatBytesToHumanReadable(uint64(m.TotalSize))) } @@ -95,7 +96,7 @@ func buildImportCmd(parentCmd *cobra.Command) { cmd.PrintLine() var choice int - fmt.Printf("Please select a snapshot [1-%d]: ", len(metadata)) + fmt.Printf("Please select a snapshot [1-%d]: ", len(metadata)) //nolint _, err = fmt.Scanf("%d", &choice) cmd.FatalErrorCheck(err) @@ -161,7 +162,7 @@ func downloadProgressBar(fileName string, totalSize, downloaded int64, percentag util.FormatBytesToHumanReadable(uint64(totalSize)), ) - fmt.Print(progressBar) + fmt.Print(progressBar) //nolint } func parseDate(dateString string) string { diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index bde76a608..11e160ac5 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -136,7 +136,7 @@ func ExtractAndStoreFile(zipFilePath, extractPath string) error { return fmt.Errorf("failed to open file in zip archive: %w", err) } - fpath := filepath.Join(extractPath, f.Name) + fpath := fmt.Sprintf("%s/%s", extractPath, f.Name) outFile, err := os.Create(fpath) if err != nil { From da348d0fc4eb4614a7c6db97e2e6d263ae0602bd Mon Sep 17 00:00:00 2001 From: Javad Date: Fri, 19 Jul 2024 12:46:41 +0330 Subject: [PATCH 07/11] fix: add prompt select and custom progress bar --- cmd/cmd.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cmd/cmd.go b/cmd/cmd.go index b59cc6bf5..9e873a1c1 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + "github.com/k0kubun/go-ansi" "github.com/manifoldco/promptui" "github.com/pactus-project/pactus/config" "github.com/pactus-project/pactus/crypto" @@ -29,6 +30,7 @@ import ( "github.com/pactus-project/pactus/wallet" "github.com/pactus-project/pactus/wallet/addresspath" "github.com/pactus-project/pactus/wallet/vault" + "github.com/schollz/progressbar/v3" ) const ( @@ -121,6 +123,20 @@ func PromptInput(label string) string { return result } +// PromptSelect prompts create choice menu for select by user. +func PromptSelect(label string, items []string) int { + prompt := promptui.Select{ + Label: label, + Items: items, + Pointer: promptui.PipeCursor, + } + + choice, _, err := prompt.Run() + FatalErrorCheck(err) + + return choice +} + // PromptInputWithSuggestion prompts the user for an input string with a suggestion. func PromptInputWithSuggestion(label, suggestion string) string { prompt := promptui.Prompt{ @@ -617,3 +633,27 @@ func MakeValidatorKey(walletInstance *wallet.Wallet, valAddrsInfo []vault.Addres return valKeys, nil } + +func TerminalProgressBar(totalSize, barWidth int, showBytes bool) *progressbar.ProgressBar { + if barWidth < 15 { + barWidth = 15 + } + + opts := []progressbar.Option{ + progressbar.OptionSetWriter(ansi.NewAnsiStdout()), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionShowBytes(showBytes), + progressbar.OptionSetWidth(barWidth), + progressbar.OptionSetElapsedTime(false), + progressbar.OptionSetPredictTime(false), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + } + + return progressbar.NewOptions(totalSize, opts...) +} From 2203b0cb08d62273cf6e611c925d6fe14277b39e Mon Sep 17 00:00:00 2001 From: Javad Date: Fri, 19 Jul 2024 12:46:55 +0330 Subject: [PATCH 08/11] fix: conver GB and TB in test --- util/utils_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/utils_test.go b/util/utils_test.go index 2c4a4763d..0b248535d 100644 --- a/util/utils_test.go +++ b/util/utils_test.go @@ -128,6 +128,8 @@ func TestFormatBytesToHumanReadable(t *testing.T) { {3145728, "3.00 MB"}, {1024, "1.00 KB"}, {512, "512.00 Bytes"}, + {1_073_741_824, "1.00 GB"}, + {1_099_511_627_776, "1.00 TB"}, } for _, test := range tests { From 36e7d3ca691a4cd40cfce1619e93e0d7e5fc4096 Mon Sep 17 00:00:00 2001 From: Javad Date: Fri, 19 Jul 2024 12:47:24 +0330 Subject: [PATCH 09/11] fix: update with new progress bar --- cmd/daemon/prune.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/cmd/daemon/prune.go b/cmd/daemon/prune.go index 80b2978c4..3fd49ade9 100644 --- a/cmd/daemon/prune.go +++ b/cmd/daemon/prune.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "github.com/gofrs/flock" "github.com/pactus-project/pactus/cmd" @@ -115,15 +114,8 @@ func buildPruneCmd(parentCmd *cobra.Command) { } func pruningProgressBar(prunedCount, skippedCount, totalCount uint32) { - percentage := float64(prunedCount+skippedCount) / float64(totalCount) * 100 - if percentage > 100 { - percentage = 100 - } - - barLength := 40 - filledLength := int(float64(barLength) * percentage / 100) - - bar := strings.Repeat("=", filledLength) + strings.Repeat(" ", barLength-filledLength) - fmt.Printf("\r [%s] %.0f%% Pruned: %d | Skipped: %d", //nolint - bar, percentage, prunedCount, skippedCount) + bar := cmd.TerminalProgressBar(int(totalCount), 30, false) + bar.Describe(fmt.Sprintf("Pruned: %d | Skipped: %d", prunedCount, skippedCount)) + err := bar.Add(int(prunedCount + skippedCount)) + cmd.FatalErrorCheck(err) } From 0f19bddef4e21894800ddea9f4a08842ff9c0495 Mon Sep 17 00:00:00 2001 From: Javad Date: Fri, 19 Jul 2024 12:48:06 +0330 Subject: [PATCH 10/11] fix: change size type to uint64 --- cmd/downlaod_mgr.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index 11e160ac5..ed00b6bba 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -17,16 +17,11 @@ import ( const maxDecompressedSize = 10 << 20 // 10 MB -var snapshotBaseUrls = []string{ - "https://download.pactus.org/files/", - "https://data.pacviewer.com/files/", -} - type Metadata struct { Name string `json:"name"` CreatedAt string `json:"created_at"` Compress string `json:"compress"` - TotalSize int `json:"total_size"` + TotalSize uint64 `json:"total_size"` Data []*SnapshotData `json:"data"` } @@ -34,7 +29,7 @@ type SnapshotData struct { Name string `json:"name"` Path string `json:"path"` Sha string `json:"sha"` - Size int `json:"size"` + Size uint64 `json:"size"` } func GetSnapshotMetadata(ctx context.Context, snapshotURL string) ([]Metadata, error) { @@ -162,10 +157,6 @@ func ExtractAndStoreFile(zipFilePath, extractPath string) error { return nil } -func SnapshotServer() string { - return snapshotBaseUrls[0] -} - // CopyAllFiles copies all files from srcDir to dstDir. func CopyAllFiles(srcDir, dstDir string) error { err := os.MkdirAll(dstDir, 0o750) From 38caa3e6a2e88bc2752036cbb7274f816046c630 Mon Sep 17 00:00:00 2001 From: Javad Date: Fri, 19 Jul 2024 12:49:07 +0330 Subject: [PATCH 11/11] fix: used prompt select for choice item --- cmd/daemon/import.go | 72 ++++++++++++++------------------------------ go.mod | 7 ++++- go.sum | 14 +++++++++ 3 files changed, 42 insertions(+), 51 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index 61bf76f33..b9eebe7f1 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -4,8 +4,6 @@ import ( "fmt" "os" "path/filepath" - "sort" - "strings" "time" "github.com/gofrs/flock" @@ -23,7 +21,8 @@ func buildImportCmd(parentCmd *cobra.Command) { parentCmd.AddCommand(importCmd) workingDirOpt := addWorkingDirOption(importCmd) - serverAddrOpt := importCmd.Flags().String("server-addr", "", "custom import server") + serverAddrOpt := importCmd.Flags().String("server-addr", "https://download.pactus.org", + "import server address") importCmd.Run = func(c *cobra.Command, _ []string) { workingDir, err := filepath.Abs(*workingDirOpt) @@ -54,19 +53,13 @@ func buildImportCmd(parentCmd *cobra.Command) { return } - serverAddr := cmd.SnapshotServer() - - if *serverAddrOpt != "" { - serverAddr = *serverAddrOpt - } - - snapshotURL := "mainnet/" + snapshotURL := *serverAddrOpt switch gen.ChainType() { case genesis.Mainnet: - snapshotURL = serverAddr + "mainnet/" + snapshotURL += "/mainnet/" case genesis.Testnet: - snapshotURL = serverAddr + "testnet/" + snapshotURL += "/testnet/" case genesis.Localnet: cmd.PrintErrorMsgf("Unsupported chain type: %s", gen.ChainType()) @@ -80,33 +73,22 @@ func buildImportCmd(parentCmd *cobra.Command) { return } - sort.Slice(metadata, func(i, j int) bool { - return i > j //nolint - }) + snapshots := make([]string, 0, len(metadata)) - cmd.PrintLine() + for _, m := range metadata { + item := fmt.Sprintf("snapshot %s (%s)", + parseTime(m.CreatedAt).Format("2006-01-02"), + util.FormatBytesToHumanReadable(m.TotalSize), + ) - for i, m := range metadata { - fmt.Printf("%d. snapshot %s (%s)\n", //nolint - i+1, - parseDate(m.CreatedAt), - util.FormatBytesToHumanReadable(uint64(m.TotalSize))) + snapshots = append(snapshots, item) } cmd.PrintLine() - var choice int - fmt.Printf("Please select a snapshot [1-%d]: ", len(metadata)) //nolint - _, err = fmt.Scanf("%d", &choice) - cmd.FatalErrorCheck(err) - - if choice < 1 || choice > len(metadata) { - cmd.PrintErrorMsgf("Invalid choice.") - - return - } + choice := cmd.PromptSelect("Please select a snapshot", snapshots) - selected := metadata[choice-1] + selected := metadata[choice] tmpDir := util.TempDirPath() extractPath := fmt.Sprintf("%s/data", tmpDir) @@ -148,30 +130,20 @@ func buildImportCmd(parentCmd *cobra.Command) { } } -func downloadProgressBar(fileName string, totalSize, downloaded int64, percentage float64) { - barWidth := 30 - completedWidth := int(float64(barWidth) * (percentage / 100)) - - progressBar := fmt.Sprintf( - "\r[%-*s] %3.0f%% %s (%s/ %s)", - barWidth, - strings.Repeat("=", completedWidth), - percentage, - fileName, - util.FormatBytesToHumanReadable(uint64(downloaded)), - util.FormatBytesToHumanReadable(uint64(totalSize)), - ) - - fmt.Print(progressBar) //nolint +func downloadProgressBar(fileName string, totalSize, downloaded int64, _ float64) { + bar := cmd.TerminalProgressBar(int(totalSize), 30, true) + bar.Describe(fileName) + err := bar.Add(int(downloaded)) + cmd.FatalErrorCheck(err) } -func parseDate(dateString string) string { +func parseTime(dateString string) time.Time { const layout = "2006-01-02T15:04:05.000000" parsedTime, err := time.Parse(layout, dateString) if err != nil { - return "" + return time.Time{} } - return parsedTime.Format("2006-01-02") + return parsedTime } diff --git a/go.mod b/go.mod index c625a42ec..c6549f1c6 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect + github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect @@ -103,6 +104,7 @@ require ( github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -144,6 +146,8 @@ require ( github.com/quic-go/quic-go v0.45.1 // indirect github.com/quic-go/webtransport-go v0.8.0 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/schollz/progressbar/v3 v3.14.4 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect @@ -160,7 +164,8 @@ require ( golang.org/x/mod v0.18.0 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.22.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect diff --git a/go.sum b/go.sum index 87f67f0e2..0e88d2230 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -311,6 +313,8 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8Rv github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -445,6 +449,8 @@ github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= @@ -454,6 +460,8 @@ github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.14.4 h1:W9ZrDSJk7eqmQhd3uxFNNcTr0QL+xuGNI9dEMrw0r74= +github.com/schollz/progressbar/v3 v3.14.4/go.mod h1:aT3UQ7yGm+2ZjeXPqsjTenwL3ddUiuZ0kfQ/2tHlyNI= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -693,8 +701,11 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -705,6 +716,9 @@ golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=