From bc1c59c01fefa2e6be93be89b800c6573e68be20 Mon Sep 17 00:00:00 2001 From: Javad Date: Sat, 20 Jul 2024 14:36:21 +0330 Subject: [PATCH 01/20] feat: add prune node widget in assitants GUI --- cmd/daemon/import.go | 19 +-- cmd/downlaod_mgr.go | 12 ++ cmd/gtk/startup_assistant.go | 232 ++++++++++++++++++++++++++++++++++- 3 files changed, 243 insertions(+), 20 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index b9eebe7f1..52be5116e 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -2,15 +2,13 @@ package main import ( "fmt" - "os" - "path/filepath" - "time" - "github.com/gofrs/flock" "github.com/pactus-project/pactus/cmd" "github.com/pactus-project/pactus/genesis" "github.com/pactus-project/pactus/util" "github.com/spf13/cobra" + "os" + "path/filepath" ) func buildImportCmd(parentCmd *cobra.Command) { @@ -77,7 +75,7 @@ func buildImportCmd(parentCmd *cobra.Command) { for _, m := range metadata { item := fmt.Sprintf("snapshot %s (%s)", - parseTime(m.CreatedAt).Format("2006-01-02"), + cmd.ParseTime(m.CreatedAt).Format("2006-01-02"), util.FormatBytesToHumanReadable(m.TotalSize), ) @@ -136,14 +134,3 @@ func downloadProgressBar(fileName string, totalSize, downloaded int64, _ float64 err := bar.Add(int(downloaded)) cmd.FatalErrorCheck(err) } - -func parseTime(dateString string) time.Time { - const layout = "2006-01-02T15:04:05.000000" - - parsedTime, err := time.Parse(layout, dateString) - if err != nil { - return time.Time{} - } - - return parsedTime -} diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index ed00b6bba..f02036d63 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -11,6 +11,7 @@ import ( "net/url" "os" "path/filepath" + "time" "github.com/pactus-project/pactus/util/downloader" ) @@ -223,3 +224,14 @@ func copyFile(src, dst string) error { return nil } + +func ParseTime(dateString string) time.Time { + const layout = "2006-01-02T15:04:05.000000" + + parsedTime, err := time.Parse(layout, dateString) + if err != nil { + return time.Time{} + } + + return parsedTime +} diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index cfa910b51..743b8671a 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -3,16 +3,19 @@ package main import ( + "context" "fmt" - "log" - "regexp" - "strings" - "github.com/gotk3/gotk3/glib" "github.com/gotk3/gotk3/gtk" "github.com/pactus-project/pactus/cmd" "github.com/pactus-project/pactus/genesis" + "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/wallet" + "log" + "os" + "path/filepath" + "regexp" + "strings" ) type assistantFunc func(assistant *gtk.Assistant, content gtk.IWidget, name, @@ -36,6 +39,9 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { assistFunc := pageAssistant() + // -- page_node_mode + snapshotWidget, snapshotGrid, snapshotRadio, snapshotPageName := pageSnapshot(assistant, assistFunc) + // --- page_mode mode, restoreRadio, pageModeName := pageMode(assistant, assistFunc) @@ -68,6 +74,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { gtk.MainQuit() }) + assistant.SetPageType(snapshotWidget, gtk.ASSISTANT_PAGE_CONTENT) assistant.SetPageType(mode, gtk.ASSISTANT_PAGE_INTRO) // page 0 assistant.SetPageType(seedGenerate, gtk.ASSISTANT_PAGE_CONTENT) // page 1 assistant.SetPageType(seedConfirm, gtk.ASSISTANT_PAGE_CONTENT) // page 2 @@ -93,6 +100,130 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { log.Printf("%v (restore: %v, prev: %v, cur: %v)\n", curPageName, isRestoreMode, prevPageIndex, curPageIndex) switch curPageName { + case snapshotPageName: + assistantPageComplete(assistant, snapshotWidget, true) + ssLabel, err := gtk.LabelNew("") + cmd.FatalErrorCheck(err) + setMargin(ssLabel, 5, 5, 1, 1) + ssLabel.SetHAlign(gtk.ALIGN_START) + + listBox, err := gtk.ListBoxNew() + cmd.FatalErrorCheck(err) + setMargin(listBox, 5, 5, 1, 1) + listBox.SetHAlign(gtk.ALIGN_CENTER) + listBox.SetSizeRequest(600, -1) + + ssDLBtn, err := gtk.ButtonNewWithLabel("⏬ Download") + cmd.FatalErrorCheck(err) + setMargin(ssDLBtn, 10, 5, 1, 1) + ssDLBtn.SetHAlign(gtk.ALIGN_CENTER) + ssDLBtn.SetSizeRequest(600, -1) + + ssPBLabel, err := gtk.LabelNew("") + cmd.FatalErrorCheck(err) + setMargin(ssPBLabel, 5, 10, 1, 1) + ssPBLabel.SetHAlign(gtk.ALIGN_START) + + snapshotGrid.Attach(ssLabel, 0, 1, 1, 1) + snapshotGrid.Attach(listBox, 0, 2, 1, 1) + snapshotGrid.Attach(ssDLBtn, 0, 3, 1, 1) + snapshotGrid.Attach(ssPBLabel, 0, 5, 1, 1) + ssLabel.SetVisible(false) + listBox.SetVisible(false) + ssDLBtn.SetVisible(false) + ssPBLabel.SetVisible(false) + + snapshotIndex := 0 + + snapshotRadio.Connect("toggled", func() { + if snapshotRadio.GetActive() { + assistantPageComplete(assistant, snapshotWidget, false) + + snapshotURL := "https://download.pactus.org/mainnet/" + + tmpDir := util.TempDirPath() + extractPath := fmt.Sprintf("%s/data", tmpDir) + + err = os.MkdirAll(extractPath, 0o750) + cmd.FatalErrorCheck(err) + + glib.IdleAdd(func() { + ssLabel.SetText(" ♻️ Please wait, loading snapshots...") + ssLabel.SetVisible(true) + }) + + mdCh := getMetadata(context.Background(), snapshotURL, listBox) + + go func() { + if md := <-mdCh; md == nil { + ssLabel.SetText(" ❌ Failed to get snapshot list, please try again later.") + } else { + ssLabel.SetText(" πŸ”½ Please select a snapshot to download:") + listBox.SetVisible(true) + + listBox.Connect("row-selected", func(box *gtk.ListBox, row *gtk.ListBoxRow) { + if row != nil { + snapshotIndex = row.GetIndex() + ssDLBtn.SetVisible(true) + } + }) + + ssDLBtn.Connect("clicked", func() { + ssDLBtn.SetVisible(false) + listBox.SetSelectionMode(gtk.SELECTION_NONE) + ssPBLabel.SetVisible(true) + + go func() { + zipFileList := cmd.DownloadManager( + context.Background(), + md[snapshotIndex], + snapshotURL, + tmpDir, + func(fileName string, totalSize, downloaded int64, percentage float64) { + percent := int(percentage) + glib.IdleAdd(func() { + dlMessage := fmt.Sprintf("🌐 Downloading file %s... %d%% (%s / %s)", + fileName, + percent, + util.FormatBytesToHumanReadable(uint64(downloaded)), + util.FormatBytesToHumanReadable(uint64(totalSize)), + ) + ssPBLabel.SetText(" " + dlMessage) + }) + }, + ) + + ssPBLabel.SetText(" " + "πŸ“‚ Extracting downloaded files...") + for _, zFile := range zipFileList { + err := cmd.ExtractAndStoreFile(zFile, extractPath) + cmd.FatalErrorCheck(err) + } + + err = os.MkdirAll(filepath.Dir("/home/javad/pactus/data/store.db"), 0o750) + cmd.FatalErrorCheck(err) + + err = cmd.CopyAllFiles(extractPath, "/home/javad/pactus/data/store.db") + cmd.FatalErrorCheck(err) + + err = os.RemoveAll(tmpDir) + cmd.FatalErrorCheck(err) + + ssPBLabel.SetText(" " + "βœ… Import completed.") + assistantPageComplete(assistant, snapshotWidget, true) + }() + }) + + } + }() + + } else { + assistantPageComplete(assistant, snapshotWidget, true) + ssLabel.SetVisible(false) + listBox.SetVisible(false) + ssDLBtn.SetVisible(false) + ssPBLabel.SetVisible(false) + } + }) case pageModeName: assistantPageComplete(assistant, mode, true) @@ -385,6 +516,57 @@ To make sure that you have properly saved your seed, please retype it here.` return pageWidget, pageSeedConfirmName } +func pageSnapshot(assistant *gtk.Assistant, assistFunc assistantFunc) ( + *gtk.Widget, + *gtk.Grid, + *gtk.RadioButton, + string, +) { + pageWidget := new(gtk.Widget) + + vbox, err := gtk.BoxNew(gtk.ORIENTATION_VERTICAL, 0) + cmd.FatalErrorCheck(err) + + grid, err := gtk.GridNew() + cmd.FatalErrorCheck(err) + + fullNode, err := gtk.RadioButtonNewWithLabel(nil, "Full node") + cmd.FatalErrorCheck(err) + fullNode.SetActive(true) + + pruneNode, err := gtk.RadioButtonNewWithLabelFromWidget(fullNode, "Prune node") + cmd.FatalErrorCheck(err) + + radioBox, err := gtk.BoxNew(gtk.ORIENTATION_VERTICAL, 0) + cmd.FatalErrorCheck(err) + + radioBox.Add(fullNode) + setMargin(fullNode, 6, 6, 6, 6) + radioBox.Add(pruneNode) + setMargin(pruneNode, 6, 10, 6, 6) + + grid.Attach(radioBox, 0, 0, 1, 1) + + vbox.PackStart(grid, true, true, 0) + + pageName := "page_snapshot" + pageTitle := "Import" + pageSubject := "" + pageDesc := "" + + // Create and return the page widget using assistFunc + pageWidget = assistFunc( + assistant, + vbox, + pageName, + pageTitle, + pageSubject, + pageDesc, + ) + + return pageWidget, grid, pruneNode, pageName +} + func pagePassword(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk.Widget, *gtk.Entry, string) { pageWidget := new(gtk.Widget) entryPassword, err := gtk.EntryNew() @@ -546,3 +728,45 @@ func assistantPageComplete(assistant *gtk.Assistant, page gtk.IWidget, completed assistant.SetPageComplete(page, completed) assistant.UpdateButtonsState() } + +func getMetadata( + ctx context.Context, + snapshotURL string, + listBox *gtk.ListBox, +) <-chan []cmd.Metadata { + mdCh := make(chan []cmd.Metadata, 1) + + go func() { + defer close(mdCh) + + children := listBox.GetChildren() + for children.Length() > 0 { + child := children.Data().(*gtk.Widget) + listBox.Remove(child) + children = children.Next() + } + + metadata, err := cmd.GetSnapshotMetadata(ctx, snapshotURL) + if err != nil { + mdCh <- nil + return + } + + for _, md := range metadata { + listBoxRow, err := gtk.ListBoxRowNew() + cmd.FatalErrorCheck(err) + + label, err := gtk.LabelNew(fmt.Sprintf("snapshot %s (%s)", + cmd.ParseTime(md.CreatedAt).Format("2006-01-02"), + util.FormatBytesToHumanReadable(md.TotalSize), + )) + cmd.FatalErrorCheck(err) + + listBoxRow.Add(label) + listBox.Add(listBoxRow) + } + listBox.ShowAll() + mdCh <- metadata + }() + return mdCh +} From 8c69c8288a29902d6fe30eaca72273c77f926745 Mon Sep 17 00:00:00 2001 From: Javad Date: Sat, 20 Jul 2024 14:37:28 +0330 Subject: [PATCH 02/20] fix: passed metadata with ref --- cmd/gtk/startup_assistant.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index 743b8671a..094c58a07 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -176,7 +176,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { go func() { zipFileList := cmd.DownloadManager( context.Background(), - md[snapshotIndex], + &md[snapshotIndex], snapshotURL, tmpDir, func(fileName string, totalSize, downloaded int64, percentage float64) { From df46aa896d25f3410208e3d1ff3fe1a371f68eb7 Mon Sep 17 00:00:00 2001 From: Javad Date: Sat, 20 Jul 2024 14:41:11 +0330 Subject: [PATCH 03/20] fix: fmt error --- cmd/daemon/import.go | 5 +++-- cmd/gtk/startup_assistant.go | 11 ++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index 52be5116e..815393626 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -2,13 +2,14 @@ package main import ( "fmt" + "os" + "path/filepath" + "github.com/gofrs/flock" "github.com/pactus-project/pactus/cmd" "github.com/pactus-project/pactus/genesis" "github.com/pactus-project/pactus/util" "github.com/spf13/cobra" - "os" - "path/filepath" ) func buildImportCmd(parentCmd *cobra.Command) { diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index 094c58a07..2264b615a 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -5,17 +5,18 @@ package main import ( "context" "fmt" + "log" + "os" + "path/filepath" + "regexp" + "strings" + "github.com/gotk3/gotk3/glib" "github.com/gotk3/gotk3/gtk" "github.com/pactus-project/pactus/cmd" "github.com/pactus-project/pactus/genesis" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/wallet" - "log" - "os" - "path/filepath" - "regexp" - "strings" ) type assistantFunc func(assistant *gtk.Assistant, content gtk.IWidget, name, From 0ba26ebf98de7249dc1846133b4aae9faa1d625a Mon Sep 17 00:00:00 2001 From: Javad Date: Sat, 20 Jul 2024 16:21:27 +0330 Subject: [PATCH 04/20] fix: add configurable struct for download manager --- cmd/downlaod_mgr.go | 147 +++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 64 deletions(-) diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index f02036d63..a3453f918 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -18,6 +18,13 @@ import ( const maxDecompressedSize = 10 << 20 // 10 MB +type DMStateFunc func( + fileName string, + totalSize, downloaded int64, + totalItem, downloadedItem int, + percentage float64, +) + type Metadata struct { Name string `json:"name"` CreatedAt string `json:"created_at"` @@ -33,9 +40,27 @@ type SnapshotData struct { Size uint64 `json:"size"` } -func GetSnapshotMetadata(ctx context.Context, snapshotURL string) ([]Metadata, error) { +type DownloadManager struct { + snapshotURL string + extractDir string + tempDir string + storeDir string + zipFileList []string +} + +func NewDownloadManager(snapshotURL, extractDir, tempDir, storeDir string) *DownloadManager { + return &DownloadManager{ + snapshotURL: snapshotURL, + extractDir: extractDir, + tempDir: tempDir, + storeDir: storeDir, + zipFileList: make([]string, 0), + } +} + +func (dl *DownloadManager) GetMetadata(ctx context.Context) ([]Metadata, error) { cli := http.DefaultClient - metaURL, err := url.JoinPath(snapshotURL, "metadata.json") + metaURL, err := url.JoinPath(dl.snapshotURL, "metadata.json") if err != nil { return nil, err } @@ -68,39 +93,37 @@ func GetSnapshotMetadata(ctx context.Context, snapshotURL string) ([]Metadata, e return metadata, nil } -func DownloadManager( +func (dl *DownloadManager) Download( ctx context.Context, metadata *Metadata, - baseURL, tempPath string, - stateFunc func(fileName string, totalSize, downloaded int64, percentage float64), -) []string { - zipFileListPath := make([]string, 0) - + stateFunc DMStateFunc, +) { + downloadedItem := 0 for _, data := range metadata.Data { done := make(chan struct{}) - dlLink, err := url.JoinPath(baseURL, data.Path) + dlLink, err := url.JoinPath(dl.snapshotURL, data.Path) FatalErrorCheck(err) fileName := filepath.Base(dlLink) - filePath := fmt.Sprintf("%s/%s", tempPath, fileName) + filePath := fmt.Sprintf("%s/%s", dl.tempDir, fileName) - dl := downloader.New( + d := downloader.New( dlLink, filePath, data.Sha, ) - dl.Start(ctx) + d.Start(ctx) go func() { - err := <-dl.Errors() + err := <-d.Errors() FatalErrorCheck(err) }() go func() { - for state := range dl.Stats() { - stateFunc(fileName, state.TotalSize, state.Downloaded, state.Percent) + for state := range d.Stats() { + stateFunc(fileName, state.TotalSize, state.Downloaded, len(metadata.Data), downloadedItem, state.Percent) if state.Completed { done <- struct{}{} close(done) @@ -111,61 +134,68 @@ func DownloadManager( }() <-done - zipFileListPath = append(zipFileListPath, filePath) + dl.zipFileList = append(dl.zipFileList, filePath) + downloadedItem++ } - - return zipFileListPath } -func ExtractAndStoreFile(zipFilePath, extractPath string) error { - r, err := zip.OpenReader(zipFilePath) - if err != nil { - return fmt.Errorf("failed to open zip file: %w", err) - } - defer func() { - _ = r.Close() - }() - - for _, f := range r.File { - rc, err := f.Open() +func (dl *DownloadManager) ExtractAndStoreFiles() error { + for _, path := range dl.zipFileList { + r, err := zip.OpenReader(path) if err != nil { - return fmt.Errorf("failed to open file in zip archive: %w", err) + return fmt.Errorf("failed to open zip file: %w", err) } - fpath := fmt.Sprintf("%s/%s", extractPath, f.Name) + for _, f := range r.File { + rc, err := f.Open() + if err != nil { + return fmt.Errorf("failed to open file in zip archive: %w", err) + } - outFile, err := os.Create(fpath) - if err != nil { - return fmt.Errorf("failed to create file: %w", err) - } + fpath := fmt.Sprintf("%s/%s", dl.extractDir, f.Name) - // fixed potential DoS vulnerability via decompression bomb - lr := io.LimitedReader{R: rc, N: maxDecompressedSize} - _, err = io.Copy(outFile, &lr) - if err != nil { - return fmt.Errorf("failed to copy file contents: %w", err) - } + outFile, err := os.Create(fpath) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + + // fixed potential DoS vulnerability via decompression bomb + lr := io.LimitedReader{R: rc, N: maxDecompressedSize} + _, err = io.Copy(outFile, &lr) + if err != nil { + return fmt.Errorf("failed to copy file contents: %w", err) + } + + // check if the file size exceeds the limit + if lr.N <= 0 { + return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fpath) + } + + _ = rc.Close() + _ = outFile.Close() - // check if the file size exceeds the limit - if lr.N <= 0 { - return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fpath) } - _ = rc.Close() - _ = outFile.Close() + _ = r.Close() } return nil } -// CopyAllFiles copies all files from srcDir to dstDir. -func CopyAllFiles(srcDir, dstDir string) error { - err := os.MkdirAll(dstDir, 0o750) +func (dl *DownloadManager) ParseTime(dateString string) time.Time { + const layout = "2006-01-02T15:04:05.000000" + + parsedTime, err := time.Parse(layout, dateString) if err != nil { - return fmt.Errorf("failed to create destination directory: %w", err) + return time.Time{} } - return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + return parsedTime +} + +// CopyAllFiles copies all files from srcDir to dstDir. +func (dl *DownloadManager) CopyAllFiles() error { + return filepath.Walk(dl.extractDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -174,12 +204,12 @@ func CopyAllFiles(srcDir, dstDir string) error { return nil // Skip directories } - relativePath, err := filepath.Rel(srcDir, path) + relativePath, err := filepath.Rel(dl.extractDir, path) if err != nil { return err } - dstPath := filepath.Join(dstDir, relativePath) + dstPath := filepath.Join(dl.storeDir, relativePath) err = os.MkdirAll(filepath.Dir(dstPath), 0o750) if err != nil { @@ -224,14 +254,3 @@ func copyFile(src, dst string) error { return nil } - -func ParseTime(dateString string) time.Time { - const layout = "2006-01-02T15:04:05.000000" - - parsedTime, err := time.Parse(layout, dateString) - if err != nil { - return time.Time{} - } - - return parsedTime -} From 49676057ec0543cb4ea6705d7ed7af655f15f0f1 Mon Sep 17 00:00:00 2001 From: Javad Date: Sat, 20 Jul 2024 16:21:45 +0330 Subject: [PATCH 05/20] fix: update download manager with new structure --- cmd/daemon/import.go | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index 815393626..4dff8b21a 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -65,18 +65,29 @@ func buildImportCmd(parentCmd *cobra.Command) { return } - metadata, err := cmd.GetSnapshotMetadata(c.Context(), snapshotURL) - if err != nil { - cmd.PrintErrorMsgf("Failed to get snapshot metadata: %s", err) + tmpDir := util.TempDirPath() + extractPath := fmt.Sprintf("%s/data", tmpDir) - return - } + err = os.MkdirAll(extractPath, 0o750) + cmd.FatalErrorCheck(err) + + cmd.PrintLine() + + dl := cmd.NewDownloadManager( + snapshotURL, + extractPath, + tmpDir, + conf.Store.StorePath(), + ) + + metadata, err := dl.GetMetadata(c.Context()) + cmd.FatalErrorCheck(err) snapshots := make([]string, 0, len(metadata)) for _, m := range metadata { item := fmt.Sprintf("snapshot %s (%s)", - cmd.ParseTime(m.CreatedAt).Format("2006-01-02"), + dl.ParseTime(m.CreatedAt).Format("2006-01-02"), util.FormatBytesToHumanReadable(m.TotalSize), ) @@ -88,31 +99,20 @@ func buildImportCmd(parentCmd *cobra.Command) { choice := cmd.PromptSelect("Please select a snapshot", snapshots) selected := metadata[choice] - tmpDir := util.TempDirPath() - extractPath := fmt.Sprintf("%s/data", tmpDir) - - err = os.MkdirAll(extractPath, 0o750) - cmd.FatalErrorCheck(err) - cmd.PrintLine() - - zipFileList := cmd.DownloadManager( + dl.Download( c.Context(), &selected, - snapshotURL, - tmpDir, downloadProgressBar, ) - for _, zFile := range zipFileList { - err := cmd.ExtractAndStoreFile(zFile, extractPath) - cmd.FatalErrorCheck(err) - } + err = dl.ExtractAndStoreFiles() + cmd.FatalErrorCheck(err) err = os.MkdirAll(filepath.Dir(conf.Store.StorePath()), 0o750) cmd.FatalErrorCheck(err) - err = cmd.CopyAllFiles(extractPath, conf.Store.StorePath()) + err = dl.CopyAllFiles() cmd.FatalErrorCheck(err) err = os.RemoveAll(tmpDir) @@ -129,9 +129,9 @@ func buildImportCmd(parentCmd *cobra.Command) { } } -func downloadProgressBar(fileName string, totalSize, downloaded int64, _ float64) { +func downloadProgressBar(fileName string, totalSize, downloaded int64, totalItem, downloadedItem int, _ float64) { bar := cmd.TerminalProgressBar(int(totalSize), 30, true) - bar.Describe(fileName) + bar.Describe(fmt.Sprintf("%s (%d/%d)", fileName, downloadedItem, totalItem)) err := bar.Add(int(downloaded)) cmd.FatalErrorCheck(err) } From 887169fcff119bc0a88943d7d94753d9c916c7fb Mon Sep 17 00:00:00 2001 From: Javad Date: Sat, 20 Jul 2024 16:22:05 +0330 Subject: [PATCH 06/20] fix: update download manager and add total item in download state --- cmd/gtk/startup_assistant.go | 64 ++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index 2264b615a..dcbc75c2d 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -40,8 +40,8 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { assistFunc := pageAssistant() - // -- page_node_mode - snapshotWidget, snapshotGrid, snapshotRadio, snapshotPageName := pageSnapshot(assistant, assistFunc) + // -- page_import + importWidget, importGrid, importRadio, importPageName := pageImport(assistant, assistFunc) // --- page_mode mode, restoreRadio, pageModeName := pageMode(assistant, assistFunc) @@ -75,7 +75,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { gtk.MainQuit() }) - assistant.SetPageType(snapshotWidget, gtk.ASSISTANT_PAGE_CONTENT) + assistant.SetPageType(importWidget, gtk.ASSISTANT_PAGE_CONTENT) assistant.SetPageType(mode, gtk.ASSISTANT_PAGE_INTRO) // page 0 assistant.SetPageType(seedGenerate, gtk.ASSISTANT_PAGE_CONTENT) // page 1 assistant.SetPageType(seedConfirm, gtk.ASSISTANT_PAGE_CONTENT) // page 2 @@ -101,8 +101,8 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { log.Printf("%v (restore: %v, prev: %v, cur: %v)\n", curPageName, isRestoreMode, prevPageIndex, curPageIndex) switch curPageName { - case snapshotPageName: - assistantPageComplete(assistant, snapshotWidget, true) + case importPageName: + assistantPageComplete(assistant, importWidget, true) ssLabel, err := gtk.LabelNew("") cmd.FatalErrorCheck(err) setMargin(ssLabel, 5, 5, 1, 1) @@ -125,10 +125,10 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { setMargin(ssPBLabel, 5, 10, 1, 1) ssPBLabel.SetHAlign(gtk.ALIGN_START) - snapshotGrid.Attach(ssLabel, 0, 1, 1, 1) - snapshotGrid.Attach(listBox, 0, 2, 1, 1) - snapshotGrid.Attach(ssDLBtn, 0, 3, 1, 1) - snapshotGrid.Attach(ssPBLabel, 0, 5, 1, 1) + importGrid.Attach(ssLabel, 0, 1, 1, 1) + importGrid.Attach(listBox, 0, 2, 1, 1) + importGrid.Attach(ssDLBtn, 0, 3, 1, 1) + importGrid.Attach(ssPBLabel, 0, 5, 1, 1) ssLabel.SetVisible(false) listBox.SetVisible(false) ssDLBtn.SetVisible(false) @@ -136,15 +136,22 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { snapshotIndex := 0 - snapshotRadio.Connect("toggled", func() { - if snapshotRadio.GetActive() { - assistantPageComplete(assistant, snapshotWidget, false) + importRadio.Connect("toggled", func() { + if importRadio.GetActive() { + assistantPageComplete(assistant, importWidget, false) snapshotURL := "https://download.pactus.org/mainnet/" tmpDir := util.TempDirPath() extractPath := fmt.Sprintf("%s/data", tmpDir) + dm := cmd.NewDownloadManager( + snapshotURL, + extractPath, + tmpDir, + "/home/javad/pactus/data/store.db", + ) + err = os.MkdirAll(extractPath, 0o750) cmd.FatalErrorCheck(err) @@ -153,7 +160,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { ssLabel.SetVisible(true) }) - mdCh := getMetadata(context.Background(), snapshotURL, listBox) + mdCh := getMetadata(context.Background(), dm, listBox) go func() { if md := <-mdCh; md == nil { @@ -175,16 +182,17 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { ssPBLabel.SetVisible(true) go func() { - zipFileList := cmd.DownloadManager( + dm.Download( context.Background(), &md[snapshotIndex], - snapshotURL, - tmpDir, - func(fileName string, totalSize, downloaded int64, percentage float64) { + func(fileName string, totalSize, downloaded int64, + totalItem, downloadedItem int, percentage float64) { percent := int(percentage) glib.IdleAdd(func() { - dlMessage := fmt.Sprintf("🌐 Downloading file %s... %d%% (%s / %s)", + dlMessage := fmt.Sprintf("🌐 Downloading %s (%d/%d)... %d%% (%s / %s)", fileName, + downloadedItem, + totalItem, percent, util.FormatBytesToHumanReadable(uint64(downloaded)), util.FormatBytesToHumanReadable(uint64(totalSize)), @@ -195,22 +203,20 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { ) ssPBLabel.SetText(" " + "πŸ“‚ Extracting downloaded files...") - for _, zFile := range zipFileList { - err := cmd.ExtractAndStoreFile(zFile, extractPath) - cmd.FatalErrorCheck(err) - } + err := dm.ExtractAndStoreFiles() + cmd.FatalErrorCheck(err) err = os.MkdirAll(filepath.Dir("/home/javad/pactus/data/store.db"), 0o750) cmd.FatalErrorCheck(err) - err = cmd.CopyAllFiles(extractPath, "/home/javad/pactus/data/store.db") + err = dm.CopyAllFiles() cmd.FatalErrorCheck(err) err = os.RemoveAll(tmpDir) cmd.FatalErrorCheck(err) ssPBLabel.SetText(" " + "βœ… Import completed.") - assistantPageComplete(assistant, snapshotWidget, true) + assistantPageComplete(assistant, importWidget, true) }() }) @@ -218,7 +224,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { }() } else { - assistantPageComplete(assistant, snapshotWidget, true) + assistantPageComplete(assistant, importWidget, true) ssLabel.SetVisible(false) listBox.SetVisible(false) ssDLBtn.SetVisible(false) @@ -517,7 +523,7 @@ To make sure that you have properly saved your seed, please retype it here.` return pageWidget, pageSeedConfirmName } -func pageSnapshot(assistant *gtk.Assistant, assistFunc assistantFunc) ( +func pageImport(assistant *gtk.Assistant, assistFunc assistantFunc) ( *gtk.Widget, *gtk.Grid, *gtk.RadioButton, @@ -732,7 +738,7 @@ func assistantPageComplete(assistant *gtk.Assistant, page gtk.IWidget, completed func getMetadata( ctx context.Context, - snapshotURL string, + dm *cmd.DownloadManager, listBox *gtk.ListBox, ) <-chan []cmd.Metadata { mdCh := make(chan []cmd.Metadata, 1) @@ -747,7 +753,7 @@ func getMetadata( children = children.Next() } - metadata, err := cmd.GetSnapshotMetadata(ctx, snapshotURL) + metadata, err := dm.GetMetadata(ctx) if err != nil { mdCh <- nil return @@ -758,7 +764,7 @@ func getMetadata( cmd.FatalErrorCheck(err) label, err := gtk.LabelNew(fmt.Sprintf("snapshot %s (%s)", - cmd.ParseTime(md.CreatedAt).Format("2006-01-02"), + dm.ParseTime(md.CreatedAt).Format("2006-01-02"), util.FormatBytesToHumanReadable(md.TotalSize), )) cmd.FatalErrorCheck(err) From 37556a52531e6fb87f74fcf35eff9055e2f84e7c Mon Sep 17 00:00:00 2001 From: Javad Date: Sat, 20 Jul 2024 16:25:15 +0330 Subject: [PATCH 07/20] fix: fmt and lint errors --- cmd/downlaod_mgr.go | 4 +--- cmd/gtk/startup_assistant.go | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index a3453f918..ebca0b56c 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -173,16 +173,14 @@ func (dl *DownloadManager) ExtractAndStoreFiles() error { _ = rc.Close() _ = outFile.Close() - } - _ = r.Close() } return nil } -func (dl *DownloadManager) ParseTime(dateString string) time.Time { +func (*DownloadManager) ParseTime(dateString string) time.Time { const layout = "2006-01-02T15:04:05.000000" parsedTime, err := time.Parse(layout, dateString) diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index dcbc75c2d..690166cbf 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -186,7 +186,8 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { context.Background(), &md[snapshotIndex], func(fileName string, totalSize, downloaded int64, - totalItem, downloadedItem int, percentage float64) { + totalItem, downloadedItem int, percentage float64, + ) { percent := int(percentage) glib.IdleAdd(func() { dlMessage := fmt.Sprintf("🌐 Downloading %s (%d/%d)... %d%% (%s / %s)", From 1320865644467cc10bf3858885113850e092aa33 Mon Sep 17 00:00:00 2001 From: Javad Date: Sun, 21 Jul 2024 09:33:07 +0330 Subject: [PATCH 08/20] fix: add new line for progress --- cmd/daemon/import.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index 4dff8b21a..b1634c11b 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -100,6 +100,8 @@ func buildImportCmd(parentCmd *cobra.Command) { selected := metadata[choice] + cmd.PrintLine() + dl.Download( c.Context(), &selected, From b7fcbcf3725db63285bd7cab51817619e0db1712 Mon Sep 17 00:00:00 2001 From: Javad Date: Sun, 21 Jul 2024 10:09:31 +0330 Subject: [PATCH 09/20] fix: remove show bytes make confused --- cmd/cmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 9e873a1c1..e1cf0fbf9 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -634,7 +634,7 @@ func MakeValidatorKey(walletInstance *wallet.Wallet, valAddrsInfo []vault.Addres return valKeys, nil } -func TerminalProgressBar(totalSize, barWidth int, showBytes bool) *progressbar.ProgressBar { +func TerminalProgressBar(totalSize int64, barWidth int) *progressbar.ProgressBar { if barWidth < 15 { barWidth = 15 } @@ -642,10 +642,10 @@ func TerminalProgressBar(totalSize, barWidth int, showBytes bool) *progressbar.P opts := []progressbar.Option{ progressbar.OptionSetWriter(ansi.NewAnsiStdout()), progressbar.OptionEnableColorCodes(true), - progressbar.OptionShowBytes(showBytes), progressbar.OptionSetWidth(barWidth), progressbar.OptionSetElapsedTime(false), progressbar.OptionSetPredictTime(false), + progressbar.OptionShowDescriptionAtLineEnd(), progressbar.OptionSetTheme(progressbar.Theme{ Saucer: "[green]=[reset]", SaucerHead: "[green]>[reset]", @@ -655,5 +655,5 @@ func TerminalProgressBar(totalSize, barWidth int, showBytes bool) *progressbar.P }), } - return progressbar.NewOptions(totalSize, opts...) + return progressbar.NewOptions64(totalSize, opts...) } From 9187cb6821a33c41bf697eb46cdb70e38b6bdefe Mon Sep 17 00:00:00 2001 From: Javad Date: Sun, 21 Jul 2024 10:10:04 +0330 Subject: [PATCH 10/20] fix: update progress bar describe --- cmd/daemon/import.go | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index b1634c11b..f0a414b1a 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -73,21 +73,21 @@ func buildImportCmd(parentCmd *cobra.Command) { cmd.PrintLine() - dl := cmd.NewDownloadManager( + dm := cmd.NewDownloadManager( snapshotURL, extractPath, tmpDir, conf.Store.StorePath(), ) - metadata, err := dl.GetMetadata(c.Context()) + metadata, err := dm.GetMetadata(c.Context()) cmd.FatalErrorCheck(err) snapshots := make([]string, 0, len(metadata)) for _, m := range metadata { item := fmt.Sprintf("snapshot %s (%s)", - dl.ParseTime(m.CreatedAt).Format("2006-01-02"), + dm.ParseTime(m.CreatedAt).Format("2006-01-02"), util.FormatBytesToHumanReadable(m.TotalSize), ) @@ -100,24 +100,29 @@ func buildImportCmd(parentCmd *cobra.Command) { selected := metadata[choice] + cmd.TrapSignal(func() { + _ = fileLock.Unlock() + _ = dm.Cleanup() + }) + cmd.PrintLine() - dl.Download( + dm.Download( c.Context(), &selected, downloadProgressBar, ) - err = dl.ExtractAndStoreFiles() + err = dm.ExtractAndStoreFiles() cmd.FatalErrorCheck(err) err = os.MkdirAll(filepath.Dir(conf.Store.StorePath()), 0o750) cmd.FatalErrorCheck(err) - err = dl.CopyAllFiles() + err = dm.CopyAllFiles() cmd.FatalErrorCheck(err) - err = os.RemoveAll(tmpDir) + err = dm.Cleanup() cmd.FatalErrorCheck(err) _ = fileLock.Unlock() @@ -132,8 +137,14 @@ func buildImportCmd(parentCmd *cobra.Command) { } func downloadProgressBar(fileName string, totalSize, downloaded int64, totalItem, downloadedItem int, _ float64) { - bar := cmd.TerminalProgressBar(int(totalSize), 30, true) - bar.Describe(fmt.Sprintf("%s (%d/%d)", fileName, downloadedItem, totalItem)) - err := bar.Add(int(downloaded)) + bar := cmd.TerminalProgressBar(totalSize, 30) + bar.Describe(fmt.Sprintf("%s (%d/%d) - %s/%s", + fileName, + downloadedItem, + totalItem, + util.FormatBytesToHumanReadable(uint64(downloaded)), + util.FormatBytesToHumanReadable(uint64(totalSize)), + )) + err := bar.Add64(downloaded) cmd.FatalErrorCheck(err) } From da0c81c06ba8d846a69842df41802dbb24f39e18 Mon Sep 17 00:00:00 2001 From: Javad Date: Sun, 21 Jul 2024 10:10:22 +0330 Subject: [PATCH 11/20] fix: remove show bytes param --- cmd/daemon/prune.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/daemon/prune.go b/cmd/daemon/prune.go index 3fd49ade9..a878d8e6d 100644 --- a/cmd/daemon/prune.go +++ b/cmd/daemon/prune.go @@ -114,7 +114,7 @@ func buildPruneCmd(parentCmd *cobra.Command) { } func pruningProgressBar(prunedCount, skippedCount, totalCount uint32) { - bar := cmd.TerminalProgressBar(int(totalCount), 30, false) + bar := cmd.TerminalProgressBar(int64(totalCount), 30) bar.Describe(fmt.Sprintf("Pruned: %d | Skipped: %d", prunedCount, skippedCount)) err := bar.Add(int(prunedCount + skippedCount)) cmd.FatalErrorCheck(err) From 213a580630aae47d95b761700ffb67439a673f43 Mon Sep 17 00:00:00 2001 From: Javad Date: Sun, 21 Jul 2024 10:10:40 +0330 Subject: [PATCH 12/20] fix: add cleanup method for dm --- cmd/downlaod_mgr.go | 4 ++++ cmd/gtk/startup_assistant.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index ebca0b56c..b66a6aee7 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -191,6 +191,10 @@ func (*DownloadManager) ParseTime(dateString string) time.Time { return parsedTime } +func (dl *DownloadManager) Cleanup() error { + return os.RemoveAll(dl.tempDir) +} + // CopyAllFiles copies all files from srcDir to dstDir. func (dl *DownloadManager) CopyAllFiles() error { return filepath.Walk(dl.extractDir, func(path string, info os.FileInfo, err error) error { diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index 690166cbf..7546e6951 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -213,7 +213,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { err = dm.CopyAllFiles() cmd.FatalErrorCheck(err) - err = os.RemoveAll(tmpDir) + err = dm.Cleanup() cmd.FatalErrorCheck(err) ssPBLabel.SetText(" " + "βœ… Import completed.") From 7f2646f0406c8618650a6a105872f429661ac996 Mon Sep 17 00:00:00 2001 From: Javad Date: Mon, 22 Jul 2024 10:47:10 +0330 Subject: [PATCH 13/20] feat: add MoveDirectory util --- util/utils.go | 20 +++++++++++++++++-- util/utils_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/util/utils.go b/util/utils.go index c84c5bc88..7d3b7dd08 100644 --- a/util/utils.go +++ b/util/utils.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "math/bits" + "os" "golang.org/x/exp/constraints" ) @@ -109,8 +110,8 @@ func IsFlagSet[T constraints.Integer](flags, mask T) bool { // OS2IP converts an octet string to a nonnegative integer. // OS2IP: https://datatracker.ietf.org/doc/html/rfc8017#section-4.2 -func OS2IP(os []byte) *big.Int { - return new(big.Int).SetBytes(os) +func OS2IP(x []byte) *big.Int { + return new(big.Int).SetBytes(x) } // I2OSP converts a nonnegative integer to an octet string of a specified length. @@ -160,3 +161,18 @@ func FormatBytesToHumanReadable(bytes uint64) string { return fmt.Sprintf("%.2f %s", value, unit) } + +// MoveDirectory moves a directory from srcDir to dstDir, including all its contents. +func MoveDirectory(srcDir, dstDir string) error { + // Ensure the destination directory does not already exist + if _, err := os.Stat(dstDir); !os.IsNotExist(err) { + return fmt.Errorf("destination directory %s already exists", dstDir) + } + + // Move the entire directory to the new location + if err := os.Rename(srcDir, dstDir); err != nil { + return fmt.Errorf("failed to move directory from %s to %s: %w", srcDir, dstDir, err) + } + + return nil +} diff --git a/util/utils_test.go b/util/utils_test.go index 0b248535d..74770cb3a 100644 --- a/util/utils_test.go +++ b/util/utils_test.go @@ -2,6 +2,8 @@ package util import ( "math/big" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -139,3 +141,50 @@ func TestFormatBytesToHumanReadable(t *testing.T) { } } } + +func TestMoveDirectory(t *testing.T) { + // Create temporary directories + srcDir := TempDirPath() + dstDir := TempDirPath() + defer func() { _ = os.RemoveAll(srcDir) }() + defer func() { _ = os.RemoveAll(dstDir) }() + + // Create a subdirectory in the source directory + subDir := filepath.Join(srcDir, "subdir") + err := Mkdir(subDir) + assert.NoError(t, err) + + // Create multiple files in the subdirectory + files := []struct { + name string + content string + }{ + {"file1.txt", "content 1"}, + {"file2.txt", "content 2"}, + } + + for _, file := range files { + filePath := filepath.Join(subDir, file.name) + err = WriteFile(filePath, []byte(file.content)) + assert.NoError(t, err) + } + + // Move the directory + dstDirPath := filepath.Join(dstDir, "movedir") + err = MoveDirectory(srcDir, dstDirPath) + assert.NoError(t, err) + + // Assert the source directory no longer exists + assert.False(t, PathExists(srcDir)) + + // Assert the destination directory exists + assert.True(t, PathExists(dstDirPath)) + + // Verify that all files have been moved and their contents are correct + for _, file := range files { + movedFilePath := filepath.Join(dstDirPath, "subdir", file.name) + data, err := ReadFile(movedFilePath) + assert.NoError(t, err) + assert.Equal(t, file.content, string(data)) + } +} From 545d681e23b453396cc77893ff8939ff1f73e8e1 Mon Sep 17 00:00:00 2001 From: Javad Date: Mon, 22 Jul 2024 10:47:25 +0330 Subject: [PATCH 14/20] fix: make single file in compression --- scripts/snapshot.py | 106 ++++++++++++++++++++++++++++---------------- 1 file changed, 69 insertions(+), 37 deletions(-) diff --git a/scripts/snapshot.py b/scripts/snapshot.py index cfcb70cd0..e84de29be 100644 --- a/scripts/snapshot.py +++ b/scripts/snapshot.py @@ -12,8 +12,8 @@ # - `--data_path`: This argument specifies the path to the Pactus data folder to create snapshots. # - Windows: `C:\Users\{user}\pactus\data` # - Linux or Mac: `/home/{user}/pactus/data` -# - `--compress`: This argument specifies the compression method based on your choice ['zip', 'tar'], -# default is zip. +# - `--compress`: This argument specifies the compression method based on your choice ['none', 'zip', 'tar'], +# with 'none' being without compression. # - `--retention`: This argument sets the number of snapshots to keep. # - `--snapshot_path`: This argument sets a custom path for snapshots, with the default being the current # working directory of the script. @@ -25,6 +25,7 @@ # sudo python3 snapshot.py --service_path /etc/systemd/system/pactus.service --data_path /home/{user}/pactus/data # --compress zip --retention 3 + import argparse import os import shutil @@ -72,7 +73,14 @@ def update_metadata_file(snapshot_path, snapshot_metadata): logging.info(f"Creating new metadata file '{metadata_file}'") metadata = [] - metadata.append(snapshot_metadata) + formatted_metadata = { + "name": snapshot_metadata["name"], + "created_at": snapshot_metadata["created_at"], + "compress": snapshot_metadata["compress"], + "data": snapshot_metadata["data"] + } + + metadata.append(formatted_metadata) with open(metadata_file, 'w') as f: json.dump(metadata, f, indent=4) @@ -92,6 +100,35 @@ def update_metadata_after_removal(snapshots_dir, removed_snapshots): with open(metadata_file, 'w') as f: json.dump(updated_metadata, f, indent=4) + @staticmethod + def create_snapshot_json(data_dir, snapshot_subdir): + files = [] + for root, _, filenames in os.walk(data_dir): + for filename in filenames: + file_path = os.path.join(root, filename) + rel_path = os.path.relpath(file_path, data_dir) + snapshot_rel_path = os.path.join(snapshot_subdir, rel_path).replace('\\', '/') + file_info = { + "name": filename, + "path": snapshot_rel_path, + "sha": Metadata.sha256(file_path) + } + files.append(file_info) + + return {"data": files} + + @staticmethod + def create_compressed_snapshot_json(compressed_file, rel_path): + compressed_file_size = os.path.getsize(compressed_file) + file_info = { + "name": os.path.basename(compressed_file), + "path": rel_path, + "sha": Metadata.sha256(compressed_file), + "size": compressed_file_size, + } + + return {"data": file_info} + def run_command(command): logging.info(f"Running command: {' '.join(command)}") @@ -160,39 +197,34 @@ def create_snapshot(self): logging.info(f"Creating snapshot directory '{snapshot_dir}'") os.makedirs(snapshot_dir, exist_ok=True) - data_dir = self.args.data_path - snapshot_metadata = {"name": timestamp_str, "created_at": get_current_time_iso(), - "compress": self.args.compress, "total_size": 0, "data": []} - - for root, _, files in os.walk(data_dir): - for file in files: - file_path = os.path.join(root, file) - file_name, file_ext = os.path.splitext(file) - compressed_file_name = f"{file_name}{file_ext}.{self.args.compress}" - compressed_file_path = os.path.join(snapshot_dir, compressed_file_name) - rel_path = os.path.relpath(compressed_file_path, self.args.snapshot_path) - - if rel_path.startswith('snapshots' + os.path.sep): - rel_path = rel_path[len('snapshots' + os.path.sep):] - - if self.args.compress == 'zip': - logging.info(f"Creating ZIP archive '{compressed_file_path}'") - with zipfile.ZipFile(compressed_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf: - zipf.write(file_path, file) - elif self.args.compress == 'tar': - logging.info(f"Creating TAR archive '{compressed_file_path}'") - subprocess.run(['tar', '-cvf', compressed_file_path, '-C', os.path.dirname(file_path), file]) - - compressed_file_size = os.path.getsize(compressed_file_path) - snapshot_metadata["total_size"] += compressed_file_size - - file_info = { - "name": file_name, - "path": rel_path, - "sha": Metadata.sha256(compressed_file_path), - "size": compressed_file_size - } - snapshot_metadata["data"].append(file_info) + data_dir = os.path.join(snapshot_dir, 'data') + if self.args.compress == 'none': + logging.info(f"Copying data from '{self.args.data_path}' to '{data_dir}'") + shutil.copytree(self.args.data_path, data_dir) + snapshot_metadata = Metadata.create_snapshot_json(data_dir, timestamp_str) + elif self.args.compress == 'zip': + zip_file = os.path.join(snapshot_dir, 'data.zip') + rel = os.path.relpath(zip_file, snapshot_dir) + meta_path = os.path.join(timestamp_str, rel) + logging.info(f"Creating ZIP archive '{zip_file}'") + with zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED) as zipf: + for root, _, files in os.walk(self.args.data_path): + for file in files: + full_path = os.path.join(root, file) + rel_path = os.path.relpath(full_path, self.args.data_path) + zipf.write(full_path, os.path.join('data', rel_path)) + snapshot_metadata = Metadata.create_compressed_snapshot_json(zip_file, meta_path) + elif self.args.compress == 'tar': + tar_file = os.path.join(snapshot_dir, 'data.tar.gz') + rel = os.path.relpath(tar_file, snapshot_dir) + meta_path = os.path.join(timestamp_str, rel) + logging.info(f"Creating TAR.GZ archive '{tar_file}'") + subprocess.run(['tar', '-czvf', tar_file, '-C', self.args.data_path, '.']) + snapshot_metadata = Metadata.create_compressed_snapshot_json(tar_file, meta_path) + + snapshot_metadata["name"] = timestamp_str + snapshot_metadata["created_at"] = get_current_time_iso() + snapshot_metadata["compress"] = self.args.compress Metadata.update_metadata_file(self.args.snapshot_path, snapshot_metadata) @@ -268,7 +300,7 @@ def parse_args(): parser = argparse.ArgumentParser(description='Pactus Blockchain Snapshot Tool') parser.add_argument('--service_path', required=True, help='Path to pactus systemctl service') parser.add_argument('--data_path', default=default_data_path, help='Path to data directory') - parser.add_argument('--compress', choices=['zip', 'tar'], default='zip', help='Compression type') + parser.add_argument('--compress', choices=['none', 'zip', 'tar'], default='none', help='Compression type') parser.add_argument('--retention', type=int, default=3, help='Number of snapshots to retain') parser.add_argument('--snapshot_path', default=os.getcwd(), help='Path to store snapshots') From f524f92cca03cf53dfa1fe09957d551e5d128373 Mon Sep 17 00:00:00 2001 From: Javad Date: Mon, 22 Jul 2024 10:48:00 +0330 Subject: [PATCH 15/20] fix: import data from single file and move after extract --- cmd/daemon/import.go | 28 +++--- cmd/daemon/prune.go | 5 +- cmd/daemon/start.go | 5 +- cmd/downlaod_mgr.go | 189 +++++++++++++---------------------- cmd/gtk/main.go | 5 +- cmd/gtk/startup_assistant.go | 26 ++--- 6 files changed, 97 insertions(+), 161 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index f0a414b1a..f036d05ae 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "os" "path/filepath" @@ -66,16 +67,11 @@ func buildImportCmd(parentCmd *cobra.Command) { } tmpDir := util.TempDirPath() - extractPath := fmt.Sprintf("%s/data", tmpDir) - - err = os.MkdirAll(extractPath, 0o750) - cmd.FatalErrorCheck(err) cmd.PrintLine() dm := cmd.NewDownloadManager( snapshotURL, - extractPath, tmpDir, conf.Store.StorePath(), ) @@ -86,9 +82,13 @@ func buildImportCmd(parentCmd *cobra.Command) { snapshots := make([]string, 0, len(metadata)) for _, m := range metadata { + if m.Data == nil { + cmd.FatalErrorCheck(errors.New("metadata is nil")) + } + item := fmt.Sprintf("snapshot %s (%s)", dm.ParseTime(m.CreatedAt).Format("2006-01-02"), - util.FormatBytesToHumanReadable(m.TotalSize), + util.FormatBytesToHumanReadable(m.Data.Size), ) snapshots = append(snapshots, item) @@ -113,13 +113,15 @@ func buildImportCmd(parentCmd *cobra.Command) { downloadProgressBar, ) - err = dm.ExtractAndStoreFiles() - cmd.FatalErrorCheck(err) + cmd.PrintLine() + cmd.PrintLine() + cmd.PrintInfoMsgf("Extracting files...") - err = os.MkdirAll(filepath.Dir(conf.Store.StorePath()), 0o750) + err = dm.ExtractAndStoreFiles() cmd.FatalErrorCheck(err) - err = dm.CopyAllFiles() + cmd.PrintInfoMsgf("Moving data...") + err = util.MoveDirectory(filepath.Join(tmpDir, "data"), filepath.Join(workingDir, "data")) cmd.FatalErrorCheck(err) err = dm.Cleanup() @@ -136,12 +138,10 @@ func buildImportCmd(parentCmd *cobra.Command) { } } -func downloadProgressBar(fileName string, totalSize, downloaded int64, totalItem, downloadedItem int, _ float64) { +func downloadProgressBar(fileName string, totalSize, downloaded int64, _ float64) { bar := cmd.TerminalProgressBar(totalSize, 30) - bar.Describe(fmt.Sprintf("%s (%d/%d) - %s/%s", + bar.Describe(fmt.Sprintf("%s (%s/%s)", fileName, - downloadedItem, - totalItem, util.FormatBytesToHumanReadable(uint64(downloaded)), util.FormatBytesToHumanReadable(uint64(totalSize)), )) diff --git a/cmd/daemon/prune.go b/cmd/daemon/prune.go index a878d8e6d..a58c2ab89 100644 --- a/cmd/daemon/prune.go +++ b/cmd/daemon/prune.go @@ -34,10 +34,7 @@ func buildPruneCmd(parentCmd *cobra.Command) { fileLock := flock.New(lockFilePath) locked, err := fileLock.TryLock() - if err != nil { - // handle unable to attempt to acquire lock - cmd.FatalErrorCheck(err) - } + cmd.FatalErrorCheck(err) if !locked { cmd.PrintWarnMsgf("Could not lock '%s', another instance is running?", lockFilePath) diff --git a/cmd/daemon/start.go b/cmd/daemon/start.go index b707cb57d..3df4b2f32 100644 --- a/cmd/daemon/start.go +++ b/cmd/daemon/start.go @@ -41,10 +41,7 @@ func buildStartCmd(parentCmd *cobra.Command) { fileLock := flock.New(lockFilePath) locked, err := fileLock.TryLock() - if err != nil { - // handle unable to attempt to acquire lock - cmd.FatalErrorCheck(err) - } + cmd.FatalErrorCheck(err) if !locked { cmd.PrintWarnMsgf("Could not lock '%s', another instance is running?", lockFilePath) diff --git a/cmd/downlaod_mgr.go b/cmd/downlaod_mgr.go index b66a6aee7..e0fba5252 100644 --- a/cmd/downlaod_mgr.go +++ b/cmd/downlaod_mgr.go @@ -13,6 +13,7 @@ import ( "path/filepath" "time" + "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/downloader" ) @@ -21,16 +22,14 @@ const maxDecompressedSize = 10 << 20 // 10 MB type DMStateFunc func( fileName string, totalSize, downloaded int64, - totalItem, downloadedItem int, percentage float64, ) type Metadata struct { - Name string `json:"name"` - CreatedAt string `json:"created_at"` - Compress string `json:"compress"` - TotalSize uint64 `json:"total_size"` - Data []*SnapshotData `json:"data"` + Name string `json:"name"` + CreatedAt string `json:"created_at"` + Compress string `json:"compress"` + Data *SnapshotData `json:"data"` } type SnapshotData struct { @@ -41,20 +40,17 @@ type SnapshotData struct { } type DownloadManager struct { - snapshotURL string - extractDir string - tempDir string - storeDir string - zipFileList []string + snapshotURL string + tempDir string + storeDir string + dataFileName string } -func NewDownloadManager(snapshotURL, extractDir, tempDir, storeDir string) *DownloadManager { +func NewDownloadManager(snapshotURL, tempDir, storeDir string) *DownloadManager { return &DownloadManager{ snapshotURL: snapshotURL, - extractDir: extractDir, tempDir: tempDir, storeDir: storeDir, - zipFileList: make([]string, 0), } } @@ -98,86 +94,42 @@ func (dl *DownloadManager) Download( metadata *Metadata, stateFunc DMStateFunc, ) { - downloadedItem := 0 - for _, data := range metadata.Data { - done := make(chan struct{}) - dlLink, err := url.JoinPath(dl.snapshotURL, data.Path) - FatalErrorCheck(err) - - fileName := filepath.Base(dlLink) - - filePath := fmt.Sprintf("%s/%s", dl.tempDir, fileName) + done := make(chan struct{}) + dlLink, err := url.JoinPath(dl.snapshotURL, metadata.Data.Path) + FatalErrorCheck(err) - d := downloader.New( - dlLink, - filePath, - data.Sha, - ) + fileName := filepath.Base(dlLink) - d.Start(ctx) + dl.dataFileName = fileName - go func() { - err := <-d.Errors() - FatalErrorCheck(err) - }() - - go func() { - for state := range d.Stats() { - stateFunc(fileName, state.TotalSize, state.Downloaded, len(metadata.Data), downloadedItem, state.Percent) - if state.Completed { - done <- struct{}{} - close(done) - - return - } - } - }() - - <-done - dl.zipFileList = append(dl.zipFileList, filePath) - downloadedItem++ - } -} - -func (dl *DownloadManager) ExtractAndStoreFiles() error { - for _, path := range dl.zipFileList { - r, err := zip.OpenReader(path) - if err != nil { - return fmt.Errorf("failed to open zip file: %w", err) - } + filePath := fmt.Sprintf("%s/%s", dl.tempDir, fileName) - for _, f := range r.File { - rc, err := f.Open() - if err != nil { - return fmt.Errorf("failed to open file in zip archive: %w", err) - } + d := downloader.New( + dlLink, + filePath, + metadata.Data.Sha, + ) - fpath := fmt.Sprintf("%s/%s", dl.extractDir, f.Name) + d.Start(ctx) - outFile, err := os.Create(fpath) - if err != nil { - return fmt.Errorf("failed to create file: %w", err) - } + go func() { + err := <-d.Errors() + FatalErrorCheck(err) + }() - // fixed potential DoS vulnerability via decompression bomb - lr := io.LimitedReader{R: rc, N: maxDecompressedSize} - _, err = io.Copy(outFile, &lr) - if err != nil { - return fmt.Errorf("failed to copy file contents: %w", err) - } + go func() { + for state := range d.Stats() { + stateFunc(fileName, state.TotalSize, state.Downloaded, state.Percent) + if state.Completed { + done <- struct{}{} + close(done) - // check if the file size exceeds the limit - if lr.N <= 0 { - return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fpath) + return } - - _ = rc.Close() - _ = outFile.Close() } - _ = r.Close() - } + }() - return nil + <-done } func (*DownloadManager) ParseTime(dateString string) time.Time { @@ -195,63 +147,60 @@ func (dl *DownloadManager) Cleanup() error { return os.RemoveAll(dl.tempDir) } -// CopyAllFiles copies all files from srcDir to dstDir. -func (dl *DownloadManager) CopyAllFiles() error { - return filepath.Walk(dl.extractDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if info.IsDir() { - return nil // Skip directories - } +func (dl *DownloadManager) ExtractAndStoreFiles() error { + zipPath := filepath.Join(dl.tempDir, dl.dataFileName) + r, err := zip.OpenReader(zipPath) + if err != nil { + return fmt.Errorf("failed to open zip file: %w", err) + } + defer func() { + _ = r.Close() + }() - relativePath, err := filepath.Rel(dl.extractDir, path) - if err != nil { + for _, f := range r.File { + if err := extractAndWriteFile(f, dl.tempDir); err != nil { return err } + } - dstPath := filepath.Join(dl.storeDir, relativePath) - - err = os.MkdirAll(filepath.Dir(dstPath), 0o750) - if err != nil { - return fmt.Errorf("failed to create directory: %w", err) - } - - err = copyFile(path, dstPath) - if err != nil { - return fmt.Errorf("failed to copy file from %s to %s: %w", path, dstPath, err) - } - - return nil - }) + return nil } -func copyFile(src, dst string) error { - sourceFile, err := os.Open(src) +func extractAndWriteFile(f *zip.File, destination string) error { + rc, err := f.Open() if err != nil { - return fmt.Errorf("failed to open source file: %w", err) + return fmt.Errorf("failed to open file in zip archive: %w", err) } defer func() { - _ = sourceFile.Close() + _ = rc.Close() }() - destinationFile, err := os.Create(dst) + fpath := fmt.Sprintf("%s/%s", destination, f.Name) + if f.FileInfo().IsDir() { + return util.Mkdir(fpath) + } + + if err := util.Mkdir(filepath.Dir(fpath)); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + outFile, err := os.Create(fpath) if err != nil { - return fmt.Errorf("failed to create destination file: %w", err) + return fmt.Errorf("failed to create file: %w", err) } defer func() { - _ = destinationFile.Close() + _ = outFile.Close() }() - _, err = io.Copy(destinationFile, sourceFile) + // Use a limited reader to prevent DoS attacks via decompression bomb + lr := &io.LimitedReader{R: rc, N: maxDecompressedSize} + written, err := io.Copy(outFile, lr) if err != nil { return fmt.Errorf("failed to copy file contents: %w", err) } - err = destinationFile.Sync() - if err != nil { - return fmt.Errorf("failed to sync destination file: %w", err) + if written >= maxDecompressedSize { + return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fpath) } return nil diff --git a/cmd/gtk/main.go b/cmd/gtk/main.go index 451466828..4f7f02b51 100644 --- a/cmd/gtk/main.go +++ b/cmd/gtk/main.go @@ -63,10 +63,7 @@ func main() { fileLock := flock.New(lockFilePath) locked, err := fileLock.TryLock() - if err != nil { - // handle unable to attempt to acquire lock - fatalErrorCheck(err) - } + fatalErrorCheck(err) if !locked { cmd.PrintWarnMsgf("Could not lock '%s', another instance is running?", lockFilePath) diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index 7546e6951..73f4bc9d1 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -4,9 +4,9 @@ package main import ( "context" + "errors" "fmt" "log" - "os" "path/filepath" "regexp" "strings" @@ -143,18 +143,13 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { snapshotURL := "https://download.pactus.org/mainnet/" tmpDir := util.TempDirPath() - extractPath := fmt.Sprintf("%s/data", tmpDir) dm := cmd.NewDownloadManager( snapshotURL, - extractPath, tmpDir, "/home/javad/pactus/data/store.db", ) - err = os.MkdirAll(extractPath, 0o750) - cmd.FatalErrorCheck(err) - glib.IdleAdd(func() { ssLabel.SetText(" ♻️ Please wait, loading snapshots...") ssLabel.SetVisible(true) @@ -186,14 +181,12 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { context.Background(), &md[snapshotIndex], func(fileName string, totalSize, downloaded int64, - totalItem, downloadedItem int, percentage float64, + percentage float64, ) { percent := int(percentage) glib.IdleAdd(func() { - dlMessage := fmt.Sprintf("🌐 Downloading %s (%d/%d)... %d%% (%s / %s)", + dlMessage := fmt.Sprintf("🌐 Downloading %s | %d%% (%s / %s)", fileName, - downloadedItem, - totalItem, percent, util.FormatBytesToHumanReadable(uint64(downloaded)), util.FormatBytesToHumanReadable(uint64(totalSize)), @@ -207,10 +200,9 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { err := dm.ExtractAndStoreFiles() cmd.FatalErrorCheck(err) - err = os.MkdirAll(filepath.Dir("/home/javad/pactus/data/store.db"), 0o750) - cmd.FatalErrorCheck(err) - - err = dm.CopyAllFiles() + ssPBLabel.SetText(" " + "πŸ“‘ Moving data...") + err = util.MoveDirectory(filepath.Join(tmpDir, "data"), + filepath.Join(workingDir, "data")) cmd.FatalErrorCheck(err) err = dm.Cleanup() @@ -761,12 +753,16 @@ func getMetadata( } for _, md := range metadata { + if md.Data == nil { + cmd.FatalErrorCheck(errors.New("metadata is nil")) + } + listBoxRow, err := gtk.ListBoxRowNew() cmd.FatalErrorCheck(err) label, err := gtk.LabelNew(fmt.Sprintf("snapshot %s (%s)", dm.ParseTime(md.CreatedAt).Format("2006-01-02"), - util.FormatBytesToHumanReadable(md.TotalSize), + util.FormatBytesToHumanReadable(md.Data.Size), )) cmd.FatalErrorCheck(err) From 55745db2b55ec4ee4711cb8c1fb151d9136a9b1d Mon Sep 17 00:00:00 2001 From: Javad Date: Mon, 22 Jul 2024 11:08:49 +0330 Subject: [PATCH 16/20] fix: change base url of snapshots --- cmd/daemon/import.go | 2 +- cmd/gtk/startup_assistant.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index f036d05ae..10c991e71 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -21,7 +21,7 @@ func buildImportCmd(parentCmd *cobra.Command) { parentCmd.AddCommand(importCmd) workingDirOpt := addWorkingDirOption(importCmd) - serverAddrOpt := importCmd.Flags().String("server-addr", "https://download.pactus.org", + serverAddrOpt := importCmd.Flags().String("server-addr", "https://snapshot.pactus.org", "import server address") importCmd.Run = func(c *cobra.Command, _ []string) { diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index 73f4bc9d1..9ec26ce73 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -140,7 +140,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { if importRadio.GetActive() { assistantPageComplete(assistant, importWidget, false) - snapshotURL := "https://download.pactus.org/mainnet/" + snapshotURL := "https://snapshot.pactus.org/mainnet/" tmpDir := util.TempDirPath() From b495c4999895814d24790038b50747fbb0393694 Mon Sep 17 00:00:00 2001 From: Mostafa Date: Wed, 24 Jul 2024 20:06:54 +0800 Subject: [PATCH 17/20] fix(cmd): fix importer issues --- cmd/daemon/import.go | 52 +--- cmd/gtk/main.go | 2 +- cmd/gtk/startup_assistant.go | 413 ++++++++++++++------------- cmd/gtk/widget_node.go | 2 +- cmd/{downlaod_mgr.go => importer.go} | 104 ++++--- util/io.go | 34 +++ util/io_test.go | 78 +++++ util/utils.go | 16 -- util/utils_test.go | 49 ---- 9 files changed, 409 insertions(+), 341 deletions(-) rename cmd/{downlaod_mgr.go => importer.go} (57%) diff --git a/cmd/daemon/import.go b/cmd/daemon/import.go index 10c991e71..c52a6cc7b 100644 --- a/cmd/daemon/import.go +++ b/cmd/daemon/import.go @@ -1,14 +1,12 @@ package main import ( - "errors" "fmt" "os" "path/filepath" "github.com/gofrs/flock" "github.com/pactus-project/pactus/cmd" - "github.com/pactus-project/pactus/genesis" "github.com/pactus-project/pactus/util" "github.com/spf13/cobra" ) @@ -21,7 +19,7 @@ func buildImportCmd(parentCmd *cobra.Command) { parentCmd.AddCommand(importCmd) workingDirOpt := addWorkingDirOption(importCmd) - serverAddrOpt := importCmd.Flags().String("server-addr", "https://snapshot.pactus.org", + serverAddrOpt := importCmd.Flags().String("server-addr", cmd.DefaultSnapshotURL, "import server address") importCmd.Run = func(c *cobra.Command, _ []string) { @@ -46,48 +44,24 @@ func buildImportCmd(parentCmd *cobra.Command) { return } - storeDir, _ := filepath.Abs(conf.Store.StorePath()) - if !util.IsDirNotExistsOrEmpty(storeDir) { - cmd.PrintErrorMsgf("The data directory is not empty: %s", conf.Store.StorePath()) - - return - } - - snapshotURL := *serverAddrOpt - - switch gen.ChainType() { - case genesis.Mainnet: - snapshotURL += "/mainnet/" - case genesis.Testnet: - snapshotURL += "/testnet/" - case genesis.Localnet: - cmd.PrintErrorMsgf("Unsupported chain type: %s", gen.ChainType()) - - return - } - - tmpDir := util.TempDirPath() - cmd.PrintLine() - dm := cmd.NewDownloadManager( + snapshotURL := *serverAddrOpt + importer, err := cmd.NewImporter( + gen.ChainType(), snapshotURL, - tmpDir, - conf.Store.StorePath(), + conf.Store.DataPath(), ) + cmd.FatalErrorCheck(err) - metadata, err := dm.GetMetadata(c.Context()) + metadata, err := importer.GetMetadata(c.Context()) cmd.FatalErrorCheck(err) snapshots := make([]string, 0, len(metadata)) for _, m := range metadata { - if m.Data == nil { - cmd.FatalErrorCheck(errors.New("metadata is nil")) - } - item := fmt.Sprintf("snapshot %s (%s)", - dm.ParseTime(m.CreatedAt).Format("2006-01-02"), + m.CreatedAtTime().Format("2006-01-02"), util.FormatBytesToHumanReadable(m.Data.Size), ) @@ -102,12 +76,12 @@ func buildImportCmd(parentCmd *cobra.Command) { cmd.TrapSignal(func() { _ = fileLock.Unlock() - _ = dm.Cleanup() + _ = importer.Cleanup() }) cmd.PrintLine() - dm.Download( + importer.Download( c.Context(), &selected, downloadProgressBar, @@ -117,14 +91,14 @@ func buildImportCmd(parentCmd *cobra.Command) { cmd.PrintLine() cmd.PrintInfoMsgf("Extracting files...") - err = dm.ExtractAndStoreFiles() + err = importer.ExtractAndStoreFiles() cmd.FatalErrorCheck(err) cmd.PrintInfoMsgf("Moving data...") - err = util.MoveDirectory(filepath.Join(tmpDir, "data"), filepath.Join(workingDir, "data")) + err = importer.MoveStore() cmd.FatalErrorCheck(err) - err = dm.Cleanup() + err = importer.Cleanup() cmd.FatalErrorCheck(err) _ = fileLock.Unlock() diff --git a/cmd/gtk/main.go b/cmd/gtk/main.go index 4f7f02b51..c9bf57945 100644 --- a/cmd/gtk/main.go +++ b/cmd/gtk/main.go @@ -48,7 +48,7 @@ func main() { } // If node is not initialized yet - if !util.PathExists(workingDir) { + if util.IsDirNotExistsOrEmpty(workingDir) { network := genesis.Mainnet if *testnetOpt { network = genesis.Testnet diff --git a/cmd/gtk/startup_assistant.go b/cmd/gtk/startup_assistant.go index 9ec26ce73..a0d0112cc 100644 --- a/cmd/gtk/startup_assistant.go +++ b/cmd/gtk/startup_assistant.go @@ -4,12 +4,12 @@ package main import ( "context" - "errors" "fmt" "log" "path/filepath" "regexp" "strings" + "time" "github.com/gotk3/gotk3/glib" "github.com/gotk3/gotk3/gtk" @@ -29,8 +29,8 @@ func setMargin(widget gtk.IWidget, top, bottom, start, end int) { widget.ToWidget().SetMarginEnd(end) } -//nolint:gocognit // complexity can't be reduced more. -func startupAssistant(workingDir string, chain genesis.ChainType) bool { +//nolint:all // complexity can't be reduced more. It needs to refactor. +func startupAssistant(workingDir string, chainType genesis.ChainType) bool { successful := false assistant, err := gtk.AssistantNew() fatalErrorCheck(err) @@ -40,29 +40,30 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { assistFunc := pageAssistant() - // -- page_import - importWidget, importGrid, importRadio, importPageName := pageImport(assistant, assistFunc) - // --- page_mode - mode, restoreRadio, pageModeName := pageMode(assistant, assistFunc) + wgtWalletMode, radioRestoreWallet, pageModeName := pageWalletMode(assistant, assistFunc) // --- page_seed_generate - seedGenerate, textViewSeed, pageSeedGenerateName := pageSeedGenerate(assistant, assistFunc) + wgtSeedGenerate, txtSeed, pageSeedGenerateName := pageSeedGenerate(assistant, assistFunc) // --- page_seed_confirm - seedConfirm, pageSeedConfirmName := pageSeedConfirm(assistant, assistFunc, textViewSeed) + wgtSeedConfirm, pageSeedConfirmName := pageSeedConfirm(assistant, assistFunc, txtSeed) // -- page_seed_restore - seedRestore, textViewRestoreSeed, pageSeedRestoreName := pageSeedRestore(assistant, assistFunc) + wgtSeedRestore, textRestoreSeed, pageSeedRestoreName := pageSeedRestore(assistant, assistFunc) // --- page_password - password, entryPassword, pagePasswordName := pagePassword(assistant, assistFunc) + wgtPassword, entryPassword, pagePasswordName := pagePassword(assistant, assistFunc) // --- page_num_validators - numValidators, lsNumValidators, comboNumValidators, pageNumValidatorsName := pageNumValidators(assistant, assistFunc) + wgtNumValidators, lsNumValidators, comboNumValidators, + pageNumValidatorsName := pageNumValidators(assistant, assistFunc) + + // -- page_node_type + wgtNodeType, gridImport, radioImport, pageNodeTypeName := pageNodeType(assistant, assistFunc) - // --- page_final - final, textViewNodeInfo, pageFinalName := pageFinal(assistant, assistFunc) + // --- page_summary + wgtSummary, txtNodeInfo, pageSummaryName := pageSummary(assistant, assistFunc) assistant.Connect("cancel", func() { assistant.Close() @@ -75,20 +76,20 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { gtk.MainQuit() }) - assistant.SetPageType(importWidget, gtk.ASSISTANT_PAGE_CONTENT) - assistant.SetPageType(mode, gtk.ASSISTANT_PAGE_INTRO) // page 0 - assistant.SetPageType(seedGenerate, gtk.ASSISTANT_PAGE_CONTENT) // page 1 - assistant.SetPageType(seedConfirm, gtk.ASSISTANT_PAGE_CONTENT) // page 2 - assistant.SetPageType(seedRestore, gtk.ASSISTANT_PAGE_CONTENT) // page 3 - assistant.SetPageType(password, gtk.ASSISTANT_PAGE_CONTENT) // page 4 - assistant.SetPageType(numValidators, gtk.ASSISTANT_PAGE_CONTENT) // page 5 - assistant.SetPageType(final, gtk.ASSISTANT_PAGE_SUMMARY) // page 6 + assistant.SetPageType(wgtWalletMode, gtk.ASSISTANT_PAGE_INTRO) // page 0 + assistant.SetPageType(wgtSeedGenerate, gtk.ASSISTANT_PAGE_CONTENT) // page 1 + assistant.SetPageType(wgtSeedConfirm, gtk.ASSISTANT_PAGE_CONTENT) // page 2 + assistant.SetPageType(wgtSeedRestore, gtk.ASSISTANT_PAGE_CONTENT) // page 3 + assistant.SetPageType(wgtPassword, gtk.ASSISTANT_PAGE_CONTENT) // page 4 + assistant.SetPageType(wgtNumValidators, gtk.ASSISTANT_PAGE_CONTENT) // page 5 + assistant.SetPageType(wgtNodeType, gtk.ASSISTANT_PAGE_CONTENT) // page 6 + assistant.SetPageType(wgtSummary, gtk.ASSISTANT_PAGE_SUMMARY) // page 7 mnemonic := "" prevPageIndex := -1 prevPageAdjust := 0 assistant.Connect("prepare", func(assistant *gtk.Assistant, page *gtk.Widget) { - isRestoreMode := restoreRadio.GetActive() + isRestoreMode := radioRestoreWallet.GetActive() curPageName, err := page.GetName() curPageIndex := assistant.GetCurrentPage() fatalErrorCheck(err) @@ -101,131 +102,8 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { log.Printf("%v (restore: %v, prev: %v, cur: %v)\n", curPageName, isRestoreMode, prevPageIndex, curPageIndex) switch curPageName { - case importPageName: - assistantPageComplete(assistant, importWidget, true) - ssLabel, err := gtk.LabelNew("") - cmd.FatalErrorCheck(err) - setMargin(ssLabel, 5, 5, 1, 1) - ssLabel.SetHAlign(gtk.ALIGN_START) - - listBox, err := gtk.ListBoxNew() - cmd.FatalErrorCheck(err) - setMargin(listBox, 5, 5, 1, 1) - listBox.SetHAlign(gtk.ALIGN_CENTER) - listBox.SetSizeRequest(600, -1) - - ssDLBtn, err := gtk.ButtonNewWithLabel("⏬ Download") - cmd.FatalErrorCheck(err) - setMargin(ssDLBtn, 10, 5, 1, 1) - ssDLBtn.SetHAlign(gtk.ALIGN_CENTER) - ssDLBtn.SetSizeRequest(600, -1) - - ssPBLabel, err := gtk.LabelNew("") - cmd.FatalErrorCheck(err) - setMargin(ssPBLabel, 5, 10, 1, 1) - ssPBLabel.SetHAlign(gtk.ALIGN_START) - - importGrid.Attach(ssLabel, 0, 1, 1, 1) - importGrid.Attach(listBox, 0, 2, 1, 1) - importGrid.Attach(ssDLBtn, 0, 3, 1, 1) - importGrid.Attach(ssPBLabel, 0, 5, 1, 1) - ssLabel.SetVisible(false) - listBox.SetVisible(false) - ssDLBtn.SetVisible(false) - ssPBLabel.SetVisible(false) - - snapshotIndex := 0 - - importRadio.Connect("toggled", func() { - if importRadio.GetActive() { - assistantPageComplete(assistant, importWidget, false) - - snapshotURL := "https://snapshot.pactus.org/mainnet/" - - tmpDir := util.TempDirPath() - - dm := cmd.NewDownloadManager( - snapshotURL, - tmpDir, - "/home/javad/pactus/data/store.db", - ) - - glib.IdleAdd(func() { - ssLabel.SetText(" ♻️ Please wait, loading snapshots...") - ssLabel.SetVisible(true) - }) - - mdCh := getMetadata(context.Background(), dm, listBox) - - go func() { - if md := <-mdCh; md == nil { - ssLabel.SetText(" ❌ Failed to get snapshot list, please try again later.") - } else { - ssLabel.SetText(" πŸ”½ Please select a snapshot to download:") - listBox.SetVisible(true) - - listBox.Connect("row-selected", func(box *gtk.ListBox, row *gtk.ListBoxRow) { - if row != nil { - snapshotIndex = row.GetIndex() - ssDLBtn.SetVisible(true) - } - }) - - ssDLBtn.Connect("clicked", func() { - ssDLBtn.SetVisible(false) - listBox.SetSelectionMode(gtk.SELECTION_NONE) - ssPBLabel.SetVisible(true) - - go func() { - dm.Download( - context.Background(), - &md[snapshotIndex], - func(fileName string, totalSize, downloaded int64, - percentage float64, - ) { - percent := int(percentage) - glib.IdleAdd(func() { - dlMessage := fmt.Sprintf("🌐 Downloading %s | %d%% (%s / %s)", - fileName, - percent, - util.FormatBytesToHumanReadable(uint64(downloaded)), - util.FormatBytesToHumanReadable(uint64(totalSize)), - ) - ssPBLabel.SetText(" " + dlMessage) - }) - }, - ) - - ssPBLabel.SetText(" " + "πŸ“‚ Extracting downloaded files...") - err := dm.ExtractAndStoreFiles() - cmd.FatalErrorCheck(err) - - ssPBLabel.SetText(" " + "πŸ“‘ Moving data...") - err = util.MoveDirectory(filepath.Join(tmpDir, "data"), - filepath.Join(workingDir, "data")) - cmd.FatalErrorCheck(err) - - err = dm.Cleanup() - cmd.FatalErrorCheck(err) - - ssPBLabel.SetText(" " + "βœ… Import completed.") - assistantPageComplete(assistant, importWidget, true) - }() - }) - - } - }() - - } else { - assistantPageComplete(assistant, importWidget, true) - ssLabel.SetVisible(false) - listBox.SetVisible(false) - ssDLBtn.SetVisible(false) - ssPBLabel.SetVisible(false) - } - }) case pageModeName: - assistantPageComplete(assistant, mode, true) + assistantPageComplete(assistant, wgtWalletMode, true) case pageSeedGenerateName: if isRestoreMode { @@ -240,11 +118,11 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { assistant.PreviousPage() prevPageAdjust = -1 } - assistantPageComplete(assistant, seedGenerate, false) + assistantPageComplete(assistant, wgtSeedGenerate, false) } else { mnemonic, _ = wallet.GenerateMnemonic(128) - setTextViewContent(textViewSeed, mnemonic) - assistantPageComplete(assistant, seedGenerate, true) + setTextViewContent(txtSeed, mnemonic) + assistantPageComplete(assistant, wgtSeedGenerate, true) } case pageSeedConfirmName: if isRestoreMode { @@ -259,9 +137,9 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { assistant.PreviousPage() prevPageAdjust = -1 } - assistantPageComplete(assistant, seedConfirm, false) + assistantPageComplete(assistant, wgtSeedConfirm, false) } else { - assistantPageComplete(assistant, seedConfirm, false) + assistantPageComplete(assistant, wgtSeedConfirm, false) } case pageSeedRestoreName: if !isRestoreMode { @@ -276,24 +154,161 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { assistant.PreviousPage() prevPageAdjust = -1 } - assistantPageComplete(assistant, seedConfirm, false) + assistantPageComplete(assistant, wgtSeedConfirm, false) } else { - assistantPageComplete(assistant, seedRestore, true) + assistantPageComplete(assistant, wgtSeedRestore, true) } case pagePasswordName: if isRestoreMode { - mnemonic = getTextViewContent(textViewRestoreSeed) + mnemonic = getTextViewContent(textRestoreSeed) if err := wallet.CheckMnemonic(mnemonic); err != nil { showErrorDialog(assistant, "mnemonic is invalid") assistant.PreviousPage() } } - assistantPageComplete(assistant, password, true) + assistantPageComplete(assistant, wgtPassword, true) case pageNumValidatorsName: - assistantPageComplete(assistant, numValidators, true) + assistantPageComplete(assistant, wgtNumValidators, true) + + case pageNodeTypeName: + assistantPageComplete(assistant, wgtNodeType, true) + ssLabel, err := gtk.LabelNew("") + fatalErrorCheck(err) + setMargin(ssLabel, 5, 5, 1, 1) + ssLabel.SetHAlign(gtk.ALIGN_START) - case pageFinalName: + listBox, err := gtk.ListBoxNew() + fatalErrorCheck(err) + setMargin(listBox, 5, 5, 1, 1) + listBox.SetHAlign(gtk.ALIGN_CENTER) + listBox.SetSizeRequest(600, -1) + + ssDLBtn, err := gtk.ButtonNewWithLabel("⏬ Download") + fatalErrorCheck(err) + setMargin(ssDLBtn, 10, 5, 1, 1) + ssDLBtn.SetHAlign(gtk.ALIGN_CENTER) + ssDLBtn.SetSizeRequest(600, -1) + + ssPBLabel, err := gtk.LabelNew("") + fatalErrorCheck(err) + setMargin(ssPBLabel, 5, 10, 1, 1) + ssPBLabel.SetHAlign(gtk.ALIGN_START) + + gridImport.Attach(ssLabel, 0, 1, 1, 1) + gridImport.Attach(listBox, 0, 2, 1, 1) + gridImport.Attach(ssDLBtn, 0, 3, 1, 1) + gridImport.Attach(ssPBLabel, 0, 5, 1, 1) + ssLabel.SetVisible(false) + listBox.SetVisible(false) + ssDLBtn.SetVisible(false) + ssPBLabel.SetVisible(false) + + snapshotIndex := 0 + + radioImport.Connect("toggled", func() { + if radioImport.GetActive() { + assistantPageComplete(assistant, wgtNodeType, false) + + ssLabel.SetVisible(true) + ssLabel.SetText(" ♻️ Please wait, loading snapshot list...") + + go func() { + time.Sleep(1 * time.Second) + + glib.IdleAdd(func() { + snapshotURL := cmd.DefaultSnapshotURL // TODO: make me optional... + + storeDir := filepath.Join(workingDir, "data") + importer, err := cmd.NewImporter( + chainType, + snapshotURL, + storeDir, + ) + fatalErrorCheck(err) + + ctx := context.Background() + mdCh := getMetadata(ctx, importer, listBox) + + if md := <-mdCh; md == nil { + ssLabel.SetText(" ❌ Failed to get snapshot list, please try again later.") + } else { + ssLabel.SetText(" πŸ”½ Please select a snapshot to download:") + listBox.SetVisible(true) + + listBox.Connect("row-selected", func(_ *gtk.ListBox, row *gtk.ListBoxRow) { + if row != nil { + snapshotIndex = row.GetIndex() + ssDLBtn.SetVisible(true) + } + }) + + ssDLBtn.Connect("clicked", func() { + radioGroup, _ := radioImport.GetParent() + radioImport.SetSensitive(false) + radioGroup.ToWidget().SetSensitive(false) + ssLabel.SetSensitive(false) + listBox.SetSensitive(false) + ssDLBtn.SetSensitive(false) + + ssDLBtn.SetVisible(false) + ssPBLabel.SetVisible(true) + listBox.SetSelectionMode(gtk.SELECTION_NONE) + + go func() { + log.Printf("start downloading...\n") + + importer.Download( + ctx, + &md[snapshotIndex], + func(fileName string, totalSize, downloaded int64, + percentage float64, + ) { + percent := int(percentage) + glib.IdleAdd(func() { + dlMessage := fmt.Sprintf("🌐 Downloading %s | %d%% (%s / %s)", + fileName, + percent, + util.FormatBytesToHumanReadable(uint64(downloaded)), + util.FormatBytesToHumanReadable(uint64(totalSize)), + ) + ssPBLabel.SetText(" " + dlMessage) + }) + }, + ) + + glib.IdleAdd(func() { + log.Printf("extracting data...\n") + ssPBLabel.SetText(" " + "πŸ“‚ Extracting downloaded files...") + err := importer.ExtractAndStoreFiles() + fatalErrorCheck(err) + + log.Printf("moving data...\n") + ssPBLabel.SetText(" " + "πŸ“‘ Moving data...") + err = importer.MoveStore() + fatalErrorCheck(err) + + log.Printf("cleanup...\n") + err = importer.Cleanup() + fatalErrorCheck(err) + + ssPBLabel.SetText(" " + "βœ… Import completed.") + assistantPageComplete(assistant, wgtNodeType, true) + }) + }() + }) + } + }) + }() + } else { + assistantPageComplete(assistant, wgtNodeType, true) + ssLabel.SetVisible(false) + listBox.SetVisible(false) + ssDLBtn.SetVisible(false) + ssPBLabel.SetVisible(false) + } + }) + case pageSummaryName: iter, err := comboNumValidators.GetActiveIter() fatalErrorCheck(err) @@ -307,13 +322,13 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { walletPassword, err := entryPassword.GetText() fatalErrorCheck(err) - validatorAddrs, rewardAddrs, err := cmd.CreateNode(numValidators, chain, workingDir, mnemonic, walletPassword) + validatorAddrs, rewardAddrs, err := cmd.CreateNode(numValidators, chainType, workingDir, mnemonic, walletPassword) fatalErrorCheck(err) // Done! showing the node information successful = true nodeInfo := fmt.Sprintf("Working directory: %s\n", workingDir) - nodeInfo += fmt.Sprintf("Network: %s\n", chain.String()) + nodeInfo += fmt.Sprintf("Network: %s\n", chainType.String()) nodeInfo += "\nValidator addresses:\n" for i, addr := range validatorAddrs { nodeInfo += fmt.Sprintf("%v- %s\n", i+1, addr) @@ -324,7 +339,7 @@ func startupAssistant(workingDir string, chain genesis.ChainType) bool { nodeInfo += fmt.Sprintf("%v- %s\n", i+1, addr) } - setTextViewContent(textViewNodeInfo, nodeInfo) + setTextViewContent(txtNodeInfo, nodeInfo) } prevPageIndex = curPageIndex + prevPageAdjust }) @@ -374,7 +389,7 @@ func pageAssistant() assistantFunc { } } -func pageMode(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk.Widget, *gtk.RadioButton, string) { +func pageWalletMode(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk.Widget, *gtk.RadioButton, string) { var mode *gtk.Widget newWalletRadio, err := gtk.RadioButtonNewWithLabel(nil, "Create a new wallet from the scratch") fatalErrorCheck(err) @@ -391,8 +406,8 @@ func pageMode(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk.Widget, radioBox.Add(restoreWalletRadio) setMargin(restoreWalletRadio, 6, 6, 6, 6) - pageModeName := "page_mode" - pageModeTitle := "Initialize mode" + pageModeName := "page_wallet_mode" + pageModeTitle := "Wallet Mode" pageModeSubject := "How to create your wallet?" pageModeDesc := "If you are running the node for the first time, choose the first option." mode = assistFunc( @@ -418,7 +433,7 @@ func pageSeedGenerate(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk. textViewSeed.SetSizeRequest(0, 80) pageSeedName := "page_seed_generate" - pageSeedTitle := "Wallet seed" + pageSeedTitle := "Wallet Seed" pageSeedSubject := "Your wallet generation seed is:" pageSeedDesc := `Please write these 12 words on paper. This seed will allow you to recover your wallet in case of computer failure. @@ -450,7 +465,7 @@ func pageSeedRestore(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk.W textViewRestoreSeed.SetSizeRequest(0, 80) pageSeedName := "page_seed_restore" - pageSeedTitle := "Wallet seed restore" + pageSeedTitle := "Wallet Seed Restore" pageSeedSubject := "Enter your wallet seed:" pageSeedDesc := "Please enter your 12 words mnemonics backup to restore your wallet." @@ -500,7 +515,7 @@ func pageSeedConfirm(assistant *gtk.Assistant, assistFunc assistantFunc, }) pageSeedConfirmName := "page_seed_confirm" - pageSeedConfirmTitle := "Confirm seed" + pageSeedConfirmTitle := "Confirm Seed" pageSeedConfirmSubject := "What was your seed?" pageSeedConfirmDesc := `Your seed is important! To make sure that you have properly saved your seed, please retype it here.` @@ -516,43 +531,45 @@ To make sure that you have properly saved your seed, please retype it here.` return pageWidget, pageSeedConfirmName } -func pageImport(assistant *gtk.Assistant, assistFunc assistantFunc) ( +func pageNodeType(assistant *gtk.Assistant, assistFunc assistantFunc) ( *gtk.Widget, *gtk.Grid, *gtk.RadioButton, string, ) { - pageWidget := new(gtk.Widget) + var pageWidget *gtk.Widget vbox, err := gtk.BoxNew(gtk.ORIENTATION_VERTICAL, 0) - cmd.FatalErrorCheck(err) + fatalErrorCheck(err) grid, err := gtk.GridNew() - cmd.FatalErrorCheck(err) + fatalErrorCheck(err) - fullNode, err := gtk.RadioButtonNewWithLabel(nil, "Full node") - cmd.FatalErrorCheck(err) - fullNode.SetActive(true) + btnFullNode, err := gtk.RadioButtonNewWithLabel(nil, "Full node") + fatalErrorCheck(err) + btnFullNode.SetActive(true) - pruneNode, err := gtk.RadioButtonNewWithLabelFromWidget(fullNode, "Prune node") - cmd.FatalErrorCheck(err) + btnPruneNode, err := gtk.RadioButtonNewWithLabelFromWidget(btnFullNode, "Pruned node") + fatalErrorCheck(err) radioBox, err := gtk.BoxNew(gtk.ORIENTATION_VERTICAL, 0) - cmd.FatalErrorCheck(err) + fatalErrorCheck(err) - radioBox.Add(fullNode) - setMargin(fullNode, 6, 6, 6, 6) - radioBox.Add(pruneNode) - setMargin(pruneNode, 6, 10, 6, 6) + radioBox.Add(btnFullNode) + setMargin(btnFullNode, 6, 6, 6, 6) + radioBox.Add(btnPruneNode) + setMargin(btnPruneNode, 6, 10, 6, 6) grid.Attach(radioBox, 0, 0, 1, 1) vbox.PackStart(grid, true, true, 0) - pageName := "page_snapshot" - pageTitle := "Import" - pageSubject := "" - pageDesc := "" + pageName := "page_node_type" + pageTitle := "Node Type" + pageSubject := "How do you want to start your node?" + pageDesc := `A pruned node doesn’t keep all the historical data. +Instead, it only retains the most recent part of the blockchain, deleting older data to save disk space. +Offline data is available at: https://snapshot.pactus.org/.` // Create and return the page widget using assistFunc pageWidget = assistFunc( @@ -564,7 +581,7 @@ func pageImport(assistant *gtk.Assistant, assistFunc assistantFunc) ( pageDesc, ) - return pageWidget, grid, pruneNode, pageName + return pageWidget, grid, btnPruneNode, pageName } func pagePassword(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk.Widget, *gtk.Entry, string) { @@ -621,7 +638,7 @@ func pagePassword(assistant *gtk.Assistant, assistFunc assistantFunc) (*gtk.Widg }) pagePasswordName := "page_password" - pagePasswordTitle := "Wallet password" + pagePasswordTitle := "Wallet Password" pagePasswordSubject := "Enter password for your wallet:" pagePsswrdDesc := "Please choose a strong password for your wallet." @@ -674,7 +691,7 @@ func pageNumValidators(assistant *gtk.Assistant, grid.Attach(comboNumValidators, 1, 0, 1, 1) pageNumValidatorsName := "page_num_validators" - pageNumValidatorsTitle := "Number of validators" + pageNumValidatorsTitle := "Number of Validators" pageNumValidatorsSubject := "How many validators do you want to create?" pageNumValidatorsDesc := `Each node can run up to 32 validators, and each validator can hold up to 1000 staked coins. You can define validators based on the amount of coins you want to stake. @@ -691,7 +708,7 @@ For more information, look = maxDecompressedSize { - return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fpath) + return fmt.Errorf("file exceeds maximum decompressed size limit: %s", fPath) } return nil } + +func (i *Importer) MoveStore() error { + return util.MoveDirectory(filepath.Join(i.tempDir, "data"), i.storeDir) +} diff --git a/util/io.go b/util/io.go index c32ce1975..1b343a9d8 100644 --- a/util/io.go +++ b/util/io.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" ) func IsAbsPath(p string) bool { @@ -72,6 +73,7 @@ func TempFilePath() string { return filepath.Join(TempDirPath(), "file") } +// IsDirEmpty checks if a directory is empty. func IsDirEmpty(name string) bool { f, err := os.Open(name) if err != nil { @@ -88,6 +90,8 @@ func IsDirEmpty(name string) bool { return errors.Is(err, io.EOF) } +// IsDirNotExistsOrEmpty returns true if a directory does not exist or is empty. +// It checks if the path exists and, if so, whether the directory is empty. func IsDirNotExistsOrEmpty(name string) bool { if !PathExists(name) { return true @@ -196,3 +200,33 @@ func NewFixedReader(max int, buf []byte) *FixedReader { return &fr } + +// MoveDirectory moves a directory from srcDir to dstDir, including all its contents. +// If dstDir already exists and is not empty, it returns an error. +func MoveDirectory(srcDir, dstDir string) error { + if !IsDirNotExistsOrEmpty(dstDir) { + return fmt.Errorf("destination directory %s already exists", dstDir) + } + + if err := os.Rename(srcDir, dstDir); err != nil { + return fmt.Errorf("failed to move directory from %s to %s: %w", srcDir, dstDir, err) + } + + return nil +} + +// SanitizeArchivePath mitigates the "Zip Slip" vulnerability by sanitizing archive file paths. +// It ensures that the file path is contained within the specified base directory to prevent directory +// traversal attacks. For more details on the vulnerability, see https://snyk.io/research/zip-slip-vulnerability. +func SanitizeArchivePath(baseDir, archivePath string) (fullPath string, err error) { + fullPath = filepath.Join(baseDir, archivePath) + if filepath.IsAbs(fullPath) { + return "", fmt.Errorf("absolute path detected: %s", fullPath) + } + + if strings.HasPrefix(fullPath, filepath.Clean(baseDir)) { + return fullPath, nil + } + + return "", fmt.Errorf("%s: %s", "content filepath is tainted", archivePath) +} diff --git a/util/io_test.go b/util/io_test.go index fec6ea052..35bfa5633 100644 --- a/util/io_test.go +++ b/util/io_test.go @@ -2,7 +2,9 @@ package util import ( "fmt" + "os" "os/exec" + "path/filepath" "runtime" "strconv" "testing" @@ -92,3 +94,79 @@ func TestIsValidPath(t *testing.T) { assert.True(t, IsValidDirPath("/tmp")) assert.True(t, IsValidDirPath("/tmp/pactus")) } + +func TestMoveDirectory(t *testing.T) { + // Create temporary directories + srcDir := TempDirPath() + dstDir := TempDirPath() + defer func() { _ = os.RemoveAll(srcDir) }() + defer func() { _ = os.RemoveAll(dstDir) }() + + // Create a subdirectory in the source directory + subDir := filepath.Join(srcDir, "subdir") + err := Mkdir(subDir) + assert.NoError(t, err) + + // Create multiple files in the subdirectory + files := []struct { + name string + content string + }{ + {"file1.txt", "content 1"}, + {"file2.txt", "content 2"}, + } + + for _, file := range files { + filePath := filepath.Join(subDir, file.name) + err = WriteFile(filePath, []byte(file.content)) + assert.NoError(t, err) + } + + // Move the directory + dstDirPath := filepath.Join(dstDir, "movedir") + err = MoveDirectory(srcDir, dstDirPath) + assert.NoError(t, err) + + // Assert the source directory no longer exists + assert.False(t, PathExists(srcDir)) + + // Assert the destination directory exists + assert.True(t, PathExists(dstDirPath)) + + // Verify that all files have been moved and their contents are correct + for _, file := range files { + movedFilePath := filepath.Join(dstDirPath, "subdir", file.name) + data, err := ReadFile(movedFilePath) + assert.NoError(t, err) + assert.Equal(t, file.content, string(data)) + } +} + +func TestSanitizeArchivePath(t *testing.T) { + baseDir := "/safe/directory" + + tests := []struct { + name string + inputPath string + expected string + expectErr bool + }{ + {"Valid path", "file.txt", "/safe/directory/file.txt", false}, + {"Valid path in subdirectory", "subdir/file.txt", "/safe/directory/subdir/file.txt", false}, + {"Path with parent directory traversal", "../outside/file.txt", "", true}, + {"Absolute path outside base directory", "/etc/passwd", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := SanitizeArchivePath(baseDir, tt.inputPath) + if tt.expectErr { + assert.Error(t, err, "Expected error but got none") + assert.Empty(t, result, "Expected empty result due to error") + } else { + assert.NoError(t, err, "Unexpected error occurred") + assert.Equal(t, tt.expected, result, "Sanitized path did not match expected") + } + }) + } +} diff --git a/util/utils.go b/util/utils.go index 7d3b7dd08..94ec20bd1 100644 --- a/util/utils.go +++ b/util/utils.go @@ -5,7 +5,6 @@ import ( "fmt" "math/big" "math/bits" - "os" "golang.org/x/exp/constraints" ) @@ -161,18 +160,3 @@ func FormatBytesToHumanReadable(bytes uint64) string { return fmt.Sprintf("%.2f %s", value, unit) } - -// MoveDirectory moves a directory from srcDir to dstDir, including all its contents. -func MoveDirectory(srcDir, dstDir string) error { - // Ensure the destination directory does not already exist - if _, err := os.Stat(dstDir); !os.IsNotExist(err) { - return fmt.Errorf("destination directory %s already exists", dstDir) - } - - // Move the entire directory to the new location - if err := os.Rename(srcDir, dstDir); err != nil { - return fmt.Errorf("failed to move directory from %s to %s: %w", srcDir, dstDir, err) - } - - return nil -} diff --git a/util/utils_test.go b/util/utils_test.go index 74770cb3a..0b248535d 100644 --- a/util/utils_test.go +++ b/util/utils_test.go @@ -2,8 +2,6 @@ package util import ( "math/big" - "os" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -141,50 +139,3 @@ func TestFormatBytesToHumanReadable(t *testing.T) { } } } - -func TestMoveDirectory(t *testing.T) { - // Create temporary directories - srcDir := TempDirPath() - dstDir := TempDirPath() - defer func() { _ = os.RemoveAll(srcDir) }() - defer func() { _ = os.RemoveAll(dstDir) }() - - // Create a subdirectory in the source directory - subDir := filepath.Join(srcDir, "subdir") - err := Mkdir(subDir) - assert.NoError(t, err) - - // Create multiple files in the subdirectory - files := []struct { - name string - content string - }{ - {"file1.txt", "content 1"}, - {"file2.txt", "content 2"}, - } - - for _, file := range files { - filePath := filepath.Join(subDir, file.name) - err = WriteFile(filePath, []byte(file.content)) - assert.NoError(t, err) - } - - // Move the directory - dstDirPath := filepath.Join(dstDir, "movedir") - err = MoveDirectory(srcDir, dstDirPath) - assert.NoError(t, err) - - // Assert the source directory no longer exists - assert.False(t, PathExists(srcDir)) - - // Assert the destination directory exists - assert.True(t, PathExists(dstDirPath)) - - // Verify that all files have been moved and their contents are correct - for _, file := range files { - movedFilePath := filepath.Join(dstDirPath, "subdir", file.name) - data, err := ReadFile(movedFilePath) - assert.NoError(t, err) - assert.Equal(t, file.content, string(data)) - } -} From a0d8bb3ed932b4d305d0998be13c97812bc51bdf Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 24 Jul 2024 18:10:34 +0330 Subject: [PATCH 18/20] fix: sort items in metadata --- cmd/importer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/importer.go b/cmd/importer.go index e43f66b50..ce8405709 100644 --- a/cmd/importer.go +++ b/cmd/importer.go @@ -118,7 +118,7 @@ func (i *Importer) GetMetadata(ctx context.Context) ([]Metadata, error) { } sort.SliceStable(metadata, func(i, j int) bool { - return metadata[i].CreatedAtTime().Before(metadata[j].CreatedAtTime()) + return metadata[i].CreatedAtTime().After(metadata[j].CreatedAtTime()) }) return metadata, nil From 0e4cb7b045f85c51bfee60c63aaff0f668e06271 Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 24 Jul 2024 18:24:51 +0330 Subject: [PATCH 19/20] fix: remove check abs fullPath --- util/io.go | 4 ---- util/io_test.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/util/io.go b/util/io.go index 1b343a9d8..9b257d997 100644 --- a/util/io.go +++ b/util/io.go @@ -220,10 +220,6 @@ func MoveDirectory(srcDir, dstDir string) error { // traversal attacks. For more details on the vulnerability, see https://snyk.io/research/zip-slip-vulnerability. func SanitizeArchivePath(baseDir, archivePath string) (fullPath string, err error) { fullPath = filepath.Join(baseDir, archivePath) - if filepath.IsAbs(fullPath) { - return "", fmt.Errorf("absolute path detected: %s", fullPath) - } - if strings.HasPrefix(fullPath, filepath.Clean(baseDir)) { return fullPath, nil } diff --git a/util/io_test.go b/util/io_test.go index 35bfa5633..de15a0f85 100644 --- a/util/io_test.go +++ b/util/io_test.go @@ -154,7 +154,7 @@ func TestSanitizeArchivePath(t *testing.T) { {"Valid path", "file.txt", "/safe/directory/file.txt", false}, {"Valid path in subdirectory", "subdir/file.txt", "/safe/directory/subdir/file.txt", false}, {"Path with parent directory traversal", "../outside/file.txt", "", true}, - {"Absolute path outside base directory", "/etc/passwd", "", true}, + {"Absolute path outside base directory", "/etc/passwd", "/safe/directory/etc/passwd", false}, } for _, tt := range tests { From fdd810ace026dab3912609d8a4cfc7cac655183e Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 24 Jul 2024 18:30:35 +0330 Subject: [PATCH 20/20] fix: don't run test on windows --- util/io_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/util/io_test.go b/util/io_test.go index de15a0f85..aec40cd69 100644 --- a/util/io_test.go +++ b/util/io_test.go @@ -143,6 +143,10 @@ func TestMoveDirectory(t *testing.T) { } func TestSanitizeArchivePath(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + baseDir := "/safe/directory" tests := []struct {