From 5201744199777772208d83ea5938d55970cd27e2 Mon Sep 17 00:00:00 2001 From: Phi Date: Mon, 19 Jun 2023 14:32:58 +0200 Subject: [PATCH 1/4] feat: snapshot: remove existing chain Remove existing chain on a snapshot-import log info instead of print log info instead of print Remove defer lock Remove defer lock, and explicitly close the lockedRepo instead --- cmd/lotus/daemon.go | 102 ++++++++++++++++++++++++++++++++++ documentation/en/cli-lotus.md | 1 + 2 files changed, 103 insertions(+) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 106446c0af3..c7762387620 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -12,6 +12,7 @@ import ( "io" "os" "path" + "path/filepath" "runtime/pprof" "strings" @@ -27,6 +28,7 @@ import ( "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" + "go.uber.org/multierr" "golang.org/x/xerrors" "gopkg.in/cheggaaa/pb.v1" @@ -128,6 +130,10 @@ var DaemonCmd = &cli.Command{ Name: "import-snapshot", Usage: "import chain state from a given chain export file or url", }, + &cli.BoolFlag{ + Name: "remove-existing-chain", + Usage: "remove existing chain and splitstore data on a snapshot-import", + }, &cli.BoolFlag{ Name: "halt-after-import", Usage: "halt the process after importing chain from file", @@ -285,6 +291,69 @@ var DaemonCmd = &cli.Command{ } } + if cctx.Bool("remove-existing-chain") { + lr, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return xerrors.Errorf("error opening fs repo: %w", err) + } + + exists, err := lr.Exists() + if err != nil { + return err + } + if !exists { + return xerrors.Errorf("lotus repo doesn't exist") + } + + lockedRepo, err := lr.Lock(repo.FullNode) + if err != nil { + return xerrors.Errorf("error locking repo: %w", err) + } + + cfg, err := lockedRepo.Config() + if err != nil { + lockedRepo.Close() + return xerrors.Errorf("error getting config: %w", err) + } + + fullNodeConfig, ok := cfg.(*config.FullNode) + if !ok { + lockedRepo.Close() + return xerrors.Errorf("wrong config type: %T", cfg) + } + + if fullNodeConfig.Chainstore.EnableSplitstore { + log.Info("removing splitstore directory...") + err = deleteSplitstoreDir(lockedRepo) + if err != nil { + lockedRepo.Close() + return xerrors.Errorf("error removing splitstore directory: %w", err) + } + } + + // Get the base repo path + repoPath := lockedRepo.Path() + + // Construct the path to the chain directory + chainPath := filepath.Join(repoPath, "datastore", "chain") + + log.Info("removing chain directory:", chainPath) + + err = os.RemoveAll(chainPath) + if err != nil { + lockedRepo.Close() + return xerrors.Errorf("error removing chain directory: %w", err) + } + + log.Info("chain and splitstore data have been removed") + + // Explicitly close the lockedRepo now that we're done with it. + err = lockedRepo.Close() + if err != nil { + return xerrors.Errorf("error releasing lock: %w", err) + } + } + chainfile := cctx.String("import-chain") snapshot := cctx.String("import-snapshot") if chainfile != "" || snapshot != "" { @@ -722,3 +791,36 @@ func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a l return nil, nil, nil } + +func deleteSplitstoreDir(lr repo.LockedRepo) error { + path, err := lr.SplitstorePath() + if err != nil { + return xerrors.Errorf("error getting splitstore path: %w", err) + } + + return os.RemoveAll(path) +} + +func clearSplitstoreDir(lr repo.LockedRepo) error { + path, err := lr.SplitstorePath() + if err != nil { + return xerrors.Errorf("error getting splitstore path: %w", err) + } + + entries, err := os.ReadDir(path) + if err != nil { + return xerrors.Errorf("error reading splitstore directory %s: %W", path, err) + } + + var result error + for _, e := range entries { + target := filepath.Join(path, e.Name()) + err = os.RemoveAll(target) + if err != nil { + log.Errorf("error removing %s: %s", target, err) + result = multierr.Append(result, err) + } + } + + return result +} diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 751182a4341..1674e2ffdda 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -65,6 +65,7 @@ OPTIONS: --bootstrap (default: true) --import-chain value on first run, load chain from given file or url and validate --import-snapshot value import chain state from a given chain export file or url + --remove-existing-chain remove existing chain and splitstore data on a snapshot-import (default: false) --halt-after-import halt the process after importing chain from file (default: false) --lite start lotus in lite mode (default: false) --pprof value specify name of file for writing cpu profile to From 13224e3e0ba23f0db280d8bd56ceeb18c583348e Mon Sep 17 00:00:00 2001 From: Phi Date: Tue, 4 Jul 2023 11:07:50 +0200 Subject: [PATCH 2/4] Remove unused function Remove unused clearSplitStoreDir function --- cmd/lotus/daemon.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index c7762387620..2193dd1a54a 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -28,7 +28,6 @@ import ( "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" - "go.uber.org/multierr" "golang.org/x/xerrors" "gopkg.in/cheggaaa/pb.v1" @@ -800,27 +799,3 @@ func deleteSplitstoreDir(lr repo.LockedRepo) error { return os.RemoveAll(path) } - -func clearSplitstoreDir(lr repo.LockedRepo) error { - path, err := lr.SplitstorePath() - if err != nil { - return xerrors.Errorf("error getting splitstore path: %w", err) - } - - entries, err := os.ReadDir(path) - if err != nil { - return xerrors.Errorf("error reading splitstore directory %s: %W", path, err) - } - - var result error - for _, e := range entries { - target := filepath.Join(path, e.Name()) - err = os.RemoveAll(target) - if err != nil { - log.Errorf("error removing %s: %s", target, err) - result = multierr.Append(result, err) - } - } - - return result -} From dedfa143ce18186f296d0ec98c67817ab3f6b817 Mon Sep 17 00:00:00 2001 From: Phi Date: Tue, 4 Jul 2023 11:30:06 +0200 Subject: [PATCH 3/4] Extract removeExistingChain to seperate func Extract removeExistingChain to seperate func --- cmd/lotus/daemon.go | 90 ++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 2193dd1a54a..9d7867cdc30 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -304,52 +304,9 @@ var DaemonCmd = &cli.Command{ return xerrors.Errorf("lotus repo doesn't exist") } - lockedRepo, err := lr.Lock(repo.FullNode) + err = removeExistingChain(cctx, lr) if err != nil { - return xerrors.Errorf("error locking repo: %w", err) - } - - cfg, err := lockedRepo.Config() - if err != nil { - lockedRepo.Close() - return xerrors.Errorf("error getting config: %w", err) - } - - fullNodeConfig, ok := cfg.(*config.FullNode) - if !ok { - lockedRepo.Close() - return xerrors.Errorf("wrong config type: %T", cfg) - } - - if fullNodeConfig.Chainstore.EnableSplitstore { - log.Info("removing splitstore directory...") - err = deleteSplitstoreDir(lockedRepo) - if err != nil { - lockedRepo.Close() - return xerrors.Errorf("error removing splitstore directory: %w", err) - } - } - - // Get the base repo path - repoPath := lockedRepo.Path() - - // Construct the path to the chain directory - chainPath := filepath.Join(repoPath, "datastore", "chain") - - log.Info("removing chain directory:", chainPath) - - err = os.RemoveAll(chainPath) - if err != nil { - lockedRepo.Close() - return xerrors.Errorf("error removing chain directory: %w", err) - } - - log.Info("chain and splitstore data have been removed") - - // Explicitly close the lockedRepo now that we're done with it. - err = lockedRepo.Close() - if err != nil { - return xerrors.Errorf("error releasing lock: %w", err) + return err } } @@ -791,6 +748,49 @@ func slashFilterMinedBlock(ctx context.Context, sf *slashfilter.SlashFilter, a l return nil, nil, nil } +func removeExistingChain(cctx *cli.Context, lr repo.Repo) error { + lockedRepo, err := lr.Lock(repo.FullNode) + if err != nil { + return xerrors.Errorf("error locking repo: %w", err) + } + // Ensure that lockedRepo is closed when this function exits + defer lockedRepo.Close() + + cfg, err := lockedRepo.Config() + if err != nil { + return xerrors.Errorf("error getting config: %w", err) + } + + fullNodeConfig, ok := cfg.(*config.FullNode) + if !ok { + return xerrors.Errorf("wrong config type: %T", cfg) + } + + if fullNodeConfig.Chainstore.EnableSplitstore { + log.Info("removing splitstore directory...") + err = deleteSplitstoreDir(lockedRepo) + if err != nil { + return xerrors.Errorf("error removing splitstore directory: %w", err) + } + } + + // Get the base repo path + repoPath := lockedRepo.Path() + + // Construct the path to the chain directory + chainPath := filepath.Join(repoPath, "datastore", "chain") + + log.Info("removing chain directory:", chainPath) + + err = os.RemoveAll(chainPath) + if err != nil { + return xerrors.Errorf("error removing chain directory: %w", err) + } + + log.Info("chain and splitstore data have been removed") + return nil +} + func deleteSplitstoreDir(lr repo.LockedRepo) error { path, err := lr.SplitstorePath() if err != nil { From 2061c32a2e9826434934540316b8eb4c853a77ef Mon Sep 17 00:00:00 2001 From: Phi Date: Tue, 4 Jul 2023 11:53:28 +0200 Subject: [PATCH 4/4] Check error Check error --- cmd/lotus/daemon.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 9d7867cdc30..14a8669b397 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -754,7 +754,11 @@ func removeExistingChain(cctx *cli.Context, lr repo.Repo) error { return xerrors.Errorf("error locking repo: %w", err) } // Ensure that lockedRepo is closed when this function exits - defer lockedRepo.Close() + defer func() { + if closeErr := lockedRepo.Close(); closeErr != nil { + log.Errorf("Error closing the lockedRepo: %v", closeErr) + } + }() cfg, err := lockedRepo.Config() if err != nil {