-
Notifications
You must be signed in to change notification settings - Fork 183
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into alex/consensus-sealing-halt-on-exec-fork
- Loading branch information
Showing
32 changed files
with
816 additions
and
531 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "os" | ||
| "path" | ||
| ) | ||
|
|
||
| func copyDir(src string, dst string) error { | ||
| var err error | ||
| var fds []os.FileInfo | ||
| var srcinfo os.FileInfo | ||
|
|
||
| if srcinfo, err = os.Stat(src); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if fds, err = ioutil.ReadDir(src); err != nil { | ||
| return err | ||
| } | ||
| for _, fd := range fds { | ||
| srcfp := path.Join(src, fd.Name()) | ||
| dstfp := path.Join(dst, fd.Name()) | ||
|
|
||
| if fd.IsDir() { | ||
| if err = copyDir(srcfp, dstfp); err != nil { | ||
| fmt.Println(err) | ||
| } | ||
| } else { | ||
| if err = copyFile(srcfp, dstfp); err != nil { | ||
| fmt.Println(err) | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // CopyFile copies a single file from src to dst | ||
| func copyFile(src, dst string) error { | ||
| var err error | ||
| var srcfd *os.File | ||
| var dstfd *os.File | ||
| var srcinfo os.FileInfo | ||
|
|
||
| if srcfd, err = os.Open(src); err != nil { | ||
| return err | ||
| } | ||
| defer srcfd.Close() | ||
|
|
||
| if dstfd, err = os.Create(dst); err != nil { | ||
| return err | ||
| } | ||
| defer dstfd.Close() | ||
|
|
||
| if _, err = io.Copy(dstfd, srcfd); err != nil { | ||
| return err | ||
| } | ||
| if srcinfo, err = os.Stat(src); err != nil { | ||
| return err | ||
| } | ||
| return os.Chmod(dst, srcinfo.Mode()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,133 +1,30 @@ | ||
| package jsonexporter | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "encoding/hex" | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "github.com/rs/zerolog/log" | ||
|
|
||
| "github.com/onflow/flow-go/cmd/util/cmd/common" | ||
| "github.com/onflow/flow-go/ledger" | ||
| "github.com/onflow/flow-go/ledger/common/pathfinder" | ||
| "github.com/onflow/flow-go/ledger/complete" | ||
| "github.com/onflow/flow-go/ledger/complete/mtrie" | ||
| "github.com/onflow/flow-go/ledger/complete/mtrie/flattener" | ||
| "github.com/onflow/flow-go/ledger/complete/wal" | ||
| "github.com/onflow/flow-go/model/flow" | ||
| "github.com/onflow/flow-go/module/metrics" | ||
| "github.com/onflow/flow-go/storage/badger" | ||
| ) | ||
|
|
||
| // ExportLedger exports ledger key value pairs at the given blockID | ||
| func ExportLedger(blockID flow.Identifier, dbPath string, ledgerPath string, outputPath string) error { | ||
| db := common.InitStorage(dbPath) | ||
| defer db.Close() | ||
| func ExportLedger(ledgerPath string, targetstate string, outputPath string) error { | ||
|
|
||
| cache := &metrics.NoopCollector{} | ||
| commits := badger.NewCommits(cache, db) | ||
|
|
||
| targetHash, err := commits.ByBlockID(blockID) | ||
| led, err := complete.NewLedger(ledgerPath, complete.DefaultCacheSize, &metrics.NoopCollector{}, log.Logger, nil, 0) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot get state commitment for block: %w", err) | ||
| return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err) | ||
| } | ||
|
|
||
| w, err := wal.NewWAL(nil, nil, ledgerPath, complete.DefaultCacheSize, pathfinder.PathByteSize, wal.SegmentSize) | ||
| state, err := hex.DecodeString(targetstate) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot create WAL: %w", err) | ||
| return fmt.Errorf("failed to decode hex code of state: %w", err) | ||
| } | ||
| defer func() { | ||
| _ = w.Close() | ||
| }() | ||
|
|
||
| // TODO port this to use new forest | ||
| forest, err := mtrie.NewForest(pathfinder.PathByteSize, outputPath, complete.DefaultCacheSize, &metrics.NoopCollector{}, nil) | ||
| err = led.DumpTrieAsJSON(ledger.State(state), outputPath) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot create mForest: %w", err) | ||
| return fmt.Errorf("cannot dump trie as json: %w", err) | ||
| } | ||
|
|
||
| i := 0 | ||
| valuesSize := 0 | ||
| valuesCount := 0 | ||
| startTime := time.Now() | ||
| found := false | ||
| FoundHashError := fmt.Errorf("found hash %s", targetHash) | ||
|
|
||
| err = w.ReplayLogsOnly( | ||
| func(forestSequencing *flattener.FlattenedForest) error { | ||
| rebuiltTries, err := flattener.RebuildTries(forestSequencing) | ||
| if err != nil { | ||
| return fmt.Errorf("rebuilding forest from sequenced nodes failed: %w", err) | ||
| } | ||
| err = forest.AddTries(rebuiltTries) | ||
| if err != nil { | ||
| return fmt.Errorf("adding rebuilt tries to forest failed: %w", err) | ||
| } | ||
| return nil | ||
| }, | ||
| func(update *ledger.TrieUpdate) error { | ||
|
|
||
| newTrieHash, err := forest.Update(update) | ||
|
|
||
| for _, value := range update.Payloads { | ||
| valuesSize += len(value.Value) | ||
| } | ||
|
|
||
| valuesCount += len(update.Payloads) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("error while updating mForest: %w", err) | ||
| } | ||
|
|
||
| if bytes.Equal(targetHash, newTrieHash) { | ||
| found = true | ||
| return FoundHashError | ||
| } | ||
|
|
||
| i++ | ||
| if i%1000 == 0 { | ||
| log.Info().Int("values_count", valuesCount).Int("values_size_bytes", valuesSize).Int("updates_count", i).Msg("progress") | ||
| } | ||
|
|
||
| return err | ||
| }, | ||
| func(commitment ledger.RootHash) error { | ||
| return nil | ||
| }) | ||
|
|
||
| duration := time.Since(startTime) | ||
|
|
||
| if !errors.Is(err, FoundHashError) { | ||
| return fmt.Errorf("error while processing WAL: %w", err) | ||
| } | ||
|
|
||
| if !found { | ||
| return fmt.Errorf("no value found: %w", err) | ||
| } | ||
|
|
||
| log.Info().Int("values_count", valuesCount).Int("values_size_bytes", valuesSize).Int("updates_count", i).Float64("total_time_s", duration.Seconds()).Msg("finished seeking") | ||
| log.Info().Msg("writing root checkpoint") | ||
|
|
||
| trie, err := forest.GetTrie(targetHash) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot get a trie with target hash: %w", err) | ||
| } | ||
|
|
||
| path := filepath.Join(outputPath, hex.EncodeToString(targetHash)+".trie.jsonl") | ||
|
|
||
| fi, err := os.Create(path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer fi.Close() | ||
|
|
||
| writer := bufio.NewWriter(fi) | ||
| defer writer.Flush() | ||
|
|
||
| return trie.DumpAsJSON(writer) | ||
| return nil | ||
| } |
Oops, something went wrong.