-
Notifications
You must be signed in to change notification settings - Fork 183
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Ledger: improving ledger export cmd functionality (#175)
- Loading branch information
Showing
4 changed files
with
61 additions
and
119 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,133 +1,30 @@ | ||
| package jsonexporter | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "encoding/hex" | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "github.com/rs/zerolog/log" | ||
|
|
||
| "github.com/onflow/flow-go/cmd/util/cmd/common" | ||
| "github.com/onflow/flow-go/ledger" | ||
| "github.com/onflow/flow-go/ledger/common/pathfinder" | ||
| "github.com/onflow/flow-go/ledger/complete" | ||
| "github.com/onflow/flow-go/ledger/complete/mtrie" | ||
| "github.com/onflow/flow-go/ledger/complete/mtrie/flattener" | ||
| "github.com/onflow/flow-go/ledger/complete/wal" | ||
| "github.com/onflow/flow-go/model/flow" | ||
| "github.com/onflow/flow-go/module/metrics" | ||
| "github.com/onflow/flow-go/storage/badger" | ||
| ) | ||
|
|
||
| // ExportLedger exports ledger key value pairs at the given blockID | ||
| func ExportLedger(blockID flow.Identifier, dbPath string, ledgerPath string, outputPath string) error { | ||
| db := common.InitStorage(dbPath) | ||
| defer db.Close() | ||
| func ExportLedger(ledgerPath string, targetstate string, outputPath string) error { | ||
|
|
||
| cache := &metrics.NoopCollector{} | ||
| commits := badger.NewCommits(cache, db) | ||
|
|
||
| targetHash, err := commits.ByBlockID(blockID) | ||
| led, err := complete.NewLedger(ledgerPath, complete.DefaultCacheSize, &metrics.NoopCollector{}, log.Logger, nil, 0) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot get state commitment for block: %w", err) | ||
| return fmt.Errorf("cannot create ledger from write-a-head logs and checkpoints: %w", err) | ||
| } | ||
|
|
||
| w, err := wal.NewWAL(nil, nil, ledgerPath, complete.DefaultCacheSize, pathfinder.PathByteSize, wal.SegmentSize) | ||
| state, err := hex.DecodeString(targetstate) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot create WAL: %w", err) | ||
| return fmt.Errorf("failed to decode hex code of state: %w", err) | ||
| } | ||
| defer func() { | ||
| _ = w.Close() | ||
| }() | ||
|
|
||
| // TODO port this to use new forest | ||
| forest, err := mtrie.NewForest(pathfinder.PathByteSize, outputPath, complete.DefaultCacheSize, &metrics.NoopCollector{}, nil) | ||
| err = led.DumpTrieAsJSON(ledger.State(state), outputPath) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot create mForest: %w", err) | ||
| return fmt.Errorf("cannot dump trie as json: %w", err) | ||
| } | ||
|
|
||
| i := 0 | ||
| valuesSize := 0 | ||
| valuesCount := 0 | ||
| startTime := time.Now() | ||
| found := false | ||
| FoundHashError := fmt.Errorf("found hash %s", targetHash) | ||
|
|
||
| err = w.ReplayLogsOnly( | ||
| func(forestSequencing *flattener.FlattenedForest) error { | ||
| rebuiltTries, err := flattener.RebuildTries(forestSequencing) | ||
| if err != nil { | ||
| return fmt.Errorf("rebuilding forest from sequenced nodes failed: %w", err) | ||
| } | ||
| err = forest.AddTries(rebuiltTries) | ||
| if err != nil { | ||
| return fmt.Errorf("adding rebuilt tries to forest failed: %w", err) | ||
| } | ||
| return nil | ||
| }, | ||
| func(update *ledger.TrieUpdate) error { | ||
|
|
||
| newTrieHash, err := forest.Update(update) | ||
|
|
||
| for _, value := range update.Payloads { | ||
| valuesSize += len(value.Value) | ||
| } | ||
|
|
||
| valuesCount += len(update.Payloads) | ||
|
|
||
| if err != nil { | ||
| return fmt.Errorf("error while updating mForest: %w", err) | ||
| } | ||
|
|
||
| if bytes.Equal(targetHash, newTrieHash) { | ||
| found = true | ||
| return FoundHashError | ||
| } | ||
|
|
||
| i++ | ||
| if i%1000 == 0 { | ||
| log.Info().Int("values_count", valuesCount).Int("values_size_bytes", valuesSize).Int("updates_count", i).Msg("progress") | ||
| } | ||
|
|
||
| return err | ||
| }, | ||
| func(commitment ledger.RootHash) error { | ||
| return nil | ||
| }) | ||
|
|
||
| duration := time.Since(startTime) | ||
|
|
||
| if !errors.Is(err, FoundHashError) { | ||
| return fmt.Errorf("error while processing WAL: %w", err) | ||
| } | ||
|
|
||
| if !found { | ||
| return fmt.Errorf("no value found: %w", err) | ||
| } | ||
|
|
||
| log.Info().Int("values_count", valuesCount).Int("values_size_bytes", valuesSize).Int("updates_count", i).Float64("total_time_s", duration.Seconds()).Msg("finished seeking") | ||
| log.Info().Msg("writing root checkpoint") | ||
|
|
||
| trie, err := forest.GetTrie(targetHash) | ||
| if err != nil { | ||
| return fmt.Errorf("cannot get a trie with target hash: %w", err) | ||
| } | ||
|
|
||
| path := filepath.Join(outputPath, hex.EncodeToString(targetHash)+".trie.jsonl") | ||
|
|
||
| fi, err := os.Create(path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer fi.Close() | ||
|
|
||
| writer := bufio.NewWriter(fi) | ||
| defer writer.Flush() | ||
|
|
||
| return trie.DumpAsJSON(writer) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters