-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix bugs in bulk loader. Add Dgraph debug tool. (#2449)
When building multiple shards, we want each predicate to only lie to one shard, and nowhere else. Bulk loader was doing that, but then adding the schema for all predicates on all shards, which causes confusion about ownership of the predicates. This changes that to only add schema for the predicates that the shard holds. Similarly, when outputting _predicate_ edge, the shard being used was the one corresponding to the original predicate. Instead, we should use the shard corresponding to _predicate_. This PR fixes #2129 . Added a new debug tool which can iterate over posting store and spit out stats per predicate. Useful for debugging.
- Loading branch information
1 parent
1a2fa20
commit 1c602b0
Showing
7 changed files
with
182 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| /* | ||
| * Copyright 2017-2018 Dgraph Labs, Inc. | ||
| * | ||
| * This file is available under the Apache License, Version 2.0, | ||
| * with the Commons Clause restriction. | ||
| */ | ||
|
|
||
| package debug | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "log" | ||
| "math" | ||
| "sort" | ||
|
|
||
| "github.com/dgraph-io/badger" | ||
| "github.com/dgraph-io/badger/options" | ||
| "github.com/dgraph-io/dgraph/x" | ||
| "github.com/spf13/cobra" | ||
| ) | ||
|
|
||
| var Debug x.SubCommand | ||
|
|
||
| func init() { | ||
| Debug.Cmd = &cobra.Command{ | ||
| Use: "debug", | ||
| Short: "Debug Dgraph instance", | ||
| Run: func(cmd *cobra.Command, args []string) { | ||
| run() | ||
| }, | ||
| } | ||
|
|
||
| flag := Debug.Cmd.Flags() | ||
| flag.StringP("postings", "p", "", "Directory where posting lists are stored.") | ||
| flag.BoolP("predicates", "s", false, "List all the predicates.") | ||
| flag.BoolP("readonly", "o", true, "Open in read only mode.") | ||
| } | ||
|
|
||
| type Stats struct { | ||
| Data int | ||
| Index int | ||
| Schema int | ||
| Reverse int | ||
| Count int | ||
| Total int | ||
| } | ||
|
|
||
| func run() { | ||
| opts := badger.DefaultOptions | ||
| opts.Dir = Debug.Conf.GetString("postings") | ||
| opts.ValueDir = Debug.Conf.GetString("postings") | ||
| opts.TableLoadingMode = options.MemoryMap | ||
| opts.ReadOnly = Debug.Conf.GetBool("readonly") | ||
|
|
||
| x.AssertTruef(len(opts.Dir) > 0, "No posting dir specified.") | ||
| fmt.Printf("Opening DB: %s\n", opts.Dir) | ||
| db, err := badger.OpenManaged(opts) | ||
| x.Check(err) | ||
| defer db.Close() | ||
|
|
||
| if Debug.Conf.GetBool("predicates") { | ||
| txn := db.NewTransactionAt(math.MaxUint64, false) | ||
| defer txn.Discard() | ||
|
|
||
| iopts := badger.DefaultIteratorOptions | ||
| iopts.PrefetchValues = false | ||
| itr := txn.NewIterator(iopts) | ||
| defer itr.Close() | ||
|
|
||
| var loop int | ||
| m := make(map[string]*Stats) | ||
| for itr.Rewind(); itr.Valid(); itr.Next() { | ||
| item := itr.Item() | ||
| pk := x.Parse(item.Key()) | ||
| stats, ok := m[pk.Attr] | ||
| if !ok { | ||
| stats = new(Stats) | ||
| m[pk.Attr] = stats | ||
| } | ||
| stats.Total += 1 | ||
| // Don't use a switch case here. Because multiple of these can be true. In particular, | ||
| // IsSchema can be true alongside IsData. | ||
| if pk.IsData() { | ||
| stats.Data += 1 | ||
| } | ||
| if pk.IsIndex() { | ||
| stats.Index += 1 | ||
| } | ||
| if pk.IsCount() { | ||
| stats.Count += 1 | ||
| } | ||
| if pk.IsSchema() { | ||
| stats.Schema += 1 | ||
| } | ||
| if pk.IsReverse() { | ||
| stats.Reverse += 1 | ||
| } | ||
| loop++ | ||
| } | ||
|
|
||
| type C struct { | ||
| pred string | ||
| stats *Stats | ||
| } | ||
|
|
||
| var counts []C | ||
| for pred, stats := range m { | ||
| counts = append(counts, C{pred, stats}) | ||
| } | ||
| sort.Slice(counts, func(i, j int) bool { | ||
| return counts[i].stats.Total > counts[j].stats.Total | ||
| }) | ||
| for _, c := range counts { | ||
| st := c.stats | ||
| fmt.Printf("Total: %-8d. Predicate: %-20s\n", st.Total, c.pred) | ||
| fmt.Printf(" Data: %d Index: %d Reverse: %d Schema: %d Count: %d Predicate: %s\n\n", | ||
| st.Data, st.Index, st.Reverse, st.Schema, st.Count, c.pred) | ||
| } | ||
| fmt.Printf("Found %d keys\n", loop) | ||
| return | ||
| } | ||
| log.Fatalln("Please provide a valid option for diagnosis.") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters