From 4ccd4e573319159507546b9617bc903f6ff32ccd Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Fri, 5 Apr 2019 14:54:31 -0700 Subject: [PATCH 1/6] Remove _predicate_ from Dgraph. From now on, expand queries will rely solely on the type of the node. This change removes all mentions of the _predicate_ internal pred that was previously being used to serve this type of queries. There is also another small change to allow the value of a list variable to be used as a list of predicates in expand queries. Most of the changes due to removing or replacing tests. These changes are due to one of the following reasons. 1. Tests that were checking for the state of _predicate_ and are no longer valid. 2. Tests that were doing checks that should be done inside of the query package. They have been removed if there was an existing query that was doing the same or a new test has been added if no similar test existed. 3. Some of the expand tests inside the query package were removed because there are tests verifying the expand functionality using types. Since types are now the only way to make expand queries work, these tests became obsolete. --- dgraph/cmd/alpha/http_test.go | 7 +- dgraph/cmd/alpha/run.go | 4 - dgraph/cmd/alpha/run_test.go | 196 +++--------------------- dgraph/cmd/bulk/loader.go | 1 - dgraph/cmd/bulk/mapper.go | 17 --- dgraph/cmd/bulk/run.go | 4 - dgraph/cmd/bulk/schema.go | 7 +- dgraph/cmd/debug/run.go | 2 +- dgraph/cmd/zero/oracle.go | 17 +-- edgraph/config.go | 12 -- posting/list.go | 5 +- query/common_test.go | 35 ++++- query/mutation.go | 47 ++---- query/query.go | 104 +++---------- query/query0_test.go | 6 +- query/query1_test.go | 53 +------ query/query3_test.go | 5 +- query/query4_test.go | 19 ++- query/query_facets_test.go | 16 ++ schema/parse.go | 4 - schema/parse_test.go | 8 - schema/schema.go | 9 -- systest/mutations_test.go | 272 +++------------------------------- systest/queries_test.go | 13 -- worker/export.go | 3 + worker/groups.go | 2 +- x/config.go | 1 - x/keys.go | 41 +++-- x/x.go | 3 - z/client.go | 27 ---- 30 files changed, 187 insertions(+), 753 deletions(-) diff --git a/dgraph/cmd/alpha/http_test.go b/dgraph/cmd/alpha/http_test.go index 00b7815ec14..2f85c2a9f31 100644 --- a/dgraph/cmd/alpha/http_test.go +++ b/dgraph/cmd/alpha/http_test.go @@ -306,15 +306,14 @@ func TestTransactionBasic(t *testing.T) { require.NoError(t, err) require.Equal(t, mts, ts) require.Equal(t, 3, len(keys)) - require.Equal(t, 3, len(preds)) + require.Equal(t, 2, len(preds)) var parsedPreds []string for _, pred := range preds { parsedPreds = append(parsedPreds, strings.Join(strings.Split(pred, "-")[1:], "-")) } sort.Strings(parsedPreds) - require.Equal(t, "_predicate_", parsedPreds[0]) - require.Equal(t, "balance", parsedPreds[1]) - require.Equal(t, "name", parsedPreds[2]) + require.Equal(t, "balance", parsedPreds[0]) + require.Equal(t, "name", parsedPreds[1]) data, _, err := queryWithTs(q1, 0) require.NoError(t, err) diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index 92178528d63..a5474d657f9 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -118,9 +118,6 @@ they form a Raft group and provide synchronous replication. "IP_ADDRESS:PORT of a Dgraph Zero.") flag.Uint64("idx", 0, "Optional Raft ID that this Dgraph Alpha will use to join RAFT groups.") - flag.Bool("expand_edge", true, - "Enables the expand() feature. This is very expensive for large data loads because it"+ - " doubles the number of mutations going on in the system.") flag.Int("max_retries", -1, "Commits to disk will give up after these number of retries to prevent locking the worker"+ " in a failed state. Use -1 to retry infinitely.") @@ -469,7 +466,6 @@ func run() { MyAddr: Alpha.Conf.GetString("my"), ZeroAddr: Alpha.Conf.GetString("zero"), RaftId: cast.ToUint64(Alpha.Conf.GetString("idx")), - ExpandEdge: Alpha.Conf.GetBool("expand_edge"), WhiteListedIPRanges: ips, MaxRetries: Alpha.Conf.GetInt("max_retries"), StrictMutations: opts.MutationsMode == edgraph.StrictMutations, diff --git a/dgraph/cmd/alpha/run_test.go b/dgraph/cmd/alpha/run_test.go index 3fdd14e9a27..ff3b6d6647e 100644 --- a/dgraph/cmd/alpha/run_test.go +++ b/dgraph/cmd/alpha/run_test.go @@ -209,14 +209,6 @@ func TestDeletePredicate(t *testing.T) { ` var q4 = ` - { - user(func: uid(0x3)) { - _predicate_ - } - } - ` - - var q5 = ` { user(func: uid(0x3)) { age @@ -260,10 +252,6 @@ func TestDeletePredicate(t *testing.T) { require.NoError(t, err) require.JSONEq(t, `{"data": {"user":[{"age": "13", "~friend" : [{"name":"Alice"}]}]}}`, output) - output, err = runQuery(q4) - require.NoError(t, err) - require.JSONEq(t, `{"data": {"user":[{"_predicate_":["name","age"]}]}}`, output) - err = deletePredicate("friend") require.NoError(t, err) err = deletePredicate("salary") @@ -272,7 +260,6 @@ func TestDeletePredicate(t *testing.T) { output, err = runQuery(`schema{}`) require.NoError(t, err) z.CompareJSON(t, `{"data":{"schema":[`+ - `{"predicate":"_predicate_","type":"string","list":true},`+ `{"predicate":"age","type":"default"},`+ `{"predicate":"name","type":"string","index":true, "tokenizer":["term"]},`+ x.AclPredicates+","+ @@ -287,13 +274,9 @@ func TestDeletePredicate(t *testing.T) { require.NoError(t, err) require.JSONEq(t, `{"data": {"user": [{"name":"Alice"},{"name":"Alice1"},{"name":"Alice2"}]}}`, output) - output, err = runQuery(q5) - require.NoError(t, err) - require.JSONEq(t, `{"data": {"user":[{"age": "13"}]}}`, output) - output, err = runQuery(q4) require.NoError(t, err) - require.JSONEq(t, `{"data": {"user":[{"_predicate_":["name","age"]}]}}`, output) + require.JSONEq(t, `{"data": {"user":[{"age": "13"}]}}`, output) // Lets try to change the type of predicates now. err = alterSchemaWithRetry(s2) @@ -1053,122 +1036,6 @@ func BenchmarkQuery(b *testing.B) { } } -func TestListPred(t *testing.T) { - require.NoError(t, alterSchema(`{"drop_all": true}`)) - var q1 = ` - { - listpred(func:anyofterms(name, "Alice")) { - _predicate_ - } - } - ` - var m = ` - { - set { - <0x1> "Alice" . - <0x1> "13" . - <0x1> <0x4> . - } - } - ` - var s = ` - name:string @index(term) . - ` - - // reset Schema - schema.ParseBytes([]byte(""), 1) - err := runMutation(m) - require.NoError(t, err) - - // add index to name - err = alterSchemaWithRetry(s) - require.NoError(t, err) - - output, err := runQuery(q1) - require.NoError(t, err) - require.JSONEq(t, `{"data": {"listpred":[{"_predicate_":["name","age","friend"]}]}}`, - output) -} - -func TestExpandPredError(t *testing.T) { - var q1 = ` - { - me(func:anyofterms(name, "Alice")) { - expand(_all_) - name - friend - } - } - ` - var m = ` - { - set { - <0x1> "Alice" . - <0x1> "13" . - <0x1> <0x4> . - <0x4> "bob" . - <0x4> "12" . - } - } - ` - var s = ` - name:string @index(term) . - ` - - // reset Schema - schema.ParseBytes([]byte(""), 1) - err := runMutation(m) - require.NoError(t, err) - - // add index to name - err = alterSchemaWithRetry(s) - require.NoError(t, err) - - _, err = runQuery(q1) - require.Error(t, err) - require.Contains(t, err.Error(), "Repeated subgraph") -} - -func TestExpandPred(t *testing.T) { - var q1 = ` - { - me(func: uid(0x11)) { - expand(_all_) { - expand(_all_) - } - } - } - ` - var m = ` - { - set { - <0x11> "Alice" . - <0x11> "13" . - <0x11> <0x4> . - <0x4> "bob" . - <0x4> "12" . - } - } - ` - var s = ` - name:string @index(term) . - ` - - // reset Schema - schema.ParseBytes([]byte(""), 1) - err := runMutation(m) - require.NoError(t, err) - - // add index to name - err = alterSchemaWithRetry(s) - require.NoError(t, err) - - output, err := runQuery(q1) - require.NoError(t, err) - require.JSONEq(t, `{"data": {"me":[{"age":"13","friend":[{"age":"12","name":"bob"}],"name":"Alice"}]}}`, - output) -} - var threeNiceFriends = `{ "data": { "me": [ @@ -1348,13 +1215,30 @@ func TestListTypeSchemaChange(t *testing.T) { require.NoError(t, err) z.CompareJSON(t, `{"data":{"schema":[`+ x.AclPredicates+","+ - `{"predicate":"_predicate_","type":"string","list":true},`+ `{"predicate":"occupations","type":"string"},`+ `{"predicate":"dgraph.type", "type":"string", "index":true, "tokenizer": ["exact"], "list":true}]}}`, res) } func TestDeleteAllSP2(t *testing.T) { + s := ` + type Node12345 { + nodeType: string + name: string + date: dateTime + weight: float + weightUnit: string + lifeLoad: int + stressLevel: int + plan: string + postMortem: string + } + ` + require.NoError(t, dropAll()) + schema.ParseBytes([]byte(""), 1) + err := alterSchemaWithRetry(s) + require.NoError(t, err) + var m = ` { set { @@ -1367,16 +1251,16 @@ func TestDeleteAllSP2(t *testing.T) { <0x12345> "3" . <0x12345> "modest day" . <0x12345> "win!" . + <0x12345> "Node12345" . } } ` - err := runMutation(m) + err = runMutation(m) require.NoError(t, err) q := fmt.Sprintf(` { me(func: uid(%s)) { - _predicate_ name date weight @@ -1387,7 +1271,7 @@ func TestDeleteAllSP2(t *testing.T) { output, err := runQuery(q) require.NoError(t, err) - require.JSONEq(t, `{"data": {"me":[{"_predicate_":["name","date","weightUnit","postMortem","lifeLoad","weight","stressLevel","nodeType","plan"],"name":"July 3 2017","date":"2017-07-03T03:49:03+00:00","weight":"262.3","lifeLoad":"5","stressLevel":"3"}]}}`, output) + require.JSONEq(t, `{"data": {"me":[{"name":"July 3 2017","date":"2017-07-03T03:49:03+00:00","weight":"262.3","lifeLoad":"5","stressLevel":"3"}]}}`, output) m = fmt.Sprintf(` { @@ -1550,7 +1434,7 @@ func TestDropAll(t *testing.T) { output, err = runQuery(q3) require.NoError(t, err) z.CompareJSON(t, - `{"data":{"schema":[{"predicate":"_predicate_","type":"string","list":true},`+ + `{"data":{"schema":[`+ x.AclPredicates+","+ `{"predicate":"dgraph.type", "type":"string", "index":true, "tokenizer":["exact"], "list":true}]}}`, output) @@ -1571,40 +1455,6 @@ func TestDropAll(t *testing.T) { require.JSONEq(t, `{"data": {"q":[]}}`, output) } -func TestRecurseExpandAll(t *testing.T) { - var q1 = ` - { - me(func:anyofterms(name, "Alica")) @recurse { - expand(_all_) - } - } - ` - var m = ` - { - set { - <0x1> "Alica" . - <0x1> "13" . - <0x1> <0x4> . - <0x4> "bob" . - <0x4> "12" . - } - } - ` - - var s = `name:string @index(term) .` - // reset Schema - schema.ParseBytes([]byte(""), 1) - err := runMutation(m) - require.NoError(t, err) - - err = alterSchemaWithRetry(s) - require.NoError(t, err) - - output, err := runQuery(q1) - require.NoError(t, err) - require.JSONEq(t, `{"data": {"me":[{"name":"Alica","age":"13","friend":[{"name":"bob","age":"12"}]}]}}`, output) -} - func TestIllegalCountInQueryFn(t *testing.T) { s := `friend: [uid] @count .` require.NoError(t, alterSchemaWithRetry(s)) diff --git a/dgraph/cmd/bulk/loader.go b/dgraph/cmd/bulk/loader.go index b59bb1aca69..50d2ff25eb1 100644 --- a/dgraph/cmd/bulk/loader.go +++ b/dgraph/cmd/bulk/loader.go @@ -50,7 +50,6 @@ type options struct { TmpDir string NumGoroutines int MapBufSize int64 - ExpandEdges bool SkipMapPhase bool CleanupTmp bool NumShufflers int diff --git a/dgraph/cmd/bulk/mapper.go b/dgraph/cmd/bulk/mapper.go index 8cdb6128d2b..62d77b17b50 100644 --- a/dgraph/cmd/bulk/mapper.go +++ b/dgraph/cmd/bulk/mapper.go @@ -208,13 +208,6 @@ func (m *mapper) processNQuad(nq gql.NQuad) { m.addMapEntry(key, rev, shard) } m.addIndexMapEntries(nq, de) - - if m.opt.ExpandEdges { - shard := m.state.shards.shardFor("_predicate_") - key = x.DataKey("_predicate_", sid) - pp := m.createPredicatePosting(nq.Predicate) - m.addMapEntry(key, pp, shard) - } } func (m *mapper) uid(xid string) uint64 { @@ -248,16 +241,6 @@ func (m *mapper) lookupUid(xid string) uint64 { return uid } -func (m *mapper) createPredicatePosting(predicate string) *pb.Posting { - fp := farm.Fingerprint64([]byte(predicate)) - return &pb.Posting{ - Uid: fp, - Value: []byte(predicate), - ValType: pb.Posting_DEFAULT, - PostingType: pb.Posting_VALUE, - } -} - func (m *mapper) createPostings(nq gql.NQuad, de *pb.DirectedEdge) (*pb.Posting, *pb.Posting) { diff --git a/dgraph/cmd/bulk/run.go b/dgraph/cmd/bulk/run.go index 917dbea1c2d..929feb63071 100644 --- a/dgraph/cmd/bulk/run.go +++ b/dgraph/cmd/bulk/run.go @@ -66,9 +66,6 @@ func init() { "Number of worker threads to use (defaults to the number of logical CPUs).") flag.Int64("mapoutput_mb", 64, "The estimated size of each map file output. Increasing this increases memory usage.") - flag.Bool("expand_edges", true, - "Generate edges that allow nodes to be expanded using _predicate_ or expand(...). "+ - "Disable to increase loading speed.") flag.Bool("skip_map_phase", false, "Skip the map phase (assumes that map output files already exist).") flag.Bool("cleanup_tmp", true, @@ -108,7 +105,6 @@ func run() { TmpDir: Bulk.Conf.GetString("tmp"), NumGoroutines: Bulk.Conf.GetInt("num_go_routines"), MapBufSize: int64(Bulk.Conf.GetInt("mapoutput_mb")), - ExpandEdges: Bulk.Conf.GetBool("expand_edges"), SkipMapPhase: Bulk.Conf.GetBool("skip_map_phase"), CleanupTmp: Bulk.Conf.GetBool("cleanup_tmp"), NumShufflers: Bulk.Conf.GetInt("shufflers"), diff --git a/dgraph/cmd/bulk/schema.go b/dgraph/cmd/bulk/schema.go index dcf28c1e9b0..31a71d0a3ad 100644 --- a/dgraph/cmd/bulk/schema.go +++ b/dgraph/cmd/bulk/schema.go @@ -37,12 +37,7 @@ type schemaStore struct { func newSchemaStore(initial []*pb.SchemaUpdate, opt options, state *state) *schemaStore { s := &schemaStore{ - m: map[string]*pb.SchemaUpdate{ - "_predicate_": { - ValueType: pb.Posting_STRING, - List: true, - }, - }, + m: map[string]*pb.SchemaUpdate{}, state: state, } if opt.StoreXids { diff --git a/dgraph/cmd/debug/run.go b/dgraph/cmd/debug/run.go index 232ae309b67..1a42bcbcee8 100644 --- a/dgraph/cmd/debug/run.go +++ b/dgraph/cmd/debug/run.go @@ -237,7 +237,7 @@ func showAllPostingsAt(db *badger.DB, readTs uint64) { } pk := x.Parse(item.Key()) - if !pk.IsData() || pk.Attr == "_predicate_" { + if !pk.IsData() { continue } diff --git a/dgraph/cmd/zero/oracle.go b/dgraph/cmd/zero/oracle.go index 08d38ec97c9..2c88794341c 100644 --- a/dgraph/cmd/zero/oracle.go +++ b/dgraph/cmd/zero/oracle.go @@ -18,7 +18,6 @@ package zero import ( "errors" - "fmt" "math/rand" "strconv" "strings" @@ -334,21 +333,7 @@ func (s *Server) commit(ctx context.Context, src *api.TxnContext) error { checkPreds := func() error { // Check if any of these tablets is being moved. If so, abort the transaction. preds := make(map[string]struct{}) - // _predicate_ would never be part of conflict detection, so keys corresponding to any - // modifications to this predicate would not be sent to Zero. But, we still need to abort - // transactions which are coming in, while this predicate is being moved. This means that if - // _predicate_ expansion is enabled, and a move for this predicate is happening, NO transactions - // across the entire cluster would commit. Sorry! But if we don't do this, we might lose commits - // which sneaked in during the move. - - // Ensure that we only consider checking _predicate_, if expand_edge flag is - // set to true, i.e. we are actually serving _predicate_. Otherwise, the - // code below, which checks if a tablet is present or is readonly, causes - // ALL txns to abort. See #2547. - if tablet := s.ServingTablet("_predicate_"); tablet != nil { - pkey := fmt.Sprintf("%d-_predicate_", tablet.GroupId) - preds[pkey] = struct{}{} - } + for _, k := range src.Preds { preds[k] = struct{}{} } diff --git a/edgraph/config.go b/edgraph/config.go index 2c4c9a3f8f5..4c4f4a0c69f 100644 --- a/edgraph/config.go +++ b/edgraph/config.go @@ -69,17 +69,6 @@ func setConfVar(conf Options) { return v } - // Expvar doesn't have bool type so we use an int. - newIntFromBool := func(b bool) *expvar.Int { - v := new(expvar.Int) - if b { - v.Set(1) - } else { - v.Set(0) - } - return v - } - // This is so we can find these options in /debug/vars. x.Conf.Set("badger.tables", newStr(conf.BadgerTables)) x.Conf.Set("badger.vlog", newStr(conf.BadgerVlog)) @@ -90,7 +79,6 @@ func setConfVar(conf Options) { // Set some vars from worker.Config. x.Conf.Set("tracing", newFloat(x.WorkerConfig.Tracing)) x.Conf.Set("num_pending_proposals", newInt(x.WorkerConfig.NumPendingProposals)) - x.Conf.Set("expand_edge", newIntFromBool(x.WorkerConfig.ExpandEdge)) } func SetConfiguration(newConfig Options) { diff --git a/posting/list.go b/posting/list.go index 44862af583c..d0f61d61a4f 100644 --- a/posting/list.go +++ b/posting/list.go @@ -310,10 +310,7 @@ func (l *List) addMutation(ctx context.Context, txn *Txn, t *pb.DirectedEdge) er // order. We can do so by proposing them in the same order as received by the Oracle delta // stream from Zero, instead of in goroutines. var conflictKey string - if t.Attr == "_predicate_" { - // Don't check for conflict. - - } else if schema.State().HasUpsert(t.Attr) { + if schema.State().HasUpsert(t.Attr) { // Consider checking to see if a email id is unique. A user adds: // "email@email.org", and there's a string equal tokenizer // and upsert directive on the schema. diff --git a/query/common_test.go b/query/common_test.go index 9fa5dc6c427..f5bfb22d3cc 100644 --- a/query/common_test.go +++ b/query/common_test.go @@ -215,6 +215,25 @@ type CarModel { previous_model: CarModel } +type SchoolInfo { + name: string + abbr: string + school: [uid] + district: [uid] + state: [uid] + county: [uid] +} + +type User { + name: string + password: password +} + +type Node { + node: uid + name: string +} + name : string @index(term, exact, trigram) @count @lang . alias : string @index(exact, term, fulltext) . dob : dateTime @index(year) . @@ -246,7 +265,7 @@ office.room : [uid] . best_friend : uid @reverse . pet : [uid] . node : [uid] . -model : string @index(term) . +model : string @index(term) @lang . make : string @index(term) . year : int . previous_model : uid @reverse . @@ -269,6 +288,7 @@ func populateCluster() { <5> "Garfield" . <6> "Bear" . <7> "Nemo" . + <11> "name" . <23> "Rick Grimes" . <24> "Glenn Rhee" . <25> "Daryl Dixon" . @@ -489,6 +509,7 @@ func populateCluster() { <23> "4" . <24> "14" . + <1> "User" . <2> "Person" . <3> "Person" . <4> "Person" . @@ -496,6 +517,12 @@ func populateCluster() { <5> "Pet" . <6> "Animal" . <6> "Pet" . + <32> "SchoolInfo" . + <33> "SchoolInfo" . + <34> "SchoolInfo" . + <35> "SchoolInfo" . + <36> "SchoolInfo" . + <11100> "Node" . <2> <5> . <3> <6> . @@ -520,6 +547,12 @@ func populateCluster() { <201> "2009" . <201> "CarModel" . <201> <200> . + + <202> "Toyota" . + <202> "2009" . + <202> "Prius" . + <202> "プリウス"@jp . + <202> "CarModel" . `) addGeoPointToCluster(1, "loc", []float64{1.1, 2.0}) diff --git a/query/mutation.go b/query/mutation.go index 55022e85722..4b29ab0eb94 100644 --- a/query/mutation.go +++ b/query/mutation.go @@ -34,20 +34,12 @@ import ( ) func ApplyMutations(ctx context.Context, m *pb.Mutations) (*api.TxnContext, error) { - if x.WorkerConfig.ExpandEdge { - edges, err := expandEdges(ctx, m) - if err != nil { - return nil, x.Wrapf(err, "While adding pb.edges") - } - m.Edges = edges - } else { - for _, mu := range m.Edges { - if mu.Attr == x.Star && !x.WorkerConfig.ExpandEdge { - return nil, x.Errorf("Expand edge (--expand_edge) is set to false." + - " Cannot perform S * * deletion.") - } - } + edges, err := expandEdges(ctx, m) + if err != nil { + return nil, x.Wrapf(err, "While adding pb.edges") } + m.Edges = edges + tctx, err := worker.MutateOverNetwork(ctx, m) if err != nil { if span := otrace.FromContext(ctx); span != nil { @@ -69,41 +61,22 @@ func expandEdges(ctx context.Context, m *pb.Mutations) ([]*pb.DirectedEdge, erro sg := &SubGraph{} sg.DestUIDs = &pb.List{Uids: []uint64{edge.GetEntity()}} sg.ReadTs = m.StartTs - valMatrix, err := getNodePredicates(ctx, sg) + + types, err := getNodeTypes(ctx, sg) if err != nil { return nil, err } - if len(valMatrix) != 1 { - return nil, x.Errorf("Expected only one list in value matrix while deleting: %v", - edge.GetEntity()) - } - for _, tv := range valMatrix[0].Values { - if len(tv.Val) > 0 { - preds = append(preds, string(tv.Val)) - } - } + preds = append(preds, getPredicatesFromTypes(types)...) + preds = append(preds, x.ReservedPredicates()...) } for _, pred := range preds { edgeCopy := *edge edgeCopy.Attr = pred edges = append(edges, &edgeCopy) - - // We only want to delete the pred from + <_predicate_> posting list if this is - // a SP* deletion operation. Otherwise we just continue. - if edge.Op == pb.DirectedEdge_DEL && string(edge.Value) != x.Star { - continue - } - - e := &pb.DirectedEdge{ - Op: edge.Op, - Entity: edge.GetEntity(), - Attr: "_predicate_", - Value: []byte(pred), - } - edges = append(edges, e) } } + return edges, nil } diff --git a/query/query.go b/query/query.go index 989de671692..ccc780493c4 100644 --- a/query/query.go +++ b/query/query.go @@ -1507,18 +1507,12 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su var v varValue var ok bool - if sg.Attr == "_predicate_" { - // This is a predicates list. - doneVars[sg.Params.Var] = varValue{ - strList: sg.valueMatrix, - path: sgPath, - Vals: make(map[uint64]types.Val), - } - } else if len(sg.counts) > 0 { + if len(sg.counts) > 0 { // This implies it is a value variable. doneVars[sg.Params.Var] = varValue{ Vals: make(map[uint64]types.Val), path: sgPath, + strList: sg.valueMatrix, } for idx, uid := range sg.SrcUIDs.Uids { val := types.Val{ @@ -1531,6 +1525,7 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su doneVars[sg.Params.Var] = varValue{ Vals: make(map[uint64]types.Val), path: sgPath, + strList: sg.valueMatrix, } val := types.Val{ @@ -1551,6 +1546,7 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su Uids: uids, path: sgPath, Vals: make(map[uint64]types.Val), + strList: sg.valueMatrix, } return nil } @@ -1565,6 +1561,7 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su if v, ok = doneVars[sg.Params.Var]; !ok { v.Vals = make(map[uint64]types.Val) v.path = sgPath + v.strList = sg.valueMatrix } for idx, uid := range sg.SrcUIDs.Uids { @@ -1592,6 +1589,7 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su doneVars[sg.Params.Var] = varValue{ path: sgPath, Vals: make(map[uint64]types.Val), + strList: sg.valueMatrix, } } return nil @@ -1868,21 +1866,11 @@ func expandSubgraph(ctx context.Context, sg *SubGraph) ([]*SubGraph, error) { for i := 0; i < len(sg.Children); i++ { child := sg.Children[i] - if !x.WorkerConfig.ExpandEdge && child.Attr == "_predicate_" { - return out, - x.Errorf("Cannot ask for _predicate_ when ExpandEdge(--expand_edge) is false.") - } - if child.Params.Expand == "" { out = append(out, child) continue } - if !x.WorkerConfig.ExpandEdge { - return out, - x.Errorf("Cannot run expand() query when ExpandEdge(--expand_edge) is false.") - } - var preds []string types, err := getNodeTypes(ctx, sg) if err != nil { @@ -1893,59 +1881,35 @@ func expandSubgraph(ctx context.Context, sg *SubGraph) ([]*SubGraph, error) { // It could be expand(_all_), expand(_forward_), expand(_reverse_) or expand(val(x)). case "_all_": span.Annotate(nil, "expand(_all_)") - if len(types) > 0 { - preds = getPredicatesFromTypes(types) - - rpreds, err := getReversePredicatesFromType(ctx, preds) - if err != nil { - return out, err - } - preds = append(preds, rpreds...) + if len(types) == 0 { break } - // Get the predicate list for expansion. - child.ExpandPreds, err = getNodePredicates(ctx, sg) - if err != nil { - return out, err - } - preds = uniqueValues(child.ExpandPreds) - - rpreds, err := getReversePredicates(ctx) + preds = getPredicatesFromTypes(types) + rpreds, err := getReversePredicates(ctx, preds) if err != nil { return out, err } preds = append(preds, rpreds...) case "_forward_": span.Annotate(nil, "expand(_forward_)") - if len(types) > 0 { - preds = getPredicatesFromTypes(types) + if len(types) == 0 { break } - child.ExpandPreds, err = getNodePredicates(ctx, sg) - if err != nil { - return out, err - } - preds = uniqueValues(child.ExpandPreds) + preds = getPredicatesFromTypes(types) case "_reverse_": span.Annotate(nil, "expand(_reverse_)") - if len(types) > 0 { - typePreds := getPredicatesFromTypes(types) - - rpreds, err := getReversePredicatesFromType(ctx, typePreds) - if err != nil { - return out, err - } - preds = append(preds, rpreds...) + if len(types) == 0 { break } - rpreds, err := getReversePredicates(ctx) + typePreds := getPredicatesFromTypes(types) + rpreds, err := getReversePredicates(ctx, typePreds) if err != nil { return out, err } - preds = rpreds + preds = append(preds, rpreds...) default: span.Annotate(nil, "expand default") // We already have the predicates populated from the var. @@ -2514,38 +2478,6 @@ func isUidFnWithoutVar(f *gql.Function) bool { return f != nil && f.Name == "uid" && len(f.NeedsVar) == 0 } -func getNodePredicates(ctx context.Context, sg *SubGraph) ([]*pb.ValueList, error) { - temp := &SubGraph{ - Attr: "_predicate_", - SrcUIDs: sg.DestUIDs, - ReadTs: sg.ReadTs, - } - taskQuery, err := createTaskQuery(temp) - if err != nil { - return nil, err - } - result, err := worker.ProcessTaskOverNetwork(ctx, taskQuery) - if err != nil { - return nil, err - } - return result.ValueMatrix, nil -} - -func getReversePredicates(ctx context.Context) ([]string, error) { - schs, err := worker.GetSchemaOverNetwork(ctx, &pb.SchemaRequest{}) - if err != nil { - return nil, err - } - preds := make([]string, 0, len(schs)) - for _, sch := range schs { - if !sch.Reverse { - continue - } - preds = append(preds, "~"+sch.Predicate) - } - return preds, nil -} - func getNodeTypes(ctx context.Context, sg *SubGraph) ([]string, error) { temp := &SubGraph{ Attr: "dgraph.type", @@ -2580,9 +2512,9 @@ func getPredicatesFromTypes(types []string) []string { return preds } -// getReversePredicatesFromType queries the schema and returns a list of the -// reverse predicates that exist within the given preds. -func getReversePredicatesFromType(ctx context.Context, preds []string) ([]string, error) { +// getReversePredicates queries the schema and returns a list of the reverse +// predicates that exist within the given preds. +func getReversePredicates(ctx context.Context, preds []string) ([]string, error) { var rpreds []string predMap := make(map[string]bool) for _, pred := range preds { diff --git a/query/query0_test.go b/query/query0_test.go index 4518b6ccf7e..05f7addeef5 100644 --- a/query/query0_test.go +++ b/query/query0_test.go @@ -152,9 +152,9 @@ func TestQueryCountEmptyNames(t *testing.T) { {in: `{q(func: has(name)) @filter(eq(name, "")) {count(uid)}}`, out: `{"data":{"q": [{"count":2}]}}`}, {in: `{q(func: has(name)) @filter(gt(name, "")) {count(uid)}}`, - out: `{"data":{"q": [{"count":45}]}}`}, + out: `{"data":{"q": [{"count":46}]}}`}, {in: `{q(func: has(name)) @filter(ge(name, "")) {count(uid)}}`, - out: `{"data":{"q": [{"count":47}]}}`}, + out: `{"data":{"q": [{"count":48}]}}`}, {in: `{q(func: has(name)) @filter(lt(name, "")) {count(uid)}}`, out: `{"data":{"q": [{"count":0}]}}`}, {in: `{q(func: has(name)) @filter(le(name, "")) {count(uid)}}`, @@ -165,7 +165,7 @@ func TestQueryCountEmptyNames(t *testing.T) { out: `{"data":{"q": [{"count":2}]}}`}, // NOTE: match with empty string filters values greater than the max distance. {in: `{q(func: has(name)) @filter(match(name, "", 8)) {count(uid)}}`, - out: `{"data":{"q": [{"count":27}]}}`}, + out: `{"data":{"q": [{"count":28}]}}`}, {in: `{q(func: has(name)) @filter(uid_in(name, "")) {count(uid)}}`, failure: `Value "" in uid_in is not a number`}, } diff --git a/query/query1_test.go b/query/query1_test.go index 01ac9eb83bf..843e4b6286b 100644 --- a/query/query1_test.go +++ b/query/query1_test.go @@ -1678,20 +1678,20 @@ func TestReturnEmptyBlock(t *testing.T) { } func TestExpandVal(t *testing.T) { - // We ignore password in expand(_all_) query := ` { - var(func: uid(1)) { - pred as _predicate_ + var(func: uid(11)) { + pred as name } - me(func: uid(1)) { + me(func: uid(11)) { expand(val(pred)) } } ` js := processQueryNoErr(t, query) - require.JSONEq(t, `{"data":{"me":[{"age":38,"full_name":"Michonne's large name for hashing","dob_day":"1910-01-01T00:00:00Z","power":13.250000,"noindex_name":"Michonne's name not indexed","survival_rate":98.990000,"name":"Michonne","sword_present":"true","alive":true,"dob":"1910-01-01T00:00:00Z","path":[{"path|weight":0.200000},{"path|weight":0.100000,"path|weight1":0.200000}],"bin_data":"YmluLWRhdGE=","loc":{"type":"Point","coordinates":[1.1,2]},"address":"31, 32 street, Jupiter","graduation":["1932-01-01T00:00:00Z"],"gender":"female","_xid_":"mich"}]}}`, js) + require.JSONEq(t, + `{"data":{"me":[{"name":"name"}]}}`, js) } func TestGroupByGeoCrash(t *testing.T) { @@ -1737,49 +1737,6 @@ func TestCountPanic(t *testing.T) { require.JSONEq(t, `{"data": {"q":[{"uid":"0x1","name":"Michonne","count(name)":1},{"uid":"0x12c","count(name)":0}]}}`, js) } -func TestExpandAll(t *testing.T) { - - query := ` - { - q(func: uid(1)) { - expand(_all_) { - name - } - } - } - ` - js := processQueryNoErr(t, query) - require.JSONEq(t, `{"data":{"q":[{"friend":[{"name":"Rick Grimes"},{"name":"Glenn Rhee"},{"name":"Daryl Dixon"},{"name":"Andrea"}],"power":13.250000,"_xid_":"mich","noindex_name":"Michonne's name not indexed","son":[{"name":"Andre"},{"name":"Helmut"}],"address":"31, 32 street, Jupiter","dob_day":"1910-01-01T00:00:00Z","follow":[{"name":"Glenn Rhee"},{"name":"Andrea"}],"name":"Michonne","path":[{"name":"Glenn Rhee","path|weight":0.200000},{"name":"Andrea","path|weight":0.100000,"path|weight1":0.200000}],"school":[{"name":"School A"}],"full_name":"Michonne's large name for hashing","alive":true,"bin_data":"YmluLWRhdGE=","gender":"female","loc":{"type":"Point","coordinates":[1.1,2]},"graduation":["1932-01-01T00:00:00Z"],"age":38,"sword_present":"true","dob":"1910-01-01T00:00:00Z","survival_rate":98.990000,"~friend":[{"name":"Rick Grimes"}]}]}}`, js) -} - -func TestExpandForward(t *testing.T) { - query := ` - { - q(func: uid(1)) { - expand(_forward_) { - name - } - } - } - ` - js := processQueryNoErr(t, query) - require.JSONEq(t, `{"data":{"q":[{"friend":[{"name":"Rick Grimes"},{"name":"Glenn Rhee"},{"name":"Daryl Dixon"},{"name":"Andrea"}],"power":13.250000,"_xid_":"mich","noindex_name":"Michonne's name not indexed","son":[{"name":"Andre"},{"name":"Helmut"}],"address":"31, 32 street, Jupiter","dob_day":"1910-01-01T00:00:00Z","follow":[{"name":"Glenn Rhee"},{"name":"Andrea"}],"name":"Michonne","path":[{"name":"Glenn Rhee","path|weight":0.200000},{"name":"Andrea","path|weight":0.100000,"path|weight1":0.200000}],"school":[{"name":"School A"}],"full_name":"Michonne's large name for hashing","alive":true,"bin_data":"YmluLWRhdGE=","gender":"female","loc":{"type":"Point","coordinates":[1.1,2]},"graduation":["1932-01-01T00:00:00Z"],"age":38,"sword_present":"true","dob":"1910-01-01T00:00:00Z","survival_rate":98.990000}]}}`, js) -} - -func TestExpandReverse(t *testing.T) { - query := ` - { - q(func: uid(1)) { - expand(_reverse_) { - name - } - } - } - ` - js := processQueryNoErr(t, query) - require.JSONEq(t, `{"data":{"q":[{"~friend":[{"name":"Rick Grimes"}]}]}}`, js) -} - func TestUidWithoutDebug(t *testing.T) { query := ` diff --git a/query/query3_test.go b/query/query3_test.go index 2b3a225e0ed..2b0314cbcdc 100644 --- a/query/query3_test.go +++ b/query/query3_test.go @@ -1190,7 +1190,6 @@ func TestQueryPassword(t *testing.T) { } func TestPasswordExpandAll1(t *testing.T) { - // We ignore password in expand(_all_) query := ` { me(func: uid(0x01)) { @@ -1199,7 +1198,7 @@ func TestPasswordExpandAll1(t *testing.T) { } ` js := processQueryNoErr(t, query) - require.JSONEq(t, `{"data":{"me":[{"path":[{"path|weight":0.200000},{"path|weight":0.100000,"path|weight1":0.200000}],"age":38,"full_name":"Michonne's large name for hashing","dob_day":"1910-01-01T00:00:00Z","_xid_":"mich","loc":{"type":"Point","coordinates":[1.1,2]},"address":"31, 32 street, Jupiter","graduation":["1932-01-01T00:00:00Z"],"dob":"1910-01-01T00:00:00Z","bin_data":"YmluLWRhdGE=","power":13.250000,"survival_rate":98.990000,"name":"Michonne","sword_present":"true","alive":true,"gender":"female","noindex_name":"Michonne's name not indexed"}]}}`, js) + require.JSONEq(t, `{"data":{"me":[{"name":"Michonne"}]}}`, js) } func TestPasswordExpandAll2(t *testing.T) { @@ -1212,7 +1211,7 @@ func TestPasswordExpandAll2(t *testing.T) { } ` js := processQueryNoErr(t, query) - require.JSONEq(t, `{"data":{"me":[{"_xid_":"mich","address":"31, 32 street, Jupiter","path":[{"path|weight":0.200000},{"path|weight":0.100000,"path|weight1":0.200000}],"sword_present":"true","dob_day":"1910-01-01T00:00:00Z","gender":"female","dob":"1910-01-01T00:00:00Z","survival_rate":98.990000,"noindex_name":"Michonne's name not indexed","name":"Michonne","graduation":["1932-01-01T00:00:00Z"],"bin_data":"YmluLWRhdGE=","loc":{"type":"Point","coordinates":[1.1,2]},"age":38,"full_name":"Michonne's large name for hashing","alive":true,"power":13.250000,"checkpwd(password)":false}]}}`, js) + require.JSONEq(t, `{"data":{"me":[{"name":"Michonne", "checkpwd(password)":false}]}}`, js) } func TestPasswordExpandError(t *testing.T) { diff --git a/query/query4_test.go b/query/query4_test.go index 4eeaa070649..f057e0a2e7f 100644 --- a/query/query4_test.go +++ b/query/query4_test.go @@ -287,7 +287,7 @@ func TestNoResultsCount(t *testing.T) { func TestTypeExpandAll(t *testing.T) { query := `{ - q(func: has(make)) { + q(func: eq(make, "Ford")) { expand(_all_) { uid } @@ -302,7 +302,7 @@ func TestTypeExpandAll(t *testing.T) { func TestTypeExpandForward(t *testing.T) { query := `{ - q(func: has(make)) { + q(func: eq(make, "Ford")) { expand(_forward_) { uid } @@ -317,7 +317,7 @@ func TestTypeExpandForward(t *testing.T) { func TestTypeExpandReverse(t *testing.T) { query := `{ - q(func: has(make)) { + q(func: eq(make, "Ford")) { expand(_reverse_) { uid } @@ -328,3 +328,16 @@ func TestTypeExpandReverse(t *testing.T) { {"~previous_model": [{"uid":"0xc9"}]} ]}}`, js) } + +func TestTypeExpandLang(t *testing.T) { + query := `{ + q(func: eq(make, "Toyota")) { + expand(_all_) { + uid + } + } + }` + js := processQueryNoErr(t, query) + require.JSONEq(t, `{"data": {"q":[ + {"make":"Toyota","model":"Prius", "model@jp":"プリウス", "year":2009}]}}`, js) +} diff --git a/query/query_facets_test.go b/query/query_facets_test.go index fbf4b6e41cd..06df0422c76 100644 --- a/query/query_facets_test.go +++ b/query/query_facets_test.go @@ -46,6 +46,8 @@ func populateClusterWithFacets() { <1> "female" . <23> "male" . + + <202> "Prius" (type = "Electric") . ` friendFacets1 := "(since = 2006-01-02T15:04:05)" @@ -902,3 +904,17 @@ func TestFacetsAlias2(t *testing.T) { js := processQueryNoErr(t, query) require.JSONEq(t, `{"data":{"me2":[{"friend":[{"friend|close":true,"f":false,"friend|since":"2005-05-02T15:04:05Z"},{"friend|since":"2006-01-02T15:04:05Z"},{"friend|since":"2006-01-02T15:04:05Z"},{"friend|close":true,"f":true,"friend|since":"2004-05-02T15:04:05Z","friend|tag":"Domain3"},{"friend|close":false,"f":true,"friend|since":"2007-05-02T15:04:05Z","friend|tag":34}]}],"me":[{"name":"Rick Grimes", "val(a)":"2006-01-02T15:04:05Z"}]}}`, js) } + +func TestTypeExpandFacets(t *testing.T) { + query := `{ + q(func: eq(make, "Toyota")) { + expand(_all_) { + uid + } + } + }` + js := processQueryNoErr(t, query) + require.JSONEq(t, `{"data": {"q":[ + {"make":"Toyota","model":"Prius", "model@jp":"プリウス", "model|type":"Electric", + "year":2009}]}}`, js) +} diff --git a/schema/parse.go b/schema/parse.go index 60ace0ed7b6..7ef3bd12e6a 100644 --- a/schema/parse.go +++ b/schema/parse.go @@ -42,10 +42,6 @@ func ParseBytes(s []byte, gid uint32) (rerr error) { for _, update := range result.Schemas { State().Set(update.Predicate, *update) } - State().Set("_predicate_", pb.SchemaUpdate{ - ValueType: pb.Posting_STRING, - List: true, - }) return nil } diff --git a/schema/parse_test.go b/schema/parse_test.go index 0176d153160..f8983c1008a 100644 --- a/schema/parse_test.go +++ b/schema/parse_test.go @@ -58,10 +58,6 @@ func TestSchema(t *testing.T) { Predicate: "name", ValueType: pb.Posting_STRING, }}, - {"_predicate_", &pb.SchemaUpdate{ - ValueType: pb.Posting_STRING, - List: true, - }}, {"address", &pb.SchemaUpdate{ Predicate: "address", ValueType: pb.Posting_STRING, @@ -173,10 +169,6 @@ friend : [uid] @reverse @count . func TestSchemaIndexCustom(t *testing.T) { require.NoError(t, ParseBytes([]byte(schemaIndexVal5), 1)) checkSchema(t, State().predicate, []nameType{ - {"_predicate_", &pb.SchemaUpdate{ - ValueType: pb.Posting_STRING, - List: true, - }}, {"name", &pb.SchemaUpdate{ Predicate: "name", ValueType: pb.Posting_STRING, diff --git a/schema/schema.go b/schema/schema.go index 27213ce4c2e..b4765d1f86a 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -411,15 +411,6 @@ func LoadTypesFromDb() error { func InitialSchema() []*pb.SchemaUpdate { var initialSchema []*pb.SchemaUpdate - // propose the schema for _predicate_ - if x.WorkerConfig.ExpandEdge { - initialSchema = append(initialSchema, &pb.SchemaUpdate{ - Predicate: x.PredicateListAttr, - ValueType: pb.Posting_STRING, - List: true, - }) - } - initialSchema = append(initialSchema, &pb.SchemaUpdate{ Predicate: "dgraph.type", ValueType: pb.Posting_STRING, diff --git a/systest/mutations_test.go b/systest/mutations_test.go index 016d86be9f6..7d3a6602386 100644 --- a/systest/mutations_test.go +++ b/systest/mutations_test.go @@ -47,10 +47,8 @@ func TestSystem(t *testing.T) { } t.Run("n-quad mutation", wrap(NQuadMutationTest)) - t.Run("expand all lang test", wrap(ExpandAllLangTest)) t.Run("list with languages", wrap(ListWithLanguagesTest)) t.Run("delete all reverse index", wrap(DeleteAllReverseIndex)) - t.Run("expand all with reverse predicates", wrap(ExpandAllReversePredicatesTest)) t.Run("normalise edge cases", wrap(NormalizeEdgeCasesTest)) t.Run("facets with order", wrap(FacetOrderTest)) t.Run("lang and sort bug", wrap(LangAndSortBugTest)) @@ -66,7 +64,6 @@ func TestSystem(t *testing.T) { t.Run("delete with expand all", wrap(DeleteWithExpandAll)) t.Run("facets using nquads", wrap(FacetsUsingNQuadsError)) t.Run("skip empty pl for has", wrap(SkipEmptyPLForHas)) - t.Run("facet expand all", wrap(FacetExpandAll)) t.Run("has with dash", wrap(HasWithDash)) t.Run("list geo filter", wrap(ListGeoFilterTest)) t.Run("list regex filter", wrap(ListRegexFilterTest)) @@ -142,70 +139,6 @@ func FacetJsonInputSupportsAnyOfTerms(t *testing.T, c *dgo.Dgraph) { `, assigned.Uids["a"], assigned.Uids["b"]), string(resp.GetJson())) } -func ExpandAllLangTest(t *testing.T, c *dgo.Dgraph) { - ctx := context.Background() - - check(t, (c.Alter(ctx, &api.Operation{ - Schema: ` - list: [string] . - name: string @lang . - `, - }))) - - txn := c.NewTxn() - defer txn.Discard(ctx) - _, err := txn.Mutate(ctx, &api.Mutation{ - CommitNow: true, - SetNquads: []byte(` - <0x1> "abc" . - <0x1> "abc_en"@en . - <0x1> "abc_nl"@nl . - <0x2> "abc_hi"@hi . - <0x2> "abc_ci"@ci . - <0x2> "abc_ja"@ja . - <0x3> "abcd" . - <0x1> "99"^^ . - <0x1> "first" . - <0x1> "second" . - `), - }) - check(t, err) - - resp, err := c.NewTxn().Query(context.Background(), ` - { - q(func: uid(0x1,0x2,0x3)) { - expand(_all_) - } - } - `) - check(t, err) - - CompareJSON(t, ` - { - "q": [ - { - "name": "abcd" - }, - { - "name@ci": "abc_ci", - "name@hi": "abc_hi", - "name@ja": "abc_ja" - }, - { - "name@en": "abc_en", - "name@nl": "abc_nl", - "name": "abc", - "number": 99, - "list": [ - "second", - "first" - ] - } - ] - } - `, string(resp.GetJson())) -} - func ListWithLanguagesTest(t *testing.T, c *dgo.Dgraph) { ctx := context.Background() @@ -340,109 +273,6 @@ func DeleteAllReverseIndex(t *testing.T, c *dgo.Dgraph) { CompareJSON(t, fmt.Sprintf(`{"q":[{"~link": [{"uid": "%s"}]}]}`, aId), string(resp.Json)) } -func ExpandAllReversePredicatesTest(t *testing.T, c *dgo.Dgraph) { - ctx := context.Background() - - require.NoError(t, c.Alter(ctx, &api.Operation{ - Schema: `link: [uid] @reverse .`, - })) - - _, err := c.NewTxn().Mutate(ctx, &api.Mutation{ - CommitNow: true, - SetNquads: []byte(` - # SPO - <0x1> <0x2> . - - # SP* - <0x3> <0x4> . - <0x3> <0x5> . - - # S** - <0x6> <0x7> . - `), - }) - require.NoError(t, err) - - // Make sure expand(_all_) follows reverse edges. - resp, err := c.NewTxn().Query(ctx, ` - { - spo(func: uid(0x2)) { - uid - expand(_all_) { - uid - } - } - } - `) - require.NoError(t, err) - CompareJSON(t, ` - { - "spo": [ - { - "uid": "0x2", - "~link": [ - { - "uid": "0x1" - } - ] - } - ] - }`, string(resp.GetJson())) - - // Delete nodes with reverse edges, and make sure the entries in - // _predicate_ are removed. - _, err = c.NewTxn().Mutate(ctx, &api.Mutation{ - CommitNow: true, - DelNquads: []byte(` - <0x1> <0x2> . - <0x3> * . - <0x6> * * . - `), - }) - require.NoError(t, err) - - resp, err = c.NewTxn().Query(ctx, ` - { - spo(func: uid(0x2)) { - expand(_all_) { - uid - } - } - spstar(func: uid(0x4, 0x5)) { - uid - expand(_all_) { - uid - } - } - sstarstar(func: uid(0x7)) { - uid - expand(_all_) { - uid - } - } - } - `) - require.NoError(t, err) - CompareJSON(t, ` - { - "spo": [], - "spstar": [ - { - "uid": "0x4" - }, - { - "uid": "0x5" - } - ], - "sstarstar": [ - { - "uid": "0x7" - } - ] - } - `, string(resp.GetJson())) -} - func NormalizeEdgeCasesTest(t *testing.T, c *dgo.Dgraph) { ctx := context.Background() require.NoError(t, c.Alter(ctx, &api.Operation{Schema: "xid: string @index(exact) ."})) @@ -680,7 +510,6 @@ func SchemaAfterDeleteNode(t *testing.T, c *dgo.Dgraph) { resp, err := c.NewTxn().Query(ctx, `schema{}`) require.NoError(t, err) z.CompareJSON(t, asJson(`[`+ - `{"predicate":"_predicate_","type":"string","list":true},`+ x.AclPredicates+","+ `{"predicate":"friend","type":"uid","list":true},`+ `{"predicate":"married","type":"bool"},`+ @@ -704,7 +533,6 @@ func SchemaAfterDeleteNode(t *testing.T, c *dgo.Dgraph) { require.NoError(t, err) z.CompareJSON(t, asJson(`[`+ x.AclPredicates+","+ - `{"predicate":"_predicate_","type":"string","list":true},`+ `{"predicate":"friend","type":"uid","list":true},`+ `{"predicate":"name","type":"default"},`+ `{"predicate":"dgraph.type","type":"string","index":true, "tokenizer":["exact"], @@ -996,9 +824,24 @@ func EmptyRoomsWithTermIndex(t *testing.T, c *dgo.Dgraph) { } func DeleteWithExpandAll(t *testing.T, c *dgo.Dgraph) { + op := &api.Operation{} + op.Schema = ` + type Node { + to: uid + name: string + } +` + ctx := context.Background() + err := c.Alter(ctx, op) + require.NoError(t, err) + + ctx = context.Background() assigned, err := c.NewTxn().Mutate(ctx, &api.Mutation{ SetNquads: []byte(` + _:a "Node" . + _:b "Node" . + _:a _:b . _:b "b" . _:a _:c . @@ -1021,16 +864,14 @@ func DeleteWithExpandAll(t *testing.T, c *dgo.Dgraph) { q := `query test($id: string) { me(func: uid($id)) { - _predicate_ + expand(_all_) { + uid + } } }` - type Preds struct { - Predicates []string `json:"_predicate_"` - } - type Root struct { - Me []Preds `json:"me"` + Me []map[string]interface{} `json:"me"` } var r Root @@ -1038,10 +879,9 @@ func DeleteWithExpandAll(t *testing.T, c *dgo.Dgraph) { require.NoError(t, err) json.Unmarshal(resp.Json, &r) // S P O deletion shouldn't delete "to" . - require.Equal(t, 1, len(r.Me[0].Predicates)) - require.Equal(t, "to", r.Me[0].Predicates[0]) + require.Equal(t, 1, len(r.Me[0])) - // b should not have any _predicate_. + // b should not have any predicates. resp, err = c.NewTxn().QueryWithVars(ctx, q, map[string]string{"$id": buid}) require.NoError(t, err) json.Unmarshal(resp.Json, &r) @@ -1131,76 +971,6 @@ func SkipEmptyPLForHas(t *testing.T, c *dgo.Dgraph) { require.JSONEq(t, `{"users": []}`, string(resp.Json)) } -func FacetExpandAll(t *testing.T, c *dgo.Dgraph) { - ctx := context.Background() - - check(t, (c.Alter(ctx, &api.Operation{ - Schema: `name: string @index(hash) . - friend: [uid] @reverse .`, - }))) - - txn := c.NewTxn() - _, err := txn.Mutate(ctx, &api.Mutation{ - SetNquads: []byte(` - _:a "Alice" (from="US",to="Canada") . - _:a _:b (age=13,car="Honda") . - _:b "Bob" (from="Toronto",to="Vancouver"). - _:a _:c (age=15,car="Tesla") . - _:c "Charlie" . - `), - }) - require.NoError(t, err) - require.NoError(t, txn.Commit(ctx)) - - const friendQuery = ` - { - q(func: eq(name, "Alice")) { - expand(_all_) { - expand(_all_) - } - } - }` - - txn = c.NewTxn() - resp, err := txn.Query(ctx, friendQuery) - require.NoError(t, err) - CompareJSON(t, `{ - "q": [ - { - "friend": [ - { - "friend|age": 13, - "friend|car": "Honda", - "name": "Bob", - "name|from": "Toronto", - "name|to": "Vancouver", - "~friend": [ - { - "~friend|age": 13, - "~friend|car": "Honda" - } - ] - }, - { - "friend|age": 15, - "friend|car": "Tesla", - "name": "Charlie", - "~friend": [ - { - "~friend|age": 15, - "~friend|car": "Tesla" - } - ] - } - ], - "name": "Alice", - "name|from": "US", - "name|to": "Canada" - } - ] -}`, string(resp.Json)) -} - func HasWithDash(t *testing.T, c *dgo.Dgraph) { ctx := context.Background() diff --git a/systest/queries_test.go b/systest/queries_test.go index 28f839c2b39..0d0991d2ab1 100644 --- a/systest/queries_test.go +++ b/systest/queries_test.go @@ -331,11 +331,6 @@ func SchemaQueryTest(t *testing.T, c *dgo.Dgraph) { js := ` { "schema": [` + x.AclPredicates + `, - { - "predicate": "_predicate_", - "type": "string", - "list": true - }, { "predicate": "dgraph.type", "type": "string", @@ -399,9 +394,6 @@ func SchemaQueryTestPredicate1(t *testing.T, c *dgo.Dgraph) { { "predicate": "dgraph.user.group" }, - { - "predicate": "_predicate_" - }, { "predicate": "friends" }, @@ -527,11 +519,6 @@ func SchemaQueryTestHTTP(t *testing.T, c *dgo.Dgraph) { js := ` { "schema": [` + x.AclPredicates + `, - { - "predicate": "_predicate_", - "type": "string", - "list": true - }, { "index": true, "predicate": "dgraph.type", diff --git a/worker/export.go b/worker/export.go index 89ef063f9b8..23143d7ce3c 100644 --- a/worker/export.go +++ b/worker/export.go @@ -287,6 +287,9 @@ func export(ctx context.Context, in *pb.ExportRequest) error { stream.LogPrefix = "Export" stream.ChooseKey = func(item *badger.Item) bool { pk := x.Parse(item.Key()) + // _predicate_ is deprecated but leaving this here so that users with a + // binary with version >= 1.1 can export data from a version < 1.1 without + // this internal data showing up. if pk.Attr == "_predicate_" { return false } diff --git a/worker/groups.go b/worker/groups.go index e421e0d41d3..d8cd1b51059 100644 --- a/worker/groups.go +++ b/worker/groups.go @@ -182,7 +182,7 @@ func (g *groupi) proposeInitialSchema() { func (g *groupi) upsertSchema(schema *pb.SchemaUpdate) { // Propose schema mutation. var m pb.Mutations - // schema for _predicate_ is not changed once set. + // schema for a reserved predicate is not changed once set. m.StartTs = 1 m.Schema = append(m.Schema, schema) diff --git a/x/config.go b/x/config.go index 80ad3e48260..1b1db2414cc 100644 --- a/x/config.go +++ b/x/config.go @@ -38,7 +38,6 @@ type WorkerOptions struct { MyAddr string ZeroAddr string RaftId uint64 - ExpandEdge bool WhiteListedIPRanges []IPRange MaxRetries int StrictMutations bool diff --git a/x/keys.go b/x/keys.go index 5f07f32253b..4043de8a62f 100644 --- a/x/keys.go +++ b/x/keys.go @@ -326,23 +326,38 @@ func Parse(key []byte) *ParsedKey { return p } -// IsReservedPredicate returns true if 'pred' is in the reserved predicate list. +var reservedPredicateMap = map[string]struct{}{ + "dgraph.type": {}, +} + +var aclPredicateMap = map[string]struct{}{ + "dgraph.xid": {}, + "dgraph.password": {}, + "dgraph.user.group": {}, + "dgraph.group.acl": {}, +} + +// IsReservedPredicate returns true if the predicate is in the reserved predicate list. func IsReservedPredicate(pred string) bool { - var m = map[string]struct{}{ - PredicateListAttr: {}, - "dgraph.type": {}, - } - _, ok := m[strings.ToLower(pred)] + _, ok := reservedPredicateMap[strings.ToLower(pred)] return ok || IsAclPredicate(pred) } +// IsAclPredicate returns true if the predicate is in the list of reserved +// predicates for the ACL feature. func IsAclPredicate(pred string) bool { - var m = map[string]struct{}{ - "dgraph.xid": {}, - "dgraph.password": {}, - "dgraph.user.group": {}, - "dgraph.group.acl": {}, - } - _, ok := m[strings.ToLower(pred)] + _, ok := aclPredicateMap[strings.ToLower(pred)] return ok } + +// ReservedPredicates returns the complete list of reserved predicates. +func ReservedPredicates() []string { + var preds []string + for pred, _ := range reservedPredicateMap { + preds = append(preds, pred) + } + for pred, _ := range aclPredicateMap { + preds = append(preds, pred) + } + return preds +} diff --git a/x/x.go b/x/x.go index ddab9881adb..3b2cfcab669 100644 --- a/x/x.go +++ b/x/x.go @@ -60,9 +60,6 @@ const ( // message sizes allowable on the client size when dialing. GrpcMaxSize = 4 << 30 - // The attr used to store list of predicates for a node. - PredicateListAttr = "_predicate_" - PortZeroGrpc = 5080 PortZeroHTTP = 6080 PortInternal = 7080 diff --git a/z/client.go b/z/client.go index 71befdb096b..264aacc36f9 100644 --- a/z/client.go +++ b/z/client.go @@ -141,33 +141,6 @@ func DgraphClientWithCerts(serviceAddr string, conf *viper.Viper) (*dgo.Dgraph, func DropAll(t *testing.T, dg *dgo.Dgraph) { err := dg.Alter(context.Background(), &api.Operation{DropAll: true}) require.NoError(t, err) - - nodes := DbNodeCount(t, dg) - // the only node left should be the groot node - require.Equal(t, 1, nodes) -} - -func DbNodeCount(t *testing.T, dg *dgo.Dgraph) int { - resp, err := dg.NewTxn().Query(context.Background(), ` - { - q(func: has(_predicate_)) { - count(uid) - } - } - `) - require.NoError(t, err) - - type count struct { - Count int - } - type root struct { - Q []count - } - var response root - err = json.Unmarshal(resp.GetJson(), &response) - require.NoError(t, err) - - return response.Q[0].Count } type LoginParams struct { From 208c9b9dc68ba353a36cf79ea38957073a245d69 Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Mon, 8 Apr 2019 10:27:52 -0700 Subject: [PATCH 2/6] go fmt --- dgraph/cmd/bulk/schema.go | 2 +- dgraph/cmd/zero/oracle.go | 2 +- ee/acl/acl_curl_test.go | 2 +- query/query.go | 18 +++++++++--------- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dgraph/cmd/bulk/schema.go b/dgraph/cmd/bulk/schema.go index 31a71d0a3ad..eca26c84ef1 100644 --- a/dgraph/cmd/bulk/schema.go +++ b/dgraph/cmd/bulk/schema.go @@ -37,7 +37,7 @@ type schemaStore struct { func newSchemaStore(initial []*pb.SchemaUpdate, opt options, state *state) *schemaStore { s := &schemaStore{ - m: map[string]*pb.SchemaUpdate{}, + m: map[string]*pb.SchemaUpdate{}, state: state, } if opt.StoreXids { diff --git a/dgraph/cmd/zero/oracle.go b/dgraph/cmd/zero/oracle.go index 2c88794341c..26da8b44501 100644 --- a/dgraph/cmd/zero/oracle.go +++ b/dgraph/cmd/zero/oracle.go @@ -333,7 +333,7 @@ func (s *Server) commit(ctx context.Context, src *api.TxnContext) error { checkPreds := func() error { // Check if any of these tablets is being moved. If so, abort the transaction. preds := make(map[string]struct{}) - + for _, k := range src.Preds { preds[k] = struct{}{} } diff --git a/ee/acl/acl_curl_test.go b/ee/acl/acl_curl_test.go index a6e89616e02..a5bf1ba478a 100644 --- a/ee/acl/acl_curl_test.go +++ b/ee/acl/acl_curl_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/require" ) -var loginEndpoint = "http://"+z.SockAddrHttp +"/login" +var loginEndpoint = "http://" + z.SockAddrHttp + "/login" func TestCurlAuthorization(t *testing.T) { if testing.Short() { diff --git a/query/query.go b/query/query.go index ccc780493c4..7a332fdcbb3 100644 --- a/query/query.go +++ b/query/query.go @@ -1510,8 +1510,8 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su if len(sg.counts) > 0 { // This implies it is a value variable. doneVars[sg.Params.Var] = varValue{ - Vals: make(map[uint64]types.Val), - path: sgPath, + Vals: make(map[uint64]types.Val), + path: sgPath, strList: sg.valueMatrix, } for idx, uid := range sg.SrcUIDs.Uids { @@ -1523,8 +1523,8 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su } } else if sg.Params.uidCount { doneVars[sg.Params.Var] = varValue{ - Vals: make(map[uint64]types.Val), - path: sgPath, + Vals: make(map[uint64]types.Val), + path: sgPath, strList: sg.valueMatrix, } @@ -1543,9 +1543,9 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su // This implies it is a entity variable. if v, ok = doneVars[sg.Params.Var]; !ok { doneVars[sg.Params.Var] = varValue{ - Uids: uids, - path: sgPath, - Vals: make(map[uint64]types.Val), + Uids: uids, + path: sgPath, + Vals: make(map[uint64]types.Val), strList: sg.valueMatrix, } return nil @@ -1587,8 +1587,8 @@ func (sg *SubGraph) populateUidValVar(doneVars map[string]varValue, sgPath []*Su } // Insert a empty entry to keep the dependency happy. doneVars[sg.Params.Var] = varValue{ - path: sgPath, - Vals: make(map[uint64]types.Val), + path: sgPath, + Vals: make(map[uint64]types.Val), strList: sg.valueMatrix, } } From fddfcf35f9dd9d0c3f3453a93b572e9ac61fc7b8 Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Wed, 24 Apr 2019 16:16:38 -0700 Subject: [PATCH 3/6] Ensure list of unique predicates during expand queries. --- query/query.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/query/query.go b/query/query.go index d05f7ac7c1a..9ebd8f9f1be 100644 --- a/query/query.go +++ b/query/query.go @@ -1821,16 +1821,23 @@ func (sg *SubGraph) appendDummyValues() { } } -func uniqueValues(vl []*pb.ValueList) []string { - predMap := make(map[string]struct{}) - +func getPredsFromVals(vl []*pb.ValueList) []string { + preds := make([]string, 0) for _, l := range vl { for _, v := range l.Values { if len(v.Val) > 0 { - predMap[string(v.Val)] = struct{}{} + preds = append(preds, string(v.Val)) } } } + return preds +} + +func uniquePreds(list []string) []string { + predMap := make(map[string]struct{}) + for _, item := range list { + predMap[item] = struct{}{} + } preds := make([]string, 0, len(predMap)) for pred := range predMap { @@ -1913,8 +1920,9 @@ func expandSubgraph(ctx context.Context, sg *SubGraph) ([]*SubGraph, error) { default: span.Annotate(nil, "expand default") // We already have the predicates populated from the var. - preds = uniqueValues(child.ExpandPreds) + preds = getPredsFromVals(child.ExpandPreds) } + preds = uniquePreds(preds) for _, pred := range preds { temp := &SubGraph{ @@ -2492,7 +2500,7 @@ func getNodeTypes(ctx context.Context, sg *SubGraph) ([]string, error) { if err != nil { return nil, err } - return uniqueValues(result.ValueMatrix), nil + return getPredsFromVals(result.ValueMatrix), nil } // getPredicatesFromTypes returns the list of preds contained in the given types. From 32a355757483f359f489439100c5e980339c0ea6 Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Mon, 6 May 2019 16:33:09 -0700 Subject: [PATCH 4/6] Update test to not include _predicate_. --- query/query0_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/query/query0_test.go b/query/query0_test.go index cb36e6f2642..680d4d2b9f1 100644 --- a/query/query0_test.go +++ b/query/query0_test.go @@ -1777,7 +1777,7 @@ func TestDefaultValueVar1(t *testing.T) { { var(func: has(pred)) { n as uid - cnt as count(_predicate_) + cnt as count(nonexistent_pred) } data(func: uid(n)) @filter(gt(val(cnt), 4)) { @@ -1792,7 +1792,7 @@ func TestDefaultValueVar2(t *testing.T) { query := ` { var(func: uid(0x1)) { - cnt as _predicate_ + cnt as nonexistent_pred } data(func: uid(0x1)) { From 3fa1d48f10c9df7e9b9a6a0fc061fe41f888df5e Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Tue, 7 May 2019 14:49:37 -0700 Subject: [PATCH 5/6] Fix test-bulk-schema.sh test --- dgraph/cmd/bulk/systest/test-bulk-schema.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/dgraph/cmd/bulk/systest/test-bulk-schema.sh b/dgraph/cmd/bulk/systest/test-bulk-schema.sh index 6bc6e496846..9412bf31fb3 100755 --- a/dgraph/cmd/bulk/systest/test-bulk-schema.sh +++ b/dgraph/cmd/bulk/systest/test-bulk-schema.sh @@ -190,7 +190,6 @@ EOF dgraph debug -p out/0/p 2>|/dev/null | grep '{s}' | cut -d' ' -f4 > all_dbs.out dgraph debug -p out/1/p 2>|/dev/null | grep '{s}' | cut -d' ' -f4 >> all_dbs.out diff <(LC_ALL=C sort all_dbs.out | uniq -c) - < Date: Wed, 8 May 2019 18:02:44 -0700 Subject: [PATCH 6/6] Remove mentions of _predicate_ from gql tests. --- gql/parser_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gql/parser_test.go b/gql/parser_test.go index 2d6e424f5aa..6d674bce683 100644 --- a/gql/parser_test.go +++ b/gql/parser_test.go @@ -167,28 +167,28 @@ func TestParseQueryAliasListPred(t *testing.T) { query := ` { me(func: uid(0x0a)) { - pred: _predicate_ + pred: some_pred } } ` res, err := Parse(Request{Str: query}) require.NoError(t, err) require.Equal(t, "pred", res.Query[0].Children[0].Alias) - require.Equal(t, "_predicate_", res.Query[0].Children[0].Attr) + require.Equal(t, "some_pred", res.Query[0].Children[0].Attr) } func TestParseQueryCountListPred(t *testing.T) { query := ` { me(func: uid(0x0a)) { - count(_predicate_) + count(some_pred) } } ` res, err := Parse(Request{Str: query}) require.NoError(t, err) require.Equal(t, true, res.Query[0].Children[0].IsCount) - require.Equal(t, "_predicate_", res.Query[0].Children[0].Attr) + require.Equal(t, "some_pred", res.Query[0].Children[0].Attr) } func TestParseQueryListPred2(t *testing.T) { @@ -199,7 +199,7 @@ func TestParseQueryListPred2(t *testing.T) { } var(func: uid(f)) { - l as _predicate_ + l as some_pred } var(func: uid( 0x0a)) { @@ -221,9 +221,9 @@ func TestParseQueryListPred_MultiVarError(t *testing.T) { } var(func: uid(f)) { - l as _predicate_ + l as some_pred friend { - g as _predicate_ + g as some_pred } }