From e6294344e997e88f45cc3cc157b9c76c5fa10927 Mon Sep 17 00:00:00 2001 From: srfrog Date: Mon, 7 Jan 2019 21:31:18 -0700 Subject: [PATCH 1/7] send error when removing last node in a group would orphan tablets. --- dgraph/cmd/zero/zero.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dgraph/cmd/zero/zero.go b/dgraph/cmd/zero/zero.go index 64bd84924bd..3245c9487b6 100644 --- a/dgraph/cmd/zero/zero.go +++ b/dgraph/cmd/zero/zero.go @@ -356,6 +356,10 @@ func (s *Server) removeNode(ctx context.Context, nodeId uint64, groupId uint32) if _, ok := s.state.Groups[groupId].Members[nodeId]; !ok { return x.Errorf("No node with nodeId %d found in group %d", nodeId, groupId) } + if len(s.state.Groups[groupId].Members) == 1 && len(s.state.Groups[groupId].Tablets) > 0 { + return x.Errorf("Move all tablets from group %d before removing the last node", groupId) + } + return s.Node.proposeAndWait(ctx, zp) } From 3020e8b63a0534d097e7ff9ea3a211139b57ca69 Mon Sep 17 00:00:00 2001 From: srfrog Date: Mon, 7 Jan 2019 22:41:16 -0700 Subject: [PATCH 2/7] reusing backup test data, moved to top systest dir. --- ee/backup/systest/backup_test.go | 2 +- .../data/goldendata_export.rdf.gz | Bin 2 files changed, 1 insertion(+), 1 deletion(-) rename {ee/backup/systest => systest}/data/goldendata_export.rdf.gz (100%) diff --git a/ee/backup/systest/backup_test.go b/ee/backup/systest/backup_test.go index 68a1e336244..6b452497768 100644 --- a/ee/backup/systest/backup_test.go +++ b/ee/backup/systest/backup_test.go @@ -59,7 +59,7 @@ func BackupSetup(t *testing.T, c *dgo.Dgraph) { require.NoError(t, err) require.NoError(t, c.Alter(ctx, &api.Operation{Schema: string(schema)})) - fp, err := os.Open(`data/goldendata_export.rdf.gz`) + fp, err := os.Open(`../../../systest/data/goldendata_export.rdf.gz`) x.Check(err) defer fp.Close() diff --git a/ee/backup/systest/data/goldendata_export.rdf.gz b/systest/data/goldendata_export.rdf.gz similarity index 100% rename from ee/backup/systest/data/goldendata_export.rdf.gz rename to systest/data/goldendata_export.rdf.gz From 81b29bd4ee4ba3cb4e37f4f266cc4b90dc0e43b8 Mon Sep 17 00:00:00 2001 From: srfrog Date: Mon, 7 Jan 2019 22:43:55 -0700 Subject: [PATCH 3/7] tests for tablet move and node removal. --- systest/nodes_test.go | 227 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 systest/nodes_test.go diff --git a/systest/nodes_test.go b/systest/nodes_test.go new file mode 100644 index 00000000000..8a3fb50674f --- /dev/null +++ b/systest/nodes_test.go @@ -0,0 +1,227 @@ +/* + * Copyright 2018 Dgraph Labs, Inc. and Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "testing" + "time" + + "github.com/dgraph-io/dgo" + "github.com/dgraph-io/dgo/protos/api" + "github.com/dgraph-io/dgraph/x" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestNodes(t *testing.T) { + wrap := func(fn func(*testing.T, *dgo.Dgraph)) func(*testing.T) { + return func(t *testing.T) { + conn, err := grpc.Dial("localhost:9180", grpc.WithInsecure()) + x.Check(err) + dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) + fn(t, dg) + } + } + + t.Run("setup test data", wrap(NodesSetup)) + t.Run("move tablets from 3", wrap(NodesMoveTablets3)) + t.Run("test query 1", wrap(NodesTestQuery)) + t.Run("move tablets from 2", wrap(NodesMoveTablets2)) + t.Run("test query 2", wrap(NodesTestQuery)) + t.Run("cleanup", wrap(NodesCleanup)) +} + +func NodesSetup(t *testing.T, c *dgo.Dgraph) { + ctx := context.Background() + + require.NoError(t, c.Alter(ctx, &api.Operation{DropAll: true})) + + schema, err := ioutil.ReadFile(`data/goldendata.schema`) + require.NoError(t, err) + require.NoError(t, c.Alter(ctx, &api.Operation{Schema: string(schema)})) + + fp, err := os.Open(`data/goldendata_export.rdf.gz`) + x.Check(err) + defer fp.Close() + + gz, err := gzip.NewReader(fp) + x.Check(err) + defer gz.Close() + + var ( + cnt int + bb bytes.Buffer + ) + + reader := bufio.NewReader(gz) + for { + b, err := reader.ReadBytes('\n') + if err != nil { + if err == io.EOF { + break + } + x.Check(err) + } + bb.Write(b) + cnt++ + if cnt%100 == 0 { + _, err = c.NewTxn().Mutate(ctx, &api.Mutation{ + CommitNow: true, + SetNquads: bb.Bytes(), + }) + x.Check(err) + cnt = 0 + bb.Reset() + } + } +} + +func NodesCleanup(t *testing.T, c *dgo.Dgraph) { + require.NoError(t, c.Alter(context.Background(), &api.Operation{DropAll: true})) +} + +type response struct { + Groups map[string]struct { + Members map[string]interface{} `json:"members"` + Tablets map[string]struct { + GroupID int `json:"groupId"` + Predicate string `json:"predicate"` + } `json:"tablets"` + } `json:"groups"` +} + +func getState() (*response, error) { + resp, err := http.Get("http://localhost:6080/state") + if err != nil { + return nil, err + } + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var st response + if err := json.Unmarshal(b, &st); err != nil { + return nil, err + } + return &st, nil +} + +func NodesMoveTablets3(t *testing.T, c *dgo.Dgraph) { + state1, err := getState() + require.NoError(t, err) + + for pred := range state1.Groups["3"].Tablets { + url := fmt.Sprintf("http://localhost:6080/moveTablet?tablet=%s&group=2", pred) + resp, err := http.Get(url) + require.NoError(t, err) + resp.Body.Close() + time.Sleep(time.Second) + } + + state2, err := getState() + require.NoError(t, err) + + if len(state2.Groups["3"].Tablets) > 0 { + t.Errorf("moving tablets failed") + } + + resp, err := http.Get("http://localhost:6080/removeNode?group=3&id=3") + require.NoError(t, err) + resp.Body.Close() + + state2, err = getState() + require.NoError(t, err) + + if _, ok := state2.Groups["3"]; ok { + t.Errorf("node removal failed") + } +} + +func NodesMoveTablets2(t *testing.T, c *dgo.Dgraph) { + state1, err := getState() + require.NoError(t, err) + + for pred := range state1.Groups["2"].Tablets { + url := fmt.Sprintf("http://localhost:6080/moveTablet?tablet=%s&group=1", pred) + resp, err := http.Get(url) + require.NoError(t, err) + resp.Body.Close() + time.Sleep(time.Second) + } + + state2, err := getState() + require.NoError(t, err) + + if len(state2.Groups["2"].Tablets) > 0 { + t.Errorf("moving tablets failed") + } + + resp, err := http.Get("http://localhost:6080/removeNode?group=2&id=2") + require.NoError(t, err) + resp.Body.Close() + + state2, err = getState() + require.NoError(t, err) + + if _, ok := state2.Groups["2"]; ok { + t.Errorf("node removal failed") + } +} + +func NodesTestQuery(t *testing.T, c *dgo.Dgraph) { + resp, err := c.NewTxn().Query(context.Background(), ` + { + q(func:anyofterms(name@en, "good bad"), first: 5) { + name@en + } + }`) + require.NoError(t, err) + + CompareJSON(t, ` + { + "q": [ + { + "name@en": "Such Good People" + }, + { + "name@en": "You will come good" + }, + { + "name@en": "A Good Match" + }, + { + "name@en": "A Good Wife" + }, + { + "name@en": "Good" + } + ] + }`, string(resp.GetJson())) +} From ebef3ba84774fe970567fd18d3e36acdfef07b7c Mon Sep 17 00:00:00 2001 From: srfrog Date: Tue, 8 Jan 2019 14:15:07 -0700 Subject: [PATCH 4/7] removed cnt reset. --- systest/nodes_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/systest/nodes_test.go b/systest/nodes_test.go index 8a3fb50674f..6da4c7651b4 100644 --- a/systest/nodes_test.go +++ b/systest/nodes_test.go @@ -94,7 +94,6 @@ func NodesSetup(t *testing.T, c *dgo.Dgraph) { SetNquads: bb.Bytes(), }) x.Check(err) - cnt = 0 bb.Reset() } } From 675dfb0b72da527b894dce0784369360b1db2a34 Mon Sep 17 00:00:00 2001 From: srfrog Date: Tue, 8 Jan 2019 21:38:58 -0700 Subject: [PATCH 5/7] restructured tests to fail in order, and observing all errors from http content. --- systest/nodes_test.go | 103 ++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/systest/nodes_test.go b/systest/nodes_test.go index 6da4c7651b4..6feb2041074 100644 --- a/systest/nodes_test.go +++ b/systest/nodes_test.go @@ -32,7 +32,6 @@ import ( "github.com/dgraph-io/dgo" "github.com/dgraph-io/dgo/protos/api" - "github.com/dgraph-io/dgraph/x" "github.com/stretchr/testify/require" "google.golang.org/grpc" ) @@ -41,17 +40,27 @@ func TestNodes(t *testing.T) { wrap := func(fn func(*testing.T, *dgo.Dgraph)) func(*testing.T) { return func(t *testing.T) { conn, err := grpc.Dial("localhost:9180", grpc.WithInsecure()) - x.Check(err) + require.NoError(t, err) dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) fn(t, dg) } } - t.Run("setup test data", wrap(NodesSetup)) - t.Run("move tablets from 3", wrap(NodesMoveTablets3)) - t.Run("test query 1", wrap(NodesTestQuery)) - t.Run("move tablets from 2", wrap(NodesMoveTablets2)) - t.Run("test query 2", wrap(NodesTestQuery)) + tests := []struct { + name string + fn func(*testing.T, *dgo.Dgraph) + }{ + {name: "setup test data", fn: NodesSetup}, + {name: "move tablets from 3", fn: NodesMoveTablets3}, + {name: "test query 1", fn: NodesTestQuery}, + {name: "move tablets from 2", fn: NodesMoveTablets2}, + {name: "test query 2", fn: NodesTestQuery}, + } + for _, tc := range tests { + if !t.Run(tc.name, wrap(tc.fn)) { + break + } + } t.Run("cleanup", wrap(NodesCleanup)) } @@ -65,11 +74,11 @@ func NodesSetup(t *testing.T, c *dgo.Dgraph) { require.NoError(t, c.Alter(ctx, &api.Operation{Schema: string(schema)})) fp, err := os.Open(`data/goldendata_export.rdf.gz`) - x.Check(err) + require.NoError(t, err) defer fp.Close() gz, err := gzip.NewReader(fp) - x.Check(err) + require.NoError(t, err) defer gz.Close() var ( @@ -84,7 +93,7 @@ func NodesSetup(t *testing.T, c *dgo.Dgraph) { if err == io.EOF { break } - x.Check(err) + require.NoError(t, err) } bb.Write(b) cnt++ @@ -93,7 +102,7 @@ func NodesSetup(t *testing.T, c *dgo.Dgraph) { CommitNow: true, SetNquads: bb.Bytes(), }) - x.Check(err) + require.NoError(t, err) bb.Reset() } } @@ -125,6 +134,10 @@ func getState() (*response, error) { return nil, err } + if bytes.Contains(b, []byte("Error")) { + return nil, fmt.Errorf("Failed to get state: %s", string(b)) + } + var st response if err := json.Unmarshal(b, &st); err != nil { return nil, err @@ -132,6 +145,18 @@ func getState() (*response, error) { return &st, nil } +func getError(rc io.ReadCloser) error { + defer rc.Close() + b, err := ioutil.ReadAll(rc) + if err != nil { + return fmt.Errorf("Read failed: %v", err) + } + if bytes.Contains(b, []byte("Error")) { + return fmt.Errorf("%s", string(b)) + } + return nil +} + func NodesMoveTablets3(t *testing.T, c *dgo.Dgraph) { state1, err := getState() require.NoError(t, err) @@ -140,7 +165,7 @@ func NodesMoveTablets3(t *testing.T, c *dgo.Dgraph) { url := fmt.Sprintf("http://localhost:6080/moveTablet?tablet=%s&group=2", pred) resp, err := http.Get(url) require.NoError(t, err) - resp.Body.Close() + require.NoError(t, getError(resp.Body)) time.Sleep(time.Second) } @@ -153,7 +178,7 @@ func NodesMoveTablets3(t *testing.T, c *dgo.Dgraph) { resp, err := http.Get("http://localhost:6080/removeNode?group=3&id=3") require.NoError(t, err) - resp.Body.Close() + require.NoError(t, getError(resp.Body)) state2, err = getState() require.NoError(t, err) @@ -171,7 +196,7 @@ func NodesMoveTablets2(t *testing.T, c *dgo.Dgraph) { url := fmt.Sprintf("http://localhost:6080/moveTablet?tablet=%s&group=1", pred) resp, err := http.Get(url) require.NoError(t, err) - resp.Body.Close() + require.NoError(t, getError(resp.Body)) time.Sleep(time.Second) } @@ -184,7 +209,7 @@ func NodesMoveTablets2(t *testing.T, c *dgo.Dgraph) { resp, err := http.Get("http://localhost:6080/removeNode?group=2&id=2") require.NoError(t, err) - resp.Body.Close() + require.NoError(t, getError(resp.Body)) state2, err = getState() require.NoError(t, err) @@ -196,31 +221,31 @@ func NodesMoveTablets2(t *testing.T, c *dgo.Dgraph) { func NodesTestQuery(t *testing.T, c *dgo.Dgraph) { resp, err := c.NewTxn().Query(context.Background(), ` - { - q(func:anyofterms(name@en, "good bad"), first: 5) { - name@en - } - }`) + { + q(func:anyofterms(name@en, "good bad"), first: -5) { + name@en + } + }`) require.NoError(t, err) CompareJSON(t, ` - { - "q": [ - { - "name@en": "Such Good People" - }, - { - "name@en": "You will come good" - }, - { - "name@en": "A Good Match" - }, - { - "name@en": "A Good Wife" - }, - { - "name@en": "Good" - } - ] - }`, string(resp.GetJson())) + { + "q": [ + { + "name@en": "Good Grief" + }, + { + "name@en": "Half Good Killer" + }, + { + "name@en": "Bad Friend" + }, + { + "name@en": "Ace of Spades: Bad Destiny" + }, + { + "name@en": "Bad Girls 6" + } + ] + }`, string(resp.GetJson())) } From c39955e10035d533c1c7c748702ab09829400acd Mon Sep 17 00:00:00 2001 From: srfrog Date: Tue, 15 Jan 2019 14:55:01 -0700 Subject: [PATCH 6/7] created new group-delete dir for custom cluster testing --- systest/group-delete/README.md | 14 ++++++++++++++ .../group_delete_test.go} | 0 2 files changed, 14 insertions(+) create mode 100644 systest/group-delete/README.md rename systest/{nodes_test.go => group-delete/group_delete_test.go} (100%) diff --git a/systest/group-delete/README.md b/systest/group-delete/README.md new file mode 100644 index 00000000000..d9e05c1f097 --- /dev/null +++ b/systest/group-delete/README.md @@ -0,0 +1,14 @@ +# Group Delete Test + +This test runs a scenario where nodes are removed from groups to make zero delete those +groups. At every stage we check the zero state to make sure the nodes and groups are deleted. + +Process: + +1. Bring up a cluster with 3 groups, 1 node each. +2. Delete node from group 3 +3. Check that zero deleted group 3 +4. Run a query to test that the cluster is viable +5. Delete node from group 2 +6. Check that zero deleted group 2 +7. Run a query to test that the cluster is viable diff --git a/systest/nodes_test.go b/systest/group-delete/group_delete_test.go similarity index 100% rename from systest/nodes_test.go rename to systest/group-delete/group_delete_test.go From f2570ef8b21b5e326fd98d78c18d87e05790d458 Mon Sep 17 00:00:00 2001 From: srfrog Date: Tue, 15 Jan 2019 14:55:49 -0700 Subject: [PATCH 7/7] docker-compose file for group-delete tests --- systest/group-delete/docker-compose.yml | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 systest/group-delete/docker-compose.yml diff --git a/systest/group-delete/docker-compose.yml b/systest/group-delete/docker-compose.yml new file mode 100644 index 00000000000..f5b4c59706e --- /dev/null +++ b/systest/group-delete/docker-compose.yml @@ -0,0 +1,70 @@ +# This file sets up the cluster required by the tests in this directory. +version: "3.5" +services: + zero1: + image: dgraph/dgraph:latest + container_name: bank-dg0.1 + working_dir: /data/dg0.1 + ports: + - 5080:5080 + - 6080:6080 + labels: + cluster: group-delete-test + volumes: + - type: bind + source: $GOPATH/bin + target: /gobin + read_only: true + command: /gobin/dgraph zero --my=zero1:5080 --bindall --logtostderr + + dg1: + image: dgraph/dgraph:latest + container_name: bank-dg1 + working_dir: /data/dg1 + volumes: + - type: bind + source: $GOPATH/bin + target: /gobin + read_only: true + ports: + - 8180:8180 + - 9180:9180 + labels: + cluster: group-delete-test + command: /gobin/dgraph alpha --my=dg1:7180 --lru_mb=1024 --zero=zero1:5080 -o 100 --logtostderr + + dg2: + image: dgraph/dgraph:latest + container_name: bank-dg2 + working_dir: /data/dg2 + depends_on: + - dg1 + volumes: + - type: bind + source: $GOPATH/bin + target: /gobin + read_only: true + ports: + - 8182:8182 + - 9182:9182 + labels: + cluster: group-delete-test + command: /gobin/dgraph alpha --my=dg2:7182 --lru_mb=1024 --zero=zero1:5080 -o 102 --logtostderr + + dg3: + image: dgraph/dgraph:latest + container_name: bank-dg3 + working_dir: /data/dg3 + depends_on: + - dg2 + volumes: + - type: bind + source: $GOPATH/bin + target: /gobin + read_only: true + ports: + - 8183:8183 + - 9183:9183 + labels: + cluster: group-delete-test + command: /gobin/dgraph alpha --my=dg3:7183 --lru_mb=1024 --zero=zero1:5080 -o 103 --logtostderr