Add --new_uids option to live loader.

dgraph/cmd/live/load-discard-uid/family.json

@@ -0,0 +1,69 @@
+[
+  {
+    "uid":"0x2001",
+    "name":"Homer",
+    "age":"38",
+    "role":"father",
+    "parent_to": [
+      { "uid":"0x3001" },
+      { "uid":"0x3002" },
+      { "uid":"0x3003" }
+    ]
+  },
+  {
+    "uid":"0x2101",
+    "name":"Marge",
+    "age":"34",
+    "role":"mother",
+    "aka":"Midge",
+    "parent_to": [
+      { "uid":"0x3001" },
+      { "uid":"0x3002" },
+      { "uid":"0x3003" }
+    ]
+  },
+  {
+    "uid":"0x3001",
+    "name":"Bart",
+    "age":"10",
+    "role":"son",
+    "aka":"El Barto",
+    "carries":"slingshot",
+    "sibling_of": [
+      { "uid":"0x3002" },
+      { "uid":"0x3003" }
+    ]
+  },
+  {
+    "uid":"0x3002",
+    "name":"Lisa",
+    "age":"8",
+    "role":"daughter",
+    "carries":"saxomophone",
+    "sibling_of": [
+      { "uid":"0x3001" },
+      { "uid":"0x3003" }
+    ]
+  },
+  {
+    "uid":"0x3003",
+    "name":"Maggie",
+    "age":"1",
+    "role":"daughter",
+    "carries":"pacifier",
+    "sibling_of": [
+      { "uid":"0x3001" },
+      { "uid":"0x3002" }
+    ]
+  },
+  {
+    "uid":"0x1001",
+    "name":"Abraham",
+    "age":"83",
+    "role":"father",
+    "aka":"Grampa",
+    "parent_to": [
+      { "uid":"0x2001" }
+    ]
+  }
+]

dgraph/cmd/live/load-discard-uid/family.rdf

@@ -0,0 +1,42 @@
+<0x1001> <age> "83"^^<xs:int> .
+<0x1001> <aka> "Grampa"^^<xs:string> .
+<0x1001> <name> "Abraham"^^<xs:string> .
+<0x1001> <parent_to> <0x2001> .
+<0x1001> <role> "father"^^<xs:string> .
+#
+<0x2001> <age> "38"^^<xs:int> .
+<0x2001> <name> "Homer"^^<xs:string> .
+<0x2001> <parent_to> <0x3001> .
+<0x2001> <parent_to> <0x3002> .
+<0x2001> <parent_to> <0x3003> .
+<0x2001> <role> "father"^^<xs:string> .
+#
+<0x2101> <age> "34"^^<xs:int> .
+<0x2101> <aka> "Midge"^^<xs:string> .
+<0x2101> <name> "Marge"^^<xs:string> .
+<0x2101> <parent_to> <0x3001> .
+<0x2101> <parent_to> <0x3002> .
+<0x2101> <parent_to> <0x3003> .
+<0x2101> <role> "mother"^^<xs:string> .
+#
+<0x3001> <age> "10"^^<xs:int> .
+<0x3001> <aka> "El Barto"^^<xs:string> .
+<0x3001> <carries> "slingshot"^^<xs:string> .
+<0x3001> <name> "Bart"^^<xs:string> .
+<0x3001> <role> "son"^^<xs:string> .
+<0x3001> <sibling_of> <0x3002> .
+<0x3001> <sibling_of> <0x3003> .
+#
+<0x3002> <age> "8"^^<xs:int> .
+<0x3002> <carries> "saxomophone"^^<xs:string> .
+<0x3002> <name> "Lisa"^^<xs:string> .
+<0x3002> <role> "daughter"^^<xs:string> .
+<0x3002> <sibling_of> <0x3001> .
+<0x3002> <sibling_of> <0x3003> .
+#
+<0x3003> <age> "1"^^<xs:int> .
+<0x3003> <carries> "pacifier"^^<xs:string> .
+<0x3003> <name> "Maggie"^^<xs:string> .
+<0x3003> <role> "daughter"^^<xs:string> .
+<0x3003> <sibling_of> <0x3001> .
+<0x3003> <sibling_of> <0x3002> .

dgraph/cmd/live/load-discard-uid/family.schema

@@ -0,0 +1,7 @@
+name:string @index(term) .
+age: int .
+role:string @index(term) .
+aka:string @index(term) .
+carries:string @index(term) .
+parent_to: [uid] @reverse .
+sibling_of: [uid] @reverse .

dgraph/cmd/live/load-discard-uid/load_test.go

@@ -0,0 +1,186 @@
+/*
+ * Copyright 2017-2018 Dgraph Labs, Inc. and Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+
+	"github.com/dgraph-io/dgo"
+	"github.com/stretchr/testify/require"
+
+	"github.com/dgraph-io/dgraph/x"
+	"github.com/dgraph-io/dgraph/z"
+)
+
+const alphaService = ":9180"
+
+var (
+	testDataDir string
+	dg          *dgo.Dgraph
+	tmpDir      string
+)
+
+func checkDifferentUid(t *testing.T, wantMap, gotMap map[string]interface{}) {
+	require.NotEqual(t, gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"],
+		wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"],
+		"new uid was assigned")
+
+	gotMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1
+	wantMap["q"].([]interface{})[0].(map[string]interface{})["uid"] = -1
+	z.CompareJSONMaps(t, wantMap, gotMap)
+}
+
+func checkLoadedData(t *testing.T, newUids bool) {
+	resp, err := dg.NewTxn().Query(context.Background(), `
+		{
+			q(func: anyofterms(name, "Homer")) {
+				uid
+				name
+				age
+				role
+			}
+		}
+	`)
+	require.NoError(t, err)
+
+	gotMap := z.UnmarshalJSON(t, string(resp.GetJson()))
+	wantMap := z.UnmarshalJSON(t, `
+		{
+		    "q": [
+					{
+					"uid": "0x2001",
+					"name": "Homer",
+					"age": 38,
+					"role": "father"
+			    }
+			]
+		}
+	`)
+	if newUids {
+		checkDifferentUid(t, wantMap, gotMap)
+	} else {
+		z.CompareJSONMaps(t, wantMap, gotMap)
+	}
+
+	resp, err = dg.NewTxn().Query(context.Background(), `
+		{
+			q(func: anyofterms(name, "Maggie")) {
+				uid
+				name
+				role
+				carries
+			}
+		}
+	`)
+	require.NoError(t, err)
+
+	gotMap = z.UnmarshalJSON(t, string(resp.GetJson()))
+	wantMap = z.UnmarshalJSON(t, `
+		{
+		    "q": [
+				{
+					"uid": "0x3003",
+					"name": "Maggie",
+					"role": "daughter",
+					"carries": "pacifier"
+			    }
+			]
+		}
+	`)
+	if newUids {
+		checkDifferentUid(t, wantMap, gotMap)
+	} else {
+		z.CompareJSONMaps(t, wantMap, gotMap)
+	}
+}
+
+func TestLiveLoadJsonUidKeep(t *testing.T) {
+	z.DropAll(t, dg)
+
+	pipeline := [][]string{
+		{os.ExpandEnv("$GOPATH/bin/dgraph"), "live",
+			"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json",
+			"--dgraph", alphaService},
+	}
+	err := z.Pipeline(pipeline)
+	require.NoError(t, err, "live loading JSON file ran successfully")
+
+	checkLoadedData(t, false)
+}
+
+func TestLiveLoadJsonUidDiscard(t *testing.T) {
+	z.DropAll(t, dg)
+
+	pipeline := [][]string{
+		{os.ExpandEnv("$GOPATH/bin/dgraph"), "live", "--new_uids",
+			"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.json",
+			"--dgraph", alphaService},
+	}
+	err := z.Pipeline(pipeline)
+	require.NoError(t, err, "live loading JSON file ran successfully")
+
+	checkLoadedData(t, true)
+}
+
+func TestLiveLoadRdfUidKeep(t *testing.T) {
+	z.DropAll(t, dg)
+
+	pipeline := [][]string{
+		{os.ExpandEnv("$GOPATH/bin/dgraph"), "live",
+			"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf",
+			"--dgraph", alphaService},
+	}
+	err := z.Pipeline(pipeline)
+	require.NoError(t, err, "live loading JSON file ran successfully")
+
+	checkLoadedData(t, false)
+}
+
+func TestLiveLoadRdfUidDiscard(t *testing.T) {
+	z.DropAll(t, dg)
+
+	pipeline := [][]string{
+		{os.ExpandEnv("$GOPATH/bin/dgraph"), "live", "--new_uids",
+			"--schema", testDataDir + "/family.schema", "--files", testDataDir + "/family.rdf",
+			"--dgraph", alphaService},
+	}
+	err := z.Pipeline(pipeline)
+	require.NoError(t, err, "live loading JSON file ran successfully")
+
+	checkLoadedData(t, true)
+}
+
+func TestMain(m *testing.M) {
+	_, thisFile, _, _ := runtime.Caller(0)
+	testDataDir = path.Dir(thisFile)
+
+	dg = z.DgraphClient(alphaService)
+
+	// Try to create any files in a dedicated temp directory that gets cleaned up
+	// instead of all over /tmp or the working directory.
+	tmpDir, err := ioutil.TempDir("", "test.tmp-")
+	x.Check(err)
+	os.Chdir(tmpDir)
+	defer os.RemoveAll(tmpDir)
+
+	os.Exit(m.Run())
+}

dgraph/cmd/live/run.go

@@ -60,6 +60,7 @@ type options struct {
 	ignoreIndexConflict bool
 	authToken           string
 	useCompression      bool
+	newUids             bool
 }
 
 var (
@@ -98,6 +99,8 @@ func init() {
 		"The auth token passed to the server for Alter operation of the schema file")
 	flag.BoolP("use_compression", "C", false,
 		"Enable compression on connection to alpha server")
+	flag.Bool("new_uids", false, ""+
+		"Ignore UIDs in load files and assign new ones.")
 
 	// TLS configuration
 	x.RegisterTLSFlags(flag)
@@ -134,7 +137,7 @@ func processSchemaFile(ctx context.Context, file string, dgraphClient *dgo.Dgrap
 	}
 
 	f, err := os.Open(file)
-	x.Check(err)
+	x.CheckfNoTrace(err)
 	defer f.Close()
 
 	var reader io.Reader
@@ -161,9 +164,11 @@ func (l *loader) uid(val string) string {
 	// to be an existing node in the graph. There is limited protection against
 	// a user selecting an unassigned UID in this way - it may be assigned
 	// later to another node. It is up to the user to avoid this.
-	if uid, err := strconv.ParseUint(val, 0, 64); err == nil {
-		l.alloc.BumpTo(uid)
-		return fmt.Sprintf("%#x", uid)
+	if !opt.newUids {
+		if uid, err := strconv.ParseUint(val, 0, 64); err == nil {
+			l.alloc.BumpTo(uid)
+			return fmt.Sprintf("%#x", uid)
+		}
 	}
 
 	uid := l.alloc.AssignUid(val)
@@ -292,6 +297,7 @@ func run() error {
 		ignoreIndexConflict: Live.Conf.GetBool("ignore_index_conflict"),
 		authToken:           Live.Conf.GetString("auth_token"),
 		useCompression:      Live.Conf.GetBool("use_compression"),
+		newUids:             Live.Conf.GetBool("new_uids"),
 	}
 	x.LoadTLSConfig(&tlsConf, Live.Conf, x.TlsClientCert, x.TlsClientKey)
 	tlsConf.ServerName = Live.Conf.GetString("tls_server_name")

z/exec.go

@@ -26,8 +26,11 @@ import (
 )
 
 // for debugging the tests
-const showOutput = false
-const showCommands = false
+
+var (
+	showOutput  bool = os.Getenv("DEBUG_SHOW_OUTPUT") != ""
+	showCommand bool = os.Getenv("DEBUG_SHOW_COMMAND") != ""
+)
 
 // Pipeline runs several commands such that the output of one command becomes the input of the next.
 // The first argument should be an two-dimensional array containing the commands.
@@ -41,7 +44,7 @@ func Pipeline(cmds [][]string) error {
 	// Run all commands in parallel, connecting stdin of each to the stdout of the previous.
 	for i, c := range cmds {
 		lastCmd := i == numCmds-1
-		if showCommands {
+		if showCommand {
 			fmt.Fprintf(os.Stderr, "%+v", c)
 		}
 
@@ -58,7 +61,7 @@ func Pipeline(cmds [][]string) error {
 			}
 		}
 
-		if showCommands {
+		if showCommand {
 			if lastCmd {
 				fmt.Fprintf(os.Stderr, "\n")
 			} else {