From fd0cecaf9945e0c9eb8fedaec5dad28f701ec151 Mon Sep 17 00:00:00 2001 From: Javier Alvarado Date: Wed, 23 Jan 2019 14:50:36 -0800 Subject: [PATCH 1/2] Add angle brackets to more exported predicates. --- worker/export.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/worker/export.go b/worker/export.go index 424c02e81f6..4385a7d8b51 100644 --- a/worker/export.go +++ b/worker/export.go @@ -27,6 +27,7 @@ import ( "strconv" "strings" "time" + "unicode" "github.com/golang/glog" "golang.org/x/net/context" @@ -54,6 +55,18 @@ var rdfTypeMap = map[types.TypeID]string{ types.PasswordID: "xs:password", } +// Having '<' and '>' around all predicates makes the exported schema harder +// for humans to look at, so only put them on predicates containing "exotic" +// characters (i.e. ones not in this list). +var predNonSpecialChars = unicode.RangeTable{ + R16: []unicode.Range16{ + unicode.Range16{'.', '.', 1}, + unicode.Range16{'0', '9', 1}, + unicode.Range16{'A', 'Z', 1}, + unicode.Range16{'a', 'z', 1}, + }, +} + func toRDF(pl *posting.List, prefix string, readTs uint64) (*bpb.KVList, error) { var buf bytes.Buffer @@ -142,7 +155,10 @@ func toRDF(pl *posting.List, prefix string, readTs uint64) (*bpb.KVList, error) func toSchema(attr string, update pb.SchemaUpdate) (*bpb.KVList, error) { // bytes.Buffer never returns error for any of the writes. So, we don't need to check them. var buf bytes.Buffer - if strings.ContainsRune(attr, ':') { + isSpecial := func(r rune) bool { + return !(unicode.In(r, &predNonSpecialChars)) + } + if strings.IndexFunc(attr, isSpecial) >= 0 { buf.WriteRune('<') buf.WriteString(attr) buf.WriteRune('>') From 6219093c27ef3950349a1b857e17002106916df4 Mon Sep 17 00:00:00 2001 From: Javier Alvarado Date: Wed, 23 Jan 2019 15:23:55 -0800 Subject: [PATCH 2/2] Add more predicate export tests. --- worker/export.go | 2 ++ worker/export_test.go | 75 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/worker/export.go b/worker/export.go index 4385a7d8b51..9d9c7c76896 100644 --- a/worker/export.go +++ b/worker/export.go @@ -60,9 +60,11 @@ var rdfTypeMap = map[types.TypeID]string{ // characters (i.e. ones not in this list). var predNonSpecialChars = unicode.RangeTable{ R16: []unicode.Range16{ + // Ranges must be in order. unicode.Range16{'.', '.', 1}, unicode.Range16{'0', '9', 1}, unicode.Range16{'A', 'Z', 1}, + unicode.Range16{'_', '_', 1}, unicode.Range16{'a', 'z', 1}, }, } diff --git a/worker/export_test.go b/worker/export_test.go index 75d5630659e..7469901a7bf 100644 --- a/worker/export_test.go +++ b/worker/export_test.go @@ -273,6 +273,81 @@ func TestToSchema(t *testing.T) { }, expected: ":string @reverse @lang . \n", }, + { + skv: &skv{ + attr: "username/password", + schema: pb.SchemaUpdate{ + Predicate: "", + ValueType: pb.Posting_STRING, + Directive: pb.SchemaUpdate_NONE, + List: false, + Count: false, + Upsert: false, + Lang: false, + }, + }, + expected: ":string . \n", + }, + { + skv: &skv{ + attr: "B*-tree", + schema: pb.SchemaUpdate{ + Predicate: "", + ValueType: pb.Posting_UID, + Directive: pb.SchemaUpdate_REVERSE, + List: true, + Count: false, + Upsert: false, + Lang: false, + }, + }, + expected: ":[uid] @reverse . \n", + }, + { + skv: &skv{ + attr: "base_de_données", + schema: pb.SchemaUpdate{ + Predicate: "", + ValueType: pb.Posting_STRING, + Directive: pb.SchemaUpdate_NONE, + List: false, + Count: false, + Upsert: false, + Lang: true, + }, + }, + expected: ":string @lang . \n", + }, + { + skv: &skv{ + attr: "data_base", + schema: pb.SchemaUpdate{ + Predicate: "", + ValueType: pb.Posting_STRING, + Directive: pb.SchemaUpdate_NONE, + List: false, + Count: false, + Upsert: false, + Lang: true, + }, + }, + expected: "data_base:string @lang . \n", + }, + { + skv: &skv{ + attr: "data.base", + schema: pb.SchemaUpdate{ + Predicate: "", + ValueType: pb.Posting_STRING, + Directive: pb.SchemaUpdate_NONE, + List: false, + Count: false, + Upsert: false, + Lang: true, + }, + }, + expected: "data.base:string @lang . \n", + }, } for _, testCase := range testCases { list, err := toSchema(testCase.skv.attr, testCase.skv.schema)