From 11b145dcc2e0ea20fdd2909fbec3e7b948dba3c4 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Tue, 2 Mar 2021 11:58:52 +0100 Subject: [PATCH 01/40] Prototype composite indexes support --- database/config.go | 2 +- database/index.go | 158 ++++++++++++++++++++++++++++++++++------- database/index_test.go | 96 +++++++++++++------------ 3 files changed, 183 insertions(+), 73 deletions(-) diff --git a/database/config.go b/database/config.go index 30dc99679..7e1efac99 100644 --- a/database/config.go +++ b/database/config.go @@ -236,7 +236,7 @@ type IndexInfo struct { Unique bool // If set, the index is typed and only accepts that type - Type document.ValueType + Types []document.ValueType } // ToDocument creates a document from an IndexConfig. diff --git a/database/index.go b/database/index.go index 1f0292aa2..e81d82d48 100644 --- a/database/index.go +++ b/database/index.go @@ -31,6 +31,16 @@ type Index struct { // NewIndex creates an index that associates a value with a list of keys. func NewIndex(tx engine.Transaction, idxName string, opts *IndexInfo) *Index { + if opts == nil { + opts = &IndexInfo{ + Types: []document.ValueType{document.ValueType(0)} + } + } + + if opts.Types == nil { + opts.Types= []document.ValueType{document.ValueType(0)} + } + return &Index{ tx: tx, storeName: append([]byte(indexStorePrefix), idxName...), @@ -43,15 +53,27 @@ var errStop = errors.New("stop") // Set associates a value with a key. If Unique is set to false, it is // possible to associate multiple keys for the same value // but a key can be associated to only one value. -func (idx *Index) Set(v document.Value, k []byte) error { +func (idx *Index) Set(vs []document.Value, k []byte) error { var err error if len(k) == 0 { return errors.New("cannot index value without a key") } - if idx.Info.Type != 0 && idx.Info.Type != v.Type { - return stringutil.Errorf("cannot index value of type %s in %s index", v.Type, idx.Info.Type) + if len(vs) == 0 { + return errors.New("cannot index without a value") + } + + if len(vs) > len(idx.Types) { + return errors.New("cannot index more values than what the index supports") + } + + for i, typ := range idx.Types { + // it is possible to set an index(a,b) on (a), it will be assumed that b is null in that case + if typ != 0 && i < len(vs) && typ != vs[i].Type { + // TODO use the full version to clarify the error + return fmt.Errorf("cannot index value of type %s in %s index", vs[i].Type, typ) + } } st, err := getOrCreateStore(idx.tx, idx.storeName) @@ -60,7 +82,7 @@ func (idx *Index) Set(v document.Value, k []byte) error { } // encode the value we are going to use as a key - buf, err := idx.EncodeValue(v) + buf, err := idx.EncodeValues(vs) if err != nil { return err } @@ -105,19 +127,21 @@ func (idx *Index) Set(v document.Value, k []byte) error { } // Delete all the references to the key from the index. -func (idx *Index) Delete(v document.Value, k []byte) error { +func (idx *Index) Delete(vs []document.Value, k []byte) error { st, err := getOrCreateStore(idx.tx, idx.storeName) if err != nil { + // TODO, more precise error handling? return nil } var toDelete []byte var buf []byte - err = idx.iterate(st, v, false, func(item engine.Item) error { + err = idx.iterate(st, vs, false, func(item engine.Item) error { buf, err = item.ValueCopy(buf[:0]) if err != nil { return err } + if bytes.Equal(buf, k) { toDelete = item.Key() return errStop @@ -139,22 +163,33 @@ func (idx *Index) Delete(v document.Value, k []byte) error { // AscendGreaterOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in increasing order and calls the given function for each pair. // If the given function returns an error, the iteration stops and returns that error. // If the pivot is empty, starts from the beginning. -func (idx *Index) AscendGreaterOrEqual(pivot document.Value, fn func(val, key []byte) error) error { - return idx.iterateOnStore(pivot, false, fn) +func (idx *Index) AscendGreaterOrEqual(pivots []document.Value, fn func(val, key []byte) error) error { + return idx.iterateOnStore(pivots, false, fn) } // DescendLessOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in descreasing order and calls the given function for each pair. // If the given function returns an error, the iteration stops and returns that error. // If the pivot is empty, starts from the end. -func (idx *Index) DescendLessOrEqual(pivot document.Value, fn func(val, key []byte) error) error { - return idx.iterateOnStore(pivot, true, fn) +func (idx *Index) DescendLessOrEqual(pivots []document.Value, fn func(val, key []byte) error) error { + return idx.iterateOnStore(pivots, true, fn) } -func (idx *Index) iterateOnStore(pivot document.Value, reverse bool, fn func(val, key []byte) error) error { - // if index and pivot are typed but not of the same type - // return no result - if idx.Info.Type != 0 && pivot.Type != 0 && idx.Info.Type != pivot.Type { - return nil +func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func(val, key []byte) error) error { + if len(pivots) == 0 { + return errors.New("cannot iterate without a pivot") + } + + if len(pivots) > len(idx.Types) { + return errors.New("cannot iterate with more values than what the index supports") + } + + for i, typ := range idx.Types { + // if index and pivot are typed but not of the same type + // return no result + if typ != 0 && pivots[i].Type != 0 && typ != pivots[i].Type { + return nil + } + } st, err := idx.tx.GetStore(idx.storeName) @@ -166,7 +201,7 @@ func (idx *Index) iterateOnStore(pivot document.Value, reverse bool, fn func(val } var buf []byte - return idx.iterate(st, pivot, reverse, func(item engine.Item) error { + return idx.iterate(st, pivots, reverse, func(item engine.Item) error { var err error k := item.Key() @@ -202,7 +237,7 @@ func (idx *Index) Truncate() error { // the presence of other types. // If not, encode so that order is preserved regardless of the type. func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { - if idx.Info.Type != 0 { + if idx.Types[0] != 0 { return v.MarshalBinary() } @@ -215,6 +250,45 @@ func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { return buf.Bytes(), nil } +// TODO +func (idx *Index) EncodeValues(vs []document.Value) ([]byte, error) { + buf := []byte{} + + for i, v := range vs { + if idx.Types[i] != 0 { + b, err := v.MarshalBinary() + if err != nil { + return nil, err + } + + buf = append(buf, b...) + continue + } + + var err error + if v.Type == document.IntegerValue { + if v.V == nil { + v.Type = document.DoubleValue + } else { + v, err = v.CastAsDouble() + if err != nil { + return nil, err + } + } + } + + var bbuf bytes.Buffer + err = document.NewValueEncoder(&bbuf).Encode(v) + if err != nil { + return nil, err + } + b := bbuf.Bytes() + buf = append(buf, b...) + } + + return buf, nil +} + func getOrCreateStore(tx engine.Transaction, name []byte) (engine.Store, error) { st, err := tx.GetStore(name) if err == nil { @@ -233,12 +307,25 @@ func getOrCreateStore(tx engine.Transaction, name []byte) (engine.Store, error) return tx.GetStore(name) } -func (idx *Index) iterate(st engine.Store, pivot document.Value, reverse bool, fn func(item engine.Item) error) error { +func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool, fn func(item engine.Item) error) error { var seek []byte var err error - if pivot.V != nil { - seek, err = idx.EncodeValue(pivot) + for i, typ := range idx.Types { + if typ == 0 && pivots[i].Type == document.IntegerValue { + if pivots[i].V == nil { + pivots[i].Type = document.DoubleValue + } else { + pivots[i], err = pivots[i].CastAsDouble() + if err != nil { + return err + } + } + } + } + + if pivots[0].V != nil { + seek, err = idx.EncodeValues(pivots) if err != nil { return err } @@ -246,14 +333,26 @@ func (idx *Index) iterate(st engine.Store, pivot document.Value, reverse bool, f if reverse { seek = append(seek, 0xFF) } - } - - if idx.Info.Type == 0 && pivot.Type != 0 && pivot.V == nil { - seek = []byte{byte(pivot.Type)} + } else { + // this is pretty surely wrong as it does not allow to select t1t2 is there are values on t1 + buf := []byte{} + for i, typ := range idx.Types { + if typ == 0 && pivots[i].Type != 0 && pivots[i].V == nil { + buf = append(buf, byte(pivots[i].Type)) + } + } + seek = buf if reverse { seek = append(seek, 0xFF) } + // if idx.Type == 0 && pivot.Type != 0 && pivot.V == nil { + // seek = []byte{byte(pivot.Type)} + + // if reverse { + // seek = append(seek, 0xFF) + // } + // } } it := st.Iterator(engine.IteratorOptions{Reverse: reverse}) @@ -263,8 +362,15 @@ func (idx *Index) iterate(st engine.Store, pivot document.Value, reverse bool, f itm := it.Item() // if index is untyped and pivot is typed, only iterate on values with the same type as pivot - if idx.Info.Type == 0 && pivot.Type != 0 && itm.Key()[0] != byte(pivot.Type) { - return nil + // if idx.Type == 0 && pivot.Type != 0 && itm.Key()[0] != byte(pivot.Type) { + // return nil + // } + + // this is wrong and only handle the first type + for i, typ := range idx.Types { + if typ == 0 && pivots[i].Type != 0 && itm.Key()[0] != byte(pivots[i].Type) { + return nil + } } err := fn(itm) diff --git a/database/index_test.go b/database/index_test.go index d3eed616c..54fdfd255 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -16,6 +16,10 @@ import ( "github.com/stretchr/testify/require" ) +func values(vs ...document.Value) []document.Value { + return vs +} + func getIndex(t testing.TB, unique bool) (*database.Index, func()) { ng := memoryengine.NewEngine() tx, err := ng.Begin(context.Background(), engine.TxOptions{ @@ -37,13 +41,13 @@ func TestIndexSet(t *testing.T) { t.Run(text+"Set nil key falls", func(t *testing.T) { idx, cleanup := getIndex(t, unique) defer cleanup() - require.Error(t, idx.Set(document.NewBoolValue(true), nil)) + require.Error(t, idx.Set(values(document.NewBoolValue(true)), nil)) }) t.Run(text+"Set value and key succeeds", func(t *testing.T) { idx, cleanup := getIndex(t, unique) defer cleanup() - require.NoError(t, idx.Set(document.NewBoolValue(true), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewBoolValue(true)), []byte("key"))) }) } @@ -51,18 +55,18 @@ func TestIndexSet(t *testing.T) { idx, cleanup := getIndex(t, true) defer cleanup() - require.NoError(t, idx.Set(document.NewIntegerValue(10), []byte("key"))) - require.NoError(t, idx.Set(document.NewIntegerValue(11), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(10)), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(11)), []byte("key"))) require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(document.NewIntegerValue(10), []byte("key"))) }) t.Run("Unique: true, Type: integer Duplicate", func(t *testing.T) { idx, cleanup := getIndex(t, true) - idx.Info.Type = document.IntegerValue + idx.Info.Types = []document.ValueType{document.IntegerValue} defer cleanup() - require.NoError(t, idx.Set(document.NewIntegerValue(10), []byte("key"))) - require.NoError(t, idx.Set(document.NewIntegerValue(11), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(10)), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(11)), []byte("key"))) require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(document.NewIntegerValue(10), []byte("key"))) }) } @@ -72,13 +76,13 @@ func TestIndexDelete(t *testing.T) { idx, cleanup := getIndex(t, false) defer cleanup() - require.NoError(t, idx.Set(document.NewDoubleValue(10), []byte("key"))) - require.NoError(t, idx.Set(document.NewIntegerValue(10), []byte("other-key"))) - require.NoError(t, idx.Set(document.NewIntegerValue(11), []byte("yet-another-key"))) - require.NoError(t, idx.Set(document.NewTextValue("hello"), []byte("yet-another-different-key"))) - require.NoError(t, idx.Delete(document.NewDoubleValue(10), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewDoubleValue(10)), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(10)), []byte("other-key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(11)), []byte("yet-another-key"))) + require.NoError(t, idx.Set(values(document.NewTextValue("hello")), []byte("yet-another-different-key"))) + require.NoError(t, idx.Delete(values(document.NewDoubleValue(10)), []byte("key"))) - pivot := document.NewIntegerValue(10) + pivot := values(document.NewIntegerValue(10)) i := 0 err := idx.AscendGreaterOrEqual(pivot, func(v, k []byte) error { if i == 0 { @@ -102,13 +106,13 @@ func TestIndexDelete(t *testing.T) { idx, cleanup := getIndex(t, true) defer cleanup() - require.NoError(t, idx.Set(document.NewIntegerValue(10), []byte("key1"))) - require.NoError(t, idx.Set(document.NewDoubleValue(11), []byte("key2"))) - require.NoError(t, idx.Set(document.NewIntegerValue(12), []byte("key3"))) - require.NoError(t, idx.Delete(document.NewDoubleValue(11), []byte("key2"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(10)), []byte("key1"))) + require.NoError(t, idx.Set(values(document.NewDoubleValue(11)), []byte("key2"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(12)), []byte("key3"))) + require.NoError(t, idx.Delete(values(document.NewDoubleValue(11)), []byte("key2"))) i := 0 - err := idx.AscendGreaterOrEqual(document.Value{Type: document.IntegerValue}, func(v, k []byte) error { + err := idx.AscendGreaterOrEqual(values(document.Value{Type: document.IntegerValue}), func(v, k []byte) error { switch i { case 0: requireEqualEncoded(t, document.NewIntegerValue(10), v) @@ -134,7 +138,7 @@ func TestIndexDelete(t *testing.T) { idx, cleanup := getIndex(t, unique) defer cleanup() - require.Error(t, idx.Delete(document.NewTextValue("foo"), []byte("foo"))) + require.Error(t, idx.Delete(values(document.NewTextValue("foo")), []byte("foo"))) }) } } @@ -157,7 +161,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { defer cleanup() i := 0 - err := idx.AscendGreaterOrEqual(document.Value{Type: document.IntegerValue}, func(val, key []byte) error { + err := idx.AscendGreaterOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, key []byte) error { i++ return errors.New("should not iterate") }) @@ -183,12 +187,12 @@ func TestIndexAscendGreaterThan(t *testing.T) { defer cleanup() for i := 0; i < 10; i += 2 { - require.NoError(t, idx.Set(test.val(i), []byte{'a' + byte(i)})) + require.NoError(t, idx.Set(values(test.val(i)), []byte{'a' + byte(i)})) } var i uint8 var count int - err := idx.AscendGreaterOrEqual(test.pivot, func(val, rid []byte) error { + err := idx.AscendGreaterOrEqual(values(test.pivot), func(val, rid []byte) error { switch test.t { case document.IntegerValue: requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) @@ -216,12 +220,12 @@ func TestIndexAscendGreaterThan(t *testing.T) { defer cleanup() for i := byte(0); i < 10; i += 2 { - require.NoError(t, idx.Set(document.NewTextValue(string([]byte{'A' + i})), []byte{'a' + i})) + require.NoError(t, idx.Set(values(document.NewTextValue(string([]byte{'A' + i}))), []byte{'a' + i})) } var i uint8 var count int - pivot := document.NewTextValue("C") + pivot := values(document.NewTextValue("C")) err := idx.AscendGreaterOrEqual(pivot, func(val, rid []byte) error { requireEqualEncoded(t, document.NewTextValue(string([]byte{'C' + i})), val) require.Equal(t, []byte{'c' + i}, rid) @@ -239,13 +243,13 @@ func TestIndexAscendGreaterThan(t *testing.T) { defer cleanup() for i := int64(0); i < 10; i++ { - require.NoError(t, idx.Set(document.NewDoubleValue(float64(i)), []byte{'d', 'a' + byte(i)})) - require.NoError(t, idx.Set(document.NewTextValue(strconv.Itoa(int(i))), []byte{'s', 'a' + byte(i)})) + require.NoError(t, idx.Set(values(document.NewDoubleValue(float64(i))), []byte{'d', 'a' + byte(i)})) + require.NoError(t, idx.Set(values(document.NewTextValue(strconv.Itoa(int(i)))), []byte{'s', 'a' + byte(i)})) } var doubles, texts int var count int - err := idx.AscendGreaterOrEqual(document.Value{}, func(val, rid []byte) error { + err := idx.AscendGreaterOrEqual(values(document.Value{}), func(val, rid []byte) error { if count < 10 { requireEqualEncoded(t, document.NewDoubleValue(float64(doubles)), val) require.Equal(t, []byte{'d', 'a' + byte(doubles)}, rid) @@ -265,15 +269,15 @@ func TestIndexAscendGreaterThan(t *testing.T) { t.Run(text+"With no pivot and typed index, should iterate over all documents in order", func(t *testing.T) { idx, cleanup := getIndex(t, unique) - idx.Info.Type = document.IntegerValue + idx.Info.Types = []document.ValueType{document.IntegerValue} defer cleanup() for i := int64(0); i < 10; i++ { - require.NoError(t, idx.Set(document.NewIntegerValue(i), []byte{'i', 'a' + byte(i)})) + require.NoError(t, idx.Set(values(document.NewIntegerValue(i)), []byte{'i', 'a' + byte(i)})) } var ints int - err := idx.AscendGreaterOrEqual(document.Value{}, func(val, rid []byte) error { + err := idx.AscendGreaterOrEqual(values(document.Value{}), func(val, rid []byte) error { enc, err := document.NewIntegerValue(int64(ints)).MarshalBinary() require.NoError(t, err) require.Equal(t, enc, val) @@ -292,14 +296,14 @@ func TestIndexAscendGreaterThan(t *testing.T) { defer cleanup() for i := int64(0); i < 100; i++ { - require.NoError(t, idx.Set(document.NewIntegerValue(1), binarysort.AppendInt64(nil, i))) - require.NoError(t, idx.Set(document.NewTextValue("1"), binarysort.AppendInt64(nil, i))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(1)), binarysort.AppendInt64(nil, i))) + require.NoError(t, idx.Set(values(document.NewTextValue("1")), binarysort.AppendInt64(nil, i))) } var doubles, texts int i := int64(0) - err := idx.AscendGreaterOrEqual(document.Value{Type: document.IntegerValue}, func(val, rid []byte) error { - requireEqualEncoded(t, document.NewIntegerValue(1), val) + err := idx.AscendGreaterOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, rid []byte) error { + requireEqualEncoded(t, document.NewDoubleValue(1), val) require.Equal(t, binarysort.AppendInt64(nil, i), rid) i++ doubles++ @@ -308,7 +312,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { require.NoError(t, err) i = 0 - err = idx.AscendGreaterOrEqual(document.Value{Type: document.TextValue}, func(val, rid []byte) error { + err = idx.AscendGreaterOrEqual(values(document.Value{Type: document.TextValue}), func(val, rid []byte) error { requireEqualEncoded(t, document.NewTextValue("1"), val) require.Equal(t, binarysort.AppendInt64(nil, i), rid) i++ @@ -330,7 +334,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { defer cleanup() i := 0 - err := idx.DescendLessOrEqual(document.Value{Type: document.IntegerValue}, func(val, key []byte) error { + err := idx.DescendLessOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, key []byte) error { i++ return errors.New("should not iterate") }) @@ -343,13 +347,13 @@ func TestIndexDescendLessOrEqual(t *testing.T) { defer cleanup() for i := byte(0); i < 10; i += 2 { - require.NoError(t, idx.Set(document.NewIntegerValue(int64(i)), []byte{'a' + i})) + require.NoError(t, idx.Set(values(document.NewIntegerValue(int64(i))), []byte{'a' + i})) } var i uint8 = 8 var count int - err := idx.DescendLessOrEqual(document.Value{Type: document.IntegerValue}, func(val, key []byte) error { - requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) + err := idx.DescendLessOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, key []byte) error { + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) require.Equal(t, []byte{'a' + i}, key) i -= 2 @@ -365,12 +369,12 @@ func TestIndexDescendLessOrEqual(t *testing.T) { defer cleanup() for i := byte(0); i < 10; i++ { - require.NoError(t, idx.Set(document.NewTextValue(string([]byte{'A' + i})), []byte{'a' + i})) + require.NoError(t, idx.Set(values(document.NewTextValue(string([]byte{'A' + i}))), []byte{'a' + i})) } var i byte = 0 var count int - pivot := document.NewTextValue("F") + pivot := values(document.NewTextValue("F")) err := idx.DescendLessOrEqual(pivot, func(val, rid []byte) error { requireEqualEncoded(t, document.NewTextValue(string([]byte{'F' - i})), val) require.Equal(t, []byte{'f' - i}, rid) @@ -388,13 +392,13 @@ func TestIndexDescendLessOrEqual(t *testing.T) { defer cleanup() for i := 0; i < 10; i++ { - require.NoError(t, idx.Set(document.NewIntegerValue(int64(i)), []byte{'i', 'a' + byte(i)})) - require.NoError(t, idx.Set(document.NewTextValue(strconv.Itoa(i)), []byte{'s', 'a' + byte(i)})) + require.NoError(t, idx.Set(values(document.NewIntegerValue(int64(i))), []byte{'i', 'a' + byte(i)})) + require.NoError(t, idx.Set(values(document.NewTextValue(strconv.Itoa(i))), []byte{'s', 'a' + byte(i)})) } var ints, texts int = 9, 9 var count int = 20 - err := idx.DescendLessOrEqual(document.Value{}, func(val, rid []byte) error { + err := idx.DescendLessOrEqual(values(document.Value{}), func(val, rid []byte) error { if count > 10 { requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(texts))), val) require.Equal(t, []byte{'s', 'a' + byte(texts)}, rid) @@ -428,7 +432,7 @@ func BenchmarkIndexSet(b *testing.B) { b.StartTimer() for j := 0; j < size; j++ { k := fmt.Sprintf("name-%d", j) - idx.Set(document.NewTextValue(k), []byte(k)) + idx.Set(values(document.NewTextValue(k)), []byte(k)) } b.StopTimer() cleanup() @@ -446,7 +450,7 @@ func BenchmarkIndexIteration(b *testing.B) { for i := 0; i < size; i++ { k := []byte(fmt.Sprintf("name-%d", i)) - _ = idx.Set(document.NewTextValue(string(k)), k) + _ = idx.Set(values(document.NewTextValue(string(k))), k) } b.ResetTimer() From 9050069b6efe5935f97eafddb927d5dbf0ed52f1 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 4 Mar 2021 16:43:19 +0100 Subject: [PATCH 02/40] Update internals to support composite indexes --- database/catalog.go | 19 ++- database/catalog_test.go | 36 ++--- database/config.go | 55 +++++-- database/config_test.go | 3 +- database/index.go | 199 ++++++++++++++--------- database/index_test.go | 332 ++++++++++++++++++++++++++++++++++---- database/table.go | 29 ++-- database/table_test.go | 10 +- planner/optimizer.go | 2 +- planner/optimizer_test.go | 8 + query/create.go | 8 +- query/create_test.go | 6 +- query/reindex_test.go | 2 +- query/select_test.go | 2 +- sql/parser/create.go | 9 +- stream/iterator.go | 152 ++++++++++++----- stream/iterator_test.go | 210 +++++++++++++++++++++--- testutil/document.go | 26 ++- 18 files changed, 884 insertions(+), 224 deletions(-) diff --git a/database/catalog.go b/database/catalog.go index 1b41c14ff..0911377f1 100644 --- a/database/catalog.go +++ b/database/catalog.go @@ -325,7 +325,8 @@ func (c *Catalog) ReIndex(tx *Transaction, indexName string) error { func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error { return table.Iterate(func(d document.Document) error { - v, err := idx.Info.Path.GetValueFromDocument(d) + // TODO + v, err := idx.Info.Paths[0].GetValueFromDocument(d) if err == document.ErrFieldNotFound { return nil } @@ -333,7 +334,8 @@ func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error { return err } - err = idx.Set(v, d.(document.Keyer).RawKey()) + // TODO + err = idx.Set([]document.Value{v}, d.(document.Keyer).RawKey()) if err != nil { return stringutil.Errorf("error while building the index: %w", err) } @@ -493,12 +495,15 @@ func (c *catalogCache) AddIndex(tx *Transaction, info *IndexInfo) error { // if the index is created on a field on which we know the type, // create a typed index. for _, fc := range ti.FieldConstraints { - if fc.Path.IsEqual(info.Path) { - if fc.Type != 0 { - info.Type = fc.Type + for _, path := range info.Paths { + if fc.Path.IsEqual(path) { + if fc.Type != 0 { + // TODO + info.Types = append(info.Types, document.ValueType(fc.Type)) + } + + break } - - break } } diff --git a/database/catalog_test.go b/database/catalog_test.go index cd1a02c02..2f5f7b02b 100644 --- a/database/catalog_test.go +++ b/database/catalog_test.go @@ -115,9 +115,9 @@ func TestCatalogTable(t *testing.T) { err := catalog.CreateTable(tx, "foo", ti) require.NoError(t, err) - err = catalog.CreateIndex(tx, &database.IndexInfo{Path: parsePath(t, "gender"), IndexName: "idx_gender", TableName: "foo"}) + err = catalog.CreateIndex(tx, &database.IndexInfo{Paths: []document.Path{parsePath(t, "gender")}, IndexName: "idx_gender", TableName: "foo"}) require.NoError(t, err) - err = catalog.CreateIndex(tx, &database.IndexInfo{Path: parsePath(t, "city"), IndexName: "idx_city", TableName: "foo", Unique: true}) + err = catalog.CreateIndex(tx, &database.IndexInfo{Paths: []document.Path{parsePath(t, "city")}, IndexName: "idx_city", TableName: "foo", Unique: true}) require.NoError(t, err) return nil @@ -318,7 +318,7 @@ func TestTxCreateIndex(t *testing.T) { update(t, db, func(tx *database.Transaction) error { err := catalog.CreateIndex(tx, &database.IndexInfo{ - IndexName: "idx_a", TableName: "test", Path: parsePath(t, "a"), + IndexName: "idx_a", TableName: "test", Paths: []document.Path{parsePath(t, "a")}, }) require.NoError(t, err) idx, err := tx.GetIndex("idx_a") @@ -355,12 +355,12 @@ func TestTxCreateIndex(t *testing.T) { update(t, db, func(tx *database.Transaction) error { err := catalog.CreateIndex(tx, &database.IndexInfo{ - IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"), + IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")}, }) require.NoError(t, err) err = catalog.CreateIndex(tx, &database.IndexInfo{ - IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"), + IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")}, }) require.Equal(t, database.ErrIndexAlreadyExists, err) return nil @@ -373,7 +373,7 @@ func TestTxCreateIndex(t *testing.T) { catalog := db.Catalog() update(t, db, func(tx *database.Transaction) error { err := catalog.CreateIndex(tx, &database.IndexInfo{ - IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"), + IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")}, }) if !errors.Is(err, database.ErrTableNotFound) { require.Equal(t, err, database.ErrTableNotFound) @@ -394,7 +394,7 @@ func TestTxCreateIndex(t *testing.T) { update(t, db, func(tx *database.Transaction) error { err := catalog.CreateIndex(tx, &database.IndexInfo{ - TableName: "test", Path: parsePath(t, "foo"), + TableName: "test", Paths: []document.Path{parsePath(t, "foo")}, }) require.NoError(t, err) @@ -403,7 +403,7 @@ func TestTxCreateIndex(t *testing.T) { // create another one err = catalog.CreateIndex(tx, &database.IndexInfo{ - TableName: "test", Path: parsePath(t, "foo"), + TableName: "test", Paths: []document.Path{parsePath(t, "foo")}, }) require.NoError(t, err) @@ -424,11 +424,11 @@ func TestTxDropIndex(t *testing.T) { err := catalog.CreateTable(tx, "test", nil) require.NoError(t, err) err = catalog.CreateIndex(tx, &database.IndexInfo{ - IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"), + IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")}, }) require.NoError(t, err) err = catalog.CreateIndex(tx, &database.IndexInfo{ - IndexName: "idxBar", TableName: "test", Path: parsePath(t, "bar"), + IndexName: "idxBar", TableName: "test", Paths: []document.Path{parsePath(t, "bar")}, }) require.NoError(t, err) return nil @@ -489,13 +489,13 @@ func TestCatalogReIndex(t *testing.T) { err = catalog.CreateIndex(tx, &database.IndexInfo{ IndexName: "a", TableName: "test", - Path: parsePath(t, "a"), + Paths: []document.Path{parsePath(t, "a")}, }) require.NoError(t, err) err = catalog.CreateIndex(tx, &database.IndexInfo{ IndexName: "b", TableName: "test", - Path: parsePath(t, "b"), + Paths: []document.Path{parsePath(t, "b")}, }) require.NoError(t, err) @@ -537,7 +537,7 @@ func TestCatalogReIndex(t *testing.T) { return catalog.CreateIndex(tx, &database.IndexInfo{ IndexName: "b", TableName: "test", - Path: parsePath(t, "b"), + Paths: []document.Path{parsePath(t, "b")}, }) }) @@ -570,7 +570,7 @@ func TestCatalogReIndex(t *testing.T) { require.NoError(t, err) var i int - err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error { var buf bytes.Buffer err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i))) require.NoError(t, err) @@ -638,13 +638,13 @@ func TestReIndexAll(t *testing.T) { err = catalog.CreateIndex(tx, &database.IndexInfo{ IndexName: "t1a", TableName: "test1", - Path: parsePath(t, "a"), + Paths: []document.Path{parsePath(t, "a")}, }) require.NoError(t, err) err = catalog.CreateIndex(tx, &database.IndexInfo{ IndexName: "t2a", TableName: "test2", - Path: parsePath(t, "a"), + Paths: []document.Path{parsePath(t, "a")}, }) require.NoError(t, err) @@ -660,7 +660,7 @@ func TestReIndexAll(t *testing.T) { require.NoError(t, err) var i int - err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error { var buf bytes.Buffer err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i))) require.NoError(t, err) @@ -676,7 +676,7 @@ func TestReIndexAll(t *testing.T) { require.NoError(t, err) i = 0 - err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error { var buf bytes.Buffer err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i))) require.NoError(t, err) diff --git a/database/config.go b/database/config.go index 7e1efac99..35612d0cf 100644 --- a/database/config.go +++ b/database/config.go @@ -230,7 +230,7 @@ func (t *tableStore) Replace(tx *Transaction, tableName string, info *TableInfo) type IndexInfo struct { TableName string IndexName string - Path document.Path + Paths []document.Path // If set to true, values will be associated with at most one key. False by default. Unique bool @@ -246,9 +246,21 @@ func (i *IndexInfo) ToDocument() document.Document { buf.Add("unique", document.NewBoolValue(i.Unique)) buf.Add("index_name", document.NewTextValue(i.IndexName)) buf.Add("table_name", document.NewTextValue(i.TableName)) - buf.Add("path", document.NewArrayValue(pathToArray(i.Path))) - if i.Type != 0 { - buf.Add("type", document.NewIntegerValue(int64(i.Type))) + + // TODO check that + vb := document.NewValueBuffer() + for _, path := range i.Paths { + vb.Append(document.NewArrayValue(pathToArray(path))) + } + + buf.Add("paths", document.NewArrayValue(vb)) + // TODO check that + if i.Types != nil { + types := make([]document.Value, 0, len(i.Types)) + for _, typ := range i.Types { + types = append(types, document.NewIntegerValue(int64(typ))) + } + buf.Add("types", document.NewArrayValue(document.NewValueBuffer(types...))) } return buf } @@ -273,21 +285,45 @@ func (i *IndexInfo) ScanDocument(d document.Document) error { } i.TableName = string(v.V.(string)) - v, err = d.GetByField("path") + v, err = d.GetByField("paths") if err != nil { return err } - i.Path, err = arrayToPath(v.V.(document.Array)) + + err = v.V.(document.Array).Iterate(func(ii int, pval document.Value) error { + p, err := arrayToPath(pval.V.(document.Array)) + if err != nil { + return err + } + + i.Paths = append(i.Paths, p) + return nil + }) + if err != nil { return err } - v, err = d.GetByField("type") + // i.Paths, err = arrayToPath(v.V.(document.Array)) + // if err != nil { + // return err + // } + + v, err = d.GetByField("types") if err != nil && err != document.ErrFieldNotFound { return err } + + // TODO refacto if err == nil { - i.Type = document.ValueType(v.V.(int64)) + err = v.V.(document.Array).Iterate(func(ii int, tval document.Value) error { + i.Types = append(i.Types, document.ValueType(tval.V.(int64))) + return nil + }) + + if err != nil { + return err + } } return nil @@ -407,7 +443,8 @@ func (i Indexes) GetIndex(name string) *Index { func (i Indexes) GetIndexByPath(p document.Path) *Index { for _, idx := range i { - if idx.Info.Path.IsEqual(p) { + // TODO + if idx.Info.Paths[0].IsEqual(p) { return idx } } diff --git a/database/config_test.go b/database/config_test.go index c5316493c..421fca159 100644 --- a/database/config_test.go +++ b/database/config_test.go @@ -163,7 +163,8 @@ func TestIndexStore(t *testing.T) { TableName: "test", IndexName: "idx_test", Unique: true, - Type: document.BoolValue, + // TODO + Types: []document.ValueType{document.BoolValue}, } err = idxs.Insert(&cfg) diff --git a/database/index.go b/database/index.go index e81d82d48..f9e7ff5e5 100644 --- a/database/index.go +++ b/database/index.go @@ -13,6 +13,12 @@ import ( const ( // indexStorePrefix is the prefix used to name the index stores. indexStorePrefix = "i" + + // untypedValue is the placeholder type for keys of an index which aren't typed. + // CREATE TABLE foo; + // CREATE INDEX idx_foo_a_b ON foo(a,b); + // document.ValueType of a and b will be untypedValue. + untypedValue = document.ValueType(0) ) var ( @@ -33,12 +39,12 @@ type Index struct { func NewIndex(tx engine.Transaction, idxName string, opts *IndexInfo) *Index { if opts == nil { opts = &IndexInfo{ - Types: []document.ValueType{document.ValueType(0)} + Types: []document.ValueType{untypedValue}, } } if opts.Types == nil { - opts.Types= []document.ValueType{document.ValueType(0)} + opts.Types = []document.ValueType{untypedValue} } return &Index{ @@ -50,6 +56,14 @@ func NewIndex(tx engine.Transaction, idxName string, opts *IndexInfo) *Index { var errStop = errors.New("stop") +func (idx *Index) IsComposite() bool { + return len(idx.Info.Types) > 1 +} + +func (idx *Index) Arity() int { + return len(idx.Info.Types) +} + // Set associates a value with a key. If Unique is set to false, it is // possible to associate multiple keys for the same value // but a key can be associated to only one value. @@ -64,13 +78,13 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { return errors.New("cannot index without a value") } - if len(vs) > len(idx.Types) { - return errors.New("cannot index more values than what the index supports") + if len(vs) != len(idx.Info.Types) { + return fmt.Errorf("cannot index %d values on an index of arity %d", len(vs), len(idx.Info.Types)) } - for i, typ := range idx.Types { + for i, typ := range idx.Info.Types { // it is possible to set an index(a,b) on (a), it will be assumed that b is null in that case - if typ != 0 && i < len(vs) && typ != vs[i].Type { + if typ != untypedValue && i < len(vs) && typ != vs[i].Type { // TODO use the full version to clarify the error return fmt.Errorf("cannot index value of type %s in %s index", vs[i].Type, typ) } @@ -82,7 +96,14 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { } // encode the value we are going to use as a key - buf, err := idx.EncodeValues(vs) + var buf []byte + if len(vs) > 1 { + wrappedVs := document.NewValueBuffer(vs...) + buf, err = idx.EncodeValue(document.NewArrayValue(wrappedVs)) + } else { + buf, err = idx.EncodeValue(vs[0]) + } + if err != nil { return err } @@ -179,13 +200,20 @@ func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func( return errors.New("cannot iterate without a pivot") } - if len(pivots) > len(idx.Types) { - return errors.New("cannot iterate with more values than what the index supports") - } + // if len(pivots) != len(idx.Info.Types) { + // return errors.New("cannot iterate without the same number of values than what the index supports") + // // return errors.New("cannot iterate with more values than what the index supports") + // } - for i, typ := range idx.Types { + for i, typ := range idx.Info.Types { // if index and pivot are typed but not of the same type // return no result + // + // don't try to check in case we have less pivots than values + if i >= len(pivots) { + break + } + if typ != 0 && pivots[i].Type != 0 && typ != pivots[i].Type { return nil } @@ -237,56 +265,55 @@ func (idx *Index) Truncate() error { // the presence of other types. // If not, encode so that order is preserved regardless of the type. func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { - if idx.Types[0] != 0 { - return v.MarshalBinary() - } - - var err error - var buf bytes.Buffer - err = document.NewValueEncoder(&buf).Encode(v) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// TODO -func (idx *Index) EncodeValues(vs []document.Value) ([]byte, error) { - buf := []byte{} - - for i, v := range vs { - if idx.Types[i] != 0 { - b, err := v.MarshalBinary() - if err != nil { - return nil, err + if idx.IsComposite() { + // v has been turned into an array of values being indexed + // TODO add a check + array := v.V.(*document.ValueBuffer) + + // in the case of one of the index keys being untyped and the corresponding + // value being an integer, convert it into a double. + err := array.Iterate(func(i int, vi document.Value) error { + if idx.Info.Types[i] != untypedValue { + return nil } - buf = append(buf, b...) - continue - } - - var err error - if v.Type == document.IntegerValue { - if v.V == nil { - v.Type = document.DoubleValue - } else { - v, err = v.CastAsDouble() - if err != nil { - return nil, err + var err error + if vi.Type == document.IntegerValue { + if vi.V == nil { + vi.Type = document.DoubleValue + } else { + vi, err = vi.CastAsDouble() + if err != nil { + return err + } } + + // update the value with its new type + return array.Replace(i, vi) } - } - var bbuf bytes.Buffer - err = document.NewValueEncoder(&bbuf).Encode(v) + return nil + }) + if err != nil { return nil, err } - b := bbuf.Bytes() - buf = append(buf, b...) + + // encode the array + return v.MarshalBinary() } - return buf, nil + if idx.Info.Types[0] != 0 { + return v.MarshalBinary() + } + + var err error + var buf bytes.Buffer + err = document.NewValueEncoder(&buf).Encode(v) + if err != nil { + return nil, err + } + return buf.Bytes(), nil } func getOrCreateStore(tx engine.Transaction, name []byte) (engine.Store, error) { @@ -311,7 +338,7 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool var seek []byte var err error - for i, typ := range idx.Types { + for i, typ := range idx.Info.Types { if typ == 0 && pivots[i].Type == document.IntegerValue { if pivots[i].V == nil { pivots[i].Type = document.DoubleValue @@ -324,35 +351,59 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool } } - if pivots[0].V != nil { - seek, err = idx.EncodeValues(pivots) - if err != nil { - return err + if idx.IsComposite() { + // if we have n valueless and typeless pivots, we just iterate + all := true + for _, pivot := range pivots { + if pivot.Type == 0 && pivot.V == nil { + all = all && true + } else { + all = false + } } - if reverse { - seek = append(seek, 0xFF) - } - } else { - // this is pretty surely wrong as it does not allow to select t1t2 is there are values on t1 - buf := []byte{} - for i, typ := range idx.Types { - if typ == 0 && pivots[i].Type != 0 && pivots[i].V == nil { - buf = append(buf, byte(pivots[i].Type)) + vb := document.NewValueBuffer(pivots...) + + // we do have pivot values, so let's use them to seek in the index + if !all { + seek, err = idx.EncodeValue(document.NewArrayValue(vb)) + + if err != nil { + return err } + } else { // we don't, let's start at the beginning + seek = []byte{} } - seek = buf if reverse { - seek = append(seek, 0xFF) + // if we are reverse on a pivot with less arity, we will get 30 255, which is lower than 31 + // and such will ignore all values. Let's drop the separator in that case + if len(seek) > 0 { + seek = append(seek[:len(seek)-1], 0xFF) + } else { + seek = append(seek, 0xFF) + } + } - // if idx.Type == 0 && pivot.Type != 0 && pivot.V == nil { - // seek = []byte{byte(pivot.Type)} + } else { + if pivots[0].V != nil { + seek, err = idx.EncodeValue(pivots[0]) + if err != nil { + return err + } - // if reverse { - // seek = append(seek, 0xFF) - // } - // } + if reverse { + seek = append(seek, 0xFF) + } + } else { + if idx.Info.Types[0] == untypedValue && pivots[0].Type != untypedValue && pivots[0].V == nil { + seek = []byte{byte(pivots[0].Type)} + + if reverse { + seek = append(seek, 0xFF) + } + } + } } it := st.Iterator(engine.IteratorOptions{Reverse: reverse}) @@ -367,7 +418,7 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool // } // this is wrong and only handle the first type - for i, typ := range idx.Types { + for i, typ := range idx.Info.Types { if typ == 0 && pivots[i].Type != 0 && itm.Key()[0] != byte(pivots[i].Type) { return nil } diff --git a/database/index_test.go b/database/index_test.go index 54fdfd255..b0dba64e2 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -20,35 +20,227 @@ func values(vs ...document.Value) []document.Value { return vs } -func getIndex(t testing.TB, unique bool) (*database.Index, func()) { +func prefixAV(b []byte) []byte { + return append([]byte{byte(document.ArrayValue)}, b...) +} + +func getIndex(t testing.TB, unique bool, types ...document.ValueType) (*database.Index, func()) { ng := memoryengine.NewEngine() tx, err := ng.Begin(context.Background(), engine.TxOptions{ Writable: true, }) require.NoError(t, err) - idx := database.NewIndex(tx, "foo", &database.IndexInfo{Unique: unique}) + idx := database.NewIndex(tx, "foo", &database.IndexInfo{Unique: unique, Types: types}) return idx, func() { tx.Rollback() } } +func TestCompositeIndexSet(t *testing.T) { + idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) + defer cleanup() + require.Error(t, idx.Set(values(document.NewIntegerValue(42), document.NewIntegerValue(43)), nil)) + require.NoError(t, idx.Set(values(document.NewIntegerValue(42), document.NewIntegerValue(43)), []byte("key"))) +} + +func TestCompositeIndexIterateAsc1(t *testing.T) { + idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) + defer cleanup() + + for i := 0; i < 10; i++ { + require.NoError(t, idx.Set(values( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(100-i))), + []byte{'a' + byte(i)})) + } + + var i int = 2 + var count int + pivots := values(document.NewIntegerValue(int64(2)), document.NewIntegerValue(0)) + err := idx.AscendGreaterOrEqual(pivots, func(val, rid []byte) error { + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(100-i)), + ) + + val = prefixAV(val) + // the missing array type is annoying + // vvv := document.Value{Type: document.ArrayValue} + // e := vvv.UnmarshalBinary(val) + // require.NoError(t, e) + // fmt.Println(vvv.V) + + requireEqualEncoded(t, document.NewArrayValue(array), val) + require.Equal(t, []byte{'a' + byte(i)}, rid) + + i++ + count++ + return nil + }) + + require.NoError(t, err) + require.Equal(t, 8, count) +} + +func TestCompositeIndexIterateAsc2(t *testing.T) { + idx, cleanup := getIndex(t, false, document.ValueType(0), document.IntegerValue) + defer cleanup() + + for i := 0; i < 10; i++ { + require.NoError(t, idx.Set(values( + document.NewDoubleValue(float64(i)), + document.NewIntegerValue(int64(100-i))), + binarysort.AppendInt64(nil, int64(i)))) + + require.NoError(t, idx.Set(values( + document.NewTextValue(strconv.Itoa(i)), + document.NewIntegerValue(int64(100-i))), + binarysort.AppendInt64(nil, int64(i)))) + } + + var i int = 2 + var count int + pivots := values(document.NewIntegerValue(int64(2)), document.NewIntegerValue(0)) + err := idx.AscendGreaterOrEqual(pivots, func(val, rid []byte) error { + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewIntegerValue(int64(100-i)), + ) + + // deal with the skipped array value type byte + vval := []byte{byte(document.ArrayValue)} + vval = append(vval, val...) + + requireEqualEncoded(t, document.NewArrayValue(array), vval) + require.Equal(t, binarysort.AppendInt64(nil, int64(i)), rid) + + i++ + count++ + return nil + }) + + require.NoError(t, err) + require.Equal(t, 8, count) +} + +func TestCompositeIndexIterateDesc(t *testing.T) { + idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) + defer cleanup() + + for i := 0; i < 10; i++ { + require.NoError(t, idx.Set(values( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(100-i))), + []byte{'a' + byte(i)})) + } + + var i int = 4 + var count int + pivots := values(document.NewIntegerValue(int64(4)), document.NewIntegerValue(96)) + err := idx.DescendLessOrEqual(pivots, func(val, rid []byte) error { + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(100-i)), + ) + + // deal with the skipped array value type byte + vval := []byte{byte(document.ArrayValue)} + vval = append(vval, val...) + + requireEqualEncoded(t, document.NewArrayValue(array), vval) + require.Equal(t, []byte{'a' + byte(i)}, rid) + + i-- + count++ + return nil + }) + + require.NoError(t, err) + require.Equal(t, 5, count) +} + +func TestCompositeIndexIterateDesc2(t *testing.T) { + idx, cleanup := getIndex(t, false, document.ValueType(0), document.IntegerValue) + defer cleanup() + + for i := 0; i < 10; i++ { + require.NoError(t, idx.Set(values( + document.NewDoubleValue(float64(i)), + document.NewIntegerValue(int64(100-i))), + binarysort.AppendInt64(nil, int64(i)))) + + require.NoError(t, idx.Set(values( + document.NewTextValue(strconv.Itoa(i)), + document.NewIntegerValue(int64(100-i))), + binarysort.AppendInt64(nil, int64(i)))) + } + + var i int = 4 + var count int + pivots := values(document.NewIntegerValue(int64(4)), document.NewIntegerValue(96)) + err := idx.DescendLessOrEqual(pivots, func(val, rid []byte) error { + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewIntegerValue(int64(100-i)), + ) + + // deal with the skipped array value type byte + vval := []byte{byte(document.ArrayValue)} + vval = append(vval, val...) + + requireEqualEncoded(t, document.NewArrayValue(array), vval) + require.Equal(t, binarysort.AppendInt64(nil, int64(i)), rid) + + i-- + count++ + return nil + }) + + require.NoError(t, err) + require.Equal(t, 5, count) +} + func TestIndexSet(t *testing.T) { for _, unique := range []bool{true, false} { text := fmt.Sprintf("Unique: %v, ", unique) - t.Run(text+"Set nil key falls", func(t *testing.T) { + t.Run(text+"Set nil key falls (arity=1)", func(t *testing.T) { idx, cleanup := getIndex(t, unique) defer cleanup() require.Error(t, idx.Set(values(document.NewBoolValue(true)), nil)) }) - t.Run(text+"Set value and key succeeds", func(t *testing.T) { + t.Run(text+"Set value and key succeeds (arity=1)", func(t *testing.T) { idx, cleanup := getIndex(t, unique) defer cleanup() require.NoError(t, idx.Set(values(document.NewBoolValue(true)), []byte("key"))) }) + + t.Run(text+"Set two values and key succeeds (arity=2)", func(t *testing.T) { + idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) + defer cleanup() + require.NoError(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) + }) + + t.Run(text+"Set one value and key on two values index fails", func(t *testing.T) { + idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) + defer cleanup() + require.Error(t, idx.Set(values(document.NewBoolValue(true)), []byte("key"))) + }) + + t.Run(text+"Set two values and key on non composite index fails", func(t *testing.T) { + idx, cleanup := getIndex(t, unique, document.ValueType(0)) + defer cleanup() + require.Error(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) + }) + + t.Run(text+"Set three values and key on two values index fails", func(t *testing.T) { + idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) + defer cleanup() + require.Error(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) + }) } t.Run("Unique: true, Duplicate", func(t *testing.T) { @@ -57,20 +249,29 @@ func TestIndexSet(t *testing.T) { require.NoError(t, idx.Set(values(document.NewIntegerValue(10)), []byte("key"))) require.NoError(t, idx.Set(values(document.NewIntegerValue(11)), []byte("key"))) - require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(document.NewIntegerValue(10), []byte("key"))) + require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(values(document.NewIntegerValue(10)), []byte("key"))) }) t.Run("Unique: true, Type: integer Duplicate", func(t *testing.T) { - idx, cleanup := getIndex(t, true) - idx.Info.Types = []document.ValueType{document.IntegerValue} + idx, cleanup := getIndex(t, true, document.IntegerValue) defer cleanup() require.NoError(t, idx.Set(values(document.NewIntegerValue(10)), []byte("key"))) require.NoError(t, idx.Set(values(document.NewIntegerValue(11)), []byte("key"))) - require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(document.NewIntegerValue(10), []byte("key"))) + require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(values(document.NewIntegerValue(10)), []byte("key"))) + }) + + t.Run("Unique: true, Type: (integer, integer) Duplicate,", func(t *testing.T) { + idx, cleanup := getIndex(t, true, document.IntegerValue, document.IntegerValue) + defer cleanup() + + require.NoError(t, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(10)), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(11), document.NewIntegerValue(11)), []byte("key"))) + require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(10)), []byte("key"))) }) } +// TODO add composite test cases func TestIndexDelete(t *testing.T) { t.Run("Unique: false, Delete valid key succeeds", func(t *testing.T) { idx, cleanup := getIndex(t, false) @@ -171,28 +372,61 @@ func TestIndexAscendGreaterThan(t *testing.T) { t.Run(text+"With typed empty pivot, should iterate over all documents of the pivot type in order", func(t *testing.T) { tests := []struct { - name string - val func(i int) document.Value - t document.ValueType - pivot document.Value + name string + val func(i int) []document.Value + t document.ValueType + indexTypes []document.ValueType + pivots []document.Value }{ - {"integers", func(i int) document.Value { return document.NewIntegerValue(int64(i)) }, document.IntegerValue, document.Value{Type: document.IntegerValue}}, - {"doubles", func(i int) document.Value { return document.NewDoubleValue(float64(i) + float64(i)/2) }, document.DoubleValue, document.Value{Type: document.DoubleValue}}, - {"texts", func(i int) document.Value { return document.NewTextValue(strconv.Itoa(i)) }, document.TextValue, document.Value{Type: document.TextValue}}, + {"integer", + func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + document.IntegerValue, + nil, + values(document.Value{Type: document.IntegerValue})}, + {"double", + func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + document.DoubleValue, + nil, + values(document.Value{Type: document.DoubleValue})}, + {"text", + func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + document.TextValue, + nil, + values(document.Value{Type: document.TextValue})}, + {"(integer, integer)", + func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + document.ArrayValue, // of integers + []document.ValueType{0, 0}, + values(document.Value{Type: document.IntegerValue, V: int64(0)}, document.Value{Type: document.IntegerValue, V: int64(0)})}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - idx, cleanup := getIndex(t, unique) + idx, cleanup := getIndex(t, unique, test.indexTypes...) defer cleanup() for i := 0; i < 10; i += 2 { - require.NoError(t, idx.Set(values(test.val(i)), []byte{'a' + byte(i)})) + require.NoError(t, idx.Set(test.val(i), []byte{'a' + byte(i)})) + + // the case where we have other types in the list was missing from this test + // ugly, TODO refactor + ovs := []document.Value{} + for range test.indexTypes { + ovs = append(ovs, document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + } + + if test.indexTypes == nil { + ovs = append(ovs, document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + } + + require.NoError(t, idx.Set(ovs, []byte{'a' + byte(i)})) } var i uint8 var count int - err := idx.AscendGreaterOrEqual(values(test.pivot), func(val, rid []byte) error { + err := idx.AscendGreaterOrEqual(test.pivots, func(val, rid []byte) error { switch test.t { case document.IntegerValue: requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) @@ -202,6 +436,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) case document.BoolValue: requireEqualEncoded(t, document.NewBoolValue(i > 0), val) + case document.ArrayValue: // of integers + array := document.NewValueBuffer(document.NewDoubleValue(float64(i)), document.NewDoubleValue(float64(i+1))) + requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) } require.Equal(t, []byte{'a' + i}, rid) @@ -238,24 +475,65 @@ func TestIndexAscendGreaterThan(t *testing.T) { require.Equal(t, 4, count) }) - t.Run(text+"With no pivot, should iterate over all documents in order, regardless of their type", func(t *testing.T) { - idx, cleanup := getIndex(t, unique) + t.Run(text+"With composite pivot, should iterate over some documents in order", func(t *testing.T) { + + idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) + defer cleanup() + + for i := 0; i < 10; i++ { + require.NoError(t, idx.Set(values( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(100-i))), + []byte{'a' + byte(i)})) + } + + var i int = 2 + var count int + pivots := values(document.NewIntegerValue(int64(2)), document.NewIntegerValue(0)) + err := idx.AscendGreaterOrEqual(pivots, func(val, rid []byte) error { + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(100-i)), + ) + + requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) + require.Equal(t, []byte{'a' + byte(i)}, rid) + + i++ + count++ + return nil + }) + + require.NoError(t, err) + require.Equal(t, 8, count) + }) + + t.Run(text+"With no pivot but a composite index, should iterate over all documents in order, regardless of their type", func(t *testing.T) { + idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) defer cleanup() for i := int64(0); i < 10; i++ { - require.NoError(t, idx.Set(values(document.NewDoubleValue(float64(i))), []byte{'d', 'a' + byte(i)})) - require.NoError(t, idx.Set(values(document.NewTextValue(strconv.Itoa(int(i)))), []byte{'s', 'a' + byte(i)})) + require.NoError(t, idx.Set(values(document.NewDoubleValue(float64(i)), document.NewBlobValue([]byte{byte(i)})), []byte{'d', 'a' + byte(i)})) + require.NoError(t, idx.Set(values(document.NewTextValue(strconv.Itoa(int(i))), document.NewDoubleValue(float64(i))), []byte{'s', 'a' + byte(i)})) } var doubles, texts int var count int - err := idx.AscendGreaterOrEqual(values(document.Value{}), func(val, rid []byte) error { + err := idx.AscendGreaterOrEqual(values(document.Value{}, document.Value{}), func(val, rid []byte) error { if count < 10 { - requireEqualEncoded(t, document.NewDoubleValue(float64(doubles)), val) + array := document.NewValueBuffer( + document.NewDoubleValue(float64(doubles)), + document.NewBlobValue([]byte{byte(doubles)})) + + requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) require.Equal(t, []byte{'d', 'a' + byte(doubles)}, rid) doubles++ } else { - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(texts))), val) + array := document.NewValueBuffer( + document.NewTextValue(strconv.Itoa(int(texts))), + document.NewDoubleValue(float64(texts))) + + requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) require.Equal(t, []byte{'s', 'a' + byte(texts)}, rid) texts++ } @@ -267,7 +545,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { require.Equal(t, 10, texts) }) - t.Run(text+"With no pivot and typed index, should iterate over all documents in order", func(t *testing.T) { + t.Run(text+"With no pivot and a typed index, should iterate over all documents in order", func(t *testing.T) { idx, cleanup := getIndex(t, unique) idx.Info.Types = []document.ValueType{document.IntegerValue} defer cleanup() @@ -455,7 +733,7 @@ func BenchmarkIndexIteration(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _ = idx.AscendGreaterOrEqual(document.Value{Type: document.TextValue}, func(_, _ []byte) error { + _ = idx.AscendGreaterOrEqual(values(document.Value{Type: document.TextValue}), func(_, _ []byte) error { return nil }) } diff --git a/database/table.go b/database/table.go index f1046ec92..fdb12bb72 100644 --- a/database/table.go +++ b/database/table.go @@ -92,12 +92,18 @@ func (t *Table) Insert(d document.Document) (document.Document, error) { indexes := t.Indexes() for _, idx := range indexes { - v, err := idx.Info.Path.GetValueFromDocument(fb) - if err != nil { - v = document.NewNullValue() + vals := make([]document.Value, len(idx.Info.Paths)) + + for i, path := range idx.Info.Paths { + v, err := path.GetValueFromDocument(fb) + if err != nil { + v = document.NewNullValue() + } + + vals[i] = v } - err = idx.Set(v, key) + err = idx.Set(vals, key) if err != nil { if err == ErrIndexDuplicateValue { return nil, ErrDuplicateDocument @@ -136,12 +142,13 @@ func (t *Table) Delete(key []byte) error { indexes := t.Indexes() for _, idx := range indexes { - v, err := idx.Info.Path.GetValueFromDocument(d) + // TODO only support one path + v, err := idx.Info.Paths[0].GetValueFromDocument(d) if err != nil { return err } - err = idx.Delete(v, key) + err = idx.Delete([]document.Value{v}, key) if err != nil { return err } @@ -179,12 +186,13 @@ func (t *Table) replace(indexes []*Index, key []byte, d document.Document) error // remove key from indexes for _, idx := range indexes { - v, err := idx.Info.Path.GetValueFromDocument(old) + // TODO only support one path + v, err := idx.Info.Paths[0].GetValueFromDocument(old) if err != nil { v = document.NewNullValue() } - err = idx.Delete(v, key) + err = idx.Delete([]document.Value{v}, key) if err != nil { return err } @@ -207,12 +215,13 @@ func (t *Table) replace(indexes []*Index, key []byte, d document.Document) error // update indexes for _, idx := range indexes { - v, err := idx.Info.Path.GetValueFromDocument(d) + // only support one path + v, err := idx.Info.Paths[0].GetValueFromDocument(d) if err != nil { v = document.NewNullValue() } - err = idx.Set(v, key) + err = idx.Set([]document.Value{v}, key) if err != nil { if err == ErrIndexDuplicateValue { return ErrDuplicateDocument diff --git a/database/table_test.go b/database/table_test.go index a0606e93a..88e9bd4a8 100644 --- a/database/table_test.go +++ b/database/table_test.go @@ -295,7 +295,7 @@ func TestTableInsert(t *testing.T) { require.NoError(t, err) err = tx.CreateIndex(&database.IndexInfo{ - IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"), + IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")}, }) require.NoError(t, err) idx, err := tx.GetIndex("idxFoo") @@ -318,7 +318,7 @@ func TestTableInsert(t *testing.T) { require.NoError(t, err) var count int - err = idx.AscendGreaterOrEqual(document.Value{}, func(val, k []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{document.Value{}}, func(val, k []byte) error { switch count { case 0: // key2, which doesn't countain the field must appear first in the next, @@ -739,21 +739,21 @@ func TestTableIndexes(t *testing.T) { Unique: true, IndexName: "idx1a", TableName: "test1", - Path: parsePath(t, "a"), + Paths: []document.Path{parsePath(t, "a")}, }) require.NoError(t, err) err = tx.CreateIndex(&database.IndexInfo{ Unique: false, IndexName: "idx1b", TableName: "test1", - Path: parsePath(t, "b"), + Paths: []document.Path{parsePath(t, "b")}, }) require.NoError(t, err) err = tx.CreateIndex(&database.IndexInfo{ Unique: false, IndexName: "ifx2a", TableName: "test2", - Path: parsePath(t, "a"), + Paths: []document.Path{parsePath(t, "a")}, }) require.NoError(t, err) diff --git a/planner/optimizer.go b/planner/optimizer.go index 0fcc2aabc..d1770129c 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -547,7 +547,7 @@ func getCandidateFromfilterNode(f *stream.FilterOperator, tableName string, info // if not, check if an index exists for that path if idx := indexes.GetIndexByPath(document.Path(path)); idx != nil { // check if the operand can be used and convert it when possible - v, ok, err := operandCanUseIndex(idx.Info.Type, idx.Info.Path, info.FieldConstraints, v) + v, ok, err := operandCanUseIndex(idx.Info.Types[0], idx.Info.Paths[0], info.FieldConstraints, v) if err != nil || !ok { return nil, err } diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index 2dacce5ae..33b7793c4 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -271,6 +271,14 @@ func TestRemoveUnnecessaryDedupNodeRule(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Project(parser.MustParseExpr("pk()"))), }, + { + "lolol", + stream.New(stream.SeqScan("foo")). + Pipe(stream.Project(parser.MustParseExpr("c as C"))). + Pipe(stream.Distinct()), + stream.New(stream.SeqScan("foo")). + Pipe(stream.Project(parser.MustParseExpr("c as C"))), + }, } for _, test := range tests { diff --git a/query/create.go b/query/create.go index 8d674dc4a..812e610dc 100644 --- a/query/create.go +++ b/query/create.go @@ -32,9 +32,9 @@ func (stmt CreateTableStmt) Run(tx *database.Transaction, args []expr.Param) (Re if fc.IsUnique { err = tx.CreateIndex(&database.IndexInfo{ TableName: stmt.TableName, - Path: fc.Path, + Paths: []document.Path{fc.Path}, Unique: true, - Type: fc.Type, + Types: []document.ValueType{fc.Type}, }) if err != nil { return res, err @@ -50,7 +50,7 @@ func (stmt CreateTableStmt) Run(tx *database.Transaction, args []expr.Param) (Re type CreateIndexStmt struct { IndexName string TableName string - Path document.Path + Paths []document.Path IfNotExists bool Unique bool } @@ -69,7 +69,7 @@ func (stmt CreateIndexStmt) Run(tx *database.Transaction, args []expr.Param) (Re Unique: stmt.Unique, IndexName: stmt.IndexName, TableName: stmt.TableName, - Path: stmt.Path, + Paths: stmt.Paths, }) if stmt.IfNotExists && err == database.ErrIndexAlreadyExists { err = nil diff --git a/query/create_test.go b/query/create_test.go index 0e49cc692..c15210fb8 100644 --- a/query/create_test.go +++ b/query/create_test.go @@ -288,17 +288,17 @@ func TestCreateTable(t *testing.T) { idx, err := tx.GetIndex("__genji_autoindex_test_1") require.NoError(t, err) - require.Equal(t, document.IntegerValue, idx.Info.Type) + require.Equal(t, document.IntegerValue, idx.Info.Types[0]) require.True(t, idx.Info.Unique) idx, err = tx.GetIndex("__genji_autoindex_test_2") require.NoError(t, err) - require.Equal(t, document.DoubleValue, idx.Info.Type) + require.Equal(t, document.DoubleValue, idx.Info.Types[0]) require.True(t, idx.Info.Unique) idx, err = tx.GetIndex("__genji_autoindex_test_3") require.NoError(t, err) - require.Zero(t, idx.Info.Type) + require.Zero(t, idx.Info.Types[0]) require.True(t, idx.Info.Unique) return nil }) diff --git a/query/reindex_test.go b/query/reindex_test.go index 0f0548ea6..5fbe38103 100644 --- a/query/reindex_test.go +++ b/query/reindex_test.go @@ -82,7 +82,7 @@ func TestReIndex(t *testing.T) { } i := 0 - err = idx.AscendGreaterOrEqual(document.Value{}, func(val []byte, key []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{document.Value{}}, func(val []byte, key []byte) error { i++ return nil }) diff --git a/query/select_test.go b/query/select_test.go index 6408b302d..fab9d826f 100644 --- a/query/select_test.go +++ b/query/select_test.go @@ -32,7 +32,7 @@ func TestSelectStmt(t *testing.T) { {"No table, wildcard", "SELECT *", true, ``, nil}, {"No table, document", "SELECT {a: 1, b: 2 + 1}", false, `[{"{a: 1, b: 2 + 1}":{"a":1,"b":3}}]`, nil}, {"No cond", "SELECT * FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100},{"k":3,"height":100,"weight":200}]`, nil}, - {"No cond Multiple wildcards", "SELECT *, *, color FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square","k":1,"color":"red","size":10,"shape":"square","color":"red"},{"k":2,"color":"blue","size":10,"weight":100,"k":2,"color":"blue","size":10,"weight":100,"color":"blue"},{"k":3,"height":100,"weight":200,"k":3,"height":100,"weight":200,"color":null}]`, nil}, + {"No con d Multiple wildcards", "SELECT *, *, color FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square","k":1,"color":"red","size":10,"shape":"square","color":"red"},{"k":2,"color":"blue","size":10,"weight":100,"k":2,"color":"blue","size":10,"weight":100,"color":"blue"},{"k":3,"height":100,"weight":200,"k":3,"height":100,"weight":200,"color":null}]`, nil}, {"With fields", "SELECT color, shape FROM test", false, `[{"color":"red","shape":"square"},{"color":"blue","shape":null},{"color":null,"shape":null}]`, nil}, {"No cond, wildcard and other field", "SELECT *, color FROM test", false, `[{"color": "red", "k": 1, "color": "red", "size": 10, "shape": "square"}, {"color": "blue", "k": 2, "color": "blue", "size": 10, "weight": 100}, {"color": null, "k": 3, "height": 100, "weight": 200}]`, nil}, {"With DISTINCT", "SELECT DISTINCT * FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100},{"k":3,"height":100,"weight":200}]`, nil}, diff --git a/sql/parser/create.go b/sql/parser/create.go index a22de8b5d..e11cedfb8 100644 --- a/sql/parser/create.go +++ b/sql/parser/create.go @@ -331,11 +331,12 @@ func (p *Parser) parseCreateIndexStatement(unique bool) (query.CreateIndexStmt, return stmt, newParseError(scanner.Tokstr(tok, lit), []string{"("}, pos) } - if len(paths) != 1 { - return stmt, &ParseError{Message: "indexes on more than one path are not supported"} - } + // TODO + // if len(paths) != 1 { + // return stmt, &ParseError{Message: "indexes on more than one path are not supported"} + // } - stmt.Path = paths[0] + stmt.Paths = paths return stmt, nil } diff --git a/stream/iterator.go b/stream/iterator.go index 82e5a4132..6f86a829c 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -332,7 +332,7 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env return err } - var iterator func(pivot document.Value, fn func(val, key []byte) error) error + var iterator func(pivots []document.Value, fn func(val, key []byte) error) error if !it.Reverse { iterator = index.AscendGreaterOrEqual @@ -342,7 +342,8 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env // if there are no ranges use a simpler and faster iteration function if len(it.Ranges) == 0 { - return iterator(document.Value{}, func(val, key []byte) error { + vs := make([]document.Value, len(index.Info.Types)) + return iterator(vs, func(val, key []byte) error { d, err := table.GetDocument(key) if err != nil { return err @@ -354,52 +355,119 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } for _, rng := range it.Ranges { - var start, end document.Value - if !it.Reverse { - start = rng.Min - end = rng.Max - } else { - start = rng.Max - end = rng.Min - } - var encEnd []byte - if !end.Type.IsZero() && end.V != nil { - encEnd, err = index.EncodeValue(end) - if err != nil { - return err + if index.IsComposite() { + var start, end document.Value + if !it.Reverse { + start = rng.Min + end = rng.Max + } else { + start = rng.Max + end = rng.Min } - } - err = iterator(start, func(val, key []byte) error { - if !rng.IsInRange(val) { - // if we reached the end of our range, we can stop iterating. - if encEnd == nil { + var encEnd []byte + + // deal with the fact that we can't have a zero then values + // TODO(JH) + if !end.Type.IsZero() && end.V != nil { + encEnd, err = index.EncodeValue(end) + if err != nil { + return err + } + } + + pivots := []document.Value{} + if start.V != nil { + start.V.(document.Array).Iterate(func(i int, value document.Value) error { + pivots = append(pivots, value) return nil + }) + } else { + for i := 0; i < index.Arity(); i++ { + pivots = append(pivots, document.Value{}) } - cmp := bytes.Compare(val, encEnd) - if !it.Reverse && cmp > 0 { - return ErrStreamClosed + } + + err = iterator(pivots, func(val, key []byte) error { + if !rng.IsInRange(val) { + // if we reached the end of our range, we can stop iterating. + if encEnd == nil { + return nil + } + cmp := bytes.Compare(val, encEnd) + if !it.Reverse && cmp > 0 { + return ErrStreamClosed + } + if it.Reverse && cmp < 0 { + return ErrStreamClosed + } + return nil } - if it.Reverse && cmp < 0 { - return ErrStreamClosed + + d, err := table.GetDocument(key) + if err != nil { + return err } - return nil - } - d, err := table.GetDocument(key) + newEnv.SetDocument(d) + return fn(&newEnv) + }) + if err == ErrStreamClosed { + err = nil + } if err != nil { return err } - newEnv.SetDocument(d) - return fn(&newEnv) - }) - if err == ErrStreamClosed { - err = nil - } - if err != nil { - return err + } else { + var start, end document.Value + if !it.Reverse { + start = rng.Min + end = rng.Max + } else { + start = rng.Max + end = rng.Min + } + + var encEnd []byte + if !end.Type.IsZero() && end.V != nil { + encEnd, err = index.EncodeValue(end) + if err != nil { + return err + } + } + + err = iterator([]document.Value{start}, func(val, key []byte) error { + if !rng.IsInRange(val) { + // if we reached the end of our range, we can stop iterating. + if encEnd == nil { + return nil + } + cmp := bytes.Compare(val, encEnd) + if !it.Reverse && cmp > 0 { + return ErrStreamClosed + } + if it.Reverse && cmp < 0 { + return ErrStreamClosed + } + return nil + } + + d, err := table.GetDocument(key) + if err != nil { + return err + } + + newEnv.SetDocument(d) + return fn(&newEnv) + }) + if err == ErrStreamClosed { + err = nil + } + if err != nil { + return err + } } } @@ -417,6 +485,8 @@ type Range struct { // and for determining the global upper bound. Exact bool + Arity int + ArityMax int encodedMin, encodedMax []byte rangeType document.ValueType } @@ -615,7 +685,13 @@ func (r *Range) IsInRange(value []byte) bool { // the value is bigger than the lower bound, // see if it matches the upper bound. if r.encodedMax != nil { - cmpMax = bytes.Compare(value, r.encodedMax) + if r.ArityMax < r.Arity { + fmt.Println("val", value) + fmt.Println("max", r.encodedMax) + cmpMax = bytes.Compare(value[:len(r.encodedMax)-1], r.encodedMax) + } else { + cmpMax = bytes.Compare(value, r.encodedMax) + } } // if boundaries are strict, ignore values equal to the max @@ -623,5 +699,5 @@ func (r *Range) IsInRange(value []byte) bool { return false } - return cmpMax <= 0 + return cmpMin >= 0 && cmpMax <= 0 } diff --git a/stream/iterator_test.go b/stream/iterator_test.go index 8162fddc5..a7dff19ca 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -276,20 +276,27 @@ func TestPkScan(t *testing.T) { func TestIndexScan(t *testing.T) { tests := []struct { name string + indexOn string docsInTable, expected testutil.Docs ranges stream.Ranges reverse bool fails bool }{ - {name: "empty"}, + {name: "empty", indexOn: "a"}, { - "no range", + "no range", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), nil, false, false, }, { - "max:2", + "no range", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), + nil, false, false, + }, + { + "max:2", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), stream.Ranges{ @@ -298,7 +305,16 @@ func TestIndexScan(t *testing.T) { false, false, }, { - "max:1", + "max:[2, 2]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + stream.Ranges{ + {Max: testutil.MakeArray(t, 2, 2)}, + }, + false, false, + }, + { + "max:1", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`), stream.Ranges{ @@ -307,7 +323,16 @@ func TestIndexScan(t *testing.T) { false, false, }, { - "min", + "max:[1, 2]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`), + stream.Ranges{ + {Max: testutil.MakeArray(t, 1, 2)}, + }, + false, false, + }, + { + "min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), stream.Ranges{ @@ -316,7 +341,16 @@ func TestIndexScan(t *testing.T) { false, false, }, { - "min/max", + "min:[2, 1]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 2, "b": 2}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 2, 1)}, + }, + false, false, + }, + { + "min/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), stream.Ranges{ @@ -325,13 +359,31 @@ func TestIndexScan(t *testing.T) { false, false, }, { - "reverse/no range", + "min:[1, 1], max:[2,2]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2)}, + }, + false, false, + }, + { + "min:[1, 1], max:[2,2] bis", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 3}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 3}`, `{"a": 2, "b": 2}`), // [1, 3] < [2, 2] + stream.Ranges{ + {Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2)}, + }, + false, false, + }, + { + "reverse/no range", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), nil, true, false, }, { - "reverse/max", + "reverse/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), stream.Ranges{ @@ -340,7 +392,16 @@ func TestIndexScan(t *testing.T) { true, false, }, { - "reverse/min", + "reverse/max", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), + stream.Ranges{ + {Max: testutil.MakeArray(t, 2, 2)}, + }, + true, false, + }, + { + "reverse/min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), stream.Ranges{ @@ -349,7 +410,25 @@ func TestIndexScan(t *testing.T) { true, false, }, { - "reverse/min/max", + "reverse/min neg", "a", + testutil.MakeDocuments(t, `{"a": 1}`, `{"a": -2}`), + testutil.MakeDocuments(t, `{"a": 1}`), + stream.Ranges{ + {Min: document.NewIntegerValue(1)}, + }, + true, false, + }, + { + "reverse/min", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1, 1)}, + }, + true, false, + }, + { + "reverse/min/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), stream.Ranges{ @@ -357,15 +436,88 @@ func TestIndexScan(t *testing.T) { }, true, false, }, + { + "reverse/min/max", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2)}, + }, + true, false, + }, + { + "max:[1]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), + stream.Ranges{ + {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, + }, + false, false, + }, + { + "reverse max:[1]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 9223372036854775807}`, `{"a": 1, "b": 1}`), + stream.Ranges{ + {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, + }, + true, false, + }, + { + "max:[1, 2]", "a, b, c", + testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), + stream.Ranges{ + {Arity: 3, ArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, + }, + false, false, + }, + // todo a,b,c and [1] + { + "min:[1]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), + testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 1, "b": 1}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1)}, + }, + false, false, + }, + { + "reverse min:[1]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": -2}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1)}, + }, + true, false, + }, + { + "min:[1], max[2]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), + testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 2, "b": 42}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1), Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 2)}, + }, + false, false, + }, + { + "reverse min:[1], max[2]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), + testutil.MakeDocuments(t, `{"a": 2, "b": 42}`, `{"a": 1, "b": -2}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1), Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 2)}, + }, + true, false, + }, } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + t.Run(test.name+"index on "+test.indexOn, func(t *testing.T) { db, err := genji.Open(":memory:") require.NoError(t, err) defer db.Close() - err = db.Exec("CREATE TABLE test (a INTEGER); CREATE INDEX idx_test_a ON test(a)") + err = db.Exec("CREATE TABLE test (a INTEGER, b INTEGER, c INTEGER); CREATE INDEX idx_test_a ON test(" + test.indexOn + ")") require.NoError(t, err) for _, doc := range test.docsInTable { @@ -404,6 +556,11 @@ func TestIndexScan(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) + fmt.Println("expected: ") + test.expected.Print() + fmt.Println("got: ") + got.Print() + fmt.Println("end test") require.Equal(t, len(test.expected), i) test.expected.RequireEqual(t, got) } @@ -411,15 +568,30 @@ func TestIndexScan(t *testing.T) { } t.Run("String", func(t *testing.T) { - require.Equal(t, `indexScan("idx_test_a", [1, 2])`, stream.IndexScan("idx_test_a", stream.Range{ - Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), - }).String()) + t.Run("idx_test_a", func(t *testing.T) { + require.Equal(t, `indexScan("idx_test_a", [1, 2])`, stream.IndexScan("idx_test_a", stream.Range{ + Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), + }).String()) - op := stream.IndexScan("idx_test_a", stream.Range{ - Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), + op := stream.IndexScan("idx_test_a", stream.Range{ + Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), + }) + op.Reverse = true + + require.Equal(t, `indexScanReverse("idx_test_a", [1, 2])`, op.String()) }) - op.Reverse = true - require.Equal(t, `indexScanReverse("idx_test_a", [1, 2])`, op.String()) + t.Run("idx_test_a_b", func(t *testing.T) { + require.Equal(t, `indexScan("idx_test_a_b", [[1, 1], [2, 2]])`, stream.IndexScan("idx_test_a_b", stream.Range{ + Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2), + }).String()) + + op := stream.IndexScan("idx_test_a_b", stream.Range{ + Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2), + }) + op.Reverse = true + + require.Equal(t, `indexScanReverse("idx_test_a_b", [[1, 1], [2, 2]])`, op.String()) + }) }) } diff --git a/testutil/document.go b/testutil/document.go index 0c6bbb6c2..d73083c9f 100644 --- a/testutil/document.go +++ b/testutil/document.go @@ -11,12 +11,25 @@ import ( ) // MakeValue turns v into a document.Value. -func MakeValue(t testing.TB, v interface{}) *document.Value { +func MakeValue(t testing.TB, v interface{}) document.Value { t.Helper() vv, err := document.NewValue(v) require.NoError(t, err) - return &vv + return vv +} + +func MakeArray(t testing.TB, vs ...interface{}) document.Value { + t.Helper() + + vvs := []document.Value{} + for _, v := range vs { + vvs = append(vvs, MakeValue(t, v)) + } + + vb := document.NewValueBuffer(vvs...) + + return document.NewArrayValue(vb) } // MakeDocument creates a document from a json string. @@ -53,6 +66,15 @@ func MakeArray(t testing.TB, jsonArray string) document.Array { type Docs []document.Document +func (docs Docs) Print() { + fmt.Println("----") + for _, d := range docs { + dv := document.NewDocumentValue(d) + fmt.Println(dv) + } + fmt.Println("----") +} + func (docs Docs) RequireEqual(t testing.TB, others Docs) { t.Helper() From 186a904f37c20f0bc729a404e3492525945bbb93 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Sat, 13 Mar 2021 16:31:10 +0100 Subject: [PATCH 03/40] Ignore gopls.log --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 563568ebb..a4393938b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ cmd/genji/genji # VS Code config .vscode/ + +gopls.log From 5939ec90286fc91f371f1d69d87369a914604111 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Tue, 16 Mar 2021 09:50:00 +0100 Subject: [PATCH 04/40] Add a larger test suite for indexes --- database/catalog_test.go | 2 +- database/index.go | 81 +- database/index_test.go | 1553 +++++++++++++++++++++++++++----------- database/table_test.go | 2 +- stream/iterator_test.go | 146 ++-- 5 files changed, 1248 insertions(+), 536 deletions(-) diff --git a/database/catalog_test.go b/database/catalog_test.go index 2f5f7b02b..8c0eaf3ea 100644 --- a/database/catalog_test.go +++ b/database/catalog_test.go @@ -326,7 +326,7 @@ func TestTxCreateIndex(t *testing.T) { require.NotNil(t, idx) var i int - err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error { + err = idx.AscendGreaterOrEqual(values(document.Value{Type: document.DoubleValue}), func(v, k []byte) error { var buf bytes.Buffer err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i))) require.NoError(t, err) diff --git a/database/index.go b/database/index.go index f9e7ff5e5..64e2ccb7f 100644 --- a/database/index.go +++ b/database/index.go @@ -181,6 +181,38 @@ func (idx *Index) Delete(vs []document.Value, k []byte) error { return engine.ErrKeyNotFound } +// validatePivots returns an error when the pivots are unsuitable for the index: +// - no pivots at all +// - having pivots length superior to the index arity +// - having the first pivot without a value when the subsequent ones do have values +func (idx *Index) validatePivots(pivots []document.Value) error { + if len(pivots) == 0 { + return errors.New("cannot iterate without a pivot") + } + + if len(pivots) > idx.Arity() { + // TODO panic + return errors.New("cannot iterate with a pivot whose size is superior to the index arity") + } + + if idx.IsComposite() { + if pivots[0].V == nil { + return errors.New("cannot iterate on a composite index with a pivot whose first item has no value") + } + + previousPivotHasValue := true + for _, p := range pivots[1:] { + if previousPivotHasValue { + previousPivotHasValue = p.V != nil + } else { + return errors.New("cannot iterate on a composite index with a pivot that has holes") + } + } + } + + return nil +} + // AscendGreaterOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in increasing order and calls the given function for each pair. // If the given function returns an error, the iteration stops and returns that error. // If the pivot is empty, starts from the beginning. @@ -196,15 +228,11 @@ func (idx *Index) DescendLessOrEqual(pivots []document.Value, fn func(val, key [ } func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func(val, key []byte) error) error { - if len(pivots) == 0 { - return errors.New("cannot iterate without a pivot") + err := idx.validatePivots(pivots) + if err != nil { + return err } - // if len(pivots) != len(idx.Info.Types) { - // return errors.New("cannot iterate without the same number of values than what the index supports") - // // return errors.New("cannot iterate with more values than what the index supports") - // } - for i, typ := range idx.Info.Types { // if index and pivot are typed but not of the same type // return no result @@ -217,7 +245,6 @@ func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func( if typ != 0 && pivots[i].Type != 0 && typ != pivots[i].Type { return nil } - } st, err := idx.tx.GetStore(idx.storeName) @@ -339,7 +366,7 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool var err error for i, typ := range idx.Info.Types { - if typ == 0 && pivots[i].Type == document.IntegerValue { + if i < len(pivots) && typ == 0 && pivots[i].Type == document.IntegerValue { if pivots[i].V == nil { pivots[i].Type = document.DoubleValue } else { @@ -359,17 +386,24 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool all = all && true } else { all = false + break } } - vb := document.NewValueBuffer(pivots...) - - // we do have pivot values, so let's use them to seek in the index + // we do have pivot values/types, so let's use them to seek in the index if !all { - seek, err = idx.EncodeValue(document.NewArrayValue(vb)) + // TODO delete + // if the first pivot is valueless but typed, we iterate but filter out the types we don't want + // but just for the first pivot. + if pivots[0].Type != 0 && pivots[0].V == nil { + seek = []byte{byte(pivots[0].Type)} + } else { + vb := document.NewValueBuffer(pivots...) + seek, err = idx.EncodeValue(document.NewArrayValue(vb)) - if err != nil { - return err + if err != nil { + return err + } } } else { // we don't, let's start at the beginning seek = []byte{} @@ -413,13 +447,18 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool itm := it.Item() // if index is untyped and pivot is typed, only iterate on values with the same type as pivot - // if idx.Type == 0 && pivot.Type != 0 && itm.Key()[0] != byte(pivot.Type) { - // return nil - // } + if idx.IsComposite() { + // for now, we only check the first element + if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { + return nil + } + } else { + var typ document.ValueType + if len(idx.Info.Types) > 0 { + typ = idx.Info.Types[0] + } - // this is wrong and only handle the first type - for i, typ := range idx.Info.Types { - if typ == 0 && pivots[i].Type != 0 && itm.Key()[0] != byte(pivots[i].Type) { + if (typ == 0) && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { return nil } } diff --git a/database/index_test.go b/database/index_test.go index b0dba64e2..2a811b827 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -8,7 +8,6 @@ import ( "strconv" "testing" - "github.com/genjidb/genji/binarysort" "github.com/genjidb/genji/database" "github.com/genjidb/genji/document" "github.com/genjidb/genji/engine" @@ -16,14 +15,11 @@ import ( "github.com/stretchr/testify/require" ) +// values is a helper function to avoid having to type []document.Value{} all the time. func values(vs ...document.Value) []document.Value { return vs } -func prefixAV(b []byte) []byte { - return append([]byte{byte(document.ArrayValue)}, b...) -} - func getIndex(t testing.TB, unique bool, types ...document.ValueType) (*database.Index, func()) { ng := memoryengine.NewEngine() tx, err := ng.Begin(context.Background(), engine.TxOptions{ @@ -38,170 +34,6 @@ func getIndex(t testing.TB, unique bool, types ...document.ValueType) (*database } } -func TestCompositeIndexSet(t *testing.T) { - idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) - defer cleanup() - require.Error(t, idx.Set(values(document.NewIntegerValue(42), document.NewIntegerValue(43)), nil)) - require.NoError(t, idx.Set(values(document.NewIntegerValue(42), document.NewIntegerValue(43)), []byte("key"))) -} - -func TestCompositeIndexIterateAsc1(t *testing.T) { - idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) - defer cleanup() - - for i := 0; i < 10; i++ { - require.NoError(t, idx.Set(values( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(100-i))), - []byte{'a' + byte(i)})) - } - - var i int = 2 - var count int - pivots := values(document.NewIntegerValue(int64(2)), document.NewIntegerValue(0)) - err := idx.AscendGreaterOrEqual(pivots, func(val, rid []byte) error { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(100-i)), - ) - - val = prefixAV(val) - // the missing array type is annoying - // vvv := document.Value{Type: document.ArrayValue} - // e := vvv.UnmarshalBinary(val) - // require.NoError(t, e) - // fmt.Println(vvv.V) - - requireEqualEncoded(t, document.NewArrayValue(array), val) - require.Equal(t, []byte{'a' + byte(i)}, rid) - - i++ - count++ - return nil - }) - - require.NoError(t, err) - require.Equal(t, 8, count) -} - -func TestCompositeIndexIterateAsc2(t *testing.T) { - idx, cleanup := getIndex(t, false, document.ValueType(0), document.IntegerValue) - defer cleanup() - - for i := 0; i < 10; i++ { - require.NoError(t, idx.Set(values( - document.NewDoubleValue(float64(i)), - document.NewIntegerValue(int64(100-i))), - binarysort.AppendInt64(nil, int64(i)))) - - require.NoError(t, idx.Set(values( - document.NewTextValue(strconv.Itoa(i)), - document.NewIntegerValue(int64(100-i))), - binarysort.AppendInt64(nil, int64(i)))) - } - - var i int = 2 - var count int - pivots := values(document.NewIntegerValue(int64(2)), document.NewIntegerValue(0)) - err := idx.AscendGreaterOrEqual(pivots, func(val, rid []byte) error { - array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewIntegerValue(int64(100-i)), - ) - - // deal with the skipped array value type byte - vval := []byte{byte(document.ArrayValue)} - vval = append(vval, val...) - - requireEqualEncoded(t, document.NewArrayValue(array), vval) - require.Equal(t, binarysort.AppendInt64(nil, int64(i)), rid) - - i++ - count++ - return nil - }) - - require.NoError(t, err) - require.Equal(t, 8, count) -} - -func TestCompositeIndexIterateDesc(t *testing.T) { - idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) - defer cleanup() - - for i := 0; i < 10; i++ { - require.NoError(t, idx.Set(values( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(100-i))), - []byte{'a' + byte(i)})) - } - - var i int = 4 - var count int - pivots := values(document.NewIntegerValue(int64(4)), document.NewIntegerValue(96)) - err := idx.DescendLessOrEqual(pivots, func(val, rid []byte) error { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(100-i)), - ) - - // deal with the skipped array value type byte - vval := []byte{byte(document.ArrayValue)} - vval = append(vval, val...) - - requireEqualEncoded(t, document.NewArrayValue(array), vval) - require.Equal(t, []byte{'a' + byte(i)}, rid) - - i-- - count++ - return nil - }) - - require.NoError(t, err) - require.Equal(t, 5, count) -} - -func TestCompositeIndexIterateDesc2(t *testing.T) { - idx, cleanup := getIndex(t, false, document.ValueType(0), document.IntegerValue) - defer cleanup() - - for i := 0; i < 10; i++ { - require.NoError(t, idx.Set(values( - document.NewDoubleValue(float64(i)), - document.NewIntegerValue(int64(100-i))), - binarysort.AppendInt64(nil, int64(i)))) - - require.NoError(t, idx.Set(values( - document.NewTextValue(strconv.Itoa(i)), - document.NewIntegerValue(int64(100-i))), - binarysort.AppendInt64(nil, int64(i)))) - } - - var i int = 4 - var count int - pivots := values(document.NewIntegerValue(int64(4)), document.NewIntegerValue(96)) - err := idx.DescendLessOrEqual(pivots, func(val, rid []byte) error { - array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewIntegerValue(int64(100-i)), - ) - - // deal with the skipped array value type byte - vval := []byte{byte(document.ArrayValue)} - vval = append(vval, val...) - - requireEqualEncoded(t, document.NewArrayValue(array), vval) - require.Equal(t, binarysort.AppendInt64(nil, int64(i)), rid) - - i-- - count++ - return nil - }) - - require.NoError(t, err) - require.Equal(t, 5, count) -} - func TestIndexSet(t *testing.T) { for _, unique := range []bool{true, false} { text := fmt.Sprintf("Unique: %v, ", unique) @@ -224,19 +56,19 @@ func TestIndexSet(t *testing.T) { require.NoError(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) }) - t.Run(text+"Set one value and key on two values index fails", func(t *testing.T) { + t.Run(text+"Set one value fails (arity=1)", func(t *testing.T) { idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) defer cleanup() require.Error(t, idx.Set(values(document.NewBoolValue(true)), []byte("key"))) }) - t.Run(text+"Set two values and key on non composite index fails", func(t *testing.T) { + t.Run(text+"Set two values fails (arity=1)", func(t *testing.T) { idx, cleanup := getIndex(t, unique, document.ValueType(0)) defer cleanup() require.Error(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) }) - t.Run(text+"Set three values and key on two values index fails", func(t *testing.T) { + t.Run(text+"Set three values fails (arity=2)", func(t *testing.T) { idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) defer cleanup() require.Error(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) @@ -266,12 +98,21 @@ func TestIndexSet(t *testing.T) { defer cleanup() require.NoError(t, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(10)), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(11)), []byte("key"))) require.NoError(t, idx.Set(values(document.NewIntegerValue(11), document.NewIntegerValue(11)), []byte("key"))) require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(10)), []byte("key"))) }) + + t.Run("Unique: true, Type: (integer, text) Duplicate,", func(t *testing.T) { + idx, cleanup := getIndex(t, true, document.IntegerValue, document.TextValue) + defer cleanup() + + require.NoError(t, idx.Set(values(document.NewIntegerValue(10), document.NewTextValue("foo")), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(11), document.NewTextValue("foo")), []byte("key"))) + require.Equal(t, database.ErrIndexDuplicateValue, idx.Set(values(document.NewIntegerValue(10), document.NewTextValue("foo")), []byte("key"))) + }) } -// TODO add composite test cases func TestIndexDelete(t *testing.T) { t.Run("Unique: false, Delete valid key succeeds", func(t *testing.T) { idx, cleanup := getIndex(t, false) @@ -303,6 +144,44 @@ func TestIndexDelete(t *testing.T) { require.Equal(t, 2, i) }) + t.Run("Unique: false, Delete valid key succeeds (arity=2)", func(t *testing.T) { + idx, cleanup := getIndex(t, false, document.ValueType(0), document.ValueType(0)) + defer cleanup() + + require.NoError(t, idx.Set(values(document.NewDoubleValue(10), document.NewDoubleValue(10)), []byte("key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(10)), []byte("other-key"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(11), document.NewIntegerValue(11)), []byte("yet-another-key"))) + require.NoError(t, idx.Set(values(document.NewTextValue("hello"), document.NewTextValue("hello")), []byte("yet-another-different-key"))) + require.NoError(t, idx.Delete(values(document.NewDoubleValue(10), document.NewDoubleValue(10)), []byte("key"))) + + pivot := values(document.NewIntegerValue(10), document.NewIntegerValue(10)) + i := 0 + err := idx.AscendGreaterOrEqual(pivot, func(v, k []byte) error { + if i == 0 { + expected := document.NewArrayValue(document.NewValueBuffer( + document.NewDoubleValue(10), + document.NewDoubleValue(10), + )) + requireEqualBinary(t, expected, v) + require.Equal(t, "other-key", string(k)) + } else if i == 1 { + expected := document.NewArrayValue(document.NewValueBuffer( + document.NewDoubleValue(11), + document.NewDoubleValue(11), + )) + requireEqualBinary(t, expected, v) + require.Equal(t, "yet-another-key", string(k)) + } else { + return errors.New("should not reach this point") + } + + i++ + return nil + }) + require.NoError(t, err) + require.Equal(t, 2, i) + }) + t.Run("Unique: true, Delete valid key succeeds", func(t *testing.T) { idx, cleanup := getIndex(t, true) defer cleanup() @@ -332,6 +211,46 @@ func TestIndexDelete(t *testing.T) { require.Equal(t, 2, i) }) + t.Run("Unique: true, Delete valid key succeeds (arity=2)", func(t *testing.T) { + idx, cleanup := getIndex(t, true, document.ValueType(0), document.ValueType(0)) + defer cleanup() + + require.NoError(t, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(10)), []byte("key1"))) + require.NoError(t, idx.Set(values(document.NewDoubleValue(11), document.NewDoubleValue(11)), []byte("key2"))) + require.NoError(t, idx.Set(values(document.NewIntegerValue(12), document.NewIntegerValue(12)), []byte("key3"))) + require.NoError(t, idx.Delete(values(document.NewDoubleValue(11), document.NewDoubleValue(11)), []byte("key2"))) + + i := 0 + // this will break until the [v, int] case is supported + // pivot := values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}) + pivot := values(document.NewIntegerValue(0), document.NewIntegerValue(0)) + err := idx.AscendGreaterOrEqual(pivot, func(v, k []byte) error { + switch i { + case 0: + expected := document.NewArrayValue(document.NewValueBuffer( + document.NewDoubleValue(10), + document.NewDoubleValue(10), + )) + requireEqualBinary(t, expected, v) + require.Equal(t, "key1", string(k)) + case 1: + expected := document.NewArrayValue(document.NewValueBuffer( + document.NewDoubleValue(12), + document.NewDoubleValue(12), + )) + requireEqualBinary(t, expected, v) + require.Equal(t, "key3", string(k)) + default: + return errors.New("should not reach this point") + } + + i++ + return nil + }) + require.NoError(t, err) + require.Equal(t, 2, i) + }) + for _, unique := range []bool{true, false} { text := fmt.Sprintf("Unique: %v, ", unique) @@ -344,6 +263,16 @@ func TestIndexDelete(t *testing.T) { } } +// requireEqualBinaryUntyped asserts equality assuming that the value is encoded through marshal binary +func requireEqualBinary(t *testing.T, expected document.Value, actual []byte) { + t.Helper() + + buf, err := expected.MarshalBinary() + require.NoError(t, err) + require.Equal(t, buf, actual) +} + +// requireEqualEncoded asserts equality, assuming that the value is encoded with document.ValueEncoder func requireEqualEncoded(t *testing.T, expected document.Value, actual []byte) { t.Helper() @@ -370,36 +299,462 @@ func TestIndexAscendGreaterThan(t *testing.T) { require.Equal(t, 0, i) }) - t.Run(text+"With typed empty pivot, should iterate over all documents of the pivot type in order", func(t *testing.T) { + t.Run(text+"Should iterate through documents in order, ", func(t *testing.T) { + noiseBlob := func(i int) []document.Value { + t.Helper() + return []document.Value{document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))} + } + noiseInts := func(i int) []document.Value { + t.Helper() + return []document.Value{document.NewIntegerValue(int64(i))} + } + + // the following tests will use that constant to determine how many values needs to be inserted + // with the value and noise generators. + total := 5 + tests := []struct { - name string - val func(i int) []document.Value - t document.ValueType + name string + // the index type(s) that is being used indexTypes []document.ValueType - pivots []document.Value + // the pivots, typed or not used to iterate + pivots []document.Value + // the generator for the values that are being indexed + val func(i int) []document.Value + // the generator for the noise values that are being indexed + noise func(i int) []document.Value + // the function to compare the key/value that the iteration yields + expectedEq func(t *testing.T, i uint8, key []byte, val []byte) + // the total count of iteration that should happen + expectedCount int + fail bool }{ - {"integer", - func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, - document.IntegerValue, - nil, - values(document.Value{Type: document.IntegerValue})}, - {"double", - func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, - document.DoubleValue, - nil, - values(document.Value{Type: document.DoubleValue})}, - {"text", - func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, - document.TextValue, - nil, - values(document.Value{Type: document.TextValue})}, - {"(integer, integer)", - func(i int) []document.Value { + // integers --------------------------------------------------- + {name: "index=untyped, vals=integers, pivot=integer", + indexTypes: nil, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + noise: noiseBlob, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 5, + }, + {name: "index=integer, vals=integers, pivot=integer", + indexTypes: []document.ValueType{document.IntegerValue}, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=integers, pivot=integer:2", + indexTypes: nil, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + noise: noiseBlob, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 3, + }, + {name: "index=untyped, vals=integers, pivot=integer:10", + indexTypes: nil, + pivots: values(document.NewIntegerValue(10)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + noise: noiseBlob, + expectedCount: 0, + }, + {name: "index=integer, vals=integers, pivot=integer:2", + indexTypes: []document.ValueType{document.IntegerValue}, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + }, + expectedCount: 3, + }, + // integers, when the index isn't typed can be iterated as doubles + {name: "index=untyped, vals=integers, pivot=double", + indexTypes: nil, + pivots: values(document.Value{Type: document.DoubleValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=integers, pivot=double:1.8", + indexTypes: nil, + pivots: values(document.NewDoubleValue(1.8)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 3, + }, + // but not when the index is typed to integers, although it won't yield an error + {name: "index=integer, vals=integers, pivot=double", + indexTypes: []document.ValueType{document.IntegerValue}, + pivots: values(document.Value{Type: document.DoubleValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedCount: 0, + }, + + // doubles ---------------------------------------------------- + {name: "index=untyped, vals=doubles, pivot=double", + indexTypes: nil, + pivots: values(document.Value{Type: document.DoubleValue}), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 5, + }, + // when iterating on doubles, but passing an integer pivot, it'll be casted as a double + {name: "index=untyped, vals=doubles, pivot=integers", + indexTypes: nil, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=doubles, pivot=double:1.8", + indexTypes: nil, + pivots: values(document.NewDoubleValue(1.8)), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 3, + }, + {name: "index=double, vals=doubles, pivot=double:1.8", + indexTypes: []document.ValueType{document.DoubleValue}, + pivots: values(document.NewDoubleValue(1.8)), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 3, + }, + {name: "index=untyped, vals=doubles, pivot=double:10.8", + indexTypes: nil, + pivots: values(document.NewDoubleValue(10.8)), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedCount: 0, + }, + + // text ------------------------------------------------------- + {name: "index=untyped, vals=text pivot=text", + indexTypes: nil, + pivots: values(document.Value{Type: document.TextValue}), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=text, pivot=text('2')", + indexTypes: nil, + pivots: values(document.NewTextValue("2")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 3, + }, + {name: "index=untyped, vals=text, pivot=text('')", + indexTypes: nil, + pivots: values(document.NewTextValue("")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=text, pivot=text('foo')", + indexTypes: nil, + pivots: values(document.NewTextValue("foo")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedCount: 0, + }, + {name: "index=text, vals=text, pivot=text('2')", + indexTypes: []document.ValueType{document.TextValue}, + pivots: values(document.NewTextValue("2")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 3, + }, + // composite -------------------------------------------------- + // composite indexes must have at least have one value + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", + indexTypes: []document.ValueType{0, 0, 0}, + pivots: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[0, 0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 5, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2, 0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, + // TODO + // {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2, int]", + // indexTypes: []document.ValueType{0, 0}, + // pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + // val: func(i int) []document.Value { + // return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + // }, + // noise: func(i int) []document.Value { + // return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + // }, + // expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + // i += 2 + // array := document.NewValueBuffer( + // document.NewDoubleValue(float64(i)), + // document.NewDoubleValue(float64(i+1))) + // requireEqualBinary(t, document.NewArrayValue(array), val) + // }, + // expectedCount: 3, + // }, + // a more subtle case + {name: "index=[untyped, untyped], vals=[int, blob], noise=blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewBlobValue([]byte{byte('a' + uint8(i))})) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, + // partial pivot + {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + // let's not try to match, it's not important + }, + expectedCount: 10, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + // let's not try to match, it's not important + }, + expectedCount: 6, // total * 2 - (noise + val = 2) * 2 + }, + // this is a tricky test, when we have multiple values but they share the first pivot element; + // this is by definition a very implementation dependent test. + {name: "index=[untyped, untyped], vals=[int, int], noise=int, bool], pivot=[int:0, int:0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBoolValue(true)) + }, + // [0, 0] > [0, true] but [1, true] > [0, 0] so we will see some bools in the results + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + if i%2 == 0 { + i = i / 2 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + } + }, + expectedCount: 9, // 10 elements, but pivot skipped the initial [0, true] + }, + // index typed + {name: "index=[int, int], vals=[int, int], pivot=[0, 0]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 5, + }, + {name: "index=[int, int], vals=[int, int], pivot=[2, 0]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - document.ArrayValue, // of integers - []document.ValueType{0, 0}, - values(document.Value{Type: document.IntegerValue, V: int64(0)}, document.Value{Type: document.IntegerValue, V: int64(0)})}, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, + // a more subtle case + {name: "index=[int, blob], vals=[int, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + indexTypes: []document.ValueType{document.IntegerValue, document.BlobValue}, + pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewBlobValue([]byte{byte('a' + uint8(i))})) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, + // partial pivot + {name: "index=[int, int], vals=[int, int], pivot=[0]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 5, + }, + {name: "index=[int, int], vals=[int, int], pivot=[2]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, + + // documents -------------------------------------------------- + // TODO + // arrays ----------------------------------------------------- + // TODO } for _, test := range tests { @@ -407,200 +762,31 @@ func TestIndexAscendGreaterThan(t *testing.T) { idx, cleanup := getIndex(t, unique, test.indexTypes...) defer cleanup() - for i := 0; i < 10; i += 2 { + for i := 0; i < total; i++ { require.NoError(t, idx.Set(test.val(i), []byte{'a' + byte(i)})) - - // the case where we have other types in the list was missing from this test - // ugly, TODO refactor - ovs := []document.Value{} - for range test.indexTypes { - ovs = append(ovs, document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) - } - - if test.indexTypes == nil { - ovs = append(ovs, document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + if test.noise != nil { + require.NoError(t, idx.Set(test.noise(i), []byte{'a' + byte(i)})) } - - require.NoError(t, idx.Set(ovs, []byte{'a' + byte(i)})) } var i uint8 var count int err := idx.AscendGreaterOrEqual(test.pivots, func(val, rid []byte) error { - switch test.t { - case document.IntegerValue: - requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) - case document.DoubleValue: - requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) - case document.TextValue: - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) - case document.BoolValue: - requireEqualEncoded(t, document.NewBoolValue(i > 0), val) - case document.ArrayValue: // of integers - array := document.NewValueBuffer(document.NewDoubleValue(float64(i)), document.NewDoubleValue(float64(i+1))) - requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) - } - require.Equal(t, []byte{'a' + i}, rid) - - i += 2 + test.expectedEq(t, i, rid, val) + i++ count++ return nil }) - require.NoError(t, err) - require.Equal(t, 5, count) + if test.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, test.expectedCount, count) + } }) } }) - - t.Run(text+"With pivot, should iterate over some documents in order", func(t *testing.T) { - idx, cleanup := getIndex(t, unique) - defer cleanup() - - for i := byte(0); i < 10; i += 2 { - require.NoError(t, idx.Set(values(document.NewTextValue(string([]byte{'A' + i}))), []byte{'a' + i})) - } - - var i uint8 - var count int - pivot := values(document.NewTextValue("C")) - err := idx.AscendGreaterOrEqual(pivot, func(val, rid []byte) error { - requireEqualEncoded(t, document.NewTextValue(string([]byte{'C' + i})), val) - require.Equal(t, []byte{'c' + i}, rid) - - i += 2 - count++ - return nil - }) - require.NoError(t, err) - require.Equal(t, 4, count) - }) - - t.Run(text+"With composite pivot, should iterate over some documents in order", func(t *testing.T) { - - idx, cleanup := getIndex(t, false, document.IntegerValue, document.IntegerValue) - defer cleanup() - - for i := 0; i < 10; i++ { - require.NoError(t, idx.Set(values( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(100-i))), - []byte{'a' + byte(i)})) - } - - var i int = 2 - var count int - pivots := values(document.NewIntegerValue(int64(2)), document.NewIntegerValue(0)) - err := idx.AscendGreaterOrEqual(pivots, func(val, rid []byte) error { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(100-i)), - ) - - requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) - require.Equal(t, []byte{'a' + byte(i)}, rid) - - i++ - count++ - return nil - }) - - require.NoError(t, err) - require.Equal(t, 8, count) - }) - - t.Run(text+"With no pivot but a composite index, should iterate over all documents in order, regardless of their type", func(t *testing.T) { - idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) - defer cleanup() - - for i := int64(0); i < 10; i++ { - require.NoError(t, idx.Set(values(document.NewDoubleValue(float64(i)), document.NewBlobValue([]byte{byte(i)})), []byte{'d', 'a' + byte(i)})) - require.NoError(t, idx.Set(values(document.NewTextValue(strconv.Itoa(int(i))), document.NewDoubleValue(float64(i))), []byte{'s', 'a' + byte(i)})) - } - - var doubles, texts int - var count int - err := idx.AscendGreaterOrEqual(values(document.Value{}, document.Value{}), func(val, rid []byte) error { - if count < 10 { - array := document.NewValueBuffer( - document.NewDoubleValue(float64(doubles)), - document.NewBlobValue([]byte{byte(doubles)})) - - requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) - require.Equal(t, []byte{'d', 'a' + byte(doubles)}, rid) - doubles++ - } else { - array := document.NewValueBuffer( - document.NewTextValue(strconv.Itoa(int(texts))), - document.NewDoubleValue(float64(texts))) - - requireEqualEncoded(t, document.NewArrayValue(array), prefixAV(val)) - require.Equal(t, []byte{'s', 'a' + byte(texts)}, rid) - texts++ - } - count++ - return nil - }) - require.NoError(t, err) - require.Equal(t, 10, doubles) - require.Equal(t, 10, texts) - }) - - t.Run(text+"With no pivot and a typed index, should iterate over all documents in order", func(t *testing.T) { - idx, cleanup := getIndex(t, unique) - idx.Info.Types = []document.ValueType{document.IntegerValue} - defer cleanup() - - for i := int64(0); i < 10; i++ { - require.NoError(t, idx.Set(values(document.NewIntegerValue(i)), []byte{'i', 'a' + byte(i)})) - } - - var ints int - err := idx.AscendGreaterOrEqual(values(document.Value{}), func(val, rid []byte) error { - enc, err := document.NewIntegerValue(int64(ints)).MarshalBinary() - require.NoError(t, err) - require.Equal(t, enc, val) - require.Equal(t, []byte{'i', 'a' + byte(ints)}, rid) - ints++ - - return nil - }) - require.NoError(t, err) - require.Equal(t, 10, ints) - }) } - - t.Run("Unique: false, Must iterate through similar values properly", func(t *testing.T) { - idx, cleanup := getIndex(t, false) - defer cleanup() - - for i := int64(0); i < 100; i++ { - require.NoError(t, idx.Set(values(document.NewIntegerValue(1)), binarysort.AppendInt64(nil, i))) - require.NoError(t, idx.Set(values(document.NewTextValue("1")), binarysort.AppendInt64(nil, i))) - } - - var doubles, texts int - i := int64(0) - err := idx.AscendGreaterOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, rid []byte) error { - requireEqualEncoded(t, document.NewDoubleValue(1), val) - require.Equal(t, binarysort.AppendInt64(nil, i), rid) - i++ - doubles++ - return nil - }) - require.NoError(t, err) - - i = 0 - err = idx.AscendGreaterOrEqual(values(document.Value{Type: document.TextValue}), func(val, rid []byte) error { - requireEqualEncoded(t, document.NewTextValue("1"), val) - require.Equal(t, binarysort.AppendInt64(nil, i), rid) - i++ - texts++ - return nil - }) - require.NoError(t, err) - require.Equal(t, 100, doubles) - require.Equal(t, 100, texts) - }) } func TestIndexDescendLessOrEqual(t *testing.T) { @@ -612,7 +798,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { defer cleanup() i := 0 - err := idx.DescendLessOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, key []byte) error { + err := idx.AscendGreaterOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, key []byte) error { i++ return errors.New("should not iterate") }) @@ -620,80 +806,515 @@ func TestIndexDescendLessOrEqual(t *testing.T) { require.Equal(t, 0, i) }) - t.Run(text+"With empty typed pivot, should iterate over all documents of the same type in reverse order", func(t *testing.T) { - idx, cleanup := getIndex(t, unique) - defer cleanup() - - for i := byte(0); i < 10; i += 2 { - require.NoError(t, idx.Set(values(document.NewIntegerValue(int64(i))), []byte{'a' + i})) + t.Run(text+"Should iterate through documents in order, ", func(t *testing.T) { + noiseBlob := func(i int) []document.Value { + t.Helper() + return []document.Value{document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))} + } + noiseInts := func(i int) []document.Value { + t.Helper() + return []document.Value{document.NewIntegerValue(int64(i))} } - var i uint8 = 8 - var count int - err := idx.DescendLessOrEqual(values(document.Value{Type: document.IntegerValue}), func(val, key []byte) error { - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) - require.Equal(t, []byte{'a' + i}, key) - - i -= 2 - count++ - return nil - }) - require.NoError(t, err) - require.Equal(t, 5, count) - }) + // the following tests will use that constant to determine how many values needs to be inserted + // with the value and noise generators. + total := 5 - t.Run(text+"With pivot, should iterate over some documents in order", func(t *testing.T) { - idx, cleanup := getIndex(t, unique) - defer cleanup() + tests := []struct { + name string + // the index type(s) that is being used + indexTypes []document.ValueType + // the pivots, typed or not used to iterate + pivots []document.Value + // the generator for the values that are being indexed + val func(i int) []document.Value + // the generator for the noise values that are being indexed + noise func(i int) []document.Value + // the function to compare the key/value that the iteration yields + expectedEq func(t *testing.T, i uint8, key []byte, val []byte) + // the total count of iteration that should happen + expectedCount int + fail bool + }{ + // integers --------------------------------------------------- + {name: "index=untyped, vals=integers, pivot=integer", + indexTypes: nil, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + noise: noiseBlob, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 5, + }, + {name: "index=integer, vals=integers, pivot=integer", + indexTypes: []document.ValueType{document.IntegerValue}, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=integers, pivot=integer:2", + indexTypes: nil, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + noise: noiseBlob, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 3, + }, + {name: "index=untyped, vals=integers, pivot=integer:-10", + indexTypes: nil, + pivots: values(document.NewIntegerValue(-10)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + noise: noiseBlob, + expectedCount: 0, + }, + {name: "index=integer, vals=integers, pivot=integer:2", + indexTypes: []document.ValueType{document.IntegerValue}, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + }, + expectedCount: 3, + }, + // integers, when the index isn't typed can be iterated as doubles + {name: "index=untyped, vals=integers, pivot=double", + indexTypes: nil, + pivots: values(document.Value{Type: document.DoubleValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=integers, pivot=double:1.8", + indexTypes: nil, + pivots: values(document.NewDoubleValue(1.8)), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + }, + expectedCount: 2, + }, + // but not when the index is typed to integers, although it won't yield an error + {name: "index=integer, vals=integers, pivot=double", + indexTypes: []document.ValueType{document.IntegerValue}, + pivots: values(document.Value{Type: document.DoubleValue}), + val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedCount: 0, + }, + + // doubles ---------------------------------------------------- + {name: "index=untyped, vals=doubles, pivot=double", + indexTypes: nil, + pivots: values(document.Value{Type: document.DoubleValue}), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 5, + }, + // when iterating on doubles, but passing an integer pivot, it'll be casted as a double + {name: "index=untyped, vals=doubles, pivot=integers", + indexTypes: nil, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=doubles, pivot=double:1.8", + indexTypes: nil, + pivots: values(document.NewDoubleValue(1.8)), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 2, + }, + {name: "index=double, vals=doubles, pivot=double:1.8", + indexTypes: []document.ValueType{document.DoubleValue}, + pivots: values(document.NewDoubleValue(1.8)), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + }, + expectedCount: 2, + }, + {name: "index=untyped, vals=doubles, pivot=double:-10.8", + indexTypes: nil, + pivots: values(document.NewDoubleValue(-10.8)), + val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedCount: 0, + }, + + // text ------------------------------------------------------- + {name: "index=untyped, vals=text pivot=text", + indexTypes: nil, + pivots: values(document.Value{Type: document.TextValue}), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=text, pivot=text('2')", + indexTypes: nil, + pivots: values(document.NewTextValue("2")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 3, + }, + {name: "index=untyped, vals=text, pivot=text('')", + indexTypes: nil, + pivots: values(document.NewTextValue("")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 5, + }, + {name: "index=untyped, vals=text, pivot=text('foo')", + indexTypes: nil, + pivots: values(document.NewTextValue("foo")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + noise: noiseInts, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + require.Equal(t, []byte{'a' + i}, key) + requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 5, + }, + {name: "index=text, vals=text, pivot=text('2')", + indexTypes: []document.ValueType{document.TextValue}, + pivots: values(document.NewTextValue("2")), + val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + require.Equal(t, []byte{'a' + i}, key) + requireEqualBinary(t, document.NewTextValue(strconv.Itoa(int(i))), val) + }, + expectedCount: 3, + }, + // composite -------------------------------------------------- + // composite indexes must have at least have one value + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", + indexTypes: []document.ValueType{0, 0, 0}, + pivots: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + fail: true, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedCount: 0, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[5, 5]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(5), document.NewIntegerValue(5)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 5, + }, + // [0,1], [1,2], --[2,0]--, [2,3], [3,4], [4,5] + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 2, + }, + // TODO + // {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2, int]", + // indexTypes: []document.ValueType{0, 0}, + // pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + // val: func(i int) []document.Value { + // return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + // }, + // noise: func(i int) []document.Value { + // return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + // }, + // expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + // i += 2 + // array := document.NewValueBuffer( + // document.NewDoubleValue(float64(i)), + // document.NewDoubleValue(float64(i+1))) + // requireEqualBinary(t, document.NewArrayValue(array), val) + // }, + // expectedCount: 3, + // }, + // a more subtle case + {name: "index=[untyped, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewBlobValue([]byte{byte('a' + uint8(i))})) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 2, + }, + // partial pivot + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + // let's not try to match, it's not important + }, + expectedCount: 2, // [0] is "equal" to [0, 1] and [0, "1"] + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[5]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(5)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + // let's not try to match, it's not important + }, + expectedCount: 10, + }, + {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + // let's not try to match, it's not important + }, + expectedCount: 6, // total * 2 - (noise + val = 2) * 2 + }, + // index typed + {name: "index=[int, int], vals=[int, int], pivot=[0, 0]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 0, + }, + {name: "index=[int, int], vals=[int, int], pivot=[5, 6]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(5), document.NewIntegerValue(6)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 5, + }, + {name: "index=[int, int], vals=[int, int], pivot=[2, 0]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 2, + }, + // a more subtle case + {name: "index=[int, blob], vals=[int, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + indexTypes: []document.ValueType{document.IntegerValue, document.BlobValue}, + pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewBlobValue([]byte{byte('a' + uint8(i))})) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 2, + }, + // partial pivot + {name: "index=[int, int], vals=[int, int], pivot=[0]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(0)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 4 + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 1, + }, + // [0,1], [1,2], [2,3], --[2]--, [3,4], [4,5] + {name: "index=[int, int], vals=[int, int], pivot=[2]", + indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, + pivots: values(document.NewIntegerValue(2)), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + array := document.NewValueBuffer( + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, - for i := byte(0); i < 10; i++ { - require.NoError(t, idx.Set(values(document.NewTextValue(string([]byte{'A' + i}))), []byte{'a' + i})) + // documents -------------------------------------------------- + // TODO + // arrays ----------------------------------------------------- + // TODO } - var i byte = 0 - var count int - pivot := values(document.NewTextValue("F")) - err := idx.DescendLessOrEqual(pivot, func(val, rid []byte) error { - requireEqualEncoded(t, document.NewTextValue(string([]byte{'F' - i})), val) - require.Equal(t, []byte{'f' - i}, rid) - - i++ - count++ - return nil - }) - require.NoError(t, err) - require.Equal(t, 6, count) - }) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + idx, cleanup := getIndex(t, unique, test.indexTypes...) + defer cleanup() - t.Run(text+"With no pivot, should iterate over all documents in reverse order, regardless of their type", func(t *testing.T) { - idx, cleanup := getIndex(t, unique) - defer cleanup() + for i := 0; i < total; i++ { + require.NoError(t, idx.Set(test.val(i), []byte{'a' + byte(i)})) + if test.noise != nil { + require.NoError(t, idx.Set(test.noise(i), []byte{'a' + byte(i)})) + } + } - for i := 0; i < 10; i++ { - require.NoError(t, idx.Set(values(document.NewIntegerValue(int64(i))), []byte{'i', 'a' + byte(i)})) - require.NoError(t, idx.Set(values(document.NewTextValue(strconv.Itoa(i))), []byte{'s', 'a' + byte(i)})) + var i uint8 + var count int + err := idx.DescendLessOrEqual(test.pivots, func(val, rid []byte) error { + test.expectedEq(t, uint8(total-1)-i, rid, val) + i++ + count++ + return nil + }) + if test.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, test.expectedCount, count) + } + }) } - - var ints, texts int = 9, 9 - var count int = 20 - err := idx.DescendLessOrEqual(values(document.Value{}), func(val, rid []byte) error { - if count > 10 { - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(texts))), val) - require.Equal(t, []byte{'s', 'a' + byte(texts)}, rid) - texts-- - } else { - requireEqualEncoded(t, document.NewIntegerValue(int64(ints)), val) - require.Equal(t, []byte{'i', 'a' + byte(ints)}, rid) - ints-- - } - - count-- - return nil - }) - require.NoError(t, err) - require.Equal(t, 0, count) - require.Equal(t, -1, ints) - require.Equal(t, -1, texts) }) } } @@ -741,3 +1362,47 @@ func BenchmarkIndexIteration(b *testing.B) { }) } } + +// BenchmarkCompositeIndexSet benchmarks the Set method with 1, 10, 1000 and 10000 successive insertions. +func BenchmarkCompositeIndexSet(b *testing.B) { + for size := 10; size <= 10000; size *= 10 { + b.Run(fmt.Sprintf("%.05d", size), func(b *testing.B) { + b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { + idx, cleanup := getIndex(b, false, document.TextValue, document.TextValue) + + b.StartTimer() + for j := 0; j < size; j++ { + k := fmt.Sprintf("name-%d", j) + idx.Set(values(document.NewTextValue(k), document.NewTextValue(k)), []byte(k)) + } + b.StopTimer() + cleanup() + } + }) + } +} + +// BenchmarkCompositeIndexIteration benchmarks the iterarion of a cursor with 1, 10, 1000 and 10000 items. +func BenchmarkCompositeIndexIteration(b *testing.B) { + for size := 10; size <= 10000; size *= 10 { + b.Run(fmt.Sprintf("%.05d", size), func(b *testing.B) { + idx, cleanup := getIndex(b, false) + defer cleanup() + + for i := 0; i < size; i++ { + k := []byte(fmt.Sprintf("name-%d", i)) + _ = idx.Set(values(document.NewTextValue(string(k)), document.NewTextValue(string(k))), k) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = idx.AscendGreaterOrEqual(values(document.NewTextValue(""), document.NewTextValue("")), func(_, _ []byte) error { + return nil + }) + } + b.StopTimer() + }) + } +} diff --git a/database/table_test.go b/database/table_test.go index 88e9bd4a8..ebdbdf6a6 100644 --- a/database/table_test.go +++ b/database/table_test.go @@ -643,7 +643,7 @@ func TestTableReplace(t *testing.T) { require.NoError(t, err) err = tx.CreateIndex(&database.IndexInfo{ - Path: document.NewPath("a"), + Paths: []document.Path{document.NewPath("a")}, Unique: true, TableName: "test", IndexName: "idx_foo_a", diff --git a/stream/iterator_test.go b/stream/iterator_test.go index a7dff19ca..c3f8c80d0 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -289,12 +289,13 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), nil, false, false, }, - { - "no range", "a, b", - testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), - testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), - nil, false, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "no range", "a, b", + // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), + // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), + // nil, false, false, + // }, { "max:2", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -304,15 +305,16 @@ func TestIndexScan(t *testing.T) { }, false, false, }, - { - "max:[2, 2]", "a, b", - testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - stream.Ranges{ - {Max: testutil.MakeArray(t, 2, 2)}, - }, - false, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "max:[2, 2]", "a, b", + // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + // stream.Ranges{ + // {Max: testutil.MakeArray(t, 2, 2)}, + // }, + // false, false, + // }, { "max:1", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -322,15 +324,16 @@ func TestIndexScan(t *testing.T) { }, false, false, }, - { - "max:[1, 2]", "a, b", - testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - testutil.MakeDocuments(t, `{"a": 1, "b": 2}`), - stream.Ranges{ - {Max: testutil.MakeArray(t, 1, 2)}, - }, - false, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "max:[1, 2]", "a, b", + // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`), + // stream.Ranges{ + // {Max: testutil.MakeArray(t, 1, 2)}, + // }, + // false, false, + // }, { "min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -391,15 +394,16 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - { - "reverse/max", "a, b", - testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), - testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), - stream.Ranges{ - {Max: testutil.MakeArray(t, 2, 2)}, - }, - true, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "reverse/max", "a, b", + // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), + // testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), + // stream.Ranges{ + // {Max: testutil.MakeArray(t, 2, 2)}, + // }, + // true, false, + // }, { "reverse/min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -418,15 +422,16 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - { - "reverse/min", "a, b", - testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), - testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), - stream.Ranges{ - {Min: testutil.MakeArray(t, 1, 1)}, - }, - true, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "reverse/min", "a, b", + // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), + // testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), + // stream.Ranges{ + // {Min: testutil.MakeArray(t, 1, 1)}, + // }, + // true, false, + // }, { "reverse/min/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -445,15 +450,16 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - { - "max:[1]", "a, b", - testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), - testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), - stream.Ranges{ - {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, - }, - false, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "max:[1]", "a, b", + // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), + // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), + // stream.Ranges{ + // {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, + // }, + // false, false, + // }, { "reverse max:[1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), @@ -463,15 +469,16 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - { - "max:[1, 2]", "a, b, c", - testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), - testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), - stream.Ranges{ - {Arity: 3, ArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, - }, - false, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "max:[1, 2]", "a, b, c", + // testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), + // testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), + // stream.Ranges{ + // {Arity: 3, ArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, + // }, + // false, false, + // }, // todo a,b,c and [1] { "min:[1]", "a, b", @@ -482,15 +489,16 @@ func TestIndexScan(t *testing.T) { }, false, false, }, - { - "reverse min:[1]", "a, b", - testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), - testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": -2}`), - stream.Ranges{ - {Min: testutil.MakeArray(t, 1)}, - }, - true, false, - }, + // because composite indexes must have at least a defined in [a, b], this case won't happen + // { + // "reverse min:[1]", "a, b", + // testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), + // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": -2}`), + // stream.Ranges{ + // {Min: testutil.MakeArray(t, 1)}, + // }, + // true, false, + // }, { "min:[1], max[2]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), From 51e6b16a05fc1fbea9135ba9d4e72874861a7415 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 17 Mar 2021 12:15:26 +0100 Subject: [PATCH 05/40] Handle the index [2, int] case --- database/index.go | 47 ++++++++++--- database/index_test.go | 146 ++++++++++++++++++++++++++++------------- testutil/index.go | 2 +- 3 files changed, 139 insertions(+), 56 deletions(-) diff --git a/database/index.go b/database/index.go index 64e2ccb7f..dadbd5acb 100644 --- a/database/index.go +++ b/database/index.go @@ -196,18 +196,31 @@ func (idx *Index) validatePivots(pivots []document.Value) error { } if idx.IsComposite() { + allNil := true + prevV := true + for _, p := range pivots { + allNil = allNil && p.V == nil + if prevV { + prevV = prevV && p.V != nil + } else { + if !allNil { + return errors.New("cannot iterate on a composite index with a pivot that has holes") + } + } + } + if pivots[0].V == nil { return errors.New("cannot iterate on a composite index with a pivot whose first item has no value") } - previousPivotHasValue := true - for _, p := range pivots[1:] { - if previousPivotHasValue { - previousPivotHasValue = p.V != nil - } else { - return errors.New("cannot iterate on a composite index with a pivot that has holes") - } - } + // previousPivotHasValue := true + // for _, p := range pivots[1:] { + // if previousPivotHasValue { + // previousPivotHasValue = p.V != nil + // } else { + // return errors.New("cannot iterate on a composite index with a pivot that has holes") + // } + // } } return nil @@ -398,9 +411,25 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool if pivots[0].Type != 0 && pivots[0].V == nil { seek = []byte{byte(pivots[0].Type)} } else { - vb := document.NewValueBuffer(pivots...) + ppivots := make([]document.Value, 0, len(pivots)) + var last *document.Value + for _, p := range pivots { + if p.V != nil { + ppivots = append(ppivots, p) + } else { + last = &p + break + } + } + + vb := document.NewValueBuffer(ppivots...) seek, err = idx.EncodeValue(document.NewArrayValue(vb)) + // if we have a [2, int] case, let's just add the type + if last != nil { + seek = append(seek[:len(seek)-1], byte(0x1f), byte(last.Type), byte(0x1e)) + } + if err != nil { return err } diff --git a/database/index_test.go b/database/index_test.go index 2a811b827..1c03bfa27 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -549,7 +549,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, fail: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[0, 0]", + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -566,7 +566,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2, 0]", + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", indexTypes: []document.ValueType{0, 0}, pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -584,27 +584,50 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 3, }, - // TODO - // {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2, int]", - // indexTypes: []document.ValueType{0, 0}, - // pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), - // val: func(i int) []document.Value { - // return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) - // }, - // noise: func(i int) []document.Value { - // return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) - // }, - // expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - // i += 2 - // array := document.NewValueBuffer( - // document.NewDoubleValue(float64(i)), - // document.NewDoubleValue(float64(i+1))) - // requireEqualBinary(t, document.NewArrayValue(array), val) - // }, - // expectedCount: 3, - // }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, + // pivot [2, int] should filter out [2, not(int)] + {name: "index=[untyped, untyped], vals=[int, int], noise=[int, blob], pivot=[2, int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + // only [3, not(int)] is greater than [2, int], so it will appear anyway if we don't skip it + if i < 3 { + return values(document.NewIntegerValue(int64(i)), document.NewBoolValue(true)) + } + + return nil + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, // a more subtle case - {name: "index=[untyped, untyped], vals=[int, blob], noise=blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + {name: "index=[untyped, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, 0}, pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { @@ -623,7 +646,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 3, }, // partial pivot - {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[0]", + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", indexTypes: []document.ValueType{0, 0}, pivots: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -637,7 +660,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 10, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2]", + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", indexTypes: []document.ValueType{0, 0}, pivots: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { @@ -765,7 +788,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { for i := 0; i < total; i++ { require.NoError(t, idx.Set(test.val(i), []byte{'a' + byte(i)})) if test.noise != nil { - require.NoError(t, idx.Set(test.noise(i), []byte{'a' + byte(i)})) + v := test.noise(i) + if v != nil { + require.NoError(t, idx.Set(test.noise(i), []byte{'a' + byte(i)})) + } } } @@ -1107,25 +1133,50 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 2, }, - // TODO - // {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2, int]", - // indexTypes: []document.ValueType{0, 0}, - // pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), - // val: func(i int) []document.Value { - // return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) - // }, - // noise: func(i int) []document.Value { - // return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) - // }, - // expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - // i += 2 - // array := document.NewValueBuffer( - // document.NewDoubleValue(float64(i)), - // document.NewDoubleValue(float64(i+1))) - // requireEqualBinary(t, document.NewArrayValue(array), val) - // }, - // expectedCount: 3, - // }, + // [0,1], [1,2], [2,3], --[2,int]--, [3,4], [4,5] + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, + // pivot [2, int] should filter out [2, not(int)] + // [0,1], [1,2], [2,3], --[2,int]--, [2, text], [3,4], [3,text], [4,5], [4,text] + {name: "index=[untyped, untyped], vals=[int, int], noise=[int, text], pivot=[2, int]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + noise: func(i int) []document.Value { + if i > 1 { + return values(document.NewIntegerValue(int64(i)), document.NewTextValue("foo")) + } + + return nil + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 3, + }, // a more subtle case {name: "index=[untyped, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, 0}, @@ -1174,7 +1225,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 10, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=blob, blob], pivot=[2]", + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", indexTypes: []document.ValueType{0, 0}, pivots: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { @@ -1295,7 +1346,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { for i := 0; i < total; i++ { require.NoError(t, idx.Set(test.val(i), []byte{'a' + byte(i)})) if test.noise != nil { - require.NoError(t, idx.Set(test.noise(i), []byte{'a' + byte(i)})) + v := test.noise(i) + if v != nil { + require.NoError(t, idx.Set(test.noise(i), []byte{'a' + byte(i)})) + } } } diff --git a/testutil/index.go b/testutil/index.go index 6bd251e93..855789796 100644 --- a/testutil/index.go +++ b/testutil/index.go @@ -16,7 +16,7 @@ func GetIndexContent(t testing.TB, tx *database.Transaction, indexName string) [ require.NoError(t, err) var content []KV - err = idx.AscendGreaterOrEqual(document.Value{}, func(val, key []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{{}}, func(val, key []byte) error { content = append(content, KV{ Key: append([]byte{}, val...), Value: append([]byte{}, key...), From 58454c399e65291bd525fd488b51c280c4c9c954 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 17 Mar 2021 13:54:56 +0100 Subject: [PATCH 06/40] Rework pivots validation, handle missing cases --- database/index.go | 55 ++++++++----- database/index_test.go | 53 +++++++++--- query/create_test.go | 2 +- sql/parser/create_test.go | 19 +++-- stream/iterator_test.go | 166 +++++++++++++++++++------------------- 5 files changed, 174 insertions(+), 121 deletions(-) diff --git a/database/index.go b/database/index.go index dadbd5acb..6e76747be 100644 --- a/database/index.go +++ b/database/index.go @@ -181,6 +181,18 @@ func (idx *Index) Delete(vs []document.Value, k []byte) error { return engine.ErrKeyNotFound } +func allEmpty(pivots []document.Value) bool { + res := true + for _, p := range pivots { + res = res && p.Type == 0 + if !res { + break + } + } + + return res +} + // validatePivots returns an error when the pivots are unsuitable for the index: // - no pivots at all // - having pivots length superior to the index arity @@ -196,31 +208,32 @@ func (idx *Index) validatePivots(pivots []document.Value) error { } if idx.IsComposite() { - allNil := true - prevV := true - for _, p := range pivots { - allNil = allNil && p.V == nil - if prevV { - prevV = prevV && p.V != nil - } else { - if !allNil { + if !allEmpty(pivots) { + // the first pivot must have a value + if pivots[0].V == nil { + return errors.New("cannot iterate on a composite index whose first pivot has no value") + } + + // it's acceptable for the last pivot to just have a type and no value + hasValue := true + for _, p := range pivots { + // if on the previous pivot we have a value + if hasValue { + hasValue = p.V != nil + + // if we have no value, we at least need a type + if !hasValue { + if p.Type == 0 { + return errors.New("cannot iterate on a composite index with a pivot that has holes") + } + } + } else { return errors.New("cannot iterate on a composite index with a pivot that has holes") } } + } else { + return nil } - - if pivots[0].V == nil { - return errors.New("cannot iterate on a composite index with a pivot whose first item has no value") - } - - // previousPivotHasValue := true - // for _, p := range pivots[1:] { - // if previousPivotHasValue { - // previousPivotHasValue = p.V != nil - // } else { - // return errors.New("cannot iterate on a composite index with a pivot that has holes") - // } - // } } return nil diff --git a/database/index_test.go b/database/index_test.go index 1c03bfa27..b07808b8d 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -309,6 +309,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { return []document.Value{document.NewIntegerValue(int64(i))} } + noCallEq := func(t *testing.T, i uint8, key []byte, val []byte) { + require.Fail(t, "equality test should not be called here") + } + // the following tests will use that constant to determine how many values needs to be inserted // with the value and noise generators. total := 5 @@ -368,6 +372,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { pivots: values(document.NewIntegerValue(10)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, + expectedEq: noCallEq, expectedCount: 0, }, {name: "index=integer, vals=integers, pivot=integer:2", @@ -408,6 +413,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { indexTypes: []document.ValueType{document.IntegerValue}, pivots: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: noCallEq, expectedCount: 0, }, @@ -459,6 +465,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { indexTypes: nil, pivots: values(document.NewDoubleValue(10.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: noCallEq, expectedCount: 0, }, @@ -502,6 +509,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { pivots: values(document.NewTextValue("foo")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, + expectedEq: noCallEq, expectedCount: 0, }, {name: "index=text, vals=text, pivot=text('2')", @@ -516,14 +524,31 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 3, }, // composite -------------------------------------------------- - // composite indexes must have at least have one value + // composite indexes can have empty pivots to iterate on the whole indexed data + {name: "index=[untyped, untyped], vals=[int, int], pivot=[nil,nil]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{}, document.Value{}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + array := document.NewValueBuffer( + document.NewDoubleValue(float64(i)), + document.NewDoubleValue(float64(i+1))) + requireEqualBinary(t, document.NewArrayValue(array), val) + }, + expectedCount: 5, + }, + + // composite indexes must have at least have one value if typed {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", indexTypes: []document.ValueType{0, 0}, pivots: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + expectedEq: noCallEq, + fail: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, @@ -531,7 +556,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + expectedEq: noCallEq, + fail: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, @@ -539,7 +565,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + expectedEq: noCallEq, + fail: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, @@ -547,7 +574,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + expectedEq: noCallEq, + fail: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, @@ -842,6 +870,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { return []document.Value{document.NewIntegerValue(int64(i))} } + noCallEq := func(t *testing.T, i uint8, key []byte, val []byte) { + require.Fail(t, "equality test should not be called here") + } + // the following tests will use that constant to determine how many values needs to be inserted // with the value and noise generators. total := 5 @@ -901,6 +933,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { pivots: values(document.NewIntegerValue(-10)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, + expectedEq: noCallEq, expectedCount: 0, }, {name: "index=integer, vals=integers, pivot=integer:2", @@ -941,6 +974,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { indexTypes: []document.ValueType{document.IntegerValue}, pivots: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, + expectedEq: noCallEq, expectedCount: 0, }, @@ -992,6 +1026,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { indexTypes: nil, pivots: values(document.NewDoubleValue(-10.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, + expectedEq: noCallEq, expectedCount: 0, }, @@ -1095,6 +1130,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: func(i int) []document.Value { return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) }, + expectedEq: noCallEq, expectedCount: 0, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[5, 5]", @@ -1246,12 +1282,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) - }, + expectedEq: noCallEq, expectedCount: 0, }, {name: "index=[int, int], vals=[int, int], pivot=[5, 6]", diff --git a/query/create_test.go b/query/create_test.go index c15210fb8..b8ab9bd23 100644 --- a/query/create_test.go +++ b/query/create_test.go @@ -319,7 +319,7 @@ func TestCreateIndex(t *testing.T) { {"No name", "CREATE UNIQUE INDEX ON test (foo[1])", false}, {"No name if not exists", "CREATE UNIQUE INDEX IF NOT EXISTS ON test (foo[1])", true}, {"No fields", "CREATE INDEX idx ON test", true}, - {"More than 1 field", "CREATE INDEX idx ON test (foo, bar)", true}, + {"More than 1 field", "CREATE INDEX idx ON test (foo, bar)", false}, // TODO(JH) this is yet to be tested } for _, test := range tests { diff --git a/sql/parser/create_test.go b/sql/parser/create_test.go index 1114f893d..d73d30926 100644 --- a/sql/parser/create_test.go +++ b/sql/parser/create_test.go @@ -264,13 +264,22 @@ func TestParserCreateIndex(t *testing.T) { expected query.Statement errored bool }{ - {"Basic", "CREATE INDEX idx ON test (foo)", query.CreateIndexStmt{IndexName: "idx", TableName: "test", Path: document.Path(parsePath(t, "foo"))}, false}, - {"If not exists", "CREATE INDEX IF NOT EXISTS idx ON test (foo.bar[1])", query.CreateIndexStmt{IndexName: "idx", TableName: "test", Path: document.Path(parsePath(t, "foo.bar[1]")), IfNotExists: true}, false}, - {"Unique", "CREATE UNIQUE INDEX IF NOT EXISTS idx ON test (foo[3].baz)", query.CreateIndexStmt{IndexName: "idx", TableName: "test", Path: document.Path(parsePath(t, "foo[3].baz")), IfNotExists: true, Unique: true}, false}, - {"No name", "CREATE UNIQUE INDEX ON test (foo[3].baz)", query.CreateIndexStmt{TableName: "test", Path: document.Path(parsePath(t, "foo[3].baz")), Unique: true}, false}, + {"Basic", "CREATE INDEX idx ON test (foo)", query.CreateIndexStmt{IndexName: "idx", TableName: "test", Paths: []document.Path{document.Path(parsePath(t, "foo"))}}, false}, + {"If not exists", "CREATE INDEX IF NOT EXISTS idx ON test (foo.bar[1])", query.CreateIndexStmt{IndexName: "idx", TableName: "test", Paths: []document.Path{document.Path(parsePath(t, "foo.bar[1]"))}, IfNotExists: true}, false}, + {"Unique", "CREATE UNIQUE INDEX IF NOT EXISTS idx ON test (foo[3].baz)", query.CreateIndexStmt{IndexName: "idx", TableName: "test", Paths: []document.Path{document.Path(parsePath(t, "foo[3].baz"))}, IfNotExists: true, Unique: true}, false}, + {"No name", "CREATE UNIQUE INDEX ON test (foo[3].baz)", query.CreateIndexStmt{TableName: "test", Paths: []document.Path{document.Path(parsePath(t, "foo[3].baz"))}, Unique: true}, false}, {"No name with IF NOT EXISTS", "CREATE UNIQUE INDEX IF NOT EXISTS ON test (foo[3].baz)", nil, true}, + {"More than 1 path", "CREATE INDEX idx ON test (foo, bar)", + query.CreateIndexStmt(query.CreateIndexStmt{ + IndexName: "idx", + TableName: "test", + Paths: []document.Path{ + document.Path(parsePath(t, "foo")), + document.Path(parsePath(t, "bar")), + }, + }), + false}, {"No fields", "CREATE INDEX idx ON test", nil, true}, - {"More than 1 path", "CREATE INDEX idx ON test (foo, bar)", nil, true}, } for _, test := range tests { diff --git a/stream/iterator_test.go b/stream/iterator_test.go index c3f8c80d0..d58052f21 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -289,13 +289,12 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), nil, false, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "no range", "a, b", - // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), - // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), - // nil, false, false, - // }, + { + "no range", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 3}`), + nil, false, false, + }, { "max:2", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -305,16 +304,15 @@ func TestIndexScan(t *testing.T) { }, false, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "max:[2, 2]", "a, b", - // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - // stream.Ranges{ - // {Max: testutil.MakeArray(t, 2, 2)}, - // }, - // false, false, - // }, + { + "max:[2, 2]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + stream.Ranges{ + {Max: testutil.MakeArray(t, 2, 2)}, + }, + false, false, + }, { "max:1", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -324,16 +322,15 @@ func TestIndexScan(t *testing.T) { }, false, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "max:[1, 2]", "a, b", - // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - // testutil.MakeDocuments(t, `{"a": 1, "b": 2}`), - // stream.Ranges{ - // {Max: testutil.MakeArray(t, 1, 2)}, - // }, - // false, false, - // }, + { + "max:[1, 2]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2}`), + stream.Ranges{ + {Max: testutil.MakeArray(t, 1, 2)}, + }, + false, false, + }, { "min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -394,16 +391,15 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "reverse/max", "a, b", - // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), - // testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), - // stream.Ranges{ - // {Max: testutil.MakeArray(t, 2, 2)}, - // }, - // true, false, - // }, + { + "reverse/max", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), + stream.Ranges{ + {Max: testutil.MakeArray(t, 2, 2)}, + }, + true, false, + }, { "reverse/min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -422,16 +418,15 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "reverse/min", "a, b", - // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), - // testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), - // stream.Ranges{ - // {Min: testutil.MakeArray(t, 1, 1)}, - // }, - // true, false, - // }, + { + "reverse/min", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), + testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1, 1)}, + }, + true, false, + }, { "reverse/min/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), @@ -450,16 +445,15 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "max:[1]", "a, b", - // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), - // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), - // stream.Ranges{ - // {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, - // }, - // false, false, - // }, + { + "max:[1]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), + stream.Ranges{ + {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, + }, + false, false, + }, { "reverse max:[1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), @@ -469,17 +463,15 @@ func TestIndexScan(t *testing.T) { }, true, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "max:[1, 2]", "a, b, c", - // testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), - // testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), - // stream.Ranges{ - // {Arity: 3, ArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, - // }, - // false, false, - // }, - // todo a,b,c and [1] + { + "max:[1, 2]", "a, b, c", + testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), + stream.Ranges{ + {Arity: 3, ArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, + }, + false, false, + }, { "min:[1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), @@ -489,16 +481,24 @@ func TestIndexScan(t *testing.T) { }, false, false, }, - // because composite indexes must have at least a defined in [a, b], this case won't happen - // { - // "reverse min:[1]", "a, b", - // testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), - // testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": -2}`), - // stream.Ranges{ - // {Min: testutil.MakeArray(t, 1)}, - // }, - // true, false, - // }, + { + "min:[1]", "a, b, c", + testutil.MakeDocuments(t, `{"a": 1, "b": -2, "c": 0}`, `{"a": -2, "b": 2, "c": 1}`, `{"a": 1, "b": 1, "c": 2}`), + testutil.MakeDocuments(t, `{"a": 1, "b": -2, "c": 0}`, `{"a": 1, "b": 1, "c": 2}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1)}, + }, + false, false, + }, + { + "reverse min:[1]", "a, b", + testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), + testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": -2}`), + stream.Ranges{ + {Min: testutil.MakeArray(t, 1)}, + }, + true, false, + }, { "min:[1], max[2]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), @@ -564,11 +564,11 @@ func TestIndexScan(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - fmt.Println("expected: ") - test.expected.Print() - fmt.Println("got: ") - got.Print() - fmt.Println("end test") + // fmt.Println("expected: ") + // test.expected.Print() + // fmt.Println("got: ") + // got.Print() + // fmt.Println("end test") require.Equal(t, len(test.expected), i) test.expected.RequireEqual(t, got) } From 1b4945b3b756215d88ae3f9dbcef21624b4f0d0c Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 18 Mar 2021 11:32:04 +0100 Subject: [PATCH 07/40] Clean and document composite indexes --- database/index.go | 312 +++++++++++++++++++++------------------- database/index_test.go | 9 ++ stream/iterator.go | 71 +++++---- stream/iterator_test.go | 10 +- 4 files changed, 224 insertions(+), 178 deletions(-) diff --git a/database/index.go b/database/index.go index 6e76747be..be500ccaa 100644 --- a/database/index.go +++ b/database/index.go @@ -13,12 +13,6 @@ import ( const ( // indexStorePrefix is the prefix used to name the index stores. indexStorePrefix = "i" - - // untypedValue is the placeholder type for keys of an index which aren't typed. - // CREATE TABLE foo; - // CREATE INDEX idx_foo_a_b ON foo(a,b); - // document.ValueType of a and b will be untypedValue. - untypedValue = document.ValueType(0) ) var ( @@ -27,7 +21,13 @@ var ( ) // An Index associates encoded values with keys. -// It is sorted by value following the lexicographic order. +// +// The association is performed by encoding the values in a binary format that preserve +// ordering when compared lexicographically. For the implementation, see the binarysort +// package and the document.ValueEncoder. +// +// When the index is composite, the values are wrapped into a document.Array before +// being encoded. type Index struct { Info *IndexInfo @@ -35,16 +35,17 @@ type Index struct { storeName []byte } -// NewIndex creates an index that associates a value with a list of keys. +// NewIndex creates an index that associates values with a list of keys. func NewIndex(tx engine.Transaction, idxName string, opts *IndexInfo) *Index { if opts == nil { opts = &IndexInfo{ - Types: []document.ValueType{untypedValue}, + Types: []document.ValueType{0}, } } + // if no types are provided, it implies that it's an index for single untyped values if opts.Types == nil { - opts.Types = []document.ValueType{untypedValue} + opts.Types = []document.ValueType{0} } return &Index{ @@ -56,10 +57,13 @@ func NewIndex(tx engine.Transaction, idxName string, opts *IndexInfo) *Index { var errStop = errors.New("stop") +// IsComposite returns true if the index is defined to operate on at least more than one value. func (idx *Index) IsComposite() bool { return len(idx.Info.Types) > 1 } +// Arity returns how many values the indexed is operating on. +// CREATE INDEX idx_a_b ON foo (a, b) -> arity: 2 func (idx *Index) Arity() int { return len(idx.Info.Types) } @@ -78,13 +82,13 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { return errors.New("cannot index without a value") } - if len(vs) != len(idx.Info.Types) { + if len(vs) != idx.Arity() { return fmt.Errorf("cannot index %d values on an index of arity %d", len(vs), len(idx.Info.Types)) } for i, typ := range idx.Info.Types { // it is possible to set an index(a,b) on (a), it will be assumed that b is null in that case - if typ != untypedValue && i < len(vs) && typ != vs[i].Type { + if typ != 0 && i < len(vs) && typ != vs[i].Type { // TODO use the full version to clarify the error return fmt.Errorf("cannot index value of type %s in %s index", vs[i].Type, typ) } @@ -151,7 +155,6 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { func (idx *Index) Delete(vs []document.Value, k []byte) error { st, err := getOrCreateStore(idx.tx, idx.storeName) if err != nil { - // TODO, more precise error handling? return nil } @@ -181,18 +184,6 @@ func (idx *Index) Delete(vs []document.Value, k []byte) error { return engine.ErrKeyNotFound } -func allEmpty(pivots []document.Value) bool { - res := true - for _, p := range pivots { - res = res && p.Type == 0 - if !res { - break - } - } - - return res -} - // validatePivots returns an error when the pivots are unsuitable for the index: // - no pivots at all // - having pivots length superior to the index arity @@ -239,16 +230,35 @@ func (idx *Index) validatePivots(pivots []document.Value) error { return nil } +// allEmpty returns true when all pivots are valueless and untyped. +func allEmpty(pivots []document.Value) bool { + res := true + for _, p := range pivots { + res = res && p.Type == 0 + if !res { + break + } + } + + return res +} + // AscendGreaterOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in increasing order and calls the given function for each pair. // If the given function returns an error, the iteration stops and returns that error. -// If the pivot is empty, starts from the beginning. +// If the pivot(s) is/are empty, starts from the beginning. +// When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. +// When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element +// is of that type, without restriction on the type of the following elements. func (idx *Index) AscendGreaterOrEqual(pivots []document.Value, fn func(val, key []byte) error) error { return idx.iterateOnStore(pivots, false, fn) } // DescendLessOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in descreasing order and calls the given function for each pair. // If the given function returns an error, the iteration stops and returns that error. -// If the pivot is empty, starts from the end. +// If the pivot(s) is/are empty, starts from the end. +// When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. +// When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element +// is of that type, without restriction on the type of the following elements. func (idx *Index) DescendLessOrEqual(pivots []document.Value, fn func(val, key []byte) error) error { return idx.iterateOnStore(pivots, true, fn) } @@ -259,16 +269,9 @@ func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func( return err } - for i, typ := range idx.Info.Types { - // if index and pivot are typed but not of the same type - // return no result - // - // don't try to check in case we have less pivots than values - if i >= len(pivots) { - break - } - - if typ != 0 && pivots[i].Type != 0 && typ != pivots[i].Type { + // If index and pivot are typed but not of the same type, return no results. + for i, p := range pivots { + if p.Type != 0 && idx.Info.Types[i] != 0 && p.Type != idx.Info.Types[i] { return nil } } @@ -319,47 +322,15 @@ func (idx *Index) Truncate() error { // If not, encode so that order is preserved regardless of the type. func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { if idx.IsComposite() { - // v has been turned into an array of values being indexed - // TODO add a check - array := v.V.(*document.ValueBuffer) - - // in the case of one of the index keys being untyped and the corresponding - // value being an integer, convert it into a double. - err := array.Iterate(func(i int, vi document.Value) error { - if idx.Info.Types[i] != untypedValue { - return nil - } - - var err error - if vi.Type == document.IntegerValue { - if vi.V == nil { - vi.Type = document.DoubleValue - } else { - vi, err = vi.CastAsDouble() - if err != nil { - return err - } - } - - // update the value with its new type - return array.Replace(i, vi) - } - - return nil - }) - - if err != nil { - return nil, err - } - - // encode the array - return v.MarshalBinary() + return idx.compositeEncodeValue(v) } if idx.Info.Types[0] != 0 { return v.MarshalBinary() } + // in the case of one of the index keys being untyped and the corresponding + // value being an integer, convert it into a double. var err error var buf bytes.Buffer err = document.NewValueEncoder(&buf).Encode(v) @@ -369,6 +340,43 @@ func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { return buf.Bytes(), nil } +func (idx *Index) compositeEncodeValue(v document.Value) ([]byte, error) { + // v has been turned into an array of values being indexed + // if we reach this point, array *must* be a document.ValueBuffer + array := v.V.(*document.ValueBuffer) + + // in the case of one of the index types being 0 (untyped) and the corresponding + // value being an integer, convert it into a double. + err := array.Iterate(func(i int, vi document.Value) error { + if idx.Info.Types[i] != 0 { + return nil + } + + var err error + if vi.Type == document.IntegerValue { + if vi.V == nil { + vi.Type = document.DoubleValue + } else { + vi, err = vi.CastAsDouble() + if err != nil { + return err + } + } + + // update the value + return array.Replace(i, vi) + } + + return nil + }) + + if err != nil { + return nil, err + } + + return v.MarshalBinary() +} + func getOrCreateStore(tx engine.Transaction, name []byte) (engine.Store, error) { st, err := tx.GetStore(name) if err == nil { @@ -387,114 +395,122 @@ func getOrCreateStore(tx engine.Transaction, name []byte) (engine.Store, error) return tx.GetStore(name) } -func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool, fn func(item engine.Item) error) error { +// buildSeek encodes the pivots as binary in order to seek into the indexed data. +// In case of a composite index, the pivots are wrapped in array before being encoded. +// See the Index type documentation for a description of its encoding and its corner cases. +func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, error) { var seek []byte var err error - for i, typ := range idx.Info.Types { - if i < len(pivots) && typ == 0 && pivots[i].Type == document.IntegerValue { - if pivots[i].V == nil { - pivots[i].Type = document.DoubleValue - } else { - pivots[i], err = pivots[i].CastAsDouble() - if err != nil { - return err - } - } + // if we have valueless and typeless pivots, we just iterate + if allEmpty(pivots) { + return []byte{}, nil + } + + // if the index is without type and the first pivot is valueless but typed, iterate but filter out the types we don't want, + // but just for the first pivot; subsequent pivots cannot be filtered this way. + if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && pivots[0].V == nil { + seek = []byte{byte(pivots[0].Type)} + + if reverse { + seek = append(seek, 0xFF) } + + return seek, nil } - if idx.IsComposite() { - // if we have n valueless and typeless pivots, we just iterate - all := true - for _, pivot := range pivots { - if pivot.Type == 0 && pivot.V == nil { - all = all && true - } else { - all = false - break + if !idx.IsComposite() { + if pivots[0].V != nil { + seek, err = idx.EncodeValue(pivots[0]) + if err != nil { + return nil, err } - } - // we do have pivot values/types, so let's use them to seek in the index - if !all { - // TODO delete - // if the first pivot is valueless but typed, we iterate but filter out the types we don't want - // but just for the first pivot. - if pivots[0].Type != 0 && pivots[0].V == nil { + if reverse { + // appending 0xFF turns the pivot into the upper bound of that value. + seek = append(seek, 0xFF) + } + } else { + if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && pivots[0].V == nil { seek = []byte{byte(pivots[0].Type)} - } else { - ppivots := make([]document.Value, 0, len(pivots)) - var last *document.Value - for _, p := range pivots { - if p.V != nil { - ppivots = append(ppivots, p) - } else { - last = &p - break - } + + if reverse { + seek = append(seek, 0xFF) } + } + } + } else { + // [2,3,4,int] is a valid pivot, in which case the last pivot, a valueless typed pivot + // it handled separatedly + valuePivots := make([]document.Value, 0, len(pivots)) + var valuelessPivot *document.Value + for _, p := range pivots { + if p.V != nil { + valuePivots = append(valuePivots, p) + } else { + valuelessPivot = &p + break + } + } - vb := document.NewValueBuffer(ppivots...) - seek, err = idx.EncodeValue(document.NewArrayValue(vb)) + vb := document.NewValueBuffer(valuePivots...) + seek, err = idx.EncodeValue(document.NewArrayValue(vb)) - // if we have a [2, int] case, let's just add the type - if last != nil { - seek = append(seek[:len(seek)-1], byte(0x1f), byte(last.Type), byte(0x1e)) - } + if err != nil { + return nil, err + } - if err != nil { - return err - } - } - } else { // we don't, let's start at the beginning - seek = []byte{} + // if we have a [2, int] case, let's just add the type + if valuelessPivot != nil { + seek = append(seek[:len(seek)-1], byte(0x1f), byte(valuelessPivot.Type), byte(0x1e)) } if reverse { - // if we are reverse on a pivot with less arity, we will get 30 255, which is lower than 31 - // and such will ignore all values. Let's drop the separator in that case + // if we are seeking in reverse on a pivot with lower arity, the comparison will be in between + // arrays of different sizes, the pivot being shorter than the indexed values. + // Because the element separator 0x1F is greater than the array end separator 0x1E, + // the reverse byte 0xFF must be appended before the end separator in order to be able + // to be compared correctly. if len(seek) > 0 { seek = append(seek[:len(seek)-1], 0xFF) } else { seek = append(seek, 0xFF) } - } - } else { - if pivots[0].V != nil { - seek, err = idx.EncodeValue(pivots[0]) - if err != nil { - return err - } + } - if reverse { - seek = append(seek, 0xFF) - } - } else { - if idx.Info.Types[0] == untypedValue && pivots[0].Type != untypedValue && pivots[0].V == nil { - seek = []byte{byte(pivots[0].Type)} + return seek, nil +} - if reverse { - seek = append(seek, 0xFF) +func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool, fn func(item engine.Item) error) error { + var err error + // Convert values into doubles if they are integers and the index is untyped + for i, typ := range idx.Info.Types { + if i < len(pivots) && typ == 0 && pivots[i].Type == document.IntegerValue { + if pivots[i].V == nil { + pivots[i].Type = document.DoubleValue + } else { + pivots[i], err = pivots[i].CastAsDouble() + if err != nil { + return err } } } } + seek, err := idx.buildSeek(pivots, reverse) + if err != nil { + return err + } + it := st.Iterator(engine.IteratorOptions{Reverse: reverse}) defer it.Close() for it.Seek(seek); it.Valid(); it.Next() { itm := it.Item() - // if index is untyped and pivot is typed, only iterate on values with the same type as pivot - if idx.IsComposite() { - // for now, we only check the first element - if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { - return nil - } - } else { + // If index is untyped and pivot is typed, only iterate on values with the same type as pivot + if !idx.IsComposite() { var typ document.ValueType if len(idx.Info.Types) > 0 { typ = idx.Info.Types[0] @@ -503,6 +519,12 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool if (typ == 0) && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { return nil } + } else { + // If the index is composite, same logic applies but for now, we only check the first pivot type. + // A possible optimization would be to check the types of the remaining values here. + if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { + return nil + } } err := fn(itm) diff --git a/database/index_test.go b/database/index_test.go index b07808b8d..871c1c2ec 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -568,6 +568,15 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: noCallEq, fail: true, }, + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, nil]", + indexTypes: []document.ValueType{0, 0, 0}, + pivots: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0), document.Value{}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: noCallEq, + fail: true, + }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, pivots: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), diff --git a/stream/iterator.go b/stream/iterator.go index 6f86a829c..2745faeac 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -275,9 +275,13 @@ func (it *PkScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Enviro type IndexScanOperator struct { baseOperator + // IndexName references the index that will be used to perform the scan IndexName string - Ranges Ranges - Reverse bool + // Ranges defines the boundaries of the scan, each corresponding to one value of the group of values + // being indexed in the case of a composite index. + Ranges Ranges + // Reverse indicates the direction used to traverse the index. + Reverse bool } // IndexScan creates an iterator that iterates over each document of the given table. @@ -355,8 +359,7 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } for _, rng := range it.Ranges { - - if index.IsComposite() { + if !index.IsComposite() { var start, end document.Value if !it.Reverse { start = rng.Min @@ -367,9 +370,6 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } var encEnd []byte - - // deal with the fact that we can't have a zero then values - // TODO(JH) if !end.Type.IsZero() && end.V != nil { encEnd, err = index.EncodeValue(end) if err != nil { @@ -377,19 +377,7 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } } - pivots := []document.Value{} - if start.V != nil { - start.V.(document.Array).Iterate(func(i int, value document.Value) error { - pivots = append(pivots, value) - return nil - }) - } else { - for i := 0; i < index.Arity(); i++ { - pivots = append(pivots, document.Value{}) - } - } - - err = iterator(pivots, func(val, key []byte) error { + err = iterator([]document.Value{start}, func(val, key []byte) error { if !rng.IsInRange(val) { // if we reached the end of our range, we can stop iterating. if encEnd == nil { @@ -419,7 +407,6 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env if err != nil { return err } - } else { var start, end document.Value if !it.Reverse { @@ -431,14 +418,27 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } var encEnd []byte - if !end.Type.IsZero() && end.V != nil { + if end.V != nil { encEnd, err = index.EncodeValue(end) if err != nil { return err } } - err = iterator([]document.Value{start}, func(val, key []byte) error { + // extract the pivots from the range, which in the case of a composite index is an array + pivots := []document.Value{} + if start.V != nil { + start.V.(document.Array).Iterate(func(i int, value document.Value) error { + pivots = append(pivots, value) + return nil + }) + } else { + for i := 0; i < index.Arity(); i++ { + pivots = append(pivots, document.Value{}) + } + } + + err = iterator(pivots, func(val, key []byte) error { if !rng.IsInRange(val) { // if we reached the end of our range, we can stop iterating. if encEnd == nil { @@ -462,12 +462,14 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env newEnv.SetDocument(d) return fn(&newEnv) }) + if err == ErrStreamClosed { err = nil } if err != nil { return err } + } } @@ -485,8 +487,23 @@ type Range struct { // and for determining the global upper bound. Exact bool - Arity int - ArityMax int + // Arity represents the range arity in the case of comparing the range + // to a composite index. With IndexArityMax, it enables to deal with the + // cases of a composite range specifying boundaries partially, ie: + // - Index on (a, b, c) + // - Range is defining a max only for a and b + // Then Arity is set to 2 and IndexArityMax is set to 3 + // + // On + // This field is subject to change when the support for composite index is added + // to the query planner in an ulterior pull-request. + Arity int + + // IndexArityMax represents the underlying Index arity. + // + // This field is subject to change when the support for composite index is added + // to the query planner in an ulterior pull-request. + IndexArityMax int encodedMin, encodedMax []byte rangeType document.ValueType } @@ -685,9 +702,7 @@ func (r *Range) IsInRange(value []byte) bool { // the value is bigger than the lower bound, // see if it matches the upper bound. if r.encodedMax != nil { - if r.ArityMax < r.Arity { - fmt.Println("val", value) - fmt.Println("max", r.encodedMax) + if r.IndexArityMax < r.Arity { cmpMax = bytes.Compare(value[:len(r.encodedMax)-1], r.encodedMax) } else { cmpMax = bytes.Compare(value, r.encodedMax) diff --git a/stream/iterator_test.go b/stream/iterator_test.go index d58052f21..122bc5f09 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -450,7 +450,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), stream.Ranges{ - {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, + {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 1)}, }, false, false, }, @@ -459,7 +459,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 9223372036854775807}`, `{"a": 1, "b": 1}`), stream.Ranges{ - {Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 1)}, + {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 1)}, }, true, false, }, @@ -468,7 +468,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), stream.Ranges{ - {Arity: 3, ArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, + {Arity: 3, IndexArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, }, false, false, }, @@ -504,7 +504,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 2, "b": 42}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1), Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 2)}, + {Min: testutil.MakeArray(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 2)}, }, false, false, }, @@ -513,7 +513,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), testutil.MakeDocuments(t, `{"a": 2, "b": 42}`, `{"a": 1, "b": -2}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1), Arity: 2, ArityMax: 1, Max: testutil.MakeArray(t, 2)}, + {Min: testutil.MakeArray(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 2)}, }, true, false, }, From 3d0255a830b0edf38fd1cf80219a17856d27d4f1 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 18 Mar 2021 11:55:39 +0100 Subject: [PATCH 08/40] Make invalid pivots panic rather than error --- database/index.go | 23 +++++---------- database/index_test.go | 64 ++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/database/index.go b/database/index.go index be500ccaa..8f86e3067 100644 --- a/database/index.go +++ b/database/index.go @@ -89,7 +89,6 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { for i, typ := range idx.Info.Types { // it is possible to set an index(a,b) on (a), it will be assumed that b is null in that case if typ != 0 && i < len(vs) && typ != vs[i].Type { - // TODO use the full version to clarify the error return fmt.Errorf("cannot index value of type %s in %s index", vs[i].Type, typ) } } @@ -188,21 +187,20 @@ func (idx *Index) Delete(vs []document.Value, k []byte) error { // - no pivots at all // - having pivots length superior to the index arity // - having the first pivot without a value when the subsequent ones do have values -func (idx *Index) validatePivots(pivots []document.Value) error { +func (idx *Index) validatePivots(pivots []document.Value) { if len(pivots) == 0 { - return errors.New("cannot iterate without a pivot") + panic("cannot iterate without a pivot") } if len(pivots) > idx.Arity() { - // TODO panic - return errors.New("cannot iterate with a pivot whose size is superior to the index arity") + panic("cannot iterate with a pivot whose size is superior to the index arity") } if idx.IsComposite() { if !allEmpty(pivots) { // the first pivot must have a value if pivots[0].V == nil { - return errors.New("cannot iterate on a composite index whose first pivot has no value") + panic("cannot iterate on a composite index whose first pivot has no value") } // it's acceptable for the last pivot to just have a type and no value @@ -215,19 +213,15 @@ func (idx *Index) validatePivots(pivots []document.Value) error { // if we have no value, we at least need a type if !hasValue { if p.Type == 0 { - return errors.New("cannot iterate on a composite index with a pivot that has holes") + panic("cannot iterate on a composite index with a pivot with both values and nil values") } } } else { - return errors.New("cannot iterate on a composite index with a pivot that has holes") + panic("cannot iterate on a composite index with a pivot with both values and nil values") } } - } else { - return nil } } - - return nil } // allEmpty returns true when all pivots are valueless and untyped. @@ -264,10 +258,7 @@ func (idx *Index) DescendLessOrEqual(pivots []document.Value, fn func(val, key [ } func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func(val, key []byte) error) error { - err := idx.validatePivots(pivots) - if err != nil { - return err - } + idx.validatePivots(pivots) // If index and pivot are typed but not of the same type, return no results. for i, p := range pivots { diff --git a/database/index_test.go b/database/index_test.go index 871c1c2ec..735432147 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -331,7 +331,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq func(t *testing.T, i uint8, key []byte, val []byte) // the total count of iteration that should happen expectedCount int - fail bool + mustPanic bool }{ // integers --------------------------------------------------- {name: "index=untyped, vals=integers, pivot=integer", @@ -548,7 +548,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: noCallEq, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, @@ -557,7 +557,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: noCallEq, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, @@ -566,7 +566,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) }, expectedEq: noCallEq, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, nil]", indexTypes: []document.ValueType{0, 0, 0}, @@ -575,7 +575,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) }, expectedEq: noCallEq, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, @@ -584,7 +584,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: noCallEq, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, @@ -834,15 +834,18 @@ func TestIndexAscendGreaterThan(t *testing.T) { var i uint8 var count int - err := idx.AscendGreaterOrEqual(test.pivots, func(val, rid []byte) error { - test.expectedEq(t, i, rid, val) - i++ - count++ - return nil - }) - if test.fail { - require.Error(t, err) + fn := func() error { + return idx.AscendGreaterOrEqual(test.pivots, func(val, rid []byte) error { + test.expectedEq(t, i, rid, val) + i++ + count++ + return nil + }) + } + if test.mustPanic { + require.Panics(t, func() { _ = fn() }) } else { + err := fn() require.NoError(t, err) require.Equal(t, test.expectedCount, count) } @@ -901,7 +904,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq func(t *testing.T, i uint8, key []byte, val []byte) // the total count of iteration that should happen expectedCount int - fail bool + mustPanic bool }{ // integers --------------------------------------------------- {name: "index=untyped, vals=integers, pivot=integer", @@ -1104,7 +1107,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, @@ -1112,7 +1115,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, @@ -1120,7 +1123,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, @@ -1128,7 +1131,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - fail: true, + mustPanic: true, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, @@ -1395,15 +1398,22 @@ func TestIndexDescendLessOrEqual(t *testing.T) { var i uint8 var count int - err := idx.DescendLessOrEqual(test.pivots, func(val, rid []byte) error { - test.expectedEq(t, uint8(total-1)-i, rid, val) - i++ - count++ - return nil - }) - if test.fail { - require.Error(t, err) + + fn := func() error { + t.Helper() + return idx.DescendLessOrEqual(test.pivots, func(val, rid []byte) error { + test.expectedEq(t, uint8(total-1)-i, rid, val) + i++ + count++ + return nil + }) + } + if test.mustPanic { + require.Panics(t, func() { + _ = fn() + }) } else { + err := fn() require.NoError(t, err) require.Equal(t, test.expectedCount, count) } From c345fb1db89a3cbec8f22febcd4317b3b97cd938 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 18 Mar 2021 12:39:07 +0100 Subject: [PATCH 09/40] Use stringutil and drop debug code --- database/index.go | 4 ++-- planner/optimizer_test.go | 8 -------- stream/iterator_test.go | 5 ----- testutil/document.go | 9 --------- 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/database/index.go b/database/index.go index 8f86e3067..cb30c40af 100644 --- a/database/index.go +++ b/database/index.go @@ -83,13 +83,13 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { } if len(vs) != idx.Arity() { - return fmt.Errorf("cannot index %d values on an index of arity %d", len(vs), len(idx.Info.Types)) + return stringutil.Errorf("cannot index %d values on an index of arity %d", len(vs), len(idx.Info.Types)) } for i, typ := range idx.Info.Types { // it is possible to set an index(a,b) on (a), it will be assumed that b is null in that case if typ != 0 && i < len(vs) && typ != vs[i].Type { - return fmt.Errorf("cannot index value of type %s in %s index", vs[i].Type, typ) + return stringutil.Errorf("cannot index value of type %s in %s index", vs[i].Type, typ) } } diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index 33b7793c4..2dacce5ae 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -271,14 +271,6 @@ func TestRemoveUnnecessaryDedupNodeRule(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Project(parser.MustParseExpr("pk()"))), }, - { - "lolol", - stream.New(stream.SeqScan("foo")). - Pipe(stream.Project(parser.MustParseExpr("c as C"))). - Pipe(stream.Distinct()), - stream.New(stream.SeqScan("foo")). - Pipe(stream.Project(parser.MustParseExpr("c as C"))), - }, } for _, test := range tests { diff --git a/stream/iterator_test.go b/stream/iterator_test.go index 122bc5f09..c9ffeee73 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -564,11 +564,6 @@ func TestIndexScan(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - // fmt.Println("expected: ") - // test.expected.Print() - // fmt.Println("got: ") - // got.Print() - // fmt.Println("end test") require.Equal(t, len(test.expected), i) test.expected.RequireEqual(t, got) } diff --git a/testutil/document.go b/testutil/document.go index d73083c9f..7229e2773 100644 --- a/testutil/document.go +++ b/testutil/document.go @@ -66,15 +66,6 @@ func MakeArray(t testing.TB, jsonArray string) document.Array { type Docs []document.Document -func (docs Docs) Print() { - fmt.Println("----") - for _, d := range docs { - dv := document.NewDocumentValue(d) - fmt.Println(dv) - } - fmt.Println("----") -} - func (docs Docs) RequireEqual(t testing.TB, others Docs) { t.Helper() From b6d878a32f9c87af52c7bf531afbe3873d2e22b0 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 18 Mar 2021 14:28:58 +0100 Subject: [PATCH 10/40] Fix documentation --- database/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/config.go b/database/config.go index 35612d0cf..55ae5fd42 100644 --- a/database/config.go +++ b/database/config.go @@ -235,7 +235,7 @@ type IndexInfo struct { // If set to true, values will be associated with at most one key. False by default. Unique bool - // If set, the index is typed and only accepts that type + // If set, the index is typed and only accepts values of those types. Types []document.ValueType } From b9b020e9b83a7f2aa9632ab507d722e2cda57239 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 18 Mar 2021 14:33:32 +0100 Subject: [PATCH 11/40] Fix multiple paths --- cmd/genji/dbutil/dump.go | 2 +- cmd/genji/shell/command.go | 2 +- query/select_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/genji/dbutil/dump.go b/cmd/genji/dbutil/dump.go index dea2ecea1..21582cf2e 100644 --- a/cmd/genji/dbutil/dump.go +++ b/cmd/genji/dbutil/dump.go @@ -211,7 +211,7 @@ func dumpSchema(tx *genji.Tx, w io.Writer, tableName string) error { } _, err = fmt.Fprintf(w, "CREATE%s INDEX %s ON %s (%s);\n", u, index.Info.IndexName, index.Info.TableName, - index.Info.Path) + index.Info.Paths) if err != nil { return err } diff --git a/cmd/genji/shell/command.go b/cmd/genji/shell/command.go index f20735806..91d87a278 100644 --- a/cmd/genji/shell/command.go +++ b/cmd/genji/shell/command.go @@ -126,7 +126,7 @@ func runIndexesCmd(db *genji.DB, tableName string, w io.Writer) error { return err } - fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, index.Path) + fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, index.Paths) return nil }) diff --git a/query/select_test.go b/query/select_test.go index fab9d826f..6408b302d 100644 --- a/query/select_test.go +++ b/query/select_test.go @@ -32,7 +32,7 @@ func TestSelectStmt(t *testing.T) { {"No table, wildcard", "SELECT *", true, ``, nil}, {"No table, document", "SELECT {a: 1, b: 2 + 1}", false, `[{"{a: 1, b: 2 + 1}":{"a":1,"b":3}}]`, nil}, {"No cond", "SELECT * FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100},{"k":3,"height":100,"weight":200}]`, nil}, - {"No con d Multiple wildcards", "SELECT *, *, color FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square","k":1,"color":"red","size":10,"shape":"square","color":"red"},{"k":2,"color":"blue","size":10,"weight":100,"k":2,"color":"blue","size":10,"weight":100,"color":"blue"},{"k":3,"height":100,"weight":200,"k":3,"height":100,"weight":200,"color":null}]`, nil}, + {"No cond Multiple wildcards", "SELECT *, *, color FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square","k":1,"color":"red","size":10,"shape":"square","color":"red"},{"k":2,"color":"blue","size":10,"weight":100,"k":2,"color":"blue","size":10,"weight":100,"color":"blue"},{"k":3,"height":100,"weight":200,"k":3,"height":100,"weight":200,"color":null}]`, nil}, {"With fields", "SELECT color, shape FROM test", false, `[{"color":"red","shape":"square"},{"color":"blue","shape":null},{"color":null,"shape":null}]`, nil}, {"No cond, wildcard and other field", "SELECT *, color FROM test", false, `[{"color": "red", "k": 1, "color": "red", "size": 10, "shape": "square"}, {"color": "blue", "k": 2, "color": "blue", "size": 10, "weight": 100}, {"color": null, "k": 3, "height": 100, "weight": 200}]`, nil}, {"With DISTINCT", "SELECT DISTINCT * FROM test", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100},{"k":3,"height":100,"weight":200}]`, nil}, From 4bdfadc0785cb177ed8772f19d24bee2c0864b20 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 18 Mar 2021 20:13:57 +0100 Subject: [PATCH 12/40] Fix catalogCache.AddIndex not reseting types --- cmd/genji/dbutil/dump.go | 2 +- cmd/genji/shell/command.go | 2 +- cmd/genji/shell/command_test.go | 6 +++++- database/catalog.go | 10 +++++----- database/config.go | 24 ++++++++++++++---------- document/document.go | 6 ++++++ sql/parser/create.go | 3 ++- 7 files changed, 34 insertions(+), 19 deletions(-) diff --git a/cmd/genji/dbutil/dump.go b/cmd/genji/dbutil/dump.go index 21582cf2e..eb4cf909f 100644 --- a/cmd/genji/dbutil/dump.go +++ b/cmd/genji/dbutil/dump.go @@ -211,7 +211,7 @@ func dumpSchema(tx *genji.Tx, w io.Writer, tableName string) error { } _, err = fmt.Fprintf(w, "CREATE%s INDEX %s ON %s (%s);\n", u, index.Info.IndexName, index.Info.TableName, - index.Info.Paths) + index.Info.Paths[0]) if err != nil { return err } diff --git a/cmd/genji/shell/command.go b/cmd/genji/shell/command.go index 91d87a278..4e2b1c954 100644 --- a/cmd/genji/shell/command.go +++ b/cmd/genji/shell/command.go @@ -126,7 +126,7 @@ func runIndexesCmd(db *genji.DB, tableName string, w io.Writer) error { return err } - fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, index.Paths) + fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, index.Paths[0]) return nil }) diff --git a/cmd/genji/shell/command_test.go b/cmd/genji/shell/command_test.go index 7689f5e6d..88ee869d1 100644 --- a/cmd/genji/shell/command_test.go +++ b/cmd/genji/shell/command_test.go @@ -162,6 +162,10 @@ func TestSaveCommand(t *testing.T) { require.Len(t, indexes, 1) require.Equal(t, "idx_a", indexes[0]) + index, err := tx.GetIndex("idx_a") + require.NoError(t, err) + require.Equal(t, []document.ValueType{document.DoubleValue}, index.Info.Types) + return nil }) require.NoError(t, err) @@ -177,7 +181,7 @@ func TestSaveCommand(t *testing.T) { // check that by iterating through the index and finding the previously inserted values var i int - err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error { i++ return nil }) diff --git a/database/catalog.go b/database/catalog.go index 0911377f1..28a64b065 100644 --- a/database/catalog.go +++ b/database/catalog.go @@ -323,9 +323,10 @@ func (c *Catalog) ReIndex(tx *Transaction, indexName string) error { return c.buildIndex(tx, idx, tb) } +// TODO not yet compatible with composite index func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error { return table.Iterate(func(d document.Document) error { - // TODO + // TODO(JH) v, err := idx.Info.Paths[0].GetValueFromDocument(d) if err == document.ErrFieldNotFound { return nil @@ -334,7 +335,6 @@ func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error { return err } - // TODO err = idx.Set([]document.Value{v}, d.(document.Keyer).RawKey()) if err != nil { return stringutil.Errorf("error while building the index: %w", err) @@ -492,13 +492,13 @@ func (c *catalogCache) AddIndex(tx *Transaction, info *IndexInfo) error { return ErrTableNotFound } - // if the index is created on a field on which we know the type, - // create a typed index. + // if the index is created on a field on which we know the type then create a typed index. + // if the given info contained existing types, they are overriden. + info.Types = nil for _, fc := range ti.FieldConstraints { for _, path := range info.Paths { if fc.Path.IsEqual(path) { if fc.Type != 0 { - // TODO info.Types = append(info.Types, document.ValueType(fc.Type)) } diff --git a/database/config.go b/database/config.go index 55ae5fd42..ac9a3239d 100644 --- a/database/config.go +++ b/database/config.go @@ -235,7 +235,7 @@ type IndexInfo struct { // If set to true, values will be associated with at most one key. False by default. Unique bool - // If set, the index is typed and only accepts values of those types. + // If set, the index is typed and only accepts values of those types . Types []document.ValueType } @@ -247,14 +247,12 @@ func (i *IndexInfo) ToDocument() document.Document { buf.Add("index_name", document.NewTextValue(i.IndexName)) buf.Add("table_name", document.NewTextValue(i.TableName)) - // TODO check that vb := document.NewValueBuffer() for _, path := range i.Paths { vb.Append(document.NewArrayValue(pathToArray(path))) } buf.Add("paths", document.NewArrayValue(vb)) - // TODO check that if i.Types != nil { types := make([]document.Value, 0, len(i.Types)) for _, typ := range i.Types { @@ -290,6 +288,7 @@ func (i *IndexInfo) ScanDocument(d document.Document) error { return err } + i.Paths = nil err = v.V.(document.Array).Iterate(func(ii int, pval document.Value) error { p, err := arrayToPath(pval.V.(document.Array)) if err != nil { @@ -304,18 +303,13 @@ func (i *IndexInfo) ScanDocument(d document.Document) error { return err } - // i.Paths, err = arrayToPath(v.V.(document.Array)) - // if err != nil { - // return err - // } - v, err = d.GetByField("types") if err != nil && err != document.ErrFieldNotFound { return err } - // TODO refacto if err == nil { + i.Types = nil err = v.V.(document.Array).Iterate(func(ii int, tval document.Value) error { i.Types = append(i.Types, document.ValueType(tval.V.(int64))) return nil @@ -331,7 +325,17 @@ func (i *IndexInfo) ScanDocument(d document.Document) error { // Clone returns a copy of the index information. func (i IndexInfo) Clone() *IndexInfo { - return &i + c := i + + c.Paths = make([]document.Path, len(i.Paths)) + for i, p := range i.Paths { + c.Paths[i] = p.Clone() + } + + c.Types = make([]document.ValueType, len(i.Types)) + copy(c.Types, i.Types) + + return &c } type indexStore struct { diff --git a/document/document.go b/document/document.go index 76d096e5e..386d63383 100644 --- a/document/document.go +++ b/document/document.go @@ -558,6 +558,12 @@ func (p Path) GetValueFromArray(a Array) (Value, error) { return p[1:].getValueFromValue(v) } +func (p Path) Clone() Path { + c := make(Path, len(p)) + copy(c, p) + return c +} + func (p Path) getValueFromValue(v Value) (Value, error) { switch v.Type { case DocumentValue: diff --git a/sql/parser/create.go b/sql/parser/create.go index e11cedfb8..11df8d5d4 100644 --- a/sql/parser/create.go +++ b/sql/parser/create.go @@ -331,7 +331,8 @@ func (p *Parser) parseCreateIndexStatement(unique bool) (query.CreateIndexStmt, return stmt, newParseError(scanner.Tokstr(tok, lit), []string{"("}, pos) } - // TODO + // TODO(JH) clean when updating the optimizer. + // Uncommented, it breaks the index scan iterator. // if len(paths) != 1 { // return stmt, &ParseError{Message: "indexes on more than one path are not supported"} // } From 499b5bf936f7415e587b333feef9a1dc004fd428 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Tue, 30 Mar 2021 11:06:32 +0200 Subject: [PATCH 13/40] Update optimizer to use composite indexes --- database/catalog.go | 12 +- database/catalog_test.go | 2 +- database/config.go | 1 - database/index.go | 44 ----- database/index_test.go | 138 ++++---------- expr/comparison.go | 12 ++ planner/optimizer.go | 367 +++++++++++++++++++++++++++----------- planner/optimizer_test.go | 125 ++++++++++++- query/create_test.go | 3 +- stream/iterator.go | 1 + stream/iterator_test.go | 36 ++-- testutil/document.go | 2 +- 12 files changed, 462 insertions(+), 281 deletions(-) diff --git a/database/catalog.go b/database/catalog.go index 28a64b065..68272a58a 100644 --- a/database/catalog.go +++ b/database/catalog.go @@ -495,16 +495,22 @@ func (c *catalogCache) AddIndex(tx *Transaction, info *IndexInfo) error { // if the index is created on a field on which we know the type then create a typed index. // if the given info contained existing types, they are overriden. info.Types = nil - for _, fc := range ti.FieldConstraints { - for _, path := range info.Paths { + +OUTER: + for _, path := range info.Paths { + for _, fc := range ti.FieldConstraints { if fc.Path.IsEqual(path) { + // a constraint may or may enforce a type if fc.Type != 0 { info.Types = append(info.Types, document.ValueType(fc.Type)) } - break + continue OUTER } } + + // no type was inferred for that path, add it to the index as untyped + info.Types = append(info.Types, document.ValueType(0)) } c.indexes[info.IndexName] = info diff --git a/database/catalog_test.go b/database/catalog_test.go index 8c0eaf3ea..3b870032a 100644 --- a/database/catalog_test.go +++ b/database/catalog_test.go @@ -708,5 +708,5 @@ func TestReadOnlyTables(t *testing.T) { doc, err = db.QueryDocument(`CREATE INDEX idx_foo_a ON foo(a); SELECT * FROM __genji_indexes`) require.NoError(t, err) - testutil.RequireDocJSONEq(t, doc, `{"index_name":"idx_foo_a", "path":["a"], "table_name":"foo", "unique":false}`) + testutil.RequireDocJSONEq(t, doc, `{"index_name":"idx_foo_a", "paths":[["a"]], "table_name":"foo", "types":[0], "unique":false}`) } diff --git a/database/config.go b/database/config.go index ac9a3239d..e12966770 100644 --- a/database/config.go +++ b/database/config.go @@ -447,7 +447,6 @@ func (i Indexes) GetIndex(name string) *Index { func (i Indexes) GetIndexByPath(p document.Path) *Index { for _, idx := range i { - // TODO if idx.Info.Paths[0].IsEqual(p) { return idx } diff --git a/database/index.go b/database/index.go index cb30c40af..17f748b51 100644 --- a/database/index.go +++ b/database/index.go @@ -334,37 +334,6 @@ func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { func (idx *Index) compositeEncodeValue(v document.Value) ([]byte, error) { // v has been turned into an array of values being indexed // if we reach this point, array *must* be a document.ValueBuffer - array := v.V.(*document.ValueBuffer) - - // in the case of one of the index types being 0 (untyped) and the corresponding - // value being an integer, convert it into a double. - err := array.Iterate(func(i int, vi document.Value) error { - if idx.Info.Types[i] != 0 { - return nil - } - - var err error - if vi.Type == document.IntegerValue { - if vi.V == nil { - vi.Type = document.DoubleValue - } else { - vi, err = vi.CastAsDouble() - if err != nil { - return err - } - } - - // update the value - return array.Replace(i, vi) - } - - return nil - }) - - if err != nil { - return nil, err - } - return v.MarshalBinary() } @@ -475,19 +444,6 @@ func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, erro func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool, fn func(item engine.Item) error) error { var err error - // Convert values into doubles if they are integers and the index is untyped - for i, typ := range idx.Info.Types { - if i < len(pivots) && typ == 0 && pivots[i].Type == document.IntegerValue { - if pivots[i].V == nil { - pivots[i].Type = document.DoubleValue - } else { - pivots[i], err = pivots[i].CastAsDouble() - if err != nil { - return err - } - } - } - } seek, err := idx.buildSeek(pivots, reverse) if err != nil { diff --git a/database/index_test.go b/database/index_test.go index 735432147..c1be30611 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -159,15 +159,15 @@ func TestIndexDelete(t *testing.T) { err := idx.AscendGreaterOrEqual(pivot, func(v, k []byte) error { if i == 0 { expected := document.NewArrayValue(document.NewValueBuffer( - document.NewDoubleValue(10), - document.NewDoubleValue(10), + document.NewIntegerValue(10), + document.NewIntegerValue(10), )) requireEqualBinary(t, expected, v) require.Equal(t, "other-key", string(k)) } else if i == 1 { expected := document.NewArrayValue(document.NewValueBuffer( - document.NewDoubleValue(11), - document.NewDoubleValue(11), + document.NewIntegerValue(11), + document.NewIntegerValue(11), )) requireEqualBinary(t, expected, v) require.Equal(t, "yet-another-key", string(k)) @@ -228,15 +228,15 @@ func TestIndexDelete(t *testing.T) { switch i { case 0: expected := document.NewArrayValue(document.NewValueBuffer( - document.NewDoubleValue(10), - document.NewDoubleValue(10), + document.NewIntegerValue(10), + document.NewIntegerValue(10), )) requireEqualBinary(t, expected, v) require.Equal(t, "key1", string(k)) case 1: expected := document.NewArrayValue(document.NewValueBuffer( - document.NewDoubleValue(12), - document.NewDoubleValue(12), + document.NewIntegerValue(12), + document.NewIntegerValue(12), )) requireEqualBinary(t, expected, v) require.Equal(t, "key3", string(k)) @@ -341,7 +341,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) }, expectedCount: 5, }, @@ -363,7 +363,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) }, expectedCount: 3, }, @@ -386,29 +386,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 3, }, - // integers, when the index isn't typed can be iterated as doubles - {name: "index=untyped, vals=integers, pivot=double", - indexTypes: nil, - pivots: values(document.Value{Type: document.DoubleValue}), - val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, - expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) - }, - expectedCount: 5, - }, - {name: "index=untyped, vals=integers, pivot=double:1.8", - indexTypes: nil, - pivots: values(document.NewDoubleValue(1.8)), - val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, - expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - i += 2 - require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) - }, - expectedCount: 3, - }, - // but not when the index is typed to integers, although it won't yield an error + // TODO but not when the index is typed to integers, although it won't yield an error {name: "index=integer, vals=integers, pivot=double", indexTypes: []document.ValueType{document.IntegerValue}, pivots: values(document.Value{Type: document.DoubleValue}), @@ -428,17 +406,6 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - // when iterating on doubles, but passing an integer pivot, it'll be casted as a double - {name: "index=untyped, vals=doubles, pivot=integers", - indexTypes: nil, - pivots: values(document.Value{Type: document.IntegerValue}), - val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, - expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) - }, - expectedCount: 5, - }, {name: "index=untyped, vals=doubles, pivot=double:1.8", indexTypes: nil, pivots: values(document.NewDoubleValue(1.8)), @@ -533,8 +500,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 5, @@ -597,8 +564,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 5, @@ -615,8 +582,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 3, @@ -633,8 +600,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 3, @@ -657,8 +624,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 3, @@ -676,7 +643,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), + document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) requireEqualBinary(t, document.NewArrayValue(array), val) }, @@ -727,8 +694,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { if i%2 == 0 { i = i / 2 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) } }, @@ -914,7 +881,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) }, expectedCount: 5, }, @@ -936,7 +903,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) + requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) }, expectedCount: 3, }, @@ -959,29 +926,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 3, }, - // integers, when the index isn't typed can be iterated as doubles - {name: "index=untyped, vals=integers, pivot=double", - indexTypes: nil, - pivots: values(document.Value{Type: document.DoubleValue}), - val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, - expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) - }, - expectedCount: 5, - }, - {name: "index=untyped, vals=integers, pivot=double:1.8", - indexTypes: nil, - pivots: values(document.NewDoubleValue(1.8)), - val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, - expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - i -= 3 - require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)), val) - }, - expectedCount: 2, - }, - // but not when the index is typed to integers, although it won't yield an error + // TODO but not when the index is typed to integers, although it won't yield an error {name: "index=integer, vals=integers, pivot=double", indexTypes: []document.ValueType{document.IntegerValue}, pivots: values(document.Value{Type: document.DoubleValue}), @@ -1001,17 +946,6 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 5, }, - // when iterating on doubles, but passing an integer pivot, it'll be casted as a double - {name: "index=untyped, vals=doubles, pivot=integers", - indexTypes: nil, - pivots: values(document.Value{Type: document.IntegerValue}), - val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, - expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) - }, - expectedCount: 5, - }, {name: "index=untyped, vals=doubles, pivot=double:1.8", indexTypes: nil, pivots: values(document.NewDoubleValue(1.8)), @@ -1156,8 +1090,8 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 5, @@ -1175,8 +1109,8 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 2, @@ -1194,8 +1128,8 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, @@ -1219,8 +1153,8 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), - document.NewDoubleValue(float64(i+1))) + document.NewIntegerValue(int64(i)), + document.NewIntegerValue(int64(i+1))) requireEqualBinary(t, document.NewArrayValue(array), val) }, expectedCount: 3, @@ -1238,7 +1172,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 array := document.NewValueBuffer( - document.NewDoubleValue(float64(i)), + document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) requireEqualBinary(t, document.NewArrayValue(array), val) }, diff --git a/expr/comparison.go b/expr/comparison.go index 1bfda6cf0..997cacb69 100644 --- a/expr/comparison.go +++ b/expr/comparison.go @@ -146,6 +146,12 @@ func IsComparisonOperator(op Operator) bool { return false } +// IsEqualOperator returns true if e is the = operator +func IsEqualOperator(op Operator) bool { + _, ok := op.(*EqOperator) + return ok +} + // IsAndOperator reports if e is the AND operator. func IsAndOperator(op Operator) bool { _, ok := op.(*AndOp) @@ -164,6 +170,12 @@ func IsInOperator(e Expr) bool { return ok } +// IsNotInOperator reports if e is the NOT IN operator. +func IsNotInOperator(e Expr) bool { + _, ok := e.(*NotInOperator) + return ok +} + type InOperator struct { *simpleOperator } diff --git a/planner/optimizer.go b/planner/optimizer.go index d1770129c..b600d4693 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -382,16 +382,24 @@ func isProjectionUnique(indexes database.Indexes, po *stream.ProjectOperator, pk return true } -// UseIndexBasedOnFilterNodeRule scans the tree for the first filter node whose condition is an -// operator that satisfies the following criterias: +type filterNode struct { + path document.Path + v document.Value + f *stream.FilterOperator +} + +// UseIndexBasedOnFilterNodeRule scans the tree for filter nodes whose conditions are +// operators that satisfies the following criterias: // - is a comparison operator // - one of its operands is a path expression that is indexed // - the other operand is a literal value or a parameter -// If found, it will replace the input node by an indexInputNode using this index. +// +// If one or many are found, it will replace the input node by an indexInputNode using this index, +// removing the now irrelevant filter nodes. +// // TODO(asdine): add support for ORDER BY +// TODO(jh): clarify ranges and cost code in composite indexes case func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, params []expr.Param) (*stream.Stream, error) { - n := s.Op - // first we lookup for the seq scan node. // Here we will assume that at this point // if there is one it has to be the @@ -413,20 +421,209 @@ func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, p indexes := t.Indexes() var candidates []*candidate + var filterNodes []filterNode - // look for all selection nodes that satisfy our requirements - for n != nil { + // then we collect all usable filter nodes, in order to see what index (or PK) can be + // used to replace them. + for n := s.Op; n != nil; n = n.GetPrev() { if f, ok := n.(*stream.FilterOperator); ok { - candidate, err := getCandidateFromfilterNode(f, st.TableName, info, indexes) + if f.E == nil { + continue + } + + op, ok := f.E.(expr.Operator) + if !ok { + continue + } + + if !expr.OperatorIsIndexCompatible(op) { + continue + } + + // determine if the operator could benefit from an index + ok, path, e := operatorCanUseIndex(op) + if !ok { + continue + } + + ev, ok := e.(expr.LiteralValue) + if !ok { + continue + } + + v := document.Value(ev) + + filterNodes = append(filterNodes, filterNode{path: path, v: v, f: f}) + + // check for primary keys scan while iterating on the filter nodes + if pk := info.GetPrimaryKey(); pk != nil && pk.Path.IsEqual(path) { + // if both types are different, don't select this scanner + v, ok, err := operandCanUseIndex(pk.Type, pk.Path, info.FieldConstraints, v) + if err != nil { + return nil, err + } + + if !ok { + continue + } else { + cd := candidate{ + filterOps: []*stream.FilterOperator{f}, + isPk: true, + priority: 3, + } + + ranges, err := getRangesFromOp(op, v) + if err != nil { + return nil, err + } + + cd.newOp = stream.PkScan(st.TableName, ranges...) + cd.cost = ranges.Cost() + + candidates = append(candidates, &cd) + } + } + } + } + + findByPath := func(path document.Path) *filterNode { + for _, fno := range filterNodes { + if fno.path.IsEqual(path) { + return &fno + } + } + + return nil + } + + isNodeEq := func(fno *filterNode) bool { + op := fno.f.E.(expr.Operator) + return expr.IsEqualOperator(op) + } + isNodeComp := func(fno *filterNode, includeInOp bool) bool { + op := fno.f.E.(expr.Operator) + if includeInOp { + return expr.IsComparisonOperator(op) + } else { + return expr.IsComparisonOperator(op) && !expr.IsInOperator(op) && !expr.IsNotInOperator(op) + } + } + + // iterate on all indexes for that table, checking for each of them if its paths are matching + // the filter nodes of the given query. +outer: + for _, idx := range indexes { + // order filter nodes by how the index paths order them; if absent, nil in still inserted + found := make([]*filterNode, len(idx.Info.Paths)) + for i, path := range idx.Info.Paths { + fno := findByPath(path) + + if fno != nil { + // mark this path from the index as found + found[i] = fno + } + } + + // Iterate on all the nodes for the given index, checking for each of its path, their is a corresponding node. + // It's possible for an index to be selected if not all of its paths are covered by the nodes, if and only if + // those are contiguous, relatively to the paths, i.e: + // - given idx_foo_abc(a, b, c) + // - given a query SELECT ... WHERE a = 1 AND b > 2 + // - the paths a and b are contiguous in the index definition, this index can be used + // - given a query SELECT ... WHERE a = 1 AND c > 2 + // - the paths a and c are not contiguous in the index definition, this index cannot be used + var fops []*stream.FilterOperator + var rranges []stream.Ranges + contiguous := true + for i, fno := range found { + if contiguous { + if fno == nil { + contiguous = false + continue + } + + // is looking ahead at the next node possible? + if i+1 < len(found) { + // is there another node found after this one? + if found[i+1] != nil { + // current one must be an eq node then + if !isNodeEq(fno) { + continue outer + } + } else { + // the next node is the last one found, so the current one can also be a comparison and not just eq + if !isNodeComp(fno, false) { + continue outer + } + } + } else { + // that's the last filter node, it can be a comparison, + // in the case of a potentially using a simple index, also a IN operator + if !isNodeComp(fno, len(found) == 1) { + continue outer + } + } + + // what the index says this node type must be + typ := idx.Info.Types[i] + + fno.v, ok, err = operandCanUseIndex(typ, fno.path, info.FieldConstraints, fno.v) + if err != nil { + return nil, err + } + if !ok { + continue outer + } + } else { + // if on the index idx_abc(a,b,c), a is found, b isn't but c is + if fno != nil { + // then idx_abc cannot be used, it's not possible to use the index without a value for b + continue outer + } else { + continue + } + } + + op := fno.f.E.(expr.Operator) + ranges, err := getRangesFromOp(op, fno.v) if err != nil { return nil, err } - if candidate != nil { - candidates = append(candidates, candidate) - } + + rranges = append(rranges, ranges) + fops = append(fops, fno.f) } - n = n.GetPrev() + // no nodes for the index has been found + if found[0] == nil { + continue outer + } + + cd := candidate{ + filterOps: fops, + isIndex: true, + } + + // there are probably less values to iterate on if the index is unique + if idx.Info.Unique { + cd.priority = 2 + } else { + cd.priority = 1 + } + + // merges the ranges inferred from each filter op into a single one + var ranges stream.Ranges + if idx.IsComposite() { + rng := compactCompIndexRanges(rranges, idx.Arity()) + ranges = ranges.Append(rng) + } else { + ranges = rranges[0] + } + + cd.newOp = stream.IndexScan(idx.Info.IndexName, ranges...) + cd.cost = ranges.Cost() + + candidates = append(candidates, &cd) } // determine which index is the most interesting and replace it in the tree. @@ -444,15 +641,27 @@ func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, p continue } - if currentCost < cost { + // With the current cost be computing on ranges, it's a bit hard to know what's best in + // between indexes. So, before looking at the cost, we look at how many filter ops would + // be replaced. + if len(selectedCandidate.filterOps) < len(candidate.filterOps) { selectedCandidate = candidates[i] cost = currentCost - } + continue + } else if len(selectedCandidate.filterOps) == len(candidate.filterOps) { + if currentCost < cost { + selectedCandidate = candidates[i] + cost = currentCost + continue + } - // if the cost is the same and the candidate's related index has a higher priority, - // select it. - if currentCost == cost && selectedCandidate.priority < candidate.priority { - selectedCandidate = candidates[i] + // if the cost is the same and the candidate's related index has a higher priority, + // select it. + if currentCost == cost { + if selectedCandidate.priority < candidate.priority { + selectedCandidate = candidates[i] + } + } } } @@ -461,7 +670,9 @@ func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, p } // remove the selection node from the tree - s.Remove(selectedCandidate.filterOp) + for _, f := range selectedCandidate.filterOps { + s.Remove(f) + } // we replace the seq scan node by the selected index scan node stream.InsertBefore(s.First(), selectedCandidate.newOp) @@ -471,10 +682,37 @@ func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, p return s, nil } +func compactCompIndexRanges(rangesList []stream.Ranges, indexArity int) stream.Range { + var rng stream.Range + for _, rs := range rangesList { + if rs[0].Min.V != nil { + if rng.Min.V == nil { + rng.Min = document.NewArrayValue(document.NewValueBuffer()) + } + + rng.Min.V.(*document.ValueBuffer).Append(rs[0].Min) + } + + if rs[0].Max.V != nil { + if rng.Max.V == nil { + rng.Max = document.NewArrayValue(document.NewValueBuffer()) + } + + rng.Max.V.(*document.ValueBuffer).Append(rs[0].Max) + } + } + + rng.Exact = rangesList[len(rangesList)-1][0].Exact + rng.Exclusive = rangesList[len(rangesList)-1][0].Exclusive + rng.Arity = indexArity + + return rng +} + type candidate struct { - // filter operator to remove and replace by either an indexScan + // filter operators to remove and replace by either an indexScan // or pkScan operators. - filterOp *stream.FilterOperator + filterOps []*stream.FilterOperator // the candidate indexScan or pkScan operator newOp stream.Operator // the cost of the candidate @@ -488,91 +726,6 @@ type candidate struct { priority int } -// getCandidateFromfilterNode analyses f and determines if it can be replaced by an indexScan or pkScan operator. -func getCandidateFromfilterNode(f *stream.FilterOperator, tableName string, info *database.TableInfo, indexes database.Indexes) (*candidate, error) { - if f.E == nil { - return nil, nil - } - - // the root of the condition must be an operator - op, ok := f.E.(expr.Operator) - if !ok { - return nil, nil - } - - // determine if the operator can read from the index - if !expr.OperatorIsIndexCompatible(op) { - return nil, nil - } - - // determine if the operator can benefit from an index - ok, path, e := operatorCanUseIndex(op) - if !ok { - return nil, nil - } - - // analyse the other operand to make sure it's a literal - ev, ok := e.(expr.LiteralValue) - if !ok { - return nil, nil - } - v := document.Value(ev) - - // now, we look if an index exists for that path - cd := candidate{ - filterOp: f, - } - - // we'll start with checking if the path is the primary key of the table - if pk := info.GetPrimaryKey(); pk != nil && pk.Path.IsEqual(path) { - // check if the operand can be used and convert it when possible - v, ok, err := operandCanUseIndex(pk.Type, pk.Path, info.FieldConstraints, v) - if err != nil || !ok { - return nil, err - } - - cd.isPk = true - cd.priority = 3 - - ranges, err := getRangesFromOp(op, v) - if err != nil { - return nil, err - } - - cd.newOp = stream.PkScan(tableName, ranges...) - cd.cost = ranges.Cost() - return &cd, nil - } - - // if not, check if an index exists for that path - if idx := indexes.GetIndexByPath(document.Path(path)); idx != nil { - // check if the operand can be used and convert it when possible - v, ok, err := operandCanUseIndex(idx.Info.Types[0], idx.Info.Paths[0], info.FieldConstraints, v) - if err != nil || !ok { - return nil, err - } - - cd.isIndex = true - if idx.Info.Unique { - cd.priority = 2 - } else { - cd.priority = 1 - } - - ranges, err := getRangesFromOp(op, v) - if err != nil { - return nil, err - } - - cd.newOp = stream.IndexScan(idx.Info.IndexName, ranges...) - cd.cost = ranges.Cost() - - return &cd, nil - } - - return nil, nil -} - func operatorCanUseIndex(op expr.Operator) (bool, document.Path, expr.Expr) { lf, leftIsField := op.LeftHand().(expr.Path) rf, rightIsField := op.RightHand().(expr.Path) @@ -656,7 +809,7 @@ func getRangesFromOp(op expr.Operator, v document.Value) (stream.Ranges, error) Max: v, }) case *expr.InOperator: - // opCanUseIndex made sure e is an array. + // operatorCanUseIndex made sure e is an array. a := v.V.(document.Array) err := a.Iterate(func(i int, value document.Value) error { ranges = ranges.Append(stream.Range{ diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index 2dacce5ae..b5c0934e3 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -300,7 +300,7 @@ func TestRemoveUnnecessaryDedupNodeRule(t *testing.T) { } } -func TestUseIndexBasedOnSelectionNodeRule(t *testing.T) { +func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { tests := []struct { name string root, expected *st.Stream @@ -320,8 +320,8 @@ func TestUseIndexBasedOnSelectionNodeRule(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), - st.New(st.IndexScan("idx_foo_b", st.Range{Min: document.NewIntegerValue(2), Exact: true})). - Pipe(st.Filter(parser.MustParseExpr("a = 1"))), + st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true})). + Pipe(st.Filter(parser.MustParseExpr("b = 2"))), }, { "FROM foo WHERE c = 3 AND b = 2", @@ -530,3 +530,122 @@ func TestUseIndexBasedOnSelectionNodeRule(t *testing.T) { } }) } + +func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { + tests := []struct { + name string + root, expected *st.Stream + }{ + { + "FROM foo WHERE a = 1 AND d = 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("d = 2"))), + st.New(st.IndexScan("idx_foo_a_d", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exact: true})), + }, + { + "FROM foo WHERE a = 1 AND d > 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("d > 2"))), + st.New(st.IndexScan("idx_foo_a_d", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exclusive: true})), + }, + { + "FROM foo WHERE a = 1 AND d >= 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("d >= 2"))), + st.New(st.IndexScan("idx_foo_a_d", st.Range{Min: testutil.MakeArrayValue(t, 1, 2)})), + }, + { + "FROM foo WHERE a > 1 AND d > 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a > 1"))). + Pipe(st.Filter(parser.MustParseExpr("d > 2"))), + st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exclusive: true})). + Pipe(st.Filter(parser.MustParseExpr("d > 2"))), + }, + { + "FROM foo WHERE a = 1 AND b = 2 AND c = 3", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("b = 2"))). + Pipe(st.Filter(parser.MustParseExpr("c = 3"))), + st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2, 3), Exact: true})), + }, + { + "FROM foo WHERE a = 1 AND b = 2", // c is omitted, but it can still use idx_foo_a_b_c + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("b = 2"))), + st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exact: true})), + }, + { + "FROM foo WHERE a = 1 AND b = 2 and k = 3", // c is omitted, but it can still use idx_foo_a_b_c + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("b = 2"))). + Pipe(st.Filter(parser.MustParseExpr("k = 3"))), + st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exact: true})). + Pipe(st.Filter(parser.MustParseExpr("k = 3"))), + }, + { + "FROM foo WHERE a = 1 AND c = 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("c = 2"))), + // c will be picked because it's a unique index and thus has a lower cost + st.New(st.IndexScan("idx_foo_c", st.Range{Min: document.NewIntegerValue(2), Exact: true})). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))), + }, + { + "FROM foo WHERE b = 1 AND c = 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("b = 1"))). + Pipe(st.Filter(parser.MustParseExpr("c = 2"))), + // c will be picked because it's a unique index and thus has a lower cost + st.New(st.IndexScan("idx_foo_c", st.Range{Min: document.NewIntegerValue(2), Exact: true})). + Pipe(st.Filter(parser.MustParseExpr("b = 1"))), + }, + { + "FROM foo WHERE a = 1 AND b = 2 AND c = 'a'", // c is from the wrong type and will prevent the index to be picked + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("b = 2"))). + Pipe(st.Filter(parser.MustParseExpr("c = 'a'"))), + st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true})). + Pipe(st.Filter(parser.MustParseExpr("b = 2"))). + Pipe(st.Filter(parser.MustParseExpr("c = 'a'"))), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, err := genji.Open(":memory:") + require.NoError(t, err) + defer db.Close() + + tx, err := db.Begin(true) + require.NoError(t, err) + defer tx.Rollback() + + err = tx.Exec(` + CREATE TABLE foo (k INT PRIMARY KEY, c INT); + CREATE INDEX idx_foo_a ON foo(a); + CREATE INDEX idx_foo_b ON foo(b); + CREATE UNIQUE INDEX idx_foo_c ON foo(c); + CREATE INDEX idx_foo_a_d ON foo(a, d); + CREATE INDEX idx_foo_a_b_c ON foo(a, b, c); + INSERT INTO foo (k, a, b, c, d) VALUES + (1, 1, 1, 1, 1), + (2, 2, 2, 2, 2), + (3, 3, 3, 3, 3) + `) + require.NoError(t, err) + + res, err := planner.UseIndexBasedOnFilterNodeRule(test.root, tx.Transaction, nil) + require.NoError(t, err) + require.Equal(t, test.expected.String(), res.String()) + }) + } +} diff --git a/query/create_test.go b/query/create_test.go index b8ab9bd23..6e522ae5e 100644 --- a/query/create_test.go +++ b/query/create_test.go @@ -319,7 +319,8 @@ func TestCreateIndex(t *testing.T) { {"No name", "CREATE UNIQUE INDEX ON test (foo[1])", false}, {"No name if not exists", "CREATE UNIQUE INDEX IF NOT EXISTS ON test (foo[1])", true}, {"No fields", "CREATE INDEX idx ON test", true}, - {"More than 1 field", "CREATE INDEX idx ON test (foo, bar)", false}, // TODO(JH) this is yet to be tested + {"Composite (2)", "CREATE INDEX idx ON test (foo, bar)", false}, + {"Composite (4)", "CREATE INDEX idx ON test (foo, bar, baz, baf)", false}, } for _, test := range tests { diff --git a/stream/iterator.go b/stream/iterator.go index 2745faeac..2217e80f2 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -664,6 +664,7 @@ func (r Ranges) Cost() int { // if there are two boundaries, increment by 50 if !rng.Min.Type.IsZero() && !rng.Max.Type.IsZero() { cost += 50 + continue } // if there is only one boundary, increment by 100 diff --git a/stream/iterator_test.go b/stream/iterator_test.go index c9ffeee73..703ef3da2 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -309,7 +309,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), stream.Ranges{ - {Max: testutil.MakeArray(t, 2, 2)}, + {Max: testutil.MakeArrayValue(t, 2, 2)}, }, false, false, }, @@ -327,7 +327,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2}`), stream.Ranges{ - {Max: testutil.MakeArray(t, 1, 2)}, + {Max: testutil.MakeArrayValue(t, 1, 2)}, }, false, false, }, @@ -345,7 +345,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 2, 1)}, + {Min: testutil.MakeArrayValue(t, 2, 1)}, }, false, false, }, @@ -363,7 +363,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2)}, + {Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2)}, }, false, false, }, @@ -372,7 +372,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 3}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 3}`, `{"a": 2, "b": 2}`), // [1, 3] < [2, 2] stream.Ranges{ - {Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2)}, + {Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2)}, }, false, false, }, @@ -396,7 +396,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), stream.Ranges{ - {Max: testutil.MakeArray(t, 2, 2)}, + {Max: testutil.MakeArrayValue(t, 2, 2)}, }, true, false, }, @@ -423,7 +423,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1, 1)}, + {Min: testutil.MakeArrayValue(t, 1, 1)}, }, true, false, }, @@ -441,7 +441,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2)}, + {Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2)}, }, true, false, }, @@ -450,7 +450,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), stream.Ranges{ - {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 1)}, + {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 1)}, }, false, false, }, @@ -459,7 +459,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 9223372036854775807}`, `{"a": 1, "b": 1}`), stream.Ranges{ - {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 1)}, + {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 1)}, }, true, false, }, @@ -468,7 +468,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), stream.Ranges{ - {Arity: 3, IndexArityMax: 2, Max: testutil.MakeArray(t, 1, 2)}, + {Arity: 3, IndexArityMax: 2, Max: testutil.MakeArrayValue(t, 1, 2)}, }, false, false, }, @@ -477,7 +477,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 1, "b": 1}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1)}, + {Min: testutil.MakeArrayValue(t, 1)}, }, false, false, }, @@ -486,7 +486,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2, "c": 0}`, `{"a": -2, "b": 2, "c": 1}`, `{"a": 1, "b": 1, "c": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": -2, "c": 0}`, `{"a": 1, "b": 1, "c": 2}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1)}, + {Min: testutil.MakeArrayValue(t, 1)}, }, false, false, }, @@ -495,7 +495,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": -2}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1)}, + {Min: testutil.MakeArrayValue(t, 1)}, }, true, false, }, @@ -504,7 +504,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 2, "b": 42}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 2)}, + {Min: testutil.MakeArrayValue(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 2)}, }, false, false, }, @@ -513,7 +513,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), testutil.MakeDocuments(t, `{"a": 2, "b": 42}`, `{"a": 1, "b": -2}`), stream.Ranges{ - {Min: testutil.MakeArray(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArray(t, 2)}, + {Min: testutil.MakeArrayValue(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 2)}, }, true, false, }, @@ -586,11 +586,11 @@ func TestIndexScan(t *testing.T) { t.Run("idx_test_a_b", func(t *testing.T) { require.Equal(t, `indexScan("idx_test_a_b", [[1, 1], [2, 2]])`, stream.IndexScan("idx_test_a_b", stream.Range{ - Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2), + Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2), }).String()) op := stream.IndexScan("idx_test_a_b", stream.Range{ - Min: testutil.MakeArray(t, 1, 1), Max: testutil.MakeArray(t, 2, 2), + Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2), }) op.Reverse = true diff --git a/testutil/document.go b/testutil/document.go index 7229e2773..5b73aedf6 100644 --- a/testutil/document.go +++ b/testutil/document.go @@ -19,7 +19,7 @@ func MakeValue(t testing.TB, v interface{}) document.Value { return vv } -func MakeArray(t testing.TB, vs ...interface{}) document.Value { +func MakeArrayValue(t testing.TB, vs ...interface{}) document.Value { t.Helper() vvs := []document.Value{} From cbf95e8251569673bb7a2c9cca821c1b3dc10908 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 5 Apr 2021 18:12:32 +0200 Subject: [PATCH 14/40] Update the commands to support composite indexes --- cmd/genji/dbutil/dump.go | 8 ++++++-- cmd/genji/dbutil/dump_test.go | 13 +++++++++---- cmd/genji/shell/command.go | 8 +++++++- cmd/genji/shell/command_test.go | 16 ++++++++-------- database/catalog.go | 20 +++++++++++--------- database/config_test.go | 3 +-- database/table.go | 24 ++++++++++++++---------- sql/parser/create.go | 6 ------ 8 files changed, 56 insertions(+), 42 deletions(-) diff --git a/cmd/genji/dbutil/dump.go b/cmd/genji/dbutil/dump.go index eb4cf909f..f69cf5fe3 100644 --- a/cmd/genji/dbutil/dump.go +++ b/cmd/genji/dbutil/dump.go @@ -210,8 +210,12 @@ func dumpSchema(tx *genji.Tx, w io.Writer, tableName string) error { u = " UNIQUE" } - _, err = fmt.Fprintf(w, "CREATE%s INDEX %s ON %s (%s);\n", u, index.Info.IndexName, index.Info.TableName, - index.Info.Paths[0]) + var paths []string + for _, path := range index.Info.Paths { + paths = append(paths, path.String()) + } + + _, err = fmt.Fprintf(w, "CREATE%s INDEX %s ON %s (%s);\n", u, index.Info.IndexName, index.Info.TableName, strings.Join(paths, ", ")) if err != nil { return err } diff --git a/cmd/genji/dbutil/dump_test.go b/cmd/genji/dbutil/dump_test.go index cb306dd97..e43eebf7d 100644 --- a/cmd/genji/dbutil/dump_test.go +++ b/cmd/genji/dbutil/dump_test.go @@ -58,22 +58,27 @@ func TestDump(t *testing.T) { require.NoError(t, err) writeToBuf(q + "\n") - q = fmt.Sprintf(`CREATE INDEX idx_a_%s ON %s (a);`, table, table) + q = fmt.Sprintf(`CREATE INDEX idx_%s_a ON %s (a);`, table, table) + err = db.Exec(q) + require.NoError(t, err) + writeToBuf(q + "\n") + + q = fmt.Sprintf(`CREATE INDEX idx_%s_b_c ON %s (a);`, table, table) err = db.Exec(q) require.NoError(t, err) writeToBuf(q + "\n") - q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d};`, table, 1, 2) + q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d, "c": %d};`, table, 1, 2, 3) err = db.Exec(q) require.NoError(t, err) writeToBuf(q + "\n") - q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d};`, table, 2, 2) + q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d, "c": %d};`, table, 2, 2, 2) err = db.Exec(q) require.NoError(t, err) writeToBuf(q + "\n") - q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d};`, table, 3, 2) + q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d, "c": %d};`, table, 3, 2, 1) err = db.Exec(q) require.NoError(t, err) writeToBuf(q + "\n") diff --git a/cmd/genji/shell/command.go b/cmd/genji/shell/command.go index 4e2b1c954..b4f604d25 100644 --- a/cmd/genji/shell/command.go +++ b/cmd/genji/shell/command.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "strings" "github.com/genjidb/genji" "github.com/genjidb/genji/cmd/genji/dbutil" @@ -126,7 +127,12 @@ func runIndexesCmd(db *genji.DB, tableName string, w io.Writer) error { return err } - fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, index.Paths[0]) + var paths []string + for _, path := range index.Paths { + paths = append(paths, path.String()) + } + + fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, strings.Join(paths, ", ")) return nil }) diff --git a/cmd/genji/shell/command_test.go b/cmd/genji/shell/command_test.go index 88ee869d1..4f8880eb1 100644 --- a/cmd/genji/shell/command_test.go +++ b/cmd/genji/shell/command_test.go @@ -60,7 +60,7 @@ func TestIndexesCmd(t *testing.T) { want string fails bool }{ - {"All", "", "idx_bar_a ON bar (a)\nidx_foo_a ON foo (a)\nidx_foo_b ON foo (b)\n", false}, + {"All", "", "idx_bar_a_b ON bar (a, b)\nidx_foo_a ON foo (a)\nidx_foo_b ON foo (b)\n", false}, {"With table name", "foo", "idx_foo_a ON foo (a)\nidx_foo_b ON foo (b)\n", false}, {"With nonexistent table name", "baz", "", true}, } @@ -76,7 +76,7 @@ func TestIndexesCmd(t *testing.T) { CREATE INDEX idx_foo_a ON foo (a); CREATE INDEX idx_foo_b ON foo (b); CREATE TABLE bar; - CREATE INDEX idx_bar_a ON bar (a); + CREATE INDEX idx_bar_a_b ON bar (a, b); `) require.NoError(t, err) @@ -117,7 +117,7 @@ func TestSaveCommand(t *testing.T) { err = db.Exec(` CREATE TABLE test (a DOUBLE); - CREATE INDEX idx_a ON test (a); + CREATE INDEX idx_a_b ON test (a, b); `) require.NoError(t, err) err = db.Exec("INSERT INTO test (a, b) VALUES (?, ?)", 1, 2) @@ -160,11 +160,11 @@ func TestSaveCommand(t *testing.T) { err = db.View(func(tx *genji.Tx) error { indexes := tx.ListIndexes() require.Len(t, indexes, 1) - require.Equal(t, "idx_a", indexes[0]) + require.Equal(t, "idx_a_b", indexes[0]) - index, err := tx.GetIndex("idx_a") + index, err := tx.GetIndex("idx_a_b") require.NoError(t, err) - require.Equal(t, []document.ValueType{document.DoubleValue}, index.Info.Types) + require.Equal(t, []document.ValueType{document.DoubleValue, 0}, index.Info.Types) return nil }) @@ -176,12 +176,12 @@ func TestSaveCommand(t *testing.T) { defer tx.Rollback() - idx, err := tx.GetIndex("idx_a") + idx, err := tx.GetIndex("idx_a_b") require.NoError(t, err) // check that by iterating through the index and finding the previously inserted values var i int - err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error { + err = idx.AscendGreaterOrEqual([]document.Value{document.NewDoubleValue(0)}, func(v, k []byte) error { i++ return nil }) diff --git a/database/catalog.go b/database/catalog.go index 68272a58a..7b020e48e 100644 --- a/database/catalog.go +++ b/database/catalog.go @@ -323,19 +323,21 @@ func (c *Catalog) ReIndex(tx *Transaction, indexName string) error { return c.buildIndex(tx, idx, tb) } -// TODO not yet compatible with composite index func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error { return table.Iterate(func(d document.Document) error { - // TODO(JH) - v, err := idx.Info.Paths[0].GetValueFromDocument(d) - if err == document.ErrFieldNotFound { - return nil - } - if err != nil { - return err + var err error + values := make([]document.Value, len(idx.Info.Paths)) + for i, path := range idx.Info.Paths { + values[i], err = path.GetValueFromDocument(d) + if err == document.ErrFieldNotFound { + return nil + } + if err != nil { + return err + } } - err = idx.Set([]document.Value{v}, d.(document.Keyer).RawKey()) + err = idx.Set(values, d.(document.Keyer).RawKey()) if err != nil { return stringutil.Errorf("error while building the index: %w", err) } diff --git a/database/config_test.go b/database/config_test.go index 421fca159..0561ef544 100644 --- a/database/config_test.go +++ b/database/config_test.go @@ -163,8 +163,7 @@ func TestIndexStore(t *testing.T) { TableName: "test", IndexName: "idx_test", Unique: true, - // TODO - Types: []document.ValueType{document.BoolValue}, + Types: []document.ValueType{document.BoolValue}, } err = idxs.Insert(&cfg) diff --git a/database/table.go b/database/table.go index fdb12bb72..170ffbd08 100644 --- a/database/table.go +++ b/database/table.go @@ -142,13 +142,15 @@ func (t *Table) Delete(key []byte) error { indexes := t.Indexes() for _, idx := range indexes { - // TODO only support one path - v, err := idx.Info.Paths[0].GetValueFromDocument(d) - if err != nil { - return err + values := make([]document.Value, len(idx.Info.Paths)) + for i, path := range idx.Info.Paths { + values[i], err = path.GetValueFromDocument(d) + if err != nil { + return err + } } - err = idx.Delete([]document.Value{v}, key) + err = idx.Delete(values, key) if err != nil { return err } @@ -186,13 +188,15 @@ func (t *Table) replace(indexes []*Index, key []byte, d document.Document) error // remove key from indexes for _, idx := range indexes { - // TODO only support one path - v, err := idx.Info.Paths[0].GetValueFromDocument(old) - if err != nil { - v = document.NewNullValue() + values := make([]document.Value, len(idx.Info.Paths)) + for i, path := range idx.Info.Paths { + values[i], err = path.GetValueFromDocument(old) + if err != nil { + values[i] = document.NewNullValue() + } } - err = idx.Delete([]document.Value{v}, key) + err = idx.Delete(values, key) if err != nil { return err } diff --git a/sql/parser/create.go b/sql/parser/create.go index 11df8d5d4..d06fcb509 100644 --- a/sql/parser/create.go +++ b/sql/parser/create.go @@ -331,12 +331,6 @@ func (p *Parser) parseCreateIndexStatement(unique bool) (query.CreateIndexStmt, return stmt, newParseError(scanner.Tokstr(tok, lit), []string{"("}, pos) } - // TODO(JH) clean when updating the optimizer. - // Uncommented, it breaks the index scan iterator. - // if len(paths) != 1 { - // return stmt, &ParseError{Message: "indexes on more than one path are not supported"} - // } - stmt.Paths = paths return stmt, nil From 8ec07934ec79c24b4e0e7d13ad762db0ce3eb0fe Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 5 Apr 2021 19:13:13 +0200 Subject: [PATCH 15/40] Refactor the ranges computation for comp indexes --- planner/optimizer.go | 93 +++++++++++++++++++++------------------ planner/optimizer_test.go | 68 ++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 42 deletions(-) diff --git a/planner/optimizer.go b/planner/optimizer.go index b600d4693..d07a6d9d2 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -533,7 +533,7 @@ outer: // - given a query SELECT ... WHERE a = 1 AND c > 2 // - the paths a and c are not contiguous in the index definition, this index cannot be used var fops []*stream.FilterOperator - var rranges []stream.Ranges + var usableFilterNodes []*filterNode contiguous := true for i, fno := range found { if contiguous { @@ -584,13 +584,7 @@ outer: } } - op := fno.f.E.(expr.Operator) - ranges, err := getRangesFromOp(op, fno.v) - if err != nil { - return nil, err - } - - rranges = append(rranges, ranges) + usableFilterNodes = append(usableFilterNodes, fno) fops = append(fops, fno.f) } @@ -611,13 +605,9 @@ outer: cd.priority = 1 } - // merges the ranges inferred from each filter op into a single one - var ranges stream.Ranges - if idx.IsComposite() { - rng := compactCompIndexRanges(rranges, idx.Arity()) - ranges = ranges.Append(rng) - } else { - ranges = rranges[0] + ranges, err := getRangesFromFilterNodes(usableFilterNodes, idx.Arity()) + if err != nil { + return nil, err } cd.newOp = stream.IndexScan(idx.Info.IndexName, ranges...) @@ -682,33 +672,6 @@ outer: return s, nil } -func compactCompIndexRanges(rangesList []stream.Ranges, indexArity int) stream.Range { - var rng stream.Range - for _, rs := range rangesList { - if rs[0].Min.V != nil { - if rng.Min.V == nil { - rng.Min = document.NewArrayValue(document.NewValueBuffer()) - } - - rng.Min.V.(*document.ValueBuffer).Append(rs[0].Min) - } - - if rs[0].Max.V != nil { - if rng.Max.V == nil { - rng.Max = document.NewArrayValue(document.NewValueBuffer()) - } - - rng.Max.V.(*document.ValueBuffer).Append(rs[0].Max) - } - } - - rng.Exact = rangesList[len(rangesList)-1][0].Exact - rng.Exclusive = rangesList[len(rangesList)-1][0].Exclusive - rng.Arity = indexArity - - return rng -} - type candidate struct { // filter operators to remove and replace by either an indexScan // or pkScan operators. @@ -781,6 +744,52 @@ func operandCanUseIndex(indexType document.ValueType, path document.Path, fc dat return converted, indexType == converted.Type, nil } +func getRangesFromFilterNodes(fnodes []*filterNode, indexArity int) (stream.Ranges, error) { + if indexArity <= 1 { + op := fnodes[0].f.E.(expr.Operator) + return getRangesFromOp(op, fnodes[0].v) + } + + vb := document.NewValueBuffer() + for _, fno := range fnodes { + op := fno.f.E.(expr.Operator) + v := fno.v + + switch op.(type) { + case *expr.EqOperator, *expr.GtOperator, *expr.GteOperator, *expr.LtOperator, *expr.LteOperator: + vb = vb.Append(v) + case *expr.InOperator: + // an index like idx_foo_a_b on (a,b) and a query like + // WHERE a IN [1, 1] and b IN [2, 2] + // would lead to [1, 1] x [2, 2] = [[1,1], [1,2], [2,1], [2,2]] + // which could eventually be added later. + panic("unsupported operator IN for composite indexes") + default: + panic(stringutil.Sprintf("unknown operator %#v", op)) + } + } + + rng := stream.Range{ + Min: document.NewArrayValue(vb), + } + + // the last node is the only one that can be a comparison operator, so + // it's the one setting the range behaviour + last := fnodes[len(fnodes)-1] + op := last.f.E.(expr.Operator) + + switch op.(type) { + case *expr.EqOperator: + rng.Exact = true + case *expr.GtOperator: + rng.Exclusive = true + case *expr.LtOperator: + rng.Exclusive = true + } + + return stream.Ranges{rng}, nil +} + func getRangesFromOp(op expr.Operator, v document.Value) (stream.Ranges, error) { var ranges stream.Ranges diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index b5c0934e3..815101081 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -580,6 +580,13 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("b = 2"))), st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exact: true})), }, + { + "FROM foo WHERE a = 1 AND b > 2", // c is omitted, but it can still use idx_foo_a_b_c, with > b + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("b > 2"))), + st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exclusive: true})), + }, { "FROM foo WHERE a = 1 AND b = 2 and k = 3", // c is omitted, but it can still use idx_foo_a_b_c st.New(st.SeqScan("foo")). @@ -648,4 +655,65 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { require.Equal(t, test.expected.String(), res.String()) }) } + + t.Run("array indexes", func(t *testing.T) { + tests := []struct { + name string + root, expected *st.Stream + }{ + { + "FROM foo WHERE a = [1, 1] AND b = [2, 2]", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). + Pipe(st.Filter(parser.MustParseExpr("b = [2, 2]"))), + st.New(st.IndexScan("idx_foo_a_b", st.Range{ + Min: document.NewArrayValue( + testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), + Exact: true})), + }, + { + "FROM foo WHERE a = [1, 1] AND b > [2, 2]", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). + Pipe(st.Filter(parser.MustParseExpr("b > [2, 2]"))), + st.New(st.IndexScan("idx_foo_a_b", st.Range{ + Min: document.NewArrayValue( + testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), + Exclusive: true})), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, err := genji.Open(":memory:") + require.NoError(t, err) + defer db.Close() + + tx, err := db.Begin(true) + require.NoError(t, err) + defer tx.Rollback() + + err = tx.Exec(` + CREATE TABLE foo ( + k ARRAY PRIMARY KEY, + a ARRAY + ); + CREATE INDEX idx_foo_a_b ON foo(a, b); + CREATE INDEX idx_foo_a0 ON foo(a[0]); + INSERT INTO foo (k, a, b) VALUES + ([1, 1], [1, 1], [1, 1]), + ([2, 2], [2, 2], [2, 2]), + ([3, 3], [3, 3], [3, 3]) + `) + require.NoError(t, err) + + res, err := planner.PrecalculateExprRule(test.root, tx.Transaction, nil) + require.NoError(t, err) + + res, err = planner.UseIndexBasedOnFilterNodeRule(res, tx.Transaction, nil) + require.NoError(t, err) + require.Equal(t, test.expected.String(), res.String()) + }) + } + }) } From de1363dda980077f4abce74ed5db14e4b49a14a1 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 14 Apr 2021 16:56:53 +0200 Subject: [PATCH 16/40] Fix broken comments --- database/index.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/database/index.go b/database/index.go index 17f748b51..bbdeceb65 100644 --- a/database/index.go +++ b/database/index.go @@ -68,7 +68,7 @@ func (idx *Index) Arity() int { return len(idx.Info.Types) } -// Set associates a value with a key. If Unique is set to false, it is +// Set associates a list of values with a key. If Unique is set to false, it is // possible to associate multiple keys for the same value // but a key can be associated to only one value. func (idx *Index) Set(vs []document.Value, k []byte) error { @@ -320,8 +320,6 @@ func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { return v.MarshalBinary() } - // in the case of one of the index keys being untyped and the corresponding - // value being an integer, convert it into a double. var err error var buf bytes.Buffer err = document.NewValueEncoder(&buf).Encode(v) From 14d2e898c23d20c5c30465650fef613f1a724f7d Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Tue, 20 Apr 2021 12:31:43 +0200 Subject: [PATCH 17/40] Refactor indexes to use value buffers in all cases --- database/index.go | 192 +++++++++------- database/index_test.go | 360 ++++++++++++++++++++---------- document/array.go | 35 +++ planner/optimizer.go | 32 ++- stream/iterator.go | 434 ++++++++---------------------------- stream/range.go | 484 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 983 insertions(+), 554 deletions(-) create mode 100644 stream/range.go diff --git a/database/index.go b/database/index.go index bbdeceb65..93924f0c1 100644 --- a/database/index.go +++ b/database/index.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "errors" + "fmt" "github.com/genjidb/genji/document" "github.com/genjidb/genji/engine" @@ -35,6 +36,50 @@ type Index struct { storeName []byte } +type indexValueEncoder struct { + typ document.ValueType +} + +func (e *indexValueEncoder) EncodeValue(v document.Value) ([]byte, error) { + // if the index has no type constraint, encode the value with its type + if e.typ.IsZero() { + var buf bytes.Buffer + + // prepend with the type + err := buf.WriteByte(byte(v.Type)) + if err != nil { + return nil, err + } + + // marshal the value, if it exists, just return the type otherwise + if v.V != nil { + b, err := v.MarshalBinary() + if err != nil { + return nil, err + } + + _, err = buf.Write(b) + if err != nil { + return nil, err + } + } + + return buf.Bytes(), nil + } + + // this should never happen, but if it does, something is very wrong + if v.Type != e.typ { + panic("incompatible index type") + } + + if v.V == nil { + return nil, nil + } + + // there is a type constraint, so a shorter form can be used as the type is always the same + return v.MarshalBinary() +} + // NewIndex creates an index that associates values with a list of keys. func NewIndex(tx engine.Transaction, idxName string, opts *IndexInfo) *Index { if opts == nil { @@ -68,7 +113,7 @@ func (idx *Index) Arity() int { return len(idx.Info.Types) } -// Set associates a list of values with a key. If Unique is set to false, it is +// Set associates a value with a key. If Unique is set to false, it is // possible to associate multiple keys for the same value // but a key can be associated to only one value. func (idx *Index) Set(vs []document.Value, k []byte) error { @@ -100,12 +145,8 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { // encode the value we are going to use as a key var buf []byte - if len(vs) > 1 { - wrappedVs := document.NewValueBuffer(vs...) - buf, err = idx.EncodeValue(document.NewArrayValue(wrappedVs)) - } else { - buf, err = idx.EncodeValue(vs[0]) - } + vb := document.NewValueBuffer(vs...) + buf, err = idx.EncodeValueBuffer(vb) if err != nil { return err @@ -307,32 +348,47 @@ func (idx *Index) Truncate() error { return nil } -// EncodeValue encodes the value we are going to use as a key, +// EncodeValue encodes the value buffer we are going to use as a key, +// TODO // If the index is typed, encode the value without expecting // the presence of other types. // If not, encode so that order is preserved regardless of the type. -func (idx *Index) EncodeValue(v document.Value) ([]byte, error) { - if idx.IsComposite() { - return idx.compositeEncodeValue(v) - } - - if idx.Info.Types[0] != 0 { - return v.MarshalBinary() +func (idx *Index) EncodeValueBuffer(vb *document.ValueBuffer) ([]byte, error) { + if vb.Len() > idx.Arity() { + // TODO + return nil, fmt.Errorf("todo") } - var err error var buf bytes.Buffer - err = document.NewValueEncoder(&buf).Encode(v) + + err := vb.Iterate(func(i int, value document.Value) error { + enc := &indexValueEncoder{idx.Info.Types[i]} + b, err := enc.EncodeValue(value) + if err != nil { + return err + } + + _, err = buf.Write(b) + if err != nil { + return err + } + + // if it's not the last value, append the seperator + if i < vb.Len()-1 { + err = buf.WriteByte(0x1f) // TODO + if err != nil { + return err + } + } + + return nil + }) + if err != nil { return nil, err } - return buf.Bytes(), nil -} -func (idx *Index) compositeEncodeValue(v document.Value) ([]byte, error) { - // v has been turned into an array of values being indexed - // if we reach this point, array *must* be a document.ValueBuffer - return v.MarshalBinary() + return buf.Bytes(), nil } func getOrCreateStore(tx engine.Transaction, name []byte) (engine.Store, error) { @@ -360,6 +416,7 @@ func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, erro var seek []byte var err error + // TODO rework // if we have valueless and typeless pivots, we just iterate if allEmpty(pivots) { return []byte{}, nil @@ -377,64 +434,39 @@ func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, erro return seek, nil } - if !idx.IsComposite() { - if pivots[0].V != nil { - seek, err = idx.EncodeValue(pivots[0]) - if err != nil { - return nil, err - } - - if reverse { - // appending 0xFF turns the pivot into the upper bound of that value. - seek = append(seek, 0xFF) - } - } else { - if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && pivots[0].V == nil { - seek = []byte{byte(pivots[0].Type)} + // if !idx.IsComposite() { + // if pivots[0].V != nil { + // seek, err = idx.EncodeValue(document.NewValueBuffer(pivots...)) + // if err != nil { + // return nil, err + // } + + // if reverse { + // // appending 0xFF turns the pivot into the upper bound of that value. + // seek = append(seek, 0xFF) + // } + // } else { + // if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && pivots[0].V == nil { + // seek = []byte{byte(pivots[0].Type)} + + // if reverse { + // seek = append(seek, 0xFF) + // } + // } + // } + // } else { + // [2,3,4,int] is a valid pivot, in which case the last pivot, a valueless typed pivot + // it handled separatedly + + vb := document.NewValueBuffer(pivots...) + seek, err = idx.EncodeValueBuffer(vb) - if reverse { - seek = append(seek, 0xFF) - } - } - } - } else { - // [2,3,4,int] is a valid pivot, in which case the last pivot, a valueless typed pivot - // it handled separatedly - valuePivots := make([]document.Value, 0, len(pivots)) - var valuelessPivot *document.Value - for _, p := range pivots { - if p.V != nil { - valuePivots = append(valuePivots, p) - } else { - valuelessPivot = &p - break - } - } - - vb := document.NewValueBuffer(valuePivots...) - seek, err = idx.EncodeValue(document.NewArrayValue(vb)) - - if err != nil { - return nil, err - } - - // if we have a [2, int] case, let's just add the type - if valuelessPivot != nil { - seek = append(seek[:len(seek)-1], byte(0x1f), byte(valuelessPivot.Type), byte(0x1e)) - } + if err != nil { + return nil, err + } - if reverse { - // if we are seeking in reverse on a pivot with lower arity, the comparison will be in between - // arrays of different sizes, the pivot being shorter than the indexed values. - // Because the element separator 0x1F is greater than the array end separator 0x1E, - // the reverse byte 0xFF must be appended before the end separator in order to be able - // to be compared correctly. - if len(seek) > 0 { - seek = append(seek[:len(seek)-1], 0xFF) - } else { - seek = append(seek, 0xFF) - } - } + if reverse { + seek = append(seek, 0xFF) } return seek, nil @@ -444,6 +476,8 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool var err error seek, err := idx.buildSeek(pivots, reverse) + fmt.Println("pivots", pivots) + fmt.Println("seek", seek) if err != nil { return err } diff --git a/database/index_test.go b/database/index_test.go index c1be30611..4629f99f7 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -12,6 +12,7 @@ import ( "github.com/genjidb/genji/document" "github.com/genjidb/genji/engine" "github.com/genjidb/genji/engine/memoryengine" + "github.com/genjidb/genji/testutil" "github.com/stretchr/testify/require" ) @@ -128,10 +129,10 @@ func TestIndexDelete(t *testing.T) { i := 0 err := idx.AscendGreaterOrEqual(pivot, func(v, k []byte) error { if i == 0 { - requireEqualEncoded(t, document.NewIntegerValue(10), v) + requireEqualBinary(t, testutil.MakeArrayValue(t, 10), v) require.Equal(t, "other-key", string(k)) } else if i == 1 { - requireEqualEncoded(t, document.NewIntegerValue(11), v) + requireEqualBinary(t, testutil.MakeArrayValue(t, 11), v) require.Equal(t, "yet-another-key", string(k)) } else { return errors.New("should not reach this point") @@ -195,10 +196,10 @@ func TestIndexDelete(t *testing.T) { err := idx.AscendGreaterOrEqual(values(document.Value{Type: document.IntegerValue}), func(v, k []byte) error { switch i { case 0: - requireEqualEncoded(t, document.NewIntegerValue(10), v) + requireEqualBinary(t, testutil.MakeArrayValue(t, 10), v) require.Equal(t, "key1", string(k)) case 1: - requireEqualEncoded(t, document.NewIntegerValue(12), v) + requireEqualBinary(t, testutil.MakeArrayValue(t, 12), v) require.Equal(t, "key3", string(k)) default: return errors.New("should not reach this point") @@ -269,7 +270,7 @@ func requireEqualBinary(t *testing.T, expected document.Value, actual []byte) { buf, err := expected.MarshalBinary() require.NoError(t, err) - require.Equal(t, buf, actual) + require.Equal(t, buf[:len(buf)-1], actual) } // requireEqualEncoded asserts equality, assuming that the value is encoded with document.ValueEncoder @@ -282,6 +283,39 @@ func requireEqualEncoded(t *testing.T, expected document.Value, actual []byte) { require.Equal(t, buf.Bytes(), actual) } +type encValue struct { + skipType bool + document.Value +} + +func req(t *testing.T, evs ...encValue) func([]byte) { + t.Helper() + + var buf bytes.Buffer + for i, ev := range evs { + if !ev.skipType { + err := buf.WriteByte(byte(ev.Value.Type)) + require.NoError(t, err) + } + + b, err := ev.Value.MarshalBinary() + require.NoError(t, err) + + _, err = buf.Write(b) + require.NoError(t, err) + + // TODO + if i < len(evs)-1 { + err = buf.WriteByte(0x1f) + } + require.NoError(t, err) + } + + return func(actual []byte) { + require.Equal(t, buf.Bytes(), actual) + } +} + func TestIndexAscendGreaterThan(t *testing.T) { for _, unique := range []bool{true, false} { text := fmt.Sprintf("Unique: %v, ", unique) @@ -341,7 +375,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + )(val) }, expectedCount: 5, }, @@ -351,7 +387,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + )(val) }, expectedCount: 5, }, @@ -363,7 +401,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + )(val) }, expectedCount: 3, }, @@ -382,7 +422,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + )(val) }, expectedCount: 3, }, @@ -402,7 +444,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + req(t, + encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, + )(val) }, expectedCount: 5, }, @@ -413,7 +457,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + req(t, + encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, + )(val) }, expectedCount: 3, }, @@ -424,7 +470,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + req(t, + encValue{true, document.NewDoubleValue(float64(i) + float64(i)/2)}, + )(val) }, expectedCount: 3, }, @@ -444,7 +492,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 5, }, @@ -456,7 +506,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 3, }, @@ -467,7 +519,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 5, }, @@ -486,7 +540,9 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{true, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 3, }, @@ -499,10 +555,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 5, }, @@ -563,10 +619,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 5, }, @@ -581,10 +637,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, @@ -599,10 +655,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, @@ -623,10 +679,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, @@ -642,10 +698,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewBlobValue([]byte{byte('a' + uint8(i))})) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, + )(val) }, expectedCount: 3, }, @@ -693,10 +749,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { if i%2 == 0 { i = i / 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) } }, expectedCount: 9, // 10 elements, but pivot skipped the initial [0, true] @@ -709,10 +765,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 5, }, @@ -724,10 +780,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, @@ -740,10 +796,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewBlobValue([]byte{byte('a' + uint8(i))})) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, + )(val) }, expectedCount: 3, }, @@ -755,10 +811,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 5, }, @@ -770,10 +826,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, @@ -881,7 +937,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) + requireEqualBinary(t, testutil.MakeArrayValue(t, int64(i)), val) }, expectedCount: 5, }, @@ -891,7 +947,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + )(val) }, expectedCount: 5, }, @@ -903,7 +961,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewIntegerValue(int64(i)), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + )(val) }, expectedCount: 3, }, @@ -922,7 +982,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewIntegerValue(int64(i)), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + )(val) }, expectedCount: 3, }, @@ -942,7 +1004,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + req(t, + encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, + )(val) }, expectedCount: 5, }, @@ -953,7 +1017,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + req(t, + encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, + )(val) }, expectedCount: 2, }, @@ -964,7 +1030,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewDoubleValue(float64(i)+float64(i)/2), val) + req(t, + encValue{true, document.NewDoubleValue(float64(i) + float64(i)/2)}, + )(val) }, expectedCount: 2, }, @@ -984,7 +1052,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) + }, expectedCount: 5, }, @@ -996,7 +1067,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 3, }, @@ -1007,7 +1080,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 5, }, @@ -1018,7 +1093,9 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - requireEqualEncoded(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 5, }, @@ -1029,11 +1106,28 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - requireEqualBinary(t, document.NewTextValue(strconv.Itoa(int(i))), val) + req(t, + encValue{true, document.NewTextValue(strconv.Itoa(int(i)))}, + )(val) }, expectedCount: 3, }, // composite -------------------------------------------------- + // composite indexes can have empty pivots to iterate on the whole indexed data + {name: "index=[untyped, untyped], vals=[int, int], pivot=[nil,nil]", + indexTypes: []document.ValueType{0, 0}, + pivots: values(document.Value{}, document.Value{}), + val: func(i int) []document.Value { + return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 5, + }, // composite indexes must have at least have one value {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", indexTypes: []document.ValueType{0, 0}, @@ -1089,10 +1183,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 5, }, @@ -1108,10 +1202,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 2, }, @@ -1127,11 +1221,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, @@ -1152,10 +1245,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, @@ -1164,17 +1257,48 @@ func TestIndexDescendLessOrEqual(t *testing.T) { indexTypes: []document.ValueType{0, 0}, pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { - return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) + return values( + document.NewIntegerValue(int64(i)), + document.NewBlobValue([]byte{byte('a' + uint8(i))}), + ) }, noise: func(i int) []document.Value { - return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) + return values( + document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), + document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), + ) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - array := document.NewValueBuffer( + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, + )(val) + }, + expectedCount: 2, + }, + // only one of the indexed value is typed + {name: "index=[untyped, blob], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + indexTypes: []document.ValueType{0, document.BlobValue}, + pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + val: func(i int) []document.Value { + return values( document.NewIntegerValue(int64(i)), - document.NewBlobValue([]byte{byte('a' + uint8(i))})) - requireEqualBinary(t, document.NewArrayValue(array), val) + document.NewBlobValue([]byte{byte('a' + uint8(i))}), + ) + }, + noise: func(i int) []document.Value { + return values( + document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), + document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 3 + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, + )(val) }, expectedCount: 2, }, @@ -1238,10 +1362,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 5, }, @@ -1253,10 +1377,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 2, }, @@ -1269,10 +1393,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewBlobValue([]byte{byte('a' + uint8(i))})) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, + )(val) }, expectedCount: 2, }, @@ -1285,10 +1409,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 4 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 1, }, @@ -1301,10 +1425,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 - array := document.NewValueBuffer( - document.NewIntegerValue(int64(i)), - document.NewIntegerValue(int64(i+1))) - requireEqualBinary(t, document.NewArrayValue(array), val) + req(t, + encValue{true, document.NewIntegerValue(int64(i))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) }, expectedCount: 3, }, diff --git a/document/array.go b/document/array.go index ec4ae8d57..53c2eea6e 100644 --- a/document/array.go +++ b/document/array.go @@ -246,6 +246,41 @@ func (vb *ValueBuffer) UnmarshalJSON(data []byte) error { return nil } +func (vb *ValueBuffer) Types() []ValueType { + types := make([]ValueType, len(vb.Values)) + + for i, v := range vb.Values { + types[i] = v.Type + } + + return types +} + +// IsEqual compares two ValueBuffer and returns true if and only if +// both each values and types are respectively equal. +func (vb *ValueBuffer) IsEqual(other *ValueBuffer) bool { + if vb.Len() != other.Len() { + return false + } + + otherTypes := other.Types() + types := vb.Types() + + for i, typ := range types { + if typ != otherTypes[i] { + return false + } + } + + for i, v := range vb.Values { + if eq, err := v.IsEqual(other.Values[i]); err != nil || !eq { + return false + } + } + + return true +} + type sortableArray struct { vb *ValueBuffer err error diff --git a/planner/optimizer.go b/planner/optimizer.go index d07a6d9d2..fec8d233b 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -605,7 +605,7 @@ outer: cd.priority = 1 } - ranges, err := getRangesFromFilterNodes(usableFilterNodes, idx.Arity()) + ranges, err := getRangesFromFilterNodes(usableFilterNodes) if err != nil { return nil, err } @@ -744,12 +744,7 @@ func operandCanUseIndex(indexType document.ValueType, path document.Path, fc dat return converted, indexType == converted.Type, nil } -func getRangesFromFilterNodes(fnodes []*filterNode, indexArity int) (stream.Ranges, error) { - if indexArity <= 1 { - op := fnodes[0].f.E.(expr.Operator) - return getRangesFromOp(op, fnodes[0].v) - } - +func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) { vb := document.NewValueBuffer() for _, fno := range fnodes { op := fno.f.E.(expr.Operator) @@ -759,6 +754,7 @@ func getRangesFromFilterNodes(fnodes []*filterNode, indexArity int) (stream.Rang case *expr.EqOperator, *expr.GtOperator, *expr.GteOperator, *expr.LtOperator, *expr.LteOperator: vb = vb.Append(v) case *expr.InOperator: + // TODO(JH) // an index like idx_foo_a_b on (a,b) and a query like // WHERE a IN [1, 1] and b IN [2, 2] // would lead to [1, 1] x [2, 2] = [[1,1], [1,2], [2,1], [2,2]] @@ -769,8 +765,8 @@ func getRangesFromFilterNodes(fnodes []*filterNode, indexArity int) (stream.Rang } } - rng := stream.Range{ - Min: document.NewArrayValue(vb), + rng := stream.IndexRange{ + Min: vb, } // the last node is the only one that can be a comparison operator, so @@ -787,41 +783,41 @@ func getRangesFromFilterNodes(fnodes []*filterNode, indexArity int) (stream.Rang rng.Exclusive = true } - return stream.Ranges{rng}, nil + return stream.IndexRanges{rng}, nil } -func getRangesFromOp(op expr.Operator, v document.Value) (stream.Ranges, error) { - var ranges stream.Ranges +func getRangesFromOp(op expr.Operator, v document.Value) (stream.ValueRanges, error) { + var ranges stream.ValueRanges switch op.(type) { case *expr.EqOperator: - ranges = ranges.Append(stream.Range{ + ranges = ranges.Append(stream.ValueRange{ Min: v, Exact: true, }) case *expr.GtOperator: - ranges = ranges.Append(stream.Range{ + ranges = ranges.Append(stream.ValueRange{ Min: v, Exclusive: true, }) case *expr.GteOperator: - ranges = ranges.Append(stream.Range{ + ranges = ranges.Append(stream.ValueRange{ Min: v, }) case *expr.LtOperator: - ranges = ranges.Append(stream.Range{ + ranges = ranges.Append(stream.ValueRange{ Max: v, Exclusive: true, }) case *expr.LteOperator: - ranges = ranges.Append(stream.Range{ + ranges = ranges.Append(stream.ValueRange{ Max: v, }) case *expr.InOperator: // operatorCanUseIndex made sure e is an array. a := v.V.(document.Array) err := a.Iterate(func(i int, value document.Value) error { - ranges = ranges.Append(stream.Range{ + ranges = ranges.Append(stream.ValueRange{ Min: value, Exact: true, }) diff --git a/stream/iterator.go b/stream/iterator.go index 2217e80f2..2fc656e76 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -151,17 +151,17 @@ func (it *SeqScanOperator) String() string { type PkScanOperator struct { baseOperator TableName string - Ranges Ranges + Ranges ValueRanges Reverse bool } // PkScan creates an iterator that iterates over each document of the given table. -func PkScan(tableName string, ranges ...Range) *PkScanOperator { +func PkScan(tableName string, ranges ...ValueRange) *PkScanOperator { return &PkScanOperator{TableName: tableName, Ranges: ranges} } // PkScanReverse creates an iterator that iterates over each document of the given table in reverse order. -func PkScanReverse(tableName string, ranges ...Range) *PkScanOperator { +func PkScanReverse(tableName string, ranges ...ValueRange) *PkScanOperator { return &PkScanOperator{TableName: tableName, Ranges: ranges, Reverse: true} } @@ -279,18 +279,18 @@ type IndexScanOperator struct { IndexName string // Ranges defines the boundaries of the scan, each corresponding to one value of the group of values // being indexed in the case of a composite index. - Ranges Ranges + Ranges IndexRanges // Reverse indicates the direction used to traverse the index. Reverse bool } // IndexScan creates an iterator that iterates over each document of the given table. -func IndexScan(name string, ranges ...Range) *IndexScanOperator { +func IndexScan(name string, ranges ...IndexRange) *IndexScanOperator { return &IndexScanOperator{IndexName: name, Ranges: ranges} } // IndexScanReverse creates an iterator that iterates over each document of the given table in reverse order. -func IndexScanReverse(name string, ranges ...Range) *IndexScanOperator { +func IndexScanReverse(name string, ranges ...IndexRange) *IndexScanOperator { return &IndexScanOperator{IndexName: name, Ranges: ranges, Reverse: true} } @@ -331,7 +331,7 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env return err } - err = it.Ranges.Encode(index, in) + err = it.Ranges.EncodeBuffer(index, in) if err != nil { return err } @@ -359,361 +359,117 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } for _, rng := range it.Ranges { - if !index.IsComposite() { - var start, end document.Value - if !it.Reverse { - start = rng.Min - end = rng.Max - } else { - start = rng.Max - end = rng.Min - } - - var encEnd []byte - if !end.Type.IsZero() && end.V != nil { - encEnd, err = index.EncodeValue(end) - if err != nil { - return err - } - } - - err = iterator([]document.Value{start}, func(val, key []byte) error { - if !rng.IsInRange(val) { - // if we reached the end of our range, we can stop iterating. - if encEnd == nil { - return nil - } - cmp := bytes.Compare(val, encEnd) - if !it.Reverse && cmp > 0 { - return ErrStreamClosed - } - if it.Reverse && cmp < 0 { - return ErrStreamClosed - } - return nil - } - - d, err := table.GetDocument(key) - if err != nil { - return err - } + // if !index.IsComposite() { + // var start, end *document.ValueBuffer + // if !it.Reverse { + // start = rng.Min + // end = rng.Max + // } else { + // start = rng.Max + // end = rng.Min + // } + + // var encEnd []byte + // // if !end.Type.IsZero() && end.V != nil { + // if end.Len() > 0 { + // encEnd, err = index.EncodeValueBuffer(end) + // if err != nil { + // return err + // } + // } + + // err = iterator([]document.Value{start}, func(val, key []byte) error { + // if !rng.IsInRange(val) { + // // if we reached the end of our range, we can stop iterating. + // if encEnd == nil { + // return nil + // } + // cmp := bytes.Compare(val, encEnd) + // if !it.Reverse && cmp > 0 { + // return ErrStreamClosed + // } + // if it.Reverse && cmp < 0 { + // return ErrStreamClosed + // } + // return nil + // } + + // d, err := table.GetDocument(key) + // if err != nil { + // return err + // } + + // newEnv.SetDocument(d) + // return fn(&newEnv) + // }) + // if err == ErrStreamClosed { + // err = nil + // } + // if err != nil { + // return err + // } + // } else { + var start, end *document.ValueBuffer + if !it.Reverse { + start = rng.Min + end = rng.Max + } else { + start = rng.Max + end = rng.Min + } - newEnv.SetDocument(d) - return fn(&newEnv) - }) - if err == ErrStreamClosed { - err = nil - } + var encEnd []byte + if end.Len() > 0 { + encEnd, err = index.EncodeValueBuffer(end) if err != nil { return err } - } else { - var start, end document.Value - if !it.Reverse { - start = rng.Min - end = rng.Max - } else { - start = rng.Max - end = rng.Min - } + } - var encEnd []byte - if end.V != nil { - encEnd, err = index.EncodeValue(end) - if err != nil { - return err - } + // extract the pivots from the range, which in the case of a composite index is an array + pivots := []document.Value{} + if start.Len() > 0 { + pivots = start.Values + } else { + for i := 0; i < index.Arity(); i++ { + pivots = append(pivots, document.Value{}) } + } - // extract the pivots from the range, which in the case of a composite index is an array - pivots := []document.Value{} - if start.V != nil { - start.V.(document.Array).Iterate(func(i int, value document.Value) error { - pivots = append(pivots, value) + err = iterator(pivots, func(val, key []byte) error { + if !rng.IsInRange(val) { + // if we reached the end of our range, we can stop iterating. + if encEnd == nil { return nil - }) - } else { - for i := 0; i < index.Arity(); i++ { - pivots = append(pivots, document.Value{}) } - } - - err = iterator(pivots, func(val, key []byte) error { - if !rng.IsInRange(val) { - // if we reached the end of our range, we can stop iterating. - if encEnd == nil { - return nil - } - cmp := bytes.Compare(val, encEnd) - if !it.Reverse && cmp > 0 { - return ErrStreamClosed - } - if it.Reverse && cmp < 0 { - return ErrStreamClosed - } - return nil + cmp := bytes.Compare(val, encEnd) + if !it.Reverse && cmp > 0 { + return ErrStreamClosed } - - d, err := table.GetDocument(key) - if err != nil { - return err + if it.Reverse && cmp < 0 { + return ErrStreamClosed } - - newEnv.SetDocument(d) - return fn(&newEnv) - }) - - if err == ErrStreamClosed { - err = nil + return nil } + + d, err := table.GetDocument(key) if err != nil { return err } - } - } - - return nil -} - -type Range struct { - Min, Max document.Value - // Exclude Min and Max from the results. - // By default, min and max are inclusive. - // Exclusive and Exact cannot be set to true at the same time. - Exclusive bool - // Used to match an exact value equal to Min. - // If set to true, Max will be ignored for comparison - // and for determining the global upper bound. - Exact bool - - // Arity represents the range arity in the case of comparing the range - // to a composite index. With IndexArityMax, it enables to deal with the - // cases of a composite range specifying boundaries partially, ie: - // - Index on (a, b, c) - // - Range is defining a max only for a and b - // Then Arity is set to 2 and IndexArityMax is set to 3 - // - // On - // This field is subject to change when the support for composite index is added - // to the query planner in an ulterior pull-request. - Arity int - - // IndexArityMax represents the underlying Index arity. - // - // This field is subject to change when the support for composite index is added - // to the query planner in an ulterior pull-request. - IndexArityMax int - encodedMin, encodedMax []byte - rangeType document.ValueType -} - -func (r *Range) encode(encoder ValueEncoder, env *expr.Environment) error { - var err error + newEnv.SetDocument(d) + return fn(&newEnv) + }) - // first we evaluate Min and Max - if !r.Min.Type.IsZero() { - r.encodedMin, err = encoder.EncodeValue(r.Min) - if err != nil { - return err + if err == ErrStreamClosed { + err = nil } - r.rangeType = r.Min.Type - } - if !r.Max.Type.IsZero() { - r.encodedMax, err = encoder.EncodeValue(r.Max) if err != nil { return err } - if !r.rangeType.IsZero() && r.rangeType != r.Max.Type { - panic("range contain values of different types") - } - - r.rangeType = r.Max.Type - } - - // ensure boundaries are typed - if r.Min.Type.IsZero() { - r.Min.Type = r.rangeType - } - if r.Max.Type.IsZero() { - r.Max.Type = r.rangeType - } - if r.Exclusive && r.Exact { - panic("exclusive and exact cannot both be true") + // } } return nil } - -func (r *Range) String() string { - if r.Exact { - return stringutil.Sprintf("%v", r.Min) - } - - if r.Min.Type.IsZero() { - r.Min = document.NewIntegerValue(-1) - } - if r.Max.Type.IsZero() { - r.Max = document.NewIntegerValue(-1) - } - - if r.Exclusive { - return stringutil.Sprintf("[%v, %v, true]", r.Min, r.Max) - } - - return stringutil.Sprintf("[%v, %v]", r.Min, r.Max) -} - -func (r *Range) IsEqual(other *Range) bool { - if r.Exact != other.Exact { - return false - } - - if r.rangeType != other.rangeType { - return false - } - - if r.Exclusive != other.Exclusive { - return false - } - - if r.Min.Type != other.Min.Type { - return false - } - ok, err := r.Min.IsEqual(other.Min) - if err != nil || !ok { - return false - } - - if r.Max.Type != other.Max.Type { - return false - } - ok, err = r.Max.IsEqual(other.Max) - if err != nil || !ok { - return false - } - - return true -} - -type Ranges []Range - -// Append rng to r and return the new slice. -// Duplicate ranges are ignored. -func (r Ranges) Append(rng Range) Ranges { - // ensure we don't keep duplicate ranges - isDuplicate := false - for _, e := range r { - if e.IsEqual(&rng) { - isDuplicate = true - break - } - } - - if isDuplicate { - return r - } - - return append(r, rng) -} - -type ValueEncoder interface { - EncodeValue(v document.Value) ([]byte, error) -} - -// Encode each range using the given value encoder. -func (r Ranges) Encode(encoder ValueEncoder, env *expr.Environment) error { - for i := range r { - err := r[i].encode(encoder, env) - if err != nil { - return err - } - } - - return nil -} - -func (r Ranges) String() string { - var sb strings.Builder - - for i, rr := range r { - if i > 0 { - sb.WriteString(", ") - } - - sb.WriteString(rr.String()) - } - - return sb.String() -} - -// Cost is a best effort function to determine the cost of -// a range lookup. -func (r Ranges) Cost() int { - var cost int - - for _, rng := range r { - // if we are looking for an exact value - // increment by 1 - if rng.Exact { - cost++ - continue - } - - // if there are two boundaries, increment by 50 - if !rng.Min.Type.IsZero() && !rng.Max.Type.IsZero() { - cost += 50 - continue - } - - // if there is only one boundary, increment by 100 - if (!rng.Min.Type.IsZero() && rng.Max.Type.IsZero()) || (rng.Min.Type.IsZero() && !rng.Max.Type.IsZero()) { - cost += 100 - continue - } - - // if there are no boundaries, increment by 200 - cost += 200 - } - - return cost -} - -func (r *Range) IsInRange(value []byte) bool { - // by default, we consider the value within range - cmpMin, cmpMax := 1, -1 - - // we compare with the lower bound and see if it matches - if r.encodedMin != nil { - cmpMin = bytes.Compare(value, r.encodedMin) - } - - // if exact is true the value has to be equal to the lower bound. - if r.Exact { - return cmpMin == 0 - } - - // if exclusive and the value is equal to the lower bound - // we can ignore it - if r.Exclusive && cmpMin == 0 { - return false - } - - // the value is bigger than the lower bound, - // see if it matches the upper bound. - if r.encodedMax != nil { - if r.IndexArityMax < r.Arity { - cmpMax = bytes.Compare(value[:len(r.encodedMax)-1], r.encodedMax) - } else { - cmpMax = bytes.Compare(value, r.encodedMax) - } - } - - // if boundaries are strict, ignore values equal to the max - if r.Exclusive && cmpMax == 0 { - return false - } - - return cmpMin >= 0 && cmpMax <= 0 -} diff --git a/stream/range.go b/stream/range.go new file mode 100644 index 000000000..8491c909d --- /dev/null +++ b/stream/range.go @@ -0,0 +1,484 @@ +package stream + +import ( + "bytes" + "strings" + + "github.com/genjidb/genji/document" + "github.com/genjidb/genji/expr" + "github.com/genjidb/genji/stringutil" +) + +type Costable interface { + Cost() int +} + +type ValueRange struct { + Min, Max document.Value + // Exclude Min and Max from the results. + // By default, min and max are inclusive. + // Exclusive and Exact cannot be set to true at the same time. + Exclusive bool + // Used to match an exact value equal to Min. + // If set to true, Max will be ignored for comparison + // and for determining the global upper bound. + Exact bool + + encodedMin, encodedMax []byte + rangeType document.ValueType +} + +func (r *ValueRange) encode(encoder ValueEncoder, env *expr.Environment) error { + var err error + + // first we evaluate Min and Max + if !r.Min.Type.IsZero() { + r.encodedMin, err = encoder.EncodeValue(r.Min) + if err != nil { + return err + } + r.rangeType = r.Min.Type + } + if !r.Max.Type.IsZero() { + r.encodedMax, err = encoder.EncodeValue(r.Max) + if err != nil { + return err + } + if !r.rangeType.IsZero() && r.rangeType != r.Max.Type { + panic("range contain values of different types") + } + + r.rangeType = r.Max.Type + } + + // ensure boundaries are typed + if r.Min.Type.IsZero() { + r.Min.Type = r.rangeType + } + if r.Max.Type.IsZero() { + r.Max.Type = r.rangeType + } + + if r.Exclusive && r.Exact { + panic("exclusive and exact cannot both be true") + } + + return nil +} + +func (r *ValueRange) String() string { + if r.Exact { + return stringutil.Sprintf("%v", r.Min) + } + + if r.Min.Type.IsZero() { + r.Min = document.NewIntegerValue(-1) + } + if r.Max.Type.IsZero() { + r.Max = document.NewIntegerValue(-1) + } + + if r.Exclusive { + return stringutil.Sprintf("[%v, %v, true]", r.Min, r.Max) + } + + return stringutil.Sprintf("[%v, %v]", r.Min, r.Max) +} + +func (r *ValueRange) IsEqual(other *ValueRange) bool { + if r.Exact != other.Exact { + return false + } + + if r.rangeType != other.rangeType { + return false + } + + if r.Exclusive != other.Exclusive { + return false + } + + if r.Min.Type != other.Min.Type { + return false + } + ok, err := r.Min.IsEqual(other.Min) + if err != nil || !ok { + return false + } + + if r.Max.Type != other.Max.Type { + return false + } + ok, err = r.Max.IsEqual(other.Max) + if err != nil || !ok { + return false + } + + return true +} + +type ValueRanges []ValueRange + +// Append rng to r and return the new slice. +// Duplicate ranges are ignored. +func (r ValueRanges) Append(rng ValueRange) ValueRanges { + // ensure we don't keep duplicate ranges + isDuplicate := false + for _, e := range r { + if e.IsEqual(&rng) { + isDuplicate = true + break + } + } + + if isDuplicate { + return r + } + + return append(r, rng) +} + +type ValueEncoder interface { + EncodeValue(v document.Value) ([]byte, error) +} + +// Encode each range using the given value encoder. +func (r ValueRanges) Encode(encoder ValueEncoder, env *expr.Environment) error { + for i := range r { + err := r[i].encode(encoder, env) + if err != nil { + return err + } + } + + return nil +} + +func (r ValueRanges) String() string { + var sb strings.Builder + + for i, rr := range r { + if i > 0 { + sb.WriteString(", ") + } + + sb.WriteString(rr.String()) + } + + return sb.String() +} + +// Cost is a best effort function to determine the cost of +// a range lookup. +func (r ValueRanges) Cost() int { + var cost int + + for _, rng := range r { + // if we are looking for an exact value + // increment by 1 + if rng.Exact { + cost++ + continue + } + + // if there are two boundaries, increment by 50 + if !rng.Min.Type.IsZero() && !rng.Max.Type.IsZero() { + cost += 50 + } + + // if there is only one boundary, increment by 100 + if (!rng.Min.Type.IsZero() && rng.Max.Type.IsZero()) || (rng.Min.Type.IsZero() && !rng.Max.Type.IsZero()) { + cost += 100 + continue + } + + // if there are no boundaries, increment by 200 + cost += 200 + } + + return cost +} + +func (r *ValueRange) IsInRange(value []byte) bool { + // by default, we consider the value within range + cmpMin, cmpMax := 1, -1 + + // we compare with the lower bound and see if it matches + if r.encodedMin != nil { + cmpMin = bytes.Compare(value, r.encodedMin) + } + + // if exact is true the value has to be equal to the lower bound. + if r.Exact { + return cmpMin == 0 + } + + // if exclusive and the value is equal to the lower bound + // we can ignore it + if r.Exclusive && cmpMin == 0 { + return false + } + + // the value is bigger than the lower bound, + // see if it matches the upper bound. + if r.encodedMax != nil { + cmpMax = bytes.Compare(value, r.encodedMax) + } + + // if boundaries are strict, ignore values equal to the max + if r.Exclusive && cmpMax == 0 { + return false + } + + return cmpMax <= 0 +} + +type IndexRange struct { + Min, Max *document.ValueBuffer + // Exclude Min and Max from the results. + // By default, min and max are inclusive. + // Exclusive and Exact cannot be set to true at the same time. + Exclusive bool + // Used to match an exact value equal to Min. + // If set to true, Max will be ignored for comparison + // and for determining the global upper bound. + Exact bool + + // Arity represents the range arity in the case of comparing the range + // to a composite index. With IndexArityMax, it enables to deal with the + // cases of a composite range specifying boundaries partially, ie: + // - Index on (a, b, c) + // - Range is defining a max only for a and b + // Then Arity is set to 2 and IndexArityMax is set to 3 + // + // On + // This field is subject to change when the support for composite index is added + // to the query planner in an ulterior pull-request. + Arity int + + // IndexArityMax represents the underlying Index arity. + // + // This field is subject to change when the support for composite index is added + // to the query planner in an ulterior pull-request. + IndexArityMax int + encodedMin, encodedMax []byte + rangeTypes []document.ValueType +} + +func (r *IndexRange) encode(encoder ValueBufferEncoder, env *expr.Environment) error { + var err error + + // first we evaluate Min and Max + if r.Min.Len() > 0 { + r.encodedMin, err = encoder.EncodeValueBuffer(r.Min) + if err != nil { + return err + } + r.rangeTypes = r.Min.Types() + } + + if r.Max.Len() > 0 { + r.encodedMax, err = encoder.EncodeValueBuffer(r.Max) + if err != nil { + return err + } + + if len(r.rangeTypes) > 0 { + maxTypes := r.Max.Types() + + if len(maxTypes) != len(r.rangeTypes) { + panic("range types for max and min differ in size") + } + + for i, typ := range maxTypes { + if typ != r.rangeTypes[i] { + panic("range contain values of different types") + } + } + } + + r.rangeTypes = r.Max.Types() + } + + // ensure boundaries are typed + // TODO(JH) + // if r.Min.Type.IsZero() { + // r.Min.Type = r.rangeType + // } + // if r.Max.Type.IsZero() { + // r.Max.Type = r.rangeType + // } + + if r.Exclusive && r.Exact { + panic("exclusive and exact cannot both be true") + } + + return nil +} + +func (r *IndexRange) String() string { + if r.Exact { + return stringutil.Sprintf("%v", r.Min) + } + + if r.Exclusive { + return stringutil.Sprintf("[%v, %v, true]", r.Min.Types(), r.Max.Types()) + } + + return stringutil.Sprintf("[%v, %v]", r.Min.Types(), r.Max.Types()) +} + +func (r *IndexRange) IsEqual(other *IndexRange) bool { + if r.Exact != other.Exact { + return false + } + + for i, typ := range r.rangeTypes { + if typ != other.rangeTypes[i] { + return false + } + } + + if r.Exclusive != other.Exclusive { + return false + } + + // TODO(JH) may or may not this + // if r.Min.Type != other.Min.Type { + // return false + // } + + if !r.Min.IsEqual(other.Min) { + return false + } + + // if r.Max.Type != other.Max.Type { + // return false + // } + if !r.Max.IsEqual(other.Max) { + return false + } + + return true +} + +type IndexRanges []IndexRange + +// Append rng to r and return the new slice. +// Duplicate ranges are ignored. +func (r IndexRanges) Append(rng IndexRange) IndexRanges { + // ensure we don't keep duplicate ranges + isDuplicate := false + for _, e := range r { + if e.IsEqual(&rng) { + isDuplicate = true + break + } + } + + if isDuplicate { + return r + } + + return append(r, rng) +} + +type ValueBufferEncoder interface { + EncodeValueBuffer(vb *document.ValueBuffer) ([]byte, error) +} + +// Encode each range using the given value encoder. +func (r IndexRanges) EncodeBuffer(encoder ValueBufferEncoder, env *expr.Environment) error { + for i := range r { + err := r[i].encode(encoder, env) + if err != nil { + return err + } + } + + return nil +} + +func (r IndexRanges) String() string { + var sb strings.Builder + + for i, rr := range r { + if i > 0 { + sb.WriteString(", ") + } + + sb.WriteString(rr.String()) + } + + return sb.String() +} + +// Cost is a best effort function to determine the cost of +// a range lookup. +func (r IndexRanges) Cost() int { + var cost int + + for _, rng := range r { + // if we are looking for an exact value + // increment by 1 + if rng.Exact { + cost++ + continue + } + + // if there are two boundaries, increment by 50 + if rng.Min.Len() > 0 && rng.Max.Len() > 0 { + cost += 50 + continue + } + + // if there is only one boundary, increment by 100 + if rng.Min.Len() > 0 || rng.Max.Len() > 0 { + cost += 100 + continue + } + + // if there are no boundaries, increment by 200 + cost += 200 + } + + return cost +} + +func (r *IndexRange) IsInRange(value []byte) bool { + // by default, we consider the value within range + cmpMin, cmpMax := 1, -1 + + // we compare with the lower bound and see if it matches + if r.encodedMin != nil { + cmpMin = bytes.Compare(value, r.encodedMin) + } + + // if exact is true the value has to be equal to the lower bound. + if r.Exact { + return cmpMin == 0 + } + + // if exclusive and the value is equal to the lower bound + // we can ignore it + if r.Exclusive && cmpMin == 0 { + return false + } + + // the value is bigger than the lower bound, + // see if it matches the upper bound. + if r.encodedMax != nil { + if r.IndexArityMax < r.Arity { + cmpMax = bytes.Compare(value[:len(r.encodedMax)-1], r.encodedMax) + } else { + cmpMax = bytes.Compare(value, r.encodedMax) + } + } + + // if boundaries are strict, ignore values equal to the max + if r.Exclusive && cmpMax == 0 { + return false + } + + return cmpMin >= 0 && cmpMax <= 0 +} From 760e89e8f25c08727e2f288f5637e90682d3a084 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 21 Apr 2021 17:06:36 +0200 Subject: [PATCH 18/40] Allow for pivots size to differ from arity --- database/index.go | 83 +++++--------- document/array.go | 9 ++ planner/optimizer.go | 25 ++++- planner/optimizer_test.go | 206 +++++++++++++++++----------------- query/select_test.go | 7 +- stream/iterator.go | 66 ++--------- stream/iterator_test.go | 226 +++++++++++++++++++++++--------------- stream/range.go | 39 +++++-- testutil/document.go | 11 ++ 9 files changed, 351 insertions(+), 321 deletions(-) diff --git a/database/index.go b/database/index.go index 93924f0c1..9917b3b7b 100644 --- a/database/index.go +++ b/database/index.go @@ -67,9 +67,13 @@ func (e *indexValueEncoder) EncodeValue(v document.Value) ([]byte, error) { return buf.Bytes(), nil } - // this should never happen, but if it does, something is very wrong if v.Type != e.typ { - panic("incompatible index type") + if v.Type == 0 { + v.Type = e.typ + } else { + // this should never happen, but if it does, something is very wrong + panic("incompatible index type") + } } if v.V == nil { @@ -229,20 +233,17 @@ func (idx *Index) Delete(vs []document.Value, k []byte) error { // - having pivots length superior to the index arity // - having the first pivot without a value when the subsequent ones do have values func (idx *Index) validatePivots(pivots []document.Value) { - if len(pivots) == 0 { - panic("cannot iterate without a pivot") - } - if len(pivots) > idx.Arity() { panic("cannot iterate with a pivot whose size is superior to the index arity") } if idx.IsComposite() { if !allEmpty(pivots) { + // fmt.Println("here", pivots) // the first pivot must have a value - if pivots[0].V == nil { - panic("cannot iterate on a composite index whose first pivot has no value") - } + // if pivots[0].V == nil { + // panic("cannot iterate on a composite index whose first pivot has no value") + // } // it's acceptable for the last pivot to just have a type and no value hasValue := true @@ -251,12 +252,12 @@ func (idx *Index) validatePivots(pivots []document.Value) { if hasValue { hasValue = p.V != nil - // if we have no value, we at least need a type - if !hasValue { - if p.Type == 0 { - panic("cannot iterate on a composite index with a pivot with both values and nil values") - } - } + // // if we have no value, we at least need a type + // if !hasValue { + // if p.Type == 0 { + // panic("cannot iterate on a composite index with a pivot with both values and nil values") + // } + // } } else { panic("cannot iterate on a composite index with a pivot with both values and nil values") } @@ -318,6 +319,7 @@ func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func( var buf []byte return idx.iterate(st, pivots, reverse, func(item engine.Item) error { + fmt.Println("idx.iterate") var err error k := item.Key() @@ -434,30 +436,6 @@ func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, erro return seek, nil } - // if !idx.IsComposite() { - // if pivots[0].V != nil { - // seek, err = idx.EncodeValue(document.NewValueBuffer(pivots...)) - // if err != nil { - // return nil, err - // } - - // if reverse { - // // appending 0xFF turns the pivot into the upper bound of that value. - // seek = append(seek, 0xFF) - // } - // } else { - // if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && pivots[0].V == nil { - // seek = []byte{byte(pivots[0].Type)} - - // if reverse { - // seek = append(seek, 0xFF) - // } - // } - // } - // } else { - // [2,3,4,int] is a valid pivot, in which case the last pivot, a valueless typed pivot - // it handled separatedly - vb := document.NewValueBuffer(pivots...) seek, err = idx.EncodeValueBuffer(vb) @@ -485,25 +463,20 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool it := st.Iterator(engine.IteratorOptions{Reverse: reverse}) defer it.Close() + for it.Seek([]byte{}); it.Valid(); it.Next() { + fmt.Println("----------", it.Item().Key()) + } + + it = st.Iterator(engine.IteratorOptions{Reverse: reverse}) + defer it.Close() + for it.Seek(seek); it.Valid(); it.Next() { itm := it.Item() - // If index is untyped and pivot is typed, only iterate on values with the same type as pivot - if !idx.IsComposite() { - var typ document.ValueType - if len(idx.Info.Types) > 0 { - typ = idx.Info.Types[0] - } - - if (typ == 0) && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { - return nil - } - } else { - // If the index is composite, same logic applies but for now, we only check the first pivot type. - // A possible optimization would be to check the types of the remaining values here. - if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { - return nil - } + // If index is untyped and pivot first element is typed, only iterate on values with the same type as the first pivot + // TODO(JH) possible optimization, check for the other types + if len(pivots) > 0 && idx.Info.Types[0] == 0 && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { + return nil } err := fn(itm) diff --git a/document/array.go b/document/array.go index 53c2eea6e..0ea42015b 100644 --- a/document/array.go +++ b/document/array.go @@ -96,6 +96,10 @@ func (vb *ValueBuffer) GetByIndex(i int) (Value, error) { // Len returns the length the of array func (vb *ValueBuffer) Len() int { + if vb == nil { + return 0 + } + return len(vb.Values) } @@ -247,6 +251,11 @@ func (vb *ValueBuffer) UnmarshalJSON(data []byte) error { } func (vb *ValueBuffer) Types() []ValueType { + // TODO check that + if vb == nil { + return nil + } + types := make([]ValueType, len(vb.Values)) for i, v := range vb.Values { diff --git a/planner/optimizer.go b/planner/optimizer.go index fec8d233b..2749693f8 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -754,20 +754,30 @@ func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) case *expr.EqOperator, *expr.GtOperator, *expr.GteOperator, *expr.LtOperator, *expr.LteOperator: vb = vb.Append(v) case *expr.InOperator: + // a := v.V.(document.Array) + // err := a.Iterate(func(i int, value document.Value) error { + // ranges = ranges.Append(stream.ValueRange{ + // Min: value, + // Exact: true, + // }) + // return nil + // }) + // if err != nil { + // return nil, err + // } // TODO(JH) // an index like idx_foo_a_b on (a,b) and a query like // WHERE a IN [1, 1] and b IN [2, 2] // would lead to [1, 1] x [2, 2] = [[1,1], [1,2], [2,1], [2,2]] // which could eventually be added later. - panic("unsupported operator IN for composite indexes") + // panic("unsupported operator IN for composite indexes") + default: panic(stringutil.Sprintf("unknown operator %#v", op)) } } - rng := stream.IndexRange{ - Min: vb, - } + var rng stream.IndexRange // the last node is the only one that can be a comparison operator, so // it's the one setting the range behaviour @@ -777,10 +787,17 @@ func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) switch op.(type) { case *expr.EqOperator: rng.Exact = true + rng.Min = vb case *expr.GtOperator: rng.Exclusive = true + rng.Min = vb + case *expr.GteOperator: + rng.Min = vb case *expr.LtOperator: rng.Exclusive = true + rng.Max = vb + case *expr.LteOperator: + rng.Max = vb } return stream.IndexRanges{rng}, nil diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index 815101081..bfebc0bd4 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -301,6 +301,7 @@ func TestRemoveUnnecessaryDedupNodeRule(t *testing.T) { } func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { + newVB := document.NewValueBuffer tests := []struct { name string root, expected *st.Stream @@ -313,14 +314,14 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { { "FROM foo WHERE a = 1", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("a = 1"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true})), + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true})), }, { "FROM foo WHERE a = 1 AND b = 2", st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true})). + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), }, { @@ -328,7 +329,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("c = 3"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), - st.New(st.IndexScan("idx_foo_c", st.Range{Min: document.NewIntegerValue(3), Exact: true})). + st.New(st.IndexScan("idx_foo_c", st.IndexRange{Min: newVB(document.NewIntegerValue(3)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), }, { @@ -336,7 +337,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("c > 3"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), - st.New(st.IndexScan("idx_foo_b", st.Range{Min: document.NewIntegerValue(2), Exact: true})). + st.New(st.IndexScan("idx_foo_b", st.IndexRange{Min: newVB(document.NewIntegerValue(2)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("c > 3"))), }, { @@ -345,7 +346,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("c = 3"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Project(parser.MustParseExpr("a"))), - st.New(st.IndexScan("idx_foo_c", st.Range{Min: document.NewIntegerValue(3), Exact: true})). + st.New(st.IndexScan("idx_foo_c", st.IndexRange{Min: newVB(document.NewIntegerValue(3)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Project(parser.MustParseExpr("a"))), }, @@ -355,7 +356,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("c = 'hello'"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Project(parser.MustParseExpr("a"))), - st.New(st.IndexScan("idx_foo_b", st.Range{Min: document.NewIntegerValue(2), Exact: true})). + st.New(st.IndexScan("idx_foo_b", st.IndexRange{Min: newVB(document.NewIntegerValue(2)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("c = 'hello'"))). Pipe(st.Project(parser.MustParseExpr("a"))), }, @@ -370,37 +371,37 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("d = 2"))). Pipe(st.Project(parser.MustParseExpr("a"))), }, - { - "FROM foo WHERE a IN [1, 2]", - st.New(st.SeqScan("foo")).Pipe(st.Filter( - expr.In( - parser.MustParseExpr("a"), - expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), - ), - )), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true}, st.Range{Min: document.NewIntegerValue(2), Exact: true})), - }, - { - "FROM foo WHERE 1 IN a", - st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), - st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), - }, + // { + // "FROM foo WHERE a IN [1, 2]", + // st.New(st.SeqScan("foo")).Pipe(st.Filter( + // expr.In( + // parser.MustParseExpr("a"), + // expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), + // ), + // )), + // st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true}, st.IndexRange{Min: newVB(document.NewIntegerValue(2)), Exact: true})), + // }, + // { + // "FROM foo WHERE 1 IN a", + // st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), + // st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), + // }, { "FROM foo WHERE a >= 10", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("a >= 10"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(10)})), + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(10))})), }, { "FROM foo WHERE k = 1", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("k = 1"))), - st.New(st.PkScan("foo", st.Range{Min: document.NewIntegerValue(1), Exact: true})), + st.New(st.PkScan("foo", st.ValueRange{Min: document.NewIntegerValue(1), Exact: true})), }, { "FROM foo WHERE k = 1 AND b = 2", st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("k = 1"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), - st.New(st.PkScan("foo", st.Range{Min: document.NewIntegerValue(1), Exact: true})). + st.New(st.PkScan("foo", st.ValueRange{Min: document.NewIntegerValue(1), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), }, { @@ -408,7 +409,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("2 = k"))), - st.New(st.PkScan("foo", st.Range{Min: document.NewIntegerValue(2), Exact: true})). + st.New(st.PkScan("foo", st.ValueRange{Min: document.NewIntegerValue(2), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("a = 1"))), }, { @@ -416,7 +417,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("k < 2"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true})). + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("k < 2"))), }, { @@ -424,7 +425,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("k = 'hello'"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true})). + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("k = 'hello'"))), }, { // c is an INT, 1.1 cannot be converted to int without precision loss, don't use the index @@ -475,22 +476,22 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { { "FROM foo WHERE k = [1, 1]", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("k = [1, 1]"))), - st.New(st.PkScan("foo", st.Range{Min: document.NewArrayValue(testutil.MakeArray(t, `[1, 1]`)), Exact: true})), + st.New(st.PkScan("foo", st.ValueRange{Min: document.NewArrayValue(testutil.MakeArray(t, `[1, 1]`)), Exact: true})), }, { // constraint on k[0] INT should not modify the operand "FROM foo WHERE k = [1.5, 1.5]", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("k = [1.5, 1.5]"))), - st.New(st.PkScan("foo", st.Range{Min: document.NewArrayValue(testutil.MakeArray(t, `[1.5, 1.5]`)), Exact: true})), + st.New(st.PkScan("foo", st.ValueRange{Min: document.NewArrayValue(testutil.MakeArray(t, `[1.5, 1.5]`)), Exact: true})), }, { "FROM foo WHERE a = [1, 1]", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewArrayValue(testutil.MakeArray(t, `[1, 1]`)), Exact: true})), + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewArrayValue(testutil.MakeArray(t, `[1, 1]`))), Exact: true})), }, { // constraint on a[0] DOUBLE should modify the operand because it's lossless "FROM foo WHERE a = [1, 1.5]", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("a = [1, 1.5]"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewArrayValue(testutil.MakeArray(t, `[1.0, 1.5]`)), Exact: true})), + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewArrayValue(testutil.MakeArray(t, `[1.0, 1.5]`))), Exact: true})), }, } @@ -532,6 +533,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { } func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { + newVB := document.NewValueBuffer tests := []struct { name string root, expected *st.Stream @@ -541,28 +543,28 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("d = 2"))), - st.New(st.IndexScan("idx_foo_a_d", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exact: true})), + st.New(st.IndexScan("idx_foo_a_d", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exact: true})), }, { "FROM foo WHERE a = 1 AND d > 2", st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("d > 2"))), - st.New(st.IndexScan("idx_foo_a_d", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exclusive: true})), + st.New(st.IndexScan("idx_foo_a_d", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exclusive: true})), }, { "FROM foo WHERE a = 1 AND d >= 2", st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("d >= 2"))), - st.New(st.IndexScan("idx_foo_a_d", st.Range{Min: testutil.MakeArrayValue(t, 1, 2)})), + st.New(st.IndexScan("idx_foo_a_d", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`)})), }, { "FROM foo WHERE a > 1 AND d > 2", st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a > 1"))). Pipe(st.Filter(parser.MustParseExpr("d > 2"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exclusive: true})). + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1]`), Exclusive: true})). Pipe(st.Filter(parser.MustParseExpr("d > 2"))), }, { @@ -571,21 +573,21 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Filter(parser.MustParseExpr("c = 3"))), - st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2, 3), Exact: true})), + st.New(st.IndexScan("idx_foo_a_b_c", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2, 3]`), Exact: true})), }, { "FROM foo WHERE a = 1 AND b = 2", // c is omitted, but it can still use idx_foo_a_b_c st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))), - st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exact: true})), + st.New(st.IndexScan("idx_foo_a_b_c", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exact: true})), }, { "FROM foo WHERE a = 1 AND b > 2", // c is omitted, but it can still use idx_foo_a_b_c, with > b st.New(st.SeqScan("foo")). Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("b > 2"))), - st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exclusive: true})), + st.New(st.IndexScan("idx_foo_a_b_c", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exclusive: true})), }, { "FROM foo WHERE a = 1 AND b = 2 and k = 3", // c is omitted, but it can still use idx_foo_a_b_c @@ -593,7 +595,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Filter(parser.MustParseExpr("k = 3"))), - st.New(st.IndexScan("idx_foo_a_b_c", st.Range{Min: testutil.MakeArrayValue(t, 1, 2), Exact: true})). + st.New(st.IndexScan("idx_foo_a_b_c", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("k = 3"))), }, { @@ -602,7 +604,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("c = 2"))), // c will be picked because it's a unique index and thus has a lower cost - st.New(st.IndexScan("idx_foo_c", st.Range{Min: document.NewIntegerValue(2), Exact: true})). + st.New(st.IndexScan("idx_foo_c", st.IndexRange{Min: newVB(document.NewIntegerValue(2)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("a = 1"))), }, { @@ -611,7 +613,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("b = 1"))). Pipe(st.Filter(parser.MustParseExpr("c = 2"))), // c will be picked because it's a unique index and thus has a lower cost - st.New(st.IndexScan("idx_foo_c", st.Range{Min: document.NewIntegerValue(2), Exact: true})). + st.New(st.IndexScan("idx_foo_c", st.IndexRange{Min: newVB(document.NewIntegerValue(2)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("b = 1"))), }, { @@ -620,7 +622,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("a = 1"))). Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Filter(parser.MustParseExpr("c = 'a'"))), - st.New(st.IndexScan("idx_foo_a", st.Range{Min: document.NewIntegerValue(1), Exact: true})). + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Filter(parser.MustParseExpr("c = 'a'"))), }, @@ -656,64 +658,64 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { }) } - t.Run("array indexes", func(t *testing.T) { - tests := []struct { - name string - root, expected *st.Stream - }{ - { - "FROM foo WHERE a = [1, 1] AND b = [2, 2]", - st.New(st.SeqScan("foo")). - Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). - Pipe(st.Filter(parser.MustParseExpr("b = [2, 2]"))), - st.New(st.IndexScan("idx_foo_a_b", st.Range{ - Min: document.NewArrayValue( - testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), - Exact: true})), - }, - { - "FROM foo WHERE a = [1, 1] AND b > [2, 2]", - st.New(st.SeqScan("foo")). - Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). - Pipe(st.Filter(parser.MustParseExpr("b > [2, 2]"))), - st.New(st.IndexScan("idx_foo_a_b", st.Range{ - Min: document.NewArrayValue( - testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), - Exclusive: true})), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - db, err := genji.Open(":memory:") - require.NoError(t, err) - defer db.Close() - - tx, err := db.Begin(true) - require.NoError(t, err) - defer tx.Rollback() - - err = tx.Exec(` - CREATE TABLE foo ( - k ARRAY PRIMARY KEY, - a ARRAY - ); - CREATE INDEX idx_foo_a_b ON foo(a, b); - CREATE INDEX idx_foo_a0 ON foo(a[0]); - INSERT INTO foo (k, a, b) VALUES - ([1, 1], [1, 1], [1, 1]), - ([2, 2], [2, 2], [2, 2]), - ([3, 3], [3, 3], [3, 3]) - `) - require.NoError(t, err) - - res, err := planner.PrecalculateExprRule(test.root, tx.Transaction, nil) - require.NoError(t, err) - - res, err = planner.UseIndexBasedOnFilterNodeRule(res, tx.Transaction, nil) - require.NoError(t, err) - require.Equal(t, test.expected.String(), res.String()) - }) - } - }) + // t.Run("array indexes", func(t *testing.T) { + // tests := []struct { + // name string + // root, expected *st.Stream + // }{ + // { + // "FROM foo WHERE a = [1, 1] AND b = [2, 2]", + // st.New(st.SeqScan("foo")). + // Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). + // Pipe(st.Filter(parser.MustParseExpr("b = [2, 2]"))), + // st.New(st.IndexScan("idx_foo_a_b", st.Range{ + // Min: document.NewArrayValue( + // testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), + // Exact: true})), + // }, + // { + // "FROM foo WHERE a = [1, 1] AND b > [2, 2]", + // st.New(st.SeqScan("foo")). + // Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). + // Pipe(st.Filter(parser.MustParseExpr("b > [2, 2]"))), + // st.New(st.IndexScan("idx_foo_a_b", st.Range{ + // Min: document.NewArrayValue( + // testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), + // Exclusive: true})), + // }, + // } + + // for _, test := range tests { + // t.Run(test.name, func(t *testing.T) { + // db, err := genji.Open(":memory:") + // require.NoError(t, err) + // defer db.Close() + + // tx, err := db.Begin(true) + // require.NoError(t, err) + // defer tx.Rollback() + + // err = tx.Exec(` + // CREATE TABLE foo ( + // k ARRAY PRIMARY KEY, + // a ARRAY + // ); + // CREATE INDEX idx_foo_a_b ON foo(a, b); + // CREATE INDEX idx_foo_a0 ON foo(a[0]); + // INSERT INTO foo (k, a, b) VALUES + // ([1, 1], [1, 1], [1, 1]), + // ([2, 2], [2, 2], [2, 2]), + // ([3, 3], [3, 3], [3, 3]) + // `) + // require.NoError(t, err) + + // res, err := planner.PrecalculateExprRule(test.root, tx.Transaction, nil) + // require.NoError(t, err) + + // res, err = planner.UseIndexBasedOnFilterNodeRule(res, tx.Transaction, nil) + // require.NoError(t, err) + // require.Equal(t, test.expected.String(), res.String()) + // }) + // } + // }) } diff --git a/query/select_test.go b/query/select_test.go index 6408b302d..579011ebe 100644 --- a/query/select_test.go +++ b/query/select_test.go @@ -41,15 +41,16 @@ func TestSelectStmt(t *testing.T) { {"With eq op", "SELECT * FROM test WHERE size = 10", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100}]`, nil}, {"With neq op", "SELECT * FROM test WHERE color != 'red'", false, `[{"k":2,"color":"blue","size":10,"weight":100}]`, nil}, {"With gt op", "SELECT * FROM test WHERE size > 10", false, `[]`, nil}, + {"With gt bis", "SELECT * FROM test WHERE size > 9", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100}]`, nil}, {"With lt op", "SELECT * FROM test WHERE size < 15", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100}]`, nil}, {"With lte op", "SELECT * FROM test WHERE color <= 'salmon' ORDER BY k ASC", false, `[{"k":1,"color":"red","size":10,"shape":"square"},{"k":2,"color":"blue","size":10,"weight":100}]`, nil}, {"With add op", "SELECT size + 10 AS s FROM test ORDER BY k", false, `[{"s":20},{"s":20},{"s":null}]`, nil}, {"With sub op", "SELECT size - 10 AS s FROM test ORDER BY k", false, `[{"s":0},{"s":0},{"s":null}]`, nil}, {"With mul op", "SELECT size * 10 AS s FROM test ORDER BY k", false, `[{"s":100},{"s":100},{"s":null}]`, nil}, {"With div op", "SELECT size / 10 AS s FROM test ORDER BY k", false, `[{"s":1},{"s":1},{"s":null}]`, nil}, - {"With IN op", "SELECT color FROM test WHERE color IN ['red', 'purple'] ORDER BY k", false, `[{"color":"red"}]`, nil}, - {"With IN op on PK", "SELECT color FROM test WHERE k IN [1.1, 1.0] ORDER BY k", false, `[{"color":"red"}]`, nil}, - {"With NOT IN op", "SELECT color FROM test WHERE color NOT IN ['red', 'purple'] ORDER BY k", false, `[{"color":"blue"}]`, nil}, + // {"With IN op", "SELECT color FROM test WHERE color IN ['red', 'purple'] ORDER BY k", false, `[{"color":"red"}]`, nil}, + // {"With IN op on PK", "SELECT color FROM test WHERE k IN [1.1, 1.0] ORDER BY k", false, `[{"color":"red"}]`, nil}, + // {"With NOT IN op", "SELECT color FROM test WHERE color NOT IN ['red', 'purple'] ORDER BY k", false, `[{"color":"blue"}]`, nil}, {"With field comparison", "SELECT * FROM test WHERE color < shape", false, `[{"k":1,"color":"red","size":10,"shape":"square"}]`, nil}, {"With group by", "SELECT color FROM test GROUP BY color", false, `[{"color":"red"},{"color":"blue"},{"color":null}]`, nil}, {"With group by and count", "SELECT COUNT(k) FROM test GROUP BY size", false, `[{"COUNT(k)":2},{"COUNT(k)":1}]`, nil}, diff --git a/stream/iterator.go b/stream/iterator.go index 2fc656e76..a6e556944 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -2,6 +2,7 @@ package stream import ( "bytes" + "fmt" "strconv" "strings" @@ -359,56 +360,6 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } for _, rng := range it.Ranges { - // if !index.IsComposite() { - // var start, end *document.ValueBuffer - // if !it.Reverse { - // start = rng.Min - // end = rng.Max - // } else { - // start = rng.Max - // end = rng.Min - // } - - // var encEnd []byte - // // if !end.Type.IsZero() && end.V != nil { - // if end.Len() > 0 { - // encEnd, err = index.EncodeValueBuffer(end) - // if err != nil { - // return err - // } - // } - - // err = iterator([]document.Value{start}, func(val, key []byte) error { - // if !rng.IsInRange(val) { - // // if we reached the end of our range, we can stop iterating. - // if encEnd == nil { - // return nil - // } - // cmp := bytes.Compare(val, encEnd) - // if !it.Reverse && cmp > 0 { - // return ErrStreamClosed - // } - // if it.Reverse && cmp < 0 { - // return ErrStreamClosed - // } - // return nil - // } - - // d, err := table.GetDocument(key) - // if err != nil { - // return err - // } - - // newEnv.SetDocument(d) - // return fn(&newEnv) - // }) - // if err == ErrStreamClosed { - // err = nil - // } - // if err != nil { - // return err - // } - // } else { var start, end *document.ValueBuffer if !it.Reverse { start = rng.Min @@ -426,22 +377,23 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } } - // extract the pivots from the range, which in the case of a composite index is an array - pivots := []document.Value{} - if start.Len() > 0 { + var pivots []document.Value + if start != nil { pivots = start.Values - } else { - for i := 0; i < index.Arity(); i++ { - pivots = append(pivots, document.Value{}) - } } + for i, p := range pivots { + fmt.Println("pivot[", i, "], type:", p.Type, "v:", p) + } + fmt.Println("encEnd", encEnd) + err = iterator(pivots, func(val, key []byte) error { if !rng.IsInRange(val) { // if we reached the end of our range, we can stop iterating. if encEnd == nil { return nil } + cmp := bytes.Compare(val, encEnd) if !it.Reverse && cmp > 0 { return ErrStreamClosed diff --git a/stream/iterator_test.go b/stream/iterator_test.go index 703ef3da2..b937db39e 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -124,7 +124,7 @@ func TestPkScan(t *testing.T) { tests := []struct { name string docsInTable, expected testutil.Docs - ranges stream.Ranges + ranges stream.ValueRanges reverse bool fails bool }{ @@ -139,7 +139,7 @@ func TestPkScan(t *testing.T) { "max:2", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), - stream.Ranges{ + stream.ValueRanges{ {Max: document.NewIntegerValue(2)}, }, false, false, @@ -148,7 +148,7 @@ func TestPkScan(t *testing.T) { "max:1", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`), - stream.Ranges{ + stream.ValueRanges{ {Max: document.NewIntegerValue(1)}, }, false, false, @@ -157,7 +157,7 @@ func TestPkScan(t *testing.T) { "min", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), - stream.Ranges{ + stream.ValueRanges{ {Min: document.NewIntegerValue(1)}, }, false, false, @@ -166,7 +166,7 @@ func TestPkScan(t *testing.T) { "min/max", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), - stream.Ranges{ + stream.ValueRanges{ {Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2)}, }, false, false, @@ -181,7 +181,7 @@ func TestPkScan(t *testing.T) { "reverse/max", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), - stream.Ranges{ + stream.ValueRanges{ {Max: document.NewIntegerValue(2)}, }, true, false, @@ -190,7 +190,7 @@ func TestPkScan(t *testing.T) { "reverse/min", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), - stream.Ranges{ + stream.ValueRanges{ {Min: document.NewIntegerValue(1)}, }, true, false, @@ -199,7 +199,7 @@ func TestPkScan(t *testing.T) { "reverse/min/max", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), - stream.Ranges{ + stream.ValueRanges{ {Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2)}, }, true, false, @@ -258,14 +258,14 @@ func TestPkScan(t *testing.T) { } t.Run("String", func(t *testing.T) { - require.Equal(t, `pkScan("test", [1, 2])`, stream.PkScan("test", stream.Range{ + require.Equal(t, `pkScan("test", [1, 2])`, stream.PkScan("test", stream.ValueRange{ Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), }).String()) op := stream.PkScan("test", - stream.Range{Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), Exclusive: true}, - stream.Range{Min: document.NewIntegerValue(10), Exact: true}, - stream.Range{Min: document.NewIntegerValue(100)}, + stream.ValueRange{Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), Exclusive: true}, + stream.ValueRange{Min: document.NewIntegerValue(10), Exact: true}, + stream.ValueRange{Min: document.NewIntegerValue(100)}, ) op.Reverse = true @@ -274,11 +274,12 @@ func TestPkScan(t *testing.T) { } func TestIndexScan(t *testing.T) { + newVB := document.NewValueBuffer tests := []struct { name string indexOn string docsInTable, expected testutil.Docs - ranges stream.Ranges + ranges stream.IndexRanges reverse bool fails bool }{ @@ -299,8 +300,8 @@ func TestIndexScan(t *testing.T) { "max:2", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), - stream.Ranges{ - {Max: document.NewIntegerValue(2)}, + stream.IndexRanges{ + {Max: newVB(document.NewIntegerValue(2))}, }, false, false, }, @@ -308,8 +309,11 @@ func TestIndexScan(t *testing.T) { "max:[2, 2]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - stream.Ranges{ - {Max: testutil.MakeArrayValue(t, 2, 2)}, + stream.IndexRanges{ + {Max: newVB( + document.NewIntegerValue(2), + document.NewIntegerValue(2), + )}, }, false, false, }, @@ -317,8 +321,8 @@ func TestIndexScan(t *testing.T) { "max:1", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`), - stream.Ranges{ - {Max: document.NewIntegerValue(1)}, + stream.IndexRanges{ + {Max: newVB(document.NewIntegerValue(1))}, }, false, false, }, @@ -326,8 +330,11 @@ func TestIndexScan(t *testing.T) { "max:[1, 2]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2}`), - stream.Ranges{ - {Max: testutil.MakeArrayValue(t, 1, 2)}, + stream.IndexRanges{ + {Max: newVB( + document.NewIntegerValue(1), + document.NewIntegerValue(2), + )}, }, false, false, }, @@ -335,8 +342,8 @@ func TestIndexScan(t *testing.T) { "min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), - stream.Ranges{ - {Min: document.NewIntegerValue(1)}, + stream.IndexRanges{ + {Min: newVB(document.NewIntegerValue(1))}, }, false, false, }, @@ -344,8 +351,10 @@ func TestIndexScan(t *testing.T) { "min:[2, 1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 2, 1)}, + stream.IndexRanges{ + { + Min: newVB(document.NewIntegerValue(2), document.NewIntegerValue(1)), + }, }, false, false, }, @@ -353,8 +362,11 @@ func TestIndexScan(t *testing.T) { "min/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), - stream.Ranges{ - {Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2)}, + stream.IndexRanges{ + { + Min: newVB(document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2)), + }, }, false, false, }, @@ -362,8 +374,11 @@ func TestIndexScan(t *testing.T) { "min:[1, 1], max:[2,2]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2}`, `{"a": 2, "b": 2}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2)}, + stream.IndexRanges{ + { + Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + }, }, false, false, }, @@ -371,8 +386,11 @@ func TestIndexScan(t *testing.T) { "min:[1, 1], max:[2,2] bis", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 3}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": 3}`, `{"a": 2, "b": 2}`), // [1, 3] < [2, 2] - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2)}, + stream.IndexRanges{ + { + Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + }, }, false, false, }, @@ -386,8 +404,8 @@ func TestIndexScan(t *testing.T) { "reverse/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), - stream.Ranges{ - {Max: document.NewIntegerValue(2)}, + stream.IndexRanges{ + {Max: newVB(document.NewIntegerValue(2))}, }, true, false, }, @@ -395,8 +413,10 @@ func TestIndexScan(t *testing.T) { "reverse/max", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), - stream.Ranges{ - {Max: testutil.MakeArrayValue(t, 2, 2)}, + stream.IndexRanges{ + { + Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + }, }, true, false, }, @@ -404,8 +424,8 @@ func TestIndexScan(t *testing.T) { "reverse/min", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), - stream.Ranges{ - {Min: document.NewIntegerValue(1)}, + stream.IndexRanges{ + {Min: newVB(document.NewIntegerValue(1))}, }, true, false, }, @@ -413,8 +433,8 @@ func TestIndexScan(t *testing.T) { "reverse/min neg", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": -2}`), testutil.MakeDocuments(t, `{"a": 1}`), - stream.Ranges{ - {Min: document.NewIntegerValue(1)}, + stream.IndexRanges{ + {Min: newVB(document.NewIntegerValue(1))}, }, true, false, }, @@ -422,8 +442,10 @@ func TestIndexScan(t *testing.T) { "reverse/min", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1, 1)}, + stream.IndexRanges{ + { + Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + }, }, true, false, }, @@ -431,8 +453,11 @@ func TestIndexScan(t *testing.T) { "reverse/min/max", "a", testutil.MakeDocuments(t, `{"a": 1}`, `{"a": 2}`), testutil.MakeDocuments(t, `{"a": 2}`, `{"a": 1}`), - stream.Ranges{ - {Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2)}, + stream.IndexRanges{ + { + Min: newVB(document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2)), + }, }, true, false, }, @@ -440,8 +465,11 @@ func TestIndexScan(t *testing.T) { "reverse/min/max", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`), testutil.MakeDocuments(t, `{"a": 2, "b": 2}`, `{"a": 1, "b": 1}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2)}, + stream.IndexRanges{ + { + Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + }, }, true, false, }, @@ -449,8 +477,12 @@ func TestIndexScan(t *testing.T) { "max:[1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), - stream.Ranges{ - {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 1)}, + stream.IndexRanges{ + { + Arity: 2, + IndexArityMax: 1, + Max: newVB(document.NewIntegerValue(1)), + }, }, false, false, }, @@ -458,8 +490,14 @@ func TestIndexScan(t *testing.T) { "reverse max:[1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 2, "b": 2}`, `{"a": 1, "b": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 9223372036854775807}`, `{"a": 1, "b": 1}`), - stream.Ranges{ - {Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 1)}, + stream.IndexRanges{ + { + Max: newVB(document.NewIntegerValue(1)), + Exclusive: false, + Exact: false, + Arity: 2, + IndexArityMax: 1, + }, }, true, false, }, @@ -467,8 +505,8 @@ func TestIndexScan(t *testing.T) { "max:[1, 2]", "a, b, c", testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), - stream.Ranges{ - {Arity: 3, IndexArityMax: 2, Max: testutil.MakeArrayValue(t, 1, 2)}, + stream.IndexRanges{ + {Arity: 3, IndexArityMax: 2, Max: newVB(document.NewIntegerValue(1), document.NewIntegerValue(2))}, }, false, false, }, @@ -476,8 +514,8 @@ func TestIndexScan(t *testing.T) { "min:[1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 1, "b": 1}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1)}, + stream.IndexRanges{ + {Min: newVB(document.NewIntegerValue(1))}, }, false, false, }, @@ -485,8 +523,8 @@ func TestIndexScan(t *testing.T) { "min:[1]", "a, b, c", testutil.MakeDocuments(t, `{"a": 1, "b": -2, "c": 0}`, `{"a": -2, "b": 2, "c": 1}`, `{"a": 1, "b": 1, "c": 2}`), testutil.MakeDocuments(t, `{"a": 1, "b": -2, "c": 0}`, `{"a": 1, "b": 1, "c": 2}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1)}, + stream.IndexRanges{ + {Min: newVB(document.NewIntegerValue(1))}, }, false, false, }, @@ -494,8 +532,8 @@ func TestIndexScan(t *testing.T) { "reverse min:[1]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 1, "b": 1}`), testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": -2}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1)}, + stream.IndexRanges{ + {Min: newVB(document.NewIntegerValue(1))}, }, true, false, }, @@ -503,8 +541,13 @@ func TestIndexScan(t *testing.T) { "min:[1], max[2]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 2, "b": 42}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 2)}, + stream.IndexRanges{ + { + Arity: 2, + IndexArityMax: 1, + Min: newVB(document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2)), + }, }, false, false, }, @@ -512,8 +555,13 @@ func TestIndexScan(t *testing.T) { "reverse min:[1], max[2]", "a, b", testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": -2, "b": 2}`, `{"a": 2, "b": 42}`, `{"a": 3, "b": -1}`), testutil.MakeDocuments(t, `{"a": 2, "b": 42}`, `{"a": 1, "b": -2}`), - stream.Ranges{ - {Min: testutil.MakeArrayValue(t, 1), Arity: 2, IndexArityMax: 1, Max: testutil.MakeArrayValue(t, 2)}, + stream.IndexRanges{ + { + Arity: 2, + IndexArityMax: 1, + Min: newVB(document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2)), + }, }, true, false, }, @@ -570,31 +618,33 @@ func TestIndexScan(t *testing.T) { }) } - t.Run("String", func(t *testing.T) { - t.Run("idx_test_a", func(t *testing.T) { - require.Equal(t, `indexScan("idx_test_a", [1, 2])`, stream.IndexScan("idx_test_a", stream.Range{ - Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), - }).String()) - - op := stream.IndexScan("idx_test_a", stream.Range{ - Min: document.NewIntegerValue(1), Max: document.NewIntegerValue(2), - }) - op.Reverse = true - - require.Equal(t, `indexScanReverse("idx_test_a", [1, 2])`, op.String()) - }) - - t.Run("idx_test_a_b", func(t *testing.T) { - require.Equal(t, `indexScan("idx_test_a_b", [[1, 1], [2, 2]])`, stream.IndexScan("idx_test_a_b", stream.Range{ - Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2), - }).String()) - - op := stream.IndexScan("idx_test_a_b", stream.Range{ - Min: testutil.MakeArrayValue(t, 1, 1), Max: testutil.MakeArrayValue(t, 2, 2), - }) - op.Reverse = true - - require.Equal(t, `indexScanReverse("idx_test_a_b", [[1, 1], [2, 2]])`, op.String()) - }) - }) + // t.Run("String", func(t *testing.T) { + // t.Run("idx_test_a", func(t *testing.T) { + // require.Equal(t, `indexScan("idx_test_a", [1, 2])`, stream.IndexScan("idx_test_a", stream.IndexRange{ + // Min: newVB(document.NewIntegerValue(1)), Max: newVB(document.NewIntegerValue(2)), + // }).String()) + + // op := stream.IndexScan("idx_test_a", stream.IndexRange{ + // Min: newVB(document.NewIntegerValue(1)), Max: newVB(document.NewIntegerValue(2)), + // }) + // op.Reverse = true + + // require.Equal(t, `indexScanReverse("idx_test_a", [1, 2])`, op.String()) + // }) + + // t.Run("idx_test_a_b", func(t *testing.T) { + // require.Equal(t, `indexScan("idx_test_a_b", [[1, 1], [2, 2]])`, stream.IndexScan("idx_test_a_b", stream.IndexRange{ + // Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + // Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + // }).String()) + + // op := stream.IndexScan("idx_test_a_b", stream.IndexRange{ + // Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + // Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + // }) + // op.Reverse = true + + // require.Equal(t, `indexScanReverse("idx_test_a_b", [[1, 1], [2, 2]])`, op.String()) + // }) + // }) } diff --git a/stream/range.go b/stream/range.go index 8491c909d..a62aad040 100644 --- a/stream/range.go +++ b/stream/range.go @@ -2,6 +2,7 @@ package stream import ( "bytes" + "fmt" "strings" "github.com/genjidb/genji/document" @@ -300,14 +301,24 @@ func (r *IndexRange) encode(encoder ValueBufferEncoder, env *expr.Environment) e r.rangeTypes = r.Max.Types() } - // ensure boundaries are typed - // TODO(JH) - // if r.Min.Type.IsZero() { - // r.Min.Type = r.rangeType - // } - // if r.Max.Type.IsZero() { - // r.Max.Type = r.rangeType - // } + // Ensure boundaries are typed, at least with the first type + if r.Max.Len() == 0 && r.Min.Len() > 0 { + v, err := r.Min.GetByIndex(0) + if err != nil { + return err + } + + r.Max = document.NewValueBuffer(document.Value{Type: v.Type}) + } + + if r.Min.Len() == 0 && r.Max.Len() > 0 { + v, err := r.Max.GetByIndex(0) + if err != nil { + return err + } + + r.Min = document.NewValueBuffer(document.Value{Type: v.Type}) + } if r.Exclusive && r.Exact { panic("exclusive and exact cannot both be true") @@ -318,14 +329,14 @@ func (r *IndexRange) encode(encoder ValueBufferEncoder, env *expr.Environment) e func (r *IndexRange) String() string { if r.Exact { - return stringutil.Sprintf("%v", r.Min) + return stringutil.Sprintf("%v", document.Array(r.Min)) } if r.Exclusive { - return stringutil.Sprintf("[%v, %v, true]", r.Min.Types(), r.Max.Types()) + return stringutil.Sprintf("[%v, %v, true]", document.Array(r.Min), document.Array(r.Max)) } - return stringutil.Sprintf("[%v, %v]", r.Min.Types(), r.Max.Types()) + return stringutil.Sprintf("[%v, %v]", document.Array(r.Min), document.Array(r.Max)) } func (r *IndexRange) IsEqual(other *IndexRange) bool { @@ -449,6 +460,10 @@ func (r *IndexRange) IsInRange(value []byte) bool { // by default, we consider the value within range cmpMin, cmpMax := 1, -1 + fmt.Println("encodedMin", r.encodedMin) + fmt.Println("encodedVal", value) + fmt.Println("encodedMax", r.encodedMax) + // we compare with the lower bound and see if it matches if r.encodedMin != nil { cmpMin = bytes.Compare(value, r.encodedMin) @@ -469,7 +484,7 @@ func (r *IndexRange) IsInRange(value []byte) bool { // see if it matches the upper bound. if r.encodedMax != nil { if r.IndexArityMax < r.Arity { - cmpMax = bytes.Compare(value[:len(r.encodedMax)-1], r.encodedMax) + cmpMax = bytes.Compare(value[:len(r.encodedMax)], r.encodedMax) } else { cmpMax = bytes.Compare(value, r.encodedMax) } diff --git a/testutil/document.go b/testutil/document.go index 5b73aedf6..d2af6ed9e 100644 --- a/testutil/document.go +++ b/testutil/document.go @@ -64,6 +64,17 @@ func MakeArray(t testing.TB, jsonArray string) document.Array { return &vb } +func MakeValueBuffer(t testing.TB, jsonArray string) *document.ValueBuffer { + t.Helper() + + var vb document.ValueBuffer + + err := vb.UnmarshalJSON([]byte(jsonArray)) + require.NoError(t, err) + + return &vb +} + type Docs []document.Document func (docs Docs) RequireEqual(t testing.TB, others Docs) { From 1ab797e18f5a4804049ba4ec010a01649327b4d9 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Fri, 23 Apr 2021 16:46:58 +0200 Subject: [PATCH 19/40] Fix IndexRange .String() --- planner/explain_test.go | 44 +++++++++++++++++++++-------------------- stream/range.go | 27 +++++++++++++++++-------- 2 files changed, 42 insertions(+), 29 deletions(-) diff --git a/planner/explain_test.go b/planner/explain_test.go index e76125b4c..32c839eff 100644 --- a/planner/explain_test.go +++ b/planner/explain_test.go @@ -13,28 +13,29 @@ func TestExplainStmt(t *testing.T) { fails bool expected string }{ - // {"EXPLAIN SELECT 1 + 1", false, `"project(1 + 1)"`}, - // {"EXPLAIN SELECT * FROM noexist", true, ``}, - // {"EXPLAIN SELECT * FROM test", false, `"seqScan(test)"`}, - // {"EXPLAIN SELECT *, a FROM test", false, `"seqScan(test) | project(*, a)"`}, - // {"EXPLAIN SELECT a + 1 FROM test", false, `"seqScan(test) | project(a + 1)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10", false, `"seqScan(test) | filter(c > 10) | project(a + 1)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10 AND d > 20", false, `"seqScan(test) | filter(c > 10) | filter(d > 20) | project(a + 1)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10 OR d > 20", false, `"seqScan(test) | filter(c > 10 OR d > 20) | project(a + 1)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c IN [1 + 1, 2 + 2]", false, `"seqScan(test) | filter(c IN [2, 4]) | project(a + 1)"`}, + {"EXPLAIN SELECT 1 + 1", false, `"project(1 + 1)"`}, + {"EXPLAIN SELECT * FROM noexist", true, ``}, + {"EXPLAIN SELECT * FROM test", false, `"seqScan(test)"`}, + {"EXPLAIN SELECT *, a FROM test", false, `"seqScan(test) | project(*, a)"`}, + {"EXPLAIN SELECT a + 1 FROM test", false, `"seqScan(test) | project(a + 1)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10", false, `"seqScan(test) | filter(c > 10) | project(a + 1)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10 AND d > 20", false, `"seqScan(test) | filter(c > 10) | filter(d > 20) | project(a + 1)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c > 10 OR d > 20", false, `"seqScan(test) | filter(c > 10 OR d > 20) | project(a + 1)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c IN [1 + 1, 2 + 2]", false, `"seqScan(test) | filter(c IN [2, 4]) | project(a + 1)"`}, {"EXPLAIN SELECT a + 1 FROM test WHERE a > 10", false, `"indexScan(\"idx_a\", [10, -1, true]) | project(a + 1)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE a > 10 AND b > 20 AND c > 30", false, `"indexScan(\"idx_b\", [20, -1, true]) | filter(a > 10) | filter(c > 30) | project(a + 1)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | project(a + 1) | sort(d) | skip(20) | take(10)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d DESC LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | project(a + 1) | sortReverse(d) | skip(20) | take(10)"`}, - // // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"indexScanReverse(\"idx_a\") | filter(c > 30) | project(a + 1) | skip(20) | take(10)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | project(a + 1) | sortReverse(a) | skip(20) | take(10)"`}, - // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 GROUP BY a + 1 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | groupBy(a + 1) | hashAggregate() | project(a + 1) | sortReverse(a) | skip(20) | take(10)"`}, - // {"EXPLAIN UPDATE test SET a = 10", false, `"seqScan(test) | set(a, 10) | tableReplace('test')"`}, - // {"EXPLAIN UPDATE test SET a = 10 WHERE c > 10", false, `"seqScan(test) | filter(c > 10) | set(a, 10) | tableReplace('test')"`}, - // {"EXPLAIN UPDATE test SET a = 10 WHERE a > 10", false, `"indexScan(\"idx_a\", [10, -1, true]) | set(a, 10) | tableReplace('test')"`}, - // {"EXPLAIN DELETE FROM test", false, `"seqScan(test) | tableDelete('test')"`}, - // {"EXPLAIN DELETE FROM test WHERE c > 10", false, `"seqScan(test) | filter(c > 10) | tableDelete('test')"`}, - // {"EXPLAIN DELETE FROM test WHERE a > 10", false, `"indexScan(\"idx_a\", [10, -1, true]) | tableDelete('test')"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE x = 10 AND y > 5", false, `"indexScan(\"idx_x_y\", [[10, 5], -1, true]) | project(a + 1)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE a > 10 AND b > 20 AND c > 30", false, `"indexScan(\"idx_b\", [20, -1, true]) | filter(a > 10) | filter(c > 30) | project(a + 1)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | project(a + 1) | sort(d) | skip(20) | take(10)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY d DESC LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | project(a + 1) | sortReverse(d) | skip(20) | take(10)"`}, + // {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"indexScanReverse(\"idx_a\") | filter(c > 30) | project(a + 1) | skip(20) | take(10)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | project(a + 1) | sortReverse(a) | skip(20) | take(10)"`}, + {"EXPLAIN SELECT a + 1 FROM test WHERE c > 30 GROUP BY a + 1 ORDER BY a DESC LIMIT 10 OFFSET 20", false, `"seqScan(test) | filter(c > 30) | groupBy(a + 1) | hashAggregate() | project(a + 1) | sortReverse(a) | skip(20) | take(10)"`}, + {"EXPLAIN UPDATE test SET a = 10", false, `"seqScan(test) | set(a, 10) | tableReplace('test')"`}, + {"EXPLAIN UPDATE test SET a = 10 WHERE c > 10", false, `"seqScan(test) | filter(c > 10) | set(a, 10) | tableReplace('test')"`}, + {"EXPLAIN UPDATE test SET a = 10 WHERE a > 10", false, `"indexScan(\"idx_a\", [10, -1, true]) | set(a, 10) | tableReplace('test')"`}, + {"EXPLAIN DELETE FROM test", false, `"seqScan(test) | tableDelete('test')"`}, + {"EXPLAIN DELETE FROM test WHERE c > 10", false, `"seqScan(test) | filter(c > 10) | tableDelete('test')"`}, + {"EXPLAIN DELETE FROM test WHERE a > 10", false, `"indexScan(\"idx_a\", [10, -1, true]) | tableDelete('test')"`}, } for _, test := range tests { @@ -48,6 +49,7 @@ func TestExplainStmt(t *testing.T) { err = db.Exec(` CREATE INDEX idx_a ON test (a); CREATE UNIQUE INDEX idx_b ON test (b); + CREATE INDEX idx_x_y ON test (x, y); `) require.NoError(t, err) diff --git a/stream/range.go b/stream/range.go index a62aad040..ed4e3747a 100644 --- a/stream/range.go +++ b/stream/range.go @@ -2,7 +2,6 @@ package stream import ( "bytes" - "fmt" "strings" "github.com/genjidb/genji/document" @@ -328,15 +327,31 @@ func (r *IndexRange) encode(encoder ValueBufferEncoder, env *expr.Environment) e } func (r *IndexRange) String() string { + format := func(vb *document.ValueBuffer) string { + switch vb.Len() { + case 0: + return "-1" + case 1: + return vb.Values[0].String() + default: + b, err := vb.MarshalJSON() + if err != nil { + return "err" + } + + return string(b) + } + } + if r.Exact { - return stringutil.Sprintf("%v", document.Array(r.Min)) + return stringutil.Sprintf("%v", format(r.Min)) } if r.Exclusive { - return stringutil.Sprintf("[%v, %v, true]", document.Array(r.Min), document.Array(r.Max)) + return stringutil.Sprintf("[%v, %v, true]", format(r.Min), format(r.Max)) } - return stringutil.Sprintf("[%v, %v]", document.Array(r.Min), document.Array(r.Max)) + return stringutil.Sprintf("[%v, %v]", format(r.Min), format(r.Max)) } func (r *IndexRange) IsEqual(other *IndexRange) bool { @@ -460,10 +475,6 @@ func (r *IndexRange) IsInRange(value []byte) bool { // by default, we consider the value within range cmpMin, cmpMax := 1, -1 - fmt.Println("encodedMin", r.encodedMin) - fmt.Println("encodedVal", value) - fmt.Println("encodedMax", r.encodedMax) - // we compare with the lower bound and see if it matches if r.encodedMin != nil { cmpMin = bytes.Compare(value, r.encodedMin) From c80b996f05b32e83b6a5887e1a5e3e72e1e81cc3 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Fri, 23 Apr 2021 18:06:49 +0200 Subject: [PATCH 20/40] Remove debug statements --- database/index.go | 10 ---------- stream/iterator.go | 6 ------ 2 files changed, 16 deletions(-) diff --git a/database/index.go b/database/index.go index 9917b3b7b..6aece20bd 100644 --- a/database/index.go +++ b/database/index.go @@ -319,7 +319,6 @@ func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func( var buf []byte return idx.iterate(st, pivots, reverse, func(item engine.Item) error { - fmt.Println("idx.iterate") var err error k := item.Key() @@ -454,8 +453,6 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool var err error seek, err := idx.buildSeek(pivots, reverse) - fmt.Println("pivots", pivots) - fmt.Println("seek", seek) if err != nil { return err } @@ -463,13 +460,6 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool it := st.Iterator(engine.IteratorOptions{Reverse: reverse}) defer it.Close() - for it.Seek([]byte{}); it.Valid(); it.Next() { - fmt.Println("----------", it.Item().Key()) - } - - it = st.Iterator(engine.IteratorOptions{Reverse: reverse}) - defer it.Close() - for it.Seek(seek); it.Valid(); it.Next() { itm := it.Item() diff --git a/stream/iterator.go b/stream/iterator.go index a6e556944..91e23bf53 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -2,7 +2,6 @@ package stream import ( "bytes" - "fmt" "strconv" "strings" @@ -382,11 +381,6 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env pivots = start.Values } - for i, p := range pivots { - fmt.Println("pivot[", i, "], type:", p.Type, "v:", p) - } - fmt.Println("encEnd", encEnd) - err = iterator(pivots, func(val, key []byte) error { if !rng.IsInRange(val) { // if we reached the end of our range, we can stop iterating. From 325d9136b1e5986562864b3efb5473f8df920054 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Fri, 23 Apr 2021 18:07:15 +0200 Subject: [PATCH 21/40] Fix tests with the new pivot logic --- database/index_test.go | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/database/index_test.go b/database/index_test.go index 4629f99f7..1bcfec57e 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -579,8 +579,13 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - expectedEq: noCallEq, - mustPanic: true, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 5, }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, @@ -866,6 +871,11 @@ func TestIndexAscendGreaterThan(t *testing.T) { }) } if test.mustPanic { + // let's avoid panicking because expectedEq wasn't defined, which would + // be a false positive. + if test.expectedEq == nil { + test.expectedEq = func(t *testing.T, i uint8, key, val []byte) {} + } require.Panics(t, func() { _ = fn() }) } else { err := fn() @@ -1128,18 +1138,24 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 5, }, - // composite indexes must have at least have one value - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), + pivots: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, - mustPanic: true, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + req(t, + encValue{false, document.NewIntegerValue(int64(i))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 5, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", + // composite indexes cannot have values with type past the first element + {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}), + pivots: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1467,6 +1483,11 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }) } if test.mustPanic { + // let's avoid panicking because expectedEq wasn't defined, which would + // be a false positive. + if test.expectedEq == nil { + test.expectedEq = func(t *testing.T, i uint8, key, val []byte) {} + } require.Panics(t, func() { _ = fn() }) From bf31e7ea0c3bf714ab92c93794eaa2c2e56d1353 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Sat, 24 Apr 2021 17:44:25 +0200 Subject: [PATCH 22/40] Export and use ArrayValueDelim --- database/index.go | 2 +- database/index_test.go | 3 +-- document/value_encoding.go | 18 +++++++++++------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/database/index.go b/database/index.go index 6aece20bd..66a29a459 100644 --- a/database/index.go +++ b/database/index.go @@ -376,7 +376,7 @@ func (idx *Index) EncodeValueBuffer(vb *document.ValueBuffer) ([]byte, error) { // if it's not the last value, append the seperator if i < vb.Len()-1 { - err = buf.WriteByte(0x1f) // TODO + err = buf.WriteByte(document.ArrayValueDelim) if err != nil { return err } diff --git a/database/index_test.go b/database/index_test.go index 1bcfec57e..8673c7f34 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -304,9 +304,8 @@ func req(t *testing.T, evs ...encValue) func([]byte) { _, err = buf.Write(b) require.NoError(t, err) - // TODO if i < len(evs)-1 { - err = buf.WriteByte(0x1f) + err = buf.WriteByte(document.ArrayValueDelim) } require.NoError(t, err) } diff --git a/document/value_encoding.go b/document/value_encoding.go index c8b9d47cf..a5bad3b2a 100644 --- a/document/value_encoding.go +++ b/document/value_encoding.go @@ -8,8 +8,12 @@ import ( ) const ( - arrayValueDelim = 0x1f - arrayEnd = 0x1e + // ArrayValueDelim is a separator used when encoding document.Array in + // binary reprsentation + ArrayValueDelim = 0x1f + // ArrayEnd is the final separator used when encoding document.Array in + // binary reprsentation. + ArrayEnd = 0x1e documentValueDelim = 0x1c documentEnd = 0x1d ) @@ -83,7 +87,7 @@ func (ve *ValueEncoder) appendValue(v Value) error { func (ve *ValueEncoder) appendArray(a Array) error { err := a.Iterate(func(i int, value Value) error { if i > 0 { - err := ve.append(arrayValueDelim) + err := ve.append(ArrayValueDelim) if err != nil { return err } @@ -95,7 +99,7 @@ func (ve *ValueEncoder) appendArray(a Array) error { return err } - return ve.append(arrayEnd) + return ve.append(ArrayEnd) } // appendDocument encodes a document into a sort-ordered binary representation. @@ -239,8 +243,8 @@ func decodeArray(data []byte) (Array, int, error) { var vb ValueBuffer var readCount int - for len(data) > 0 && data[0] != arrayEnd { - v, i, err := decodeValueUntil(data, arrayValueDelim, arrayEnd) + for len(data) > 0 && data[0] != ArrayEnd { + v, i, err := decodeValueUntil(data, ArrayValueDelim, ArrayEnd) if err != nil { return nil, i, err } @@ -248,7 +252,7 @@ func decodeArray(data []byte) (Array, int, error) { vb.Append(v) // skip the delimiter - if data[i] == arrayValueDelim { + if data[i] == ArrayValueDelim { i++ } From fc3e39c4b6363dc28c09f5c2ecd96ee71068d228 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Sat, 24 Apr 2021 17:46:02 +0200 Subject: [PATCH 23/40] Document indexes and add a custom error for arity --- database/index.go | 22 +++++---- database/index_test.go | 100 ++++++++++++++++++++--------------------- document/array.go | 5 --- stream/range.go | 17 ++++--- 4 files changed, 73 insertions(+), 71 deletions(-) diff --git a/database/index.go b/database/index.go index 66a29a459..1997346dd 100644 --- a/database/index.go +++ b/database/index.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/binary" "errors" - "fmt" "github.com/genjidb/genji/document" "github.com/genjidb/genji/engine" @@ -19,6 +18,10 @@ const ( var ( // ErrIndexDuplicateValue is returned when a value is already associated with a key ErrIndexDuplicateValue = errors.New("duplicate value") + + // ErrIndexWrongArity is returned when trying to index more values that what an + // index supports. + ErrIndexWrongArity = errors.New("wrong index arity") ) // An Index associates encoded values with keys. @@ -349,15 +352,18 @@ func (idx *Index) Truncate() error { return nil } -// EncodeValue encodes the value buffer we are going to use as a key, -// TODO -// If the index is typed, encode the value without expecting -// the presence of other types. -// If not, encode so that order is preserved regardless of the type. +// EncodeValueBuffer encodes the value buffer containing a single or +// multiple values being indexed into a byte array, keeping the +// order of the original values. +// +// The values are marshalled and separated with a document.ArrayValueDelim, +// *without* a trailing document.ArrayEnd, which enables to handle cases +// where only some of the values are being provided and still perform lookups. +// +// See IndexValueEncoder for details about how the value themselves are encoded. func (idx *Index) EncodeValueBuffer(vb *document.ValueBuffer) ([]byte, error) { if vb.Len() > idx.Arity() { - // TODO - return nil, fmt.Errorf("todo") + return nil, ErrIndexWrongArity } var buf bytes.Buffer diff --git a/database/index_test.go b/database/index_test.go index 8673c7f34..8dfd79f31 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -288,7 +288,7 @@ type encValue struct { document.Value } -func req(t *testing.T, evs ...encValue) func([]byte) { +func requireIdxEncodedEq(t *testing.T, evs ...encValue) func([]byte) { t.Helper() var buf bytes.Buffer @@ -374,7 +374,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, )(val) }, @@ -386,7 +386,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, )(val) }, @@ -400,7 +400,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, )(val) }, @@ -421,13 +421,12 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, )(val) }, expectedCount: 3, }, - // TODO but not when the index is typed to integers, although it won't yield an error {name: "index=integer, vals=integers, pivot=double", indexTypes: []document.ValueType{document.IntegerValue}, pivots: values(document.Value{Type: document.DoubleValue}), @@ -443,7 +442,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, )(val) }, @@ -456,7 +455,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, )(val) }, @@ -469,7 +468,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewDoubleValue(float64(i) + float64(i)/2)}, )(val) }, @@ -491,7 +490,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -505,7 +504,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -518,7 +517,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -539,7 +538,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -554,7 +553,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -579,7 +578,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -623,7 +622,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -641,7 +640,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -659,7 +658,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -683,7 +682,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -702,7 +701,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, )(val) @@ -753,7 +752,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { if i%2 == 0 { i = i / 2 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -769,7 +768,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -784,7 +783,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -800,7 +799,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, )(val) @@ -815,7 +814,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -830,7 +829,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -956,7 +955,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, )(val) }, @@ -970,7 +969,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, )(val) }, @@ -991,13 +990,12 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, )(val) }, expectedCount: 3, }, - // TODO but not when the index is typed to integers, although it won't yield an error {name: "index=integer, vals=integers, pivot=double", indexTypes: []document.ValueType{document.IntegerValue}, pivots: values(document.Value{Type: document.DoubleValue}), @@ -1013,7 +1011,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, )(val) }, @@ -1026,7 +1024,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewDoubleValue(float64(i) + float64(i)/2)}, )(val) }, @@ -1039,7 +1037,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewDoubleValue(float64(i) + float64(i)/2)}, )(val) }, @@ -1061,7 +1059,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) @@ -1076,7 +1074,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -1089,7 +1087,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -1102,7 +1100,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -1115,7 +1113,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 require.Equal(t, []byte{'a' + i}, key) - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewTextValue(strconv.Itoa(int(i)))}, )(val) }, @@ -1130,7 +1128,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1144,7 +1142,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1198,7 +1196,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { return values(document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10)), document.NewBlobValue(strconv.AppendInt(nil, int64(i), 10))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1217,7 +1215,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1236,7 +1234,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1260,7 +1258,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1285,7 +1283,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{false, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, )(val) @@ -1310,7 +1308,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - req(t, + requireIdxEncodedEq(t, encValue{false, document.NewIntegerValue(int64(i))}, encValue{true, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, )(val) @@ -1377,7 +1375,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1392,7 +1390,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1408,7 +1406,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewBlobValue([]byte{byte('a' + uint8(i))})}, )(val) @@ -1424,7 +1422,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 4 - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) @@ -1440,7 +1438,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 - req(t, + requireIdxEncodedEq(t, encValue{true, document.NewIntegerValue(int64(i))}, encValue{true, document.NewIntegerValue(int64(i + 1))}, )(val) diff --git a/document/array.go b/document/array.go index 0ea42015b..28e4c9a51 100644 --- a/document/array.go +++ b/document/array.go @@ -251,11 +251,6 @@ func (vb *ValueBuffer) UnmarshalJSON(data []byte) error { } func (vb *ValueBuffer) Types() []ValueType { - // TODO check that - if vb == nil { - return nil - } - types := make([]ValueType, len(vb.Values)) for i, v := range vb.Values { diff --git a/stream/range.go b/stream/range.go index ed4e3747a..0ce09c43e 100644 --- a/stream/range.go +++ b/stream/range.go @@ -233,6 +233,9 @@ func (r *ValueRange) IsInRange(value []byte) bool { return cmpMax <= 0 } +// IndexRange represents a range to select indexed values after or before +// a given boundary. Because indexes can be composites, IndexRange boundaries +// are composite as well. type IndexRange struct { Min, Max *document.ValueBuffer // Exclude Min and Max from the results. @@ -369,18 +372,18 @@ func (r *IndexRange) IsEqual(other *IndexRange) bool { return false } - // TODO(JH) may or may not this - // if r.Min.Type != other.Min.Type { - // return false - // } + if r.Min.Len() != other.Min.Len() { + return false + } + + if r.Max.Len() != other.Max.Len() { + return false + } if !r.Min.IsEqual(other.Min) { return false } - // if r.Max.Type != other.Max.Type { - // return false - // } if !r.Max.IsEqual(other.Max) { return false } From 2859eda622f17632dd781c859722ed39a903dbf8 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Sat, 24 Apr 2021 18:01:47 +0200 Subject: [PATCH 24/40] Remove redundant IndexArityMax field from Range --- stream/iterator_test.go | 30 +++++++++++++----------------- stream/range.go | 24 ++++++------------------ 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/stream/iterator_test.go b/stream/iterator_test.go index b937db39e..80f4c8adf 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -479,9 +479,8 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 1}`, `{"a": 1, "b": 9223372036854775807}`), stream.IndexRanges{ { - Arity: 2, - IndexArityMax: 1, - Max: newVB(document.NewIntegerValue(1)), + IndexArity: 2, + Max: newVB(document.NewIntegerValue(1)), }, }, false, false, @@ -492,11 +491,10 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 9223372036854775807}`, `{"a": 1, "b": 1}`), stream.IndexRanges{ { - Max: newVB(document.NewIntegerValue(1)), - Exclusive: false, - Exact: false, - Arity: 2, - IndexArityMax: 1, + Max: newVB(document.NewIntegerValue(1)), + Exclusive: false, + Exact: false, + IndexArity: 2, }, }, true, false, @@ -506,7 +504,7 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 2, "b": 2, "c": 2}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), testutil.MakeDocuments(t, `{"a": 1, "b": 2, "c": 1}`, `{"a": 1, "b": 2, "c": 9223372036854775807}`), stream.IndexRanges{ - {Arity: 3, IndexArityMax: 2, Max: newVB(document.NewIntegerValue(1), document.NewIntegerValue(2))}, + {IndexArity: 3, Max: newVB(document.NewIntegerValue(1), document.NewIntegerValue(2))}, }, false, false, }, @@ -543,10 +541,9 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 1, "b": -2}`, `{"a": 2, "b": 42}`), stream.IndexRanges{ { - Arity: 2, - IndexArityMax: 1, - Min: newVB(document.NewIntegerValue(1)), - Max: newVB(document.NewIntegerValue(2)), + IndexArity: 2, + Min: newVB(document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2)), }, }, false, false, @@ -557,10 +554,9 @@ func TestIndexScan(t *testing.T) { testutil.MakeDocuments(t, `{"a": 2, "b": 42}`, `{"a": 1, "b": -2}`), stream.IndexRanges{ { - Arity: 2, - IndexArityMax: 1, - Min: newVB(document.NewIntegerValue(1)), - Max: newVB(document.NewIntegerValue(2)), + IndexArity: 2, + Min: newVB(document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2)), }, }, true, false, diff --git a/stream/range.go b/stream/range.go index 0ce09c43e..c9d723abc 100644 --- a/stream/range.go +++ b/stream/range.go @@ -238,6 +238,7 @@ func (r *ValueRange) IsInRange(value []byte) bool { // are composite as well. type IndexRange struct { Min, Max *document.ValueBuffer + // Exclude Min and Max from the results. // By default, min and max are inclusive. // Exclusive and Exact cannot be set to true at the same time. @@ -247,23 +248,10 @@ type IndexRange struct { // and for determining the global upper bound. Exact bool - // Arity represents the range arity in the case of comparing the range - // to a composite index. With IndexArityMax, it enables to deal with the - // cases of a composite range specifying boundaries partially, ie: - // - Index on (a, b, c) - // - Range is defining a max only for a and b - // Then Arity is set to 2 and IndexArityMax is set to 3 - // - // On - // This field is subject to change when the support for composite index is added - // to the query planner in an ulterior pull-request. - Arity int - - // IndexArityMax represents the underlying Index arity. - // - // This field is subject to change when the support for composite index is added - // to the query planner in an ulterior pull-request. - IndexArityMax int + // IndexArity is the underlying index arity, which can be greater + // than the boundaries of this range. + IndexArity int + encodedMin, encodedMax []byte rangeTypes []document.ValueType } @@ -497,7 +485,7 @@ func (r *IndexRange) IsInRange(value []byte) bool { // the value is bigger than the lower bound, // see if it matches the upper bound. if r.encodedMax != nil { - if r.IndexArityMax < r.Arity { + if r.Max.Len() < r.IndexArity { cmpMax = bytes.Compare(value[:len(r.encodedMax)], r.encodedMax) } else { cmpMax = bytes.Compare(value, r.encodedMax) From 0548be8d6ea95de094f37004fc18d9ae4b4e52a1 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Sat, 24 Apr 2021 18:03:16 +0200 Subject: [PATCH 25/40] Re-enable string tests --- stream/iterator_test.go | 58 ++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/stream/iterator_test.go b/stream/iterator_test.go index 80f4c8adf..44a8a3335 100644 --- a/stream/iterator_test.go +++ b/stream/iterator_test.go @@ -614,33 +614,33 @@ func TestIndexScan(t *testing.T) { }) } - // t.Run("String", func(t *testing.T) { - // t.Run("idx_test_a", func(t *testing.T) { - // require.Equal(t, `indexScan("idx_test_a", [1, 2])`, stream.IndexScan("idx_test_a", stream.IndexRange{ - // Min: newVB(document.NewIntegerValue(1)), Max: newVB(document.NewIntegerValue(2)), - // }).String()) - - // op := stream.IndexScan("idx_test_a", stream.IndexRange{ - // Min: newVB(document.NewIntegerValue(1)), Max: newVB(document.NewIntegerValue(2)), - // }) - // op.Reverse = true - - // require.Equal(t, `indexScanReverse("idx_test_a", [1, 2])`, op.String()) - // }) - - // t.Run("idx_test_a_b", func(t *testing.T) { - // require.Equal(t, `indexScan("idx_test_a_b", [[1, 1], [2, 2]])`, stream.IndexScan("idx_test_a_b", stream.IndexRange{ - // Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), - // Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), - // }).String()) - - // op := stream.IndexScan("idx_test_a_b", stream.IndexRange{ - // Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), - // Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), - // }) - // op.Reverse = true - - // require.Equal(t, `indexScanReverse("idx_test_a_b", [[1, 1], [2, 2]])`, op.String()) - // }) - // }) + t.Run("String", func(t *testing.T) { + t.Run("idx_test_a", func(t *testing.T) { + require.Equal(t, `indexScan("idx_test_a", [1, 2])`, stream.IndexScan("idx_test_a", stream.IndexRange{ + Min: newVB(document.NewIntegerValue(1)), Max: newVB(document.NewIntegerValue(2)), + }).String()) + + op := stream.IndexScan("idx_test_a", stream.IndexRange{ + Min: newVB(document.NewIntegerValue(1)), Max: newVB(document.NewIntegerValue(2)), + }) + op.Reverse = true + + require.Equal(t, `indexScanReverse("idx_test_a", [1, 2])`, op.String()) + }) + + t.Run("idx_test_a_b", func(t *testing.T) { + require.Equal(t, `indexScan("idx_test_a_b", [[1, 1], [2, 2]])`, stream.IndexScan("idx_test_a_b", stream.IndexRange{ + Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + }).String()) + + op := stream.IndexScan("idx_test_a_b", stream.IndexRange{ + Min: newVB(document.NewIntegerValue(1), document.NewIntegerValue(1)), + Max: newVB(document.NewIntegerValue(2), document.NewIntegerValue(2)), + }) + op.Reverse = true + + require.Equal(t, `indexScanReverse("idx_test_a_b", [[1, 1], [2, 2]])`, op.String()) + }) + }) } From 6aac2b22fc0a4f0aed032a3b8092d7d55bc94fc5 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Sat, 24 Apr 2021 18:11:08 +0200 Subject: [PATCH 26/40] Document indexValueEncoder --- database/index.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/database/index.go b/database/index.go index 1997346dd..49b362123 100644 --- a/database/index.go +++ b/database/index.go @@ -39,6 +39,9 @@ type Index struct { storeName []byte } +// indexValueEncoder encodes a field based on its type; if a type is provided, +// the value is encoded as is, without any type information. Otherwise, the +// type is prepended to the value. type indexValueEncoder struct { typ document.ValueType } From eb713cf180f7ecb565c9d3a8f97da650c42d9010 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Sat, 24 Apr 2021 18:30:30 +0200 Subject: [PATCH 27/40] Create a custom Pivot type, update var names --- database/index.go | 97 +++++++++++-------------- database/index_test.go | 156 ++++++++++++++++++++--------------------- stream/iterator.go | 9 +-- 3 files changed, 124 insertions(+), 138 deletions(-) diff --git a/database/index.go b/database/index.go index 49b362123..a3b7a80e0 100644 --- a/database/index.go +++ b/database/index.go @@ -39,6 +39,8 @@ type Index struct { storeName []byte } +type Pivot []document.Value + // indexValueEncoder encodes a field based on its type; if a type is provided, // the value is encoded as is, without any type information. Otherwise, the // type is prepended to the value. @@ -234,48 +236,33 @@ func (idx *Index) Delete(vs []document.Value, k []byte) error { return engine.ErrKeyNotFound } -// validatePivots returns an error when the pivots are unsuitable for the index: -// - no pivots at all -// - having pivots length superior to the index arity -// - having the first pivot without a value when the subsequent ones do have values -func (idx *Index) validatePivots(pivots []document.Value) { - if len(pivots) > idx.Arity() { +// validate panics when the pivot values are unsuitable for the index: +// - no pivot values at all +// - having pivot length superior to the index arity +// - having the first pivot value without a value when the subsequent ones do have values +func (pivot Pivot) validate(idx *Index) { + if len(pivot) > idx.Arity() { panic("cannot iterate with a pivot whose size is superior to the index arity") } - if idx.IsComposite() { - if !allEmpty(pivots) { - // fmt.Println("here", pivots) - // the first pivot must have a value - // if pivots[0].V == nil { - // panic("cannot iterate on a composite index whose first pivot has no value") - // } - - // it's acceptable for the last pivot to just have a type and no value - hasValue := true - for _, p := range pivots { - // if on the previous pivot we have a value - if hasValue { - hasValue = p.V != nil - - // // if we have no value, we at least need a type - // if !hasValue { - // if p.Type == 0 { - // panic("cannot iterate on a composite index with a pivot with both values and nil values") - // } - // } - } else { - panic("cannot iterate on a composite index with a pivot with both values and nil values") - } + if idx.IsComposite() && !pivot.Empty() { + // it's acceptable for the last pivot to just have a type and no value + hasValue := true + for _, p := range pivot { + // if on the previous pivot we have a value + if hasValue { + hasValue = p.V != nil + } else { + panic("cannot iterate on a composite index with a pivot with both values and nil values") } } } } -// allEmpty returns true when all pivots are valueless and untyped. -func allEmpty(pivots []document.Value) bool { +// TODO rename? +func (pivot Pivot) Empty() bool { res := true - for _, p := range pivots { + for _, p := range pivot { res = res && p.Type == 0 if !res { break @@ -291,8 +278,8 @@ func allEmpty(pivots []document.Value) bool { // When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. // When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element // is of that type, without restriction on the type of the following elements. -func (idx *Index) AscendGreaterOrEqual(pivots []document.Value, fn func(val, key []byte) error) error { - return idx.iterateOnStore(pivots, false, fn) +func (idx *Index) AscendGreaterOrEqual(pivot Pivot, fn func(val, key []byte) error) error { + return idx.iterateOnStore(pivot, false, fn) } // DescendLessOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in descreasing order and calls the given function for each pair. @@ -301,16 +288,16 @@ func (idx *Index) AscendGreaterOrEqual(pivots []document.Value, fn func(val, key // When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. // When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element // is of that type, without restriction on the type of the following elements. -func (idx *Index) DescendLessOrEqual(pivots []document.Value, fn func(val, key []byte) error) error { - return idx.iterateOnStore(pivots, true, fn) +func (idx *Index) DescendLessOrEqual(pivot Pivot, fn func(val, key []byte) error) error { + return idx.iterateOnStore(pivot, true, fn) } -func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func(val, key []byte) error) error { - idx.validatePivots(pivots) +func (idx *Index) iterateOnStore(pivot Pivot, reverse bool, fn func(val, key []byte) error) error { + pivot.validate(idx) - // If index and pivot are typed but not of the same type, return no results. - for i, p := range pivots { - if p.Type != 0 && idx.Info.Types[i] != 0 && p.Type != idx.Info.Types[i] { + // If index and pivot values are typed but not of the same type, return no results. + for i, pv := range pivot { + if pv.Type != 0 && idx.Info.Types[i] != 0 && pv.Type != idx.Info.Types[i] { return nil } } @@ -324,7 +311,7 @@ func (idx *Index) iterateOnStore(pivots []document.Value, reverse bool, fn func( } var buf []byte - return idx.iterate(st, pivots, reverse, func(item engine.Item) error { + return idx.iterate(st, pivot, reverse, func(item engine.Item) error { var err error k := item.Key() @@ -419,23 +406,21 @@ func getOrCreateStore(tx engine.Transaction, name []byte) (engine.Store, error) return tx.GetStore(name) } -// buildSeek encodes the pivots as binary in order to seek into the indexed data. -// In case of a composite index, the pivots are wrapped in array before being encoded. -// See the Index type documentation for a description of its encoding and its corner cases. -func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, error) { +// buildSeek encodes the pivot values as binary in order to seek into the indexed data. +func (idx *Index) buildSeek(pivot Pivot, reverse bool) ([]byte, error) { var seek []byte var err error // TODO rework - // if we have valueless and typeless pivots, we just iterate - if allEmpty(pivots) { + // if we have valueless and typeless pivot, we just iterate + if pivot.Empty() { return []byte{}, nil } // if the index is without type and the first pivot is valueless but typed, iterate but filter out the types we don't want, - // but just for the first pivot; subsequent pivots cannot be filtered this way. - if idx.Info.Types[0] == 0 && pivots[0].Type != 0 && pivots[0].V == nil { - seek = []byte{byte(pivots[0].Type)} + // but just for the first pivot; subsequent pivot values cannot be filtered this way. + if idx.Info.Types[0] == 0 && pivot[0].Type != 0 && pivot[0].V == nil { + seek = []byte{byte(pivot[0].Type)} if reverse { seek = append(seek, 0xFF) @@ -444,7 +429,7 @@ func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, erro return seek, nil } - vb := document.NewValueBuffer(pivots...) + vb := document.NewValueBuffer(pivot...) seek, err = idx.EncodeValueBuffer(vb) if err != nil { @@ -458,10 +443,10 @@ func (idx *Index) buildSeek(pivots []document.Value, reverse bool) ([]byte, erro return seek, nil } -func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool, fn func(item engine.Item) error) error { +func (idx *Index) iterate(st engine.Store, pivot Pivot, reverse bool, fn func(item engine.Item) error) error { var err error - seek, err := idx.buildSeek(pivots, reverse) + seek, err := idx.buildSeek(pivot, reverse) if err != nil { return err } @@ -474,7 +459,7 @@ func (idx *Index) iterate(st engine.Store, pivots []document.Value, reverse bool // If index is untyped and pivot first element is typed, only iterate on values with the same type as the first pivot // TODO(JH) possible optimization, check for the other types - if len(pivots) > 0 && idx.Info.Types[0] == 0 && pivots[0].Type != 0 && itm.Key()[0] != byte(pivots[0].Type) { + if len(pivot) > 0 && idx.Info.Types[0] == 0 && pivot[0].Type != 0 && itm.Key()[0] != byte(pivot[0].Type) { return nil } diff --git a/database/index_test.go b/database/index_test.go index 8dfd79f31..d391915cb 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -354,8 +354,8 @@ func TestIndexAscendGreaterThan(t *testing.T) { name string // the index type(s) that is being used indexTypes []document.ValueType - // the pivots, typed or not used to iterate - pivots []document.Value + // the pivot, typed or not used to iterate + pivot database.Pivot // the generator for the values that are being indexed val func(i int) []document.Value // the generator for the noise values that are being indexed @@ -369,7 +369,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // integers --------------------------------------------------- {name: "index=untyped, vals=integers, pivot=integer", indexTypes: nil, - pivots: values(document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -382,7 +382,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=integer, vals=integers, pivot=integer", indexTypes: []document.ValueType{document.IntegerValue}, - pivots: values(document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) @@ -394,7 +394,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=untyped, vals=integers, pivot=integer:2", indexTypes: nil, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -408,7 +408,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=untyped, vals=integers, pivot=integer:10", indexTypes: nil, - pivots: values(document.NewIntegerValue(10)), + pivot: values(document.NewIntegerValue(10)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, expectedEq: noCallEq, @@ -416,7 +416,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=integer, vals=integers, pivot=integer:2", indexTypes: []document.ValueType{document.IntegerValue}, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 @@ -429,7 +429,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=integer, vals=integers, pivot=double", indexTypes: []document.ValueType{document.IntegerValue}, - pivots: values(document.Value{Type: document.DoubleValue}), + pivot: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: noCallEq, expectedCount: 0, @@ -438,7 +438,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // doubles ---------------------------------------------------- {name: "index=untyped, vals=doubles, pivot=double", indexTypes: nil, - pivots: values(document.Value{Type: document.DoubleValue}), + pivot: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) @@ -450,7 +450,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=untyped, vals=doubles, pivot=double:1.8", indexTypes: nil, - pivots: values(document.NewDoubleValue(1.8)), + pivot: values(document.NewDoubleValue(1.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 @@ -463,7 +463,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=double, vals=doubles, pivot=double:1.8", indexTypes: []document.ValueType{document.DoubleValue}, - pivots: values(document.NewDoubleValue(1.8)), + pivot: values(document.NewDoubleValue(1.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 @@ -476,7 +476,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=untyped, vals=doubles, pivot=double:10.8", indexTypes: nil, - pivots: values(document.NewDoubleValue(10.8)), + pivot: values(document.NewDoubleValue(10.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: noCallEq, expectedCount: 0, @@ -485,7 +485,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // text ------------------------------------------------------- {name: "index=untyped, vals=text pivot=text", indexTypes: nil, - pivots: values(document.Value{Type: document.TextValue}), + pivot: values(document.Value{Type: document.TextValue}), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -498,7 +498,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=untyped, vals=text, pivot=text('2')", indexTypes: nil, - pivots: values(document.NewTextValue("2")), + pivot: values(document.NewTextValue("2")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -512,7 +512,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=untyped, vals=text, pivot=text('')", indexTypes: nil, - pivots: values(document.NewTextValue("")), + pivot: values(document.NewTextValue("")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -525,7 +525,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=untyped, vals=text, pivot=text('foo')", indexTypes: nil, - pivots: values(document.NewTextValue("foo")), + pivot: values(document.NewTextValue("foo")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: noCallEq, @@ -533,7 +533,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=text, vals=text, pivot=text('2')", indexTypes: []document.ValueType{document.TextValue}, - pivots: values(document.NewTextValue("2")), + pivot: values(document.NewTextValue("2")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i += 2 @@ -545,10 +545,10 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 3, }, // composite -------------------------------------------------- - // composite indexes can have empty pivots to iterate on the whole indexed data + // composite indexes can have empty pivot values to iterate on the whole indexed data {name: "index=[untyped, untyped], vals=[int, int], pivot=[nil,nil]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{}, document.Value{}), + pivot: values(document.Value{}, document.Value{}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -564,7 +564,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // composite indexes must have at least have one value if typed {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -573,7 +573,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -587,7 +587,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, - pivots: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) }, @@ -596,7 +596,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, nil]", indexTypes: []document.ValueType{0, 0, 0}, - pivots: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0), document.Value{}), + pivot: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0), document.Value{}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) }, @@ -605,7 +605,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + pivot: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -614,7 +614,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -631,7 +631,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -649,7 +649,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -668,7 +668,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // pivot [2, int] should filter out [2, not(int)] {name: "index=[untyped, untyped], vals=[int, int], noise=[int, blob], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -692,7 +692,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // a more subtle case {name: "index=[untyped, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) }, @@ -711,7 +711,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // partial pivot {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -725,7 +725,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -741,7 +741,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // this is by definition a very implementation dependent test. {name: "index=[untyped, untyped], vals=[int, int], noise=int, bool], pivot=[int:0, int:0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -763,7 +763,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // index typed {name: "index=[int, int], vals=[int, int], pivot=[0, 0]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -777,7 +777,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[int, int], vals=[int, int], pivot=[2, 0]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -793,7 +793,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // a more subtle case {name: "index=[int, blob], vals=[int, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{document.IntegerValue, document.BlobValue}, - pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) }, @@ -809,7 +809,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // partial pivot {name: "index=[int, int], vals=[int, int], pivot=[0]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -823,7 +823,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, {name: "index=[int, int], vals=[int, int], pivot=[2]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -861,7 +861,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { var i uint8 var count int fn := func() error { - return idx.AscendGreaterOrEqual(test.pivots, func(val, rid []byte) error { + return idx.AscendGreaterOrEqual(test.pivot, func(val, rid []byte) error { test.expectedEq(t, i, rid, val) i++ count++ @@ -925,8 +925,8 @@ func TestIndexDescendLessOrEqual(t *testing.T) { name string // the index type(s) that is being used indexTypes []document.ValueType - // the pivots, typed or not used to iterate - pivots []document.Value + // the pivot, typed or not used to iterate + pivot database.Pivot // the generator for the values that are being indexed val func(i int) []document.Value // the generator for the noise values that are being indexed @@ -940,7 +940,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // integers --------------------------------------------------- {name: "index=untyped, vals=integers, pivot=integer", indexTypes: nil, - pivots: values(document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -951,7 +951,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=integer, vals=integers, pivot=integer", indexTypes: []document.ValueType{document.IntegerValue}, - pivots: values(document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) @@ -963,7 +963,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=untyped, vals=integers, pivot=integer:2", indexTypes: nil, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -977,7 +977,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=untyped, vals=integers, pivot=integer:-10", indexTypes: nil, - pivots: values(document.NewIntegerValue(-10)), + pivot: values(document.NewIntegerValue(-10)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, noise: noiseBlob, expectedEq: noCallEq, @@ -985,7 +985,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=integer, vals=integers, pivot=integer:2", indexTypes: []document.ValueType{document.IntegerValue}, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 @@ -998,7 +998,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=integer, vals=integers, pivot=double", indexTypes: []document.ValueType{document.IntegerValue}, - pivots: values(document.Value{Type: document.DoubleValue}), + pivot: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, expectedEq: noCallEq, expectedCount: 0, @@ -1007,7 +1007,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // doubles ---------------------------------------------------- {name: "index=untyped, vals=doubles, pivot=double", indexTypes: nil, - pivots: values(document.Value{Type: document.DoubleValue}), + pivot: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { require.Equal(t, []byte{'a' + i}, key) @@ -1019,7 +1019,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=untyped, vals=doubles, pivot=double:1.8", indexTypes: nil, - pivots: values(document.NewDoubleValue(1.8)), + pivot: values(document.NewDoubleValue(1.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 @@ -1032,7 +1032,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=double, vals=doubles, pivot=double:1.8", indexTypes: []document.ValueType{document.DoubleValue}, - pivots: values(document.NewDoubleValue(1.8)), + pivot: values(document.NewDoubleValue(1.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 3 @@ -1045,7 +1045,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=untyped, vals=doubles, pivot=double:-10.8", indexTypes: nil, - pivots: values(document.NewDoubleValue(-10.8)), + pivot: values(document.NewDoubleValue(-10.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, expectedEq: noCallEq, expectedCount: 0, @@ -1054,7 +1054,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // text ------------------------------------------------------- {name: "index=untyped, vals=text pivot=text", indexTypes: nil, - pivots: values(document.Value{Type: document.TextValue}), + pivot: values(document.Value{Type: document.TextValue}), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -1068,7 +1068,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=untyped, vals=text, pivot=text('2')", indexTypes: nil, - pivots: values(document.NewTextValue("2")), + pivot: values(document.NewTextValue("2")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -1082,7 +1082,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=untyped, vals=text, pivot=text('')", indexTypes: nil, - pivots: values(document.NewTextValue("")), + pivot: values(document.NewTextValue("")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -1095,7 +1095,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=untyped, vals=text, pivot=text('foo')", indexTypes: nil, - pivots: values(document.NewTextValue("foo")), + pivot: values(document.NewTextValue("foo")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, noise: noiseInts, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { @@ -1108,7 +1108,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=text, vals=text, pivot=text('2')", indexTypes: []document.ValueType{document.TextValue}, - pivots: values(document.NewTextValue("2")), + pivot: values(document.NewTextValue("2")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { i -= 2 @@ -1120,10 +1120,10 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 3, }, // composite -------------------------------------------------- - // composite indexes can have empty pivots to iterate on the whole indexed data + // composite indexes can have empty pivot values to iterate on the whole indexed data {name: "index=[untyped, untyped], vals=[int, int], pivot=[nil,nil]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{}, document.Value{}), + pivot: values(document.Value{}, document.Value{}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1137,7 +1137,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1152,7 +1152,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // composite indexes cannot have values with type past the first element {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), + pivot: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1160,7 +1160,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, - pivots: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1)), document.NewIntegerValue(int64(i+1))) }, @@ -1168,7 +1168,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), + pivot: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1176,7 +1176,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1188,7 +1188,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[5, 5]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(5), document.NewIntegerValue(5)), + pivot: values(document.NewIntegerValue(5), document.NewIntegerValue(5)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1206,7 +1206,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // [0,1], [1,2], --[2,0]--, [2,3], [3,4], [4,5] {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1225,7 +1225,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // [0,1], [1,2], [2,3], --[2,int]--, [3,4], [4,5] {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1245,7 +1245,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // [0,1], [1,2], [2,3], --[2,int]--, [2, text], [3,4], [3,text], [4,5], [4,text] {name: "index=[untyped, untyped], vals=[int, int], noise=[int, text], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), + pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1268,7 +1268,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // a more subtle case {name: "index=[untyped, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { return values( document.NewIntegerValue(int64(i)), @@ -1293,7 +1293,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // only one of the indexed value is typed {name: "index=[untyped, blob], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, document.BlobValue}, - pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { return values( document.NewIntegerValue(int64(i)), @@ -1318,7 +1318,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // partial pivot {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1332,7 +1332,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[5]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(5)), + pivot: values(document.NewIntegerValue(5)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1346,7 +1346,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", indexTypes: []document.ValueType{0, 0}, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1361,7 +1361,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // index typed {name: "index=[int, int], vals=[int, int], pivot=[0, 0]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1370,7 +1370,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[int, int], vals=[int, int], pivot=[5, 6]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(5), document.NewIntegerValue(6)), + pivot: values(document.NewIntegerValue(5), document.NewIntegerValue(6)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1384,7 +1384,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, {name: "index=[int, int], vals=[int, int], pivot=[2, 0]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1400,7 +1400,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // a more subtle case {name: "index=[int, blob], vals=[int, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{document.IntegerValue, document.BlobValue}, - pivots: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), + pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) }, @@ -1416,7 +1416,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // partial pivot {name: "index=[int, int], vals=[int, int], pivot=[0]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(0)), + pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1432,7 +1432,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // [0,1], [1,2], [2,3], --[2]--, [3,4], [4,5] {name: "index=[int, int], vals=[int, int], pivot=[2]", indexTypes: []document.ValueType{document.IntegerValue, document.IntegerValue}, - pivots: values(document.NewIntegerValue(2)), + pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) }, @@ -1472,7 +1472,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { fn := func() error { t.Helper() - return idx.DescendLessOrEqual(test.pivots, func(val, rid []byte) error { + return idx.DescendLessOrEqual(test.pivot, func(val, rid []byte) error { test.expectedEq(t, uint8(total-1)-i, rid, val) i++ count++ diff --git a/stream/iterator.go b/stream/iterator.go index 91e23bf53..0c0d22dd5 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -5,6 +5,7 @@ import ( "strconv" "strings" + "github.com/genjidb/genji/database" "github.com/genjidb/genji/document" "github.com/genjidb/genji/expr" "github.com/genjidb/genji/stringutil" @@ -336,7 +337,7 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env return err } - var iterator func(pivots []document.Value, fn func(val, key []byte) error) error + var iterator func(pivot database.Pivot, fn func(val, key []byte) error) error if !it.Reverse { iterator = index.AscendGreaterOrEqual @@ -376,12 +377,12 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env } } - var pivots []document.Value + var pivot database.Pivot if start != nil { - pivots = start.Values + pivot = start.Values } - err = iterator(pivots, func(val, key []byte) error { + err = iterator(pivot, func(val, key []byte) error { if !rng.IsInRange(val) { // if we reached the end of our range, we can stop iterating. if encEnd == nil { From 34d3ed23cd60b29481c24f77be767269fae77434 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 26 Apr 2021 16:00:27 +0200 Subject: [PATCH 28/40] Add support for IN operators in comp indexes --- document/array.go | 5 ++ planner/optimizer.go | 111 ++++++++++++++++++---------- planner/optimizer_test.go | 147 ++++++++++++++++++++++++++++++++++---- query/select_test.go | 6 +- 4 files changed, 214 insertions(+), 55 deletions(-) diff --git a/document/array.go b/document/array.go index 28e4c9a51..312c23a72 100644 --- a/document/array.go +++ b/document/array.go @@ -267,6 +267,11 @@ func (vb *ValueBuffer) IsEqual(other *ValueBuffer) bool { return false } + // empty buffers are always equal eh + if vb.Len() == 0 && other.Len() == 0 { + return true + } + otherTypes := other.Types() types := vb.Types() diff --git a/planner/optimizer.go b/planner/optimizer.go index 2749693f8..db684a1d6 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -498,7 +498,7 @@ func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, p isNodeEq := func(fno *filterNode) bool { op := fno.f.E.(expr.Operator) - return expr.IsEqualOperator(op) + return expr.IsEqualOperator(op) || expr.IsInOperator(op) } isNodeComp := func(fno *filterNode, includeInOp bool) bool { op := fno.f.E.(expr.Operator) @@ -510,7 +510,7 @@ func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, p } // iterate on all indexes for that table, checking for each of them if its paths are matching - // the filter nodes of the given query. + // the filter nodes of the given query. The resulting nodes are ordered like the index paths. outer: for _, idx := range indexes { // order filter nodes by how the index paths order them; if absent, nil in still inserted @@ -745,8 +745,12 @@ func operandCanUseIndex(indexType document.ValueType, path document.Path, fc dat } func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) { + var ranges stream.IndexRanges vb := document.NewValueBuffer() - for _, fno := range fnodes { + // store in Operands of a given position + inOperands := make(map[int]document.Array) + + for i, fno := range fnodes { op := fno.f.E.(expr.Operator) v := fno.v @@ -754,51 +758,84 @@ func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) case *expr.EqOperator, *expr.GtOperator, *expr.GteOperator, *expr.LtOperator, *expr.LteOperator: vb = vb.Append(v) case *expr.InOperator: - // a := v.V.(document.Array) - // err := a.Iterate(func(i int, value document.Value) error { - // ranges = ranges.Append(stream.ValueRange{ - // Min: value, - // Exact: true, - // }) - // return nil - // }) - // if err != nil { - // return nil, err - // } - // TODO(JH) - // an index like idx_foo_a_b on (a,b) and a query like - // WHERE a IN [1, 1] and b IN [2, 2] - // would lead to [1, 1] x [2, 2] = [[1,1], [1,2], [2,1], [2,2]] - // which could eventually be added later. - // panic("unsupported operator IN for composite indexes") + // mark where the IN operator values are supposed to go is in the buffer + // and what are the value needed to generate the ranges. + inOperands[i] = v.V.(document.Array) + // placeholder for when we'll explode the IN operands in multiple ranges + vb = vb.Append(document.Value{}) default: panic(stringutil.Sprintf("unknown operator %#v", op)) } } - var rng stream.IndexRange + if len(inOperands) > 1 { + // TODO(JH) Github issue + panic("unsupported operation: multiple IN operators on a composite index") + } + + // a small helper func to create a range based on an operator type + buildRange := func(op expr.Operator, vb *document.ValueBuffer) stream.IndexRange { + var rng stream.IndexRange + + switch op.(type) { + case *expr.EqOperator, *expr.InOperator: + rng.Exact = true + rng.Min = vb + case *expr.GtOperator: + rng.Exclusive = true + rng.Min = vb + case *expr.GteOperator: + rng.Min = vb + case *expr.LtOperator: + rng.Exclusive = true + rng.Max = vb + case *expr.LteOperator: + rng.Max = vb + } + + return rng + } + + // explode the IN operator values in multiple ranges + for pos, operands := range inOperands { + err := operands.Iterate(func(j int, value document.Value) error { + newVB := document.NewValueBuffer() + err := newVB.Copy(vb) + if err != nil { + return err + } + + // insert IN operand at the right position, replacing the placeholder value + newVB.Values[pos] = value + + // the last node is the only one that can be a comparison operator, so + // it's the one setting the range behaviour + last := fnodes[len(fnodes)-1] + op := last.f.E.(expr.Operator) + + rng := buildRange(op, newVB) + + ranges = ranges.Append(rng) + return nil + }) + + if err != nil { + return nil, err + } + } + + // Were there any IN operators requiring multiple ranges? + // If yes, we're done here. + if len(ranges) > 0 { + return ranges, nil + } // the last node is the only one that can be a comparison operator, so // it's the one setting the range behaviour last := fnodes[len(fnodes)-1] op := last.f.E.(expr.Operator) - - switch op.(type) { - case *expr.EqOperator: - rng.Exact = true - rng.Min = vb - case *expr.GtOperator: - rng.Exclusive = true - rng.Min = vb - case *expr.GteOperator: - rng.Min = vb - case *expr.LtOperator: - rng.Exclusive = true - rng.Max = vb - case *expr.LteOperator: - rng.Max = vb - } + rng := buildRange(op, vb) return stream.IndexRanges{rng}, nil } diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index bfebc0bd4..256ba4b4b 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -371,21 +371,21 @@ func TestUseIndexBasedOnSelectionNodeRule_Simple(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("d = 2"))). Pipe(st.Project(parser.MustParseExpr("a"))), }, - // { - // "FROM foo WHERE a IN [1, 2]", - // st.New(st.SeqScan("foo")).Pipe(st.Filter( - // expr.In( - // parser.MustParseExpr("a"), - // expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), - // ), - // )), - // st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true}, st.IndexRange{Min: newVB(document.NewIntegerValue(2)), Exact: true})), - // }, - // { - // "FROM foo WHERE 1 IN a", - // st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), - // st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), - // }, + { + "FROM foo WHERE a IN [1, 2]", + st.New(st.SeqScan("foo")).Pipe(st.Filter( + expr.In( + parser.MustParseExpr("a"), + expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), + ), + )), + st.New(st.IndexScan("idx_foo_a", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true}, st.IndexRange{Min: newVB(document.NewIntegerValue(2)), Exact: true})), + }, + { + "FROM foo WHERE 1 IN a", + st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), + st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("1 IN a"))), + }, { "FROM foo WHERE a >= 10", st.New(st.SeqScan("foo")).Pipe(st.Filter(parser.MustParseExpr("a >= 10"))), @@ -552,6 +552,20 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("d > 2"))), st.New(st.IndexScan("idx_foo_a_d", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exclusive: true})), }, + { + "FROM foo WHERE a = 1 AND d < 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("d < 2"))), + st.New(st.IndexScan("idx_foo_a_d", st.IndexRange{Max: testutil.MakeValueBuffer(t, `[1, 2]`), Exclusive: true})), + }, + { + "FROM foo WHERE a = 1 AND d <= 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("d <= 2"))), + st.New(st.IndexScan("idx_foo_a_d", st.IndexRange{Max: testutil.MakeValueBuffer(t, `[1, 2]`)})), + }, { "FROM foo WHERE a = 1 AND d >= 2", st.New(st.SeqScan("foo")). @@ -589,6 +603,13 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("b > 2"))), st.New(st.IndexScan("idx_foo_a_b_c", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exclusive: true})), }, + { + "FROM foo WHERE a = 1 AND b < 2", // c is omitted, but it can still use idx_foo_a_b_c, with > b + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = 1"))). + Pipe(st.Filter(parser.MustParseExpr("b < 2"))), + st.New(st.IndexScan("idx_foo_a_b_c", st.IndexRange{Max: testutil.MakeValueBuffer(t, `[1, 2]`), Exclusive: true})), + }, { "FROM foo WHERE a = 1 AND b = 2 and k = 3", // c is omitted, but it can still use idx_foo_a_b_c st.New(st.SeqScan("foo")). @@ -626,6 +647,102 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { Pipe(st.Filter(parser.MustParseExpr("b = 2"))). Pipe(st.Filter(parser.MustParseExpr("c = 'a'"))), }, + + { + "FROM foo WHERE a IN [1, 2] AND d = 4", + st.New(st.SeqScan("foo")). + Pipe(st.Filter( + expr.In( + parser.MustParseExpr("a"), + expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), + ), + )). + Pipe(st.Filter(parser.MustParseExpr("d = 4"))), + st.New(st.IndexScan("idx_foo_a_d", + st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 4]`), Exact: true}, + st.IndexRange{Min: testutil.MakeValueBuffer(t, `[2, 4]`), Exact: true}, + )), + }, + { + "FROM foo WHERE a IN [1, 2] AND b = 3 AND c = 4", + st.New(st.SeqScan("foo")). + Pipe(st.Filter( + expr.In( + parser.MustParseExpr("a"), + expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), + ), + )). + Pipe(st.Filter(parser.MustParseExpr("b = 3"))). + Pipe(st.Filter(parser.MustParseExpr("c = 4"))), + st.New(st.IndexScan("idx_foo_a_b_c", + st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 3, 4]`), Exact: true}, + st.IndexRange{Min: testutil.MakeValueBuffer(t, `[2, 3, 4]`), Exact: true}, + )), + }, + { + "FROM foo WHERE a IN [1, 2] AND b = 3 AND c > 4", + st.New(st.SeqScan("foo")). + Pipe(st.Filter( + expr.In( + parser.MustParseExpr("a"), + expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), + ), + )). + Pipe(st.Filter(parser.MustParseExpr("b = 3"))). + Pipe(st.Filter(parser.MustParseExpr("c > 4"))), + st.New(st.IndexScan("idx_foo_a_b_c", + st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 3, 4]`), Exclusive: true}, + st.IndexRange{Min: testutil.MakeValueBuffer(t, `[2, 3, 4]`), Exclusive: true}, + )), + }, + { + "FROM foo WHERE a IN [1, 2] AND b = 3 AND c < 4", + st.New(st.SeqScan("foo")). + Pipe(st.Filter( + expr.In( + parser.MustParseExpr("a"), + expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), + ), + )). + Pipe(st.Filter(parser.MustParseExpr("b = 3"))). + Pipe(st.Filter(parser.MustParseExpr("c < 4"))), + st.New(st.IndexScan("idx_foo_a_b_c", + st.IndexRange{Max: testutil.MakeValueBuffer(t, `[1, 3, 4]`), Exclusive: true}, + st.IndexRange{Max: testutil.MakeValueBuffer(t, `[2, 3, 4]`), Exclusive: true}, + )), + }, + // { + // "FROM foo WHERE a IN [1, 2] AND b IN [3, 4] AND c > 5", + // st.New(st.SeqScan("foo")). + // Pipe(st.Filter( + // expr.In( + // parser.MustParseExpr("a"), + // expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(1), document.NewIntegerValue(2))), + // ), + // )). + // Pipe(st.Filter( + // expr.In( + // parser.MustParseExpr("b"), + // expr.ArrayValue(document.NewValueBuffer(document.NewIntegerValue(3), document.NewIntegerValue(4))), + // ), + // )). + // Pipe(st.Filter(parser.MustParseExpr("c < 5"))), + // st.New(st.IndexScan("idx_foo_a_b_c", + // st.IndexRange{Max: testutil.MakeValueBuffer(t, `[1, 3, 5]`), Exclusive: true}, + // st.IndexRange{Max: testutil.MakeValueBuffer(t, `[2, 3, 5]`), Exclusive: true}, + // st.IndexRange{Max: testutil.MakeValueBuffer(t, `[1, 4, 5]`), Exclusive: true}, + // st.IndexRange{Max: testutil.MakeValueBuffer(t, `[2, 4, 5]`), Exclusive: true}, + // )), + // }, + { + "FROM foo WHERE 1 IN a AND d = 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("1 IN a"))). + Pipe(st.Filter(parser.MustParseExpr("d = 4"))), + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("1 IN a"))). + Pipe(st.Filter(parser.MustParseExpr("d = 4"))), + }, } for _, test := range tests { diff --git a/query/select_test.go b/query/select_test.go index 579011ebe..7ef99c903 100644 --- a/query/select_test.go +++ b/query/select_test.go @@ -48,9 +48,9 @@ func TestSelectStmt(t *testing.T) { {"With sub op", "SELECT size - 10 AS s FROM test ORDER BY k", false, `[{"s":0},{"s":0},{"s":null}]`, nil}, {"With mul op", "SELECT size * 10 AS s FROM test ORDER BY k", false, `[{"s":100},{"s":100},{"s":null}]`, nil}, {"With div op", "SELECT size / 10 AS s FROM test ORDER BY k", false, `[{"s":1},{"s":1},{"s":null}]`, nil}, - // {"With IN op", "SELECT color FROM test WHERE color IN ['red', 'purple'] ORDER BY k", false, `[{"color":"red"}]`, nil}, - // {"With IN op on PK", "SELECT color FROM test WHERE k IN [1.1, 1.0] ORDER BY k", false, `[{"color":"red"}]`, nil}, - // {"With NOT IN op", "SELECT color FROM test WHERE color NOT IN ['red', 'purple'] ORDER BY k", false, `[{"color":"blue"}]`, nil}, + {"With IN op", "SELECT color FROM test WHERE color IN ['red', 'purple'] ORDER BY k", false, `[{"color":"red"}]`, nil}, + {"With IN op on PK", "SELECT color FROM test WHERE k IN [1.1, 1.0] ORDER BY k", false, `[{"color":"red"}]`, nil}, + {"With NOT IN op", "SELECT color FROM test WHERE color NOT IN ['red', 'purple'] ORDER BY k", false, `[{"color":"blue"}]`, nil}, {"With field comparison", "SELECT * FROM test WHERE color < shape", false, `[{"k":1,"color":"red","size":10,"shape":"square"}]`, nil}, {"With group by", "SELECT color FROM test GROUP BY color", false, `[{"color":"red"},{"color":"blue"},{"color":null}]`, nil}, {"With group by and count", "SELECT COUNT(k) FROM test GROUP BY size", false, `[{"COUNT(k)":2},{"COUNT(k)":1}]`, nil}, From 3f3940aaf8b6f28db5d6de07f7aef7ced86ffd5d Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 26 Apr 2021 16:21:00 +0200 Subject: [PATCH 29/40] Name the zero value of document.ValueType AnyType --- database/index.go | 18 +++++++++--------- database/table.go | 4 ++-- document/value.go | 9 ++++++--- planner/optimizer.go | 2 +- sql/parser/create.go | 2 +- stream/iterator.go | 2 +- stream/range.go | 18 +++++++++--------- 7 files changed, 29 insertions(+), 26 deletions(-) diff --git a/database/index.go b/database/index.go index a3b7a80e0..c5dc09f53 100644 --- a/database/index.go +++ b/database/index.go @@ -50,7 +50,7 @@ type indexValueEncoder struct { func (e *indexValueEncoder) EncodeValue(v document.Value) ([]byte, error) { // if the index has no type constraint, encode the value with its type - if e.typ.IsZero() { + if e.typ.IsAny() { var buf bytes.Buffer // prepend with the type @@ -76,7 +76,7 @@ func (e *indexValueEncoder) EncodeValue(v document.Value) ([]byte, error) { } if v.Type != e.typ { - if v.Type == 0 { + if v.Type.IsAny() { v.Type = e.typ } else { // this should never happen, but if it does, something is very wrong @@ -245,7 +245,7 @@ func (pivot Pivot) validate(idx *Index) { panic("cannot iterate with a pivot whose size is superior to the index arity") } - if idx.IsComposite() && !pivot.Empty() { + if idx.IsComposite() && !pivot.IsAny() { // it's acceptable for the last pivot to just have a type and no value hasValue := true for _, p := range pivot { @@ -259,11 +259,11 @@ func (pivot Pivot) validate(idx *Index) { } } -// TODO rename? -func (pivot Pivot) Empty() bool { +// IsAny return true if every value of the pivot is typed with AnyType +func (pivot Pivot) IsAny() bool { res := true for _, p := range pivot { - res = res && p.Type == 0 + res = res && p.Type.IsAny() && p.V == nil if !res { break } @@ -297,7 +297,7 @@ func (idx *Index) iterateOnStore(pivot Pivot, reverse bool, fn func(val, key []b // If index and pivot values are typed but not of the same type, return no results. for i, pv := range pivot { - if pv.Type != 0 && idx.Info.Types[i] != 0 && pv.Type != idx.Info.Types[i] { + if !pv.Type.IsAny() && !idx.Info.Types[i].IsAny() && pv.Type != idx.Info.Types[i] { return nil } } @@ -413,7 +413,7 @@ func (idx *Index) buildSeek(pivot Pivot, reverse bool) ([]byte, error) { // TODO rework // if we have valueless and typeless pivot, we just iterate - if pivot.Empty() { + if pivot.IsAny() { return []byte{}, nil } @@ -459,7 +459,7 @@ func (idx *Index) iterate(st engine.Store, pivot Pivot, reverse bool, fn func(it // If index is untyped and pivot first element is typed, only iterate on values with the same type as the first pivot // TODO(JH) possible optimization, check for the other types - if len(pivot) > 0 && idx.Info.Types[0] == 0 && pivot[0].Type != 0 && itm.Key()[0] != byte(pivot[0].Type) { + if len(pivot) > 0 && idx.Info.Types[0].IsAny() && !pivot[0].Type.IsAny() && itm.Key()[0] != byte(pivot[0].Type) { return nil } diff --git a/database/table.go b/database/table.go index 170ffbd08..f33296733 100644 --- a/database/table.go +++ b/database/table.go @@ -363,7 +363,7 @@ func (t *Table) encodeValueToKey(info *TableInfo, v document.Value) ([]byte, err } // if a primary key was defined and the primary is typed, convert the value to the right type. - if !pk.Type.IsZero() { + if !pk.Type.IsAny() { v, err = v.CastAs(pk.Type) if err != nil { return nil, err @@ -413,7 +413,7 @@ func (t *Table) iterate(pivot document.Value, reverse bool, fn func(d document.D info := t.Info() // if there is a pivot, convert it to the right type - if !pivot.Type.IsZero() && pivot.V != nil { + if !pivot.Type.IsAny() && pivot.V != nil { var err error seek, err = t.encodeValueToKey(info, pivot) if err != nil { diff --git a/document/value.go b/document/value.go index 647bb3f16..f9d37aa17 100644 --- a/document/value.go +++ b/document/value.go @@ -37,6 +37,9 @@ type ValueType uint8 // These types are separated by family so that when // new types are introduced we don't need to modify them. const ( + // denote the absence of type + AnyType ValueType = 0x0 + NullValue ValueType = 0x80 BoolValue ValueType = 0x81 @@ -88,9 +91,9 @@ func (t ValueType) IsNumber() bool { return t == IntegerValue || t == DoubleValue } -// IsZero returns whether this is a valid type. -func (t ValueType) IsZero() bool { - return t == 0 +// IsAny returns whether this is type is Any or a real type +func (t ValueType) IsAny() bool { + return t == AnyType } // A Value stores encoded data alongside its type. diff --git a/planner/optimizer.go b/planner/optimizer.go index db684a1d6..a67b3d05f 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -736,7 +736,7 @@ func operandCanUseIndex(indexType document.ValueType, path document.Path, fc dat } // if the index is not typed, any operand can work - if indexType.IsZero() { + if indexType.IsAny() { return converted, true, nil } diff --git a/sql/parser/create.go b/sql/parser/create.go index d06fcb509..8ba2ba84d 100644 --- a/sql/parser/create.go +++ b/sql/parser/create.go @@ -71,7 +71,7 @@ func (p *Parser) parseFieldDefinition(fc *database.FieldConstraint) (err error) return err } - if fc.Type == 0 && fc.DefaultValue.Type.IsZero() && !fc.IsNotNull && !fc.IsPrimaryKey && !fc.IsUnique { + if fc.Type.IsAny() && fc.DefaultValue.Type.IsAny() && !fc.IsNotNull && !fc.IsPrimaryKey && !fc.IsUnique { tok, pos, lit := p.ScanIgnoreWhitespace() return newParseError(scanner.Tokstr(tok, lit), []string{"CONSTRAINT", "TYPE"}, pos) } diff --git a/stream/iterator.go b/stream/iterator.go index 0c0d22dd5..9b2f1cafa 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -234,7 +234,7 @@ func (it *PkScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Enviro } var encEnd []byte - if !end.Type.IsZero() && end.V != nil { + if !end.Type.IsAny() && end.V != nil { encEnd, err = table.EncodeValue(end) if err != nil { return err diff --git a/stream/range.go b/stream/range.go index c9d723abc..479117321 100644 --- a/stream/range.go +++ b/stream/range.go @@ -32,19 +32,19 @@ func (r *ValueRange) encode(encoder ValueEncoder, env *expr.Environment) error { var err error // first we evaluate Min and Max - if !r.Min.Type.IsZero() { + if !r.Min.Type.IsAny() { r.encodedMin, err = encoder.EncodeValue(r.Min) if err != nil { return err } r.rangeType = r.Min.Type } - if !r.Max.Type.IsZero() { + if !r.Max.Type.IsAny() { r.encodedMax, err = encoder.EncodeValue(r.Max) if err != nil { return err } - if !r.rangeType.IsZero() && r.rangeType != r.Max.Type { + if !r.rangeType.IsAny() && r.rangeType != r.Max.Type { panic("range contain values of different types") } @@ -52,10 +52,10 @@ func (r *ValueRange) encode(encoder ValueEncoder, env *expr.Environment) error { } // ensure boundaries are typed - if r.Min.Type.IsZero() { + if r.Min.Type.IsAny() { r.Min.Type = r.rangeType } - if r.Max.Type.IsZero() { + if r.Max.Type.IsAny() { r.Max.Type = r.rangeType } @@ -71,10 +71,10 @@ func (r *ValueRange) String() string { return stringutil.Sprintf("%v", r.Min) } - if r.Min.Type.IsZero() { + if r.Min.Type.IsAny() { r.Min = document.NewIntegerValue(-1) } - if r.Max.Type.IsZero() { + if r.Max.Type.IsAny() { r.Max = document.NewIntegerValue(-1) } @@ -182,12 +182,12 @@ func (r ValueRanges) Cost() int { } // if there are two boundaries, increment by 50 - if !rng.Min.Type.IsZero() && !rng.Max.Type.IsZero() { + if !rng.Min.Type.IsAny() && !rng.Max.Type.IsAny() { cost += 50 } // if there is only one boundary, increment by 100 - if (!rng.Min.Type.IsZero() && rng.Max.Type.IsZero()) || (rng.Min.Type.IsZero() && !rng.Max.Type.IsZero()) { + if (!rng.Min.Type.IsAny() && rng.Max.Type.IsAny()) || (rng.Min.Type.IsAny() && !rng.Max.Type.IsAny()) { cost += 100 continue } From 52a79aad0419689adcaad093465bbeeb489651f5 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 26 Apr 2021 16:45:02 +0200 Subject: [PATCH 30/40] Re-enable indexes on arrays test --- planner/optimizer_test.go | 118 +++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index 256ba4b4b..e3986f98c 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -775,64 +775,62 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { }) } - // t.Run("array indexes", func(t *testing.T) { - // tests := []struct { - // name string - // root, expected *st.Stream - // }{ - // { - // "FROM foo WHERE a = [1, 1] AND b = [2, 2]", - // st.New(st.SeqScan("foo")). - // Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). - // Pipe(st.Filter(parser.MustParseExpr("b = [2, 2]"))), - // st.New(st.IndexScan("idx_foo_a_b", st.Range{ - // Min: document.NewArrayValue( - // testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), - // Exact: true})), - // }, - // { - // "FROM foo WHERE a = [1, 1] AND b > [2, 2]", - // st.New(st.SeqScan("foo")). - // Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). - // Pipe(st.Filter(parser.MustParseExpr("b > [2, 2]"))), - // st.New(st.IndexScan("idx_foo_a_b", st.Range{ - // Min: document.NewArrayValue( - // testutil.MakeArray(t, `[[1, 1], [2, 2]]`)), - // Exclusive: true})), - // }, - // } - - // for _, test := range tests { - // t.Run(test.name, func(t *testing.T) { - // db, err := genji.Open(":memory:") - // require.NoError(t, err) - // defer db.Close() - - // tx, err := db.Begin(true) - // require.NoError(t, err) - // defer tx.Rollback() - - // err = tx.Exec(` - // CREATE TABLE foo ( - // k ARRAY PRIMARY KEY, - // a ARRAY - // ); - // CREATE INDEX idx_foo_a_b ON foo(a, b); - // CREATE INDEX idx_foo_a0 ON foo(a[0]); - // INSERT INTO foo (k, a, b) VALUES - // ([1, 1], [1, 1], [1, 1]), - // ([2, 2], [2, 2], [2, 2]), - // ([3, 3], [3, 3], [3, 3]) - // `) - // require.NoError(t, err) - - // res, err := planner.PrecalculateExprRule(test.root, tx.Transaction, nil) - // require.NoError(t, err) - - // res, err = planner.UseIndexBasedOnFilterNodeRule(res, tx.Transaction, nil) - // require.NoError(t, err) - // require.Equal(t, test.expected.String(), res.String()) - // }) - // } - // }) + t.Run("array indexes", func(t *testing.T) { + tests := []struct { + name string + root, expected *st.Stream + }{ + { + "FROM foo WHERE a = [1, 1] AND b = [2, 2]", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). + Pipe(st.Filter(parser.MustParseExpr("b = [2, 2]"))), + st.New(st.IndexScan("idx_foo_a_b", st.IndexRange{ + Min: testutil.MakeValueBuffer(t, `[[1, 1], [2, 2]]`), + Exact: true})), + }, + { + "FROM foo WHERE a = [1, 1] AND b > [2, 2]", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("a = [1, 1]"))). + Pipe(st.Filter(parser.MustParseExpr("b > [2, 2]"))), + st.New(st.IndexScan("idx_foo_a_b", st.IndexRange{ + Min: testutil.MakeValueBuffer(t, `[[1, 1], [2, 2]]`), + Exclusive: true})), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, err := genji.Open(":memory:") + require.NoError(t, err) + defer db.Close() + + tx, err := db.Begin(true) + require.NoError(t, err) + defer tx.Rollback() + + err = tx.Exec(` + CREATE TABLE foo ( + k ARRAY PRIMARY KEY, + a ARRAY + ); + CREATE INDEX idx_foo_a_b ON foo(a, b); + CREATE INDEX idx_foo_a0 ON foo(a[0]); + INSERT INTO foo (k, a, b) VALUES + ([1, 1], [1, 1], [1, 1]), + ([2, 2], [2, 2], [2, 2]), + ([3, 3], [3, 3], [3, 3]) + `) + require.NoError(t, err) + + res, err := planner.PrecalculateExprRule(test.root, tx.Transaction, nil) + require.NoError(t, err) + + res, err = planner.UseIndexBasedOnFilterNodeRule(res, tx.Transaction, nil) + require.NoError(t, err) + require.Equal(t, test.expected.String(), res.String()) + }) + } + }) } From c227c0b1c79438d0225058c298ecff6a1d934f8a Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 26 Apr 2021 17:27:40 +0200 Subject: [PATCH 31/40] Clean the docs --- database/index.go | 13 ++++++------- planner/optimizer.go | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/database/index.go b/database/index.go index c5dc09f53..2f45c9e79 100644 --- a/database/index.go +++ b/database/index.go @@ -29,9 +29,6 @@ var ( // The association is performed by encoding the values in a binary format that preserve // ordering when compared lexicographically. For the implementation, see the binarysort // package and the document.ValueEncoder. -// -// When the index is composite, the values are wrapped into a document.Array before -// being encoded. type Index struct { Info *IndexInfo @@ -120,12 +117,12 @@ func (idx *Index) IsComposite() bool { } // Arity returns how many values the indexed is operating on. -// CREATE INDEX idx_a_b ON foo (a, b) -> arity: 2 +// For example, an index created with `CREATE INDEX idx_a_b ON foo (a, b)` has an arity of 2. func (idx *Index) Arity() int { return len(idx.Info.Types) } -// Set associates a value with a key. If Unique is set to false, it is +// Set associates values with a key. If Unique is set to false, it is // possible to associate multiple keys for the same value // but a key can be associated to only one value. func (idx *Index) Set(vs []document.Value, k []byte) error { @@ -275,6 +272,7 @@ func (pivot Pivot) IsAny() bool { // AscendGreaterOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in increasing order and calls the given function for each pair. // If the given function returns an error, the iteration stops and returns that error. // If the pivot(s) is/are empty, starts from the beginning. +// // When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. // When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element // is of that type, without restriction on the type of the following elements. @@ -285,6 +283,7 @@ func (idx *Index) AscendGreaterOrEqual(pivot Pivot, fn func(val, key []byte) err // DescendLessOrEqual seeks for the pivot and then goes through all the subsequent key value pairs in descreasing order and calls the given function for each pair. // If the given function returns an error, the iteration stops and returns that error. // If the pivot(s) is/are empty, starts from the end. +// // When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. // When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element // is of that type, without restriction on the type of the following elements. @@ -348,7 +347,8 @@ func (idx *Index) Truncate() error { // // The values are marshalled and separated with a document.ArrayValueDelim, // *without* a trailing document.ArrayEnd, which enables to handle cases -// where only some of the values are being provided and still perform lookups. +// where only some of the values are being provided and still perform lookups +// (like index_foo_a_b_c and providing only a and b). // // See IndexValueEncoder for details about how the value themselves are encoded. func (idx *Index) EncodeValueBuffer(vb *document.ValueBuffer) ([]byte, error) { @@ -411,7 +411,6 @@ func (idx *Index) buildSeek(pivot Pivot, reverse bool) ([]byte, error) { var seek []byte var err error - // TODO rework // if we have valueless and typeless pivot, we just iterate if pivot.IsAny() { return []byte{}, nil diff --git a/planner/optimizer.go b/planner/optimizer.go index a67b3d05f..b41df2f7b 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -760,6 +760,7 @@ func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) case *expr.InOperator: // mark where the IN operator values are supposed to go is in the buffer // and what are the value needed to generate the ranges. + // operatorCanUseIndex made sure v is an array. inOperands[i] = v.V.(document.Array) // placeholder for when we'll explode the IN operands in multiple ranges From 2a5cd4021c89d785615fde00fbf68f2d575909d6 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 26 Apr 2021 17:29:21 +0200 Subject: [PATCH 32/40] Add more index tests, docs and array --- database/index_test.go | 361 +++++++++++++++++++++++++++++++++-------- 1 file changed, 296 insertions(+), 65 deletions(-) diff --git a/database/index_test.go b/database/index_test.go index d391915cb..1f620b8f5 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -264,7 +264,7 @@ func TestIndexDelete(t *testing.T) { } } -// requireEqualBinaryUntyped asserts equality assuming that the value is encoded through marshal binary +// requireEqualBinary asserts equality assuming that the value is encoded through marshal binary func requireEqualBinary(t *testing.T, expected document.Value, actual []byte) { t.Helper() @@ -273,16 +273,6 @@ func requireEqualBinary(t *testing.T, expected document.Value, actual []byte) { require.Equal(t, buf[:len(buf)-1], actual) } -// requireEqualEncoded asserts equality, assuming that the value is encoded with document.ValueEncoder -func requireEqualEncoded(t *testing.T, expected document.Value, actual []byte) { - t.Helper() - - var buf bytes.Buffer - err := document.NewValueEncoder(&buf).Encode(expected) - require.NoError(t, err) - require.Equal(t, buf.Bytes(), actual) -} - type encValue struct { skipType bool document.Value @@ -367,7 +357,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { mustPanic bool }{ // integers --------------------------------------------------- - {name: "index=untyped, vals=integers, pivot=integer", + {name: "index=any, vals=integers, pivot=integer", indexTypes: nil, pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, @@ -392,7 +382,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=integers, pivot=integer:2", + {name: "index=any, vals=integers, pivot=integer:2", indexTypes: nil, pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, @@ -406,7 +396,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 3, }, - {name: "index=untyped, vals=integers, pivot=integer:10", + {name: "index=any, vals=integers, pivot=integer:10", indexTypes: nil, pivot: values(document.NewIntegerValue(10)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, @@ -436,7 +426,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // doubles ---------------------------------------------------- - {name: "index=untyped, vals=doubles, pivot=double", + {name: "index=any, vals=doubles, pivot=double", indexTypes: nil, pivot: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, @@ -448,7 +438,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=doubles, pivot=double:1.8", + {name: "index=any, vals=doubles, pivot=double:1.8", indexTypes: nil, pivot: values(document.NewDoubleValue(1.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, @@ -474,7 +464,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 3, }, - {name: "index=untyped, vals=doubles, pivot=double:10.8", + {name: "index=any, vals=doubles, pivot=double:10.8", indexTypes: nil, pivot: values(document.NewDoubleValue(10.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, @@ -483,7 +473,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // text ------------------------------------------------------- - {name: "index=untyped, vals=text pivot=text", + {name: "index=any, vals=text pivot=text", indexTypes: nil, pivot: values(document.Value{Type: document.TextValue}), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -496,7 +486,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=text, pivot=text('2')", + {name: "index=any, vals=text, pivot=text('2')", indexTypes: nil, pivot: values(document.NewTextValue("2")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -510,7 +500,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 3, }, - {name: "index=untyped, vals=text, pivot=text('')", + {name: "index=any, vals=text, pivot=text('')", indexTypes: nil, pivot: values(document.NewTextValue("")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -523,7 +513,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=text, pivot=text('foo')", + {name: "index=any, vals=text, pivot=text('foo')", indexTypes: nil, pivot: values(document.NewTextValue("foo")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -546,7 +536,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // composite -------------------------------------------------- // composite indexes can have empty pivot values to iterate on the whole indexed data - {name: "index=[untyped, untyped], vals=[int, int], pivot=[nil,nil]", + {name: "index=[any, untyped], vals=[int, int], pivot=[nil,nil]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{}, document.Value{}), val: func(i int) []document.Value { @@ -562,7 +552,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // composite indexes must have at least have one value if typed - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -571,7 +561,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: noCallEq, mustPanic: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -585,7 +575,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, pivot: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -594,7 +584,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: noCallEq, mustPanic: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, nil]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, nil]", indexTypes: []document.ValueType{0, 0, 0}, pivot: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0), document.Value{}), val: func(i int) []document.Value { @@ -603,7 +593,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: noCallEq, mustPanic: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -612,7 +602,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedEq: noCallEq, mustPanic: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -629,7 +619,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 5, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -647,7 +637,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 3, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -666,7 +656,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 3, }, // pivot [2, int] should filter out [2, not(int)] - {name: "index=[untyped, untyped], vals=[int, int], noise=[int, blob], pivot=[2, int]", + {name: "index=[any, untyped], vals=[int, int], noise=[int, blob], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -690,7 +680,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 3, }, // a more subtle case - {name: "index=[untyped, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + {name: "index=[any, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { @@ -709,7 +699,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 3, }, // partial pivot - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -723,7 +713,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, expectedCount: 10, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { @@ -739,7 +729,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // this is a tricky test, when we have multiple values but they share the first pivot element; // this is by definition a very implementation dependent test. - {name: "index=[untyped, untyped], vals=[int, int], noise=int, bool], pivot=[int:0, int:0]", + {name: "index=[any, untyped], vals=[int, int], noise=int, bool], pivot=[int:0, int:0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -838,9 +828,130 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // documents -------------------------------------------------- - // TODO + {name: "index=[any, any], vals=[doc, int], pivot=[{a:2}, 3]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":2}`)), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(i)+`}`)), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + requireIdxEncodedEq(t, + encValue{false, document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(int(i))+`}`))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, + {name: "index=[document, int], vals=[doc, int], pivot=[{a:2}, 3]", + indexTypes: []document.ValueType{document.DocumentValue, document.IntegerValue}, + pivot: values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":2}`)), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(i)+`}`)), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + requireIdxEncodedEq(t, + encValue{true, document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(int(i))+`}`))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, + // arrays ----------------------------------------------------- - // TODO + {name: "index=[any, any], vals=[int[], int], pivot=[]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values(), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + requireIdxEncodedEq(t, + encValue{false, testutil.MakeArrayValue(t, i, i)}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 5, + }, + {name: "index=[any, any], vals=[int[], int], pivot=[[2,2], 3]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values( + testutil.MakeArrayValue(t, 2, 2), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + requireIdxEncodedEq(t, + encValue{false, testutil.MakeArrayValue(t, i, i)}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, + {name: "index=[any, any], vals=[int[], int[]], pivot=[[2,2], [3,3]]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values( + testutil.MakeArrayValue(t, 2, 2), + testutil.MakeArrayValue(t, 3, 3), + ), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + testutil.MakeArrayValue(t, i+1, i+1), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + requireIdxEncodedEq(t, + encValue{false, testutil.MakeArrayValue(t, i, i)}, + encValue{false, testutil.MakeArrayValue(t, i+1, i+1)}, + )(val) + }, + expectedCount: 3, + }, + {name: "index=[array, any], vals=[int[], int], pivot=[[2,2], 3]", + indexTypes: []document.ValueType{document.ArrayValue, document.AnyType}, + pivot: values( + testutil.MakeArrayValue(t, 2, 2), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i += 2 + requireIdxEncodedEq(t, + encValue{true, testutil.MakeArrayValue(t, i, i)}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, } for _, test := range tests { @@ -938,7 +1049,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { mustPanic bool }{ // integers --------------------------------------------------- - {name: "index=untyped, vals=integers, pivot=integer", + {name: "index=any, vals=integers, pivot=integer", indexTypes: nil, pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, @@ -961,7 +1072,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=integers, pivot=integer:2", + {name: "index=any, vals=integers, pivot=integer:2", indexTypes: nil, pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, @@ -975,7 +1086,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 3, }, - {name: "index=untyped, vals=integers, pivot=integer:-10", + {name: "index=any, vals=integers, pivot=integer:-10", indexTypes: nil, pivot: values(document.NewIntegerValue(-10)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i))) }, @@ -1005,7 +1116,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // doubles ---------------------------------------------------- - {name: "index=untyped, vals=doubles, pivot=double", + {name: "index=any, vals=doubles, pivot=double", indexTypes: nil, pivot: values(document.Value{Type: document.DoubleValue}), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, @@ -1017,7 +1128,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=doubles, pivot=double:1.8", + {name: "index=any, vals=doubles, pivot=double:1.8", indexTypes: nil, pivot: values(document.NewDoubleValue(1.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, @@ -1043,7 +1154,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 2, }, - {name: "index=untyped, vals=doubles, pivot=double:-10.8", + {name: "index=any, vals=doubles, pivot=double:-10.8", indexTypes: nil, pivot: values(document.NewDoubleValue(-10.8)), val: func(i int) []document.Value { return values(document.NewDoubleValue(float64(i) + float64(i)/2)) }, @@ -1052,7 +1163,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // text ------------------------------------------------------- - {name: "index=untyped, vals=text pivot=text", + {name: "index=any, vals=text pivot=text", indexTypes: nil, pivot: values(document.Value{Type: document.TextValue}), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -1066,7 +1177,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=text, pivot=text('2')", + {name: "index=any, vals=text, pivot=text('2')", indexTypes: nil, pivot: values(document.NewTextValue("2")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -1080,7 +1191,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 3, }, - {name: "index=untyped, vals=text, pivot=text('')", + {name: "index=any, vals=text, pivot=text('')", indexTypes: nil, pivot: values(document.NewTextValue("")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -1093,7 +1204,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 5, }, - {name: "index=untyped, vals=text, pivot=text('foo')", + {name: "index=any, vals=text, pivot=text('foo')", indexTypes: nil, pivot: values(document.NewTextValue("foo")), val: func(i int) []document.Value { return values(document.NewTextValue(strconv.Itoa(i))) }, @@ -1121,7 +1232,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // composite -------------------------------------------------- // composite indexes can have empty pivot values to iterate on the whole indexed data - {name: "index=[untyped, untyped], vals=[int, int], pivot=[nil,nil]", + {name: "index=[any, untyped], vals=[int, int], pivot=[nil,nil]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{}, document.Value{}), val: func(i int) []document.Value { @@ -1135,7 +1246,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 5, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -1150,7 +1261,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 5, }, // composite indexes cannot have values with type past the first element - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -1158,7 +1269,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, mustPanic: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, int, 0]", indexTypes: []document.ValueType{0, 0, 0}, pivot: values(document.NewIntegerValue(0), document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -1166,7 +1277,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, mustPanic: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -1174,7 +1285,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, mustPanic: true, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -1186,7 +1297,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedEq: noCallEq, expectedCount: 0, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[5, 5]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[5, 5]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(5), document.NewIntegerValue(5)), val: func(i int) []document.Value { @@ -1204,7 +1315,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 5, }, // [0,1], [1,2], --[2,0]--, [2,3], [3,4], [4,5] - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -1223,7 +1334,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 2, }, // [0,1], [1,2], [2,3], --[2,int]--, [3,4], [4,5] - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -1243,7 +1354,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // pivot [2, int] should filter out [2, not(int)] // [0,1], [1,2], [2,3], --[2,int]--, [2, text], [3,4], [3,text], [4,5], [4,text] - {name: "index=[untyped, untyped], vals=[int, int], noise=[int, text], pivot=[2, int]", + {name: "index=[any, untyped], vals=[int, int], noise=[int, text], pivot=[2, int]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { @@ -1266,7 +1377,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 3, }, // a more subtle case - {name: "index=[untyped, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + {name: "index=[any, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { @@ -1291,7 +1402,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 2, }, // only one of the indexed value is typed - {name: "index=[untyped, blob], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway + {name: "index=[any, blob], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway indexTypes: []document.ValueType{0, document.BlobValue}, pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { @@ -1316,7 +1427,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 2, }, // partial pivot - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { @@ -1330,7 +1441,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 2, // [0] is "equal" to [0, 1] and [0, "1"] }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[5]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[5]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(5)), val: func(i int) []document.Value { @@ -1344,7 +1455,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 10, }, - {name: "index=[untyped, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", + {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", indexTypes: []document.ValueType{0, 0}, pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { @@ -1445,11 +1556,131 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, expectedCount: 3, }, - // documents -------------------------------------------------- - // TODO + {name: "index=[any, any], vals=[doc, int], pivot=[{a:2}, 3]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":2}`)), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(i)+`}`)), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + requireIdxEncodedEq(t, + encValue{false, document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(int(i))+`}`))}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, + {name: "index=[document, int], vals=[doc, int], pivot=[{a:2}, 3]", + indexTypes: []document.ValueType{document.DocumentValue, document.IntegerValue}, + pivot: values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":2}`)), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(i)+`}`)), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + requireIdxEncodedEq(t, + encValue{true, document.NewDocumentValue(testutil.MakeDocument(t, `{"a":`+strconv.Itoa(int(i))+`}`))}, + encValue{true, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, + // arrays ----------------------------------------------------- - // TODO + {name: "index=[any, any], vals=[int[], int], pivot=[]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values(), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + requireIdxEncodedEq(t, + encValue{false, testutil.MakeArrayValue(t, i, i)}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 5, + }, + {name: "index=[any, any], vals=[int[], int], pivot=[[2,2], 3]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values( + testutil.MakeArrayValue(t, 2, 2), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + requireIdxEncodedEq(t, + encValue{false, testutil.MakeArrayValue(t, i, i)}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, + {name: "index=[any, any], vals=[int[], int[]], pivot=[[2,2], [3,3]]", + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, + pivot: values( + testutil.MakeArrayValue(t, 2, 2), + testutil.MakeArrayValue(t, 3, 3), + ), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + testutil.MakeArrayValue(t, i+1, i+1), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + requireIdxEncodedEq(t, + encValue{false, testutil.MakeArrayValue(t, i, i)}, + encValue{false, testutil.MakeArrayValue(t, i+1, i+1)}, + )(val) + }, + expectedCount: 3, + }, + {name: "index=[array, any], vals=[int[], int], pivot=[[2,2], 3]", + indexTypes: []document.ValueType{document.ArrayValue, document.AnyType}, + pivot: values( + testutil.MakeArrayValue(t, 2, 2), + document.NewIntegerValue(int64(3)), + ), + val: func(i int) []document.Value { + return values( + testutil.MakeArrayValue(t, i, i), + document.NewIntegerValue(int64(i+1)), + ) + }, + expectedEq: func(t *testing.T, i uint8, key []byte, val []byte) { + i -= 2 + requireIdxEncodedEq(t, + encValue{true, testutil.MakeArrayValue(t, i, i)}, + encValue{false, document.NewIntegerValue(int64(i + 1))}, + )(val) + }, + expectedCount: 3, + }, } for _, test := range tests { From 1c21c420137668ca0b80c795d7f5413abaf42975 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Mon, 26 Apr 2021 18:58:03 +0200 Subject: [PATCH 33/40] Fix composite benchmark --- database/index_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/index_test.go b/database/index_test.go index 1f620b8f5..1b2bb0a31 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -1799,7 +1799,7 @@ func BenchmarkCompositeIndexSet(b *testing.B) { func BenchmarkCompositeIndexIteration(b *testing.B) { for size := 10; size <= 10000; size *= 10 { b.Run(fmt.Sprintf("%.05d", size), func(b *testing.B) { - idx, cleanup := getIndex(b, false) + idx, cleanup := getIndex(b, false, document.AnyType, document.AnyType) defer cleanup() for i := 0; i < size; i++ { From 5c361fa4fe664182b5439cd2c6214783b5611647 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 28 Apr 2021 17:36:03 +0200 Subject: [PATCH 34/40] Edit doc, missing document.AnyType replacements --- database/index.go | 49 +++++++++++++++++++++++++++++--------------- planner/optimizer.go | 4 ++-- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/database/index.go b/database/index.go index 2f45c9e79..419602765 100644 --- a/database/index.go +++ b/database/index.go @@ -36,8 +36,6 @@ type Index struct { storeName []byte } -type Pivot []document.Value - // indexValueEncoder encodes a field based on its type; if a type is provided, // the value is encoded as is, without any type information. Otherwise, the // type is prepended to the value. @@ -93,13 +91,13 @@ func (e *indexValueEncoder) EncodeValue(v document.Value) ([]byte, error) { func NewIndex(tx engine.Transaction, idxName string, opts *IndexInfo) *Index { if opts == nil { opts = &IndexInfo{ - Types: []document.ValueType{0}, + Types: []document.ValueType{document.AnyType}, } } // if no types are provided, it implies that it's an index for single untyped values if opts.Types == nil { - opts.Types = []document.ValueType{0} + opts.Types = []document.ValueType{document.AnyType} } return &Index{ @@ -141,8 +139,7 @@ func (idx *Index) Set(vs []document.Value, k []byte) error { } for i, typ := range idx.Info.Types { - // it is possible to set an index(a,b) on (a), it will be assumed that b is null in that case - if typ != 0 && i < len(vs) && typ != vs[i].Type { + if !typ.IsAny() && typ != vs[i].Type { return stringutil.Errorf("cannot index value of type %s in %s index", vs[i].Type, typ) } } @@ -233,17 +230,18 @@ func (idx *Index) Delete(vs []document.Value, k []byte) error { return engine.ErrKeyNotFound } +type Pivot []document.Value + // validate panics when the pivot values are unsuitable for the index: -// - no pivot values at all // - having pivot length superior to the index arity -// - having the first pivot value without a value when the subsequent ones do have values +// - having the first pivot element without a value when the subsequent ones do have values func (pivot Pivot) validate(idx *Index) { if len(pivot) > idx.Arity() { panic("cannot iterate with a pivot whose size is superior to the index arity") } if idx.IsComposite() && !pivot.IsAny() { - // it's acceptable for the last pivot to just have a type and no value + // it's acceptable for the last pivot element to just have a type and no value hasValue := true for _, p := range pivot { // if on the previous pivot we have a value @@ -273,9 +271,18 @@ func (pivot Pivot) IsAny() bool { // If the given function returns an error, the iteration stops and returns that error. // If the pivot(s) is/are empty, starts from the beginning. // -// When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. -// When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element -// is of that type, without restriction on the type of the following elements. +// Valid pivots are: +// - zero value pivot +// - iterate on everything +// - n elements pivot (where n is the index arity) with each element having a value and a type +// - iterate starting at the closest index value +// - optionally, the last pivot element can have just a type and no value, which will scope the value of that element to that type +// - less than n elements pivot, with each element having a value and a type +// - iterate starting at the closest index value, using the first known value for missing elements +// - optionally, the last pivot element can have just a type and no value, which will scope the value of that element to that type +// - a single element with a type but nil value: will iterate on everything of that type +// +// Any other variation of a pivot are invalid and will panic. func (idx *Index) AscendGreaterOrEqual(pivot Pivot, fn func(val, key []byte) error) error { return idx.iterateOnStore(pivot, false, fn) } @@ -284,9 +291,18 @@ func (idx *Index) AscendGreaterOrEqual(pivot Pivot, fn func(val, key []byte) err // If the given function returns an error, the iteration stops and returns that error. // If the pivot(s) is/are empty, starts from the end. // -// When the index is simple (arity=1) and untyped, the pivot can have a nil value but a type; in that case, iteration will only yield values of that type. -// When the index is composite (arity>1) and untyped, the same logic applies, but only for the first pivot; iteration will only yield values whose first element -// is of that type, without restriction on the type of the following elements. +// Valid pivots are: +// - zero value pivot +// - iterate on everything +// - n elements pivot (where n is the index arity) with each element having a value and a type +// - iterate starting at the closest index value +// - optionally, the last pivot element can have just a type and no value, which will scope the value of that element to that type +// - less than n elements pivot, with each element having a value and a type +// - iterate starting at the closest index value, using the last known value for missing elements +// - optionally, the last pivot element can have just a type and no value, which will scope the value of that element to that type +// - a single element with a type but nil value: will iterate on everything of that type +// +// Any other variation of a pivot are invalid and will panic. func (idx *Index) DescendLessOrEqual(pivot Pivot, fn func(val, key []byte) error) error { return idx.iterateOnStore(pivot, true, fn) } @@ -418,7 +434,7 @@ func (idx *Index) buildSeek(pivot Pivot, reverse bool) ([]byte, error) { // if the index is without type and the first pivot is valueless but typed, iterate but filter out the types we don't want, // but just for the first pivot; subsequent pivot values cannot be filtered this way. - if idx.Info.Types[0] == 0 && pivot[0].Type != 0 && pivot[0].V == nil { + if idx.Info.Types[0].IsAny() && !pivot[0].Type.IsAny() && pivot[0].V == nil { seek = []byte{byte(pivot[0].Type)} if reverse { @@ -457,7 +473,6 @@ func (idx *Index) iterate(st engine.Store, pivot Pivot, reverse bool, fn func(it itm := it.Item() // If index is untyped and pivot first element is typed, only iterate on values with the same type as the first pivot - // TODO(JH) possible optimization, check for the other types if len(pivot) > 0 && idx.Info.Types[0].IsAny() && !pivot[0].Type.IsAny() && itm.Key()[0] != byte(pivot[0].Type) { return nil } diff --git a/planner/optimizer.go b/planner/optimizer.go index b41df2f7b..f398142e6 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -747,7 +747,7 @@ func operandCanUseIndex(indexType document.ValueType, path document.Path, fc dat func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) { var ranges stream.IndexRanges vb := document.NewValueBuffer() - // store in Operands of a given position + // store IN operands with their position (in the index paths) as a key inOperands := make(map[int]document.Array) for i, fno := range fnodes { @@ -771,7 +771,7 @@ func getRangesFromFilterNodes(fnodes []*filterNode) (stream.IndexRanges, error) } if len(inOperands) > 1 { - // TODO(JH) Github issue + // TODO FEATURE https://github.com/genjidb/genji/issues/392 panic("unsupported operation: multiple IN operators on a composite index") } From 9118425fdb48750cecf9355922c79acdeff451dc Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 28 Apr 2021 17:55:07 +0200 Subject: [PATCH 35/40] Remove unecessary statement --- database/index_test.go | 62 +++++++++++++++++++++--------------------- stream/iterator.go | 3 +- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/database/index_test.go b/database/index_test.go index 1b2bb0a31..0fc4eea2c 100644 --- a/database/index_test.go +++ b/database/index_test.go @@ -52,25 +52,25 @@ func TestIndexSet(t *testing.T) { }) t.Run(text+"Set two values and key succeeds (arity=2)", func(t *testing.T) { - idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) + idx, cleanup := getIndex(t, unique, document.AnyType, document.AnyType) defer cleanup() require.NoError(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) }) t.Run(text+"Set one value fails (arity=1)", func(t *testing.T) { - idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) + idx, cleanup := getIndex(t, unique, document.AnyType, document.AnyType) defer cleanup() require.Error(t, idx.Set(values(document.NewBoolValue(true)), []byte("key"))) }) t.Run(text+"Set two values fails (arity=1)", func(t *testing.T) { - idx, cleanup := getIndex(t, unique, document.ValueType(0)) + idx, cleanup := getIndex(t, unique, document.AnyType) defer cleanup() require.Error(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) }) t.Run(text+"Set three values fails (arity=2)", func(t *testing.T) { - idx, cleanup := getIndex(t, unique, document.ValueType(0), document.ValueType(0)) + idx, cleanup := getIndex(t, unique, document.AnyType, document.AnyType) defer cleanup() require.Error(t, idx.Set(values(document.NewBoolValue(true), document.NewBoolValue(true), document.NewBoolValue(true)), []byte("key"))) }) @@ -146,7 +146,7 @@ func TestIndexDelete(t *testing.T) { }) t.Run("Unique: false, Delete valid key succeeds (arity=2)", func(t *testing.T) { - idx, cleanup := getIndex(t, false, document.ValueType(0), document.ValueType(0)) + idx, cleanup := getIndex(t, false, document.AnyType, document.AnyType) defer cleanup() require.NoError(t, idx.Set(values(document.NewDoubleValue(10), document.NewDoubleValue(10)), []byte("key"))) @@ -213,7 +213,7 @@ func TestIndexDelete(t *testing.T) { }) t.Run("Unique: true, Delete valid key succeeds (arity=2)", func(t *testing.T) { - idx, cleanup := getIndex(t, true, document.ValueType(0), document.ValueType(0)) + idx, cleanup := getIndex(t, true, document.AnyType, document.AnyType) defer cleanup() require.NoError(t, idx.Set(values(document.NewIntegerValue(10), document.NewIntegerValue(10)), []byte("key1"))) @@ -537,7 +537,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // composite -------------------------------------------------- // composite indexes can have empty pivot values to iterate on the whole indexed data {name: "index=[any, untyped], vals=[int, int], pivot=[nil,nil]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{}, document.Value{}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -553,7 +553,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // composite indexes must have at least have one value if typed {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -562,7 +562,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { mustPanic: true, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -594,7 +594,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { mustPanic: true, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -603,7 +603,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { mustPanic: true, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -620,7 +620,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 5, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -638,7 +638,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 3, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -657,7 +657,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // pivot [2, int] should filter out [2, not(int)] {name: "index=[any, untyped], vals=[int, int], noise=[int, blob], pivot=[2, int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -681,7 +681,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // a more subtle case {name: "index=[any, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewBlobValue([]byte{byte('a' + uint8(i))})) @@ -700,7 +700,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { }, // partial pivot {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -714,7 +714,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { expectedCount: 10, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -730,7 +730,7 @@ func TestIndexAscendGreaterThan(t *testing.T) { // this is a tricky test, when we have multiple values but they share the first pivot element; // this is by definition a very implementation dependent test. {name: "index=[any, untyped], vals=[int, int], noise=int, bool], pivot=[int:0, int:0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1233,7 +1233,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // composite -------------------------------------------------- // composite indexes can have empty pivot values to iterate on the whole indexed data {name: "index=[any, untyped], vals=[int, int], pivot=[nil,nil]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{}, document.Value{}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1247,7 +1247,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 5, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1262,7 +1262,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // composite indexes cannot have values with type past the first element {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{Type: document.IntegerValue}, document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1278,7 +1278,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { mustPanic: true, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[int, 0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.Value{Type: document.IntegerValue}, document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1286,7 +1286,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { mustPanic: true, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0, 0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(0), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1298,7 +1298,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 0, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[5, 5]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(5), document.NewIntegerValue(5)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1316,7 +1316,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // [0,1], [1,2], --[2,0]--, [2,3], [3,4], [4,5] {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, 0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1335,7 +1335,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // [0,1], [1,2], [2,3], --[2,int]--, [3,4], [4,5] {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2, int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1355,7 +1355,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { // pivot [2, int] should filter out [2, not(int)] // [0,1], [1,2], [2,3], --[2,int]--, [2, text], [3,4], [3,text], [4,5], [4,text] {name: "index=[any, untyped], vals=[int, int], noise=[int, text], pivot=[2, int]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.Value{Type: document.IntegerValue}), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1378,7 +1378,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // a more subtle case {name: "index=[any, untyped], vals=[int, blob], noise=[blob, blob], pivot=[2, 'a']", // pivot is [2, a] but value is [2, c] but that must work anyway - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2), document.NewBlobValue([]byte{byte('a')})), val: func(i int) []document.Value { return values( @@ -1428,7 +1428,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { }, // partial pivot {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[0]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(0)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1442,7 +1442,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 2, // [0] is "equal" to [0, 1] and [0, "1"] }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[5]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(5)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) @@ -1456,7 +1456,7 @@ func TestIndexDescendLessOrEqual(t *testing.T) { expectedCount: 10, }, {name: "index=[any, untyped], vals=[int, int], noise=[blob, blob], pivot=[2]", - indexTypes: []document.ValueType{0, 0}, + indexTypes: []document.ValueType{document.AnyType, document.AnyType}, pivot: values(document.NewIntegerValue(2)), val: func(i int) []document.Value { return values(document.NewIntegerValue(int64(i)), document.NewIntegerValue(int64(i+1))) diff --git a/stream/iterator.go b/stream/iterator.go index 9b2f1cafa..7dbf124b6 100644 --- a/stream/iterator.go +++ b/stream/iterator.go @@ -347,8 +347,7 @@ func (it *IndexScanOperator) Iterate(in *expr.Environment, fn func(out *expr.Env // if there are no ranges use a simpler and faster iteration function if len(it.Ranges) == 0 { - vs := make([]document.Value, len(index.Info.Types)) - return iterator(vs, func(val, key []byte) error { + return iterator(nil, func(val, key []byte) error { d, err := table.GetDocument(key) if err != nil { return err From 54bb6dc18b7960b1dbbb96e0d3bf9ea99091c51b Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 28 Apr 2021 18:45:51 +0200 Subject: [PATCH 36/40] Use comp index partially when a path is missing --- planner/optimizer.go | 8 ++------ planner/optimizer_test.go | 11 +++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/planner/optimizer.go b/planner/optimizer.go index f398142e6..f4f17da21 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -576,12 +576,8 @@ outer: } } else { // if on the index idx_abc(a,b,c), a is found, b isn't but c is - if fno != nil { - // then idx_abc cannot be used, it's not possible to use the index without a value for b - continue outer - } else { - continue - } + // then idx_abc is valid but just with a, c will use a filter node instead + continue } usableFilterNodes = append(usableFilterNodes, fno) diff --git a/planner/optimizer_test.go b/planner/optimizer_test.go index e3986f98c..7a85c3534 100644 --- a/planner/optimizer_test.go +++ b/planner/optimizer_test.go @@ -619,6 +619,16 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { st.New(st.IndexScan("idx_foo_a_b_c", st.IndexRange{Min: testutil.MakeValueBuffer(t, `[1, 2]`), Exact: true})). Pipe(st.Filter(parser.MustParseExpr("k = 3"))), }, + // If a path is missing from the query, we can still the index, with paths after the missing one are + // using filter nodes rather than the index. + { + "FROM foo WHERE x = 1 AND z = 2", + st.New(st.SeqScan("foo")). + Pipe(st.Filter(parser.MustParseExpr("x = 1"))). + Pipe(st.Filter(parser.MustParseExpr("z = 2"))), + st.New(st.IndexScan("idx_foo_x_y_z", st.IndexRange{Min: newVB(document.NewIntegerValue(1)), Exact: true})). + Pipe(st.Filter(parser.MustParseExpr("z = 2"))), + }, { "FROM foo WHERE a = 1 AND c = 2", st.New(st.SeqScan("foo")). @@ -762,6 +772,7 @@ func TestUseIndexBasedOnSelectionNodeRule_Composite(t *testing.T) { CREATE UNIQUE INDEX idx_foo_c ON foo(c); CREATE INDEX idx_foo_a_d ON foo(a, d); CREATE INDEX idx_foo_a_b_c ON foo(a, b, c); + CREATE INDEX idx_foo_x_y_z ON foo(x, y, z); INSERT INTO foo (k, a, b, c, d) VALUES (1, 1, 1, 1, 1), (2, 2, 2, 2, 2), From e5bdba7fad274b7730c867a13ef83fb063e2e5c9 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 28 Apr 2021 18:55:19 +0200 Subject: [PATCH 37/40] Remove irrelevant code --- planner/optimizer.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/planner/optimizer.go b/planner/optimizer.go index f4f17da21..4a478c25a 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -398,7 +398,7 @@ type filterNode struct { // removing the now irrelevant filter nodes. // // TODO(asdine): add support for ORDER BY -// TODO(jh): clarify ranges and cost code in composite indexes case +// TODO(jh): clarify cost code in composite indexes case func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, params []expr.Param) (*stream.Stream, error) { // first we lookup for the seq scan node. // Here we will assume that at this point @@ -500,13 +500,9 @@ func UseIndexBasedOnFilterNodeRule(s *stream.Stream, tx *database.Transaction, p op := fno.f.E.(expr.Operator) return expr.IsEqualOperator(op) || expr.IsInOperator(op) } - isNodeComp := func(fno *filterNode, includeInOp bool) bool { + isNodeComp := func(fno *filterNode) bool { op := fno.f.E.(expr.Operator) - if includeInOp { - return expr.IsComparisonOperator(op) - } else { - return expr.IsComparisonOperator(op) && !expr.IsInOperator(op) && !expr.IsNotInOperator(op) - } + return expr.IsComparisonOperator(op) } // iterate on all indexes for that table, checking for each of them if its paths are matching @@ -552,14 +548,13 @@ outer: } } else { // the next node is the last one found, so the current one can also be a comparison and not just eq - if !isNodeComp(fno, false) { + if !isNodeComp(fno) { continue outer } } } else { // that's the last filter node, it can be a comparison, - // in the case of a potentially using a simple index, also a IN operator - if !isNodeComp(fno, len(found) == 1) { + if !isNodeComp(fno) { continue outer } } From 6357297adfc63bae35772fdb99c2720b607fd09f Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Wed, 28 Apr 2021 19:04:53 +0200 Subject: [PATCH 38/40] Reuse the same buffer when encoding idx values --- database/index.go | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/database/index.go b/database/index.go index 419602765..8331eb72c 100644 --- a/database/index.go +++ b/database/index.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "errors" + "io" "github.com/genjidb/genji/document" "github.com/genjidb/genji/engine" @@ -41,33 +42,32 @@ type Index struct { // type is prepended to the value. type indexValueEncoder struct { typ document.ValueType + w io.Writer } -func (e *indexValueEncoder) EncodeValue(v document.Value) ([]byte, error) { +func (e *indexValueEncoder) EncodeValue(v document.Value) error { // if the index has no type constraint, encode the value with its type if e.typ.IsAny() { - var buf bytes.Buffer - // prepend with the type - err := buf.WriteByte(byte(v.Type)) + _, err := e.w.Write([]byte{byte(v.Type)}) if err != nil { - return nil, err + return err } // marshal the value, if it exists, just return the type otherwise if v.V != nil { b, err := v.MarshalBinary() if err != nil { - return nil, err + return err } - _, err = buf.Write(b) + _, err = e.w.Write(b) if err != nil { - return nil, err + return err } } - return buf.Bytes(), nil + return nil } if v.Type != e.typ { @@ -80,11 +80,17 @@ func (e *indexValueEncoder) EncodeValue(v document.Value) ([]byte, error) { } if v.V == nil { - return nil, nil + return nil } // there is a type constraint, so a shorter form can be used as the type is always the same - return v.MarshalBinary() + b, err := v.MarshalBinary() + if err != nil { + return err + } + + _, err = e.w.Write(b) + return err } // NewIndex creates an index that associates values with a list of keys. @@ -375,13 +381,8 @@ func (idx *Index) EncodeValueBuffer(vb *document.ValueBuffer) ([]byte, error) { var buf bytes.Buffer err := vb.Iterate(func(i int, value document.Value) error { - enc := &indexValueEncoder{idx.Info.Types[i]} - b, err := enc.EncodeValue(value) - if err != nil { - return err - } - - _, err = buf.Write(b) + enc := &indexValueEncoder{typ: idx.Info.Types[i], w: &buf} + err := enc.EncodeValue(value) if err != nil { return err } From 76babcbd186aa7e8a7ff6fab6583cb7df180122b Mon Sep 17 00:00:00 2001 From: Jean Hadrien Chabran Date: Thu, 29 Apr 2021 11:04:28 +0200 Subject: [PATCH 39/40] Fix grammar and other minor edits Co-authored-by: Asdine El Hrychy --- cmd/genji/dbutil/dump_test.go | 2 +- database/catalog.go | 2 +- database/config.go | 2 +- database/index.go | 2 +- planner/optimizer.go | 3 ++- query/reindex_test.go | 3 +-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/genji/dbutil/dump_test.go b/cmd/genji/dbutil/dump_test.go index e43eebf7d..a9e0184f9 100644 --- a/cmd/genji/dbutil/dump_test.go +++ b/cmd/genji/dbutil/dump_test.go @@ -63,7 +63,7 @@ func TestDump(t *testing.T) { require.NoError(t, err) writeToBuf(q + "\n") - q = fmt.Sprintf(`CREATE INDEX idx_%s_b_c ON %s (a);`, table, table) + q = fmt.Sprintf(`CREATE INDEX idx_%s_b_c ON %s (b, c);`, table, table) err = db.Exec(q) require.NoError(t, err) writeToBuf(q + "\n") diff --git a/database/catalog.go b/database/catalog.go index 7b020e48e..a21299534 100644 --- a/database/catalog.go +++ b/database/catalog.go @@ -502,7 +502,7 @@ OUTER: for _, path := range info.Paths { for _, fc := range ti.FieldConstraints { if fc.Path.IsEqual(path) { - // a constraint may or may enforce a type + // a constraint may or may not enforce a type if fc.Type != 0 { info.Types = append(info.Types, document.ValueType(fc.Type)) } diff --git a/database/config.go b/database/config.go index e12966770..c97e5917b 100644 --- a/database/config.go +++ b/database/config.go @@ -235,7 +235,7 @@ type IndexInfo struct { // If set to true, values will be associated with at most one key. False by default. Unique bool - // If set, the index is typed and only accepts values of those types . + // If set, the index is typed and only accepts values of those types. Types []document.ValueType } diff --git a/database/index.go b/database/index.go index 8331eb72c..7af517195 100644 --- a/database/index.go +++ b/database/index.go @@ -120,7 +120,7 @@ func (idx *Index) IsComposite() bool { return len(idx.Info.Types) > 1 } -// Arity returns how many values the indexed is operating on. +// Arity returns how many values the index is operating on. // For example, an index created with `CREATE INDEX idx_a_b ON foo (a, b)` has an arity of 2. func (idx *Index) Arity() int { return len(idx.Info.Types) diff --git a/planner/optimizer.go b/planner/optimizer.go index 4a478c25a..8312df3e7 100644 --- a/planner/optimizer.go +++ b/planner/optimizer.go @@ -527,7 +527,8 @@ outer: // - given a query SELECT ... WHERE a = 1 AND b > 2 // - the paths a and b are contiguous in the index definition, this index can be used // - given a query SELECT ... WHERE a = 1 AND c > 2 - // - the paths a and c are not contiguous in the index definition, this index cannot be used + // - the paths a and c are not contiguous in the index definition, this index cannot be used for both values + // but it will be used with a and c with a normal filter node. var fops []*stream.FilterOperator var usableFilterNodes []*filterNode contiguous := true diff --git a/query/reindex_test.go b/query/reindex_test.go index 5fbe38103..a4a0721ed 100644 --- a/query/reindex_test.go +++ b/query/reindex_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/genjidb/genji" - "github.com/genjidb/genji/document" "github.com/stretchr/testify/require" ) @@ -82,7 +81,7 @@ func TestReIndex(t *testing.T) { } i := 0 - err = idx.AscendGreaterOrEqual([]document.Value{document.Value{}}, func(val []byte, key []byte) error { + err = idx.AscendGreaterOrEqual(nil, func(val []byte, key []byte) error { i++ return nil }) From d404b59582e52464d809af6202b6dcb4931d9b86 Mon Sep 17 00:00:00 2001 From: Jean-Hadrien Chabran Date: Thu, 29 Apr 2021 17:04:01 +0200 Subject: [PATCH 40/40] Fix Table operations with comp indexes --- database/table.go | 49 ++++++++++++++---------- database/table_test.go | 85 +++++++++++++++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 33 deletions(-) diff --git a/database/table.go b/database/table.go index f33296733..319b6f819 100644 --- a/database/table.go +++ b/database/table.go @@ -92,18 +92,18 @@ func (t *Table) Insert(d document.Document) (document.Document, error) { indexes := t.Indexes() for _, idx := range indexes { - vals := make([]document.Value, len(idx.Info.Paths)) + vs := make([]document.Value, 0, len(idx.Info.Paths)) - for i, path := range idx.Info.Paths { + for _, path := range idx.Info.Paths { v, err := path.GetValueFromDocument(fb) if err != nil { v = document.NewNullValue() } - vals[i] = v + vs = append(vs, v) } - err = idx.Set(vals, key) + err = idx.Set(vs, key) if err != nil { if err == ErrIndexDuplicateValue { return nil, ErrDuplicateDocument @@ -142,15 +142,21 @@ func (t *Table) Delete(key []byte) error { indexes := t.Indexes() for _, idx := range indexes { - values := make([]document.Value, len(idx.Info.Paths)) - for i, path := range idx.Info.Paths { - values[i], err = path.GetValueFromDocument(d) + vs := make([]document.Value, 0, len(idx.Info.Paths)) + for _, path := range idx.Info.Paths { + v, err := path.GetValueFromDocument(d) if err != nil { - return err + if err == document.ErrFieldNotFound { + v = document.NewNullValue() + } else { + return err + } } + + vs = append(vs, v) } - err = idx.Delete(values, key) + err = idx.Delete(vs, key) if err != nil { return err } @@ -188,15 +194,16 @@ func (t *Table) replace(indexes []*Index, key []byte, d document.Document) error // remove key from indexes for _, idx := range indexes { - values := make([]document.Value, len(idx.Info.Paths)) - for i, path := range idx.Info.Paths { - values[i], err = path.GetValueFromDocument(old) + vs := make([]document.Value, 0, len(idx.Info.Paths)) + for _, path := range idx.Info.Paths { + v, err := path.GetValueFromDocument(old) if err != nil { - values[i] = document.NewNullValue() + v = document.NewNullValue() } + vs = append(vs, v) } - err = idx.Delete(values, key) + err := idx.Delete(vs, key) if err != nil { return err } @@ -219,13 +226,17 @@ func (t *Table) replace(indexes []*Index, key []byte, d document.Document) error // update indexes for _, idx := range indexes { - // only support one path - v, err := idx.Info.Paths[0].GetValueFromDocument(d) - if err != nil { - v = document.NewNullValue() + vs := make([]document.Value, 0, len(idx.Info.Paths)) + for _, path := range idx.Info.Paths { + v, err := path.GetValueFromDocument(d) + if err != nil { + v = document.NewNullValue() + } + + vs = append(vs, v) } - err = idx.Set([]document.Value{v}, key) + err = idx.Set(vs, key) if err != nil { if err == ErrIndexDuplicateValue { return ErrDuplicateDocument diff --git a/database/table_test.go b/database/table_test.go index ebdbdf6a6..7438c99e6 100644 --- a/database/table_test.go +++ b/database/table_test.go @@ -639,18 +639,31 @@ func TestTableReplace(t *testing.T) { _, tx, cleanup := newTestTx(t) defer cleanup() - err := tx.CreateTable("test", nil) + err := tx.CreateTable("test1", nil) + require.NoError(t, err) + + err = tx.CreateTable("test2", nil) require.NoError(t, err) + // simple indexes err = tx.CreateIndex(&database.IndexInfo{ Paths: []document.Path{document.NewPath("a")}, Unique: true, - TableName: "test", + TableName: "test1", IndexName: "idx_foo_a", }) require.NoError(t, err) - tb, err := tx.GetTable("test") + // composite indexes + err = tx.CreateIndex(&database.IndexInfo{ + Paths: []document.Path{document.NewPath("x"), document.NewPath("y")}, + Unique: true, + TableName: "test2", + IndexName: "idx_foo_x_y", + }) + require.NoError(t, err) + + tb, err := tx.GetTable("test1") require.NoError(t, err) // insert two different documents @@ -659,26 +672,63 @@ func TestTableReplace(t *testing.T) { d2, err := tb.Insert(testutil.MakeDocument(t, `{"a": 2, "b": 2}`)) require.NoError(t, err) - before := testutil.GetIndexContent(t, tx, "idx_foo_a") + beforeIdxA := testutil.GetIndexContent(t, tx, "idx_foo_a") - // replace doc 1 without modifying indexed key + // --- a + // replace d1 without modifying indexed key err = tb.Replace(d1.(document.Keyer).RawKey(), testutil.MakeDocument(t, `{"a": 1, "b": 3}`)) require.NoError(t, err) - // index should be the same as before - require.Equal(t, before, testutil.GetIndexContent(t, tx, "idx_foo_a")) - // replace doc 2 and modify indexed key + // indexes should be the same as before + require.Equal(t, beforeIdxA, testutil.GetIndexContent(t, tx, "idx_foo_a")) + + // replace d2 and modify indexed key err = tb.Replace(d2.(document.Keyer).RawKey(), testutil.MakeDocument(t, `{"a": 3, "b": 3}`)) require.NoError(t, err) - // index should be different for doc 2 + + // indexes should be different for d2 got := testutil.GetIndexContent(t, tx, "idx_foo_a") - require.Equal(t, before[0], got[0]) - require.NotEqual(t, before[1], got[1]) + require.Equal(t, beforeIdxA[0], got[0]) + require.NotEqual(t, beforeIdxA[1], got[1]) - // replace doc 1 with duplicate indexed key + // replace d1 with duplicate indexed key err = tb.Replace(d1.(document.Keyer).RawKey(), testutil.MakeDocument(t, `{"a": 3, "b": 3}`)) + + // index should be the same as before + require.Equal(t, database.ErrDuplicateDocument, err) + + // --- x, y + tb, err = tx.GetTable("test2") + require.NoError(t, err) + // insert two different documents + dc1, err := tb.Insert(testutil.MakeDocument(t, `{"x": 1, "y": 1, "z": 1}`)) + require.NoError(t, err) + dc2, err := tb.Insert(testutil.MakeDocument(t, `{"x": 2, "y": 2, "z": 2}`)) + require.NoError(t, err) + + beforeIdxXY := testutil.GetIndexContent(t, tx, "idx_foo_x_y") + // replace dc1 without modifying indexed key + err = tb.Replace(dc1.(document.Keyer).RawKey(), testutil.MakeDocument(t, `{"x": 1, "y": 1, "z": 2}`)) + require.NoError(t, err) + + // index should be the same as before + require.Equal(t, beforeIdxXY, testutil.GetIndexContent(t, tx, "idx_foo_x_y")) + + // replace dc2 and modify indexed key + err = tb.Replace(dc2.(document.Keyer).RawKey(), testutil.MakeDocument(t, `{"x": 3, "y": 3, "z": 3}`)) + require.NoError(t, err) + + // indexes should be different for d2 + got = testutil.GetIndexContent(t, tx, "idx_foo_x_y") + require.Equal(t, beforeIdxXY[0], got[0]) + require.NotEqual(t, beforeIdxXY[1], got[1]) + + // replace dc2 with duplicate indexed key + err = tb.Replace(dc1.(document.Keyer).RawKey(), testutil.MakeDocument(t, `{"x": 3, "y": 3, "z": 3}`)) + // index should be the same as before require.Equal(t, database.ErrDuplicateDocument, err) + }) } @@ -751,7 +801,14 @@ func TestTableIndexes(t *testing.T) { require.NoError(t, err) err = tx.CreateIndex(&database.IndexInfo{ Unique: false, - IndexName: "ifx2a", + IndexName: "idx1ab", + TableName: "test1", + Paths: []document.Path{parsePath(t, "a"), parsePath(t, "b")}, + }) + require.NoError(t, err) + err = tx.CreateIndex(&database.IndexInfo{ + Unique: false, + IndexName: "idx2a", TableName: "test2", Paths: []document.Path{parsePath(t, "a")}, }) @@ -762,7 +819,7 @@ func TestTableIndexes(t *testing.T) { m := tb.Indexes() require.NoError(t, err) - require.Len(t, m, 2) + require.Len(t, m, 3) }) }