Support for composite indexes

.gitignore

@@ -26,3 +26,5 @@ cmd/genji/genji
 
 # VS Code config
 .vscode/
+
+gopls.log

cmd/genji/dbutil/dump.go

@@ -210,8 +210,12 @@ func dumpSchema(tx *genji.Tx, w io.Writer, tableName string) error {
 			u = " UNIQUE"
 		}
 
-		_, err = fmt.Fprintf(w, "CREATE%s INDEX %s ON %s (%s);\n", u, index.Info.IndexName, index.Info.TableName,
-			index.Info.Path)
+		var paths []string
+		for _, path := range index.Info.Paths {
+			paths = append(paths, path.String())
+		}
+
+		_, err = fmt.Fprintf(w, "CREATE%s INDEX %s ON %s (%s);\n", u, index.Info.IndexName, index.Info.TableName, strings.Join(paths, ", "))
 		if err != nil {
 			return err
 		}

cmd/genji/dbutil/dump_test.go

@@ -58,22 +58,27 @@ func TestDump(t *testing.T) {
 				require.NoError(t, err)
 				writeToBuf(q + "\n")
 
-				q = fmt.Sprintf(`CREATE INDEX idx_a_%s ON %s (a);`, table, table)
+				q = fmt.Sprintf(`CREATE INDEX idx_%s_a ON %s (a);`, table, table)
+				err = db.Exec(q)
+				require.NoError(t, err)
+				writeToBuf(q + "\n")
+
+				q = fmt.Sprintf(`CREATE INDEX idx_%s_b_c ON %s (b, c);`, table, table)
 				err = db.Exec(q)
 				require.NoError(t, err)
 				writeToBuf(q + "\n")
 
-				q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d};`, table, 1, 2)
+				q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d, "c": %d};`, table, 1, 2, 3)
 				err = db.Exec(q)
 				require.NoError(t, err)
 				writeToBuf(q + "\n")
 
-				q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d};`, table, 2, 2)
+				q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d, "c": %d};`, table, 2, 2, 2)
 				err = db.Exec(q)
 				require.NoError(t, err)
 				writeToBuf(q + "\n")
 
-				q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d};`, table, 3, 2)
+				q = fmt.Sprintf(`INSERT INTO %s VALUES {"a": %d, "b": %d, "c": %d};`, table, 3, 2, 1)
 				err = db.Exec(q)
 				require.NoError(t, err)
 				writeToBuf(q + "\n")

cmd/genji/shell/command.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/genjidb/genji"
 	"github.com/genjidb/genji/cmd/genji/dbutil"
@@ -126,7 +127,12 @@ func runIndexesCmd(db *genji.DB, tableName string, w io.Writer) error {
 				return err
 			}
 
-			fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, index.Path)
+			var paths []string
+			for _, path := range index.Paths {
+				paths = append(paths, path.String())
+			}
+
+			fmt.Fprintf(w, "%s ON %s (%s)\n", index.IndexName, index.TableName, strings.Join(paths, ", "))
 
 			return nil
 		})

cmd/genji/shell/command_test.go

@@ -60,7 +60,7 @@ func TestIndexesCmd(t *testing.T) {
 		want      string
 		fails     bool
 	}{
-		{"All", "", "idx_bar_a ON bar (a)\nidx_foo_a ON foo (a)\nidx_foo_b ON foo (b)\n", false},
+		{"All", "", "idx_bar_a_b ON bar (a, b)\nidx_foo_a ON foo (a)\nidx_foo_b ON foo (b)\n", false},
 		{"With table name", "foo", "idx_foo_a ON foo (a)\nidx_foo_b ON foo (b)\n", false},
 		{"With nonexistent table name", "baz", "", true},
 	}
@@ -76,7 +76,7 @@ func TestIndexesCmd(t *testing.T) {
 				CREATE INDEX idx_foo_a ON foo (a);
 				CREATE INDEX idx_foo_b ON foo (b);
 				CREATE TABLE bar;
-				CREATE INDEX idx_bar_a ON bar (a);
+				CREATE INDEX idx_bar_a_b ON bar (a, b);
 			`)
 			require.NoError(t, err)
 
@@ -117,7 +117,7 @@ func TestSaveCommand(t *testing.T) {
 
 			err = db.Exec(`
 				CREATE TABLE test (a DOUBLE);
-				CREATE INDEX idx_a ON test (a);
+				CREATE INDEX idx_a_b ON test (a, b);
 			`)
 			require.NoError(t, err)
 			err = db.Exec("INSERT INTO test (a, b) VALUES (?, ?)", 1, 2)
@@ -160,7 +160,11 @@ func TestSaveCommand(t *testing.T) {
 			err = db.View(func(tx *genji.Tx) error {
 				indexes := tx.ListIndexes()
 				require.Len(t, indexes, 1)
-				require.Equal(t, "idx_a", indexes[0])
+				require.Equal(t, "idx_a_b", indexes[0])
+
+				index, err := tx.GetIndex("idx_a_b")
+				require.NoError(t, err)
+				require.Equal(t, []document.ValueType{document.DoubleValue, 0}, index.Info.Types)
 
 				return nil
 			})
@@ -172,12 +176,12 @@ func TestSaveCommand(t *testing.T) {
 
 			defer tx.Rollback()
 
-			idx, err := tx.GetIndex("idx_a")
+			idx, err := tx.GetIndex("idx_a_b")
 			require.NoError(t, err)
 
 			// check that by iterating through the index and finding the previously inserted values
 			var i int
-			err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error {
+			err = idx.AscendGreaterOrEqual([]document.Value{document.NewDoubleValue(0)}, func(v, k []byte) error {
 				i++
 				return nil
 			})

database/catalog.go

@@ -325,15 +325,19 @@ func (c *Catalog) ReIndex(tx *Transaction, indexName string) error {
 
 func (c *Catalog) buildIndex(tx *Transaction, idx *Index, table *Table) error {
 	return table.Iterate(func(d document.Document) error {
-		v, err := idx.Info.Path.GetValueFromDocument(d)
-		if err == document.ErrFieldNotFound {
-			return nil
-		}
-		if err != nil {
-			return err
+		var err error
+		values := make([]document.Value, len(idx.Info.Paths))
+		for i, path := range idx.Info.Paths {
+			values[i], err = path.GetValueFromDocument(d)
+			if err == document.ErrFieldNotFound {
+				return nil
+			}
+			if err != nil {
+				return err
+			}
 		}
 
-		err = idx.Set(v, d.(document.Keyer).RawKey())
+		err = idx.Set(values, d.(document.Keyer).RawKey())
 		if err != nil {
 			return stringutil.Errorf("error while building the index: %w", err)
 		}
@@ -490,16 +494,25 @@ func (c *catalogCache) AddIndex(tx *Transaction, info *IndexInfo) error {
 		return ErrTableNotFound
 	}
 
-	// if the index is created on a field on which we know the type,
-	// create a typed index.
-	for _, fc := range ti.FieldConstraints {
-		if fc.Path.IsEqual(info.Path) {
-			if fc.Type != 0 {
-				info.Type = fc.Type
-			}
+	// if the index is created on a field on which we know the type then create a typed index.
+	// if the given info contained existing types, they are overriden.
+	info.Types = nil
+
+OUTER:
+	for _, path := range info.Paths {
+		for _, fc := range ti.FieldConstraints {
+			if fc.Path.IsEqual(path) {
+				// a constraint may or may not enforce a type
+				if fc.Type != 0 {
+					info.Types = append(info.Types, document.ValueType(fc.Type))
+				}
 
-			break
+				continue OUTER
+			}
 		}
+
+		// no type was inferred for that path, add it to the index as untyped
+		info.Types = append(info.Types, document.ValueType(0))
 	}
 
 	c.indexes[info.IndexName] = info

database/catalog_test.go

@@ -115,9 +115,9 @@ func TestCatalogTable(t *testing.T) {
 			err := catalog.CreateTable(tx, "foo", ti)
 			require.NoError(t, err)
 
-			err = catalog.CreateIndex(tx, &database.IndexInfo{Path: parsePath(t, "gender"), IndexName: "idx_gender", TableName: "foo"})
+			err = catalog.CreateIndex(tx, &database.IndexInfo{Paths: []document.Path{parsePath(t, "gender")}, IndexName: "idx_gender", TableName: "foo"})
 			require.NoError(t, err)
-			err = catalog.CreateIndex(tx, &database.IndexInfo{Path: parsePath(t, "city"), IndexName: "idx_city", TableName: "foo", Unique: true})
+			err = catalog.CreateIndex(tx, &database.IndexInfo{Paths: []document.Path{parsePath(t, "city")}, IndexName: "idx_city", TableName: "foo", Unique: true})
 			require.NoError(t, err)
 
 			return nil
@@ -318,15 +318,15 @@ func TestTxCreateIndex(t *testing.T) {
 
 		update(t, db, func(tx *database.Transaction) error {
 			err := catalog.CreateIndex(tx, &database.IndexInfo{
-				IndexName: "idx_a", TableName: "test", Path: parsePath(t, "a"),
+				IndexName: "idx_a", TableName: "test", Paths: []document.Path{parsePath(t, "a")},
 			})
 			require.NoError(t, err)
 			idx, err := tx.GetIndex("idx_a")
 			require.NoError(t, err)
 			require.NotNil(t, idx)
 
 			var i int
-			err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error {
+			err = idx.AscendGreaterOrEqual(values(document.Value{Type: document.DoubleValue}), func(v, k []byte) error {
 				var buf bytes.Buffer
 				err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i)))
 				require.NoError(t, err)
@@ -355,12 +355,12 @@ func TestTxCreateIndex(t *testing.T) {
 
 		update(t, db, func(tx *database.Transaction) error {
 			err := catalog.CreateIndex(tx, &database.IndexInfo{
-				IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"),
+				IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")},
 			})
 			require.NoError(t, err)
 
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
-				IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"),
+				IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")},
 			})
 			require.Equal(t, database.ErrIndexAlreadyExists, err)
 			return nil
@@ -373,7 +373,7 @@ func TestTxCreateIndex(t *testing.T) {
 		catalog := db.Catalog()
 		update(t, db, func(tx *database.Transaction) error {
 			err := catalog.CreateIndex(tx, &database.IndexInfo{
-				IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"),
+				IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")},
 			})
 			if !errors.Is(err, database.ErrTableNotFound) {
 				require.Equal(t, err, database.ErrTableNotFound)
@@ -394,7 +394,7 @@ func TestTxCreateIndex(t *testing.T) {
 
 		update(t, db, func(tx *database.Transaction) error {
 			err := catalog.CreateIndex(tx, &database.IndexInfo{
-				TableName: "test", Path: parsePath(t, "foo"),
+				TableName: "test", Paths: []document.Path{parsePath(t, "foo")},
 			})
 			require.NoError(t, err)
 
@@ -403,7 +403,7 @@ func TestTxCreateIndex(t *testing.T) {
 
 			// create another one
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
-				TableName: "test", Path: parsePath(t, "foo"),
+				TableName: "test", Paths: []document.Path{parsePath(t, "foo")},
 			})
 			require.NoError(t, err)
 
@@ -424,11 +424,11 @@ func TestTxDropIndex(t *testing.T) {
 			err := catalog.CreateTable(tx, "test", nil)
 			require.NoError(t, err)
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
-				IndexName: "idxFoo", TableName: "test", Path: parsePath(t, "foo"),
+				IndexName: "idxFoo", TableName: "test", Paths: []document.Path{parsePath(t, "foo")},
 			})
 			require.NoError(t, err)
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
-				IndexName: "idxBar", TableName: "test", Path: parsePath(t, "bar"),
+				IndexName: "idxBar", TableName: "test", Paths: []document.Path{parsePath(t, "bar")},
 			})
 			require.NoError(t, err)
 			return nil
@@ -489,13 +489,13 @@ func TestCatalogReIndex(t *testing.T) {
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
 				IndexName: "a",
 				TableName: "test",
-				Path:      parsePath(t, "a"),
+				Paths:     []document.Path{parsePath(t, "a")},
 			})
 			require.NoError(t, err)
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
 				IndexName: "b",
 				TableName: "test",
-				Path:      parsePath(t, "b"),
+				Paths:     []document.Path{parsePath(t, "b")},
 			})
 			require.NoError(t, err)
 
@@ -537,7 +537,7 @@ func TestCatalogReIndex(t *testing.T) {
 			return catalog.CreateIndex(tx, &database.IndexInfo{
 				IndexName: "b",
 				TableName: "test",
-				Path:      parsePath(t, "b"),
+				Paths:     []document.Path{parsePath(t, "b")},
 			})
 		})
 
@@ -570,7 +570,7 @@ func TestCatalogReIndex(t *testing.T) {
 			require.NoError(t, err)
 
 			var i int
-			err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error {
+			err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error {
 				var buf bytes.Buffer
 				err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i)))
 				require.NoError(t, err)
@@ -638,13 +638,13 @@ func TestReIndexAll(t *testing.T) {
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
 				IndexName: "t1a",
 				TableName: "test1",
-				Path:      parsePath(t, "a"),
+				Paths:     []document.Path{parsePath(t, "a")},
 			})
 			require.NoError(t, err)
 			err = catalog.CreateIndex(tx, &database.IndexInfo{
 				IndexName: "t2a",
 				TableName: "test2",
-				Path:      parsePath(t, "a"),
+				Paths:     []document.Path{parsePath(t, "a")},
 			})
 			require.NoError(t, err)
 
@@ -660,7 +660,7 @@ func TestReIndexAll(t *testing.T) {
 			require.NoError(t, err)
 
 			var i int
-			err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error {
+			err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error {
 				var buf bytes.Buffer
 				err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i)))
 				require.NoError(t, err)
@@ -676,7 +676,7 @@ func TestReIndexAll(t *testing.T) {
 			require.NoError(t, err)
 
 			i = 0
-			err = idx.AscendGreaterOrEqual(document.Value{Type: document.DoubleValue}, func(v, k []byte) error {
+			err = idx.AscendGreaterOrEqual([]document.Value{document.Value{Type: document.DoubleValue}}, func(v, k []byte) error {
 				var buf bytes.Buffer
 				err = document.NewValueEncoder(&buf).Encode(document.NewDoubleValue(float64(i)))
 				require.NoError(t, err)
@@ -708,5 +708,5 @@ func TestReadOnlyTables(t *testing.T) {
 	doc, err = db.QueryDocument(`CREATE INDEX idx_foo_a ON foo(a); SELECT * FROM __genji_indexes`)
 	require.NoError(t, err)
 
-	testutil.RequireDocJSONEq(t, doc, `{"index_name":"idx_foo_a", "path":["a"], "table_name":"foo", "unique":false}`)
+	testutil.RequireDocJSONEq(t, doc, `{"index_name":"idx_foo_a", "paths":[["a"]], "table_name":"foo", "types":[0], "unique":false}`)
 }