[full-ci] enhancement: add more kql spec tests and simplify ast normalization

changelog/unreleased/kql-search-query-language.md

@@ -19,7 +19,11 @@ Complex queries:
 
 https://github.com/owncloud/ocis/pull/7212
 https://github.com/owncloud/ocis/pull/7043
+https://github.com/owncloud/ocis/pull/7247
+https://github.com/owncloud/ocis/pull/7248
+https://github.com/owncloud/ocis/pull/7254
 https://github.com/owncloud/web/pull/9653
+https://github.com/owncloud/web/pull/9672
 https://github.com/owncloud/ocis/issues/7042
 https://github.com/owncloud/ocis/issues/7179
 https://github.com/owncloud/ocis/issues/7114

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/MicahParks/keyfunc v1.9.0
 	github.com/Nerzal/gocloak/v13 v13.8.0
+	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/bbalet/stopwords v1.0.0
 	github.com/blevesearch/bleve/v2 v2.3.9
 	github.com/coreos/go-oidc v2.2.1+incompatible
@@ -54,6 +55,7 @@ require (
 	github.com/libregraph/idm v0.4.1-0.20230221143410-3503963047a5
 	github.com/libregraph/lico v0.60.1-0.20230811070109-1d4140be554d
 	github.com/mitchellh/mapstructure v1.5.0
+	github.com/mna/pigeon v1.1.0
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/nats-io/nats-server/v2 v2.9.21
 	github.com/oklog/run v1.1.0

go.sum

@@ -859,6 +859,8 @@ github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4x
 github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
+github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
 github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -1640,6 +1642,7 @@ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
@@ -1697,6 +1700,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/mna/pigeon v1.1.0 h1:EjlvVbkGnNGemf8OrjeJX0nH8orujY/HkJgzJtd7kxc=
+github.com/mna/pigeon v1.1.0/go.mod h1:rkFeDZ0gc+YbnrXPw0q2RlI0QRuKBBPu67fgYIyGRNg=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -1861,6 +1866,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qq
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/riandyrn/otelchi v0.5.1 h1:0/45omeqpP7f/cvdL16GddQBfAEmZvUyl2QzLSE6uYo=
 github.com/riandyrn/otelchi v0.5.1/go.mod h1:ZxVxNEl+jQ9uHseRYIxKWRb3OY8YXFEu+EkNiiSNUEA=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=
 github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -1892,6 +1898,7 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
 github.com/sciencemesh/meshdirectory-web v1.0.4 h1:1YSctF6PAXhoHUYCaeRTj7rHaF7b3rYrZf2R0VXBIbo=
 github.com/sciencemesh/meshdirectory-web v1.0.4/go.mod h1:fJSThTS3xf+sTdL0iXQoaQJssLI7tn7DetHMHUl4SRk=
+github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
@@ -2525,6 +2532,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190830223141-573d9926052a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

ocis-pkg/tools/tools.go

@@ -1,7 +1,8 @@
-// +build tools
+//go:build tools
 
 package tools
 
 import (
+	_ "github.com/mna/pigeon"
 	_ "github.com/onsi/ginkgo/ginkgo"
 )

services/search/Makefile

@@ -29,7 +29,7 @@ ci-go-generate: $(PIGEON) $(MOCKERY) # CI runs ci-node-generate automatically be
 	$(MOCKERY) --dir pkg/content --output pkg/content/mocks --case underscore --name Extractor
 	$(MOCKERY) --dir pkg/content --output pkg/content/mocks --case underscore --name Retriever
 	$(MOCKERY) --dir pkg/search --output pkg/search/mocks --case underscore --name Searcher
-	$(PIGEON) -o pkg/query/kql/dictionary_gen.go pkg/query/kql/dictionary.peg
+	$(PIGEON) -optimize-grammar -optimize-parser -o pkg/query/kql/dictionary_gen.go pkg/query/kql/dictionary.peg
 
 .PHONY: ci-node-generate
 ci-node-generate:

services/search/pkg/query/ast/ast.go

@@ -73,3 +73,35 @@ type GroupNode struct {
 	Key   string
 	Nodes []Node
 }
+
+// NodeKey tries to return the node key
+func NodeKey(n Node) string {
+	switch node := n.(type) {
+	case *StringNode:
+		return node.Key
+	case *DateTimeNode:
+		return node.Key
+	case *BooleanNode:
+		return node.Key
+	case *GroupNode:
+		return node.Key
+	default:
+		return ""
+	}
+}
+
+// NodeValue tries to return the node key
+func NodeValue(n Node) interface{} {
+	switch node := n.(type) {
+	case *StringNode:
+		return node.Value
+	case *DateTimeNode:
+		return node.Value
+	case *BooleanNode:
+		return node.Value
+	case *GroupNode:
+		return node.Nodes
+	default:
+		return ""
+	}
+}

services/search/pkg/query/kql/cast.go

@@ -4,16 +4,11 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/araddon/dateparse"
+
 	"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
 )
 
-func toIfaceSlice(in interface{}) []interface{} {
-	if in == nil {
-		return nil
-	}
-	return in.([]interface{})
-}
-
 func toNode[T ast.Node](in interface{}) (T, error) {
 	var t T
 	out, ok := in.(T)
@@ -25,25 +20,27 @@ func toNode[T ast.Node](in interface{}) (T, error) {
 }
 
 func toNodes[T ast.Node](in interface{}) ([]T, error) {
-
 	switch v := in.(type) {
+	case []T:
+		return v, nil
+	case T:
+		return []T{v}, nil
 	case []interface{}:
-		var nodes []T
-
-		for _, el := range toIfaceSlice(v) {
-			node, err := toNode[T](el)
+		var ts []T
+		for _, inter := range v {
+			n, err := toNodes[T](inter)
 			if err != nil {
 				return nil, err
 			}
 
-			nodes = append(nodes, node)
+			ts = append(ts, n...)
 		}
-
-		return nodes, nil
-	case []T:
-		return v, nil
+		return ts, nil
+	case nil:
+		return nil, nil
 	default:
-		return nil, fmt.Errorf("can't convert '%T' to []ast.Node", in)
+		var t T
+		return nil, fmt.Errorf("can't convert '%T' to '%T'", in, t)
 	}
 }
 
@@ -77,5 +74,5 @@ func toTime(in interface{}) (time.Time, error) {
 		return time.Time{}, err
 	}
 
-	return time.Parse(time.RFC3339Nano, ts)
+	return dateparse.ParseLocal(ts)
 }

services/search/pkg/query/kql/connect.go

@@ -0,0 +1,129 @@
+package kql
+
+import (
+	"strings"
+
+	"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
+)
+
+// connectNodes connects given nodes
+func connectNodes(c Connector, nodes ...ast.Node) []ast.Node {
+	var connectedNodes []ast.Node
+
+	for i := range nodes {
+		ri := len(nodes) - 1 - i
+		head := nodes[ri]
+
+		if connectionNodes := connectNode(c, head, connectedNodes...); len(connectionNodes) > 0 {
+			connectedNodes = append(connectionNodes, connectedNodes...)
+		}
+
+		connectedNodes = append([]ast.Node{head}, connectedNodes...)
+	}
+
+	return connectedNodes
+}
+
+// connectNode connects a tip node with the rest
+func connectNode(c Connector, headNode ast.Node, tailNodes ...ast.Node) []ast.Node {
+	var nearestNeighborNode ast.Node
+	var nearestNeighborOperators []*ast.OperatorNode
+
+l:
+	for _, tailNode := range tailNodes {
+		switch node := tailNode.(type) {
+		case *ast.OperatorNode:
+			nearestNeighborOperators = append(nearestNeighborOperators, node)
+		default:
+			nearestNeighborNode = node
+			break l
+		}
+	}
+
+	if nearestNeighborNode == nil {
+		return nil
+	}
+
+	return c.Connect(headNode, nearestNeighborNode, nearestNeighborOperators)
+}
+
+// Connector is responsible to decide what node connections are needed
+type Connector interface {
+	Connect(head ast.Node, neighbor ast.Node, connections []*ast.OperatorNode) []ast.Node
+}
+
+// DefaultConnector is the default node connector
+type DefaultConnector struct {
+	sameKeyOPValue string
+}
+
+// Connect implements the Connector interface and is used to connect the nodes using
+// the default logic defined by the kql spec.
+func (c DefaultConnector) Connect(head ast.Node, neighbor ast.Node, connections []*ast.OperatorNode) []ast.Node {
+	switch head.(type) {
+	case *ast.OperatorNode:
+		return nil
+	}
+
+	headKey := strings.ToLower(ast.NodeKey(head))
+	neighborKey := strings.ToLower(ast.NodeKey(neighbor))
+
+	connection := &ast.OperatorNode{
+		Base:  &ast.Base{Loc: &ast.Location{Source: &[]string{"implicitly operator"}[0]}},
+		Value: BoolAND,
+	}
+
+	// if the current node and the neighbor node have the same key
+	// the connection is of type OR
+	//
+	// spec: same
+	//		author:"John Smith" author:"Jane Smith"
+	//		author:"John Smith" OR author:"Jane Smith"
+	//
+	// if the nodes have NO key, the edge is a AND connection
+	//
+	// spec: same
+	//		cat dog
+	//		cat AND dog
+	// from the spec:
+	// 		To construct complex queries, you can combine multiple
+	// 		free-text expressions with KQL query operators.
+	// 		If there are multiple free-text expressions without any
+	// 		operators in between them, the query behavior is the same
+	// 		as using the AND operator.
+	//
+	// nodes inside of group node are handled differently,
+	// if no explicit operator given, it uses AND
+	//
+	// spec: same
+	// 		author:"John Smith" AND author:"Jane Smith"
+	// 		author:("John Smith" "Jane Smith")
+	if headKey == neighborKey && headKey != "" && neighborKey != "" {
+		connection.Value = c.sameKeyOPValue
+	}
+
+	// decisions based on nearest neighbor operators
+	for i, node := range connections {
+		// consider direct neighbor operator only
+		if i == 0 {
+			// no connection is necessary here because an `AND` or `OR` edge is already present
+			// exit
+			for _, skipValue := range []string{BoolOR, BoolAND} {
+				if node.Value == skipValue {
+					return nil
+				}
+			}
+
+			// if neighbor node negotiates, an AND edge is needed
+			//
+			// spec: same
+			// 		cat -dog
+			// 		cat AND NOT dog
+			if node.Value == BoolNOT {
+				connection.Value = BoolAND
+			}
+		}
+	}
+
+	return []ast.Node{connection}
+}