Skip to content

Commit

Permalink
New implementation for Syntax option #1
Browse files Browse the repository at this point in the history 
Signed-off-by: forestmvey <[email protected]>
  • Loading branch information
@forestmvey
forestmvey committed May 15, 2023
1 parent afd0540 commit 68821a3
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 119 deletions.
24 changes: 0 additions & 24 deletions core/src/main/java/org/opensearch/sql/analysis/Analyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,36 +220,13 @@ public LogicalPlan visitLimit(Limit node, AnalysisContext context) {
public LogicalPlan visitFilter(Filter node, AnalysisContext context) {
LogicalPlan child = node.getChild().get(0).accept(this, context);
Expression condition = expressionAnalyzer.analyze(node.getCondition(), context);
verifySupportsCondition(condition);

ExpressionReferenceOptimizer optimizer =
new ExpressionReferenceOptimizer(expressionAnalyzer.getRepository(), child);
Expression optimized = optimizer.optimize(condition, context);
return new LogicalFilter(child, optimized);
}

/**
* Ensure NESTED function is not used in WHERE, GROUP BY, and HAVING clauses.
* Fallback to legacy engine. Can remove when support is added for NESTED function in WHERE,
* GROUP BY, ORDER BY, and HAVING clauses.
* @param condition : Filter condition
*/
private void verifySupportsCondition(Expression condition) {
if (condition instanceof FunctionExpression) {
if (((FunctionExpression) condition).getFunctionName().getFunctionName().equalsIgnoreCase(
BuiltinFunctionName.NESTED.name()
)) {
throw new SyntaxCheckException(
"Falling back to legacy engine. Nested function is not supported in WHERE,"
+ " GROUP BY, and HAVING clauses."
);
}
((FunctionExpression)condition).getArguments().stream()
.forEach(e -> verifySupportsCondition(e)
);
}
}

/**
* Build {@link LogicalRename}.
*/
Expand Down Expand Up @@ -300,7 +277,6 @@ public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) {

for (UnresolvedExpression expr : node.getGroupExprList()) {
NamedExpression resolvedExpr = namedExpressionAnalyzer.analyze(expr, context);
verifySupportsCondition(resolvedExpr.getDelegated());
groupbyBuilder.add(resolvedExpr);
}
ImmutableList<NamedExpression> groupBys = groupbyBuilder.build();
Expand Down
37 changes: 37 additions & 0 deletions ...src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.opensearch.sql.expression.Expression;
import org.opensearch.sql.expression.ExpressionNodeVisitor;
import org.opensearch.sql.expression.FunctionExpression;
import org.opensearch.sql.expression.LiteralExpression;
import org.opensearch.sql.expression.ReferenceExpression;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.FunctionName;
Expand Down Expand Up @@ -123,6 +124,10 @@ public QueryBuilder visitFunction(FunctionExpression func, Object context) {

// example: WHERE nested(foo.bar, nested(zoo.blah, condition))

if (func.getArguments().size() == 1) { // Syntax: nested(field | field, path) OPERATOR LITERAL
LuceneQuery query = luceneQueries.get(name);
return query.build(func);
}
if (func.getArguments().size() > 1) {
Expression secondArgument = func.getArguments().get(1);
if (secondArgument instanceof FunctionExpression) {
Expand All @@ -140,11 +145,43 @@ public QueryBuilder visitFunction(FunctionExpression func, Object context) {
if (query != null && query.canSupport(func)) {
return query.build(func);
}
if (query != null && query.isNestedFunction(func)) {
QueryBuilder outerQuery = query.buildNested(func);
boolean hasPathParam = (((FunctionExpression)func.getArguments().get(0)).getArguments().size() == 2);
String pathStr = !hasPathParam ?
getNestedPathString((ReferenceExpression) ((FunctionExpression)func.getArguments().get(0)).getArguments().get(0)) :
((FunctionExpression)func.getArguments().get(0)).getArguments().get(0).toString();
NestedQuery innerQuery = (NestedQuery) luceneQueries.get(((FunctionExpression)func.getArguments().get(0)).getFunctionName());
return innerQuery.adInnerQuery(outerQuery, pathStr);
}

// Nested used in predicate expression with syntax 'WHERE nested(field | field, path) = ...'
if (func.getArguments().get(0) instanceof FunctionExpression
&& ((FunctionExpression)func.getArguments().get(0)).getFunctionName().getFunctionName().equalsIgnoreCase(BuiltinFunctionName.NESTED.name())) {
LuceneQuery innerQuery = luceneQueries.get(((FunctionExpression)func.getArguments().get(0)).getFunctionName());
return innerQuery.build(func);
// containsInnerNestedQuery()
// innerQuery.buildPredicateExpression()
} else if (query instanceof NestedQuery) {
// TODO Throw exception if does not have conditional parameter.
return query.build(func);
}
return buildScriptQuery(func);
}
}
}

private boolean funcArgsIsPredicateExpression(FunctionExpression func) {
func.getArguments().stream().forEach(
a -> {
if (a instanceof FunctionExpression) {
funcArgsIsPredicateExpression((FunctionExpression) a);
}
}
);
return false;
}

private String getNestedPathString(ReferenceExpression field) {
String ret = "";
for (int i = 0; i < field.getPaths().size() - 1; i++) {
Expand Down
16 changes: 13 additions & 3 deletions ...src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/LuceneQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import com.google.common.collect.ImmutableMap;

import java.sql.Ref;
import java.util.ArrayList;
import java.util.Map;
import java.util.function.BiFunction;
Expand Down Expand Up @@ -58,11 +59,10 @@ public boolean canSupport(FunctionExpression func) {
&& (func.getArguments().get(0) instanceof ReferenceExpression)
&& (func.getArguments().get(1) instanceof LiteralExpression
|| literalExpressionWrappedByCast(func))
|| isMultiParameterQuery(func)
|| isNestedFunction(func);
|| isMultiParameterQuery(func);
}

private boolean isNestedFunction(FunctionExpression func) {
public boolean isNestedFunction(FunctionExpression func) {
return ((func.getArguments().get(0) instanceof FunctionExpression
&& ((FunctionExpression)func.getArguments().get(0)).getFunctionName().getFunctionName().equalsIgnoreCase("nested"))
|| func.getFunctionName().getFunctionName().equalsIgnoreCase("nested"));
Expand Down Expand Up @@ -110,6 +110,16 @@ public QueryBuilder build(FunctionExpression func) {
return doBuild(ref.getAttr(), ref.type(), literalValue);
}

public QueryBuilder buildNested(FunctionExpression func) {
FunctionExpression ref = (FunctionExpression) func.getArguments().get(0);
Expression expr = func.getArguments().get(1);
ExprValue literalValue = expr instanceof LiteralExpression ? expr
.valueOf() : cast((FunctionExpression) expr);

ReferenceExpression funcExpr = (ReferenceExpression) ref.getArguments().get(0);

return doBuild(funcExpr.getAttr(), ref.type(), literalValue);
}

private ExprValue cast(FunctionExpression castFunction) {
return castMap.get(castFunction.getFunctionName()).apply(
Expand Down
105 changes: 13 additions & 92 deletions ...src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/NestedQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,7 @@ public class NestedQuery extends LuceneQuery {
@Override
public QueryBuilder build(FunctionExpression func) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
// WHERE nested(message, message.info = '' AND comment.data = '')
if (func.getFunctionName().getFunctionName().equalsIgnoreCase(BuiltinFunctionName.NESTED.name())) {
switch (((FunctionExpression)func.getArguments().get(1)).getFunctionName().getFunctionName()) {
case "and":
applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::must, boolQuery);
break;
case "or":
applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::should, boolQuery);
break;
case "not":
applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::mustNot, boolQuery);
break;
default:
applyInnerQueryOrRecurse(((FunctionExpression)func.getArguments().get(1)), BoolQueryBuilder::filter, boolQuery);
}
// No recursion for operators needed
} else if (func.getArguments().get(0) instanceof ReferenceExpression) { // TODO can this be handled in just the else?
if (func.getArguments().get(0) instanceof ReferenceExpression) { // TODO can this be handled in just the else?
applyInnerQuery(func, BoolQueryBuilder::filter, boolQuery);
} else if (func.getArguments().get(0) instanceof FunctionExpression &&
((FunctionExpression)func.getArguments().get(0)).getFunctionName().getFunctionName().equalsIgnoreCase("nested")) { // Is predicate expression
Expand All @@ -53,85 +37,22 @@ public QueryBuilder build(FunctionExpression func) {
return boolQuery;
}

private QueryBuilder applyInnerQueryOrRecurse(FunctionExpression func, BiFunction<BoolQueryBuilder, QueryBuilder,
QueryBuilder> accumulator, BoolQueryBuilder boolQuery) {
if (func.getFunctionName().getFunctionName().equalsIgnoreCase("nested")) {
ReferenceExpression nestedPath = (ReferenceExpression) func.getArguments().get(0);
ReferenceExpression nestedField = (ReferenceExpression) ((FunctionExpression)func.getArguments().get(1)).getArguments().get(0);
ExprValue literal = ((FunctionExpression)func.getArguments().get(1)).getArguments().get(1).valueOf();
String fieldName = convertTextToKeyword(nestedField.toString(), nestedField.type());
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(literal));
NestedQueryBuilder ret = QueryBuilders.nestedQuery(nestedPath.toString(), termQuery, ScoreMode.None);
return ret;
} else if (func.getArguments().get(0) instanceof ReferenceExpression) {
// innerNestedQuery
// nested(message, message.info = 'a' AND comment.data = 'ab' OR message.dayOfWeek = 1)
// nested(message, message.info = 'a' AND message.dayOfWeek = 1)
ReferenceExpression field = (ReferenceExpression)func.getArguments().get(0);
String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type?
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf()));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None);
return nestedQueryBuilder;
} else if (func.getArguments().get(1) instanceof LiteralExpression) { // Syntax: 'WHERE nested(message.info) = 'a'
ReferenceExpression field = (ReferenceExpression)((FunctionExpression)func.getArguments().get(0)).getArguments().get(0);
String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type?
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf()));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None);
return nestedQueryBuilder;
} else { // Syntax: recursion...
for (Expression arg : func.getArguments()) {
if (arg instanceof FunctionExpression) {
switch (((FunctionExpression)arg).getFunctionName().getFunctionName()) {
case "and":
accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::must, QueryBuilders.boolQuery()));
break;
case "or":
accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::should, QueryBuilders.boolQuery()));
break;
case "not":
accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::mustNot, QueryBuilders.boolQuery()));
break;
default:
accumulator.apply(boolQuery, applyInnerQueryOrRecurse(((FunctionExpression)arg), BoolQueryBuilder::filter, QueryBuilders.boolQuery()));
}
} else {
// Throw exception? Shouldn't get here.
}
}
}
return boolQuery;
public QueryBuilder adInnerQuery(QueryBuilder builder, String path) {
// BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(path, builder, ScoreMode.None);
return nestedQueryBuilder;
}

private QueryBuilder applyInnerQuery(FunctionExpression func, BiFunction<BoolQueryBuilder, QueryBuilder,
QueryBuilder> accumulator, BoolQueryBuilder boolQuery) {
if (func.getFunctionName().getFunctionName().equalsIgnoreCase("nested")) { // Not sure if we need this case anymore.
ReferenceExpression nestedPath = (ReferenceExpression) func.getArguments().get(0);
ReferenceExpression nestedField = (ReferenceExpression) ((FunctionExpression)func.getArguments().get(1)).getArguments().get(0);
ExprValue literal = ((FunctionExpression)func.getArguments().get(1)).getArguments().get(1).valueOf();
String fieldName = convertTextToKeyword(nestedField.toString(), nestedField.type());
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(literal));
NestedQueryBuilder ret = QueryBuilders.nestedQuery(nestedPath.toString(), termQuery, ScoreMode.None);
return accumulator.apply(boolQuery, ret);
} else if (func.getArguments().get(0) instanceof ReferenceExpression) {
// innerNestedQuery
// nested(message, message.info = 'a' AND comment.data = 'ab' OR message.dayOfWeek = 1)
ReferenceExpression field = (ReferenceExpression)func.getArguments().get(0);
String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type?
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf()));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None);
return accumulator.apply(boolQuery, nestedQueryBuilder);
} else if (func.getArguments().get(1) instanceof LiteralExpression) {
// Syntax: 'WHERE nested(message.info) = 'a'
// nestedFunctionAsPredicateExpression
ReferenceExpression field = (ReferenceExpression)((FunctionExpression)func.getArguments().get(0)).getArguments().get(0);
String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type?
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf()));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None);
return accumulator.apply(boolQuery, nestedQueryBuilder);
} else {
// Error? Shouldn't get here.
return null;
}
// Syntax: 'WHERE nested(message.info) = 'a'
// nestedFunctionAsPredicateExpression
ReferenceExpression field = (ReferenceExpression)((FunctionExpression)func.getArguments().get(0)).getArguments().get(0);
String fieldName = convertTextToKeyword(field.toString(), field.type());// function ret type?
TermQueryBuilder termQuery = QueryBuilders.termQuery(fieldName, value(func.getArguments().get(1).valueOf()));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(getNestedPathString(field), termQuery, ScoreMode.None);
return accumulator.apply(boolQuery, nestedQueryBuilder);
// TODO add range query and others that may apply...
}

private String getNestedPathString(ReferenceExpression field) {
Expand Down

0 comments on commit 68821a3

Please sign in to comment.