From 71b92f8699417b286ff28cf5f4fafe95b4087a95 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Thu, 24 Sep 2020 13:32:56 +0300 Subject: [PATCH] QL: Optimize Like/Rlike all (#62682) Replace common Like and RLike queries that match all characters with IsNotNull (exists) queries Fix #62585 (cherry picked from commit 4c23fad0468a9edd7325b06c6a96f7af37625dbf) --- .../xpack/eql/optimizer/Optimizer.java | 10 +++-- .../xpack/eql/planner/QueryTranslator.java | 2 + .../eql/planner/QueryTranslationTests.java | 28 ++++++++++++ .../predicate/regex/LikePattern.java | 11 +++++ .../predicate/regex/RLikePattern.java | 8 ++++ .../predicate/regex/RegexMatch.java | 8 +++- .../predicate/regex/StringPattern.java | 7 +++ .../xpack/ql/optimizer/OptimizerRules.java | 33 +++++++++++--- .../ql/planner/ExpressionTranslators.java | 45 +++++++++++++++++++ .../xpack/ql/util/StringUtils.java | 4 +- .../predicate/regex/StringPatternTests.java | 43 ++++++++++++++++++ .../ql/optimizer/OptimizerRulesTests.java | 26 +++++++++++ .../xpack/sql/optimizer/Optimizer.java | 10 +++-- .../xpack/sql/planner/QueryTranslator.java | 19 +------- 14 files changed, 219 insertions(+), 35 deletions(-) create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java create mode 100644 x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPatternTests.java diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index b6246156e3a2b..f6a51254fc5ae 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneLiteralsInOrderBy; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceSurrogateFunction; +import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceMatchAll; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SetAsOptimized; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.TransformDirection; import org.elasticsearch.xpack.ql.plan.logical.Filter; @@ -66,7 +67,11 @@ public LogicalPlan optimize(LogicalPlan verified) { @Override protected Iterable.Batch> batches() { Batch substitutions = new Batch("Substitution", Limiter.ONCE, - new ReplaceSurrogateFunction()); + // needed for replace wildcards + new BooleanLiteralsOnTheRight(), + new ReplaceWildcards(), + new ReplaceSurrogateFunction(), + new ReplaceMatchAll()); Batch operators = new Batch("Operator Optimization", new ConstantFolding(), @@ -75,7 +80,6 @@ protected Iterable.Batch> batches() { new BooleanLiteralsOnTheRight(), new BooleanEqualsSimplification(), // needs to occur before BinaryComparison combinations - new ReplaceWildcards(), new ReplaceNullChecks(), new PropagateEquals(), new CombineBinaryComparisons(), @@ -106,7 +110,7 @@ protected Iterable.Batch> batches() { private static class ReplaceWildcards extends OptimizerRule { private static boolean isWildcard(Expression expr) { - if (expr.foldable()) { + if (expr instanceof Literal) { Object value = expr.fold(); return value instanceof String && value.toString().contains("*"); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java index 2f6cc8ae60dc6..2a8a0294046a5 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java @@ -40,6 +40,8 @@ final class QueryTranslator { new ExpressionTranslators.BinaryComparisons(), new ExpressionTranslators.Ranges(), new BinaryLogic(), + new ExpressionTranslators.IsNotNulls(), + new ExpressionTranslators.IsNulls(), new ExpressionTranslators.Nots(), new ExpressionTranslators.Likes(), new ExpressionTranslators.InComparisons(), diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java new file mode 100644 index 0000000000000..7e007b4918c50 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.planner; + +import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; + +import static org.hamcrest.Matchers.containsString; + +public class QueryTranslationTests extends AbstractQueryFolderTestCase { + + public void testLikeOptimization() throws Exception { + PhysicalPlan plan = plan("process where process_name == \"*\" "); + assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\"")); + } + + public void testMatchOptimization() throws Exception { + PhysicalPlan plan = plan("process where match(process_name, \".*\") "); + assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\"")); + } + + private static String asQuery(PhysicalPlan plan) { + return plan.toString().replaceAll("\\s+", ""); + } +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java index 096f7d8ae27e1..a3972382c58bf 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/LikePattern.java @@ -5,6 +5,11 @@ */ package org.elasticsearch.xpack.ql.expression.predicate.regex; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.MinimizationOperations; +import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.xpack.ql.util.StringUtils; import java.util.Objects; @@ -48,6 +53,12 @@ public String asJavaRegex() { return regex; } + @Override + public boolean matchesAll() { + Automaton automaton = WildcardQuery.toAutomaton(new Term(null, wildcard)); + return Operations.isTotal(MinimizationOperations.minimize(automaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES)); + } + /** * Returns the pattern in (Lucene) wildcard format. */ diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java index 69747f03e2150..b2459f835bad1 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RLikePattern.java @@ -5,6 +5,9 @@ */ package org.elasticsearch.xpack.ql.expression.predicate.regex; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; + public class RLikePattern implements StringPattern { private final String regexpPattern; @@ -17,4 +20,9 @@ public RLikePattern(String regexpPattern) { public String asJavaRegex() { return regexpPattern; } + + @Override + public boolean matchesAll() { + return Operations.isTotal(new RegExp(regexpPattern).toAutomaton()); + } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java index 565a75d489765..6b00c2dde38d9 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/RegexMatch.java @@ -25,12 +25,12 @@ public abstract class RegexMatch extends UnaryScalarFunction { private final T pattern; - + protected RegexMatch(Source source, Expression value, T pattern) { super(source, value); this.pattern = pattern; } - + public T pattern() { return pattern; } @@ -65,6 +65,10 @@ public Boolean fold() { return RegexProcessor.RegexOperation.match(val, pattern().asJavaRegex()); } + public boolean matchesAll() { + return pattern.matchesAll(); + } + @Override protected Processor makeProcessor() { return new RegexProcessor(pattern().asJavaRegex()); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java index 6dac865b4ac5b..486000599fa38 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPattern.java @@ -10,4 +10,11 @@ interface StringPattern { * Returns the pattern in (Java) regex format. */ String asJavaRegex(); + + /** + * Hint method on whether this pattern matches everything or not. + */ + default boolean matchesAll() { + return false; + } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java index 3b9df1d4f56d5..e42a92dafec2b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; @@ -26,6 +27,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; +import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; @@ -50,7 +52,7 @@ public final class OptimizerRules { - + public static final class ConstantFolding extends OptimizerExpressionRule { public ConstantFolding() { @@ -90,7 +92,7 @@ protected Expression rule(Expression e) { return e; } } - + public static final class BooleanSimplification extends OptimizerExpressionRule { public BooleanSimplification() { @@ -230,7 +232,7 @@ private Expression literalToTheRight(BinaryOperator be) { return be.left() instanceof Literal && !(be.right() instanceof Literal) ? be.swapLeftAndRight() : be; } } - + /** * Propagate Equals to eliminate conjuncted Ranges or BinaryComparisons. * When encountering a different Equals, non-containing {@link Range} or {@link BinaryComparison}, the conjunction becomes false. @@ -537,7 +539,7 @@ private Expression propagate(Or or) { return updated ? Predicates.combineOr(CollectionUtils.combine(exps, equals, notEquals, inequalities, ranges)) : or; } } - + public static final class CombineBinaryComparisons extends OptimizerExpressionRule { public CombineBinaryComparisons() { @@ -1046,7 +1048,7 @@ protected Expression rule(Expression e) { return e; } } - + public abstract static class PruneFilters extends OptimizerRule { @Override @@ -1094,7 +1096,7 @@ private static Expression foldBinaryLogic(Expression expression) { return expression; } } - + public static final class PruneLiteralsInOrderBy extends OptimizerRule { @Override @@ -1120,7 +1122,7 @@ protected LogicalPlan rule(OrderBy ob) { return ob; } } - + public abstract static class SkipQueryOnLimitZero extends OptimizerRule { @Override @@ -1136,6 +1138,23 @@ protected LogicalPlan rule(Limit limit) { protected abstract LogicalPlan skipPlan(Limit limit); } + public static class ReplaceMatchAll extends OptimizerExpressionRule { + + public ReplaceMatchAll() { + super(TransformDirection.DOWN); + } + + protected Expression rule(Expression e) { + if (e instanceof RegexMatch) { + RegexMatch regexMatch = (RegexMatch) e; + if (regexMatch.matchesAll()) { + return new IsNotNull(e.source(), regexMatch.field()); + } + } + return e; + } + } + public static final class SetAsOptimized extends Rule { @Override diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java index 98b9c2f2b57b3..0562bc4fabbd8 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java @@ -20,6 +20,8 @@ import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; @@ -34,6 +36,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.ql.querydsl.query.BoolQuery; +import org.elasticsearch.xpack.ql.querydsl.query.ExistsQuery; import org.elasticsearch.xpack.ql.querydsl.query.MatchQuery; import org.elasticsearch.xpack.ql.querydsl.query.MultiMatchQuery; import org.elasticsearch.xpack.ql.querydsl.query.NotQuery; @@ -74,6 +77,8 @@ public final class ExpressionTranslators { new BinaryComparisons(), new Ranges(), new BinaryLogic(), + new IsNulls(), + new IsNotNulls(), new Nots(), new Likes(), new InComparisons(), @@ -207,6 +212,46 @@ public static Query doTranslate(Not not, TranslatorHandler handler) { } } + public static class IsNotNulls extends ExpressionTranslator { + + @Override + protected Query asQuery(IsNotNull isNotNull, TranslatorHandler handler) { + return doTranslate(isNotNull, handler); + } + + public static Query doTranslate(IsNotNull isNotNull, TranslatorHandler handler) { + Query query = null; + + if (isNotNull.field() instanceof FieldAttribute) { + query = new ExistsQuery(isNotNull.source(), handler.nameOf(isNotNull.field())); + } else { + query = new ScriptQuery(isNotNull.source(), isNotNull.asScript()); + } + + return handler.wrapFunctionQuery(isNotNull, isNotNull.field(), query); + } + } + + public static class IsNulls extends ExpressionTranslator { + + @Override + protected Query asQuery(IsNull isNull, TranslatorHandler handler) { + return doTranslate(isNull, handler); + } + + public static Query doTranslate(IsNull isNull, TranslatorHandler handler) { + Query query = null; + + if (isNull.field() instanceof FieldAttribute) { + query = new NotQuery(isNull.source(), new ExistsQuery(isNull.source(), handler.nameOf(isNull.field()))); + } else { + query = new ScriptQuery(isNull.source(), isNull.asScript()); + } + + return handler.wrapFunctionQuery(isNull, isNull.field(), query); + } + } + // assume the Optimizer properly orders the predicates to ease the translation public static class BinaryComparisons extends ExpressionTranslator { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java index 638a1b4efbb79..5fd369c1e3639 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java @@ -62,7 +62,7 @@ public static String camelCaseToUnderscore(String string) { } return sb.toString().toUpperCase(Locale.ROOT); } - + //CAMEL_CASE to camelCase public static String underscoreToLowerCamelCase(String string) { if (!Strings.hasText(string)) { @@ -337,4 +337,4 @@ public static String ordinal(int i) { } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPatternTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPatternTests.java new file mode 100644 index 0000000000000..66c48e3ee08ed --- /dev/null +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/predicate/regex/StringPatternTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ql.expression.predicate.regex; + +import org.elasticsearch.test.ESTestCase; + +public class StringPatternTests extends ESTestCase { + + private boolean isTotalWildcard(String pattern, char escape) { + return new LikePattern(pattern, escape).matchesAll(); + } + + private boolean isTotalRegex(String pattern) { + return new RLikePattern(pattern).matchesAll(); + } + + public void testWildcardMatchAll() throws Exception { + assertTrue(isTotalWildcard("%", '0')); + assertTrue(isTotalWildcard("%%", '0')); + + assertFalse(isTotalWildcard("a%", '0')); + assertFalse(isTotalWildcard("%_", '0')); + assertFalse(isTotalWildcard("%_%_%", '0')); + assertFalse(isTotalWildcard("_%", '0')); + assertFalse(isTotalWildcard("0%", '0')); + } + + public void testRegexMatchAll() throws Exception { + assertTrue(isTotalRegex(".*")); + assertTrue(isTotalRegex(".*.*")); + assertTrue(isTotalRegex(".*.?")); + assertTrue(isTotalRegex(".?.*")); + assertTrue(isTotalRegex(".*.?.*")); + + assertFalse(isTotalRegex("..*")); + assertFalse(isTotalRegex("ab.")); + assertFalse(isTotalRegex("..?")); + } +} diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java index 8e9f04925a072..266d45874ac7c 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRulesTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Div; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mod; @@ -1318,4 +1319,29 @@ public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() { Expression exp = rule.rule(Predicates.combineOr(Arrays.asList(eq, range, neq, gt))); assertEquals(TRUE, exp); } + + // + // + // + public void testMatchAllLikeToExist() throws Exception { + for (String s : Arrays.asList("%", "%%", "%%%")) { + LikePattern pattern = new LikePattern(s, (char) 0); + FieldAttribute fa = getFieldAttribute(); + Like l = new Like(EMPTY, fa, pattern); + Expression e = new OptimizerRules.ReplaceMatchAll().rule(l); + assertEquals(IsNotNull.class, e.getClass()); + IsNotNull inn = (IsNotNull) e; + assertEquals(fa, inn.field()); + } + } + + public void testMatchAllRLikeToExist() throws Exception { + RLikePattern pattern = new RLikePattern(".*"); + FieldAttribute fa = getFieldAttribute(); + RLike l = new RLike(EMPTY, fa, pattern); + Expression e = new OptimizerRules.ReplaceMatchAll().rule(l); + assertEquals(IsNotNull.class, e.getClass()); + IsNotNull inn = (IsNotNull) e; + assertEquals(fa, inn.field()); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java index 3bbc125ef0a58..0687ad220e727 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; +import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceMatchAll; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons; @@ -116,8 +117,9 @@ public LogicalPlan optimize(LogicalPlan verified) { @Override protected Iterable.Batch> batches() { - Batch pivot = new Batch("Pivot Rewrite", Limiter.ONCE, - new RewritePivot()); + Batch substitutions = new Batch("Substitutions", Limiter.ONCE, + new RewritePivot(), + new ReplaceMatchAll()); Batch refs = new Batch("Replace References", Limiter.ONCE, new ReplaceReferenceAttributeWithSource(), @@ -171,7 +173,7 @@ protected Iterable.Batch> batches() { CleanAliases.INSTANCE, new SetAsOptimized()); - return Arrays.asList(pivot, refs, operators, aggregate, local, label); + return Arrays.asList(substitutions, refs, operators, aggregate, local, label); } static class RewritePivot extends OptimizerRule { @@ -826,7 +828,7 @@ private static class ReplaceCountInLocalRelation extends OptimizerRule { return c instanceof Count ? new Literal(c.source(), 1, c.dataType()) : c; }) : a; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index a7aa2b6010483..1a254cbed9b8f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -31,11 +31,8 @@ import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.ql.planner.ExpressionTranslators; import org.elasticsearch.xpack.ql.planner.TranslatorHandler; -import org.elasticsearch.xpack.ql.querydsl.query.ExistsQuery; import org.elasticsearch.xpack.ql.querydsl.query.GeoDistanceQuery; -import org.elasticsearch.xpack.ql.querydsl.query.NotQuery; import org.elasticsearch.xpack.ql.querydsl.query.Query; -import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.ReflectionUtils; @@ -349,13 +346,7 @@ protected QueryTranslation asQuery(IsNotNull isNotNull, boolean onAggs, Translat if (onAggs) { aggFilter = new AggFilter(id(isNotNull), isNotNull.asScript()); } else { - Query q = null; - if (isNotNull.field() instanceof FieldAttribute) { - q = new ExistsQuery(isNotNull.source(), handler.nameOf(isNotNull.field())); - } else { - q = new ScriptQuery(isNotNull.source(), isNotNull.asScript()); - } - query = handler.wrapFunctionQuery(isNotNull, isNotNull.field(), q); + query = ExpressionTranslators.IsNotNulls.doTranslate(isNotNull, handler); } return new QueryTranslation(query, aggFilter); @@ -372,13 +363,7 @@ protected QueryTranslation asQuery(IsNull isNull, boolean onAggs, TranslatorHand if (onAggs) { aggFilter = new AggFilter(id(isNull), isNull.asScript()); } else { - Query q = null; - if (isNull.field() instanceof FieldAttribute) { - q = new NotQuery(isNull.source(), new ExistsQuery(isNull.source(), handler.nameOf(isNull.field()))); - } else { - q = new ScriptQuery(isNull.source(), isNull.asScript()); - } - query = handler.wrapFunctionQuery(isNull, isNull.field(), q); + query = ExpressionTranslators.IsNulls.doTranslate(isNull, handler); } return new QueryTranslation(query, aggFilter);