Merge pull request #1 from opensearch-project/2.x

update latest changes from 2.x

build.gradle

@@ -10,8 +10,8 @@ buildscript {
         opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT")
         opensearch_jdbc_version = System.getProperty("JDBC_VERSION", "2.0.0.0")
         spring_version = "5.3.22"
-        jackson_version = "2.14.0"
-        jackson_databind_version = "2.14.0"
+        jackson_version = "2.14.1"
+        jackson_databind_version = "2.14.1"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')

core/src/main/java/org/opensearch/sql/expression/DSL.java

@@ -486,6 +486,10 @@ public static FunctionExpression replace(Expression... expressions) {
     return compile(FunctionProperties.None, BuiltinFunctionName.REPLACE, expressions);
   }
 
+  public static FunctionExpression reverse(Expression... expressions) {
+    return compile(FunctionProperties.None, BuiltinFunctionName.REVERSE, expressions);
+  }
+
   public static FunctionExpression and(Expression... expressions) {
     return compile(FunctionProperties.None, BuiltinFunctionName.AND, expressions);
   }
@@ -715,6 +719,10 @@ public static FunctionExpression match_bool_prefix(Expression... args) {
     return compile(FunctionProperties.None, BuiltinFunctionName.MATCH_BOOL_PREFIX, args);
   }
 
+  public static FunctionExpression wildcard_query(Expression... args) {
+    return compile(FunctionProperties.None,BuiltinFunctionName.WILDCARD_QUERY, args);
+  }
+
   public static FunctionExpression now(FunctionProperties functionProperties,
                                        Expression... args) {
     return compile(functionProperties, BuiltinFunctionName.NOW, args);

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -167,6 +167,7 @@ public enum BuiltinFunctionName {
   POSITION(FunctionName.of("position")),
   REGEXP(FunctionName.of("regexp")),
   REPLACE(FunctionName.of("replace")),
+  REVERSE(FunctionName.of("reverse")),
   RIGHT(FunctionName.of("right")),
   RTRIM(FunctionName.of("rtrim")),
   STRCMP(FunctionName.of("strcmp")),
@@ -228,7 +229,9 @@ public enum BuiltinFunctionName {
   MATCHQUERY(FunctionName.of("matchquery")),
   MULTI_MATCH(FunctionName.of("multi_match")),
   MULTIMATCH(FunctionName.of("multimatch")),
-  MULTIMATCHQUERY(FunctionName.of("multimatchquery"));
+  MULTIMATCHQUERY(FunctionName.of("multimatchquery")),
+  WILDCARDQUERY(FunctionName.of("wildcardquery")),
+  WILDCARD_QUERY(FunctionName.of("wildcard_query"));
 
   private final FunctionName name;
 

core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java

@@ -42,6 +42,8 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASE));
     repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASEQUERY));
     repository.register(match_phrase_prefix());
+    repository.register(wildcard_query(BuiltinFunctionName.WILDCARD_QUERY));
+    repository.register(wildcard_query(BuiltinFunctionName.WILDCARDQUERY));
   }
 
   private static FunctionResolver match_bool_prefix() {
@@ -83,6 +85,11 @@ private static FunctionResolver query_string() {
     return new RelevanceFunctionResolver(funcName, STRUCT);
   }
 
+  private static FunctionResolver wildcard_query(BuiltinFunctionName wildcardQuery) {
+    FunctionName funcName = wildcardQuery.getName();
+    return new RelevanceFunctionResolver(funcName, STRING);
+  }
+
   public static class OpenSearchFunction extends FunctionExpression {
     private final FunctionName functionName;
     private final List<Expression> arguments;

core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java

@@ -49,6 +49,7 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(ltrim());
     repository.register(position());
     repository.register(replace());
+    repository.register(reverse());
     repository.register(right());
     repository.register(rtrim());
     repository.register(strcmp());
@@ -268,6 +269,17 @@ private DefaultFunctionResolver replace() {
         impl(nullMissingHandling(TextFunction::exprReplace), STRING, STRING, STRING, STRING));
   }
 
+  /**
+   * REVERSE(str) returns reversed string of the string supplied as an argument
+   * Returns NULL if the argument is NULL.
+   * Supports the following signature:
+   * (STRING) -> STRING
+   */
+  private DefaultFunctionResolver reverse() {
+    return define(BuiltinFunctionName.REVERSE.getName(),
+        impl(nullMissingHandling(TextFunction::exprReverse), STRING, STRING));
+  }
+
   private static ExprValue exprSubstrStart(ExprValue exprValue, ExprValue start) {
     int startIdx = start.integerValue();
     if (startIdx == 0) {
@@ -331,5 +343,9 @@ private static ExprValue exprLocate(ExprValue subStr, ExprValue str, ExprValue p
   private static ExprValue exprReplace(ExprValue str, ExprValue from, ExprValue to) {
     return new ExprStringValue(str.stringValue().replaceAll(from.stringValue(), to.stringValue()));
   }
+
+  private static ExprValue exprReverse(ExprValue str) {
+    return new ExprStringValue(new StringBuilder(str.stringValue()).reverse().toString());
+  }
 }
 

core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java

@@ -34,6 +34,7 @@
 import org.opensearch.sql.planner.physical.SortOperator;
 import org.opensearch.sql.planner.physical.ValuesOperator;
 import org.opensearch.sql.planner.physical.WindowOperator;
+import org.opensearch.sql.storage.read.TableScanBuilder;
 
 /**
  * Default implementor for implementing logical to physical translation. "Default" here means all
@@ -123,6 +124,11 @@ public PhysicalPlan visitLimit(LogicalLimit node, C context) {
     return new LimitOperator(visitChild(node, context), node.getLimit(), node.getOffset());
   }
 
+  @Override
+  public PhysicalPlan visitTableScanBuilder(TableScanBuilder plan, C context) {
+    return plan.build();
+  }
+
   @Override
   public PhysicalPlan visitRelation(LogicalRelation node, C context) {
     throw new UnsupportedOperationException("Storage engine is responsible for "

core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java

@@ -6,6 +6,8 @@
 
 package org.opensearch.sql.planner.logical;
 
+import org.opensearch.sql.storage.read.TableScanBuilder;
+
 /**
  * The visitor of {@link LogicalPlan}.
  *
@@ -22,6 +24,10 @@ public R visitRelation(LogicalRelation plan, C context) {
     return visitNode(plan, context);
   }
 
+  public R visitTableScanBuilder(TableScanBuilder plan, C context) {
+    return visitNode(plan, context);
+  }
+
   public R visitFilter(LogicalFilter plan, C context) {
     return visitNode(plan, context);
   }

core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java

@@ -15,6 +15,8 @@
 import org.opensearch.sql.planner.logical.LogicalPlan;
 import org.opensearch.sql.planner.optimizer.rule.MergeFilterAndFilter;
 import org.opensearch.sql.planner.optimizer.rule.PushFilterUnderSort;
+import org.opensearch.sql.planner.optimizer.rule.read.CreateTableScanBuilder;
+import org.opensearch.sql.planner.optimizer.rule.read.TableScanPushDown;
 
 /**
  * {@link LogicalPlan} Optimizer.
@@ -39,8 +41,21 @@ public LogicalPlanOptimizer(List<Rule<?>> rules) {
    */
   public static LogicalPlanOptimizer create() {
     return new LogicalPlanOptimizer(Arrays.asList(
+        /*
+         * Phase 1: Transformations that rely on relational algebra equivalence
+         */
         new MergeFilterAndFilter(),
-        new PushFilterUnderSort()));
+        new PushFilterUnderSort(),
+        /*
+         * Phase 2: Transformations that rely on data source push down capability
+         */
+        new CreateTableScanBuilder(),
+        TableScanPushDown.PUSH_DOWN_FILTER,
+        TableScanPushDown.PUSH_DOWN_AGGREGATION,
+        TableScanPushDown.PUSH_DOWN_SORT,
+        TableScanPushDown.PUSH_DOWN_LIMIT,
+        TableScanPushDown.PUSH_DOWN_HIGHLIGHT,
+        TableScanPushDown.PUSH_DOWN_PROJECT));
   }
 
   /**
@@ -63,7 +78,14 @@ private LogicalPlan internalOptimize(LogicalPlan plan) {
         Match match = DEFAULT_MATCHER.match(rule.pattern(), node);
         if (match.isPresent()) {
           node = rule.apply(match.value(), match.captures());
-          done = false;
+
+          // For new TableScanPushDown impl, pattern match doesn't necessarily cause
+          // push down to happen. So reiterate all rules against the node only if the node
+          // is actually replaced by any rule.
+          // TODO: may need to introduce fixed point or maximum iteration limit in future
+          if (node != match.value()) {
+            done = false;
+          }
         }
       }
     }

core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java

@@ -6,17 +6,78 @@
 
 package org.opensearch.sql.planner.optimizer.pattern;
 
+import com.facebook.presto.matching.Capture;
+import com.facebook.presto.matching.Pattern;
 import com.facebook.presto.matching.Property;
+import com.facebook.presto.matching.PropertyPattern;
 import java.util.Optional;
 import lombok.experimental.UtilityClass;
+import org.opensearch.sql.planner.logical.LogicalAggregation;
+import org.opensearch.sql.planner.logical.LogicalFilter;
+import org.opensearch.sql.planner.logical.LogicalHighlight;
+import org.opensearch.sql.planner.logical.LogicalLimit;
 import org.opensearch.sql.planner.logical.LogicalPlan;
+import org.opensearch.sql.planner.logical.LogicalProject;
+import org.opensearch.sql.planner.logical.LogicalRelation;
+import org.opensearch.sql.planner.logical.LogicalSort;
+import org.opensearch.sql.storage.Table;
+import org.opensearch.sql.storage.read.TableScanBuilder;
 
 /**
  * Pattern helper class.
  */
 @UtilityClass
 public class Patterns {
 
+  /**
+   * Logical filter with a given pattern on inner field.
+   */
+  public static <T extends LogicalPlan> Pattern<LogicalFilter> filter(Pattern<T> pattern) {
+    return Pattern.typeOf(LogicalFilter.class).with(source(pattern));
+  }
+
+  /**
+   * Logical aggregate operator with a given pattern on inner field.
+   */
+  public static <T extends LogicalPlan> Pattern<LogicalAggregation> aggregate(Pattern<T> pattern) {
+    return Pattern.typeOf(LogicalAggregation.class).with(source(pattern));
+  }
+
+  /**
+   * Logical sort operator with a given pattern on inner field.
+   */
+  public static <T extends LogicalPlan> Pattern<LogicalSort> sort(Pattern<T> pattern) {
+    return Pattern.typeOf(LogicalSort.class).with(source(pattern));
+  }
+
+  /**
+   * Logical limit operator with a given pattern on inner field.
+   */
+  public static <T extends LogicalPlan> Pattern<LogicalLimit> limit(Pattern<T> pattern) {
+    return Pattern.typeOf(LogicalLimit.class).with(source(pattern));
+  }
+
+  /**
+   * Logical highlight operator with a given pattern on inner field.
+   */
+  public static <T extends LogicalPlan> Pattern<LogicalHighlight> highlight(Pattern<T> pattern) {
+    return Pattern.typeOf(LogicalHighlight.class).with(source(pattern));
+  }
+
+  /**
+   * Logical project operator with a given pattern on inner field.
+   */
+  public static <T extends LogicalPlan> Pattern<LogicalProject> project(Pattern<T> pattern) {
+    return Pattern.typeOf(LogicalProject.class).with(source(pattern));
+  }
+
+  /**
+   * Pattern for {@link TableScanBuilder} and capture it meanwhile.
+   */
+  public static Pattern<TableScanBuilder> scanBuilder() {
+    return Pattern.typeOf(TableScanBuilder.class).capturedAs(Capture.newCapture());
+  }
+
   /**
    * LogicalPlan source {@link Property}.
    */
@@ -25,4 +86,28 @@ public static Property<LogicalPlan, LogicalPlan> source() {
         ? Optional.of(plan.getChild().get(0))
         : Optional.empty());
   }
+
+  /**
+   * Source (children field) with a given pattern.
+   */
+  @SuppressWarnings("unchecked")
+  public static <T extends LogicalPlan>
+      PropertyPattern<LogicalPlan, T> source(Pattern<T> pattern) {
+    Property<LogicalPlan, T> property = Property.optionalProperty("source",
+        plan -> plan.getChild().size() == 1
+            ? Optional.of((T) plan.getChild().get(0))
+            : Optional.empty());
+
+    return property.matching(pattern);
+  }
+
+  /**
+   * Logical relation with table field.
+   */
+  public static Property<LogicalPlan, Table> table() {
+    return Property.optionalProperty("table",
+        plan -> plan instanceof LogicalRelation
+            ? Optional.of(((LogicalRelation) plan).getTable())
+            : Optional.empty());
+  }
 }

core/src/main/java/org/opensearch/sql/planner/optimizer/rule/read/CreateTableScanBuilder.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.planner.optimizer.rule.read;
+
+import static org.opensearch.sql.planner.optimizer.pattern.Patterns.table;
+
+import com.facebook.presto.matching.Capture;
+import com.facebook.presto.matching.Captures;
+import com.facebook.presto.matching.Pattern;
+import lombok.Getter;
+import lombok.experimental.Accessors;
+import org.opensearch.sql.planner.logical.LogicalPlan;
+import org.opensearch.sql.planner.logical.LogicalRelation;
+import org.opensearch.sql.planner.optimizer.Rule;
+import org.opensearch.sql.storage.Table;
+import org.opensearch.sql.storage.read.TableScanBuilder;
+
+/**
+ * Rule that replace logical relation operator to {@link TableScanBuilder} for later
+ * push down optimization. All push down optimization rules that depends on table scan
+ * builder needs to run after this.
+ */
+public class CreateTableScanBuilder implements Rule<LogicalRelation> {
+
+  /** Capture the table inside matched logical relation operator. */
+  private final Capture<Table> capture;
+
+  /** Pattern that matches logical relation operator. */
+  @Accessors(fluent = true)
+  @Getter
+  private final Pattern<LogicalRelation> pattern;
+
+  /**
+   * Construct create table scan builder rule.
+   */
+  public CreateTableScanBuilder() {
+    this.capture = Capture.newCapture();
+    this.pattern = Pattern.typeOf(LogicalRelation.class)
+        .with(table().capturedAs(capture));
+  }
+
+  @Override
+  public LogicalPlan apply(LogicalRelation plan, Captures captures) {
+    TableScanBuilder scanBuilder = captures.get(capture).createScanBuilder();
+    // TODO: Remove this after Prometheus refactored to new table scan builder too
+    return (scanBuilder == null) ? plan : scanBuilder;
+  }
+}