diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml index 07f5ff79b5..858e663d6b 100644 --- a/.github/workflows/draft-release-notes-workflow.yml +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -18,6 +18,6 @@ jobs: with: config-name: draft-release-notes-config.yml tag: (None) - version: 2.4.0.0 + version: 3.0.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index 25e0387cf3..14ce737bc3 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -70,6 +70,7 @@ jobs: - name: Upload test reports if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }} uses: actions/upload-artifact@v2 + continue-on-error: true with: name: test-reports path: | diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index b5a0c4c852..150d5be7dc 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -21,13 +21,13 @@ jobs: steps: - uses: actions/checkout@v3 - + - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v3 with: distribution: 'temurin' java-version: ${{ matrix.java }} - + - name: Run tests id: tests run: | @@ -68,7 +68,7 @@ jobs: ./gradlew :integ-test:integTest || echo "* Integration test failed" >> report.log ./gradlew :doctest:doctest || echo "* Doctest failed" >> report.log ./scripts/bwctest.sh || echo "* Backward compatibility test failed" >> report.log - + - name: Verify test results run: | if [[ -e report.log ]] diff --git a/.github/workflows/sql-workbench-test-and-build-workflow.yml b/.github/workflows/sql-workbench-test-and-build-workflow.yml index 06005f3a3a..5e0670e9d5 100644 --- a/.github/workflows/sql-workbench-test-and-build-workflow.yml +++ b/.github/workflows/sql-workbench-test-and-build-workflow.yml @@ -12,7 +12,7 @@ on: env: PLUGIN_NAME: query-workbench-dashboards OPENSEARCH_VERSION: 'main' - OPENSEARCH_PLUGIN_VERSION: 2.4.0.0 + OPENSEARCH_PLUGIN_VERSION: 3.0.0.0 jobs: build: @@ -77,4 +77,3 @@ jobs: with: name: workbench-${{ matrix.os }} path: ../OpenSearch-Dashboards/plugins/workbench/build - \ No newline at end of file diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ba4ce45209..073b4e206c 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,14 +1,16 @@ -# OpenSearch SQL Maintainers +## Overview -## Maintainers +This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). -| Maintainer | GitHub ID | Affiliation | -| --------------- | --------- | ----------- | -| Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | -| Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | -| Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | -| Chloe Zhang | [chloe-zh](https://github.com/chloe-zh) | Amazon | -| Nick Knize | [nknize](https://github.com/nknize) | Amazon | -| Charlotte Henkle | [CEHENKLE](https://github.com/CEHENKLE) | Amazon | -| Max Ksyunz | [MaxKsyunz](https://github.com/MaxKsyunz) | BitQuill | -| Yury Fridlyand | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | BitQuill | \ No newline at end of file +## Current Maintainers + +| Maintainer | GitHub ID | Affiliation | +| --------------------- | --------------------------------------------------- | ----------- | +| Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | +| Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | +| Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | +| Chloe Zhang | [chloe-zh](https://github.com/chloe-zh) | Amazon | +| Nick Knize | [nknize](https://github.com/nknize) | Amazon | +| Charlotte Henkle | [CEHENKLE](https://github.com/CEHENKLE) | Amazon | +| Max Ksyunz | [MaxKsyunz](https://github.com/MaxKsyunz) | BitQuill | +| Yury Fridlyand | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | BitQuill | \ No newline at end of file diff --git a/build.gradle b/build.gradle index a9a171da01..6f227f89b8 100644 --- a/build.gradle +++ b/build.gradle @@ -6,10 +6,7 @@ buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") - spring_version = "5.3.22" - jackson_version = "2.14.0" - jackson_databind_version = "2.14.0" + opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') @@ -69,9 +66,15 @@ plugins { id 'jacoco' } +// import versions defined in https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java#L94 +// versions https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/version.properties +apply plugin: 'opensearch.java' + // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{} repositories { mavenLocal() + // todo. remove this when lucene 9.4.0 is released + maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() // For Elastic Libs that you can use to get started coding until open OpenSearch libs are available } @@ -94,6 +97,8 @@ allprojects { subprojects { repositories { mavenLocal() + // todo. remove this when lucene 9.4.0 is released + maven { url "https://d1nvenhzbhpy0q.cloudfront.net/snapshots/lucene/" } maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" } mavenCentral() } diff --git a/core/build.gradle b/core/build.gradle index fe7126fed5..791d64df65 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -35,19 +35,16 @@ repositories { dependencies { api group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - api group: 'org.springframework', name: 'spring-context', version: "${spring_version}" - api group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' - api "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" - api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" + api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api project(':common') testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' - testImplementation group: 'org.springframework', name: 'spring-test', version: "${spring_version}" testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' } diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 719c3adbce..ff3c01d5b8 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -6,6 +6,11 @@ package org.opensearch.sql.analysis; +import static org.opensearch.sql.ast.dsl.AstDSL.and; +import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTE; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -22,6 +27,7 @@ import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.Between; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; @@ -229,6 +235,14 @@ public Expression visitCompare(Compare node, AnalysisContext context) { functionName, Arrays.asList(left, right)); } + @Override + public Expression visitBetween(Between node, AnalysisContext context) { + return and( + compare(">=", node.getValue(), node.getLowerBound()), + compare("<=", node.getValue(), node.getUpperBound()) + ).accept(this, context); + } + @Override public Expression visitCase(Case node, AnalysisContext context) { List whens = new ArrayList<>(); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index fe993c899e..393de05164 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -12,6 +12,7 @@ import org.opensearch.sql.ast.expression.And; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.AttributeList; +import org.opensearch.sql.ast.expression.Between; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; @@ -173,6 +174,10 @@ public T visitCompare(Compare node, C context) { return visitChildren(node, context); } + public T visitBetween(Between node, C context) { + return visitChildren(node, context); + } + public T visitArgument(Argument node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 2959cae4a1..039b6380f7 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -16,6 +16,7 @@ import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; import org.opensearch.sql.ast.expression.Argument; +import org.opensearch.sql.ast.expression.Between; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; @@ -59,6 +60,7 @@ import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.expression.function.BuiltinFunctionName; /** * Class of static methods to create specific node instances. @@ -320,6 +322,12 @@ public static UnresolvedExpression compare( return new Compare(operator, left, right); } + public static UnresolvedExpression between(UnresolvedExpression value, + UnresolvedExpression lowerBound, + UnresolvedExpression upperBound) { + return new Between(value, lowerBound, upperBound); + } + public static Argument argument(String argName, Literal argValue) { return new Argument(argName, argValue); } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Between.java b/core/src/main/java/org/opensearch/sql/ast/expression/Between.java new file mode 100644 index 0000000000..886c9a9282 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Between.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.expression; + +import java.util.Arrays; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; + +/** + * Unresolved expression for BETWEEN. + */ +@Data +@EqualsAndHashCode(callSuper = false) +public class Between extends UnresolvedExpression { + + /** Value for range check. */ + private final UnresolvedExpression value; + + /** Lower bound of the range (inclusive). */ + private final UnresolvedExpression lowerBound; + + /** Upper bound of the range (inclusive). */ + private final UnresolvedExpression upperBound; + + @Override + public List getChild() { + return Arrays.asList(value, lowerBound, upperBound); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitBetween(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index f5fd1e3315..dfe380b507 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -354,6 +354,10 @@ public static FunctionExpression minute(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.MINUTE, expressions); } + public static FunctionExpression minute_of_day(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.MINUTE_OF_DAY, expressions); + } + public static FunctionExpression month(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.MONTH, expressions); } @@ -486,6 +490,10 @@ public static FunctionExpression replace(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.REPLACE, expressions); } + public static FunctionExpression reverse(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.REVERSE, expressions); + } + public static FunctionExpression and(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.AND, expressions); } @@ -715,6 +723,10 @@ public static FunctionExpression match_bool_prefix(Expression... args) { return compile(FunctionProperties.None, BuiltinFunctionName.MATCH_BOOL_PREFIX, args); } + public static FunctionExpression wildcard_query(Expression... args) { + return compile(FunctionProperties.None,BuiltinFunctionName.WILDCARD_QUERY, args); + } + public static FunctionExpression now(FunctionProperties functionProperties, Expression... args) { return compile(functionProperties, BuiltinFunctionName.NOW, args); @@ -760,6 +772,22 @@ public static FunctionExpression current_date(FunctionProperties functionPropert return compile(functionProperties, BuiltinFunctionName.CURRENT_DATE, args); } + public static FunctionExpression utc_date(FunctionProperties functionProperties, + Expression... args) { + return compile(functionProperties, BuiltinFunctionName.UTC_DATE, args); + } + + public static FunctionExpression utc_time(FunctionProperties functionProperties, + Expression... args) { + return compile(functionProperties, BuiltinFunctionName.UTC_TIME, args); + } + + public static FunctionExpression utc_timestamp(FunctionProperties functionProperties, + Expression... args) { + return compile(functionProperties, BuiltinFunctionName.UTC_TIMESTAMP, args); + + } + @SuppressWarnings("unchecked") private static T compile(FunctionProperties functionProperties, diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index df4dd0a96b..f2f56e28a1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -6,6 +6,8 @@ package org.opensearch.sql.expression.datetime; +import static java.time.temporal.ChronoUnit.DAYS; +import static java.time.temporal.ChronoUnit.MINUTES; import static java.time.temporal.ChronoUnit.MONTHS; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; @@ -27,6 +29,7 @@ import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_SHORT_YEAR; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_STRICT_WITH_TZ; import static org.opensearch.sql.utils.DateTimeUtils.extractDateTime; +import static org.opensearch.sql.utils.DateTimeUtils.extractDate; import java.math.BigDecimal; import java.math.RoundingMode; @@ -100,6 +103,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(current_time()); repository.register(current_timestamp()); repository.register(date()); + repository.register(datediff()); repository.register(datetime()); repository.register(date_add()); repository.register(date_sub()); @@ -118,6 +122,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(maketime()); repository.register(microsecond()); repository.register(minute()); + repository.register(minute_of_day()); repository.register(month(BuiltinFunctionName.MONTH)); repository.register(month(BuiltinFunctionName.MONTH_OF_YEAR)); repository.register(monthName()); @@ -131,7 +136,11 @@ public void register(BuiltinFunctionRepository repository) { repository.register(sysdate()); repository.register(time()); repository.register(time_to_sec()); + repository.register(timediff()); repository.register(timestamp()); + repository.register(utc_date()); + repository.register(utc_time()); + repository.register(utc_timestamp()); repository.register(date_format()); repository.register(to_days()); repository.register(unix_timestamp()); @@ -313,6 +322,46 @@ private DefaultFunctionResolver date() { impl(nullMissingHandling(DateTimeFunction::exprDate), DATE, TIMESTAMP)); } + /* + * Calculates the difference of date part of given values. + * (DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME) -> LONG + */ + private DefaultFunctionResolver datediff() { + return define(BuiltinFunctionName.DATEDIFF.getName(), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATE, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATETIME, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATE, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATETIME, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATE, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIME, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIME, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIMESTAMP, DATE), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATE, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIMESTAMP, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIMESTAMP, TIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIME, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIMESTAMP, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATETIME, TIMESTAMP), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, TIME, DATETIME), + implWithProperties(nullMissingHandlingWithProperties(DateTimeFunction::exprDateDiff), + LONG, DATETIME, TIME)); + } + /** * Specify a datetime with time zone field and a time zone to convert to. * Returns a local date time. @@ -484,6 +533,19 @@ private DefaultFunctionResolver minute() { ); } + /** + * MINUTE(STRING/TIME/DATETIME/TIMESTAMP). return the minute value for time. + */ + private DefaultFunctionResolver minute_of_day() { + return define(BuiltinFunctionName.MINUTE_OF_DAY.getName(), + impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, STRING), + impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, TIME), + impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, DATE), + impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, DATETIME), + impl(nullMissingHandling(DateTimeFunction::exprMinuteOfDay), INTEGER, TIMESTAMP) + ); + } + /** * MONTH(STRING/DATE/DATETIME/TIMESTAMP). return the month for date (1-12). */ @@ -617,6 +679,22 @@ private DefaultFunctionResolver time() { impl(nullMissingHandling(DateTimeFunction::exprTime), TIME, TIMESTAMP)); } + /** + * Returns different between two times as a time. + * (TIME, TIME) -> TIME + * MySQL has these signatures too + * (DATE, DATE) -> TIME // result is > 24 hours + * (DATETIME, DATETIME) -> TIME // result is > 24 hours + * (TIMESTAMP, TIMESTAMP) -> TIME // result is > 24 hours + * (x, x) -> NULL // when args have different types + * (STRING, STRING) -> TIME // argument strings contain same types only + * (STRING, STRING) -> NULL // argument strings are different types + */ + private DefaultFunctionResolver timediff() { + return define(BuiltinFunctionName.TIMEDIFF.getName(), + impl(nullMissingHandling(DateTimeFunction::exprTimeDiff), TIME, TIME, TIME)); + } + /** * TIME_TO_SEC(STRING/TIME/DATETIME/TIMESTAMP). return the time argument, converted to seconds. */ @@ -664,6 +742,33 @@ private FunctionResolver unix_timestamp() { ); } + /** + * UTC_DATE(). return the current UTC Date in format yyyy-MM-dd + */ + private DefaultFunctionResolver utc_date() { + return define(BuiltinFunctionName.UTC_DATE.getName(), + implWithProperties(functionProperties + -> exprUtcDate(functionProperties), DATE)); + } + + /** + * UTC_TIME(). return the current UTC Time in format HH:mm:ss + */ + private DefaultFunctionResolver utc_time() { + return define(BuiltinFunctionName.UTC_TIME.getName(), + implWithProperties(functionProperties + -> exprUtcTime(functionProperties), TIME)); + } + + /** + * UTC_TIMESTAMP(). return the current UTC TimeStamp in format yyyy-MM-dd HH:mm:ss + */ + private DefaultFunctionResolver utc_timestamp() { + return define(BuiltinFunctionName.UTC_TIMESTAMP.getName(), + implWithProperties(functionProperties + -> exprUtcTimeStamp(functionProperties), DATETIME)); + } + /** * WEEK(DATE[,mode]). return the week number for date. */ @@ -822,6 +927,22 @@ private ExprValue exprDate(ExprValue exprValue) { } } + /** + * Calculate the value in days from one date to the other. + * Only the date parts of the values are used in the calculation. + * + * @param first The first value. + * @param second The second value. + * @return The diff. + */ + private ExprValue exprDateDiff(FunctionProperties functionProperties, + ExprValue first, ExprValue second) { + // java inverses the value, so we have to swap 1 and 2 + return new ExprLongValue(DAYS.between( + extractDate(second, functionProperties), + extractDate(first, functionProperties))); + } + /** * DateTime implementation for ExprValue. * @@ -1033,6 +1154,17 @@ private ExprValue exprMinute(ExprValue time) { return new ExprIntegerValue(time.timeValue().getMinute()); } + /** + * Minute_of_day implementation for ExprValue. + * + * @param time ExprValue of Time/String type. + * @return ExprValue. + */ + private ExprValue exprMinuteOfDay(ExprValue time) { + return new ExprIntegerValue( + MINUTES.between(LocalTime.MIN, time.timeValue())); + } + /** * Month for date implementation for ExprValue. * @@ -1182,6 +1314,19 @@ private ExprValue exprTime(ExprValue exprValue) { } } + /** + * Calculate the time difference between two times. + * + * @param first The first value. + * @param second The second value. + * @return The diff. + */ + private ExprValue exprTimeDiff(ExprValue first, ExprValue second) { + // java inverses the value, so we have to swap 1 and 2 + return new ExprTimeValue(LocalTime.MIN.plus( + Duration.between(second.timeValue(), first.timeValue()))); + } + /** * Timestamp implementation for ExprValue. * @@ -1206,6 +1351,38 @@ private ExprValue exprTimeToSec(ExprValue time) { return new ExprLongValue(time.timeValue().toSecondOfDay()); } + /** + * UTC_DATE implementation for ExprValue. + * + * @param functionProperties FunctionProperties. + * @return ExprValue. + */ + private ExprValue exprUtcDate(FunctionProperties functionProperties) { + return new ExprDateValue(exprUtcTimeStamp(functionProperties).dateValue()); + } + + /** + * UTC_TIME implementation for ExprValue. + * + * @param functionProperties FunctionProperties. + * @return ExprValue. + */ + private ExprValue exprUtcTime(FunctionProperties functionProperties) { + return new ExprTimeValue(exprUtcTimeStamp(functionProperties).timeValue()); + } + + /** + * UTC_TIMESTAMP implementation for ExprValue. + * + * @param functionProperties FunctionProperties. + * @return ExprValue. + */ + private ExprValue exprUtcTimeStamp(FunctionProperties functionProperties) { + var zdt = ZonedDateTime.now(functionProperties.getQueryStartClock()) + .withZoneSameInstant(ZoneId.of("UTC")); + return new ExprDatetimeValue(zdt.toLocalDateTime()); + } + /** * To_days implementation for ExprValue. * diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index a7b4f2e9c8..caf3317f29 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -62,6 +62,7 @@ public enum BuiltinFunctionName { ADDTIME(FunctionName.of("addtime")), CONVERT_TZ(FunctionName.of("convert_tz")), DATE(FunctionName.of("date")), + DATEDIFF(FunctionName.of("datediff")), DATETIME(FunctionName.of("datetime")), DATE_ADD(FunctionName.of("date_add")), DATE_SUB(FunctionName.of("date_sub")), @@ -78,6 +79,7 @@ public enum BuiltinFunctionName { MAKETIME(FunctionName.of("maketime")), MICROSECOND(FunctionName.of("microsecond")), MINUTE(FunctionName.of("minute")), + MINUTE_OF_DAY(FunctionName.of("minute_of_day")), MONTH(FunctionName.of("month")), MONTH_OF_YEAR(FunctionName.of("month_of_year")), MONTHNAME(FunctionName.of("monthname")), @@ -88,10 +90,14 @@ public enum BuiltinFunctionName { SUBDATE(FunctionName.of("subdate")), SUBTIME(FunctionName.of("subtime")), TIME(FunctionName.of("time")), + TIMEDIFF(FunctionName.of("timediff")), TIME_TO_SEC(FunctionName.of("time_to_sec")), TIMESTAMP(FunctionName.of("timestamp")), DATE_FORMAT(FunctionName.of("date_format")), TO_DAYS(FunctionName.of("to_days")), + UTC_DATE(FunctionName.of("utc_date")), + UTC_TIME(FunctionName.of("utc_time")), + UTC_TIMESTAMP(FunctionName.of("utc_timestamp")), UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")), WEEK(FunctionName.of("week")), WEEK_OF_YEAR(FunctionName.of("week_of_year")), @@ -169,6 +175,7 @@ public enum BuiltinFunctionName { POSITION(FunctionName.of("position")), REGEXP(FunctionName.of("regexp")), REPLACE(FunctionName.of("replace")), + REVERSE(FunctionName.of("reverse")), RIGHT(FunctionName.of("right")), RTRIM(FunctionName.of("rtrim")), STRCMP(FunctionName.of("strcmp")), @@ -230,7 +237,9 @@ public enum BuiltinFunctionName { MATCHQUERY(FunctionName.of("matchquery")), MULTI_MATCH(FunctionName.of("multi_match")), MULTIMATCH(FunctionName.of("multimatch")), - MULTIMATCHQUERY(FunctionName.of("multimatchquery")); + MULTIMATCHQUERY(FunctionName.of("multimatchquery")), + WILDCARDQUERY(FunctionName.of("wildcardquery")), + WILDCARD_QUERY(FunctionName.of("wildcard_query")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index 2041b9762e..842cf25cd6 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -5,10 +5,6 @@ package org.opensearch.sql.expression.function; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; - -import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; @@ -42,45 +38,52 @@ public void register(BuiltinFunctionRepository repository) { repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASE)); repository.register(match_phrase(BuiltinFunctionName.MATCHPHRASEQUERY)); repository.register(match_phrase_prefix()); + repository.register(wildcard_query(BuiltinFunctionName.WILDCARD_QUERY)); + repository.register(wildcard_query(BuiltinFunctionName.WILDCARDQUERY)); } private static FunctionResolver match_bool_prefix() { FunctionName name = BuiltinFunctionName.MATCH_BOOL_PREFIX.getName(); - return new RelevanceFunctionResolver(name, STRING); + return new RelevanceFunctionResolver(name); } private static FunctionResolver match(BuiltinFunctionName match) { FunctionName funcName = match.getName(); - return new RelevanceFunctionResolver(funcName, STRING); + return new RelevanceFunctionResolver(funcName); } private static FunctionResolver match_phrase_prefix() { FunctionName funcName = BuiltinFunctionName.MATCH_PHRASE_PREFIX.getName(); - return new RelevanceFunctionResolver(funcName, STRING); + return new RelevanceFunctionResolver(funcName); } private static FunctionResolver match_phrase(BuiltinFunctionName matchPhrase) { FunctionName funcName = matchPhrase.getName(); - return new RelevanceFunctionResolver(funcName, STRING); + return new RelevanceFunctionResolver(funcName); } private static FunctionResolver multi_match(BuiltinFunctionName multiMatchName) { - return new RelevanceFunctionResolver(multiMatchName.getName(), STRUCT); + return new RelevanceFunctionResolver(multiMatchName.getName()); } private static FunctionResolver simple_query_string() { FunctionName funcName = BuiltinFunctionName.SIMPLE_QUERY_STRING.getName(); - return new RelevanceFunctionResolver(funcName, STRUCT); + return new RelevanceFunctionResolver(funcName); } private static FunctionResolver query() { FunctionName funcName = BuiltinFunctionName.QUERY.getName(); - return new RelevanceFunctionResolver(funcName, STRING); + return new RelevanceFunctionResolver(funcName); } private static FunctionResolver query_string() { FunctionName funcName = BuiltinFunctionName.QUERY_STRING.getName(); - return new RelevanceFunctionResolver(funcName, STRUCT); + return new RelevanceFunctionResolver(funcName); + } + + private static FunctionResolver wildcard_query(BuiltinFunctionName wildcardQuery) { + FunctionName funcName = wildcardQuery.getName(); + return new RelevanceFunctionResolver(funcName); } public static class OpenSearchFunction extends FunctionExpression { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/RelevanceFunctionResolver.java b/core/src/main/java/org/opensearch/sql/expression/function/RelevanceFunctionResolver.java index 7066622e1b..ef0ac9226c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/RelevanceFunctionResolver.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/RelevanceFunctionResolver.java @@ -20,9 +20,6 @@ public class RelevanceFunctionResolver @Getter private final FunctionName functionName; - @Getter - private final ExprType declaredFirstParamType; - @Override public Pair resolve(FunctionSignature unresolvedSignature) { if (!unresolvedSignature.getFunctionName().equals(functionName)) { @@ -30,14 +27,6 @@ public Pair resolve(FunctionSignature unreso functionName.getFunctionName(), unresolvedSignature.getFunctionName().getFunctionName())); } List paramTypes = unresolvedSignature.getParamTypeList(); - ExprType providedFirstParamType = paramTypes.get(0); - - // Check if the first parameter is of the specified type. - if (!declaredFirstParamType.equals(providedFirstParamType)) { - throw new SemanticCheckException( - getWrongParameterErrorMessage(0, providedFirstParamType, declaredFirstParamType)); - } - // Check if all but the first parameter are of type STRING. for (int i = 1; i < paramTypes.size(); i++) { ExprType paramType = paramTypes.get(i); diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java index 0e4df086fb..4424243860 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunction.java @@ -500,26 +500,26 @@ private static DefaultFunctionResolver truncate() { FunctionDSL.impl( FunctionDSL.nullMissingHandling( (x, y) -> new ExprLongValue( - new BigDecimal(x.integerValue()).setScale(y.integerValue(), - RoundingMode.DOWN).longValue())), + BigDecimal.valueOf(x.integerValue()).setScale(y.integerValue(), + RoundingMode.DOWN).longValue())), LONG, INTEGER, INTEGER), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (x, y) -> new ExprLongValue( - new BigDecimal(x.integerValue()).setScale(y.integerValue(), - RoundingMode.DOWN).longValue())), + BigDecimal.valueOf(x.longValue()).setScale(y.integerValue(), + RoundingMode.DOWN).longValue())), LONG, LONG, INTEGER), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (x, y) -> new ExprDoubleValue( - new BigDecimal(x.floatValue()).setScale(y.integerValue(), - RoundingMode.DOWN).doubleValue())), + BigDecimal.valueOf(x.floatValue()).setScale(y.integerValue(), + RoundingMode.DOWN).doubleValue())), DOUBLE, FLOAT, INTEGER), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (x, y) -> new ExprDoubleValue( - new BigDecimal(x.doubleValue()).setScale(y.integerValue(), - RoundingMode.DOWN).doubleValue())), + BigDecimal.valueOf(x.doubleValue()).setScale(y.integerValue(), + RoundingMode.DOWN).doubleValue())), DOUBLE, DOUBLE, INTEGER)); } diff --git a/core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java b/core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java index 5915700bf1..25eb25489c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/text/TextFunction.java @@ -49,6 +49,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(ltrim()); repository.register(position()); repository.register(replace()); + repository.register(reverse()); repository.register(right()); repository.register(rtrim()); repository.register(strcmp()); @@ -268,6 +269,17 @@ private DefaultFunctionResolver replace() { impl(nullMissingHandling(TextFunction::exprReplace), STRING, STRING, STRING, STRING)); } + /** + * REVERSE(str) returns reversed string of the string supplied as an argument + * Returns NULL if the argument is NULL. + * Supports the following signature: + * (STRING) -> STRING + */ + private DefaultFunctionResolver reverse() { + return define(BuiltinFunctionName.REVERSE.getName(), + impl(nullMissingHandling(TextFunction::exprReverse), STRING, STRING)); + } + private static ExprValue exprSubstrStart(ExprValue exprValue, ExprValue start) { int startIdx = start.integerValue(); if (startIdx == 0) { @@ -331,5 +343,9 @@ private static ExprValue exprLocate(ExprValue subStr, ExprValue str, ExprValue p private static ExprValue exprReplace(ExprValue str, ExprValue from, ExprValue to) { return new ExprStringValue(str.stringValue().replaceAll(from.stringValue(), to.stringValue())); } + + private static ExprValue exprReverse(ExprValue str) { + return new ExprStringValue(new StringBuilder(str.stringValue()).reverse().toString()); + } } diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java index 4a5276418d..4a6d4d8222 100644 --- a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java @@ -35,6 +35,7 @@ import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; /** * Default implementor for implementing logical to physical translation. "Default" here means all @@ -129,6 +130,11 @@ public PhysicalPlan visitTableScanBuilder(TableScanBuilder plan, C context) { return plan.build(); } + @Override + public PhysicalPlan visitTableWriteBuilder(TableWriteBuilder plan, C context) { + return plan.build(visitChild(plan, context)); + } + @Override public PhysicalPlan visitRelation(LogicalRelation node, C context) { throw new UnsupportedOperationException("Storage engine is responsible for " diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index b18e099afa..a192966287 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -32,6 +32,10 @@ @UtilityClass public class LogicalPlanDSL { + public static LogicalPlan write(LogicalPlan input, Table table, List columns) { + return new LogicalWrite(input, table, columns); + } + public static LogicalPlan aggregation( LogicalPlan input, List aggregatorList, List groupByList) { return new LogicalAggregation(input, aggregatorList, groupByList); diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 0386eb6e2a..9a41072fe7 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -7,6 +7,7 @@ package org.opensearch.sql.planner.logical; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; /** * The visitor of {@link LogicalPlan}. @@ -28,6 +29,14 @@ public R visitTableScanBuilder(TableScanBuilder plan, C context) { return visitNode(plan, context); } + public R visitWrite(LogicalWrite plan, C context) { + return visitNode(plan, context); + } + + public R visitTableWriteBuilder(TableWriteBuilder plan, C context) { + return visitNode(plan, context); + } + public R visitFilter(LogicalFilter plan, C context) { return visitNode(plan, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalWrite.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalWrite.java new file mode 100644 index 0000000000..496e6009e3 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalWrite.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.logical; + +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.storage.Table; + +/** + * Logical operator for insert statement. + */ +@EqualsAndHashCode(callSuper = true) +@Getter +@ToString +public class LogicalWrite extends LogicalPlan { + + /** Table that handles the write operation. */ + private final Table table; + + /** Optional column name list specified in insert statement. */ + private final List columns; + + /** + * Construct a logical write with given child node, table and column name list. + */ + public LogicalWrite(LogicalPlan child, Table table, List columns) { + super(Collections.singletonList(child)); + this.table = table; + this.columns = columns; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitWrite(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java index f241e76993..70847b869b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java @@ -17,6 +17,7 @@ import org.opensearch.sql.planner.optimizer.rule.PushFilterUnderSort; import org.opensearch.sql.planner.optimizer.rule.read.CreateTableScanBuilder; import org.opensearch.sql.planner.optimizer.rule.read.TableScanPushDown; +import org.opensearch.sql.planner.optimizer.rule.write.CreateTableWriteBuilder; /** * {@link LogicalPlan} Optimizer. @@ -55,7 +56,8 @@ public static LogicalPlanOptimizer create() { TableScanPushDown.PUSH_DOWN_SORT, TableScanPushDown.PUSH_DOWN_LIMIT, TableScanPushDown.PUSH_DOWN_HIGHLIGHT, - TableScanPushDown.PUSH_DOWN_PROJECT)); + TableScanPushDown.PUSH_DOWN_PROJECT, + new CreateTableWriteBuilder())); } /** diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java index 0ba478594a..856d8df7ea 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java @@ -20,6 +20,7 @@ import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.planner.logical.LogicalWrite; import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.read.TableScanBuilder; @@ -110,4 +111,14 @@ public static Property table() { ? Optional.of(((LogicalRelation) plan).getTable()) : Optional.empty()); } + + /** + * Logical write with table field. + */ + public static Property writeTable() { + return Property.optionalProperty("table", + plan -> plan instanceof LogicalWrite + ? Optional.of(((LogicalWrite) plan).getTable()) + : Optional.empty()); + } } diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/write/CreateTableWriteBuilder.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/write/CreateTableWriteBuilder.java new file mode 100644 index 0000000000..4fbf676862 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/write/CreateTableWriteBuilder.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.optimizer.rule.write; + +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.writeTable; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import lombok.Getter; +import lombok.experimental.Accessors; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalWrite; +import org.opensearch.sql.planner.optimizer.Rule; +import org.opensearch.sql.storage.Table; +import org.opensearch.sql.storage.write.TableWriteBuilder; + +/** + * Rule that replaces logical write operator with {@link TableWriteBuilder} for later optimization + * and transforming to physical operator. + */ +public class CreateTableWriteBuilder implements Rule { + + /** Capture the table inside matched logical relation operator. */ + private final Capture capture; + + /** Pattern that matches logical relation operator. */ + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + /** + * Construct create table write builder rule. + */ + public CreateTableWriteBuilder() { + this.capture = Capture.newCapture(); + this.pattern = Pattern.typeOf(LogicalWrite.class) + .with(writeTable().capturedAs(capture)); + } + + @Override + public LogicalPlan apply(LogicalWrite plan, Captures captures) { + return captures.get(capture).createWriteBuilder(plan); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 63dd05cc6b..d4bc4a1ea9 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -7,6 +7,7 @@ package org.opensearch.sql.planner.physical; import org.opensearch.sql.storage.TableScanOperator; +import org.opensearch.sql.storage.write.TableWriteOperator; /** * The visitor of {@link PhysicalPlan}. @@ -36,6 +37,10 @@ public R visitTableScan(TableScanOperator node, C context) { return visitNode(node, context); } + public R visitTableWrite(TableWriteOperator node, C context) { + return visitNode(node, context); + } + public R visitProject(ProjectOperator node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/storage/Table.java b/core/src/main/java/org/opensearch/sql/storage/Table.java index ae0aaaf17b..496281fa8d 100644 --- a/core/src/main/java/org/opensearch/sql/storage/Table.java +++ b/core/src/main/java/org/opensearch/sql/storage/Table.java @@ -10,8 +10,10 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.streaming.StreamingSource; import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalWrite; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; /** * Table. @@ -73,6 +75,17 @@ default TableScanBuilder createScanBuilder() { return null; // TODO: Enforce all subclasses to implement this later } + /* + * Create table write builder for logical to physical transformation. + * + * @param plan logical write plan + * @return table write builder + */ + default TableWriteBuilder createWriteBuilder(LogicalWrite plan) { + throw new UnsupportedOperationException( + "Write operation is not supported on current table"); + } + /** * Translate {@link Table} to {@link StreamingSource} if possible. */ diff --git a/core/src/main/java/org/opensearch/sql/storage/write/TableWriteBuilder.java b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteBuilder.java new file mode 100644 index 0000000000..54dfa5d557 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteBuilder.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.write; + +import java.util.Collections; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalPlanNodeVisitor; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A {@link TableWriteBuilder} represents transition state between logical planning and physical + * planning for table write operator. The concrete implementation class gets involved in the logical + * optimization through this abstraction and thus transform to specific {@link TableWriteOperator} + * in a certain storage engine. + */ +public abstract class TableWriteBuilder extends LogicalPlan { + + /** + * Construct table write builder with child node. + */ + public TableWriteBuilder(LogicalPlan child) { + super(Collections.singletonList(child)); + } + + /** + * Build table write operator with given child node. + * + * @param child child operator node + * @return table write operator + */ + public abstract TableWriteOperator build(PhysicalPlan child); + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitTableWriteBuilder(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/storage/write/TableWriteOperator.java b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteOperator.java new file mode 100644 index 0000000000..92cdc6eb41 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteOperator.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.write; + +import java.util.Collections; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlanNodeVisitor; + +/** + * {@link TableWriteOperator} is the abstraction for data source to implement different physical + * write operator on a data source. This is also to avoid "polluting" physical plan visitor by + * concrete table scan implementation. + */ +@RequiredArgsConstructor +public abstract class TableWriteOperator extends PhysicalPlan { + + /** Input physical node. */ + protected final PhysicalPlan input; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitTableWrite(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + /** + * Explain the execution plan. + * + * @return explain output + */ + public abstract String explain(); +} diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java index 6ca9486de6..06d40ea6aa 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeUtils.java @@ -6,6 +6,7 @@ package org.opensearch.sql.utils; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -139,4 +140,15 @@ public static LocalDateTime extractDateTime(ExprValue value, ? ((ExprTimeValue) value).datetimeValue(functionProperties) : value.datetimeValue(); } + + /** + * Extracts LocalDate from a datetime ExprValue. + * Uses `FunctionProperties` for `ExprTimeValue`. + */ + public static LocalDate extractDate(ExprValue value, + FunctionProperties functionProperties) { + return value instanceof ExprTimeValue + ? ((ExprTimeValue) value).dateValue(functionProperties) + : value.dateValue(); + } } diff --git a/core/src/main/java/org/opensearch/sql/utils/OperatorUtils.java b/core/src/main/java/org/opensearch/sql/utils/OperatorUtils.java index 26a21251eb..f4ece6a190 100644 --- a/core/src/main/java/org/opensearch/sql/utils/OperatorUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/OperatorUtils.java @@ -22,8 +22,9 @@ public class OperatorUtils { * @return if text matches pattern returns true; else return false. */ public static ExprBooleanValue matches(ExprValue text, ExprValue pattern) { - return ExprBooleanValue - .of(Pattern.compile(patternToRegex(pattern.stringValue())).matcher(text.stringValue()) + return ExprBooleanValue.of( + Pattern.compile(patternToRegex(pattern.stringValue()), Pattern.CASE_INSENSITIVE) + .matcher(text.stringValue()) .matches()); } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 044949ea35..1db29a6a42 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -65,7 +65,6 @@ import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.DataType; @@ -89,15 +88,7 @@ import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.physical.datasource.DataSourceTable; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {AnalyzerTest.class}) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) + class AnalyzerTest extends AnalyzerTestBase { @Test diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index c68ba2653e..51c1f06433 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.analysis.symbol.Namespace; @@ -39,8 +40,6 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; public class AnalyzerTestBase { @@ -49,14 +48,12 @@ protected Map typeMapping() { return TestConfig.typeMapping; } - @Bean protected StorageEngine storageEngine() { return (dataSourceSchemaName, tableName) -> table; } - @Bean protected Table table() { - return new Table() { + return Optional.ofNullable(table).orElseGet(() -> new Table() { @Override public boolean exists() { return true; @@ -76,31 +73,13 @@ public Map getFieldTypes() { public PhysicalPlan implement(LogicalPlan plan) { throw new UnsupportedOperationException(); } - }; - } - - @Bean - protected Table dataSourceTable() { - return new Table() { - @Override - public Map getFieldTypes() { - return typeMapping(); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - throw new UnsupportedOperationException(); - } - }; + }); } - @Bean protected DataSourceService dataSourceService() { - return new DefaultDataSourceService(); + return Optional.ofNullable(dataSourceService).orElseGet(DefaultDataSourceService::new); } - - @Bean protected SymbolTable symbolTable() { SymbolTable symbolTable = new SymbolTable(); typeMapping().entrySet() @@ -110,7 +89,6 @@ protected SymbolTable symbolTable() { return symbolTable; } - @Bean protected Environment typeEnv() { return var -> { if (var instanceof ReferenceExpression) { @@ -123,25 +101,16 @@ protected Environment typeEnv() { }; } - @Autowired - protected AnalysisContext analysisContext; - - @Autowired - protected ExpressionAnalyzer expressionAnalyzer; + protected AnalysisContext analysisContext = analysisContext(typeEnvironment(symbolTable())); - @Autowired - protected Analyzer analyzer; + protected ExpressionAnalyzer expressionAnalyzer = expressionAnalyzer(); - @Autowired - protected Table table; + protected Table table = table(); - @Autowired - protected DataSourceService dataSourceService; + protected DataSourceService dataSourceService = dataSourceService(); - @Autowired - protected Environment typeEnv; + protected Analyzer analyzer = analyzer(expressionAnalyzer(), dataSourceService, table); - @Bean protected Analyzer analyzer(ExpressionAnalyzer expressionAnalyzer, DataSourceService dataSourceService, Table table) { @@ -167,17 +136,14 @@ public FunctionName getFunctionName() { return new Analyzer(expressionAnalyzer, dataSourceService, functionRepository); } - @Bean protected TypeEnvironment typeEnvironment(SymbolTable symbolTable) { return new TypeEnvironment(null, symbolTable); } - @Bean protected AnalysisContext analysisContext(TypeEnvironment typeEnvironment) { return new AnalysisContext(typeEnvironment); } - @Bean protected ExpressionAnalyzer expressionAnalyzer() { return new ExpressionAnalyzer(BuiltinFunctionRepository.getInstance()); } diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index dfb7a7239f..c7a11658e3 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -31,7 +31,6 @@ import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.ast.dsl.AstDSL; @@ -48,15 +47,8 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.function.FunctionPropertiesTestConfig; import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {FunctionPropertiesTestConfig.class, AnalyzerTestBase.class}) class ExpressionAnalyzerTest extends AnalyzerTestBase { @Test @@ -107,6 +99,22 @@ public void qualified_name() { ); } + @Test + public void between() { + assertAnalyzeEqual( + DSL.and( + DSL.gte( + DSL.ref("integer_value", INTEGER), + DSL.literal(20)), + DSL.lte( + DSL.ref("integer_value", INTEGER), + DSL.literal(30))), + AstDSL.between( + qualifiedName("integer_value"), + AstDSL.intLiteral(20), + AstDSL.intLiteral(30))); + } + @Test public void case_value() { assertAnalyzeEqual( @@ -358,10 +366,10 @@ public void named_non_parse_expression() { void match_bool_prefix_expression() { assertAnalyzeEqual( DSL.match_bool_prefix( - DSL.namedArgument("field", DSL.literal("fieldA")), + DSL.namedArgument("field", DSL.literal("field_value1")), DSL.namedArgument("query", DSL.literal("sample query"))), AstDSL.function("match_bool_prefix", - AstDSL.unresolvedArg("field", stringLiteral("fieldA")), + AstDSL.unresolvedArg("field", stringLiteral("field_value1")), AstDSL.unresolvedArg("query", stringLiteral("sample query")))); } @@ -402,11 +410,11 @@ void multi_match_expression() { DSL.multi_match( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field", ExprValueUtils.floatValue(1.F)))))), + "field_value1", ExprValueUtils.floatValue(1.F)))))), DSL.namedArgument("query", DSL.literal("sample query"))), AstDSL.function("multi_match", AstDSL.unresolvedArg("fields", new RelevanceFieldList(Map.of( - "field", 1.F))), + "field_value1", 1.F))), AstDSL.unresolvedArg("query", stringLiteral("sample query")))); } @@ -416,12 +424,12 @@ void multi_match_expression_with_params() { DSL.multi_match( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field", ExprValueUtils.floatValue(1.F)))))), + "field_value1", ExprValueUtils.floatValue(1.F)))))), DSL.namedArgument("query", DSL.literal("sample query")), DSL.namedArgument("analyzer", DSL.literal("keyword"))), AstDSL.function("multi_match", AstDSL.unresolvedArg("fields", new RelevanceFieldList(Map.of( - "field", 1.F))), + "field_value1", 1.F))), AstDSL.unresolvedArg("query", stringLiteral("sample query")), AstDSL.unresolvedArg("analyzer", stringLiteral("keyword")))); } @@ -432,12 +440,12 @@ void multi_match_expression_two_fields() { DSL.multi_match( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field1", ExprValueUtils.floatValue(1.F), - "field2", ExprValueUtils.floatValue(.3F)))))), + "field_value1", ExprValueUtils.floatValue(1.F), + "field_value2", ExprValueUtils.floatValue(.3F)))))), DSL.namedArgument("query", DSL.literal("sample query"))), AstDSL.function("multi_match", AstDSL.unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of( - "field1", 1.F, "field2", .3F))), + "field_value1", 1.F, "field_value2", .3F))), AstDSL.unresolvedArg("query", stringLiteral("sample query")))); } @@ -447,11 +455,11 @@ void simple_query_string_expression() { DSL.simple_query_string( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field", ExprValueUtils.floatValue(1.F)))))), + "field_value1", ExprValueUtils.floatValue(1.F)))))), DSL.namedArgument("query", DSL.literal("sample query"))), AstDSL.function("simple_query_string", AstDSL.unresolvedArg("fields", new RelevanceFieldList(Map.of( - "field", 1.F))), + "field_value1", 1.F))), AstDSL.unresolvedArg("query", stringLiteral("sample query")))); } @@ -461,12 +469,12 @@ void simple_query_string_expression_with_params() { DSL.simple_query_string( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field", ExprValueUtils.floatValue(1.F)))))), + "field_value1", ExprValueUtils.floatValue(1.F)))))), DSL.namedArgument("query", DSL.literal("sample query")), DSL.namedArgument("analyzer", DSL.literal("keyword"))), AstDSL.function("simple_query_string", AstDSL.unresolvedArg("fields", new RelevanceFieldList(Map.of( - "field", 1.F))), + "field_value1", 1.F))), AstDSL.unresolvedArg("query", stringLiteral("sample query")), AstDSL.unresolvedArg("analyzer", stringLiteral("keyword")))); } @@ -477,12 +485,12 @@ void simple_query_string_expression_two_fields() { DSL.simple_query_string( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field1", ExprValueUtils.floatValue(1.F), - "field2", ExprValueUtils.floatValue(.3F)))))), + "field_value1", ExprValueUtils.floatValue(1.F), + "field_value2", ExprValueUtils.floatValue(.3F)))))), DSL.namedArgument("query", DSL.literal("sample query"))), AstDSL.function("simple_query_string", AstDSL.unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of( - "field1", 1.F, "field2", .3F))), + "field_value1", 1.F, "field_value2", .3F))), AstDSL.unresolvedArg("query", stringLiteral("sample query")))); } @@ -501,11 +509,11 @@ void query_string_expression() { DSL.query_string( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field", ExprValueUtils.floatValue(1.F)))))), + "field_value1", ExprValueUtils.floatValue(1.F)))))), DSL.namedArgument("query", DSL.literal("query_value"))), AstDSL.function("query_string", AstDSL.unresolvedArg("fields", new RelevanceFieldList(Map.of( - "field", 1.F))), + "field_value1", 1.F))), AstDSL.unresolvedArg("query", stringLiteral("query_value")))); } @@ -515,12 +523,12 @@ void query_string_expression_with_params() { DSL.query_string( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field", ExprValueUtils.floatValue(1.F)))))), + "field_value1", ExprValueUtils.floatValue(1.F)))))), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("escape", DSL.literal("false"))), AstDSL.function("query_string", AstDSL.unresolvedArg("fields", new RelevanceFieldList(Map.of( - "field", 1.F))), + "field_value1", 1.F))), AstDSL.unresolvedArg("query", stringLiteral("query_value")), AstDSL.unresolvedArg("escape", stringLiteral("false")))); } @@ -531,20 +539,48 @@ void query_string_expression_two_fields() { DSL.query_string( DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "field1", ExprValueUtils.floatValue(1.F), - "field2", ExprValueUtils.floatValue(.3F)))))), + "field_value1", ExprValueUtils.floatValue(1.F), + "field_value2", ExprValueUtils.floatValue(.3F)))))), DSL.namedArgument("query", DSL.literal("query_value"))), AstDSL.function("query_string", AstDSL.unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of( - "field1", 1.F, "field2", .3F))), + "field_value1", 1.F, "field_value2", .3F))), AstDSL.unresolvedArg("query", stringLiteral("query_value")))); } + @Test + void wildcard_query_expression() { + assertAnalyzeEqual( + DSL.wildcard_query( + DSL.namedArgument("field", DSL.literal("test")), + DSL.namedArgument("query", DSL.literal("query_value*"))), + AstDSL.function("wildcard_query", + unresolvedArg("field", stringLiteral("test")), + unresolvedArg("query", stringLiteral("query_value*")))); + } + + @Test + void wildcard_query_expression_all_params() { + assertAnalyzeEqual( + DSL.wildcard_query( + DSL.namedArgument("field", DSL.literal("test")), + DSL.namedArgument("query", DSL.literal("query_value*")), + DSL.namedArgument("boost", DSL.literal("1.5")), + DSL.namedArgument("case_insensitive", DSL.literal("true")), + DSL.namedArgument("rewrite", DSL.literal("scoring_boolean"))), + AstDSL.function("wildcard_query", + unresolvedArg("field", stringLiteral("test")), + unresolvedArg("query", stringLiteral("query_value*")), + unresolvedArg("boost", stringLiteral("1.5")), + unresolvedArg("case_insensitive", stringLiteral("true")), + unresolvedArg("rewrite", stringLiteral("scoring_boolean")))); + } + @Test public void match_phrase_prefix_all_params() { assertAnalyzeEqual( DSL.match_phrase_prefix( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", "field_value1"), DSL.namedArgument("query", "search query"), DSL.namedArgument("slop", "3"), DSL.namedArgument("boost", "1.5"), @@ -553,7 +589,7 @@ public void match_phrase_prefix_all_params() { DSL.namedArgument("zero_terms_query", "NONE") ), AstDSL.function("match_phrase_prefix", - unresolvedArg("field", stringLiteral("test")), + unresolvedArg("field", stringLiteral("field_value1")), unresolvedArg("query", stringLiteral("search query")), unresolvedArg("slop", stringLiteral("3")), unresolvedArg("boost", stringLiteral("1.5")), diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java index 287cff0e3a..89d5f699e3 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionReferenceOptimizerTest.java @@ -14,20 +14,13 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalPlanDSL; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {AnalyzerTest.class}) class ExpressionReferenceOptimizerTest extends AnalyzerTestBase { @Test diff --git a/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java index 913593add3..e9c891905c 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/NamedExpressionAnalyzerTest.java @@ -11,19 +11,12 @@ import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Alias; import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.expression.NamedExpression; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {AnalyzerTestBase.class}) class NamedExpressionAnalyzerTest extends AnalyzerTestBase { @Test void visit_named_select_item() { diff --git a/core/src/test/java/org/opensearch/sql/analysis/QualifierAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/QualifierAnalyzerTest.java index 8c87b24d3e..5833ef6ae4 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/QualifierAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/QualifierAnalyzerTest.java @@ -13,18 +13,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.type.ExprType; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {AnalyzerTestBase.class}) class QualifierAnalyzerTest extends AnalyzerTestBase { private QualifierAnalyzer qualifierAnalyzer; diff --git a/core/src/test/java/org/opensearch/sql/analysis/SelectAnalyzeTest.java b/core/src/test/java/org/opensearch/sql/analysis/SelectAnalyzeTest.java index 82f3a87e30..3bd90f0081 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/SelectAnalyzeTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/SelectAnalyzeTest.java @@ -17,20 +17,13 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.planner.logical.LogicalPlanDSL; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {SelectAnalyzeTest.class}) public class SelectAnalyzeTest extends AnalyzerTestBase { @Override diff --git a/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java index a9fb03d2d5..b2fe29b509 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java @@ -24,14 +24,8 @@ import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.NamedExpression; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) @ExtendWith(MockitoExtension.class) -@ContextConfiguration(classes = {SelectExpressionAnalyzerTest.class}) public class SelectExpressionAnalyzerTest extends AnalyzerTestBase { @Mock diff --git a/core/src/test/java/org/opensearch/sql/analysis/WindowExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/WindowExpressionAnalyzerTest.java index cf0a90a436..dd4361ad6a 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/WindowExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/WindowExpressionAnalyzerTest.java @@ -21,10 +21,7 @@ import java.util.Collections; import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayNameGeneration; -import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.Alias; import org.opensearch.sql.ast.tree.Sort.SortOption; @@ -34,14 +31,7 @@ import org.opensearch.sql.planner.logical.LogicalPlanDSL; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.logical.LogicalSort; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {SelectExpressionAnalyzerTest.class}) -@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class WindowExpressionAnalyzerTest extends AnalyzerTestBase { private LogicalPlan child; diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index a0ef436162..74dde6c2e9 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -22,13 +22,10 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; /** * Configuration will be used for UT. */ -@Configuration public class TestConfig { public static final String INT_TYPE_NULL_VALUE_FIELD = "int_null_value"; public static final String INT_TYPE_MISSING_VALUE_FIELD = "int_missing_value"; @@ -57,9 +54,10 @@ public class TestConfig { .put("struct_value", ExprCoreType.STRUCT) .put("array_value", ExprCoreType.ARRAY) .put("timestamp_value", ExprCoreType.TIMESTAMP) + .put("field_value1", ExprCoreType.STRING) + .put("field_value2", ExprCoreType.STRING) .build(); - @Bean protected StorageEngine storageEngine() { return new StorageEngine() { @Override @@ -89,8 +87,6 @@ public PhysicalPlan implement(LogicalPlan plan) { }; } - - @Bean protected SymbolTable symbolTable() { SymbolTable symbolTable = new SymbolTable(); typeMapping.entrySet() @@ -100,7 +96,6 @@ protected SymbolTable symbolTable() { return symbolTable; } - @Bean protected Environment typeEnv() { return var -> { if (var instanceof ReferenceExpression) { diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index 431efd9fb4..4df38027f4 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -32,11 +32,9 @@ import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.split.Split; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; @ExtendWith(MockitoExtension.class) class QueryServiceTest { - private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); private QueryService queryService; diff --git a/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java b/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java index c73bd8ac18..3d735d6762 100644 --- a/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java @@ -29,33 +29,18 @@ import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.function.Function; -import org.junit.jupiter.api.extension.ExtendWith; -import org.opensearch.sql.config.TestConfig; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionProperties; -import org.opensearch.sql.expression.function.FunctionPropertiesTestConfig; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {FunctionPropertiesTestConfig.class, ExpressionTestBase.class, - TestConfig.class}) public class ExpressionTestBase { - @Autowired - protected FunctionProperties functionProperties; + protected FunctionProperties functionProperties = new FunctionProperties(); - @Autowired protected Environment typeEnv; - @Bean protected static Environment valueEnv() { return var -> { if (var instanceof ReferenceExpression) { @@ -95,7 +80,6 @@ protected static Environment valueEnv() { }; } - @Bean protected Environment typeEnv() { return typeEnv; } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateDiffTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateDiffTest.java new file mode 100644 index 0000000000..72c1ceba03 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateDiffTest.java @@ -0,0 +1,77 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.datetime; + +import static java.time.temporal.ChronoUnit.DAYS; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.temporal.Temporal; +import java.util.TimeZone; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class DateDiffTest extends DateTimeTestBase { + + private static final LocalTime timeSample1 = LocalTime.of(12, 42); + private static final LocalTime timeSample2 = LocalTime.of(7, 40); + private static final LocalDate dateSample1 = LocalDate.of(2022, 6, 6); + private static final LocalDate dateSample2 = LocalDate.of(1961, 4, 12); + private static final LocalDate dateSample3 = LocalDate.of(1993, 3, 4); + private static final LocalDate epochStart = LocalDate.of(1970, 1, 1); + private static final LocalDate dateNow = LocalDate.now(); + private static final LocalDateTime dateTimeSample1 = LocalDateTime.of(1961, 4, 12, 9, 7); + private static final LocalDateTime dateTimeSample2 = LocalDateTime.of(1993, 3, 4, 5, 6); + + // Function signature is: + // (DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME) -> LONG + private static Stream getTestData() { + // Arguments are: first argument for `DATE_DIFF` function, second argument and expected result. + return Stream.of( + Arguments.of(timeSample1, timeSample2, 0L), + Arguments.of(timeSample1, dateNow, 0L), + Arguments.of(timeSample1, LocalDateTime.now(), 0L), + Arguments.of(timeSample1, + Instant.now().plusMillis(TimeZone.getDefault().getRawOffset()), 0L), + Arguments.of(dateSample1, timeSample1, + -DAYS.between(dateSample1, dateNow)), + Arguments.of(dateSample1, dateSample3, + -DAYS.between(dateSample1, dateSample3)), + Arguments.of(dateSample1, dateTimeSample1, + -DAYS.between(dateSample1, dateSample2)), + Arguments.of(dateSample1, Instant.ofEpochSecond(42), + -DAYS.between(dateSample1, epochStart)), + Arguments.of(dateTimeSample1, LocalTime.now(), + -DAYS.between(dateSample2, dateNow)), + Arguments.of(dateTimeSample1, dateSample3, + -DAYS.between(dateSample2, dateSample3)), + Arguments.of(dateTimeSample1, dateTimeSample2, + -DAYS.between(dateSample2, dateSample3)), + Arguments.of(dateTimeSample1, Instant.ofEpochSecond(0), + -DAYS.between(dateSample2, epochStart)), + Arguments.of(Instant.ofEpochSecond(0), LocalTime.MAX, + -DAYS.between(epochStart, dateNow)), + Arguments.of(Instant.ofEpochSecond(0), dateSample3, + -DAYS.between(epochStart, dateSample3)), + Arguments.of(Instant.ofEpochSecond(0), dateTimeSample2, + -DAYS.between(epochStart, dateSample3)), + Arguments.of(Instant.ofEpochSecond(0), Instant.now(), + -DAYS.between(epochStart, LocalDateTime.now(ZoneId.of("UTC")))) + ); + } + + @ParameterizedTest + @MethodSource("getTestData") + public void try_different_data(Temporal arg1, Temporal arg2, Long expectedResult) { + assertEquals(expectedResult, datediff(arg1, arg2)); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 092b64d5d7..29a0843287 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -614,6 +614,37 @@ public void hour() { assertEquals("hour(\"2020-08-17 01:02:03\")", expression.toString()); } + private void testInvalidMinuteOfDay(String date) { + FunctionExpression expression = DSL.minute_of_day(DSL.literal(new ExprDateValue(date))); + eval(expression); + } + + @Test + public void invalidMinuteOfDay() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("2022-12-14 12:23:3400")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("2022-12-14 12:2300:34")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("2022-12-14 1200:23:34")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("2022-12-1400 12:23:34")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("2022-1200-14 12:23:34")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("12:23:3400")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("12:2300:34")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("1200:23:34")); + assertThrows(SemanticCheckException.class, + () -> testInvalidMinuteOfDay("asdfasdfasdf")); + + } + @Test public void microsecond() { when(nullRef.type()).thenReturn(TIME); @@ -691,6 +722,48 @@ public void minute() { assertEquals("minute(\"2020-08-17 01:02:03\")", expression.toString()); } + private void testMinuteOfDay(String date, int value) { + FunctionExpression expression = DSL.minute_of_day(DSL.literal(new ExprTimeValue(date))); + assertEquals(INTEGER, expression.type()); + assertEquals(integerValue(value), eval(expression)); + } + + @Test + public void minuteOfDay() { + when(nullRef.type()).thenReturn(TIME); + when(missingRef.type()).thenReturn(TIME); + assertEquals(nullValue(), eval(DSL.minute_of_day(nullRef))); + assertEquals(missingValue(), eval(DSL.minute_of_day(missingRef))); + + FunctionExpression expression = DSL.minute_of_day(DSL.literal(new ExprTimeValue("01:02:03"))); + assertEquals(INTEGER, expression.type()); + assertEquals(integerValue(62), eval(expression)); + assertEquals("minute_of_day(TIME '01:02:03')", expression.toString()); + + expression = DSL.minute_of_day(DSL.literal("01:02:03")); + assertEquals(INTEGER, expression.type()); + assertEquals(integerValue(62), eval(expression)); + assertEquals("minute_of_day(\"01:02:03\")", expression.toString()); + + expression = DSL.minute_of_day(DSL.literal(new ExprTimestampValue("2020-08-17 01:02:03"))); + assertEquals(INTEGER, expression.type()); + assertEquals(integerValue(62), expression.valueOf(env)); + assertEquals("minute_of_day(TIMESTAMP '2020-08-17 01:02:03')", expression.toString()); + + expression = DSL.minute_of_day(DSL.literal(new ExprDatetimeValue("2020-08-17 01:02:03"))); + assertEquals(INTEGER, expression.type()); + assertEquals(integerValue(62), expression.valueOf(env)); + assertEquals("minute_of_day(DATETIME '2020-08-17 01:02:03')", expression.toString()); + + expression = DSL.minute_of_day(DSL.literal("2020-08-17 01:02:03")); + assertEquals(INTEGER, expression.type()); + assertEquals(integerValue(62), expression.valueOf(env)); + assertEquals("minute_of_day(\"2020-08-17 01:02:03\")", expression.toString()); + + testMinuteOfDay("2020-08-17 23:59:59", 1439); + testMinuteOfDay("2020-08-17 00:00:01", 0); + } + @Test public void month() { when(nullRef.type()).thenReturn(DATE); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index 38beaad131..d8829ea41a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.datetime; import static org.opensearch.sql.data.model.ExprValueUtils.fromObjectValue; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import java.time.Instant; import java.time.LocalDate; @@ -17,6 +18,7 @@ import org.opensearch.sql.data.model.ExprDatetimeValue; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.model.ExprNullValue; +import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.expression.DSL; @@ -50,6 +52,17 @@ protected ExprValue addtime(Temporal first, Temporal second) { .valueOf(null); } + protected FunctionExpression datediff(Expression first, Expression second) { + return (FunctionExpression) functionRepository.compile( + functionProperties, + BuiltinFunctionName.DATEDIFF.getName(), List.of(first, second)); + } + + protected Long datediff(Temporal first, Temporal second) { + return datediff(DSL.literal(fromObjectValue(first)), DSL.literal(fromObjectValue(second))) + .valueOf(null).longValue(); + } + protected LocalDateTime fromUnixTime(Double value) { return fromUnixTime(DSL.literal(value)).valueOf().datetimeValue(); } @@ -88,7 +101,6 @@ protected FunctionExpression maketime(Expression hour, Expression minute, Expres BuiltinFunctionName.MAKETIME.getName(), List.of(hour, minute, second)); } - protected LocalTime maketime(Double hour, Double minute, Double second) { return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) .valueOf().timeValue(); @@ -137,6 +149,17 @@ protected ExprValue subtime(Temporal first, Temporal second) { .valueOf(null); } + protected FunctionExpression timediff(Expression first, Expression second) { + return (FunctionExpression) functionRepository.compile( + functionProperties, + BuiltinFunctionName.TIMEDIFF.getName(), List.of(first, second)); + } + + protected LocalTime timediff(LocalTime first, LocalTime second) { + return timediff(DSL.literal(new ExprTimeValue(first)), DSL.literal(new ExprTimeValue(second))) + .valueOf(null).timeValue(); + } + protected FunctionExpression unixTimeStampExpr() { return (FunctionExpression) functionRepository.compile( functionProperties, BuiltinFunctionName.UNIX_TIMESTAMP.getName(), List.of()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java index 6f7548b5cb..4b8a4aac5e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java @@ -19,10 +19,13 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.Period; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.time.temporal.Temporal; import java.time.temporal.TemporalUnit; import java.util.List; +import java.util.TimeZone; import java.util.concurrent.Callable; import java.util.function.BiFunction; import java.util.function.Function; @@ -103,6 +106,31 @@ void current_date() { () -> LocalDate.now(functionProperties.getQueryStartClock())); } + @Test + void utc_date() { + test_now_like_functions(DSL::utc_date, DATE, false, + () -> utcDateTimeNow(functionProperties).toLocalDate()); + } + + @Test + void utc_time() { + test_now_like_functions(DSL::utc_time, TIME, false, + () -> utcDateTimeNow(functionProperties).toLocalTime()); + } + + @Test + void utc_timestamp() { + test_now_like_functions(DSL::utc_timestamp, DATETIME, false, + () -> utcDateTimeNow(functionProperties)); + } + + private static LocalDateTime utcDateTimeNow(FunctionProperties functionProperties) { + ZonedDateTime zonedDateTime = + LocalDateTime.now(functionProperties.getQueryStartClock()) + .atZone(TimeZone.getDefault().toZoneId()); + return zonedDateTime.withZoneSameInstant(ZoneId.of("UTC")).toLocalDateTime(); + } + /** * Check how NOW-like functions are processed. * diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/TimeDiffTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/TimeDiffTest.java new file mode 100644 index 0000000000..8bfb09bd49 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/TimeDiffTest.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.time.LocalTime; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.model.ExprTimeValue; +import org.opensearch.sql.expression.DSL; + +public class TimeDiffTest extends DateTimeTestBase { + + private static Stream getTestData() { + return Stream.of( + Arguments.of(LocalTime.of(12, 42), LocalTime.of(7, 40), LocalTime.of(5, 2)), + Arguments.of(LocalTime.of(7, 40), LocalTime.of(12, 42), LocalTime.of(18, 58)), + Arguments.of(LocalTime.of(7, 40), LocalTime.of(7, 40), LocalTime.of(0, 0)), + Arguments.of(LocalTime.MAX, LocalTime.MIN, LocalTime.MAX) + ); + } + + /** + * Test `TIME_DIFF` function with different data. + * @param arg1 First argument. + * @param arg2 Second argument. + * @param expectedResult Expected result. + */ + @ParameterizedTest + @MethodSource("getTestData") + public void try_different_data(LocalTime arg1, LocalTime arg2, LocalTime expectedResult) { + assertEquals(expectedResult, timediff(arg1, arg2)); + assertEquals(expectedResult, eval(timediff(DSL.literal(new ExprTimeValue(arg1)), + DSL.literal(new ExprTimeValue(arg2)))).timeValue()); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/FunctionPropertiesTestConfig.java b/core/src/test/java/org/opensearch/sql/expression/function/FunctionPropertiesTestConfig.java deleted file mode 100644 index dfc9b543ae..0000000000 --- a/core/src/test/java/org/opensearch/sql/expression/function/FunctionPropertiesTestConfig.java +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class FunctionPropertiesTestConfig { - @Bean - FunctionProperties functionProperties() { - return new FunctionProperties(); - } -} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java index 787ca016c9..6e4fff2fb0 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java @@ -197,4 +197,12 @@ void query_string() { fields.getValue(), query.getValue()), expr.toString()); } + + @Test + void wildcard_query() { + FunctionExpression expr = DSL.wildcard_query(field, query); + assertEquals(String.format("wildcard_query(field=%s, query=%s)", + field.getValue(), query.getValue()), + expr.toString()); + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/RelevanceFunctionResolverTest.java b/core/src/test/java/org/opensearch/sql/expression/function/RelevanceFunctionResolverTest.java index d8547057c4..deba721481 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/RelevanceFunctionResolverTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/RelevanceFunctionResolverTest.java @@ -24,7 +24,7 @@ class RelevanceFunctionResolverTest { @BeforeEach void setUp() { - resolver = new RelevanceFunctionResolver(sampleFuncName, STRING); + resolver = new RelevanceFunctionResolver(sampleFuncName); } @Test @@ -44,15 +44,6 @@ void resolve_invalid_name_test() { exception.getMessage()); } - @Test - void resolve_invalid_first_param_type_test() { - var sig = new FunctionSignature(sampleFuncName, List.of(INTEGER)); - Exception exception = assertThrows(SemanticCheckException.class, - () -> resolver.resolve(sig)); - assertEquals("Expected type STRING instead of INTEGER for parameter #1", - exception.getMessage()); - } - @Test void resolve_invalid_third_param_type_test() { var sig = new FunctionSignature(sampleFuncName, List.of(STRING, STRING, INTEGER, STRING)); diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java index 3d7cdaeb41..d6d15e9315 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java @@ -192,12 +192,12 @@ public void ceil_int_value(Integer value) { assertThat( ceil.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); + assertEquals(String.format("ceil(%s)", value), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); + assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); } /** @@ -209,12 +209,12 @@ public void ceil_long_value(Long value) { FunctionExpression ceil = DSL.ceil(DSL.literal(value)); assertThat( ceil.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); + assertEquals(String.format("ceil(%s)", value), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); + assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); } /** @@ -226,12 +226,12 @@ public void ceil_float_value(Float value) { FunctionExpression ceil = DSL.ceil(DSL.literal(value)); assertThat( ceil.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); + assertEquals(String.format("ceil(%s)", value), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); + assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); } /** @@ -243,12 +243,12 @@ public void ceil_double_value(Double value) { FunctionExpression ceil = DSL.ceil(DSL.literal(value)); assertThat( ceil.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceil(%s)", value.toString()), ceil.toString()); + assertEquals(String.format("ceil(%s)", value), ceil.toString()); FunctionExpression ceiling = DSL.ceiling(DSL.literal(value)); assertThat( ceiling.valueOf(valueEnv()), allOf(hasType(INTEGER), hasValue((int) Math.ceil(value)))); - assertEquals(String.format("ceiling(%s)", value.toString()), ceiling.toString()); + assertEquals(String.format("ceiling(%s)", value), ceiling.toString()); } /** @@ -1721,12 +1721,12 @@ public void sqrt_missing_value() { * Test truncate with integer value. */ @ParameterizedTest(name = "truncate({0}, {1})") - @ValueSource(ints = {2, -2}) + @ValueSource(ints = {2, -2, Integer.MAX_VALUE, Integer.MIN_VALUE}) public void truncate_int_value(Integer value) { FunctionExpression truncate = DSL.truncate(DSL.literal(value), DSL.literal(1)); assertThat( truncate.valueOf(valueEnv()), allOf(hasType(LONG), - hasValue(new BigDecimal(value).setScale(1, RoundingMode.DOWN).longValue()))); + hasValue(BigDecimal.valueOf(value).setScale(1, RoundingMode.DOWN).longValue()))); assertEquals(String.format("truncate(%s, 1)", value), truncate.toString()); } @@ -1734,12 +1734,12 @@ public void truncate_int_value(Integer value) { * Test truncate with long value. */ @ParameterizedTest(name = "truncate({0}, {1})") - @ValueSource(longs = {2L, -2L}) + @ValueSource(longs = {2L, -2L, Long.MAX_VALUE, Long.MIN_VALUE}) public void truncate_long_value(Long value) { FunctionExpression truncate = DSL.truncate(DSL.literal(value), DSL.literal(1)); assertThat( truncate.valueOf(valueEnv()), allOf(hasType(LONG), - hasValue(new BigDecimal(value).setScale(1, RoundingMode.DOWN).longValue()))); + hasValue(BigDecimal.valueOf(value).setScale(1, RoundingMode.DOWN).longValue()))); assertEquals(String.format("truncate(%s, 1)", value), truncate.toString()); } @@ -1747,12 +1747,12 @@ public void truncate_long_value(Long value) { * Test truncate with float value. */ @ParameterizedTest(name = "truncate({0}, {1})") - @ValueSource(floats = {2F, -2F}) + @ValueSource(floats = {2F, -2F, Float.MAX_VALUE, Float.MIN_VALUE}) public void truncate_float_value(Float value) { FunctionExpression truncate = DSL.truncate(DSL.literal(value), DSL.literal(1)); assertThat( truncate.valueOf(valueEnv()), allOf(hasType(DOUBLE), - hasValue(new BigDecimal(value).setScale(1, RoundingMode.DOWN).doubleValue()))); + hasValue(BigDecimal.valueOf(value).setScale(1, RoundingMode.DOWN).doubleValue()))); assertEquals(String.format("truncate(%s, 1)", value), truncate.toString()); } @@ -1760,12 +1760,15 @@ public void truncate_float_value(Float value) { * Test truncate with double value. */ @ParameterizedTest(name = "truncate({0}, {1})") - @ValueSource(doubles = {2D, -2D}) + @ValueSource(doubles = {2D, -9.223372036854776e+18D, -2147483649.0D, -2147483648.0D, + -32769.0D, -32768.0D, -34.84D, -2.0D, -1.2D, -1.0D, 0.0D, 1.0D, + 1.3D, 2.0D, 1004.3D, 32767.0D, 32768.0D, 2147483647.0D, 2147483648.0D, + 9.223372036854776e+18D, Double.MAX_VALUE, Double.MIN_VALUE}) public void truncate_double_value(Double value) { FunctionExpression truncate = DSL.truncate(DSL.literal(value), DSL.literal(1)); assertThat( truncate.valueOf(valueEnv()), allOf(hasType(DOUBLE), - hasValue(new BigDecimal(value).setScale(1, RoundingMode.DOWN).doubleValue()))); + hasValue(BigDecimal.valueOf(value).setScale(1, RoundingMode.DOWN).doubleValue()))); assertEquals(String.format("truncate(%s, 1)", value), truncate.toString()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/text/TextFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/text/TextFunctionTest.java index 5e32678b94..515b436c82 100644 --- a/core/src/test/java/org/opensearch/sql/expression/text/TextFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/text/TextFunctionTest.java @@ -412,6 +412,18 @@ void replace() { assertEquals(missingValue(), eval(DSL.replace(missingRef, DSL.literal("a"), DSL.literal("b")))); } + @Test + void reverse() { + FunctionExpression expression = DSL.reverse(DSL.literal("abcde")); + assertEquals(STRING, expression.type()); + assertEquals("edcba", eval(expression).stringValue()); + + when(nullRef.type()).thenReturn(STRING); + assertEquals(nullValue(), eval(DSL.reverse(nullRef))); + when(missingRef.type()).thenReturn(STRING); + assertEquals(missingValue(), eval(DSL.reverse(missingRef))); + } + void testConcatString(List strings) { String expected = null; if (strings.stream().noneMatch(Objects::isNull)) { diff --git a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java index 2322e4684e..017cfb60ea 100644 --- a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java @@ -58,6 +58,8 @@ import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.TableScanOperator; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; +import org.opensearch.sql.storage.write.TableWriteOperator; @ExtendWith(MockitoExtension.class) class DefaultImplementorTest { @@ -212,4 +214,17 @@ public TableScanOperator build() { }; assertEquals(tableScanOperator, tableScanBuilder.accept(implementor, null)); } + + @Test + public void visitTableWriteBuilderShouldBuildTableWriteOperator() { + LogicalPlan child = values(); + TableWriteOperator tableWriteOperator = Mockito.mock(TableWriteOperator.class); + TableWriteBuilder logicalPlan = new TableWriteBuilder(child) { + @Override + public TableWriteOperator build(PhysicalPlan child) { + return tableWriteOperator; + } + }; + assertEquals(tableWriteOperator, logicalPlan.accept(implementor, null)); + } } diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalDedupeTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalDedupeTest.java index 84cef2cbf3..fe751506ce 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalDedupeTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalDedupeTest.java @@ -18,16 +18,9 @@ import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.analysis.AnalyzerTestBase; import org.opensearch.sql.expression.DSL; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {AnalyzerTestBase.class}) class LogicalDedupeTest extends AnalyzerTestBase { @Test public void analyze_dedup_with_two_field_with_default_option() { diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalEvalTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalEvalTest.java index 63429453f9..55bc793045 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalEvalTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalEvalTest.java @@ -16,13 +16,7 @@ import org.opensearch.sql.analysis.AnalyzerTestBase; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.expression.DSL; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {AnalyzerTestBase.class}) @ExtendWith(MockitoExtension.class) public class LogicalEvalTest extends AnalyzerTestBase { diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index 33c6b02c38..341bcbc29e 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -12,6 +12,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -31,9 +32,12 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.Aggregator; import org.opensearch.sql.expression.window.WindowDefinition; +import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.TableScanOperator; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; +import org.opensearch.sql.storage.write.TableWriteOperator; /** * Todo. Temporary added for UT coverage, Will be removed. @@ -83,6 +87,19 @@ public TableScanOperator build() { assertNull(tableScanBuilder.accept(new LogicalPlanNodeVisitor() { }, null)); + LogicalPlan write = LogicalPlanDSL.write(null, table, Collections.emptyList()); + assertNull(write.accept(new LogicalPlanNodeVisitor() { + }, null)); + + TableWriteBuilder tableWriteBuilder = new TableWriteBuilder(null) { + @Override + public TableWriteOperator build(PhysicalPlan child) { + return null; + } + }; + assertNull(tableWriteBuilder.accept(new LogicalPlanNodeVisitor() { + }, null)); + LogicalPlan filter = LogicalPlanDSL.filter(relation, expression); assertNull(filter.accept(new LogicalPlanNodeVisitor() { }, null)); diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalSortTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalSortTest.java index deeda2b82c..1a33795009 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalSortTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalSortTest.java @@ -19,17 +19,10 @@ import org.apache.commons.lang3.tuple.ImmutablePair; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.opensearch.sql.analysis.AnalyzerTestBase; import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.expression.DSL; -import org.springframework.context.annotation.Configuration; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@Configuration -@ExtendWith(SpringExtension.class) -@ContextConfiguration(classes = {AnalyzerTestBase.class}) class LogicalSortTest extends AnalyzerTestBase { @Test public void analyze_sort_with_two_field_with_default_option() { diff --git a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java index e2510ec464..7516aa1809 100644 --- a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.planner.optimizer; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; @@ -20,6 +21,8 @@ import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.sort; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.values; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.write; import com.google.common.collect.ImmutableList; import java.util.Collections; @@ -39,6 +42,7 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; @ExtendWith(MockitoExtension.class) class LogicalPlanOptimizerTest { @@ -270,6 +274,37 @@ public PhysicalPlan implement(LogicalPlan plan) { ); } + @Test + void table_support_write_builder_should_be_replaced() { + Mockito.reset(table, tableScanBuilder); + TableWriteBuilder writeBuilder = Mockito.mock(TableWriteBuilder.class); + when(table.createWriteBuilder(any())).thenReturn(writeBuilder); + + assertEquals( + writeBuilder, + optimize(write(values(), table, Collections.emptyList())) + ); + } + + @Test + void table_not_support_write_builder_should_report_error() { + Mockito.reset(table, tableScanBuilder); + Table table = new Table() { + @Override + public Map getFieldTypes() { + return null; + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return null; + } + }; + + assertThrows(UnsupportedOperationException.class, + () -> table.createWriteBuilder(null)); + } + private LogicalPlan optimize(LogicalPlan plan) { final LogicalPlanOptimizer optimizer = LogicalPlanOptimizer.create(); final LogicalPlan optimize = optimizer.optimize(plan); diff --git a/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java b/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java index 61d192362a..9f90fd8d05 100644 --- a/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java @@ -35,5 +35,6 @@ void source_is_empty() { void table_is_empty() { plan = Mockito.mock(LogicalFilter.class); assertFalse(Patterns.table().getFunction().apply(plan).isPresent()); + assertFalse(Patterns.writeTable().getFunction().apply(plan).isPresent()); } } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java index 97bf8ebb79..a5bb61f2f7 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanTestBase.java @@ -22,7 +22,6 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; -import org.springframework.context.annotation.Bean; public class PhysicalPlanTestBase { @@ -183,7 +182,6 @@ public class PhysicalPlanTestBase { "errors", 8))) .build(); - @Bean protected Environment typeEnv() { return var -> { if (var instanceof ReferenceExpression) { diff --git a/core/src/test/java/org/opensearch/sql/storage/write/TableWriteOperatorTest.java b/core/src/test/java/org/opensearch/sql/storage/write/TableWriteOperatorTest.java new file mode 100644 index 0000000000..8780b08276 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/storage/write/TableWriteOperatorTest.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.write; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlanNodeVisitor; + +@ExtendWith(MockitoExtension.class) +class TableWriteOperatorTest { + + @Mock + private PhysicalPlan child; + + private TableWriteOperator tableWrite; + + @BeforeEach + void setUp() { + tableWrite = new TableWriteOperator(child) { + @Override + public String explain() { + return "explain"; + } + + @Override + public boolean hasNext() { + return false; + } + + @Override + public ExprValue next() { + return null; + } + }; + } + + @Test + void testAccept() { + Boolean isVisited = tableWrite.accept(new PhysicalPlanNodeVisitor<>() { + @Override + protected Boolean visitNode(PhysicalPlan node, Object context) { + return (node instanceof TableWriteOperator); + } + + @Override + public Boolean visitTableWrite(TableWriteOperator node, Object context) { + return super.visitTableWrite(node, context); + } + }, null); + + assertTrue(isVisited); + } + + @Test + void testGetChild() { + assertEquals(Collections.singletonList(child), tableWrite.getChild()); + } +} \ No newline at end of file diff --git a/docs/category.json b/docs/category.json index b8feaa654a..e90c674a2e 100644 --- a/docs/category.json +++ b/docs/category.json @@ -46,6 +46,7 @@ "user/dql/window.rst", "user/beyond/partiql.rst", "user/dql/aggregations.rst", - "user/dql/complex.rst" + "user/dql/complex.rst", + "user/dql/metadata.rst" ] } diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index a167ce29b5..09818c902d 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -132,6 +132,10 @@ Operators +----------------+----------------------------------------+ | NOT IN | NOT IN value list test | +----------------+----------------------------------------+ +| BETWEEN | Between a range (endpoint inclusive) | ++----------------+----------------------------------------+ +| NOT BETWEEN | Not between a range (BETWEEN negation) | ++----------------+----------------------------------------+ Basic Comparison Operator ------------------------- @@ -149,12 +153,12 @@ Here is an example for different type of comparison operators:: LIKE ---- -expr LIKE pattern. The expr is string value, pattern is supports literal text, a percent ( % ) character for a wildcard, and an underscore ( _ ) character for a single character match:: +expr LIKE pattern. The expr is string value, pattern is supports literal text, a percent ( % ) character for a wildcard, and an underscore ( _ ) character for a single character match, pattern is case insensitive:: - os> SELECT 'axyzb' LIKE 'a%b', 'acb' LIKE 'a_b', 'axyzb' NOT LIKE 'a%b', 'acb' NOT LIKE 'a_b'; + os> SELECT 'axyzb' LIKE 'a%b', 'acb' LIKE 'A_B', 'axyzb' NOT LIKE 'a%b', 'acb' NOT LIKE 'a_b'; fetched rows / total rows = 1/1 +----------------------+--------------------+--------------------------+------------------------+ - | 'axyzb' LIKE 'a%b' | 'acb' LIKE 'a_b' | 'axyzb' NOT LIKE 'a%b' | 'acb' NOT LIKE 'a_b' | + | 'axyzb' LIKE 'a%b' | 'acb' LIKE 'A_B' | 'axyzb' NOT LIKE 'a%b' | 'acb' NOT LIKE 'a_b' | |----------------------+--------------------+--------------------------+------------------------| | True | True | False | False | +----------------------+--------------------+--------------------------+------------------------+ @@ -199,6 +203,23 @@ Here is an example for IN value test:: | True | True | +---------------+-------------------+ +BETWEEN range test +------------------ + +Here is an example for range test by BETWEEN expression:: + + os> SELECT + ... 1 BETWEEN 1 AND 3, + ... 4 BETWEEN 1 AND 3, + ... 4 NOT BETWEEN 1 AND 3; + fetched rows / total rows = 1/1 + +---------------------+---------------------+-------------------------+ + | 1 BETWEEN 1 AND 3 | 4 BETWEEN 1 AND 3 | 4 NOT BETWEEN 1 AND 3 | + |---------------------+---------------------+-------------------------| + | True | False | True | + +---------------------+---------------------+-------------------------+ + + Function Call ============= diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index f7383d862d..1e08a729c0 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1414,6 +1414,26 @@ Example:: +-------------------------------------------------+-----------------------------------+-------------------------------------------------+ +DATEDIFF +-------- + +Usage: Calculates the difference of date parts of the given values. If the first argument is time, today's date is used. + +Argument type: DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME + +Return type: LONG + +Example:: + + os> SELECT DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')) AS `'2000-01-02' - '2000-01-01'`, DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00')) AS `'2001-02-01' - '2004-01-01'`, DATEDIFF(TIME('23:59:59'), TIME('00:00:00')) AS `today - today` + fetched rows / total rows = 1/1 + +-------------------------------+-------------------------------+-----------------+ + | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | + |-------------------------------+-------------------------------+-----------------| + | 1 | -1064 | 0 | + +-------------------------------+-------------------------------+-----------------+ + + DAY --- @@ -1823,6 +1843,28 @@ Example:: | 2 | +-----------------------------+ +MINUTE_OF_DAY +------ + +Description +>>>>>>>>>>> + +Usage: minute_of_day(time) returns the minute value for time within a 24 hour day, in the range 0 to 1439. + +Argument type: STRING/TIME/DATETIME/TIMESTAMP + +Return type: INTEGER + +Example:: + + os> SELECT MINUTE_OF_DAY((TIME '01:02:03')) + fetched rows / total rows = 1/1 + +------------------------------------+ + | MINUTE_OF_DAY((TIME '01:02:03')) | + |------------------------------------| + | 62 | + +------------------------------------+ + MONTH ----- @@ -2161,6 +2203,29 @@ Example:: +--------------------------------+ +TIMEDIFF +-------- + +Description +>>>>>>>>>>> + +Usage: returns the difference between two time expressions as a time. + +Argument type: TIME, TIME + +Return type: TIME + +Example:: + + os> SELECT TIMEDIFF('23:59:59', '13:00:00') + fetched rows / total rows = 1/1 + +------------------------------------+ + | TIMEDIFF('23:59:59', '13:00:00') | + |------------------------------------| + | 10:59:59 | + +------------------------------------+ + + TIMESTAMP --------- @@ -2241,6 +2306,74 @@ Examples:: +----------------------------------------------------+ +UTC_DATE +-------- + +Description +>>>>>>>>>>> + +Returns the current UTC date as a value in 'YYYY-MM-DD'. + +Return type: DATE + +Specification: UTC_DATE() -> DATE + +Example:: + + > SELECT UTC_DATE(); + fetched rows / total rows = 1/1 + +--------------+ + | utc_date() | + |--------------| + | 2022-10-03 | + +--------------+ + + +UTC_TIME +-------- + +Description +>>>>>>>>>>> + +Returns the current UTC time as a value in 'hh:mm:ss'. + +Return type: TIME + +Specification: UTC_TIME() -> TIME + +Example:: + + > SELECT UTC_TIME(); + fetched rows / total rows = 1/1 + +--------------+ + | utc_time() | + |--------------| + | 17:54:27 | + +--------------+ + + +UTC_TIMESTAMP +------------- + +Description +>>>>>>>>>>> + +Returns the current UTC timestamp as a value in 'YYYY-MM-DD hh:mm:ss'. + +Return type: DATETIME + +Specification: UTC_TIMESTAMP() -> DATETIME + +Example:: + + > SELECT UTC_TIMESTAMP(); + fetched rows / total rows = 1/1 + +---------------------+ + | utc_timestamp() | + |---------------------| + | 2022-10-03 17:54:28 | + +---------------------+ + WEEK ---- @@ -2590,6 +2723,29 @@ Example:: +--------------------------------------------------+ +REVERSE +------- + +Description +>>>>>>>>>>> + +Usage: REVERSE(str) returns reversed string of the string supplied as an argument. Returns NULL if the argument is NULL. + +Argument type: STRING + +Return type: STRING + +Example:: + + os> SELECT REVERSE('abcde'), REVERSE(null) + fetched rows / total rows = 1/1 + +--------------------+-----------------+ + | REVERSE('abcde') | REVERSE(null) | + |--------------------+-----------------| + | edcba | null | + +--------------------+-----------------+ + + RIGHT ----- @@ -3010,6 +3166,16 @@ Another example to show how to set custom values for the optional parameters:: | Bond | +------------+ + The matchquery function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = matchquery('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + MATCH_QUERY ----- @@ -3039,6 +3205,16 @@ Another example to show how to set custom values for the optional parameters:: | Bond | +------------+ +The match_query function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = match_query('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + MATCH_PHRASE ------------ @@ -3078,6 +3254,23 @@ Another example to show how to set custom values for the optional parameters:: | Alan Alexander Milne | Winnie-the-Pooh | +----------------------+--------------------------+ +The match_phrase function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = match_phrase('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + + os> SELECT firstname FROM accounts WHERE firstname = matchphrase('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ MATCH_BOOL_PREFIX ----- @@ -3221,6 +3414,24 @@ Another example to show how to set custom values for the optional parameters:: | 1 | The House at Pooh Corner | Alan Alexander Milne | +------+--------------------------+----------------------+ +The multi_match function also supports an alternative syntax:: + + os> SELECT firstname FROM accounts WHERE firstname = multi_match('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + + os> SELECT firstname FROM accounts WHERE firstname = multimatch('Hattie'); + fetched rows / total rows = 1/1 + +-------------+ + | firstname | + |-------------| + | Hattie | + +-------------+ + SIMPLE_QUERY_STRING ------------------- @@ -3421,6 +3632,59 @@ Example searching for field Tags:: | [Winnie-the-Pooh] | +----------------------------------------------+ +WILDCARD_QUERY +------------ + +Description +>>>>>>>>>>> + +``wildcard_query(field_expression, query_expression[, option=]*)`` + +The ``wildcard_query`` function maps to the ``wildcard_query`` query used in search engine. It returns documents that match provided text in the specified field. +OpenSearch supports wildcard characters ``*`` and ``?``. See the full description here: https://opensearch.org/docs/latest/opensearch/query-dsl/term/#wildcards. +You may include a backslash ``\`` to escape SQL wildcard characters ``\%`` and ``\_``. + +Available parameters include: + +- boost +- case_insensitive +- rewrite + +For backward compatibility, ``wildcardquery`` is also supported and mapped to ``wildcard_query`` query as well. + +Example with only ``field`` and ``query`` expressions, and all other parameters are set default values:: + + os> select Body from wildcard where wildcard_query(Body, 'test wildcard*'); + fetched rows / total rows = 7/7 + +-------------------------------------------+ + | Body | + |-------------------------------------------| + | test wildcard | + | test wildcard in the end of the text% | + | test wildcard in % the middle of the text | + | test wildcard %% beside each other | + | test wildcard in the end of the text_ | + | test wildcard in _ the middle of the text | + | test wildcard __ beside each other | + +-------------------------------------------+ + +Another example to show how to set custom values for the optional parameters:: + + os> select Body from wildcard where wildcard_query(Body, 'test wildcard*', boost=0.7, case_insensitive=true, rewrite='constant_score'); + fetched rows / total rows = 8/8 + +-------------------------------------------+ + | Body | + |-------------------------------------------| + | test wildcard | + | test wildcard in the end of the text% | + | test wildcard in % the middle of the text | + | test wildcard %% beside each other | + | test wildcard in the end of the text_ | + | test wildcard in _ the middle of the text | + | test wildcard __ beside each other | + | tEsT wIlDcArD sensitive cases | + +-------------------------------------------+ + System Functions ================ @@ -3445,3 +3709,5 @@ Example:: |----------------+---------------+-----------------+------------------| | DATE | INTEGER | DATETIME | STRUCT | +----------------+---------------+-----------------+------------------+ + + diff --git a/docs/user/dql/metadata.rst b/docs/user/dql/metadata.rst index 7ffc2ffe38..5b34455974 100644 --- a/docs/user/dql/metadata.rst +++ b/docs/user/dql/metadata.rst @@ -21,13 +21,11 @@ You can query your indices metadata by ``SHOW`` and ``DESCRIBE`` statement. Thes Syntax ------ -Rule ``showStatement``: +``SHOW TABLES LIKE ""`` -.. image:: /docs/user/img/rdd/showStatement.png +``DESCRIBE TABLES LIKE "" [COLUMNS LIKE ""]`` -Rule ``showFilter``: - -.. image:: /docs/user/img/rdd/showFilter.png +Pattern accepts SQL style wildcards where `_` mathes any character and `%` matches any characters. Example 1: Show All Indices Information --------------------------------------- @@ -36,21 +34,19 @@ Example 1: Show All Indices Information SQL query:: - POST /_plugins/_sql - { - "query" : "SHOW TABLES LIKE %" - } - -Result set: - -+---------+-----------+----------------+----------+-------+--------+----------+---------+-------------------------+--------------+ -|TABLE_CAT|TABLE_SCHEM| TABLE_NAME|TABLE_TYPE|REMARKS|TYPE_CAT|TYPE_SCHEM|TYPE_NAME|SELF_REFERENCING_COL_NAME|REF_GENERATION| -+=========+===========+================+==========+=======+========+==========+=========+=========================+==============+ -|integTest| null| accounts|BASE TABLE| null| null| null| null| null| null| -+---------+-----------+----------------+----------+-------+--------+----------+---------+-------------------------+--------------+ -|integTest| null|employees_nested|BASE TABLE| null| null| null| null| null| null| -+---------+-----------+----------------+----------+-------+--------+----------+---------+-------------------------+--------------+ - + os> SHOW TABLES LIKE '%' + fetched rows / total rows = 7/7 + +----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | + |----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------| + | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | apache | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | books | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | nyc_taxi | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | people | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | wildcard | BASE TABLE | null | null | null | null | null | null | + +----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ Example 2: Show Specific Index Information ------------------------------------------ @@ -59,19 +55,14 @@ Here is an example that searches metadata for index name prefixed by 'acc'. Besi SQL query:: - POST /_plugins/_sql - { - "query" : "SHOW TABLES LIKE acc%" - } - -Result set: - -+---------+-----------+----------+----------+-------+--------+----------+---------+-------------------------+--------------+ -|TABLE_CAT|TABLE_SCHEM|TABLE_NAME|TABLE_TYPE|REMARKS|TYPE_CAT|TYPE_SCHEM|TYPE_NAME|SELF_REFERENCING_COL_NAME|REF_GENERATION| -+=========+===========+==========+==========+=======+========+==========+=========+=========================+==============+ -|integTest| null| accounts|BASE TABLE| null| null| null| null| null| null| -+---------+-----------+----------+----------+-------+--------+----------+---------+-------------------------+--------------+ - + os> SHOW TABLES LIKE "acc%" + fetched rows / total rows = 2/2 + +----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | TABLE_TYPE | REMARKS | TYPE_CAT | TYPE_SCHEM | TYPE_NAME | SELF_REFERENCING_COL_NAME | REF_GENERATION | + |----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------| + | docTestCluster | null | account2 | BASE TABLE | null | null | null | null | null | null | + | docTestCluster | null | accounts | BASE TABLE | null | null | null | null | null | null | + +----------------+---------------+--------------+--------------+-----------+------------+--------------+-------------+-----------------------------+------------------+ Example 3: Describe Index Fields Information -------------------------------------------- @@ -80,37 +71,36 @@ Example 3: Describe Index Fields Information SQL query:: - POST /_plugins/_sql - { - "query" : "DESCRIBE TABLES LIKE accounts" - } - -Result set: - -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|TABLE_CAT|TABLE_SCHEM|TABLE_NAME| COLUMN_NAME|DATA_TYPE|TYPE_NAME|COLUMN_SIZE|BUFFER_LENGTH|DECIMAL_DIGITS|NUM_PREC_RADIX|NULLABLE|REMARKS|COLUMN_DEF|SQL_DATA_TYPE|SQL_DATETIME_SUB|CHAR_OCTET_LENGTH|ORDINAL_POSITION|IS_NULLABLE|SCOPE_CATALOG|SCOPE_SCHEMA|SCOPE_TABLE|SOURCE_DATA_TYPE|IS_AUTOINCREMENT|IS_GENERATEDCOLUMN| -+=========+===========+==========+==============+=========+=========+===========+=============+==============+==============+========+=======+==========+=============+================+=================+================+===========+=============+============+===========+================+================+==================+ -|integTest| null| accounts|account_number| null| long| null| null| null| 10| 2| null| null| null| null| null| 1| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| firstname| null| text| null| null| null| 10| 2| null| null| null| null| null| 2| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| address| null| text| null| null| null| 10| 2| null| null| null| null| null| 3| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| balance| null| long| null| null| null| 10| 2| null| null| null| null| null| 4| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| gender| null| text| null| null| null| 10| 2| null| null| null| null| null| 5| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| city| null| text| null| null| null| 10| 2| null| null| null| null| null| 6| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| employer| null| text| null| null| null| 10| 2| null| null| null| null| null| 7| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| state| null| text| null| null| null| 10| 2| null| null| null| null| null| 8| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| age| null| long| null| null| null| 10| 2| null| null| null| null| null| 9| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| email| null| text| null| null| null| 10| 2| null| null| null| null| null| 10| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ -|integTest| null| accounts| lastname| null| text| null| null| null| 10| 2| null| null| null| null| null| 11| | null| null| null| null| NO| | -+---------+-----------+----------+--------------+---------+---------+-----------+-------------+--------------+--------------+--------+-------+----------+-------------+----------------+-----------------+----------------+-----------+-------------+------------+-----------+----------------+----------------+------------------+ + os> DESCRIBE TABLES LIKE 'accounts' + fetched rows / total rows = 11/11 + +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------| + | docTestCluster | null | accounts | account_number | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 0 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | address | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 2 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | balance | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 3 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | gender | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 4 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | city | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 5 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | employer | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 6 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | state | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 7 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | age | null | long | null | null | null | 10 | 2 | null | null | null | null | null | 8 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | email | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 9 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+---------------+--------------+----------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ + +Example 4: Describe Index With Fields Filter +-------------------------------------------- +``DESCRIBE`` statement fields that can match the search pattern for indices that can match the search pattern. + +SQL query:: + os> DESCRIBE TABLES LIKE "accounts" COLUMNS LIKE "%name" + fetched rows / total rows = 2/2 + +----------------+---------------+--------------+---------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ + | TABLE_CAT | TABLE_SCHEM | TABLE_NAME | COLUMN_NAME | DATA_TYPE | TYPE_NAME | COLUMN_SIZE | BUFFER_LENGTH | DECIMAL_DIGITS | NUM_PREC_RADIX | NULLABLE | REMARKS | COLUMN_DEF | SQL_DATA_TYPE | SQL_DATETIME_SUB | CHAR_OCTET_LENGTH | ORDINAL_POSITION | IS_NULLABLE | SCOPE_CATALOG | SCOPE_SCHEMA | SCOPE_TABLE | SOURCE_DATA_TYPE | IS_AUTOINCREMENT | IS_GENERATEDCOLUMN | + |----------------+---------------+--------------+---------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------| + | docTestCluster | null | accounts | firstname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 1 | | null | null | null | null | NO | | + | docTestCluster | null | accounts | lastname | null | text | null | null | null | 10 | 2 | null | null | null | null | null | 10 | | null | null | null | null | NO | | + +----------------+---------------+--------------+---------------+-------------+-------------+---------------+-----------------+------------------+------------------+------------+-----------+--------------+-----------------+--------------------+---------------------+--------------------+---------------+-----------------+----------------+---------------+--------------------+--------------------+----------------------+ diff --git a/docs/user/img/rdd/showFilter.png b/docs/user/img/rdd/showFilter.png deleted file mode 100644 index 47dbc0746e..0000000000 Binary files a/docs/user/img/rdd/showFilter.png and /dev/null differ diff --git a/docs/user/img/rdd/showStatement.png b/docs/user/img/rdd/showStatement.png deleted file mode 100644 index a1939e7d8c..0000000000 Binary files a/docs/user/img/rdd/showStatement.png and /dev/null differ diff --git a/docs/user/limitations/limitations.rst b/docs/user/limitations/limitations.rst index d6e763a73d..8ce75a0e25 100644 --- a/docs/user/limitations/limitations.rst +++ b/docs/user/limitations/limitations.rst @@ -34,23 +34,6 @@ Aggregation over expression is not supported for now. You can only apply aggrega Here's a link to the Github issue - [Issue #288](https://github.com/opendistro-for-elasticsearch/sql/issues/288). -Limitations on Subqueries -========================= - -Subqueries in the FROM clause ------------------------------ - -Subquery in the `FROM` clause in this format: `SELECT outer FROM (SELECT inner)` is supported only when the query is merged into one query. For example, the following query is supported:: - - SELECT t.f, t.d - FROM ( - SELECT FlightNum as f, DestCountry as d - FROM opensearch_dashboards_sample_data_flights - WHERE OriginCountry = 'US') t - -But, if the outer query has `GROUP BY` or `ORDER BY`, then it's not supported. - - Limitations on JOINs ==================== diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 2cc83eba97..5e54d8e80f 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -599,6 +599,26 @@ Example:: +-------------------------------------------------+-----------------------------------+-------------------------------------------------+ +DATEDIFF +-------- + +Usage: Calculates the difference of date parts of given values. If the first argument is time, today's date is used. + +Argument type: DATE/DATETIME/TIMESTAMP/TIME, DATE/DATETIME/TIMESTAMP/TIME + +Return type: LONG + +Example:: + + os> source=people | eval `'2000-01-02' - '2000-01-01'` = DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')), `'2001-02-01' - '2004-01-01'` = DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00')), `today - today` = DATEDIFF(TIME('23:59:59'), TIME('00:00:00')) | fields `'2000-01-02' - '2000-01-01'`, `'2001-02-01' - '2004-01-01'`, `today - today` + fetched rows / total rows = 1/1 + +-------------------------------+-------------------------------+-----------------+ + | '2000-01-02' - '2000-01-01' | '2001-02-01' - '2004-01-01' | today - today | + |-------------------------------+-------------------------------+-----------------| + | 1 | -1064 | 0 | + +-------------------------------+-------------------------------+-----------------+ + + DAY --- @@ -1302,6 +1322,29 @@ Example:: +---------------------------------+ +TIMEDIFF +-------- + +Description +>>>>>>>>>>> + +Usage: returns the difference between two time expressions as a time. + +Argument type: TIME, TIME + +Return type: TIME + +Example:: + + os> source=people | eval `TIMEDIFF('23:59:59', '13:00:00')` = TIMEDIFF('23:59:59', '13:00:00') | fields `TIMEDIFF('23:59:59', '13:00:00')` + fetched rows / total rows = 1/1 + +------------------------------------+ + | TIMEDIFF('23:59:59', '13:00:00') | + |------------------------------------| + | 10:59:59 | + +------------------------------------+ + + TIMESTAMP --------- @@ -1374,6 +1417,75 @@ Example:: +--------------------------+-----------------------------+ +UTC_DATE +-------- + +Description +>>>>>>>>>>> + +Returns the current UTC date as a value in 'YYYY-MM-DD'. + +Return type: DATE + +Specification: UTC_DATE() -> DATE + +Example:: + + > source=people | eval `UTC_DATE()` = UTC_DATE() | fields `UTC_DATE()` + fetched rows / total rows = 1/1 + +--------------+ + | UTC_DATE() | + |--------------| + | 2022-10-03 | + +--------------+ + + +UTC_TIME +-------- + +Description +>>>>>>>>>>> + +Returns the current UTC time as a value in 'hh:mm:ss'. + +Return type: TIME + +Specification: UTC_TIME() -> TIME + +Example:: + + > source=people | eval `UTC_TIME()` = UTC_TIME() | fields `UTC_TIME()` + fetched rows / total rows = 1/1 + +--------------+ + | UTC_TIME() | + |--------------| + | 17:54:27 | + +--------------+ + + +UTC_TIMESTAMP +------------- + +Description +>>>>>>>>>>> + +Returns the current UTC timestamp as a value in 'YYYY-MM-DD hh:mm:ss'. + +Return type: DATETIME + +Specification: UTC_TIMESTAMP() -> DATETIME + +Example:: + + > source=people | eval `UTC_TIMESTAMP()` = UTC_TIMESTAMP() | fields `UTC_TIMESTAMP()` + fetched rows / total rows = 1/1 + +---------------------+ + | UTC_TIMESTAMP() | + |---------------------| + | 2022-10-03 17:54:28 | + +---------------------+ + + WEEK ---- diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index b14acc88e0..0503759cbd 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -87,7 +87,7 @@ LIKE Description >>>>>>>>>>> -Usage: like(string, PATTERN) return true if the string match the PATTERN. +Usage: like(string, PATTERN) return true if the string match the PATTERN, PATTERN is case insensitive. There are two wildcards often used in conjunction with the LIKE operator: @@ -96,7 +96,7 @@ There are two wildcards often used in conjunction with the LIKE operator: Example:: - os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ello%') | fields `LIKE('hello world', '_ello%')` + os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ELLO%') | fields `LIKE('hello world', '_ello%')` fetched rows / total rows = 1/1 +---------------------------------+ | LIKE('hello world', '_ello%') | @@ -175,6 +175,29 @@ Example:: +-------------------------------------+---------------------------------------+ +REVERSE +----- + +Description +>>>>>>>>>>> + +Usage: REVERSE(str) returns reversed string of the string supplied as an argument. + +Argument type: STRING + +Return type: STRING + +Example:: + + os> source=people | eval `REVERSE('abcde')` = REVERSE('abcde') | fields `REVERSE('abcde')` + fetched rows / total rows = 1/1 + +--------------------+ + | REVERSE('abcde') | + |--------------------| + | edcba | + +--------------------+ + + RIGHT ----- diff --git a/doctest/test_data/wildcard.json b/doctest/test_data/wildcard.json new file mode 100644 index 0000000000..c91778d8ab --- /dev/null +++ b/doctest/test_data/wildcard.json @@ -0,0 +1,22 @@ +{"index":{"_id":"0"}} +{"Body":"test wildcard"} +{"index":{"_id":"1"}} +{"Body":"test wildcard in the end of the text%"} +{"index":{"_id":"2"}} +{"Body":"%test wildcard in the beginning of the text"} +{"index":{"_id":"3"}} +{"Body":"test wildcard in % the middle of the text"} +{"index":{"_id":"4"}} +{"Body":"test wildcard %% beside each other"} +{"index":{"_id":"5"}} +{"Body":"test wildcard in the end of the text_"} +{"index":{"_id":"6"}} +{"Body":"_test wildcard in the beginning of the text"} +{"index":{"_id":"7"}} +{"Body":"test wildcard in _ the middle of the text"} +{"index":{"_id":"8"}} +{"Body":"test wildcard __ beside each other"} +{"index":{"_id":"9"}} +{"Body":"test backslash wildcard \\_"} +{"index":{"_id":"10"}} +{"Body":"tEsT wIlDcArD sensitive cases"} diff --git a/doctest/test_docs.py b/doctest/test_docs.py index 6d2538196a..b5edf46de9 100644 --- a/doctest/test_docs.py +++ b/doctest/test_docs.py @@ -26,6 +26,7 @@ NYC_TAXI = "nyc_taxi" BOOKS = "books" APACHE = "apache" +WILDCARD = "wildcard" class DocTestConnection(OpenSearchConnection): @@ -92,6 +93,7 @@ def set_up_test_indices(test): load_file("nyc_taxi.json", index_name=NYC_TAXI) load_file("books.json", index_name=BOOKS) load_file("apache.json", index_name=APACHE) + load_file("wildcard.json", index_name=WILDCARD) def load_file(filename, index_name): @@ -120,7 +122,7 @@ def set_up(test): def tear_down(test): # drop leftover tables after each test - test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE], ignore_unavailable=True) + test_data_client.indices.delete(index=[ACCOUNTS, EMPLOYEES, PEOPLE, ACCOUNT2, NYC_TAXI, BOOKS, APACHE, WILDCARD], ignore_unavailable=True) docsuite = partial(doctest.DocFileSuite, diff --git a/doctest/test_mapping/wildcard.json b/doctest/test_mapping/wildcard.json new file mode 100644 index 0000000000..670a774ae1 --- /dev/null +++ b/doctest/test_mapping/wildcard.json @@ -0,0 +1,9 @@ +{ + "mappings" : { + "properties" : { + "Body" : { + "type" : "keyword" + } + } + } +} diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 6c10d55262..d4737e58f3 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -38,9 +38,6 @@ apply plugin: 'java' apply plugin: 'io.freefair.lombok' apply plugin: 'com.wiredforcode.spawn' - - - repositories { mavenCentral() maven { url 'https://jitpack.io' } @@ -61,13 +58,15 @@ loggerUsageCheck.enabled = false configurations.all { resolutionStrategy.force 'junit:junit:4.13.2' - exclude group: "commons-logging", module: "commons-logging" + resolutionStrategy.force "commons-logging:commons-logging:1.2" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" @@ -243,17 +242,18 @@ task compileJdbc(type: Exec) { } } -String bwcVersion = "1.1.0.0"; +String baseVersion = "2.5.0" +String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" String bwcSqlPlugin = "opensearch-sql-" + bwcVersion + ".zip" -String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/20210930/linux/x64/builds/opensearch/plugins/" + bwcSqlPlugin +String bwcRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${baseVersion}/latest/linux/x64/tar/builds/opensearch/plugins/" + bwcSqlPlugin 2.times { i -> testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" - versions = ["1.1.0", opensearch_version] + versions = [baseVersion, opensearch_version] numberOfNodes = 3 plugin(provider(new Callable() { @Override diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java b/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java index a6f3e561b3..889f60f4ad 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/CorrectnessIT.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; import com.google.common.collect.Maps; +import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -18,7 +19,7 @@ import java.util.Date; import java.util.Map; import java.util.TimeZone; -import org.apache.http.HttpHost; +import org.apache.hc.core5.http.HttpHost; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONObject; @@ -43,7 +44,7 @@ public class CorrectnessIT extends OpenSearchIntegTestCase { private static final Logger LOG = LogManager.getLogger(); @Test - public void performComparisonTest() { + public void performComparisonTest() throws URISyntaxException { TestConfig config = new TestConfig(getCmdLineArgs()); LOG.info("Starting comparison test {}", config); @@ -73,7 +74,7 @@ private Map getCmdLineArgs() { return Maps.fromProperties(System.getProperties()); } - private DBConnection getThisDBConnection(TestConfig config) { + private DBConnection getThisDBConnection(TestConfig config) throws URISyntaxException { String dbUrl = config.getDbConnectionUrl(); if (dbUrl.isEmpty()) { return getOpenSearchConnection(config); @@ -84,7 +85,7 @@ private DBConnection getThisDBConnection(TestConfig config) { /** * Use OpenSearch cluster given on CLI arg or internal embedded in SQLIntegTestCase */ - private DBConnection getOpenSearchConnection(TestConfig config) { + private DBConnection getOpenSearchConnection(TestConfig config) throws URISyntaxException { RestClient client; String openSearchHost = config.getOpenSearchHostUrl(); if (openSearchHost.isEmpty()) { diff --git a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java index 73659525f2..5b33884814 100644 --- a/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java +++ b/integ-test/src/test/java/org/opensearch/sql/correctness/tests/OpenSearchConnectionTest.java @@ -17,8 +17,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.Arrays; -import org.apache.http.ProtocolVersion; -import org.apache.http.message.BasicStatusLine; +import org.apache.hc.core5.http.ProtocolVersion; +import org.apache.hc.core5.http.message.StatusLine; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -48,7 +48,7 @@ public void setUp() throws IOException { Response response = mock(Response.class); when(client.performRequest(any(Request.class))).thenReturn(response); when(response.getStatusLine()) - .thenReturn(new BasicStatusLine(new ProtocolVersion("HTTP", 2, 0), 200, "")); + .thenReturn(new StatusLine(new ProtocolVersion("HTTP", 2, 0), 200, "")); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java index d4374dcbaf..854a33fc91 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationExpressionIT.java @@ -121,6 +121,7 @@ public void hasGroupKeyMaxAddLiteralShouldPass() { rows("f", 1)); } + @Ignore("Handled by v2 engine which returns 'name': 'Log(MAX(age) + MIN(age))' instead") @Test public void noGroupKeyLogMaxAddMinShouldPass() { JSONObject response = executeJdbcRequest(String.format( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java index ba007f43f9..dd78ad1cf9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java @@ -166,13 +166,13 @@ public void percentileTest() throws IOException { 0.6); Assert .assertEquals(35.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "75.0"), - 0.001); + 0.6); Assert .assertEquals(39.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "95.0"), - 0.001); + 0.6); Assert .assertEquals(40.0, getDoubleAggregationValue(result, "PERCENTILES(age)", "values", "99.0"), - 0.001); + 0.6); } @Test @@ -183,9 +183,9 @@ public void percentileTestSpecific() throws IOException { Assert.assertThat(getTotalHits(result), equalTo(1000)); Assert.assertEquals(25.0, - getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.001); + getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "25.0"), 0.6); Assert.assertEquals(35.0, - getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.001); + getDoubleAggregationValue(result, "PERCENTILES(age,25.0,75.0)", "values", "75.0"), 0.6); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index 9a08302577..d562794409 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -486,7 +486,36 @@ public void percentileAggregationTest() throws Exception { List lines = csvResult.getLines(); Assert.assertEquals(1, lines.size()); - Assert.assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0)); + + List result = + Arrays.stream(lines.get(0).split(",")) + .mapToDouble(Double::valueOf) + .boxed() + .collect(Collectors.toList()); + assertEquals(7, result.size()); + assertEquals(32.0, result.get(0), 0.6); + assertEquals(32.0, result.get(1), 0.6); + assertEquals(34.0, result.get(2), 0.6); + + assertEquals("32.0,32.0,34.0,36.0,38.0,40.0,40.0", lines.get(0), 0.6); + } + + private void assertEquals(String expected, String actual, Double delta) { + List actualList = + Arrays.stream(actual.split(",")) + .mapToDouble(Double::valueOf) + .boxed() + .collect(Collectors.toList()); + List expectedList = + Arrays.stream(expected.split(",")) + .mapToDouble(Double::valueOf) + .boxed() + .collect(Collectors.toList()); + + assertEquals(expectedList.size(), actualList.size()); + for (int i = 0; i < expectedList.size(); i++) { + assertEquals(expectedList.get(i), actualList.get(i), delta); + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java index 74f999f368..bd72877e1c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/JdbcTestIT.java @@ -35,10 +35,10 @@ public void testPercentilesQuery() { JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0"); - assertThat(percentileRow.getDouble("25.0"), equalTo(31.5)); - assertThat(percentileRow.getDouble("50.0"), equalTo(33.5)); - assertThat(percentileRow.getDouble("75.0"), equalTo(36.5)); - assertThat(percentileRow.getDouble("99.9"), equalTo(39.0)); + assertEquals(31.5, percentileRow.getDouble("25.0"), 0.6); + assertEquals(33.5, percentileRow.getDouble("50.0"), 0.6); + assertEquals(36.5, percentileRow.getDouble("75.0"), 0.6); + assertEquals(39.0, percentileRow.getDouble("99.9"), 0.6); } // https://github.com/opensearch-project/sql/issues/537 @@ -53,10 +53,10 @@ public void testSlowQuery() throws IOException { assertThat(response.getJSONArray("datarows").length(), equalTo(1)); JSONObject percentileRow = (JSONObject) response.query("/datarows/0/0"); - assertThat(percentileRow.getDouble("25.0"), equalTo(31.5)); - assertThat(percentileRow.getDouble("50.0"), equalTo(33.5)); - assertThat(percentileRow.getDouble("75.0"), equalTo(36.5)); - assertThat(percentileRow.getDouble("99.9"), equalTo(39.0)); + assertEquals(31.5, percentileRow.getDouble("25.0"), 0.6); + assertEquals(33.5, percentileRow.getDouble("50.0"), 0.6); + assertEquals(36.5, percentileRow.getDouble("75.0"), 0.6); + assertEquals(39.0, percentileRow.getDouble("99.9"), 0.6); wipeAllClusterSettings(); } @@ -165,6 +165,7 @@ public void functionWithoutAliasShouldHaveEntireFunctionAsNameInSchema() { ); } + @Ignore("Handled by v2 engine which returns 'name': 'substring(lastname, 1, 2)' instead") @Test public void functionWithAliasShouldHaveAliasAsNameInSchema() { assertThat( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java index b1b6b24ada..fdbbb0f6ba 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java @@ -54,7 +54,7 @@ public void matchQueryTest() throws IOException { "select address from %s where address= matchQuery('880 Holmes Lane') limit 3", TestsConstants.TEST_INDEX_ACCOUNT)); Assert.assertThat(result, - containsString("{\"match\":{\"address\":{\"query\":\"880 Holmes Lane\"")); + containsString("{\\\"match\\\":{\\\"address\\\":{\\\"query\\\":\\\"880 Holmes Lane\\\"")); } /** diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 0369ee715a..72daa286e8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -9,16 +9,20 @@ import java.io.IOException; import java.util.Map; import java.util.Optional; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.message.BasicHeader; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.util.EntityUtils; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.util.Timeout; import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.client.Request; @@ -56,7 +60,7 @@ protected String getProtocol() { protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); if (isHttps()) { - configureHttpsClient(builder, settings); + configureHttpsClient(builder, settings, hosts[0]); } else { configureClient(builder, settings); } @@ -68,19 +72,24 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE protected static void wipeAllOpenSearchIndices() throws IOException { // include all the indices, included hidden indices. // https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html#cat-indices-api-query-params - Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); - JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8")); - for (Object object : jsonArray) { - JSONObject jsonObject = (JSONObject) object; - String indexName = jsonObject.getString("index"); - //.opendistro_security isn't allowed to delete from cluster - if (!indexName.startsWith(".opensearch_dashboards") && !indexName.startsWith(".opendistro")) { - client().performRequest(new Request("DELETE", "/" + indexName)); + try { + Response response = client().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all")); + JSONArray jsonArray = new JSONArray(EntityUtils.toString(response.getEntity(), "UTF-8")); + for (Object object : jsonArray) { + JSONObject jsonObject = (JSONObject) object; + String indexName = jsonObject.getString("index"); + //.opendistro_security isn't allowed to delete from cluster + if (!indexName.startsWith(".opensearch_dashboards") && !indexName.startsWith(".opendistro")) { + client().performRequest(new Request("DELETE", "/" + indexName)); + } } + } catch (ParseException e) { + throw new IOException(e); } } - protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) + protected static void configureHttpsClient(RestClientBuilder builder, Settings settings, + HttpHost httpHost) throws IOException { Map headers = ThreadContext.buildDefaultHeaders(settings); Header[] defaultHeaders = new Header[headers.size()]; @@ -94,15 +103,21 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s .orElseThrow(() -> new RuntimeException("user name is missing")); String password = Optional.ofNullable(System.getProperty("password")) .orElseThrow(() -> new RuntimeException("password is missing")); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider - .setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password)); + .setCredentials(new AuthScope(httpHost), new UsernamePasswordCredentials(userName, + password.toCharArray())); try { - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) - //disable the certificate since our testing cluster just uses the default security configuration - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(SSLContextBuilder.create() + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setSslContext(SSLContextBuilder.create() .loadTrustMaterial(null, (chains, authType) -> true) + .build()) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build(); + + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager(PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) .build()); } catch (Exception e) { throw new RuntimeException(e); @@ -114,7 +129,7 @@ protected static void configureHttpsClient(RestClientBuilder builder, Settings s TimeValue.parseTimeValue(socketTimeoutString == null ? "60s" : socketTimeoutString, CLIENT_SOCKET_TIMEOUT); builder.setRequestConfigCallback( - conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); + conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis())))); if (settings.hasValue(CLIENT_PATH_PREFIX)) { builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index f03acbbbfd..80348b2a8b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -584,7 +584,11 @@ public enum Index { CALCS(TestsConstants.TEST_INDEX_CALCS, "calcs", getMappingFile("calcs_index_mappings.json"), - "src/test/resources/calcs.json"),; + "src/test/resources/calcs.json"), + WILDCARD(TestsConstants.TEST_INDEX_WILDCARD, + "wildcard", + getMappingFile("wildcard_index_mappings.json"), + "src/test/resources/wildcard.json"),; private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index a9f81c68fe..aff269fcce 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -53,6 +53,7 @@ public class TestsConstants { public final static String TEST_INDEX_BEER = TEST_INDEX + "_beer"; public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing"; public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs"; + public final static String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 57745e2e2a..23f2df69c8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -6,8 +6,11 @@ package org.opensearch.sql.ppl; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE2; +import static org.opensearch.sql.sql.DateTimeFunctionIT.utcDateTimeNow; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifySchema; @@ -754,6 +757,33 @@ private List> nowLikeFunctionsData() { .put("referenceGetter", (Supplier) LocalDate::now) .put("parser", (BiFunction) LocalDate::parse) .put("serializationPattern", "uuuu-MM-dd") + .build(), + ImmutableMap.builder() + .put("name", "utc_date") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) ()-> utcDateTimeNow().toLocalDate()) + .put("parser", (BiFunction) LocalDate::parse) + .put("serializationPattern", "uuuu-MM-dd") + .build(), + ImmutableMap.builder() + .put("name", "utc_time") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) ()-> utcDateTimeNow().toLocalTime()) + .put("parser", (BiFunction) LocalTime::parse) + .put("serializationPattern", "HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "utc_timestamp") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) ()-> utcDateTimeNow()) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") .build() ); } @@ -912,4 +942,27 @@ public void testPeriodDiff() throws IOException { verifySchema(result, schema("f1", null, "integer"), schema("f2", null, "integer")); verifySome(result.getJSONArray("datarows"), rows(11, -25)); } + + public void testDateDiff() throws IOException { + var result = executeQuery(String.format("source=%s | eval" + + " `'2000-01-02' - '2000-01-01'` = DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59'))," + + " `'2001-02-01' - '2004-01-01'` = DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00'))," + + " `'2004-01-01' - '2002-02-01'` = DATEDIFF(TIMESTAMP('2004-01-01 00:00:00'), DATETIME('2002-02-01 14:25:30'))," + + " `today - today` = DATEDIFF(TIME('23:59:59'), TIME('00:00:00'))" + + " | fields `'2000-01-02' - '2000-01-01'`, `'2001-02-01' - '2004-01-01'`, `'2004-01-01' - '2002-02-01'`, `today - today`", TEST_INDEX_DATE)); + verifySchema(result, + schema("'2000-01-02' - '2000-01-01'", null, "long"), + schema("'2001-02-01' - '2004-01-01'", null, "long"), + schema("'2004-01-01' - '2002-02-01'", null, "long"), + schema("today - today", null, "long")); + verifySome(result.getJSONArray("datarows"), rows(1, -1064, 699, 0)); + } + + @Test + public void testTimeDiff() throws IOException { + var result = executeQuery(String.format( + "source=%s | eval f = TIMEDIFF('23:59:59', '13:00:00') | fields f", TEST_INDEX_DATE)); + verifySchema(result, schema("f", null, "time")); + verifySome(result.getJSONArray("datarows"), rows("10:59:59")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java new file mode 100644 index 0000000000..67ad553689 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/LikeQueryIT.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; + +public class LikeQueryIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(Index.WILDCARD); + } + + @Test + public void test_like_with_percent() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(KeywordBody, 'test wildcard%') | fields KeywordBody"; + JSONObject result = executeQuery(query); + verifyDataRows(result, + rows("test wildcard"), + rows("test wildcard in the end of the text%"), + rows("test wildcard in % the middle of the text"), + rows("test wildcard %% beside each other"), + rows("test wildcard in the end of the text_"), + rows("test wildcard in _ the middle of the text"), + rows("test wildcard __ beside each other")); + } + + @Test + public void test_like_with_escaped_percent() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(KeywordBody, '\\\\%test wildcard%') | fields KeywordBody"; + JSONObject result = executeQuery(query); + verifyDataRows(result, + rows("%test wildcard in the beginning of the text")); + } + + @Test + public void test_like_in_where_with_escaped_underscore() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(KeywordBody, '\\\\_test wildcard%') | fields KeywordBody"; + JSONObject result = executeQuery(query); + verifyDataRows(result, + rows("_test wildcard in the beginning of the text")); + } + + @Test + public void test_like_on_text_field_with_one_word() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextBody, 'test*') | fields TextBody"; + JSONObject result = executeQuery(query); + assertEquals(9, result.getInt("total")); + } + + @Test + public void test_like_on_text_keyword_field_with_one_word() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextKeywordBody, 'test*') | fields TextKeywordBody"; + JSONObject result = executeQuery(query); + assertEquals(8, result.getInt("total")); + } + + @Test + public void test_like_on_text_keyword_field_with_greater_than_one_word() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextKeywordBody, 'test wild*') | fields TextKeywordBody"; + JSONObject result = executeQuery(query); + assertEquals(7, result.getInt("total")); + } + + @Test + public void test_like_on_text_field_with_greater_than_one_word() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextBody, 'test wild*') | fields TextBody"; + JSONObject result = executeQuery(query); + assertEquals(0, result.getInt("total")); + } + + @Test + public void test_convert_field_text_to_keyword() throws IOException { + String query = "source=" + TEST_INDEX_WILDCARD + " | WHERE Like(TextKeywordBody, '*') | fields TextKeywordBody"; + String result = explainQueryToString(query); + assertTrue(result.contains("TextKeywordBody.keyword")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index 33050e7200..71988a8e31 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -16,10 +16,16 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import lombok.RequiredArgsConstructor; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.client.RestClient; import org.opensearch.client.RestHighLevelClient; +import org.opensearch.common.inject.AbstractModule; +import org.opensearch.common.inject.Injector; +import org.opensearch.common.inject.ModulesBuilder; +import org.opensearch.common.inject.Provides; +import org.opensearch.common.inject.Singleton; import org.opensearch.sql.analysis.Analyzer; import org.opensearch.sql.analysis.ExpressionAnalyzer; import org.opensearch.sql.common.response.ResponseListener; @@ -32,27 +38,27 @@ import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.execution.QueryPlanFactory; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.FunctionProperties; import org.opensearch.sql.monitor.AlwaysHealthyMonitor; +import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.client.OpenSearchRestClient; import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; +import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; +import org.opensearch.sql.opensearch.security.SecurityAccess; import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory; +import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; import org.opensearch.sql.planner.Planner; import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; -import org.opensearch.sql.ppl.config.PPLServiceConfig; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.QueryResult; import org.opensearch.sql.protocol.response.format.SimpleJsonResponseFormatter; +import org.opensearch.sql.sql.SQLService; +import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.storage.DataSourceFactory; +import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.util.ExecuteOnCallerThreadQueryManager; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; /** * Run PPL with query engine outside OpenSearch cluster. This IT doesn't require our plugin @@ -67,27 +73,19 @@ public class StandaloneIT extends PPLIntegTestCase { @Override public void init() { - // Using client() defined in ODFERestTestCase. restClient = new InternalRestHighLevelClient(client()); - OpenSearchClient client = new OpenSearchRestClient(restClient); - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - context.registerBean(ExecutionEngine.class, () -> new OpenSearchExecutionEngine(client, - new OpenSearchExecutionProtector(new AlwaysHealthyMonitor()))); - context.registerBean(OpenSearchClient.class, () -> client); - context.registerBean(Settings.class, () -> defaultSettings()); - context.registerBean(FunctionProperties.class, FunctionProperties::new); DataSourceService dataSourceService = new DataSourceServiceImpl( new ImmutableSet.Builder() .add(new OpenSearchDataSourceFactory(client, defaultSettings())) .build()); dataSourceService.addDataSource(defaultOpenSearchDataSourceMetadata()); - context.registerBean(DataSourceService.class, () -> dataSourceService); - context.register(StandaloneConfig.class); - context.register(PPLServiceConfig.class); - context.refresh(); - pplService = context.getBean(PPLService.class); + ModulesBuilder modules = new ModulesBuilder(); + modules.add(new StandaloneModule(new InternalRestHighLevelClient(client()), defaultSettings(), dataSourceService)); + Injector injector = modules.createInjector(); + pplService = + SecurityAccess.doPrivileged(() -> injector.getInstance(PPLService.class)); } @Test @@ -170,27 +168,68 @@ public InternalRestHighLevelClient(RestClient restClient) { } } - @Configuration - static class StandaloneConfig { - @Autowired - private DataSourceService dataSourceService; + @RequiredArgsConstructor + public class StandaloneModule extends AbstractModule { + + private final RestHighLevelClient client; + + private final Settings settings; + + private final DataSourceService dataSourceService; + + private final BuiltinFunctionRepository functionRepository = + BuiltinFunctionRepository.getInstance(); - @Autowired - private ExecutionEngine executionEngine; + @Override + protected void configure() {} - @Bean - QueryManager queryManager() { + @Provides + public OpenSearchClient openSearchClient() { + return new OpenSearchRestClient(client); + } + + @Provides + public StorageEngine storageEngine(OpenSearchClient client) { + return new OpenSearchStorageEngine(client, settings); + } + + @Provides + public ExecutionEngine executionEngine(OpenSearchClient client, ExecutionProtector protector) { + return new OpenSearchExecutionEngine(client, protector); + } + + @Provides + public ResourceMonitor resourceMonitor() { + return new AlwaysHealthyMonitor(); + } + + @Provides + public ExecutionProtector protector(ResourceMonitor resourceMonitor) { + return new OpenSearchExecutionProtector(resourceMonitor); + } + + @Provides + @Singleton + public QueryManager queryManager() { return new ExecuteOnCallerThreadQueryManager(); } - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - QueryPlanFactory queryExecutionFactory() { - BuiltinFunctionRepository functionRepository = BuiltinFunctionRepository.getInstance(); - Analyzer analyzer = new Analyzer(new ExpressionAnalyzer(functionRepository), - dataSourceService, functionRepository); - Planner planner = - new Planner(LogicalPlanOptimizer.create()); + @Provides + public PPLService pplService(QueryManager queryManager, QueryPlanFactory queryPlanFactory) { + return new PPLService(new PPLSyntaxParser(), queryManager, queryPlanFactory); + } + + @Provides + public SQLService sqlService(QueryManager queryManager, QueryPlanFactory queryPlanFactory) { + return new SQLService(new SQLSyntaxParser(), queryManager, queryPlanFactory); + } + + @Provides + public QueryPlanFactory queryPlanFactory(ExecutionEngine executionEngine) { + Analyzer analyzer = + new Analyzer( + new ExpressionAnalyzer(functionRepository), dataSourceService, functionRepository); + Planner planner = new Planner(LogicalPlanOptimizer.create()); return new QueryPlanFactory(new QueryService(analyzer, executionEngine, planner)); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StreamingQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StreamingQueryIT.java index 2a00e14d34..50929782c8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StreamingQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StreamingQueryIT.java @@ -41,7 +41,6 @@ import org.opensearch.sql.planner.Planner; import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; import org.opensearch.sql.storage.DataSourceFactory; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; @ThreadLeakFilters(filters = {StreamingQueryIT.HadoopFSThreadsFilter.class}) public class StreamingQueryIT extends PPLIntegTestCase { @@ -111,9 +110,6 @@ void close() throws IOException { } class StreamingQuery { - - final AnnotationConfigApplicationContext context; - final DefaultQueryManager queryManager; final QueryService queryService; @@ -124,15 +120,13 @@ class StreamingQuery { public StreamingQuery(java.nio.file.Path tempDir) { result.set(0); - context = new AnnotationConfigApplicationContext(); - context.refresh(); + DataSourceService dataSourceService = new DataSourceServiceImpl( new ImmutableSet.Builder() .add(new FSDataSourceFactory(tempDir.toUri(), result)) .build()); dataSourceService.addDataSource(fsDataSourceMetadata()); - context.registerBean(DataSourceService.class, () -> dataSourceService); storageEngine = (FSStorageEngine) dataSourceService.getDataSource(DATASOURCE_NAME).getStorageEngine(); final BuiltinFunctionRepository functionRepository = BuiltinFunctionRepository.getInstance(); @@ -175,7 +169,6 @@ void close() throws InterruptedException, IOException { assertTrue(queryManager.cancel(queryId)); storageEngine.close(); - context.close(); queryManager.awaitTermination(5, TimeUnit.SECONDS); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java index 84717900ca..7c48bceab0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/TextFunctionIT.java @@ -139,4 +139,9 @@ public void testLocate() throws IOException { public void testReplace() throws IOException { verifyQuery("replace", "", ", 'world', ' opensearch'", "hello", " opensearch", "hello opensearch"); } + + @Test + public void testReverse() throws IOException { + verifyQuery("reverse", "", "", "olleh", "dlrow", "dlrowolleh"); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/AdminIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/AdminIT.java index ed7ec600a3..243432790d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/AdminIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/AdminIT.java @@ -7,17 +7,23 @@ package org.opensearch.sql.sql; import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.assertJsonEquals; +import static org.opensearch.sql.util.TestUtils.getResponseBody; import com.google.common.io.Resources; import java.io.IOException; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Locale; + import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.legacy.SQLIntegTestCase; import org.opensearch.sql.legacy.TestsConstants; @@ -34,7 +40,7 @@ public void init() throws Exception { public void showSingleIndexAlias() throws IOException { String alias = "acc"; addAlias(TestsConstants.TEST_INDEX_ACCOUNT, alias); - JSONObject response = new JSONObject(executeQuery("SHOW TABLES LIKE acc", "jdbc")); + JSONObject response = new JSONObject(executeQuery("SHOW TABLES LIKE 'acc'", "jdbc")); /* * Assumed indices of fields in dataRows based on "schema" output for SHOW given above: @@ -48,7 +54,7 @@ public void showSingleIndexAlias() throws IOException { public void describeSingleIndexAlias() throws IOException { String alias = "acc"; addAlias(TestsConstants.TEST_INDEX_ACCOUNT, alias); - JSONObject response = new JSONObject(executeQuery("DESCRIBE TABLES LIKE acc", "jdbc")); + JSONObject response = new JSONObject(executeQuery("DESCRIBE TABLES LIKE 'acc'", "jdbc")); /* * Assumed indices of fields in dataRows based on "schema" output for DESCRIBE given above: @@ -58,15 +64,25 @@ public void describeSingleIndexAlias() throws IOException { assertThat(row.get(2), equalTo(alias)); } + @Test + public void describeSingleIndexWildcard() throws IOException { + JSONObject response1 = executeQuery("DESCRIBE TABLES LIKE \\\"%account\\\""); + JSONObject response2 = executeQuery("DESCRIBE TABLES LIKE '%account'"); + JSONObject response3 = executeQuery("DESCRIBE TABLES LIKE '%account' COLUMNS LIKE \\\"%name\\\""); + JSONObject response4 = executeQuery("DESCRIBE TABLES LIKE \\\"%account\\\" COLUMNS LIKE '%name'"); + // 11 rows in the output, each corresponds to a column in the table + assertEquals(11, response1.getJSONArray("datarows").length()); + assertTrue(response1.similar(response2)); + // 2 columns should match the wildcard + assertEquals(2, response3.getJSONArray("datarows").length()); + assertTrue(response3.similar(response4)); + } + @Test public void explainShow() throws Exception { String expected = loadFromFile("expectedOutput/sql/explain_show.json"); - - final String actual = explainQuery("SHOW TABLES LIKE %"); - assertJsonEquals( - expected, - explainQuery("SHOW TABLES LIKE %") - ); + String actual = explainQuery("SHOW TABLES LIKE '%'"); + assertTrue(new JSONObject(expected).similar(new JSONObject(actual))); } private void addAlias(String index, String alias) throws IOException { @@ -77,4 +93,16 @@ private String loadFromFile(String filename) throws Exception { URI uri = Resources.getResource(filename).toURI(); return new String(Files.readAllBytes(Paths.get(uri))); } + + protected JSONObject executeQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index f10596e21c..56a7cea9cb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -24,6 +24,8 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.Period; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.temporal.ChronoField; @@ -364,6 +366,30 @@ public void testMinute() throws IOException { verifyDataRows(result, rows(30)); } + + @Test + public void testMinuteOfDay() throws IOException { + JSONObject result = executeQuery("select minute_of_day(timestamp('2020-09-16 17:30:00'))"); + verifySchema(result, schema("minute_of_day(timestamp('2020-09-16 17:30:00'))", null, "integer")); + verifyDataRows(result, rows(1050)); + + result = executeQuery("select minute_of_day(datetime('2020-09-16 17:30:00'))"); + verifySchema(result, schema("minute_of_day(datetime('2020-09-16 17:30:00'))", null, "integer")); + verifyDataRows(result, rows(1050)); + + result = executeQuery("select minute_of_day(time('17:30:00'))"); + verifySchema(result, schema("minute_of_day(time('17:30:00'))", null, "integer")); + verifyDataRows(result, rows(1050)); + + result = executeQuery("select minute_of_day('2020-09-16 17:30:00')"); + verifySchema(result, schema("minute_of_day('2020-09-16 17:30:00')", null, "integer")); + verifyDataRows(result, rows(1050)); + + result = executeQuery("select minute_of_day('17:30:00')"); + verifySchema(result, schema("minute_of_day('17:30:00')", null, "integer")); + verifyDataRows(result, rows(1050)); + } + @Test public void testMonth() throws IOException { JSONObject result = executeQuery("select month(date('2020-09-16'))"); @@ -640,6 +666,12 @@ public void testMakeDate() throws IOException { verifyDataRows(result, rows("1945-01-06", "1989-06-06")); } + public static LocalDateTime utcDateTimeNow() { + ZonedDateTime zonedDateTime = + LocalDateTime.now().atZone(TimeZone.getDefault().toZoneId()); + return zonedDateTime.withZoneSameInstant(ZoneId.of("UTC")).toLocalDateTime(); + } + private List> nowLikeFunctionsData() { return List.of( ImmutableMap.builder() @@ -722,6 +754,33 @@ private List> nowLikeFunctionsData() { .put("referenceGetter", (Supplier) LocalDate::now) .put("parser", (BiFunction) LocalDate::parse) .put("serializationPattern", "uuuu-MM-dd") + .build(), + ImmutableMap.builder() + .put("name", "utc_date") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) (()-> utcDateTimeNow().toLocalDate())) + .put("parser", (BiFunction) LocalDate::parse) + .put("serializationPattern", "uuuu-MM-dd") + .build(), + ImmutableMap.builder() + .put("name", "utc_time") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) (()-> utcDateTimeNow().toLocalTime())) + .put("parser", (BiFunction) LocalTime::parse) + .put("serializationPattern", "HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "utc_timestamp") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) DateTimeFunctionIT::utcDateTimeNow) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") .build() ); } @@ -878,6 +937,27 @@ public void testSubTime() throws IOException { verifyDataRows(result, rows("2008-12-12 00:00:00", "23:59:59", "2003-12-31 00:00:01", "10:14:48", "1999-12-31 06:35:13")); } + public void testDateDiff() throws IOException { + var result = executeQuery("SELECT" + + " DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59')) AS `'2000-01-02' - '2000-01-01'`," + + " DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00')) AS `'2001-02-01' - '2004-01-01'`," + + " DATEDIFF(TIMESTAMP('2004-01-01 00:00:00'), DATETIME('2002-02-01 14:25:30')) AS `'2004-01-01' - '2002-02-01'`," + + " DATEDIFF(TIME('23:59:59'), TIME('00:00:00')) AS `today - today`"); + verifySchema(result, + schema("DATEDIFF(TIMESTAMP('2000-01-02 00:00:00'), TIMESTAMP('2000-01-01 23:59:59'))", "'2000-01-02' - '2000-01-01'", "long"), + schema("DATEDIFF(DATE('2001-02-01'), TIMESTAMP('2004-01-01 00:00:00'))", "'2001-02-01' - '2004-01-01'", "long"), + schema("DATEDIFF(TIMESTAMP('2004-01-01 00:00:00'), DATETIME('2002-02-01 14:25:30'))", "'2004-01-01' - '2002-02-01'", "long"), + schema("DATEDIFF(TIME('23:59:59'), TIME('00:00:00'))", "today - today", "long")); + verifyDataRows(result, rows(1, -1064, 699, 0)); + } + + @Test + public void testTimeDiff() throws IOException { + var result = executeQuery("select TIMEDIFF('23:59:59', '13:00:00') as f"); + verifySchema(result, schema("TIMEDIFF('23:59:59', '13:00:00')", "f", "time")); + verifyDataRows(result, rows("10:59:59")); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java new file mode 100644 index 0000000000..f0e82adb6f --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/LikeQueryIT.java @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.sql; + +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +import java.io.IOException; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +public class LikeQueryIT extends SQLIntegTestCase { + @Override + protected void init() throws Exception { + loadIndex(Index.WILDCARD); + } + + @Test + public void test_like_in_select() throws IOException { + String query = "SELECT KeywordBody, KeywordBody LIKE 'test wildcard%' FROM " + TEST_INDEX_WILDCARD; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, + rows("test wildcard", true), + rows("test wildcard in the end of the text%", true), + rows("%test wildcard in the beginning of the text", false), + rows("test wildcard in % the middle of the text", true), + rows("test wildcard %% beside each other", true), + rows("test wildcard in the end of the text_", true), + rows("_test wildcard in the beginning of the text", false), + rows("test wildcard in _ the middle of the text", true), + rows("test wildcard __ beside each other", true), + rows("test backslash wildcard \\_", false)); + } + + @Test + public void test_like_in_select_with_escaped_percent() throws IOException { + String query = "SELECT KeywordBody, KeywordBody LIKE '\\\\%test wildcard%' FROM " + TEST_INDEX_WILDCARD; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, + rows("test wildcard", false), + rows("test wildcard in the end of the text%", false), + rows("%test wildcard in the beginning of the text", true), + rows("test wildcard in % the middle of the text", false), + rows("test wildcard %% beside each other", false), + rows("test wildcard in the end of the text_", false), + rows("_test wildcard in the beginning of the text", false), + rows("test wildcard in _ the middle of the text", false), + rows("test wildcard __ beside each other", false), + rows("test backslash wildcard \\_", false)); + } + + @Test + public void test_like_in_select_with_escaped_underscore() throws IOException { + String query = "SELECT KeywordBody, KeywordBody LIKE '\\\\_test wildcard%' FROM " + TEST_INDEX_WILDCARD; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, + rows("test wildcard", false), + rows("test wildcard in the end of the text%", false), + rows("%test wildcard in the beginning of the text", false), + rows("test wildcard in % the middle of the text", false), + rows("test wildcard %% beside each other", false), + rows("test wildcard in the end of the text_", false), + rows("_test wildcard in the beginning of the text", true), + rows("test wildcard in _ the middle of the text", false), + rows("test wildcard __ beside each other", false), + rows("test backslash wildcard \\_", false)); + } + + @Test + public void test_like_in_where() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE KeywordBody LIKE 'test wildcard%'"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, + rows("test wildcard"), + rows("test wildcard in the end of the text%"), + rows("test wildcard in % the middle of the text"), + rows("test wildcard %% beside each other"), + rows("test wildcard in the end of the text_"), + rows("test wildcard in _ the middle of the text"), + rows("test wildcard __ beside each other")); + } + + @Test + public void test_like_in_where_with_escaped_percent() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE KeywordBody LIKE '\\\\%test wildcard%'"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, + rows("%test wildcard in the beginning of the text")); + } + + @Test + public void test_like_in_where_with_escaped_underscore() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE KeywordBody LIKE '\\\\_test wildcard%'"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, + rows("_test wildcard in the beginning of the text")); + } + + @Test + public void test_like_on_text_field_with_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextBody LIKE 'test*'"; + JSONObject result = executeJdbcRequest(query); + assertEquals(9, result.getInt("total")); + } + + @Test + public void test_like_on_text_keyword_field_with_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE 'test*'"; + JSONObject result = executeJdbcRequest(query); + assertEquals(8, result.getInt("total")); + } + + @Test + public void test_like_on_text_keyword_field_with_greater_than_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE 'test wild*'"; + JSONObject result = executeJdbcRequest(query); + assertEquals(7, result.getInt("total")); + } + + @Test + public void test_like_on_text_field_with_greater_than_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextBody LIKE 'test wild*'"; + JSONObject result = executeJdbcRequest(query); + assertEquals(0, result.getInt("total")); + } + + @Test + public void test_convert_field_text_to_keyword() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE TextKeywordBody LIKE '*'"; + String result = explainQuery(query); + assertTrue(result.contains("TextKeywordBody.keyword")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java index 09e3504e4a..28573fdd10 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MatchIT.java @@ -15,6 +15,7 @@ import org.json.JSONObject; import org.junit.Test; import org.opensearch.sql.legacy.SQLIntegTestCase; +import org.opensearch.sql.legacy.utils.StringUtils; public class MatchIT extends SQLIntegTestCase { @Override @@ -36,6 +37,42 @@ public void match_in_having() throws IOException { verifyDataRows(result, rows("Bates")); } + @Test + public void missing_field_test() { + String query = StringUtils.format("SELECT * FROM %s WHERE match(invalid, 'Bates')", TEST_INDEX_ACCOUNT); + final RuntimeException exception = + expectThrows(RuntimeException.class, () -> executeJdbcRequest(query)); + + assertTrue(exception.getMessage() + .contains("can't resolve Symbol(namespace=FIELD_NAME, name=invalid) in type env")); + + assertTrue(exception.getMessage().contains("SemanticCheckException")); + } + + @Test + public void missing_quoted_field_test() { + String query = StringUtils.format("SELECT * FROM %s WHERE match('invalid', 'Bates')", TEST_INDEX_ACCOUNT); + final RuntimeException exception = + expectThrows(RuntimeException.class, () -> executeJdbcRequest(query)); + + assertTrue(exception.getMessage() + .contains("can't resolve Symbol(namespace=FIELD_NAME, name=invalid) in type env")); + + assertTrue(exception.getMessage().contains("SemanticCheckException")); + } + + @Test + public void missing_backtick_field_test() { + String query = StringUtils.format("SELECT * FROM %s WHERE match(`invalid`, 'Bates')", TEST_INDEX_ACCOUNT); + final RuntimeException exception = + expectThrows(RuntimeException.class, () -> executeJdbcRequest(query)); + + assertTrue(exception.getMessage() + .contains("can't resolve Symbol(namespace=FIELD_NAME, name=invalid) in type env")); + + assertTrue(exception.getMessage().contains("SemanticCheckException")); + } + @Test public void matchquery_in_where() throws IOException { JSONObject result = executeJdbcRequest("SELECT firstname FROM " + TEST_INDEX_ACCOUNT + " WHERE matchquery(lastname, 'Bates')"); @@ -66,7 +103,7 @@ public void match_query_in_having() throws IOException { } @Test - public void alternate_syntaxes_return_the_same_results() throws IOException { + public void match_aliases_return_the_same_results() throws IOException { String query1 = "SELECT lastname FROM " + TEST_INDEX_ACCOUNT + " HAVING match(firstname, 'Nanette')"; JSONObject result1 = executeJdbcRequest(query1); @@ -79,4 +116,35 @@ public void alternate_syntaxes_return_the_same_results() throws IOException { assertEquals(result1.getInt("total"), result2.getInt("total")); assertEquals(result1.getInt("total"), result3.getInt("total")); } + + @Test + public void match_query_alternate_syntax() throws IOException { + JSONObject result = executeJdbcRequest( + "SELECT lastname FROM " + TEST_INDEX_ACCOUNT + " WHERE lastname = match_query('Bates')"); + verifySchema(result, schema("lastname", "text")); + verifyDataRows(result, rows("Bates")); + } + + @Test + public void matchquery_alternate_syntax() throws IOException { + JSONObject result = executeJdbcRequest( + "SELECT lastname FROM " + TEST_INDEX_ACCOUNT + " WHERE lastname = matchquery('Bates')"); + verifySchema(result, schema("lastname", "text")); + verifyDataRows(result, rows("Bates")); + } + + @Test + public void match_alternate_syntaxes_return_the_same_results() throws IOException { + String query1 = "SELECT * FROM " + + TEST_INDEX_ACCOUNT + " WHERE match(firstname, 'Nanette')"; + JSONObject result1 = executeJdbcRequest(query1); + String query2 = "SELECT * FROM " + + TEST_INDEX_ACCOUNT + " WHERE firstname = match_query('Nanette')"; + JSONObject result2 = executeJdbcRequest(query2); + String query3 = "SELECT * FROM " + + TEST_INDEX_ACCOUNT + " WHERE firstname = matchquery('Nanette')"; + JSONObject result3 = executeJdbcRequest(query3); + assertEquals(result1.getInt("total"), result2.getInt("total")); + assertEquals(result1.getInt("total"), result3.getInt("total")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java index b870a60604..3b7e65dcc6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MatchPhraseIT.java @@ -51,7 +51,7 @@ public void test_match_phrase_with_slop() throws IOException { } @Test - public void test_alternate_syntax_for_match_phrase_returns_same_result() throws IOException { + public void test_aliases_for_match_phrase_returns_same_result() throws IOException { String query1 = "SELECT phrase FROM %s WHERE matchphrase(phrase, 'quick fox')"; String query2 = "SELECT phrase FROM %s WHERE match_phrase(phrase, 'quick fox')"; String query3 = "SELECT phrase FROM %s WHERE matchphrasequery(phrase, 'quick fox')"; @@ -61,4 +61,30 @@ public void test_alternate_syntax_for_match_phrase_returns_same_result() throws assertTrue(result1.similar(result2)); assertTrue(result1.similar(result3)); } + + @Test + public void match_phrase_alternate_syntax() throws IOException { + String query = "SELECT phrase FROM %s WHERE phrase = match_phrase('quick fox')"; + JSONObject result = executeJdbcRequest(String.format(query, TEST_INDEX_PHRASE)); + verifyDataRows(result, rows("quick fox"), rows("quick fox here")); + } + + @Test + public void matchphrase_alternate_syntax() throws IOException { + String query = "SELECT phrase FROM %s WHERE phrase = matchphrase('quick fox')"; + JSONObject result = executeJdbcRequest(String.format(query, TEST_INDEX_PHRASE)); + verifyDataRows(result, rows("quick fox"), rows("quick fox here")); + } + + @Test + public void match_phrase_alternate_syntaxes_return_the_same_results() throws IOException { + String query1 = "SELECT phrase FROM %s WHERE matchphrase(phrase, 'quick fox')"; + String query2 = "SELECT phrase FROM %s WHERE phrase = matchphrase('quick fox')"; + String query3 = "SELECT phrase FROM %s WHERE phrase = match_phrase('quick fox')"; + JSONObject result1 = executeJdbcRequest(String.format(query1, TEST_INDEX_PHRASE)); + JSONObject result2 = executeJdbcRequest(String.format(query2, TEST_INDEX_PHRASE)); + JSONObject result3 = executeJdbcRequest(String.format(query3, TEST_INDEX_PHRASE)); + assertTrue(result1.similar(result2)); + assertTrue(result1.similar(result3)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java index efa16ba9d7..f2d1bb7d28 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java @@ -142,6 +142,30 @@ public void testTruncate() throws IOException { result = executeQuery("select truncate(-56, -1)"); verifySchema(result, schema("truncate(-56, -1)", null, "long")); verifyDataRows(result, rows(-50)); + + result = executeQuery("select truncate(33.33344, -1)"); + verifySchema(result, schema("truncate(33.33344, -1)", null, "double")); + verifyDataRows(result, rows(30.0)); + + result = executeQuery("select truncate(33.33344, 2)"); + verifySchema(result, schema("truncate(33.33344, 2)", null, "double")); + verifyDataRows(result, rows(33.33)); + + result = executeQuery("select truncate(33.33344, 100)"); + verifySchema(result, schema("truncate(33.33344, 100)", null, "double")); + verifyDataRows(result, rows(33.33344)); + + result = executeQuery("select truncate(33.33344, 0)"); + verifySchema(result, schema("truncate(33.33344, 0)", null, "double")); + verifyDataRows(result, rows(33.0)); + + result = executeQuery("select truncate(33.33344, 4)"); + verifySchema(result, schema("truncate(33.33344, 4)", null, "double")); + verifyDataRows(result, rows(33.3334)); + + result = executeQuery(String.format("select truncate(%s, 6)", Math.PI)); + verifySchema(result, schema(String.format("truncate(%s, 6)", Math.PI), null, "double")); + verifyDataRows(result, rows(3.141592)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java index 24ce45fd20..07c89b4cdf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java @@ -124,4 +124,35 @@ public void test_all_params_multimatchquery_alternate_parameter_syntax() { JSONObject result = executeJdbcRequest(query); assertEquals(2, result.getInt("total")); } + + @Test + public void multi_match_alternate_syntax() throws IOException { + String query = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multi_match('2014-01-22');"; + var result = new JSONObject(executeQuery(query, "jdbc")); + assertEquals(8, result.getInt("total")); + } + + @Test + public void multimatch_alternate_syntax() throws IOException { + String query = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multimatch('2014-01-22');"; + var result = new JSONObject(executeQuery(query, "jdbc")); + assertEquals(8, result.getInt("total")); + } + + @Test + public void multi_match_alternate_syntaxes_return_the_same_results() throws IOException { + String query1 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE multi_match(['CreationDate'], '2014-01-22');"; + String query2 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multi_match('2014-01-22');"; + String query3 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE CreationDate = multimatch('2014-01-22');"; + var result1 = new JSONObject(executeQuery(query1, "jdbc")); + var result2 = new JSONObject(executeQuery(query2, "jdbc")); + var result3 = new JSONObject(executeQuery(query3, "jdbc")); + assertEquals(result1.getInt("total"), result2.getInt("total")); + assertEquals(result1.getInt("total"), result3.getInt("total")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java index c907b36a63..175cafd31e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/TextFunctionIT.java @@ -41,12 +41,26 @@ void verifyQuery(String query, String type, Integer output) throws IOException { verifyDataRows(result, rows(output)); } + void verifyQueryWithNullOutput(String query, String type) throws IOException { + JSONObject result = executeQuery("select 'test null'," + query); + verifySchema(result, schema(query, null, type), + schema("'test null'", null, type)); + verifyDataRows(result, rows("test null", null)); + } + @Test public void testRegexp() throws IOException { verifyQuery("'a' regexp 'b'", "integer", 0); verifyQuery("'a' regexp '.*'", "integer", 1); } + @Test + public void testReverse() throws IOException { + verifyQuery("reverse('hello')", "keyword", "olleh"); + verifyQuery("reverse('')", "keyword", ""); + verifyQueryWithNullOutput("reverse(null)", "keyword"); + } + @Test public void testSubstr() throws IOException { verifyQuery("substr('hello', 2)", "keyword", "ello"); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/WildcardQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/WildcardQueryIT.java new file mode 100644 index 0000000000..ee636ed5ce --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/WildcardQueryIT.java @@ -0,0 +1,183 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +public class WildcardQueryIT extends SQLIntegTestCase { + @Override + protected void init() throws Exception { + loadIndex(Index.WILDCARD); + } + + @Test + public void test_wildcard_query_asterisk_function() throws IOException { + String expected = "test wildcard"; + + String query1 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, 't*') LIMIT 1"; + JSONObject result1 = executeJdbcRequest(query1); + verifyDataRows(result1, rows(expected)); + + String query2 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcardquery(KeywordBody, 't*') LIMIT 1"; + JSONObject result2 = executeJdbcRequest(query2); + verifyDataRows(result2, rows(expected)); + } + + @Test + public void test_wildcard_query_question_mark_function() throws IOException { + String expected = "test wildcard"; + + String query1 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, 'test wild??rd')"; + JSONObject result1 = executeJdbcRequest(query1); + verifyDataRows(result1, rows(expected)); + + String query2 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcardquery(KeywordBody, 'test wild??rd')"; + JSONObject result2 = executeJdbcRequest(query2); + verifyDataRows(result2, rows(expected)); + } + + // SQL uses ? as a wildcard which is converted to * in WildcardQuery.java + @Test + public void test_wildcard_query_sql_wildcard_percent_conversion() throws IOException { + String query1 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, 'test%')"; + JSONObject result1 = executeJdbcRequest(query1); + assertEquals(8, result1.getInt("total")); + + String query2 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, 'test*')"; + JSONObject result2 = executeJdbcRequest(query2); + assertEquals(result1.getInt("total"), result2.getInt("total")); + } + + // SQL uses _ as a wildcard which is converted to ? in WildcardQuery.java + @Test + public void test_wildcard_query_sql_wildcard_underscore_conversion() throws IOException { + String query1 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, 'test wild_ard*')"; + JSONObject result1 = executeJdbcRequest(query1); + assertEquals(7, result1.getInt("total")); + + String query2 = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, 'test wild?ard*')"; + JSONObject result2 = executeJdbcRequest(query2); + assertEquals(result1.getInt("total"), result2.getInt("total")); + } + + @Test + public void test_escaping_wildcard_percent_in_the_beginning_of_text() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '\\\\%*')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("%test wildcard in the beginning of the text")); + } + + @Test + public void test_escaping_wildcard_percent_in_text() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '*\\\\%%')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("test wildcard in % the middle of the text"), + rows("test wildcard %% beside each other"), + rows("test wildcard in the end of the text%"), + rows("%test wildcard in the beginning of the text")); + } + + @Test + public void test_escaping_wildcard_percent_in_the_end_of_text() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '*\\\\%')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("test wildcard in the end of the text%")); + } + + @Test + public void test_double_escaped_wildcard_percent() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '*\\\\%\\\\%*')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("test wildcard %% beside each other")); + } + + @Test + public void test_escaping_wildcard_underscore_in_the_beginning_of_text() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '\\\\_*')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("_test wildcard in the beginning of the text")); + } + + @Test + public void test_escaping_wildcard_underscore_in_text() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '*\\\\_*')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("test wildcard in _ the middle of the text"), + rows("test wildcard __ beside each other"), + rows("test wildcard in the end of the text_"), + rows("_test wildcard in the beginning of the text"), + rows("test backslash wildcard \\_")); + } + + @Test + public void test_escaping_wildcard_underscore_in_the_end_of_text() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '*\\\\_')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, + rows("test wildcard in the end of the text_"), + rows("test backslash wildcard \\_")); + } + + @Test + public void test_double_escaped_wildcard_underscore() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '*\\\\_\\\\_*')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("test wildcard __ beside each other")); + } + + @Test + public void test_backslash_wildcard() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(KeywordBody, '*\\\\\\\\\\\\_')"; + JSONObject result = executeJdbcRequest(query); + verifyDataRows(result, rows("test backslash wildcard \\_")); + } + + @Test + public void all_params_test() throws IOException { + String query = "SELECT KeywordBody FROM " + TEST_INDEX_WILDCARD + + " WHERE wildcard_query(KeywordBody, 'test*', boost = 0.9," + + " case_insensitive=true, rewrite='constant_score')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(8, result.getInt("total")); + } + + @Test + public void test_wildcard_query_on_text_field_with_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(TextBody, 'test*')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(9, result.getInt("total")); + } + + @Test + public void test_wildcard_query_on_text_keyword_field_with_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(TextKeywordBody, 'test*')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(9, result.getInt("total")); + } + + @Test + public void test_wildcard_query_on_text_field_with_greater_than_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(TextBody, 'test wild*')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(0, result.getInt("total")); + } + + @Test + public void test_wildcard_query_on_text_keyword_field_with_greater_than_one_word() throws IOException { + String query = "SELECT * FROM " + TEST_INDEX_WILDCARD + " WHERE wildcard_query(TextKeywordBody, 'test wild*')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(0, result.getInt("total")); + } +} diff --git a/integ-test/src/test/resources/correctness/bugfixes/550.txt b/integ-test/src/test/resources/correctness/bugfixes/550.txt new file mode 100644 index 0000000000..902e3b0d20 --- /dev/null +++ b/integ-test/src/test/resources/correctness/bugfixes/550.txt @@ -0,0 +1,2 @@ +SELECT ABS(`flights`.`AvgTicketPrice`) FROM (SELECT `AvgTicketPrice` FROM `opensearch_dashboards_sample_data_flights`) AS `flights` GROUP BY ABS(`flights`.`AvgTicketPrice`) +SELECT `b`.`Origin`, `b`.`avgPrice` FROM (SELECT `a`.`Origin` AS `Origin`, AVG(`AvgTicketPrice`) AS `avgPrice` FROM (SELECT `Origin`, `AvgTicketPrice` FROM `opensearch_dashboards_sample_data_flights` WHERE `FlightDelay` = True) AS `a` GROUP BY `a`.`Origin`) AS `b` ORDER BY `b`.`avgPrice` DESC diff --git a/integ-test/src/test/resources/correctness/queries/aggregation.txt b/integ-test/src/test/resources/correctness/queries/aggregation.txt index 0c0648a937..9b610cb885 100644 --- a/integ-test/src/test/resources/correctness/queries/aggregation.txt +++ b/integ-test/src/test/resources/correctness/queries/aggregation.txt @@ -11,4 +11,5 @@ SELECT VAR_SAMP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights SELECT STDDEV_POP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights SELECT STDDEV_SAMP(AvgTicketPrice) FROM opensearch_dashboards_sample_data_flights SELECT COUNT(DISTINCT Origin), COUNT(DISTINCT Dest) FROM opensearch_dashboards_sample_data_flights -SELECT COUNT(DISTINCT Origin) FROM (SELECT * FROM opensearch_dashboards_sample_data_flights) AS flights \ No newline at end of file +SELECT COUNT(DISTINCT Origin) FROM (SELECT * FROM opensearch_dashboards_sample_data_flights) AS flights +SELECT LOG(MAX(AvgTicketPrice) + MIN(AvgTicketPrice)) FROM opensearch_dashboards_sample_data_flights \ No newline at end of file diff --git a/integ-test/src/test/resources/indexDefinitions/wildcard_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/wildcard_index_mappings.json new file mode 100644 index 0000000000..b9974e9548 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/wildcard_index_mappings.json @@ -0,0 +1,21 @@ +{ + "mappings" : { + "properties" : { + "KeywordBody" : { + "type" : "keyword" + }, + "TextKeywordBody" : { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword", + "ignore_above":256 + } + } + }, + "TextBody" : { + "type" : "text" + } + } + } +} diff --git a/integ-test/src/test/resources/wildcard.json b/integ-test/src/test/resources/wildcard.json new file mode 100644 index 0000000000..b25772a47e --- /dev/null +++ b/integ-test/src/test/resources/wildcard.json @@ -0,0 +1,20 @@ +{"index":{"_id":"0"}} +{"KeywordBody":"test wildcard", "TextKeywordBody":"test wildcard", "TextBody":"test wildcard"} +{"index":{"_id":"1"}} +{"KeywordBody":"test wildcard in the end of the text%", "TextKeywordBody":"test wildcard in the end of the text%", "TextBody":"test wildcard in the end of the text%"} +{"index":{"_id":"2"}} +{"KeywordBody":"%test wildcard in the beginning of the text", "TextKeywordBody":"%test wildcard in the beginning of the text", "TextBody":"%test wildcard in the beginning of the text"} +{"index":{"_id":"3"}} +{"KeywordBody":"test wildcard in % the middle of the text", "TextKeywordBody":"test wildcard in % the middle of the text", "TextBody":"test wildcard in % the middle of the text"} +{"index":{"_id":"4"}} +{"KeywordBody":"test wildcard %% beside each other", "TextKeywordBody":"test wildcard %% beside each other", "TextBody":"test wildcard %% beside each other"} +{"index":{"_id":"5"}} +{"KeywordBody":"test wildcard in the end of the text_", "TextKeywordBody":"test wildcard in the end of the text_", "TextBody":"test wildcard in the end of the text_"} +{"index":{"_id":"6"}} +{"KeywordBody":"_test wildcard in the beginning of the text", "TextKeywordBody":"_test wildcard in the beginning of the text", "TextBody":"_test wildcard in the beginning of the text"} +{"index":{"_id":"7"}} +{"KeywordBody":"test wildcard in _ the middle of the text", "TextKeywordBody":"test wildcard in _ the middle of the text", "TextBody":"test wildcard in _ the middle of the text"} +{"index":{"_id":"8"}} +{"KeywordBody":"test wildcard __ beside each other", "TextKeywordBody":"test wildcard __ beside each other", "TextBody":"test wildcard __ beside each other"} +{"index":{"_id":"9"}} +{"KeywordBody":"test backslash wildcard \\_", "TextKeywordBody":"test backslash wildcard \\_", "TextBody":"test backslash wildcard \\_"} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java index a5a3ac5a4f..bc97f71b47 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.Injector; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestChannel; @@ -36,7 +37,6 @@ import org.opensearch.sql.protocol.response.format.ResponseFormatter; import org.opensearch.sql.sql.SQLService; import org.opensearch.sql.sql.domain.SQLQueryRequest; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; /** * New SQL REST action handler. This will not be registered to OpenSearch unless: @@ -49,14 +49,14 @@ public class RestSQLQueryAction extends BaseRestHandler { public static final RestChannelConsumer NOT_SUPPORTED_YET = null; - private final AnnotationConfigApplicationContext applicationContext; + private final Injector injector; /** * Constructor of RestSQLQueryAction. */ - public RestSQLQueryAction(AnnotationConfigApplicationContext applicationContext) { + public RestSQLQueryAction(Injector injector) { super(); - this.applicationContext = applicationContext; + this.injector = injector; } @Override @@ -91,7 +91,7 @@ public RestChannelConsumer prepareRequest( } SQLService sqlService = - SecurityAccess.doPrivileged(() -> applicationContext.getBean(SQLService.class)); + SecurityAccess.doPrivileged(() -> injector.getInstance(SQLService.class)); if (request.isExplainRequest()) { return channel -> diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java index de09bcee1a..88ed42010b 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/plugin/RestSqlAction.java @@ -26,6 +26,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.Injector; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; import org.opensearch.rest.BaseRestHandler; @@ -62,7 +63,6 @@ import org.opensearch.sql.legacy.utils.JsonPrettyFormatter; import org.opensearch.sql.legacy.utils.QueryDataAnonymizer; import org.opensearch.sql.sql.domain.SQLQueryRequest; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; public class RestSqlAction extends BaseRestHandler { @@ -87,16 +87,10 @@ public class RestSqlAction extends BaseRestHandler { */ private final RestSQLQueryAction newSqlQueryHandler; - /** - * Application context used to create SQLService for each request. - */ - private final AnnotationConfigApplicationContext applicationContext; - - public RestSqlAction(Settings settings, AnnotationConfigApplicationContext applicationContext) { + public RestSqlAction(Settings settings, Injector injector) { super(); this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings); - this.newSqlQueryHandler = new RestSQLQueryAction(applicationContext); - this.applicationContext = applicationContext; + this.newSqlQueryHandler = new RestSQLQueryAction(injector); } @Override diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java index 3eafbd32d9..1bc34edf50 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/plugin/RestSQLQueryActionTest.java @@ -23,20 +23,19 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.Injector; +import org.opensearch.common.inject.ModulesBuilder; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestChannel; import org.opensearch.rest.RestRequest; -import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.common.antlr.SyntaxCheckException; -import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.execution.QueryPlanFactory; -import org.opensearch.sql.sql.config.SQLServiceConfig; +import org.opensearch.sql.sql.SQLService; +import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.domain.SQLQueryRequest; -import org.opensearch.sql.storage.StorageEngine; import org.opensearch.threadpool.ThreadPool; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; @RunWith(MockitoJUnitRunner.class) public class RestSQLQueryActionTest extends BaseRestHandler { @@ -52,25 +51,19 @@ public class RestSQLQueryActionTest extends BaseRestHandler { @Mock private QueryPlanFactory factory; - @Mock - private ExecutionEngine.Schema schema; - @Mock private RestChannel restChannel; - private AnnotationConfigApplicationContext context; + private Injector injector; @Before public void setup() { nodeClient = new NodeClient(org.opensearch.common.settings.Settings.EMPTY, threadPool); - context = new AnnotationConfigApplicationContext(); - context.registerBean(StorageEngine.class, () -> Mockito.mock(StorageEngine.class)); - context.registerBean(ExecutionEngine.class, () -> Mockito.mock(ExecutionEngine.class)); - context.registerBean(DataSourceService.class, () -> Mockito.mock(DataSourceService.class)); - context.registerBean(QueryManager.class, () -> queryManager); - context.registerBean(QueryPlanFactory.class, () -> factory); - context.register(SQLServiceConfig.class); - context.refresh(); + ModulesBuilder modules = new ModulesBuilder(); + modules.add(b -> { + b.bind(SQLService.class).toInstance(new SQLService(new SQLSyntaxParser(), queryManager, factory)); + }); + injector = modules.createInjector(); Mockito.lenient().when(threadPool.getThreadContext()) .thenReturn(new ThreadContext(org.opensearch.common.settings.Settings.EMPTY)); } @@ -83,7 +76,7 @@ public void handleQueryThatCanSupport() throws Exception { QUERY_API_ENDPOINT, ""); - RestSQLQueryAction queryAction = new RestSQLQueryAction(context); + RestSQLQueryAction queryAction = new RestSQLQueryAction(injector); queryAction.prepareRequest(request, (channel, exception) -> { fail(); }, (channel, exception) -> { @@ -99,7 +92,7 @@ public void handleExplainThatCanSupport() throws Exception { EXPLAIN_API_ENDPOINT, ""); - RestSQLQueryAction queryAction = new RestSQLQueryAction(context); + RestSQLQueryAction queryAction = new RestSQLQueryAction(injector); queryAction.prepareRequest(request, (channel, exception) -> { fail(); }, (channel, exception) -> { @@ -117,7 +110,7 @@ public void queryThatNotSupportIsHandledByFallbackHandler() throws Exception { ""); AtomicBoolean fallback = new AtomicBoolean(false); - RestSQLQueryAction queryAction = new RestSQLQueryAction(context); + RestSQLQueryAction queryAction = new RestSQLQueryAction(injector); queryAction.prepareRequest(request, (channel, exception) -> { fallback.set(true); assertTrue(exception instanceof SyntaxCheckException); @@ -142,7 +135,7 @@ public void queryExecutionFailedIsHandledByExecutionErrorHandler() throws Except .submit(any()); AtomicBoolean executionErrorHandler = new AtomicBoolean(false); - RestSQLQueryAction queryAction = new RestSQLQueryAction(context); + RestSQLQueryAction queryAction = new RestSQLQueryAction(injector); queryAction.prepareRequest(request, (channel, exception) -> { assertTrue(exception instanceof SyntaxCheckException); }, (channel, exception) -> { diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java index 67ad4b9ba2..b23b24413a 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/unittest/LocalClusterStateTest.java @@ -106,7 +106,8 @@ public class LocalClusterStateTest { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + //======================================================= " }\n" + "}"; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java index f427170cfc..0f9ee88c69 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java @@ -189,7 +189,8 @@ public static void stubMockClient(Client mockClient) { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + //======================================================= " }\n" + "}"; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java index 0213cac01b..1a3ee07f88 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/MultipleIndexClusterUtils.java @@ -83,7 +83,8 @@ public class MultipleIndexClusterUtils { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + " }\n" + "}"; @@ -134,7 +135,8 @@ public class MultipleIndexClusterUtils { " }\n" + " },\n" + " \"mapping_version\": \"1\",\n" + - " \"settings_version\": \"1\"\n" + + " \"settings_version\": \"1\",\n" + + " \"aliases_version\": \"1\"\n" + " }\n" + "}"; diff --git a/legacy/src/test/resources/mappings/field_mappings.json b/legacy/src/test/resources/mappings/field_mappings.json index bf059d0ae9..997e794d1b 100644 --- a/legacy/src/test/resources/mappings/field_mappings.json +++ b/legacy/src/test/resources/mappings/field_mappings.json @@ -67,6 +67,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} diff --git a/legacy/src/test/resources/mappings/semantics.json b/legacy/src/test/resources/mappings/semantics.json index 46de8f2eaa..e6e7be958e 100644 --- a/legacy/src/test/resources/mappings/semantics.json +++ b/legacy/src/test/resources/mappings/semantics.json @@ -87,6 +87,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 7ad7d63546..6eeab86fff 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -32,9 +32,9 @@ dependencies { api project(':core') api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" implementation group: 'org.json', name: 'json', version:'20180813' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" implementation group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java new file mode 100644 index 0000000000..7b68bd5c92 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/StringUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.storage.script; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class StringUtils { + /** + * Converts sql wildcard character % and _ to * and ?. + * @param text string to be converted + * @return converted string + */ + public static String convertSqlWildcardToLucene(String text) { + final char DEFAULT_ESCAPE = '\\'; + StringBuilder convertedString = new StringBuilder(text.length()); + boolean escaped = false; + + for (char currentChar : text.toCharArray()) { + switch (currentChar) { + case DEFAULT_ESCAPE: + escaped = true; + convertedString.append(currentChar); + break; + case '%': + if (escaped) { + convertedString.deleteCharAt(convertedString.length() - 1); + convertedString.append("%"); + } else { + convertedString.append("*"); + } + escaped = false; + break; + case '_': + if (escaped) { + convertedString.deleteCharAt(convertedString.length() - 1); + convertedString.append("_"); + } else { + convertedString.append('?'); + } + escaped = false; + break; + default: + convertedString.append(currentChar); + escaped = false; + } + } + return convertedString.toString(); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java index 2c55a28b88..5f36954d4a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java @@ -24,11 +24,11 @@ import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.storage.script.filter.lucene.LikeQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.RangeQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.RangeQuery.Comparison; import org.opensearch.sql.opensearch.storage.script.filter.lucene.TermQuery; -import org.opensearch.sql.opensearch.storage.script.filter.lucene.WildcardQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchBoolPrefixQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhrasePrefixQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhraseQuery; @@ -37,6 +37,7 @@ import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryStringQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.SimpleQueryStringQuery; +import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.WildcardQuery; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @RequiredArgsConstructor @@ -57,7 +58,7 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor b.quoteFieldSuffix(v.stringValue())) .build(); + public static final Map> + WildcardQueryBuildActions = ImmutableMap.>builder() + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("case_insensitive", (b, v) -> b.caseInsensitive(convertBoolValue(v, "case_insensitive"))) + .put("rewrite", (b, v) -> b.rewrite(checkRewrite(v, "rewrite"))) + .build(); + public static final Map ArgumentLimitations = ImmutableMap.builder() .put("boost", "Accepts only floating point values greater than 0.") diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java index a7d7584d4f..ec110dfd8b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java @@ -10,6 +10,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.ReferenceExpression; /** * Base class to represent builder class for relevance queries like match_query, match_bool_prefix, @@ -36,7 +37,7 @@ protected T createQueryBuilder(List arguments) { .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); return createBuilder( - field.getValue().valueOf().stringValue(), + ((ReferenceExpression)field.getValue()).getAttr(), query.getValue().valueOf().stringValue()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/WildcardQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/WildcardQuery.java new file mode 100644 index 0000000000..9fd37e3de7 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/WildcardQuery.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.WildcardQueryBuilder; +import org.opensearch.sql.opensearch.storage.script.StringUtils; + +/** + * Lucene query that builds wildcard query. + */ +public class WildcardQuery extends SingleFieldQuery { + /** + * Default constructor for WildcardQuery configures how RelevanceQuery.build() handles + * named arguments. + */ + public WildcardQuery() { + super(FunctionParameterRepository.WildcardQueryBuildActions); + } + + @Override + protected String getQueryName() { + return WildcardQueryBuilder.NAME; + } + + @Override + protected WildcardQueryBuilder createBuilder(String field, String query) { + String matchText = StringUtils.convertSqlWildcardToLucene(query); + return QueryBuilders.wildcardQuery(field, matchText); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java new file mode 100644 index 0000000000..24c809ebab --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/StringUtilsTest.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script; + +import static org.junit.Assert.assertEquals; + +import org.junit.jupiter.api.Test; + +public class StringUtilsTest { + @Test + public void test_escaping_sql_wildcards() { + assertEquals("%", StringUtils.convertSqlWildcardToLucene("\\%")); + assertEquals("\\*", StringUtils.convertSqlWildcardToLucene("\\*")); + assertEquals("_", StringUtils.convertSqlWildcardToLucene("\\_")); + assertEquals("\\?", StringUtils.convertSqlWildcardToLucene("\\?")); + assertEquals("%*", StringUtils.convertSqlWildcardToLucene("\\%%")); + assertEquals("*%", StringUtils.convertSqlWildcardToLucene("%\\%")); + assertEquals("%*%", StringUtils.convertSqlWildcardToLucene("\\%%\\%")); + assertEquals("*%*", StringUtils.convertSqlWildcardToLucene("%\\%%")); + assertEquals("_?", StringUtils.convertSqlWildcardToLucene("\\__")); + assertEquals("?_", StringUtils.convertSqlWildcardToLucene("_\\_")); + assertEquals("_?_", StringUtils.convertSqlWildcardToLucene("\\__\\_")); + assertEquals("?_?", StringUtils.convertSqlWildcardToLucene("_\\__")); + assertEquals("%\\*_\\?", StringUtils.convertSqlWildcardToLucene("\\%\\*\\_\\?")); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 737e61f54b..2ad1f59d39 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -53,6 +53,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -132,6 +133,7 @@ void should_build_wildcard_query_for_like_expression() { + " \"wildcard\" : {\n" + " \"name\" : {\n" + " \"wildcard\" : \"*John?\",\n" + + " \"case_insensitive\" : true,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -282,6 +284,7 @@ void should_use_keyword_for_multi_field_in_like_expression() { + " \"wildcard\" : {\n" + " \"name.keyword\" : {\n" + " \"wildcard\" : \"John*\",\n" + + " \"case_insensitive\" : true,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" @@ -311,7 +314,8 @@ void should_build_match_query_with_default_parameters() { + "}", buildQuery( DSL.match( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query"))))); } @@ -339,7 +343,8 @@ void should_build_match_query_with_custom_parameters() { + "}", buildQuery( DSL.match( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("operator", literal("AND")), DSL.namedArgument("analyzer", literal("keyword")), @@ -358,7 +363,8 @@ void should_build_match_query_with_custom_parameters() { @Test void match_invalid_parameter() { FunctionExpression expr = DSL.match( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); @@ -431,7 +437,8 @@ void should_build_match_phrase_query_with_default_parameters() { + "}", buildQuery( DSL.match_phrase( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query"))))); } @@ -623,14 +630,148 @@ void should_build_match_phrase_query_with_custom_parameters() { + "}", buildQuery( DSL.match_phrase( + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("boost", literal("1.2")), - DSL.namedArgument("field", literal("message")), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("analyzer", literal("keyword")), DSL.namedArgument("slop", literal("2")), DSL.namedArgument("zero_terms_query", literal("ALL"))))); } + @Test + void wildcard_query_invalid_parameter() { + FunctionExpression expr = DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query*")), + DSL.namedArgument("invalid_parameter", literal("invalid_value"))); + assertThrows(SemanticCheckException.class, () -> buildQuery(expr), + "Parameter invalid_parameter is invalid for wildcard_query function."); + } + + @Test + void wildcard_query_convert_sql_wildcard_to_lucene() { + // Test conversion of % wildcard to * + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query*\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query%"))))); + + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query?\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query_"))))); + } + + @Test + void wildcard_query_escape_wildcards_characters() { + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query%\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query\\%"))))); + + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query_\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query\\_"))))); + + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query\\\\*\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query\\*"))))); + + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query\\\\?\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query\\?"))))); + } + + @Test + void should_build_wildcard_query_with_default_parameters() { + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query*\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query*"))))); + } + + @Test + void should_build_wildcard_query_query_with_custom_parameters() { + assertJsonEquals("{\n" + + " \"wildcard\" : {\n" + + " \"field\" : {\n" + + " \"wildcard\" : \"search query*\",\n" + + " \"boost\" : 0.6,\n" + + " \"case_insensitive\" : true,\n" + + " \"rewrite\" : \"constant_score_boolean\"\n" + + " }\n" + + " }\n" + + "}", + buildQuery(DSL.wildcard_query( + DSL.namedArgument("field", + new ReferenceExpression("field", OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", literal("search query*")), + DSL.namedArgument("boost", literal("0.6")), + DSL.namedArgument("case_insensitive", literal("true")), + DSL.namedArgument("rewrite", literal("constant_score_boolean"))))); + } + @Test void query_invalid_parameter() { FunctionExpression expr = DSL.query( @@ -962,7 +1103,8 @@ void simple_query_string_invalid_parameter() { @Test void match_phrase_invalid_parameter() { FunctionExpression expr = DSL.match_phrase( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); @@ -971,7 +1113,8 @@ void match_phrase_invalid_parameter() { @Test void relevancy_func_invalid_arg_values() { - final var field = DSL.namedArgument("field", literal("message")); + final var field = DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)); final var fields = DSL.namedArgument("fields", DSL.literal( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( "field1", ExprValueUtils.floatValue(1.F), @@ -1049,18 +1192,11 @@ void should_build_match_bool_prefix_query_with_default_parameters() { + "}", buildQuery( DSL.match_bool_prefix( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query"))))); } - @Test - void multi_match_missing_fields() { - var msg = assertThrows(SemanticCheckException.class, () -> - DSL.multi_match( - DSL.namedArgument("query", literal("search query")))).getMessage(); - assertEquals("Expected type STRUCT instead of STRING for parameter #1", msg); - } - @Test void multi_match_missing_fields_even_with_struct() { FunctionExpression expr = DSL.multi_match( @@ -1101,7 +1237,8 @@ void should_build_match_phrase_prefix_query_with_default_parameters() { + "}", buildQuery( DSL.match_phrase_prefix( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query"))))); } @@ -1122,7 +1259,8 @@ void should_build_match_phrase_prefix_query_with_non_default_parameters() { + "}", buildQuery( DSL.match_phrase_prefix( - DSL.namedArgument("field", literal("message")), + DSL.namedArgument("field", + new ReferenceExpression("message", OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", literal("search query")), DSL.namedArgument("boost", literal("1.2")), DSL.namedArgument("max_expansions", literal("42")), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java index 162c55fcaf..34ef29f091 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchBoolPrefixQueryTest.java @@ -23,8 +23,10 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchBoolPrefixQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -33,7 +35,8 @@ public class MatchBoolPrefixQueryTest { private final FunctionName matchBoolPrefix = FunctionName.of("match_bool_prefix"); static Stream> generateValidData() { - NamedArgumentExpression field = DSL.namedArgument("field", DSL.literal("field_value")); + NamedArgumentExpression field = DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)); NamedArgumentExpression query = DSL.namedArgument("query", DSL.literal("query_value")); return List.of( DSL.namedArgument("fuzziness", DSL.literal("AUTO")), @@ -58,7 +61,8 @@ public void test_valid_arguments(List validArgs) { @Test public void test_valid_when_two_arguments() { List arguments = List.of( - DSL.namedArgument("field", "field_value"), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "query_value")); Assertions.assertNotNull(matchBoolPrefixQuery.build(new MatchExpression(arguments))); } @@ -80,7 +84,8 @@ public void test_SyntaxCheckException_when_one_argument() { @Test public void test_SemanticCheckException_when_invalid_argument() { List arguments = List.of( - DSL.namedArgument("field", "field_value"), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "query_value"), DSL.namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java index a0b9e5f318..e02f112677 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhrasePrefixQueryTest.java @@ -20,8 +20,10 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhrasePrefixQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -39,7 +41,8 @@ public void test_SyntaxCheckException_when_no_arguments() { @Test public void test_SyntaxCheckException_when_one_argument() { - List arguments = List.of(DSL.namedArgument("field", "test")); + List arguments = List.of(DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); assertThrows(SyntaxCheckException.class, () -> matchPhrasePrefixQuery.build(new MatchPhraseExpression(arguments))); } @@ -47,7 +50,8 @@ public void test_SyntaxCheckException_when_one_argument() { @Test public void test_SyntaxCheckException_when_invalid_parameter() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -57,7 +61,8 @@ public void test_SyntaxCheckException_when_invalid_parameter() { @Test public void test_analyzer_parameter() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -67,7 +72,8 @@ public void test_analyzer_parameter() { @Test public void build_succeeds_with_two_arguments() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhrasePrefixQuery.build(new MatchPhraseExpression(arguments))); } @@ -75,7 +81,8 @@ public void build_succeeds_with_two_arguments() { @Test public void test_slop_parameter() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -85,7 +92,8 @@ public void test_slop_parameter() { @Test public void test_zero_terms_query_parameter() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -95,7 +103,8 @@ public void test_zero_terms_query_parameter() { @Test public void test_zero_terms_query_parameter_lower_case() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); @@ -105,7 +114,8 @@ public void test_zero_terms_query_parameter_lower_case() { @Test public void test_boost_parameter() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("boost", "0.1") ); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java index 6a298326b7..dd6296279c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchPhraseQueryTest.java @@ -20,8 +20,10 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhraseQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -41,7 +43,8 @@ public void test_SyntaxCheckException_when_no_arguments() { @Test public void test_SyntaxCheckException_when_one_argument() { - List arguments = List.of(DSL.namedArgument("field", "test")); + List arguments = List.of(DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); assertThrows(SyntaxCheckException.class, () -> matchPhraseQuery.build(new MatchPhraseExpression(arguments))); } @@ -49,7 +52,8 @@ public void test_SyntaxCheckException_when_one_argument() { @Test public void test_SyntaxCheckException_when_invalid_parameter() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -59,7 +63,8 @@ public void test_SyntaxCheckException_when_invalid_parameter() { @Test public void test_analyzer_parameter() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -69,7 +74,8 @@ public void test_analyzer_parameter() { @Test public void build_succeeds_with_two_arguments() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression(arguments))); } @@ -77,7 +83,8 @@ public void build_succeeds_with_two_arguments() { @Test public void test_slop_parameter() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -87,7 +94,8 @@ public void test_slop_parameter() { @Test public void test_zero_terms_query_parameter() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -97,7 +105,8 @@ public void test_zero_terms_query_parameter() { @Test public void test_zero_terms_query_parameter_lower_case() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); @@ -114,7 +123,8 @@ public void test_SyntaxCheckException_when_no_arguments_match_phrase_syntax() { @Test public void test_SyntaxCheckException_when_one_argument_match_phrase_syntax() { - List arguments = List.of(DSL.namedArgument("field", "test")); + List arguments = List.of(DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); assertThrows(SyntaxCheckException.class, () -> matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseWithUnderscoreName))); @@ -124,7 +134,8 @@ public void test_SyntaxCheckException_when_one_argument_match_phrase_syntax() { @Test public void test_SyntaxCheckException_when_invalid_parameter_match_phrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -135,7 +146,8 @@ public void test_SyntaxCheckException_when_invalid_parameter_match_phrase_syntax @Test public void test_analyzer_parameter_match_phrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -146,7 +158,8 @@ public void test_analyzer_parameter_match_phrase_syntax() { @Test public void build_succeeds_with_two_arguments_match_phrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseWithUnderscoreName))); @@ -155,7 +168,8 @@ public void build_succeeds_with_two_arguments_match_phrase_syntax() { @Test public void test_slop_parameter_match_phrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -166,7 +180,8 @@ public void test_slop_parameter_match_phrase_syntax() { @Test public void test_zero_terms_query_parameter_match_phrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -177,7 +192,8 @@ public void test_zero_terms_query_parameter_match_phrase_syntax() { @Test public void test_zero_terms_query_parameter_lower_case_match_phrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); @@ -195,7 +211,8 @@ public void test_SyntaxCheckException_when_no_arguments_matchphrase_syntax() { @Test public void test_SyntaxCheckException_when_one_argument_matchphrase_syntax() { - List arguments = List.of(DSL.namedArgument("field", "test")); + List arguments = List.of(DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); assertThrows(SyntaxCheckException.class, () -> matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseQueryName))); @@ -205,7 +222,8 @@ public void test_SyntaxCheckException_when_one_argument_matchphrase_syntax() { @Test public void test_SyntaxCheckException_when_invalid_parameter_matchphrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2"), DSL.namedArgument("unsupported", "3")); Assertions.assertThrows(SemanticCheckException.class, @@ -216,7 +234,8 @@ public void test_SyntaxCheckException_when_invalid_parameter_matchphrase_syntax( @Test public void test_analyzer_parameter_matchphrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("analyzer", "standard") ); @@ -227,7 +246,8 @@ public void test_analyzer_parameter_matchphrase_syntax() { @Test public void build_succeeds_with_two_arguments_matchphrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "test"), + DSL.namedArgument("field", + new ReferenceExpression("test", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "test2")); Assertions.assertNotNull(matchPhraseQuery.build(new MatchPhraseExpression( arguments, matchPhraseQueryName))); @@ -236,7 +256,8 @@ public void build_succeeds_with_two_arguments_matchphrase_syntax() { @Test public void test_slop_parameter_matchphrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("slop", "2") ); @@ -247,7 +268,8 @@ public void test_slop_parameter_matchphrase_syntax() { @Test public void test_zero_terms_query_parameter_matchphrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "ALL") ); @@ -258,7 +280,8 @@ public void test_zero_terms_query_parameter_matchphrase_syntax() { @Test public void test_zero_terms_query_parameter_lower_case_matchphrase_syntax() { List arguments = List.of( - DSL.namedArgument("field", "t1"), + DSL.namedArgument("field", + new ReferenceExpression("t1", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", "t2"), DSL.namedArgument("zero_terms_query", "all") ); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java index e18b477745..f7d1f1aa9a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MatchQueryTest.java @@ -23,8 +23,10 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchQuery; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -39,71 +41,85 @@ public class MatchQueryTest { static Stream> generateValidData() { return Stream.of( List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("analyzer", DSL.literal("standard")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("auto_generate_synonyms_phrase_query", DSL.literal("true")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("fuzziness", DSL.literal("AUTO")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("max_expansions", DSL.literal("50")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("prefix_length", DSL.literal("0")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("fuzzy_transpositions", DSL.literal("true")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("fuzzy_rewrite", DSL.literal("constant_score")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("lenient", DSL.literal("false")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("operator", DSL.literal("OR")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("minimum_should_match", DSL.literal("3")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("zero_terms_query", DSL.literal("NONE")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("zero_terms_query", DSL.literal("none")) ), List.of( - DSL.namedArgument("field", DSL.literal("field_value")), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), DSL.namedArgument("query", DSL.literal("query_value")), DSL.namedArgument("boost", DSL.literal("1")) ) @@ -133,7 +149,8 @@ public void test_SyntaxCheckException_when_one_argument() { @Test public void test_SemanticCheckException_when_invalid_parameter() { List arguments = List.of( - namedArgument("field", "field_value"), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), namedArgument("query", "query_value"), namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, @@ -166,7 +183,8 @@ public void test_SyntaxCheckException_when_one_argument_matchquery_syntax() { @Test public void test_SemanticCheckException_when_invalid_parameter_matchquery_syntax() { List arguments = List.of( - namedArgument("field", "field_value"), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), namedArgument("query", "query_value"), namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, @@ -200,7 +218,8 @@ public void test_SyntaxCheckException_when_one_argument_match_query_syntax() { @Test public void test_SemanticCheckException_when_invalid_parameter_match_query_syntax() { List arguments = List.of( - namedArgument("field", "field_value"), + DSL.namedArgument("field", + new ReferenceExpression("field_value", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), namedArgument("query", "query_value"), namedArgument("unsupported", "unsupported_value")); Assertions.assertThrows(SemanticCheckException.class, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/WildcardQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/WildcardQueryTest.java new file mode 100644 index 0000000000..684036595c --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/WildcardQueryTest.java @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.expression.DSL.namedArgument; + +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.WildcardQuery; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class WildcardQueryTest { + private final WildcardQuery wildcardQueryQuery = new WildcardQuery(); + private static final FunctionName wildcardQueryFunc = FunctionName.of("wildcard_query"); + + static Stream> generateValidData() { + return Stream.of( + List.of( + namedArgument("field", + new ReferenceExpression("title", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + namedArgument("query", "query_value*"), + namedArgument("boost", "0.7"), + namedArgument("case_insensitive", "false"), + namedArgument("rewrite", "constant_score_boolean") + ) + ); + } + + @ParameterizedTest + @MethodSource("generateValidData") + public void test_valid_parameters(List validArgs) { + Assertions.assertNotNull(wildcardQueryQuery.build( + new WildcardQueryExpression(validArgs))); + } + + @Test + public void test_SyntaxCheckException_when_no_arguments() { + List arguments = List.of(); + assertThrows(SyntaxCheckException.class, + () -> wildcardQueryQuery.build(new WildcardQueryExpression(arguments))); + } + + @Test + public void test_SyntaxCheckException_when_one_argument() { + List arguments = List.of(namedArgument("field", + new ReferenceExpression("title", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD))); + assertThrows(SyntaxCheckException.class, + () -> wildcardQueryQuery.build(new WildcardQueryExpression(arguments))); + } + + @Test + public void test_SemanticCheckException_when_invalid_parameter() { + List arguments = List.of( + namedArgument("field", + new ReferenceExpression("title", OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + namedArgument("query", "query_value*"), + namedArgument("unsupported", "unsupported_value")); + Assertions.assertThrows(SemanticCheckException.class, + () -> wildcardQueryQuery.build(new WildcardQueryExpression(arguments))); + } + + private class WildcardQueryExpression extends FunctionExpression { + public WildcardQueryExpression(List arguments) { + super(WildcardQueryTest.this.wildcardQueryFunc, arguments); + } + + @Override + public ExprValue valueOf(Environment valueEnv) { + throw new UnsupportedOperationException("Invalid function call, " + + "valueOf function need implementation only to support Expression interface"); + } + + @Override + public ExprType type() { + throw new UnsupportedOperationException("Invalid function call, " + + "type function need implementation only to support Expression interface"); + } + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java index b2d650602b..67f22178bc 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java @@ -19,6 +19,8 @@ import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; class SingleFieldQueryTest { SingleFieldQuery query; @@ -35,12 +37,26 @@ void setUp() { } @Test - void createQueryBuilderTest() { + void createQueryBuilderTestTypeTextKeyword() { String sampleQuery = "sample query"; String sampleField = "fieldA"; query.createQueryBuilder(List.of(DSL.namedArgument("field", - new LiteralExpression(ExprValueUtils.stringValue(sampleField))), + new ReferenceExpression(sampleField, OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD)), + DSL.namedArgument("query", + new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); + + verify(query).createBuilder(eq(sampleField), + eq(sampleQuery)); + } + + @Test + void createQueryBuilderTestTypeText() { + String sampleQuery = "sample query"; + String sampleField = "fieldA"; + + query.createQueryBuilder(List.of(DSL.namedArgument("field", + new ReferenceExpression(sampleField, OpenSearchDataType.OPENSEARCH_TEXT)), DSL.namedArgument("query", new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); diff --git a/opensearch/src/test/resources/mappings/accounts.json b/opensearch/src/test/resources/mappings/accounts.json index a7024b6490..5431568304 100644 --- a/opensearch/src/test/resources/mappings/accounts.json +++ b/opensearch/src/test/resources/mappings/accounts.json @@ -87,6 +87,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} diff --git a/opensearch/src/test/resources/mappings/accounts2.json b/opensearch/src/test/resources/mappings/accounts2.json index d300b8c523..0a5937fd36 100644 --- a/opensearch/src/test/resources/mappings/accounts2.json +++ b/opensearch/src/test/resources/mappings/accounts2.json @@ -88,6 +88,7 @@ } }, "mapping_version": "1", - "settings_version": "1" + "settings_version": "1", + "aliases_version": "1" } -} \ No newline at end of file +} diff --git a/plugin/build.gradle b/plugin/build.gradle index d2bdb87275..04f57b0ca6 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -83,14 +83,16 @@ thirdPartyAudit.enabled = false configurations.all { resolutionStrategy.force 'junit:junit:4.13.2' // conflict with spring-jcl - exclude group: "commons-logging", module: "commons-logging" + resolutionStrategy.force "commons-logging:commons-logging:1.2" // enforce 2.12.6, https://github.com/opensearch-project/sql/issues/424 - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" @@ -108,10 +110,9 @@ compileTestJava { } dependencies { - api group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" - api "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" - api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" + api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api project(":ppl") api project(':legacy') diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index fab14966d8..98826c6f62 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -31,6 +31,8 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Injector; +import org.opensearch.common.inject.ModulesBuilder; import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; @@ -67,7 +69,7 @@ import org.opensearch.sql.opensearch.storage.OpenSearchDataSourceFactory; import org.opensearch.sql.opensearch.storage.script.ExpressionScriptEngine; import org.opensearch.sql.opensearch.storage.serialization.DefaultExpressionSerializer; -import org.opensearch.sql.plugin.config.OpenSearchPluginConfig; +import org.opensearch.sql.plugin.config.OpenSearchPluginModule; import org.opensearch.sql.plugin.datasource.DataSourceSettings; import org.opensearch.sql.plugin.rest.RestPPLQueryAction; import org.opensearch.sql.plugin.rest.RestPPLStatsAction; @@ -75,15 +77,12 @@ import org.opensearch.sql.plugin.transport.PPLQueryAction; import org.opensearch.sql.plugin.transport.TransportPPLQueryAction; import org.opensearch.sql.plugin.transport.TransportPPLQueryResponse; -import org.opensearch.sql.ppl.config.PPLServiceConfig; import org.opensearch.sql.prometheus.storage.PrometheusStorageFactory; -import org.opensearch.sql.sql.config.SQLServiceConfig; import org.opensearch.sql.storage.DataSourceFactory; import org.opensearch.threadpool.ExecutorBuilder; import org.opensearch.threadpool.FixedExecutorBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; public class SQLPlugin extends Plugin implements ActionPlugin, ScriptPlugin, ReloadablePlugin { @@ -98,10 +97,10 @@ public class SQLPlugin extends Plugin implements ActionPlugin, ScriptPlugin, Rel private NodeClient client; - private AnnotationConfigApplicationContext applicationContext; - private DataSourceService dataSourceService; + private Injector injector; + public String name() { return "sql"; } @@ -127,7 +126,7 @@ public List getRestHandlers( return Arrays.asList( new RestPPLQueryAction(pluginSettings, settings), - new RestSqlAction(settings, applicationContext), + new RestSqlAction(settings, injector), new RestSqlStatsAction(settings, restController), new RestPPLStatsAction(settings, restController), new RestQuerySettingsAction(settings, restController)); @@ -172,25 +171,16 @@ public Collection createComponents( LocalClusterState.state().setClusterService(clusterService); LocalClusterState.state().setPluginSettings((OpenSearchSettings) pluginSettings); - this.applicationContext = new AnnotationConfigApplicationContext(); - SecurityAccess.doPrivileged( - () -> { - applicationContext.registerBean(ClusterService.class, () -> clusterService); - applicationContext.registerBean(NodeClient.class, () -> (NodeClient) client); - applicationContext.registerBean( - org.opensearch.sql.common.setting.Settings.class, () -> pluginSettings); - applicationContext.registerBean( - DataSourceService.class, () -> dataSourceService); - applicationContext.register(OpenSearchPluginConfig.class); - applicationContext.register(PPLServiceConfig.class); - applicationContext.register(SQLServiceConfig.class); - applicationContext.refresh(); - return null; - }); + ModulesBuilder modules = new ModulesBuilder(); + modules.add(new OpenSearchPluginModule()); + modules.add(b -> { + b.bind(NodeClient.class).toInstance((NodeClient) client); + b.bind(org.opensearch.sql.common.setting.Settings.class).toInstance(pluginSettings); + b.bind(DataSourceService.class).toInstance(dataSourceService); + }); - // return objects used by Guice to inject dependencies for e.g., - // transport action handler constructors - return ImmutableList.of(applicationContext); + injector = modules.createInjector(); + return ImmutableList.of(dataSourceService); } @Override diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java deleted file mode 100644 index 1a98510aad..0000000000 --- a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginConfig.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - - -package org.opensearch.sql.plugin.config; - -import org.opensearch.client.node.NodeClient; -import org.opensearch.sql.analysis.Analyzer; -import org.opensearch.sql.analysis.ExpressionAnalyzer; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.datasource.DataSourceService; -import org.opensearch.sql.executor.ExecutionEngine; -import org.opensearch.sql.executor.QueryManager; -import org.opensearch.sql.executor.QueryService; -import org.opensearch.sql.executor.execution.QueryPlanFactory; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.monitor.ResourceMonitor; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; -import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; -import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager; -import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; -import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; -import org.opensearch.sql.planner.Planner; -import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; -import org.opensearch.sql.storage.StorageEngine; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; - -/** - * OpenSearch plugin config that injects cluster service and node client from plugin - * and initialize OpenSearch storage and execution engine. - */ -@Configuration -public class OpenSearchPluginConfig { - - @Autowired - private NodeClient nodeClient; - - @Autowired - private Settings settings; - - @Autowired - private DataSourceService dataSourceService; - - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public OpenSearchClient client() { - return new OpenSearchNodeClient(nodeClient); - } - - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public StorageEngine storageEngine() { - return new OpenSearchStorageEngine(client(), settings); - } - - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public ExecutionEngine executionEngine() { - return new OpenSearchExecutionEngine(client(), protector()); - } - - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public ResourceMonitor resourceMonitor() { - return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy()); - } - - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public ExecutionProtector protector() { - return new OpenSearchExecutionProtector(resourceMonitor()); - } - - /** - * Per node singleton object. - */ - @Bean - public QueryManager queryManager() { - return new OpenSearchQueryManager(nodeClient); - } - - /** - * QueryPlanFactory. - */ - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public QueryPlanFactory queryExecutionFactory() { - BuiltinFunctionRepository functionRepository = BuiltinFunctionRepository.getInstance(); - Analyzer analyzer = new Analyzer(new ExpressionAnalyzer(functionRepository), - dataSourceService, functionRepository); - Planner planner = - new Planner(LogicalPlanOptimizer.create()); - return new QueryPlanFactory(new QueryService(analyzer, executionEngine(), planner)); - } -} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java new file mode 100644 index 0000000000..5ab4bbaecd --- /dev/null +++ b/plugin/src/main/java/org/opensearch/sql/plugin/config/OpenSearchPluginModule.java @@ -0,0 +1,103 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.plugin.config; + +import lombok.RequiredArgsConstructor; +import org.opensearch.client.node.NodeClient; +import org.opensearch.common.inject.AbstractModule; +import org.opensearch.common.inject.Provides; +import org.opensearch.common.inject.Singleton; +import org.opensearch.sql.analysis.Analyzer; +import org.opensearch.sql.analysis.ExpressionAnalyzer; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.executor.ExecutionEngine; +import org.opensearch.sql.executor.QueryManager; +import org.opensearch.sql.executor.QueryService; +import org.opensearch.sql.executor.execution.QueryPlanFactory; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.monitor.ResourceMonitor; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.client.OpenSearchNodeClient; +import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; +import org.opensearch.sql.opensearch.executor.OpenSearchQueryManager; +import org.opensearch.sql.opensearch.executor.protector.ExecutionProtector; +import org.opensearch.sql.opensearch.executor.protector.OpenSearchExecutionProtector; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.opensearch.storage.OpenSearchStorageEngine; +import org.opensearch.sql.planner.Planner; +import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; +import org.opensearch.sql.ppl.PPLService; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; +import org.opensearch.sql.sql.SQLService; +import org.opensearch.sql.sql.antlr.SQLSyntaxParser; +import org.opensearch.sql.storage.StorageEngine; + +@RequiredArgsConstructor +public class OpenSearchPluginModule extends AbstractModule { + + private final BuiltinFunctionRepository functionRepository = + BuiltinFunctionRepository.getInstance(); + + @Override + protected void configure() { + } + + @Provides + public OpenSearchClient openSearchClient(NodeClient nodeClient) { + return new OpenSearchNodeClient(nodeClient); + } + + @Provides + public StorageEngine storageEngine(OpenSearchClient client, Settings settings) { + return new OpenSearchStorageEngine(client, settings); + } + + @Provides + public ExecutionEngine executionEngine(OpenSearchClient client, ExecutionProtector protector) { + return new OpenSearchExecutionEngine(client, protector); + } + + @Provides + public ResourceMonitor resourceMonitor(Settings settings) { + return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy()); + } + + @Provides + public ExecutionProtector protector(ResourceMonitor resourceMonitor) { + return new OpenSearchExecutionProtector(resourceMonitor); + } + + @Provides + @Singleton + public QueryManager queryManager(NodeClient nodeClient) { + return new OpenSearchQueryManager(nodeClient); + } + + @Provides + public PPLService pplService(QueryManager queryManager, QueryPlanFactory queryPlanFactory) { + return new PPLService(new PPLSyntaxParser(), queryManager, queryPlanFactory); + } + + @Provides + public SQLService sqlService(QueryManager queryManager, QueryPlanFactory queryPlanFactory) { + return new SQLService(new SQLSyntaxParser(), queryManager, queryPlanFactory); + } + + /** + * {@link QueryPlanFactory}. + */ + @Provides + public QueryPlanFactory queryPlanFactory( + DataSourceService dataSourceService, ExecutionEngine executionEngine) { + Analyzer analyzer = + new Analyzer( + new ExpressionAnalyzer(functionRepository), dataSourceService, functionRepository); + Planner planner = new Planner(LogicalPlanOptimizer.create()); + return new QueryPlanFactory(new QueryService(analyzer, executionEngine, planner)); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java index af57c91e5c..6825b2ac92 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/transport/TransportPPLQueryAction.java @@ -16,14 +16,18 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.inject.Inject; +import org.opensearch.common.inject.Injector; +import org.opensearch.common.inject.ModulesBuilder; import org.opensearch.sql.common.response.ResponseListener; -import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.utils.QueryContext; +import org.opensearch.sql.datasource.DataSourceService; +import org.opensearch.sql.datasource.DataSourceServiceImpl; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.legacy.metrics.MetricName; import org.opensearch.sql.legacy.metrics.Metrics; import org.opensearch.sql.opensearch.security.SecurityAccess; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; +import org.opensearch.sql.plugin.config.OpenSearchPluginModule; import org.opensearch.sql.ppl.PPLService; import org.opensearch.sql.ppl.domain.PPLQueryRequest; import org.opensearch.sql.protocol.response.QueryResult; @@ -36,21 +40,12 @@ import org.opensearch.sql.protocol.response.format.VisualizationResponseFormatter; import org.opensearch.tasks.Task; import org.opensearch.transport.TransportService; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; /** Send PPL query transport action. */ public class TransportPPLQueryAction extends HandledTransportAction { - private final NodeClient client; - - /** Cluster service required by bean initialization. */ - private final ClusterService clusterService; - - /** Settings required by been initialization. */ - private final Settings pluginSettings; - - private final AnnotationConfigApplicationContext applicationContext; + private final Injector injector; /** Constructor of TransportPPLQueryAction. */ @Inject @@ -59,12 +54,19 @@ public TransportPPLQueryAction( ActionFilters actionFilters, NodeClient client, ClusterService clusterService, - AnnotationConfigApplicationContext applicationContext) { + DataSourceServiceImpl dataSourceService) { super(PPLQueryAction.NAME, transportService, actionFilters, TransportPPLQueryRequest::new); - this.client = client; - this.clusterService = clusterService; - this.pluginSettings = new OpenSearchSettings(clusterService.getClusterSettings()); - this.applicationContext = applicationContext; + + ModulesBuilder modules = new ModulesBuilder(); + modules.add(new OpenSearchPluginModule()); + modules.add( + b -> { + b.bind(NodeClient.class).toInstance(client); + b.bind(org.opensearch.sql.common.setting.Settings.class) + .toInstance(new OpenSearchSettings(clusterService.getClusterSettings())); + b.bind(DataSourceService.class).toInstance(dataSourceService); + }); + this.injector = modules.createInjector(); } /** @@ -80,7 +82,7 @@ protected void doExecute( QueryContext.addRequestId(); PPLService pplService = - SecurityAccess.doPrivileged(() -> applicationContext.getBean(PPLService.class)); + SecurityAccess.doPrivileged(() -> injector.getInstance(PPLService.class)); TransportPPLQueryRequest transportRequest = TransportPPLQueryRequest.fromActionRequest(request); // in order to use PPL service, we need to convert TransportPPLQueryRequest to PPLQueryRequest PPLQueryRequest transformedRequest = transportRequest.toPPLQueryRequest(); diff --git a/ppl/build.gradle b/ppl/build.gradle index 1c605f4372..9e2ce321a4 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -47,8 +47,6 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' api group: 'org.json', name: 'json', version: '20180813' - implementation group: 'org.springframework', name: 'spring-context', version: "${spring_version}" - implementation group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version:'2.17.1' api project(':common') api project(':core') diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index c356099225..12c24bd531 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -248,6 +248,7 @@ DATE: 'DATE'; DATE_ADD: 'DATE_ADD'; DATE_FORMAT: 'DATE_FORMAT'; DATE_SUB: 'DATE_SUB'; +DATEDIFF: 'DATEDIFF'; DAYNAME: 'DAYNAME'; DAYOFMONTH: 'DAYOFMONTH'; DAYOFWEEK: 'DAYOFWEEK'; @@ -267,6 +268,7 @@ SUBTIME: 'SUBTIME'; SYSDATE: 'SYSDATE'; TIME: 'TIME'; TIME_TO_SEC: 'TIME_TO_SEC'; +TIMEDIFF: 'TIMEDIFF'; TIMESTAMP: 'TIMESTAMP'; TO_DAYS: 'TO_DAYS'; UTC_DATE: 'UTC_DATE'; @@ -292,6 +294,7 @@ LEFT: 'LEFT'; ASCII: 'ASCII'; LOCATE: 'LOCATE'; REPLACE: 'REPLACE'; +REVERSE: 'REVERSE'; CAST: 'CAST'; // BOOL FUNCTIONS diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 9cb539cfba..dbb63ef12a 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -254,11 +254,15 @@ comparisonExpression ; valueExpression - : left=valueExpression binaryOperator right=valueExpression #binaryArithmetic - | LT_PRTHS left=valueExpression binaryOperator - right=valueExpression RT_PRTHS #parentheticBinaryArithmetic + : left=valueExpression + binaryOperator=(STAR | DIVIDE | MODULE) + right=valueExpression #binaryArithmetic + | left=valueExpression + binaryOperator=(PLUS | MINUS) + right=valueExpression #binaryArithmetic | primaryExpression #valueExpressionDefault | positionFunction #positionFunctionCall + | LT_PRTHS valueExpression RT_PRTHS #parentheticValueExpr ; primaryExpression @@ -438,6 +442,7 @@ dateAndTimeFunctionBase | DATE_ADD | DATE_FORMAT | DATE_SUB + | DATEDIFF | DATETIME | DAY | DAYNAME @@ -467,6 +472,7 @@ dateAndTimeFunctionBase | SYSDATE | TIME | TIME_TO_SEC + | TIMEDIFF | TIMESTAMP | TO_DAYS | UNIX_TIMESTAMP @@ -489,7 +495,7 @@ systemFunctionBase textFunctionBase : SUBSTR | SUBSTRING | TRIM | LTRIM | RTRIM | LOWER | UPPER | CONCAT | CONCAT_WS | LENGTH | STRCMP - | RIGHT | LEFT | ASCII | LOCATE | REPLACE + | RIGHT | LEFT | ASCII | LOCATE | REPLACE | REVERSE ; positionFunctionName @@ -501,10 +507,6 @@ comparisonOperator : EQUAL | NOT_EQUAL | LESS | NOT_LESS | GREATER | NOT_GREATER | REGEXP ; -binaryOperator - : PLUS | MINUS | STAR | DIVIDE | MODULE - ; - singleFieldRelevanceFunctionName : MATCH diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java deleted file mode 100644 index 1067bbaa6b..0000000000 --- a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.ppl.config; - -import org.opensearch.sql.executor.QueryManager; -import org.opensearch.sql.executor.execution.QueryPlanFactory; -import org.opensearch.sql.ppl.PPLService; -import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; - -@Configuration -public class PPLServiceConfig { - - @Autowired - private QueryManager queryManager; - - @Autowired - private QueryPlanFactory queryPlanFactory; - - /** - * The registration of OpenSearch storage engine happens here because - * OpenSearchStorageEngine is dependent on NodeClient. - * - * @return PPLService. - */ - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public PPLService pplService() { - return new PPLService(new PPLSyntaxParser(), queryManager, queryPlanFactory); - } - -} diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 68608e23ad..c9823b67f9 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -33,7 +33,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalOrContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.LogicalXorContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.MultiFieldRelevanceFunctionContext; -import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ParentheticBinaryArithmeticContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ParentheticValueExprContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.PercentileAggFunctionContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SingleFieldRelevanceFunctionContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.SortFieldContext; @@ -154,18 +154,14 @@ public UnresolvedExpression visitInExpr(InExprContext ctx) { @Override public UnresolvedExpression visitBinaryArithmetic(BinaryArithmeticContext ctx) { return new Function( - ctx.binaryOperator().getText(), + ctx.binaryOperator.getText(), Arrays.asList(visit(ctx.left), visit(ctx.right)) ); } @Override - public UnresolvedExpression visitParentheticBinaryArithmetic( - ParentheticBinaryArithmeticContext ctx) { - return new Function( - ctx.binaryOperator().getText(), - Arrays.asList(visit(ctx.left), visit(ctx.right)) - ); + public UnresolvedExpression visitParentheticValueExpr(ParentheticValueExprContext ctx) { + return visit(ctx.valueExpression()); // Discard parenthesis around } /** @@ -372,7 +368,7 @@ private List singleFieldRelevanceArguments( // to skip environment resolving and function signature resolving ImmutableList.Builder builder = ImmutableList.builder(); builder.add(new UnresolvedArgument("field", - new Literal(StringUtils.unquoteText(ctx.field.getText()), DataType.STRING))); + new QualifiedName(StringUtils.unquoteText(ctx.field.getText())))); builder.add(new UnresolvedArgument("query", new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); ctx.relevanceArg().forEach(v -> builder.add(new UnresolvedArgument( diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index a1b1ccaf14..178335a126 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -19,20 +19,15 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.opensearch.sql.common.response.ResponseListener; -import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.executor.DefaultQueryManager; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; import org.opensearch.sql.executor.ExecutionEngine.QueryResponse; -import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.execution.QueryPlanFactory; -import org.opensearch.sql.expression.function.FunctionProperties; -import org.opensearch.sql.ppl.config.PPLServiceConfig; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.ppl.domain.PPLQueryRequest; -import org.opensearch.sql.storage.StorageEngine; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; @RunWith(MockitoJUnitRunner.class) public class PPLServiceTest { @@ -41,42 +36,24 @@ public class PPLServiceTest { private static String EXPLAIN = "/_plugins/_ppl/_explain"; - private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - private PPLService pplService; - @Mock - private QueryService queryService; - - @Mock - private StorageEngine storageEngine; - - @Mock - private ExecutionEngine executionEngine; + private DefaultQueryManager queryManager; @Mock - private DataSourceService dataSourceService; + private QueryService queryService; @Mock private ExecutionEngine.Schema schema; - private DefaultQueryManager queryManager; - /** * Setup the test context. */ @Before public void setUp() { queryManager = DefaultQueryManager.defaultQueryManager(); - context.registerBean(QueryManager.class, () -> queryManager); - context.registerBean(QueryPlanFactory.class, () -> new QueryPlanFactory(queryService)); - context.registerBean(StorageEngine.class, () -> storageEngine); - context.registerBean(ExecutionEngine.class, () -> executionEngine); - context.registerBean(DataSourceService.class, () -> dataSourceService); - context.registerBean(FunctionProperties.class, FunctionProperties::new); - context.register(PPLServiceConfig.class); - context.refresh(); - pplService = context.getBean(PPLService.class); + pplService = new PPLService(new PPLSyntaxParser(), queryManager, + new QueryPlanFactory(queryService)); } @After diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java index dda404f29a..fb849f6de7 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java @@ -46,7 +46,10 @@ public static Iterable functionNames() { {"curtime", true, false}, {"current_time", true, true}, {"curdate", false, false}, - {"current_date", false, true} + {"current_date", false, true}, + {"utc_date", false, false}, + {"utc_time", false, false}, + {"utc_timestamp", false, false} }); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index dbdfb71aa7..aa573449b6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -226,6 +226,30 @@ public void testLiteralValueBinaryOperationExpr() { )); } + @Test + public void testBinaryOperationExprWithParentheses() { + assertEqual("source = t | where a = (1 + 2) * 3", + filter( + relation("t"), + compare("=", + field("a"), + function("*", + function("+", intLiteral(1), intLiteral(2)), + intLiteral(3))))); + } + + @Test + public void testBinaryOperationExprPrecedence() { + assertEqual("source = t | where a = 1 + 2 * 3", + filter( + relation("t"), + compare("=", + field("a"), + function("+", + intLiteral(1), + function("*", intLiteral(2), intLiteral(3)))))); + } + @Test public void testCompareExpr() { assertEqual("source=t a='b'", @@ -705,7 +729,7 @@ public void canBuildMatchRelevanceFunctionWithArguments() { relation("test"), function( "match", - unresolvedArg("field", stringLiteral("message")), + unresolvedArg("field", qualifiedName("message")), unresolvedArg("query", stringLiteral("test query")), unresolvedArg("analyzer", stringLiteral("keyword")) ) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java index 711e780f3b..ddcde513dd 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java @@ -56,7 +56,10 @@ public static Iterable functionNames() { {"curtime", false, false}, {"current_time", false, false}, {"curdate", false, false}, - {"current_date", false, false} + {"current_date", false, false}, + {"utc_date", false, false}, + {"utc_time", false, false}, + {"utc_timestamp", false, false} }); } diff --git a/prometheus/build.gradle b/prometheus/build.gradle index b11c23fd25..7cf1e56085 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -17,9 +17,9 @@ repositories { dependencies { api project(':core') implementation "io.github.resilience4j:resilience4j-retry:1.5.0" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' implementation 'com.github.babbel:okhttp-aws-signer:1.0.2' implementation group: 'org.json', name: 'json', version: '20180813' diff --git a/protocol/build.gradle b/protocol/build.gradle index 9c41fbf101..5d32a235ea 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -30,9 +30,9 @@ plugins { dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" implementation 'com.google.code.gson:gson:2.8.9' implementation project(':core') implementation project(':opensearch') @@ -44,7 +44,7 @@ dependencies { } configurations.all { - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" } test { diff --git a/sql-cli/MAINTAINERS.md b/sql-cli/MAINTAINERS.md index 098ae18733..e32b745ea4 100644 --- a/sql-cli/MAINTAINERS.md +++ b/sql-cli/MAINTAINERS.md @@ -1,13 +1,16 @@ -# OpenSearch Maintainers - -## Maintainers -| Maintainer | GitHub ID | Affiliation | -| --------------- | --------- | ----------- | -| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon | -| Alolita Sharma | [alolita](https://github.com/alolita) | Amazon | -| Anirudha Jadhav | [anirudha](https://github.com/anirudha) | Amazon | -| Chloe Zhang | [chloe-zh](https://github.com/chloe-zh) | Amazon | -| Joshua Li | [joshuali925](https://github.com/joshuali925) | Amazon | -| Zhongnan Su | [zhongnansu](https://github.com/CEHENKLE) | Amazon | +## Overview + +This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). + +## Current Maintainers + +| Maintainer | GitHub ID | Affiliation | +| --------------- | --------------------------------------------- | ----------- | +| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon | +| Alolita Sharma | [alolita](https://github.com/alolita) | Amazon | +| Anirudha Jadhav | [anirudha](https://github.com/anirudha) | Amazon | +| Chloe Zhang | [chloe-zh](https://github.com/chloe-zh) | Amazon | +| Joshua Li | [joshuali925](https://github.com/joshuali925) | Amazon | +| Zhongnan Su | [zhongnansu](https://github.com/CEHENKLE) | Amazon | diff --git a/sql-jdbc/MAINTAINERS.md b/sql-jdbc/MAINTAINERS.md index ff390ab74f..76346cc04c 100644 --- a/sql-jdbc/MAINTAINERS.md +++ b/sql-jdbc/MAINTAINERS.md @@ -1,11 +1,13 @@ -# OpenSearch JDBC Maintainers +## Overview -## Maintainers +This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). -| Maintainer | GitHub ID | Affiliation | -| --------------- | --------- | ----------- | -| Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | -| Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | -| Chloe | [chloe-zh](https://github.com/chloe-zh) | Amazon | -| Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | -| Harold Wang | [harold-wang](https://github.com/harold-wang) | Amazon | +## Current Maintainers + +| Maintainer | GitHub ID | Affiliation | +| --------------------- | --------------------------------------------- | ----------- | +| Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | +| Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | +| Chloe | [chloe-zh](https://github.com/chloe-zh) | Amazon | +| Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | +| Harold Wang | [harold-wang](https://github.com/harold-wang) | Amazon | diff --git a/sql-odbc/MAINTAINERS.md b/sql-odbc/MAINTAINERS.md index 59838756d9..76346cc04c 100644 --- a/sql-odbc/MAINTAINERS.md +++ b/sql-odbc/MAINTAINERS.md @@ -1,11 +1,13 @@ -# OpenSearch ODBC Maintainers +## Overview -## Maintainers +This document contains a list of maintainers in this repo. See [opensearch-project/.github/RESPONSIBILITIES.md](https://github.com/opensearch-project/.github/blob/main/RESPONSIBILITIES.md#maintainer-responsibilities) that explains what the role of maintainer means, what maintainers do in this and other repos, and how they should be doing it. If you're interested in contributing, and becoming a maintainer, see [CONTRIBUTING](CONTRIBUTING.md). -| Maintainer | GitHub ID | Affiliation | -| --------------- | --------- | ----------- | -| Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | -| Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | -| Chloe | [chloe-zh](https://github.com/chloe-zh) | Amazon | -| Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | -| Harold Wang | [harold-wang](https://github.com/harold-wang) | Amazon | +## Current Maintainers + +| Maintainer | GitHub ID | Affiliation | +| --------------------- | --------------------------------------------- | ----------- | +| Anirudha (Ani) Jadhav | [anirudha](https://github.com/anirudha) | Amazon | +| Peng Huo | [penghuo](https://github.com/penghuo) | Amazon | +| Chloe | [chloe-zh](https://github.com/chloe-zh) | Amazon | +| Chen Dai | [dai-chen](https://github.com/dai-chen) | Amazon | +| Harold Wang | [harold-wang](https://github.com/harold-wang) | Amazon | diff --git a/sql/build.gradle b/sql/build.gradle index 13a7ceba53..5c85231b77 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -47,8 +47,6 @@ dependencies { implementation "org.antlr:antlr4-runtime:4.7.1" implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' implementation group: 'org.json', name: 'json', version:'20180813' - implementation group: 'org.springframework', name: 'spring-context', version: "${spring_version}" - implementation group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" implementation project(':common') implementation project(':core') api project(':protocol') diff --git a/sql/src/main/antlr/OpenSearchSQLIdentifierParser.g4 b/sql/src/main/antlr/OpenSearchSQLIdentifierParser.g4 deleted file mode 100644 index cd65e5066c..0000000000 --- a/sql/src/main/antlr/OpenSearchSQLIdentifierParser.g4 +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -/* -MySQL (Positive Technologies) grammar -The MIT License (MIT). -Copyright (c) 2015-2017, Ivan Kochurkin (kvanttt@gmail.com), Positive Technologies. -Copyright (c) 2017, Ivan Khudyashev (IHudyashov@ptsecurity.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -parser grammar OpenSearchSQLIdentifierParser; - -options { tokenVocab=OpenSearchSQLLexer; } - - -// Identifiers - -tableName - : qualifiedName - ; - -columnName - : qualifiedName - ; - -alias - : ident - ; - -qualifiedName - : ident (DOT ident)* - ; - -ident - : DOT? ID - | BACKTICK_QUOTE_ID - | keywordsCanBeId - ; - -keywordsCanBeId - : FULL - | FIELD | D | T | TS // OD SQL and ODBC special - | COUNT | SUM | AVG | MAX | MIN - | TIMESTAMP | DATE | TIME | DAYOFWEEK - | FIRST | LAST - | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME - | CURDATE | CURTIME | NOW - ; diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index aa6972c06b..941c1a4c4e 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -201,6 +201,7 @@ DATE: 'DATE'; DATE_ADD: 'DATE_ADD'; DATE_FORMAT: 'DATE_FORMAT'; DATE_SUB: 'DATE_SUB'; +DATEDIFF: 'DATEDIFF'; DAYNAME: 'DAYNAME'; DAYOFMONTH: 'DAYOFMONTH'; DAYOFWEEK: 'DAYOFWEEK'; @@ -244,6 +245,7 @@ REPLACE: 'REPLACE'; RINT: 'RINT'; ROUND: 'ROUND'; RTRIM: 'RTRIM'; +REVERSE: 'REVERSE'; SIGN: 'SIGN'; SIGNUM: 'SIGNUM'; SIN: 'SIN'; @@ -255,13 +257,14 @@ SUBTRACT: 'SUBTRACT'; SYSDATE: 'SYSDATE'; TAN: 'TAN'; TIME: 'TIME'; +TIMEDIFF: 'TIMEDIFF'; TIME_TO_SEC: 'TIME_TO_SEC'; TIMESTAMP: 'TIMESTAMP'; TRUNCATE: 'TRUNCATE'; TO_DAYS: 'TO_DAYS'; -UTC_DATE: 'UTC_DATE'; UNIX_TIMESTAMP: 'UNIX_TIMESTAMP'; UPPER: 'UPPER'; +UTC_DATE: 'UTC_DATE'; UTC_TIME: 'UTC_TIME'; UTC_TIMESTAMP: 'UTC_TIMESTAMP'; @@ -340,6 +343,7 @@ ANALYZER: 'ANALYZER'; ANALYZE_WILDCARD: 'ANALYZE_WILDCARD'; AUTO_GENERATE_SYNONYMS_PHRASE_QUERY:'AUTO_GENERATE_SYNONYMS_PHRASE_QUERY'; BOOST: 'BOOST'; +CASE_INSENSITIVE: 'CASE_INSENSITIVE'; CUTOFF_FREQUENCY: 'CUTOFF_FREQUENCY'; DEFAULT_FIELD: 'DEFAULT_FIELD'; DEFAULT_OPERATOR: 'DEFAULT_OPERATOR'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 1598e7676c..d493731550 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -30,8 +30,6 @@ THE SOFTWARE. parser grammar OpenSearchSQLParser; -import OpenSearchSQLIdentifierParser; - options { tokenVocab=OpenSearchSQLLexer; } @@ -82,13 +80,8 @@ tableFilter ; showDescribePattern - : oldID=compatibleID | stringLiteral - ; - -compatibleID - : (MODULE | ID)+? + : stringLiteral ; - // Select Statement's Details querySpecification @@ -278,6 +271,7 @@ predicate : expressionAtom #expressionAtomPredicate | left=predicate comparisonOperator right=predicate #binaryComparisonPredicate | predicate IS nullNotnull #isNullPredicate + | predicate NOT? BETWEEN predicate AND predicate #betweenPredicate | left=predicate NOT? LIKE right=predicate #likePredicate | left=predicate REGEXP right=predicate #regexpPredicate | predicate NOT? IN '(' expressions ')' #inPredicate @@ -292,11 +286,12 @@ expressionAtom | columnName #fullColumnNameExpressionAtom | functionCall #functionCallExpressionAtom | LR_BRACKET expression RR_BRACKET #nestedExpressionAtom - | left=expressionAtom mathOperator right=expressionAtom #mathExpressionAtom - ; - -mathOperator - : PLUS | MINUS | STAR | DIVIDE | MODULE + | left=expressionAtom + mathOperator=(STAR | DIVIDE | MODULE) + right=expressionAtom #mathExpressionAtom + | left=expressionAtom + mathOperator=(PLUS | MINUS) + right=expressionAtom #mathExpressionAtom ; comparisonOperator @@ -328,6 +323,10 @@ positionFunction : POSITION LR_BRACKET functionArg IN functionArg RR_BRACKET ; +matchQueryAltSyntaxFunction + : field=relevanceField EQUAL_SYMBOL MATCH_QUERY LR_BRACKET query=relevanceQuery RR_BRACKET + ; + scalarFunctionName : mathematicalFunctionName | dateTimeFunctionName @@ -345,7 +344,8 @@ specificFunction ; relevanceFunction - : noFieldRelevanceFunction | singleFieldRelevanceFunction | multiFieldRelevanceFunction + : noFieldRelevanceFunction | singleFieldRelevanceFunction | multiFieldRelevanceFunction | altSingleFieldRelevanceFunction | altMultiFieldRelevanceFunction + ; noFieldRelevanceFunction @@ -367,6 +367,14 @@ multiFieldRelevanceFunction alternateMultiMatchQuery COMMA alternateMultiMatchField (COMMA relevanceArg)* RR_BRACKET ; +altSingleFieldRelevanceFunction + : field=relevanceField EQUAL_SYMBOL altSyntaxFunctionName=altSingleFieldRelevanceFunctionName LR_BRACKET query=relevanceQuery (COMMA relevanceArg)* RR_BRACKET + ; + +altMultiFieldRelevanceFunction + : field=relevanceField EQUAL_SYMBOL altSyntaxFunctionName=altMultiFieldRelevanceFunctionName LR_BRACKET query=relevanceQuery (COMMA relevanceArg)* RR_BRACKET + ; + convertedDataType : typeName=DATE | typeName=TIME @@ -421,6 +429,7 @@ dateTimeFunctionName | DATE_ADD | DATE_FORMAT | DATE_SUB + | DATEDIFF | DATETIME | DAY | DAYNAME @@ -434,6 +443,7 @@ dateTimeFunctionName | MAKETIME | MICROSECOND | MINUTE + | MINUTE_OF_DAY | MONTH | MONTHNAME | NOW @@ -446,6 +456,7 @@ dateTimeFunctionName | SYSDATE | TIME | TIME_TO_SEC + | TIMEDIFF | TIMESTAMP | TO_DAYS | UNIX_TIMESTAMP @@ -456,7 +467,7 @@ dateTimeFunctionName textFunctionName : SUBSTR | SUBSTRING | TRIM | LTRIM | RTRIM | LOWER | UPPER | CONCAT | CONCAT_WS | SUBSTR | LENGTH | STRCMP | RIGHT | LEFT - | ASCII | LOCATE | REPLACE + | ASCII | LOCATE | REPLACE | REVERSE ; flowControlFunctionName @@ -475,6 +486,7 @@ singleFieldRelevanceFunctionName : MATCH | MATCHQUERY | MATCH_QUERY | MATCH_PHRASE | MATCHPHRASE | MATCHPHRASEQUERY | MATCH_BOOL_PREFIX | MATCH_PHRASE_PREFIX + | WILDCARD_QUERY | WILDCARDQUERY ; multiFieldRelevanceFunctionName @@ -485,6 +497,18 @@ multiFieldRelevanceFunctionName | QUERY_STRING ; +altSingleFieldRelevanceFunctionName + : MATCH_QUERY + | MATCHQUERY + | MATCH_PHRASE + | MATCHPHRASE + ; + +altMultiFieldRelevanceFunctionName + : MULTI_MATCH + | MULTIMATCH + ; + functionArgs : (functionArg (COMMA functionArg)*)? ; @@ -504,7 +528,7 @@ highlightArg relevanceArgName : ALLOW_LEADING_WILDCARD | ANALYZER | ANALYZE_WILDCARD | AUTO_GENERATE_SYNONYMS_PHRASE_QUERY - | BOOST | CUTOFF_FREQUENCY | DEFAULT_FIELD | DEFAULT_OPERATOR | ENABLE_POSITION_INCREMENTS + | BOOST | CASE_INSENSITIVE | CUTOFF_FREQUENCY | DEFAULT_FIELD | DEFAULT_OPERATOR | ENABLE_POSITION_INCREMENTS | ESCAPE | FIELDS | FLAGS | FUZZINESS | FUZZY_MAX_EXPANSIONS | FUZZY_PREFIX_LENGTH | FUZZY_REWRITE | FUZZY_TRANSPOSITIONS | LENIENT | LOW_FREQ_OPERATOR | MAX_DETERMINIZED_STATES | MAX_EXPANSIONS | MINIMUM_SHOULD_MATCH | OPERATOR | PHRASE_SLOP | PREFIX_LENGTH @@ -560,3 +584,37 @@ alternateMultiMatchField | argName=alternateMultiMatchArgName EQUAL_SYMBOL LT_SQR_PRTHS argVal=relevanceArgValue RT_SQR_PRTHS ; + + +// Identifiers + +tableName + : qualifiedName + ; + +columnName + : qualifiedName + ; + +alias + : ident + ; + +qualifiedName + : ident (DOT ident)* + ; + +ident + : DOT? ID + | BACKTICK_QUOTE_ID + | keywordsCanBeId + | scalarFunctionName + ; + +keywordsCanBeId + : FULL + | FIELD | D | T | TS // OD SQL and ODBC special + | COUNT | SUM | AVG | MAX | MIN + | FIRST | LAST + | TYPE // TODO: Type is keyword required by relevancy function. Remove this when relevancy functions moved out + ; diff --git a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java b/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java deleted file mode 100644 index 4287883c34..0000000000 --- a/sql/src/main/java/org/opensearch/sql/sql/config/SQLServiceConfig.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.sql.config; - -import org.opensearch.sql.executor.QueryManager; -import org.opensearch.sql.executor.execution.QueryPlanFactory; -import org.opensearch.sql.sql.SQLService; -import org.opensearch.sql.sql.antlr.SQLSyntaxParser; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.config.ConfigurableBeanFactory; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Scope; - -/** - * SQL service configuration for Spring container initialization. - */ -@Configuration -public class SQLServiceConfig { - - @Autowired - private QueryManager queryManager; - - @Autowired - private QueryPlanFactory queryExecutionFactory; - - /** - * The registration of OpenSearch storage engine happens here because - * OpenSearchStorageEngine is dependent on NodeClient. - * - * @return SQLService. - */ - @Bean - @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) - public SQLService sqlService() { - return new SQLService( - new SQLSyntaxParser(), - queryManager, - queryExecutionFactory); - } - -} - diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java index 6edce6eb15..02a7f7d1b9 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstBuilder.java @@ -170,7 +170,8 @@ public UnresolvedPlan visitTableAsRelation(TableAsRelationContext ctx) { @Override public UnresolvedPlan visitSubqueryAsRelation(SubqueryAsRelationContext ctx) { - return new RelationSubquery(visit(ctx.subquery), ctx.alias().getText()); + String subqueryAlias = StringUtils.unquoteIdentifier(ctx.alias().getText()); + return new RelationSubquery(visit(ctx.subquery), subqueryAlias); } @Override diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index bae22595ca..1c01f8aaf7 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -6,6 +6,8 @@ package org.opensearch.sql.sql.parser; +import static org.opensearch.sql.ast.dsl.AstDSL.between; +import static org.opensearch.sql.ast.dsl.AstDSL.not; import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; @@ -14,6 +16,9 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT_LIKE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.POSITION; import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEXP; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchFieldContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AlternateMultiMatchQueryContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BetweenPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BinaryComparisonPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.BooleanContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CaseFuncAlternativeContext; @@ -24,16 +29,25 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DataTypeFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DateLiteralContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DistinctCountFunctionCallContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FilterClauseContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FilteredAggregationFunctionCallContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.FunctionArgContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.HighlightFunctionCallContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.InPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IsNullPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.LikePredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.MathExpressionAtomContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.MultiFieldRelevanceFunctionContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.NoFieldRelevanceFunctionContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.NotExpressionContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.NullLiteralContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.OverClauseContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.PositionFunctionContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.QualifiedNameContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.RegexpPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.RegularAggregateFunctionCallContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.RelevanceArgContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.RelevanceFieldAndWeightContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ScalarFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ScalarWindowFunctionContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ShowDescribePatternContext; @@ -82,7 +96,6 @@ import org.opensearch.sql.ast.tree.Sort.SortOption; import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.AndExpressionContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnNameContext; import org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IdentContext; @@ -120,7 +133,7 @@ public UnresolvedExpression visitQualifiedName(QualifiedNameContext ctx) { @Override public UnresolvedExpression visitMathExpressionAtom(MathExpressionAtomContext ctx) { return new Function( - ctx.mathOperator().getText(), + ctx.mathOperator.getText(), Arrays.asList(visit(ctx.left), visit(ctx.right)) ); } @@ -137,7 +150,7 @@ public UnresolvedExpression visitScalarFunctionCall(ScalarFunctionCallContext ct @Override public UnresolvedExpression visitHighlightFunctionCall( - OpenSearchSQLParser.HighlightFunctionCallContext ctx) { + HighlightFunctionCallContext ctx) { ImmutableMap.Builder builder = ImmutableMap.builder(); ctx.highlightFunction().highlightArg().forEach(v -> builder.put( v.highlightArgName().getText().toLowerCase(), @@ -151,7 +164,7 @@ public UnresolvedExpression visitHighlightFunctionCall( @Override public UnresolvedExpression visitPositionFunction( - OpenSearchSQLParser.PositionFunctionContext ctx) { + PositionFunctionContext ctx) { return new Function( POSITION.getName().getFunctionName(), Arrays.asList(visitFunctionArg(ctx.functionArg(0)), @@ -175,16 +188,12 @@ public UnresolvedExpression visitColumnFilter(ColumnFilterContext ctx) { @Override public UnresolvedExpression visitShowDescribePattern( ShowDescribePatternContext ctx) { - if (ctx.compatibleID() != null) { - return stringLiteral(ctx.compatibleID().getText()); - } else { - return visit(ctx.stringLiteral()); - } + return visit(ctx.stringLiteral()); } @Override public UnresolvedExpression visitFilteredAggregationFunctionCall( - OpenSearchSQLParser.FilteredAggregationFunctionCallContext ctx) { + FilteredAggregationFunctionCallContext ctx) { AggregateFunction agg = (AggregateFunction) visit(ctx.aggregateFunction()); return agg.condition(visit(ctx.filterClause())); } @@ -241,7 +250,7 @@ public UnresolvedExpression visitCountStarFunctionCall(CountStarFunctionCallCont } @Override - public UnresolvedExpression visitFilterClause(OpenSearchSQLParser.FilterClauseContext ctx) { + public UnresolvedExpression visitFilterClause(FilterClauseContext ctx) { return visit(ctx.expression()); } @@ -253,6 +262,20 @@ public UnresolvedExpression visitIsNullPredicate(IsNullPredicateContext ctx) { Arrays.asList(visit(ctx.predicate()))); } + @Override + public UnresolvedExpression visitBetweenPredicate(BetweenPredicateContext ctx) { + UnresolvedExpression func = + between( + visit(ctx.predicate(0)), + visit(ctx.predicate(1)), + visit(ctx.predicate(2))); + + if (ctx.NOT() != null) { + func = not(func); + } + return func; + } + @Override public UnresolvedExpression visitLikePredicate(LikePredicateContext ctx) { return new Function( @@ -268,7 +291,7 @@ public UnresolvedExpression visitRegexpPredicate(RegexpPredicateContext ctx) { } @Override - public UnresolvedExpression visitInPredicate(OpenSearchSQLParser.InPredicateContext ctx) { + public UnresolvedExpression visitInPredicate(InPredicateContext ctx) { UnresolvedExpression field = visit(ctx.predicate()); List inLists = ctx .expressions() @@ -392,7 +415,7 @@ public UnresolvedExpression visitConvertedDataType( @Override public UnresolvedExpression visitNoFieldRelevanceFunction( - OpenSearchSQLParser.NoFieldRelevanceFunctionContext ctx) { + NoFieldRelevanceFunctionContext ctx) { return new Function( ctx.noFieldRelevanceFunctionName().getText().toLowerCase(), noFieldRelevanceArguments(ctx)); @@ -406,6 +429,14 @@ public UnresolvedExpression visitSingleFieldRelevanceFunction( singleFieldRelevanceArguments(ctx)); } + @Override + public UnresolvedExpression visitAltSingleFieldRelevanceFunction( + OpenSearchSQLParser.AltSingleFieldRelevanceFunctionContext ctx) { + return new Function( + ctx.altSyntaxFunctionName.getText().toLowerCase(), + altSingleFieldRelevanceFunctionArguments(ctx)); + } + @Override public UnresolvedExpression visitMultiFieldRelevanceFunction( MultiFieldRelevanceFunctionContext ctx) { @@ -415,7 +446,7 @@ public UnresolvedExpression visitMultiFieldRelevanceFunction( if ((funcName.equalsIgnoreCase(BuiltinFunctionName.MULTI_MATCH.toString()) || funcName.equalsIgnoreCase(BuiltinFunctionName.MULTIMATCH.toString()) || funcName.equalsIgnoreCase(BuiltinFunctionName.MULTIMATCHQUERY.toString())) - && ! ctx.getRuleContexts(OpenSearchSQLParser.AlternateMultiMatchQueryContext.class) + && ! ctx.getRuleContexts(AlternateMultiMatchQueryContext.class) .isEmpty()) { return new Function( ctx.multiFieldRelevanceFunctionName().getText().toLowerCase(), @@ -427,8 +458,16 @@ public UnresolvedExpression visitMultiFieldRelevanceFunction( } } + @Override + public UnresolvedExpression visitAltMultiFieldRelevanceFunction( + OpenSearchSQLParser.AltMultiFieldRelevanceFunctionContext ctx) { + return new Function( + ctx.altSyntaxFunctionName.getText().toLowerCase(), + altMultiFieldRelevanceFunctionArguments(ctx)); + } + private Function buildFunction(String functionName, - List arg) { + List arg) { return new Function( functionName, arg @@ -447,7 +486,7 @@ private QualifiedName visitIdentifiers(List identifiers) { ); } - private void fillRelevanceArgs(List args, + private void fillRelevanceArgs(List args, ImmutableList.Builder builder) { // To support old syntax we must support argument keys as quoted strings. args.forEach(v -> builder.add(v.argName == null @@ -459,7 +498,7 @@ private void fillRelevanceArgs(List arg } private List noFieldRelevanceArguments( - OpenSearchSQLParser.NoFieldRelevanceFunctionContext ctx) { + NoFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values // to skip environment resolving and function signature resolving ImmutableList.Builder builder = ImmutableList.builder(); @@ -470,12 +509,12 @@ private List noFieldRelevanceArguments( } private List singleFieldRelevanceArguments( - OpenSearchSQLParser.SingleFieldRelevanceFunctionContext ctx) { + SingleFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values // to skip environment resolving and function signature resolving ImmutableList.Builder builder = ImmutableList.builder(); builder.add(new UnresolvedArgument("field", - new Literal(StringUtils.unquoteText(ctx.field.getText()), DataType.STRING))); + new QualifiedName(StringUtils.unquoteText(ctx.field.getText())))); builder.add(new UnresolvedArgument("query", new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); fillRelevanceArgs(ctx.relevanceArg(), builder); @@ -483,14 +522,26 @@ private List singleFieldRelevanceArguments( } + private List altSingleFieldRelevanceFunctionArguments( + OpenSearchSQLParser.AltSingleFieldRelevanceFunctionContext ctx) { + // all the arguments are defaulted to string values + // to skip environment resolving and function signature resolving + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(new UnresolvedArgument("field", + new Literal(StringUtils.unquoteText(ctx.field.getText()), DataType.STRING))); + builder.add(new UnresolvedArgument("query", + new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); + fillRelevanceArgs(ctx.relevanceArg(), builder); + return builder.build(); + } private List multiFieldRelevanceArguments( - OpenSearchSQLParser.MultiFieldRelevanceFunctionContext ctx) { + MultiFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values // to skip environment resolving and function signature resolving ImmutableList.Builder builder = ImmutableList.builder(); var fields = new RelevanceFieldList(ctx - .getRuleContexts(OpenSearchSQLParser.RelevanceFieldAndWeightContext.class) + .getRuleContexts(RelevanceFieldAndWeightContext.class) .stream() .collect(Collectors.toMap( f -> StringUtils.unquoteText(f.field.getText()), @@ -509,14 +560,14 @@ private List multiFieldRelevanceArguments( * @return : Returns list of all arguments for relevance function. */ private List alternateMultiMatchArguments( - OpenSearchSQLParser.MultiFieldRelevanceFunctionContext ctx) { + MultiFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values // to skip environment resolving and function signature resolving ImmutableList.Builder builder = ImmutableList.builder(); Map fieldAndWeightMap = new HashMap<>(); String[] fieldAndWeights = StringUtils.unquoteText( - ctx.getRuleContexts(OpenSearchSQLParser.AlternateMultiMatchFieldContext.class) + ctx.getRuleContexts(AlternateMultiMatchFieldContext.class) .stream().findFirst().get().argVal.getText()).split(","); for (var fieldAndWeight : fieldAndWeights) { @@ -527,7 +578,7 @@ private List alternateMultiMatchArguments( builder.add(new UnresolvedArgument("fields", new RelevanceFieldList(fieldAndWeightMap))); - ctx.getRuleContexts(OpenSearchSQLParser.AlternateMultiMatchQueryContext.class) + ctx.getRuleContexts(AlternateMultiMatchQueryContext.class) .stream().findFirst().ifPresent( arg -> builder.add(new UnresolvedArgument("query", @@ -538,4 +589,19 @@ private List alternateMultiMatchArguments( return builder.build(); } + + private List altMultiFieldRelevanceFunctionArguments( + OpenSearchSQLParser.AltMultiFieldRelevanceFunctionContext ctx) { + // all the arguments are defaulted to string values + // to skip environment resolving and function signature resolving + var map = new HashMap(); + map.put(ctx.field.getText(), 1F); + ImmutableList.Builder builder = ImmutableList.builder(); + var fields = new RelevanceFieldList(map); + builder.add(new UnresolvedArgument("fields", fields)); + builder.add(new UnresolvedArgument("query", + new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); + fillRelevanceArgs(ctx.relevanceArg(), builder); + return builder.build(); + } } diff --git a/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java b/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java index 9abe37cd06..a351c30609 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/SQLServiceTest.java @@ -26,12 +26,10 @@ import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponse; import org.opensearch.sql.executor.ExecutionEngine.ExplainResponseNode; -import org.opensearch.sql.executor.QueryManager; import org.opensearch.sql.executor.QueryService; import org.opensearch.sql.executor.execution.QueryPlanFactory; -import org.opensearch.sql.sql.config.SQLServiceConfig; +import org.opensearch.sql.sql.antlr.SQLSyntaxParser; import org.opensearch.sql.sql.domain.SQLQueryRequest; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; @ExtendWith(MockitoExtension.class) class SQLServiceTest { @@ -40,8 +38,6 @@ class SQLServiceTest { private static String EXPLAIN = "/_plugins/_sql/_explain"; - private AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - private SQLService sqlService; private DefaultQueryManager queryManager; @@ -55,11 +51,8 @@ class SQLServiceTest { @BeforeEach public void setUp() { queryManager = DefaultQueryManager.defaultQueryManager(); - context.registerBean(QueryManager.class, () -> queryManager); - context.registerBean(QueryPlanFactory.class, () -> new QueryPlanFactory(queryService)); - context.register(SQLServiceConfig.class); - context.refresh(); - sqlService = context.getBean(SQLService.class); + sqlService = new SQLService(new SQLSyntaxParser(), queryManager, + new QueryPlanFactory(queryService)); } @AfterEach diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index bf2c9af623..82e1b0b848 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql.antlr; +import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -15,7 +16,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Random; import java.util.stream.Stream; @@ -168,7 +168,10 @@ private static Stream nowLikeFunctionsData() { Arguments.of("curtime", true, false), Arguments.of("current_time", true, true), Arguments.of("curdate", false, false), - Arguments.of("current_date", false, true) + Arguments.of("current_date", false, true), + Arguments.of("utc_date", false, true), + Arguments.of("utc_time", false, true), + Arguments.of("utc_timestamp", false, true) ); } @@ -449,6 +452,61 @@ public void can_parse_match_phrase_relevance_function() { assertNotNull(parser.parse("SELECT * FROM test WHERE match_phrase(column, 100500)")); } + @Test + public void can_parse_minute_of_day_function() { + assertNotNull(parser.parse("SELECT minute_of_day(\"12:23:34\");")); + assertNotNull(parser.parse("SELECT minute_of_day('12:23:34');"));; + assertNotNull(parser.parse("SELECT minute_of_day(\"2022-12-14 12:23:34\");"));; + assertNotNull(parser.parse("SELECT minute_of_day('2022-12-14 12:23:34');"));; + } + + @Test + public void can_parse_wildcard_query_relevance_function() { + assertNotNull( + parser.parse("SELECT * FROM test WHERE wildcard_query(column, \"this is a test*\")")); + assertNotNull( + parser.parse("SELECT * FROM test WHERE wildcard_query(column, 'this is a test*')")); + assertNotNull( + parser.parse("SELECT * FROM test WHERE wildcard_query(`column`, \"this is a test*\")")); + assertNotNull( + parser.parse("SELECT * FROM test WHERE wildcard_query(`column`, 'this is a test*')")); + assertNotNull( + parser.parse("SELECT * FROM test WHERE wildcard_query(`column`, 'this is a test*', " + + "boost=1.5, case_insensitive=true, rewrite=\"scoring_boolean\")")); + } + + @Test + public void describe_request_accepts_only_quoted_string_literals() { + assertAll( + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("DESCRIBE TABLES LIKE bank")), + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("DESCRIBE TABLES LIKE %bank%")), + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("DESCRIBE TABLES LIKE `bank`")), + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("DESCRIBE TABLES LIKE %bank% COLUMNS LIKE %status%")), + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("DESCRIBE TABLES LIKE 'bank' COLUMNS LIKE status")), + () -> assertNotNull(parser.parse("DESCRIBE TABLES LIKE 'bank' COLUMNS LIKE \"status\"")), + () -> assertNotNull(parser.parse("DESCRIBE TABLES LIKE \"bank\" COLUMNS LIKE 'status'")) + ); + } + + @Test + public void show_request_accepts_only_quoted_string_literals() { + assertAll( + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("SHOW TABLES LIKE bank")), + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("SHOW TABLES LIKE %bank%")), + () -> assertThrows(SyntaxCheckException.class, + () -> parser.parse("SHOW TABLES LIKE `bank`")), + () -> assertNotNull(parser.parse("SHOW TABLES LIKE 'bank'")), + () -> assertNotNull(parser.parse("SHOW TABLES LIKE \"bank\"")) + ); + } + @ParameterizedTest @MethodSource({ "matchPhraseComplexQueries", @@ -484,6 +542,30 @@ private static Stream matchPhraseComplexQueries() { ); } + @Test + public void canParseMatchQueryAlternateSyntax() { + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchquery('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchquery(\"query\")")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_query('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_query(\"query\")")); + } + + @Test + public void canParseMatchPhraseAlternateSyntax() { + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_phrase('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = match_phrase(\"query\")")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchphrase('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = matchphrase(\"query\")")); + } + + @Test + public void canParseMultiMatchAlternateSyntax() { + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multi_match('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multi_match(\"query\")")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multimatch('query')")); + assertNotNull(parser.parse("SELECT * FROM test WHERE Field = multimatch(\"query\")")); + } + private static Stream matchPhraseQueryComplexQueries() { return Stream.of( "SELECT * FROM t WHERE matchphrasequery(c, 3)", diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 2aed4f2834..64a7445dc8 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -518,6 +518,27 @@ public void can_build_from_subquery() { ); } + @Test + public void can_build_from_subquery_with_backquoted_alias() { + assertEquals( + project( + relationSubquery( + project( + relation("test"), + alias("firstname", qualifiedName("firstname"), "firstName")), + "a"), + alias("a.firstName", qualifiedName("a", "firstName")) + ), + buildAST( + "SELECT a.firstName " + + "FROM ( " + + " SELECT `firstname` AS `firstName` " + + " FROM `test` " + + ") AS `a`" + ) + ); + } + @Test public void can_build_show_all_tables() { assertEquals( @@ -546,10 +567,6 @@ public void can_build_show_selected_tables() { ); } - /** - * Todo, ideally the identifier (%) couldn't be used in LIKE operator, only the string literal - * is allowed. - */ @Test public void show_compatible_with_old_engine_syntax() { assertEquals( @@ -560,18 +577,7 @@ public void show_compatible_with_old_engine_syntax() { ), AllFields.of() ), - buildAST("SHOW TABLES LIKE %") - ); - } - - @Test - public void describe_compatible_with_old_engine_syntax() { - assertEquals( - project( - relation(mappingTable("a_c%")), - AllFields.of() - ), - buildAST("DESCRIBE TABLES LIKE a_c%") + buildAST("SHOW TABLES LIKE '%'") ); } @@ -600,24 +606,6 @@ public void can_build_describe_selected_tables_field_filter() { ); } - /** - * Todo, ideally the identifier (%) couldn't be used in LIKE operator, only the string literal - * is allowed. - */ - @Test - public void describe_and_column_compatible_with_old_engine_syntax() { - assertEquals( - project( - filter( - relation(mappingTable("a_c%")), - function("like", qualifiedName("COLUMN_NAME"), stringLiteral("name%")) - ), - AllFields.of() - ), - buildAST("DESCRIBE TABLES LIKE a_c% COLUMNS LIKE name%") - ); - } - @Test public void can_build_alias_by_keywords() { assertEquals( diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index ac68d146b2..0bc44cdffd 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.and; +import static org.opensearch.sql.ast.dsl.AstDSL.between; import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.caseWhen; import static org.opensearch.sql.ast.dsl.AstDSL.dateLiteral; @@ -153,6 +154,17 @@ public void canBuildArithmeticExpression() { ); } + @Test + public void canBuildArithmeticExpressionPrecedence() { + assertEquals( + function("+", + intLiteral(1), + function("*", + intLiteral(2), intLiteral(3))), + buildExprAst("1 + 2 * 3") + ); + } + @Test public void canBuildFunctionWithoutArguments() { assertEquals( @@ -261,6 +273,25 @@ public void canBuildRegexpExpression() { ); } + @Test + public void canBuildBetweenExpression() { + assertEquals( + between( + qualifiedName("age"), intLiteral(10), intLiteral(30)), + buildExprAst("age BETWEEN 10 AND 30") + ); + } + + @Test + public void canBuildNotBetweenExpression() { + assertEquals( + not( + between( + qualifiedName("age"), intLiteral(10), intLiteral(30))), + buildExprAst("age NOT BETWEEN 10 AND 30") + ); + } + @Test public void canBuildLogicalExpression() { assertEquals( @@ -463,7 +494,7 @@ public void filteredDistinctCount() { public void matchPhraseQueryAllParameters() { assertEquals( AstDSL.function("matchphrasequery", - unresolvedArg("field", stringLiteral("test")), + unresolvedArg("field", qualifiedName("test")), unresolvedArg("query", stringLiteral("search query")), unresolvedArg("slop", stringLiteral("3")), unresolvedArg("analyzer", stringLiteral("standard")), @@ -479,7 +510,7 @@ public void matchPhraseQueryAllParameters() { public void matchPhrasePrefixAllParameters() { assertEquals( AstDSL.function("match_phrase_prefix", - unresolvedArg("field", stringLiteral("test")), + unresolvedArg("field", qualifiedName("test")), unresolvedArg("query", stringLiteral("search query")), unresolvedArg("slop", stringLiteral("3")), unresolvedArg("boost", stringLiteral("1.5")), @@ -496,13 +527,13 @@ public void matchPhrasePrefixAllParameters() { @Test public void relevanceMatch() { assertEquals(AstDSL.function("match", - unresolvedArg("field", stringLiteral("message")), + unresolvedArg("field", qualifiedName("message")), unresolvedArg("query", stringLiteral("search query"))), buildExprAst("match('message', 'search query')") ); assertEquals(AstDSL.function("match", - unresolvedArg("field", stringLiteral("message")), + unresolvedArg("field", qualifiedName("message")), unresolvedArg("query", stringLiteral("search query")), unresolvedArg("analyzer", stringLiteral("keyword")), unresolvedArg("operator", stringLiteral("AND"))), @@ -512,13 +543,13 @@ public void relevanceMatch() { @Test public void relevanceMatchQuery() { assertEquals(AstDSL.function("matchquery", - unresolvedArg("field", stringLiteral("message")), + unresolvedArg("field", qualifiedName("message")), unresolvedArg("query", stringLiteral("search query"))), buildExprAst("matchquery('message', 'search query')") ); assertEquals(AstDSL.function("matchquery", - unresolvedArg("field", stringLiteral("message")), + unresolvedArg("field", qualifiedName("message")), unresolvedArg("query", stringLiteral("search query")), unresolvedArg("analyzer", stringLiteral("keyword")), unresolvedArg("operator", stringLiteral("AND"))), @@ -528,19 +559,100 @@ public void relevanceMatchQuery() { @Test public void relevanceMatch_Query() { assertEquals(AstDSL.function("match_query", - unresolvedArg("field", stringLiteral("message")), + unresolvedArg("field", qualifiedName("message")), unresolvedArg("query", stringLiteral("search query"))), buildExprAst("match_query('message', 'search query')") ); assertEquals(AstDSL.function("match_query", - unresolvedArg("field", stringLiteral("message")), + unresolvedArg("field", qualifiedName("message")), unresolvedArg("query", stringLiteral("search query")), unresolvedArg("analyzer", stringLiteral("keyword")), unresolvedArg("operator", stringLiteral("AND"))), buildExprAst("match_query('message', 'search query', analyzer='keyword', operator='AND')")); } + @Test + public void relevanceMatchQueryAltSyntax() { + assertEquals(AstDSL.function("match_query", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_query('search query')") + ); + + assertEquals(AstDSL.function("match_query", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_query(\"search query\")") + ); + + assertEquals(AstDSL.function("matchquery", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchquery('search query')") + ); + + assertEquals(AstDSL.function("matchquery", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchquery(\"search query\")") + ); + } + + @Test + public void relevanceMatchPhraseAltSyntax() { + assertEquals(AstDSL.function("match_phrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_phrase('search query')") + ); + + assertEquals(AstDSL.function("match_phrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = match_phrase(\"search query\")") + ); + + assertEquals(AstDSL.function("matchphrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchphrase('search query')") + ); + + assertEquals(AstDSL.function("matchphrase", + unresolvedArg("field", stringLiteral("message")), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("message = matchphrase(\"search query\")") + ); + } + + @Test + public void relevanceMultiMatchAltSyntax() { + assertEquals(AstDSL.function("multi_match", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multi_match('search query')") + ); + + assertEquals(AstDSL.function("multi_match", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multi_match(\"search query\")") + ); + + assertEquals(AstDSL.function("multimatch", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multimatch('search query')") + ); + + assertEquals(AstDSL.function("multimatch", + unresolvedArg("fields", new RelevanceFieldList(ImmutableMap.of("field1", 1.F))), + unresolvedArg("query", stringLiteral("search query"))), + buildExprAst("field1 = multimatch(\"search query\")") + ); + } + @Test public void relevanceMulti_match() { assertEquals(AstDSL.function("multi_match", @@ -637,6 +749,19 @@ public void relevanceQuery_string() { + "analyzer='keyword', time_zone='Canada/Pacific', tie_breaker='1.3')")); } + @Test + public void relevanceWildcard_query() { + assertEquals(AstDSL.function("wildcard_query", + unresolvedArg("field", qualifiedName("field")), + unresolvedArg("query", stringLiteral("search query*")), + unresolvedArg("boost", stringLiteral("1.5")), + unresolvedArg("case_insensitive", stringLiteral("true")), + unresolvedArg("rewrite", stringLiteral("scoring_boolean"))), + buildExprAst("wildcard_query(field, 'search query*', boost=1.5," + + "case_insensitive=true, rewrite='scoring_boolean'))") + ); + } + @Test public void relevanceQuery() { assertEquals(AstDSL.function("query", diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstNowLikeFunctionTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstNowLikeFunctionTest.java index 19b48ca0bd..4ce2a2d3f7 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstNowLikeFunctionTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstNowLikeFunctionTest.java @@ -32,8 +32,11 @@ private static Stream allFunctions() { "curtime", "localtimestamp", "localtime", - "now", - "sysdate") + "now", + "sysdate", + "utc_date", + "utc_time", + "utc_timestamp") .map(Arguments::of); } diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java index 92b535144f..28665dd7ef 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java @@ -51,6 +51,10 @@ public void canBuildQualifiedIdentifier() { buildFromQualifiers("account.location.city").expectQualifiedName("account", "location", "city"); } + @Test + public void commonKeywordCanBeUsedAsIdentifier() { + buildFromIdentifier("type").expectQualifiedName("type"); + } @Test public void functionNameCanBeUsedAsIdentifier() { diff --git a/workbench/opensearch_dashboards.json b/workbench/opensearch_dashboards.json index c8203b8db1..972bedd4ef 100644 --- a/workbench/opensearch_dashboards.json +++ b/workbench/opensearch_dashboards.json @@ -1,7 +1,7 @@ { "id": "queryWorkbenchDashboards", - "version": "2.4.0.0", - "opensearchDashboardsVersion": "2.4.0", + "version": "3.0.0.0", + "opensearchDashboardsVersion": "3.0.0", "server": true, "ui": true, "requiredPlugins": ["navigation"], diff --git a/workbench/package.json b/workbench/package.json index 8a2f5c6c71..6f2acd0b29 100644 --- a/workbench/package.json +++ b/workbench/package.json @@ -1,6 +1,6 @@ { "name": "opensearch-query-workbench", - "version": "2.4.0.0", + "version": "3.0.0.0", "description": "Query Workbench", "main": "index.js", "license": "Apache-2.0", diff --git a/workbench/yarn.lock b/workbench/yarn.lock index 58182914c4..ce780beeef 100644 --- a/workbench/yarn.lock +++ b/workbench/yarn.lock @@ -1678,9 +1678,9 @@ json-stringify-safe@~5.0.1: integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== json5@2.x: - version "2.2.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.1.tgz#655d50ed1e6f95ad1a3caababd2b0efda10b395c" - integrity sha512-1hqLFMSrGHRHxav9q9gNjJ5EXznIxGVO09xQRrwplcS8qs28pZ8s8hupZAmqDwZUmVZ2Qb2jnyPOWcDH8m8dlA== + version "2.2.3" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== jsonfile@^6.0.1: version "6.1.0"