From 3c6b37ae15e293abcd54660631aa69aa43523c31 Mon Sep 17 00:00:00 2001 From: Chen Dai Date: Wed, 9 Nov 2022 13:23:19 -0800 Subject: [PATCH] Merge 2.x to Maximus feature branch (#1062) * Fix `FLOAT` -> `DOUBLE` cast. (#1025) Signed-off-by: Yury-Fridlyand * Fix error messaging from prometheus. (#1029) (#1037) Signed-off-by: vamsi-amazon (cherry picked from commit 4a9cef32ac6aa565d1e92faf72a472e727b205e6) Co-authored-by: vamsi-amazon * Add `query` function as alternate syntax to `query_string` function (#1010) This maintains backwards compatibility with the v1 engine. * Update DATE and TIME functions to parse string input as datetime (#991) Add option to accept datetime like string in both TIME and DATE (eg. accept "1999-01-02 12:12:12" for both TIME and DATE. Strict check on date for testing for valid dates (eg. Don't accept Feb 30th as a valid date) and throws a SemanticCheckException. Co-authored-by: Yury-Fridlyand Signed-off-by: MitchellGale-BitQuill * back quote fix (#1041) (#1050) Signed-off-by: vamsi-amazon (cherry picked from commit d3bb902f66b1232dd8565890452c58bd9ca4d529) Co-authored-by: vamsi-amazon * Catalog to Datasource changes (#1027) (#1049) Signed-off-by: vamsi-amazon (cherry picked from commit 3e30379903ba596b8565e680af69073a16cdf940) * Bump jackson to 2.14.0 (#1058) Signed-off-by: Joshua Li (cherry picked from commit 5a1adb236f5589dfd10aa17783f13684b70a3c51) * Add valueOf() to Expression (#1055) Signed-off-by: Joshua Li Signed-off-by: Yury-Fridlyand Signed-off-by: MitchellGale-BitQuill Signed-off-by: Joshua Li Signed-off-by: Chen Dai Co-authored-by: Yury-Fridlyand Co-authored-by: opensearch-trigger-bot[bot] <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Co-authored-by: vamsi-amazon Co-authored-by: GabeFernandez310 <64448015+GabeFernandez310@users.noreply.github.com> Co-authored-by: MitchellGale-BitQuill <104795536+MitchellGale-BitQuill@users.noreply.github.com> Co-authored-by: Joshua Li --- build.gradle | 4 +- .../sql/analysis/ExpressionAnalyzer.java | 2 +- .../sql/data/model/ExprDateValue.java | 4 +- .../sql/data/model/ExprTimeValue.java | 16 +- .../org/opensearch/sql/expression/DSL.java | 6 +- .../opensearch/sql/expression/Expression.java | 7 + .../aggregation/TakeAggregator.java | 2 +- .../expression/datetime/DateTimeFunction.java | 1 + .../function/OpenSearchFunctions.java | 6 + .../operator/convert/TypeCastOperator.java | 2 +- .../sql/expression/parse/GrokExpression.java | 2 +- .../sql/expression/parse/ParseExpression.java | 2 +- .../expression/parse/PatternsExpression.java | 2 +- .../sql/expression/parse/RegexExpression.java | 2 +- .../sql/planner/physical/ValuesOperator.java | 2 +- .../physical/catalog/CatalogTable.java | 2 +- .../physical/catalog/CatalogTableScan.java | 2 +- .../physical/catalog/CatalogTableSchema.java | 2 +- .../planner/physical/collector/Rounding.java | 2 +- .../sql/utils/DateTimeFormatters.java | 12 +- .../sql/analysis/ExpressionAnalyzerTest.java | 19 +- .../sql/expression/NamedExpressionTest.java | 2 +- .../datetime/DateTimeFunctionTest.java | 37 ++++ .../expression/datetime/DateTimeTestBase.java | 26 +-- .../expression/datetime/FromUnixTimeTest.java | 36 ++-- .../datetime/NowLikeFunctionTest.java | 8 +- .../datetime/PeriodFunctionsTest.java | 6 +- .../datetime/UnixTimeStampTest.java | 2 +- .../function/OpenSearchFunctionsTest.java | 7 + .../arthmetic/ArithmeticFunctionTest.java | 10 +- .../convert/TypeCastOperatorTest.java | 98 ++++----- .../system/SystemFunctionsTest.java | 2 +- .../catalog/CatalogTableScanTest.java | 2 +- .../physical/catalog/CatalogTableTest.java | 2 +- docs/category.json | 2 +- docs/user/dql/functions.rst | 85 ++++++-- docs/user/general/identifiers.rst | 30 +-- docs/user/ppl/admin/catalog.rst | 83 -------- docs/user/ppl/admin/datasources.rst | 83 ++++++++ docs/user/ppl/cmd/information_schema.rst | 14 +- docs/user/ppl/cmd/showcatalogs.rst | 36 ---- docs/user/ppl/cmd/showdatasources.rst | 36 ++++ docs/user/ppl/functions/datetime.rst | 73 +++++-- docs/user/ppl/index.rst | 4 +- doctest/build.gradle | 2 +- integ-test/build.gradle | 2 +- .../opensearch/sql/legacy/MethodQueryIT.java | 2 +- .../sql/ppl/PrometheusCatalogCommandsIT.java | 12 +- .../sql/ppl/ShowCatalogsCommandIT.java | 8 +- .../java/org/opensearch/sql/sql/QueryIT.java | 84 ++++++++ .../dsl/BucketAggregationBuilder.java | 2 +- .../dsl/MetricAggregationBuilder.java | 2 +- .../script/filter/FilterQueryBuilder.java | 3 +- .../script/filter/lucene/LuceneQuery.java | 64 +++--- .../lucene/relevance/MultiFieldQuery.java | 4 +- .../filter/lucene/relevance/NoFieldQuery.java | 72 +++++++ .../filter/lucene/relevance/QueryQuery.java | 40 ++++ .../lucene/relevance/RelevanceQuery.java | 53 +++-- .../lucene/relevance/SingleFieldQuery.java | 4 +- .../OpenSearchDataTypeRecognitionTest.java | 2 +- .../script/filter/FilterQueryBuilderTest.java | 188 +++++++++++++++--- .../script/filter/lucene/QueryTest.java | 150 ++++++++++++++ .../lucene/relevance/NoFieldQueryTest.java | 49 +++++ .../sql/plugin/catalog/CatalogSettings.java | 2 +- .../catalog/CatalogServiceImplTest.java | 2 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- .../sql/ppl/parser/AstBuilderTest.java | 2 +- .../client/PrometheusClientImpl.java | 3 +- .../QueryRangeFunctionImplementation.java | 2 +- .../response/PrometheusResponse.java | 20 +- .../PrometheusDefaultImplementor.java | 8 +- .../model/PrometheusResponseFieldNames.java | 3 + .../querybuilder/AggregationQueryBuilder.java | 8 +- .../TimeRangeParametersResolver.java | 2 +- .../QueryRangeFunctionImplementationTest.java | 2 +- .../storage/PrometheusMetricScanTest.java | 47 +++++ .../storage/PrometheusMetricTableTest.java | 27 +++ sql/src/main/antlr/OpenSearchSQLParser.g4 | 10 +- .../sql/sql/parser/AstExpressionBuilder.java | 36 +++- .../sql/sql/antlr/SQLSyntaxParserTest.java | 43 ++++ .../sql/parser/AstExpressionBuilderTest.java | 16 ++ 81 files changed, 1347 insertions(+), 414 deletions(-) delete mode 100644 docs/user/ppl/admin/catalog.rst create mode 100644 docs/user/ppl/admin/datasources.rst delete mode 100644 docs/user/ppl/cmd/showcatalogs.rst create mode 100644 docs/user/ppl/cmd/showdatasources.rst create mode 100644 integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java diff --git a/build.gradle b/build.gradle index b25c36f687..bbfc122005 100644 --- a/build.gradle +++ b/build.gradle @@ -8,8 +8,8 @@ buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") spring_version = "5.3.22" - jackson_version = "2.13.4" - jackson_databind_version = "2.13.4.2" + jackson_version = "2.14.0" + jackson_databind_version = "2.14.0" isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 061c4b505f..9885302a69 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -184,7 +184,7 @@ public Expression visitConstantFunction(ConstantFunction node, AnalysisContext c } var value = visitFunction(node, context); - value = DSL.literal(value.valueOf(null)); + value = DSL.literal(value.valueOf()); context.getConstantFunctionValues().put(valueName, value); return value; } diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index 09b2e56b44..7617e156ba 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -6,6 +6,8 @@ package org.opensearch.sql.data.model; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL; + import com.google.common.base.Objects; import java.time.Instant; import java.time.LocalDate; @@ -33,7 +35,7 @@ public class ExprDateValue extends AbstractExprValue { */ public ExprDateValue(String date) { try { - this.date = LocalDate.parse(date); + this.date = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("date:%s in unsupported format, please use " + "yyyy-MM-dd", date)); diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index 9c5f5c5d55..6cc4021d2e 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -6,10 +6,15 @@ package org.opensearch.sql.data.model; -import static org.opensearch.sql.utils.DateTimeFormatters.TIME_FORMATTER_VARIABLE_NANOS; +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; +import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; -import java.time.format.DateTimeFormatter; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.format.DateTimeParseException; import java.util.Objects; import lombok.RequiredArgsConstructor; @@ -22,14 +27,15 @@ */ @RequiredArgsConstructor public class ExprTimeValue extends AbstractExprValue { + private final LocalTime time; /** - * Constructor. + * Constructor of ExprTimeValue. */ public ExprTimeValue(String time) { try { - this.time = LocalTime.parse(time, TIME_FORMATTER_VARIABLE_NANOS); + this.time = LocalTime.parse(time, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL); } catch (DateTimeParseException e) { throw new SemanticCheckException(String.format("time:%s in unsupported format, please use " + "HH:mm:ss[.SSSSSSSSS]", time)); @@ -38,7 +44,7 @@ public ExprTimeValue(String time) { @Override public String value() { - return DateTimeFormatter.ISO_LOCAL_TIME.format(time); + return ISO_LOCAL_TIME.format(time); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 961608d104..e7c94bea7c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -102,7 +102,7 @@ public static NamedExpression named(Expression expression) { return (NamedExpression) expression; } if (expression instanceof ParseExpression) { - return named(((ParseExpression) expression).getIdentifier().valueOf(null).stringValue(), + return named(((ParseExpression) expression).getIdentifier().valueOf().stringValue(), expression); } return named(expression.toString(), expression); @@ -707,6 +707,10 @@ public FunctionExpression simple_query_string(Expression... args) { return compile(BuiltinFunctionName.SIMPLE_QUERY_STRING, args); } + public FunctionExpression query(Expression... args) { + return compile(BuiltinFunctionName.QUERY, args); + } + public FunctionExpression query_string(Expression... args) { return compile(BuiltinFunctionName.QUERY_STRING, args); } diff --git a/core/src/main/java/org/opensearch/sql/expression/Expression.java b/core/src/main/java/org/opensearch/sql/expression/Expression.java index e5efed200a..25a8173efa 100644 --- a/core/src/main/java/org/opensearch/sql/expression/Expression.java +++ b/core/src/main/java/org/opensearch/sql/expression/Expression.java @@ -16,6 +16,13 @@ */ public interface Expression extends Serializable { + /** + * Evaluate the value of expression that does not depend on value environment. + */ + default ExprValue valueOf() { + return valueOf(null); + } + /** * Evaluate the value of expression in the value environment. */ diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java index 4ac0991979..cff08bb098 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java @@ -29,7 +29,7 @@ public TakeAggregator(List arguments, ExprCoreType returnType) { @Override public TakeState create() { - return new TakeState(getArguments().get(1).valueOf(null).integerValue()); + return new TakeState(getArguments().get(1).valueOf().integerValue()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 43f5234d31..42274c0756 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -56,6 +56,7 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java index 43a722b838..97afe3675e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java @@ -30,6 +30,7 @@ public void register(BuiltinFunctionRepository repository) { repository.register(match()); repository.register(multi_match()); repository.register(simple_query_string()); + repository.register(query()); repository.register(query_string()); // Register MATCHPHRASE as MATCH_PHRASE as well for backwards // compatibility. @@ -68,6 +69,11 @@ private static FunctionResolver simple_query_string() { return new RelevanceFunctionResolver(funcName, STRUCT); } + private static FunctionResolver query() { + FunctionName funcName = BuiltinFunctionName.QUERY.getName(); + return new RelevanceFunctionResolver(funcName, STRING); + } + private static FunctionResolver query_string() { FunctionName funcName = BuiltinFunctionName.QUERY_STRING.getName(); return new RelevanceFunctionResolver(funcName, STRUCT); diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java index 23508406ac..8f904bfbf7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java @@ -125,7 +125,7 @@ private static DefaultFunctionResolver castToFloat() { impl(nullMissingHandling( (v) -> new ExprFloatValue(Float.valueOf(v.stringValue()))), FLOAT, STRING), impl(nullMissingHandling( - (v) -> new ExprFloatValue(v.longValue())), FLOAT, DOUBLE), + (v) -> new ExprFloatValue(v.floatValue())), FLOAT, DOUBLE), impl(nullMissingHandling( (v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, BOOLEAN) ); diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java index 12c803dcbc..9797832f07 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java @@ -45,7 +45,7 @@ public class GrokExpression extends ParseExpression { */ public GrokExpression(Expression sourceField, Expression pattern, Expression identifier) { super("grok", sourceField, pattern, identifier); - this.grok = grokCompiler.compile(pattern.valueOf(null).stringValue()); + this.grok = grokCompiler.compile(pattern.valueOf().stringValue()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java index 822651be20..d99242bdde 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java @@ -48,7 +48,7 @@ public ParseExpression(String functionName, Expression sourceField, Expression p this.sourceField = sourceField; this.pattern = pattern; this.identifier = identifier; - this.identifierStr = identifier.valueOf(null).stringValue(); + this.identifierStr = identifier.valueOf().stringValue(); } @Override diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java index c60588c910..67160dad58 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java @@ -44,7 +44,7 @@ public class PatternsExpression extends ParseExpression { */ public PatternsExpression(Expression sourceField, Expression pattern, Expression identifier) { super("patterns", sourceField, pattern, identifier); - String patternStr = pattern.valueOf(null).stringValue(); + String patternStr = pattern.valueOf().stringValue(); useCustomPattern = !patternStr.isEmpty(); if (useCustomPattern) { this.pattern = Pattern.compile(patternStr); diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java index 850f9f8d08..f3a3ff0b66 100644 --- a/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java @@ -40,7 +40,7 @@ public class RegexExpression extends ParseExpression { */ public RegexExpression(Expression sourceField, Expression pattern, Expression identifier) { super("regex", sourceField, pattern, identifier); - this.regexPattern = Pattern.compile(pattern.valueOf(null).stringValue()); + this.regexPattern = Pattern.compile(pattern.valueOf().stringValue()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java index d0aeceecba..51d2850df7 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java @@ -58,7 +58,7 @@ public boolean hasNext() { @Override public ExprValue next() { List values = valuesIterator.next().stream() - .map(expr -> expr.valueOf(null)) + .map(expr -> expr.valueOf()) .collect(Collectors.toList()); return new ExprCollectionValue(values); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java index 26c9c3d04f..4e6a87e21b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java @@ -21,7 +21,7 @@ /** - * Table implementation to handle show catalogs command. + * Table implementation to handle show datasources command. * Since catalog information is not tied to any storage engine, this info * is handled via Catalog Table. * diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java index 894ff9f216..efc59c97ec 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java @@ -51,7 +51,7 @@ public void open() { for (Catalog catalog : catalogs) { exprValues.add( new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "DATASOURCE_NAME", ExprValueUtils.stringValue(catalog.getName()), "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name()))))); } iterator = exprValues.iterator(); diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java index 35fd63f098..b360eb87db 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java @@ -22,7 +22,7 @@ public enum CatalogTableSchema { CATALOG_TABLE_SCHEMA(new LinkedHashMap<>() { { - put("CATALOG_NAME", STRING); + put("DATASOURCE_NAME", STRING); put("CONNECTOR_TYPE", STRING); } } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java index 4eaafea59b..71ae5dc7c3 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java @@ -44,7 +44,7 @@ public abstract class Rounding { * Create Rounding instance. */ public static Rounding createRounding(SpanExpression span) { - ExprValue interval = span.getValue().valueOf(null); + ExprValue interval = span.getValue().valueOf(); ExprType type = span.type(); if (LONG.isCompatible(type)) { diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java index 2efdbb3755..2556aed8d8 100644 --- a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java +++ b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java @@ -113,23 +113,25 @@ public class DateTimeFormatters { public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder() - .appendPattern("yyyy-MM-dd HH:mm:ss") + .appendPattern("uuuu-MM-dd HH:mm:ss") .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) - .toFormatter(Locale.ROOT); + .toFormatter(Locale.ROOT) + .withResolverStyle(ResolverStyle.STRICT); - public static final DateTimeFormatter TIME_FORMATTER_VARIABLE_NANOS = + public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL = new DateTimeFormatterBuilder() - .appendPattern("HH:mm:ss") + .appendPattern("[uuuu-MM-dd HH:mm:ss][uuuu-MM-dd HH:mm][HH:mm:ss][HH:mm][uuuu-MM-dd]") .appendFraction( ChronoField.NANO_OF_SECOND, MIN_FRACTION_SECONDS, MAX_FRACTION_SECONDS, true) - .toFormatter(); + .toFormatter(Locale.ROOT) + .withResolverStyle(ResolverStyle.STRICT); // YYMMDD public static final DateTimeFormatter DATE_FORMATTER_SHORT_YEAR = diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index d1cef1edb3..84c5005eff 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -487,6 +487,15 @@ void simple_query_string_expression_two_fields() { AstDSL.unresolvedArg("query", stringLiteral("sample query")))); } + @Test + void query_expression() { + assertAnalyzeEqual( + dsl.query( + dsl.namedArgument("query", DSL.literal("field:query"))), + AstDSL.function("query", + AstDSL.unresolvedArg("query", stringLiteral("field:query")))); + } + @Test void query_string_expression() { assertAnalyzeEqual( @@ -574,7 +583,7 @@ public void constant_function_returns_constant_cached_value() { var values = List.of(analyze(AstDSL.constantFunction("now")), analyze(AstDSL.constantFunction("now")), analyze(AstDSL.constantFunction("now"))); assertTrue(values.stream().allMatch(v -> - v.valueOf(null) == analyze(AstDSL.constantFunction("now")).valueOf(null))); + v.valueOf() == analyze(AstDSL.constantFunction("now")).valueOf())); } @Test @@ -584,8 +593,8 @@ public void function_returns_non_constant_value() { // different values var values = List.of(analyze(function("sysdate")), analyze(function("sysdate")), analyze(function("sysdate")), analyze(function("sysdate"))); - var referenceValue = analyze(function("sysdate")).valueOf(null); - assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue)); + var referenceValue = analyze(function("sysdate")).valueOf(); + assertTrue(values.stream().noneMatch(v -> v.valueOf() == referenceValue)); } @Test @@ -593,8 +602,8 @@ public void now_as_a_function_not_cached() { // // We can call `now()` as a function, in that case nothing should be cached var values = List.of(analyze(function("now")), analyze(function("now")), analyze(function("now")), analyze(function("now"))); - var referenceValue = analyze(function("now")).valueOf(null); - assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue)); + var referenceValue = analyze(function("now")).valueOf(); + assertTrue(values.stream().noneMatch(v -> v.valueOf() == referenceValue)); } protected Expression analyze(UnresolvedExpression unresolvedExpression) { diff --git a/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java index 7dc56fdf89..38f1ce3ca9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java @@ -59,7 +59,7 @@ void name_a_parse_expression() { DSL.literal("group")); NamedExpression named = DSL.named(parse); assertEquals(parse, named.getDelegated()); - assertEquals(parse.getIdentifier().valueOf(null).stringValue(), named.getName()); + assertEquals(parse.getIdentifier().valueOf().stringValue(), named.getName()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 79efa2a015..29bfa3d91b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -24,6 +24,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; +import java.time.LocalDate; import java.util.List; import lombok.AllArgsConstructor; import org.junit.jupiter.api.BeforeEach; @@ -240,6 +241,18 @@ public void date() { assertEquals(DATE, expr.type()); assertEquals(new ExprDateValue("2020-08-17"), eval(expr)); assertEquals("date(DATE '2020-08-17')", expr.toString()); + + expr = dsl.date(DSL.literal(new ExprDateValue("2020-08-17 12:12:00"))); + assertEquals(DATE, expr.type()); + assertEquals(new ExprDateValue("2020-08-17 12:12:00"), eval(expr)); + assertEquals("date(DATE '2020-08-17')", expr.toString()); + + expr = dsl.date(DSL.literal(new ExprDateValue("2020-08-17 12:12"))); + assertEquals(DATE, expr.type()); + assertEquals(new ExprDateValue("2020-08-17 12:12"), eval(expr)); + assertEquals("date(DATE '2020-08-17')", expr.toString()); + + } @Test @@ -795,6 +808,30 @@ public void time() { assertEquals(TIME, expr.type()); assertEquals(new ExprTimeValue("01:01:01"), eval(expr)); assertEquals("time(TIME '01:01:01')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("01:01"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("01:01"), eval(expr)); + assertEquals("time(TIME '01:01:00')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("2019-04-19 01:01:01"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("2019-04-19 01:01:01"), eval(expr)); + assertEquals("time(TIME '01:01:01')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("2019-04-19 01:01"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("2019-04-19 01:01"), eval(expr)); + assertEquals("time(TIME '01:01:00')", expr.toString()); + + expr = dsl.time(DSL.literal(new ExprTimeValue("01:01:01.0123"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("01:01:01.0123"), eval(expr)); + assertEquals("time(TIME '01:01:01.0123')", expr.toString()); + + expr = dsl.time(dsl.date(DSL.literal("2020-01-02"))); + assertEquals(TIME, expr.type()); + assertEquals(new ExprTimeValue("00:00:00"), expr.valueOf()); } @Test diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index 91d1fc4b0f..bee16cb92a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -66,19 +66,19 @@ protected FunctionExpression fromUnixTime(Expression value, Expression format) { } protected LocalDateTime fromUnixTime(Long value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + return fromUnixTime(DSL.literal(value)).valueOf().datetimeValue(); } protected LocalDateTime fromUnixTime(Double value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + return fromUnixTime(DSL.literal(value)).valueOf().datetimeValue(); } protected String fromUnixTime(Long value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf().stringValue(); } protected String fromUnixTime(Double value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf().stringValue(); } protected FunctionExpression makedate(Expression year, Expression dayOfYear) { @@ -89,7 +89,7 @@ protected FunctionExpression makedate(Expression year, Expression dayOfYear) { } protected LocalDate makedate(Double year, Double dayOfYear) { - return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); + return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf().dateValue(); } protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { @@ -101,7 +101,7 @@ protected FunctionExpression maketime(Expression hour, Expression minute, Expres protected LocalTime maketime(Double hour, Double minute, Double second) { return maketime(DSL.literal(hour), DSL.literal(minute), DSL.literal(second)) - .valueOf(null).timeValue(); + .valueOf().timeValue(); } protected FunctionExpression period_add(Expression period, Expression months) { @@ -112,7 +112,7 @@ protected FunctionExpression period_add(Expression period, Expression months) { } protected Integer period_add(Integer period, Integer months) { - return period_add(DSL.literal(period), DSL.literal(months)).valueOf(null).integerValue(); + return period_add(DSL.literal(period), DSL.literal(months)).valueOf().integerValue(); } protected FunctionExpression period_diff(Expression first, Expression second) { @@ -123,7 +123,7 @@ protected FunctionExpression period_diff(Expression first, Expression second) { } protected Integer period_diff(Integer first, Integer second) { - return period_diff(DSL.literal(first), DSL.literal(second)).valueOf(null).integerValue(); + return period_diff(DSL.literal(first), DSL.literal(second)).valueOf().integerValue(); } protected FunctionExpression unixTimeStampExpr() { @@ -133,7 +133,7 @@ protected FunctionExpression unixTimeStampExpr() { } protected Long unixTimeStamp() { - return unixTimeStampExpr().valueOf(null).longValue(); + return unixTimeStampExpr().valueOf().longValue(); } protected FunctionExpression unixTimeStampOf(Expression value) { @@ -144,18 +144,18 @@ protected FunctionExpression unixTimeStampOf(Expression value) { } protected Double unixTimeStampOf(Double value) { - return unixTimeStampOf(DSL.literal(value)).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(value)).valueOf().doubleValue(); } protected Double unixTimeStampOf(LocalDate value) { - return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(new ExprDateValue(value))).valueOf().doubleValue(); } protected Double unixTimeStampOf(LocalDateTime value) { - return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(new ExprDatetimeValue(value))).valueOf().doubleValue(); } protected Double unixTimeStampOf(Instant value) { - return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); + return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf().doubleValue(); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java index b6d04cbc97..58387ef04f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/FromUnixTimeTest.java @@ -132,54 +132,54 @@ public void checkOfDoubleWithFormat(Double value, String format, String expected @Test public void checkInvalidFormat() { assertEquals(new ExprStringValue("q"), - fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf(null)); + fromUnixTime(DSL.literal(0L), DSL.literal("%q")).valueOf()); assertEquals(new ExprStringValue(""), - fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf(null)); + fromUnixTime(DSL.literal(0L), DSL.literal("")).valueOf()); } @Test public void checkValueOutsideOfTheRangeWithoutFormat() { - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L)).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d)).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L)).valueOf(null)); - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d)).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1L)).valueOf()); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(-1.5d)).valueOf()); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200L)).valueOf()); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771200d)).valueOf()); } @Test public void checkInsideTheRangeWithoutFormat() { - assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L)).valueOf(null)); - assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d)).valueOf(null)); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199L)).valueOf()); + assertNotEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(32536771199d)).valueOf()); } @Test public void checkValueOutsideOfTheRangeWithFormat() { assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771200L), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771200d), DSL.literal("%d")).valueOf()); } @Test public void checkInsideTheRangeWithFormat() { assertNotEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771199L), DSL.literal("%d")).valueOf()); assertNotEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(32536771199d), DSL.literal("%d")).valueOf()); } @Test public void checkNullOrNegativeValues() { - assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of())).valueOf(null)); + assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of())).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(-1L), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(-1.5d), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(-1.5d), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(42L), DSL.literal(ExprNullValue.of())).valueOf(null)); + fromUnixTime(DSL.literal(42L), DSL.literal(ExprNullValue.of())).valueOf()); assertEquals(ExprNullValue.of(), - fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal("%d")).valueOf(null)); + fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal("%d")).valueOf()); assertEquals(ExprNullValue.of(), fromUnixTime(DSL.literal(ExprNullValue.of()), DSL.literal(ExprNullValue.of())) - .valueOf(null)); + .valueOf()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java index e8f5c16025..b6b82fe8fb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java @@ -60,9 +60,9 @@ private static Stream functionNames() { private Temporal extractValue(FunctionExpression func) { switch ((ExprCoreType)func.type()) { - case DATE: return func.valueOf(null).dateValue(); - case DATETIME: return func.valueOf(null).datetimeValue(); - case TIME: return func.valueOf(null).timeValue(); + case DATE: return func.valueOf().dateValue(); + case DATETIME: return func.valueOf().datetimeValue(); + case TIME: return func.valueOf().timeValue(); // unreachable code default: throw new IllegalArgumentException(String.format("%s", func.type())); } @@ -105,7 +105,7 @@ public void test_now_like_functions(Function f for (var wrongFspValue: List.of(-1, 10)) { var exception = assertThrows(IllegalArgumentException.class, - () -> function.apply(new Expression[]{DSL.literal(wrongFspValue)}).valueOf(null)); + () -> function.apply(new Expression[]{DSL.literal(wrongFspValue)}).valueOf()); assertEquals(String.format("Invalid `fsp` value: %d, allowed 0 to 6", wrongFspValue), exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java index ff63cb6f0f..bf228dd509 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java @@ -96,8 +96,8 @@ public static Stream getInvalidTestData() { @ParameterizedTest @MethodSource("getInvalidTestData") public void period_add_returns_null_on_invalid_input(int period) { - assertNull(period_add(DSL.literal(period), DSL.literal(1)).valueOf(null).value()); - assertNull(period_diff(DSL.literal(period), DSL.literal(1)).valueOf(null).value()); - assertNull(period_diff(DSL.literal(1), DSL.literal(period)).valueOf(null).value()); + assertNull(period_add(DSL.literal(period), DSL.literal(1)).valueOf().value()); + assertNull(period_diff(DSL.literal(period), DSL.literal(1)).valueOf().value()); + assertNull(period_diff(DSL.literal(1), DSL.literal(period)).valueOf().value()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java index 437e195f3e..4e7541177f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/UnixTimeStampTest.java @@ -229,7 +229,7 @@ private static Stream getInvalidDoubleSamples() { @MethodSource("getInvalidDoubleSamples") public void checkInvalidDoubleCausesNull(Double value) { assertEquals(ExprNullValue.of(), - unixTimeStampOf(DSL.literal(new ExprDoubleValue(value))).valueOf(null), + unixTimeStampOf(DSL.literal(new ExprDoubleValue(value))).valueOf(), new DecimalFormat("0.#").format(value)); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java index 741caa5f91..620bdbf1b4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/OpenSearchFunctionsTest.java @@ -183,6 +183,13 @@ void simple_query_string() { expr.toString()); } + @Test + void query() { + FunctionExpression expr = dsl.query(query); + assertEquals(String.format("query(query=%s)", query.getValue()), + expr.toString()); + } + @Test void query_string() { FunctionExpression expr = dsl.query_string(fields, query); diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java index b90314f9c1..44af20a690 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java @@ -71,7 +71,7 @@ public void add(ExprValue op1, ExprValue op2) { FunctionExpression expression = dsl.add(literal(op1), literal(op2)); ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); - assertValueEqual(BuiltinFunctionName.ADD, expectedType, op1, op2, expression.valueOf(null)); + assertValueEqual(BuiltinFunctionName.ADD, expectedType, op1, op2, expression.valueOf()); assertEquals(String.format("+(%s, %s)", op1.toString(), op2.toString()), expression.toString()); } @@ -147,7 +147,7 @@ public void subtract(ExprValue op1, ExprValue op2) { ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); assertValueEqual(BuiltinFunctionName.SUBTRACT, expectedType, op1, op2, - expression.valueOf(null)); + expression.valueOf()); assertEquals(String.format("-(%s, %s)", op1.toString(), op2.toString()), expression.toString()); } @@ -159,7 +159,7 @@ public void multiply(ExprValue op1, ExprValue op2) { ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); assertValueEqual(BuiltinFunctionName.MULTIPLY, expectedType, op1, op2, - expression.valueOf(null)); + expression.valueOf()); assertEquals(String.format("*(%s, %s)", op1.toString(), op2.toString()), expression.toString()); } @@ -170,7 +170,7 @@ public void divide(ExprValue op1, ExprValue op2) { FunctionExpression expression = dsl.divide(literal(op1), literal(op2)); ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); - assertValueEqual(BuiltinFunctionName.DIVIDE, expectedType, op1, op2, expression.valueOf(null)); + assertValueEqual(BuiltinFunctionName.DIVIDE, expectedType, op1, op2, expression.valueOf()); assertEquals(String.format("/(%s, %s)", op1.toString(), op2.toString()), expression.toString()); @@ -187,7 +187,7 @@ public void module(ExprValue op1, ExprValue op2) { FunctionExpression expression = dsl.module(literal(op1), literal(op2)); ExprType expectedType = WideningTypeRule.max(op1.type(), op2.type()); assertEquals(expectedType, expression.type()); - assertValueEqual(BuiltinFunctionName.MODULES, expectedType, op1, op2, expression.valueOf(null)); + assertValueEqual(BuiltinFunctionName.MODULES, expectedType, op1, op2, expression.valueOf()); assertEquals(String.format("%%(%s, %s)", op1.toString(), op2.toString()), expression.toString()); diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index c2ca793e39..1f3748709e 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -48,8 +48,8 @@ class TypeCastOperatorTest { private static Stream numberData() { return Stream.of(new ExprByteValue(3), new ExprShortValue(3), - new ExprIntegerValue(3), new ExprLongValue(3L), new ExprFloatValue(3f), - new ExprDoubleValue(3D)); + new ExprIntegerValue(3), new ExprLongValue(3L), new ExprFloatValue(3.14f), + new ExprDoubleValue(3.1415D)); } private static Stream stringData() { @@ -81,7 +81,7 @@ private static Stream datetime() { void castToString(ExprValue value) { FunctionExpression expression = dsl.castString(DSL.literal(value)); assertEquals(STRING, expression.type()); - assertEquals(new ExprStringValue(value.value().toString()), expression.valueOf(null)); + assertEquals(new ExprStringValue(value.value().toString()), expression.valueOf()); } @ParameterizedTest(name = "castToByte({0})") @@ -89,7 +89,7 @@ void castToString(ExprValue value) { void castToByte(ExprValue value) { FunctionExpression expression = dsl.castByte(DSL.literal(value)); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(value.byteValue()), expression.valueOf(null)); + assertEquals(new ExprByteValue(value.byteValue()), expression.valueOf()); } @ParameterizedTest(name = "castToShort({0})") @@ -97,7 +97,7 @@ void castToByte(ExprValue value) { void castToShort(ExprValue value) { FunctionExpression expression = dsl.castShort(DSL.literal(value)); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(value.shortValue()), expression.valueOf(null)); + assertEquals(new ExprShortValue(value.shortValue()), expression.valueOf()); } @ParameterizedTest(name = "castToInt({0})") @@ -105,67 +105,67 @@ void castToShort(ExprValue value) { void castToInt(ExprValue value) { FunctionExpression expression = dsl.castInt(DSL.literal(value)); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(value.integerValue()), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(value.integerValue()), expression.valueOf()); } @Test void castStringToByte() { FunctionExpression expression = dsl.castByte(DSL.literal("100")); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(100), expression.valueOf(null)); + assertEquals(new ExprByteValue(100), expression.valueOf()); } @Test void castStringToShort() { FunctionExpression expression = dsl.castShort(DSL.literal("100")); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(100), expression.valueOf(null)); + assertEquals(new ExprShortValue(100), expression.valueOf()); } @Test void castStringToInt() { FunctionExpression expression = dsl.castInt(DSL.literal("100")); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(100), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(100), expression.valueOf()); } @Test void castStringToIntException() { FunctionExpression expression = dsl.castInt(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToByte() { FunctionExpression expression = dsl.castByte(DSL.literal(true)); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(1), expression.valueOf(null)); + assertEquals(new ExprByteValue(1), expression.valueOf()); expression = dsl.castByte(DSL.literal(false)); assertEquals(BYTE, expression.type()); - assertEquals(new ExprByteValue(0), expression.valueOf(null)); + assertEquals(new ExprByteValue(0), expression.valueOf()); } @Test void castBooleanToShort() { FunctionExpression expression = dsl.castShort(DSL.literal(true)); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(1), expression.valueOf(null)); + assertEquals(new ExprShortValue(1), expression.valueOf()); expression = dsl.castShort(DSL.literal(false)); assertEquals(SHORT, expression.type()); - assertEquals(new ExprShortValue(0), expression.valueOf(null)); + assertEquals(new ExprShortValue(0), expression.valueOf()); } @Test void castBooleanToInt() { FunctionExpression expression = dsl.castInt(DSL.literal(true)); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(1), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(1), expression.valueOf()); expression = dsl.castInt(DSL.literal(false)); assertEquals(INTEGER, expression.type()); - assertEquals(new ExprIntegerValue(0), expression.valueOf(null)); + assertEquals(new ExprIntegerValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToLong({0})") @@ -173,31 +173,31 @@ void castBooleanToInt() { void castToLong(ExprValue value) { FunctionExpression expression = dsl.castLong(DSL.literal(value)); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(value.longValue()), expression.valueOf(null)); + assertEquals(new ExprLongValue(value.longValue()), expression.valueOf()); } @Test void castStringToLong() { FunctionExpression expression = dsl.castLong(DSL.literal("100")); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(100), expression.valueOf(null)); + assertEquals(new ExprLongValue(100), expression.valueOf()); } @Test void castStringToLongException() { FunctionExpression expression = dsl.castLong(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToLong() { FunctionExpression expression = dsl.castLong(DSL.literal(true)); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(1), expression.valueOf(null)); + assertEquals(new ExprLongValue(1), expression.valueOf()); expression = dsl.castLong(DSL.literal(false)); assertEquals(LONG, expression.type()); - assertEquals(new ExprLongValue(0), expression.valueOf(null)); + assertEquals(new ExprLongValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToFloat({0})") @@ -205,31 +205,31 @@ void castBooleanToLong() { void castToFloat(ExprValue value) { FunctionExpression expression = dsl.castFloat(DSL.literal(value)); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(value.floatValue()), expression.valueOf(null)); + assertEquals(new ExprFloatValue(value.floatValue()), expression.valueOf()); } @Test void castStringToFloat() { FunctionExpression expression = dsl.castFloat(DSL.literal("100.0")); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(100.0), expression.valueOf(null)); + assertEquals(new ExprFloatValue(100.0), expression.valueOf()); } @Test void castStringToFloatException() { FunctionExpression expression = dsl.castFloat(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToFloat() { FunctionExpression expression = dsl.castFloat(DSL.literal(true)); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(1), expression.valueOf(null)); + assertEquals(new ExprFloatValue(1), expression.valueOf()); expression = dsl.castFloat(DSL.literal(false)); assertEquals(FLOAT, expression.type()); - assertEquals(new ExprFloatValue(0), expression.valueOf(null)); + assertEquals(new ExprFloatValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToDouble({0})") @@ -237,31 +237,31 @@ void castBooleanToFloat() { void castToDouble(ExprValue value) { FunctionExpression expression = dsl.castDouble(DSL.literal(value)); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(value.doubleValue()), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(value.doubleValue()), expression.valueOf()); } @Test void castStringToDouble() { FunctionExpression expression = dsl.castDouble(DSL.literal("100.0")); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(100), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(100), expression.valueOf()); } @Test void castStringToDoubleException() { FunctionExpression expression = dsl.castDouble(DSL.literal("invalid")); - assertThrows(RuntimeException.class, () -> expression.valueOf(null)); + assertThrows(RuntimeException.class, () -> expression.valueOf()); } @Test void castBooleanToDouble() { FunctionExpression expression = dsl.castDouble(DSL.literal(true)); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(1), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(1), expression.valueOf()); expression = dsl.castDouble(DSL.literal(false)); assertEquals(DOUBLE, expression.type()); - assertEquals(new ExprDoubleValue(0), expression.valueOf(null)); + assertEquals(new ExprDoubleValue(0), expression.valueOf()); } @ParameterizedTest(name = "castToBoolean({0})") @@ -269,96 +269,96 @@ void castBooleanToDouble() { void castToBoolean(ExprValue value) { FunctionExpression expression = dsl.castBoolean(DSL.literal(value)); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(true), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } @Test void castZeroToBoolean() { FunctionExpression expression = dsl.castBoolean(DSL.literal(0)); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(false), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(false), expression.valueOf()); } @Test void castStringToBoolean() { FunctionExpression expression = dsl.castBoolean(DSL.literal("True")); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(true), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } @Test void castBooleanToBoolean() { FunctionExpression expression = dsl.castBoolean(DSL.literal(true)); assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(true), expression.valueOf(null)); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } @Test void castToDate() { FunctionExpression expression = dsl.castDate(DSL.literal("2012-08-07")); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); expression = dsl.castDate(DSL.literal(new ExprDatetimeValue("2012-08-07 01:01:01"))); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); expression = dsl.castDate(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); expression = dsl.castDate(DSL.literal(new ExprDateValue("2012-08-07"))); assertEquals(DATE, expression.type()); - assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf(null)); + assertEquals(new ExprDateValue("2012-08-07"), expression.valueOf()); } @Test void castToTime() { FunctionExpression expression = dsl.castTime(DSL.literal("01:01:01")); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); expression = dsl.castTime(DSL.literal(new ExprDatetimeValue("2012-08-07 01:01:01"))); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); expression = dsl.castTime(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); expression = dsl.castTime(DSL.literal(new ExprTimeValue("01:01:01"))); assertEquals(TIME, expression.type()); - assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimeValue("01:01:01"), expression.valueOf()); } @Test void castToTimestamp() { FunctionExpression expression = dsl.castTimestamp(DSL.literal("2012-08-07 01:01:01")); assertEquals(TIMESTAMP, expression.type()); - assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castTimestamp(DSL.literal(new ExprDatetimeValue("2012-08-07 01:01:01"))); assertEquals(TIMESTAMP, expression.type()); - assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castTimestamp(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(TIMESTAMP, expression.type()); - assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprTimestampValue("2012-08-07 01:01:01"), expression.valueOf()); } @Test void castToDatetime() { FunctionExpression expression = dsl.castDatetime(DSL.literal("2012-08-07 01:01:01")); assertEquals(DATETIME, expression.type()); - assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castDatetime(DSL.literal(new ExprTimestampValue("2012-08-07 01:01:01"))); assertEquals(DATETIME, expression.type()); - assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf(null)); + assertEquals(new ExprDatetimeValue("2012-08-07 01:01:01"), expression.valueOf()); expression = dsl.castDatetime(DSL.literal(new ExprDateValue("2012-08-07"))); assertEquals(DATETIME, expression.type()); - assertEquals(new ExprDatetimeValue("2012-08-07 00:00:00"), expression.valueOf(null)); + assertEquals(new ExprDatetimeValue("2012-08-07 00:00:00"), expression.valueOf()); } } diff --git a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java index 453018a700..e6cf4ce585 100644 --- a/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/system/SystemFunctionsTest.java @@ -88,6 +88,6 @@ public ExprType type() { } private String typeofGetValue(ExprValue input) { - return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue(); + return dsl.typeof(DSL.literal(input)).valueOf().stringValue(); } } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java index 26374ff042..cf9b5fe016 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScanTest.java @@ -61,7 +61,7 @@ void testIterator() { assertTrue(catalogTableScan.hasNext()); for (Catalog catalog : catalogSet) { assertEquals(new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( - "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()), + "DATASOURCE_NAME", ExprValueUtils.stringValue(catalog.getName()), "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name())))), catalogTableScan.next()); } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java index a35c94a21c..1c069005f0 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/catalog/CatalogTableTest.java @@ -34,7 +34,7 @@ void testGetFieldTypes() { CatalogTable catalogTable = new CatalogTable(catalogService); Map fieldTypes = catalogTable.getFieldTypes(); Map expectedTypes = new HashMap<>(); - expectedTypes.put("CATALOG_NAME", ExprCoreType.STRING); + expectedTypes.put("DATASOURCE_NAME", ExprCoreType.STRING); expectedTypes.put("CONNECTOR_TYPE", ExprCoreType.STRING); assertEquals(expectedTypes, fieldTypes); } diff --git a/docs/category.json b/docs/category.json index 0b11209111..b8feaa654a 100644 --- a/docs/category.json +++ b/docs/category.json @@ -10,7 +10,7 @@ "user/ppl/cmd/ad.rst", "user/ppl/cmd/dedup.rst", "user/ppl/cmd/describe.rst", - "user/ppl/cmd/showcatalogs.rst", + "user/ppl/cmd/showdatasources.rst", "user/ppl/cmd/information_schema.rst", "user/ppl/cmd/eval.rst", "user/ppl/cmd/fields.rst", diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index a04f01446e..788cac0433 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1076,13 +1076,13 @@ Return type: DATE Example:: - >od SELECT DATE('2020-08-26'), DATE(TIMESTAMP('2020-08-26 13:49:00')) + os> SELECT DATE('2020-08-26'), DATE(TIMESTAMP('2020-08-26 13:49:00')), DATE('2020-08-26 13:49:00'), DATE('2020-08-26 13:49') fetched rows / total rows = 1/1 - +----------------------+------------------------------------------+ - | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | - |----------------------+------------------------------------------| - | DATE '2020-08-26' | DATE '2020-08-26' | - +----------------------+------------------------------------------+ + +----------------------+------------------------------------------+-------------------------------+----------------------------+ + | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | DATE('2020-08-26 13:49:00') | DATE('2020-08-26 13:49') | + |----------------------+------------------------------------------+-------------------------------+----------------------------| + | 2020-08-26 | 2020-08-26 | 2020-08-26 | 2020-08-26 | + +----------------------+------------------------------------------+-------------------------------+----------------------------+ DATETIME @@ -1926,13 +1926,13 @@ Return type: TIME Example:: - >od SELECT TIME('13:49:00'), TIME(TIMESTAMP('2020-08-26 13:49:00')) + os> SELECT TIME('13:49:00'), TIME('13:49'), TIME(TIMESTAMP('2020-08-26 13:49:00')), TIME('2020-08-26 13:49:00') fetched rows / total rows = 1/1 - +--------------------+------------------------------------------+ - | TIME('13:49:00') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | - |--------------------+------------------------------------------| - | TIME '13:49:00' | TIME '13:49:00' | - +--------------------+------------------------------------------+ + +--------------------+-----------------+------------------------------------------+-------------------------------+ + | TIME('13:49:00') | TIME('13:49') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | TIME('2020-08-26 13:49:00') | + |--------------------+-----------------+------------------------------------------+-------------------------------| + | 13:49:00 | 13:49:00 | 13:49:00 | 13:49:00 | + +--------------------+-----------------+------------------------------------------+-------------------------------+ TIME_TO_SEC @@ -3017,6 +3017,67 @@ Another example to show how to set custom values for the optional parameters:: +------+--------------------------+----------------------+ +QUERY +----- + +Description +>>>>>>>>>>> + +``query("query_expression" [, option=]*)`` + +The `query` function is an alternative syntax to the `query_string`_ function. It maps to the query_string query used in search engine, to return the documents that match a provided text, number, date or boolean value with a given query expression. +``query_expression`` must be a string provided in Lucene query string syntax. Please refer to examples below: + +| ``query('Tags:taste OR Body:taste', ...)`` +| ``query("Tags:taste AND Body:taste", ...)`` + +Available parameters include: + +- analyzer +- escape +- allow_leading_wildcard +- analyze_wildcard +- auto_generate_synonyms_phrase_query +- boost +- default_operator +- enable_position_increments +- fuzziness +- fuzzy_max_expansions +- fuzzy_prefix_length +- fuzzy_transpositions +- fuzzy_rewrite +- tie_breaker +- lenient +- type +- max_determinized_states +- minimum_should_match +- quote_analyzer +- phrase_slop +- quote_field_suffix +- rewrite +- time_zone + +Example with only ``query_expressions``, and all other parameters are set default values:: + + os> select * from books where query('title:Pooh House'); + fetched rows / total rows = 2/2 + +------+--------------------------+----------------------+ + | id | title | author | + |------+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + | 2 | Winnie-the-Pooh | Alan Alexander Milne | + +------+--------------------------+----------------------+ + +Another example to show how to set custom values for the optional parameters:: + + os> select * from books where query('title:Pooh House', default_operator='AND'); + fetched rows / total rows = 1/1 + +------+--------------------------+----------------------+ + | id | title | author | + |------+--------------------------+----------------------| + | 1 | The House at Pooh Corner | Alan Alexander Milne | + +------+--------------------------+----------------------+ + HIGHLIGHT ------------ diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index affd381d41..8bb42bb7e7 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -184,18 +184,18 @@ Fully Qualified Table Names Description ----------- -With the introduction of different datasource catalogs along with Opensearch, support for fully qualified table names became compulsory to resolve tables to a catalog. +With the introduction of different datasources along with Opensearch, support for fully qualified table names became compulsory to resolve tables to a datasource. Format for fully qualified table name. -``..`` +``..`` -* catalogName:[Mandatory] Catalog information is mandatory when querying over tables from catalogs other than opensearch connector. +* datasourceName:[Mandatory] Datasource information is mandatory when querying over tables from datasources other than opensearch connector. * schemaName:[Optional] Schema is a logical abstraction for a group of tables. In the current state, we only support ``default`` and ``information_schema``. Any schema mentioned in the fully qualified name other than these two will be resolved to be part of tableName. * tableName:[Mandatory] tableName is mandatory. -The current resolution algorithm works in such a way, the old queries on opensearch work without specifying any catalog name. +The current resolution algorithm works in such a way, the old queries on opensearch work without specifying any datasource name. So queries on opensearch indices doesn't need a fully qualified table name. Table Name Resolution Algorithm. @@ -205,24 +205,24 @@ Fully qualified Name is divided into parts based on ``.`` character. TableName resolution algorithm works in the following manner. -1. Take the first part of the qualified name and resolve it to a catalog from the list of catalogs configured. -If it doesn't resolve to any of the catalog names configured, catalog name will default to ``@opensearch`` catalog. +1. Take the first part of the qualified name and resolve it to a datasource from the list of datasources configured. +If it doesn't resolve to any of the datasource names configured, datasource name will default to ``@opensearch`` datasource. -2. Take the first part of the remaining qualified name after capturing the catalog name. -If this part represents any of the supported schemas under catalog, it will resolve to the same otherwise schema name will resolve to ``default`` schema. +2. Take the first part of the remaining qualified name after capturing the datasource name. +If this part represents any of the supported schemas under datasource, it will resolve to the same otherwise schema name will resolve to ``default`` schema. Currently ``default`` and ``information_schema`` are the only schemas supported. 3. Rest of the parts are combined to resolve tablename. -** Only table name identifiers are supported with fully qualified names, identifiers used for columns and other attributes doesn't require prefixing with catalog and schema information.** +** Only table name identifiers are supported with fully qualified names, identifiers used for columns and other attributes doesn't require prefixing with datasource and schema information.** Examples -------- -Assume [my_prometheus] is the only catalog configured other than default opensearch engine. +Assume [my_prometheus] is the only datasource configured other than default opensearch engine. 1. ``my_prometheus.default.http_requests_total`` -catalogName = ``my_prometheus`` [Is in the list of catalogs configured]. +datasourceName = ``my_prometheus`` [Is in the list of datasources configured]. schemaName = ``default`` [Is in the list of schemas supported]. @@ -231,7 +231,7 @@ tableName = ``http_requests_total``. 2. ``logs.12.13.1`` -catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. @@ -241,7 +241,7 @@ tableName = ``logs.12.13.1``. 3. ``my_prometheus.http_requests_total`` -catalogName = ```my_prometheus`` [Is in the list of catalogs configured]. +datasourceName = ```my_prometheus`` [Is in the list of datasources configured]. schemaName = ``default`` [No supported schema found, so default to `default`]. @@ -249,7 +249,7 @@ tableName = ``http_requests_total``. 4. ``prometheus.http_requests_total`` -catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only datasource configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. @@ -257,7 +257,7 @@ tableName = ``prometheus.http_requests_total``. 5. ``prometheus.default.http_requests_total.1.2.3`` -catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] +datasourceName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] schemaName = ``default`` [No supported schema found, so default to `default`]. diff --git a/docs/user/ppl/admin/catalog.rst b/docs/user/ppl/admin/catalog.rst deleted file mode 100644 index ccaab342a5..0000000000 --- a/docs/user/ppl/admin/catalog.rst +++ /dev/null @@ -1,83 +0,0 @@ -.. highlight:: sh - -================= -Catalog Settings -================= - -.. rubric:: Table of contents - -.. contents:: - :local: - :depth: 1 - -Introduction -============ - -The concept of ``catalog`` is introduced to support the federation of SQL/PPL query engine to multiple data sources. -This helps PPL users to leverage data from multiple data sources and derive correlation and insights. -Catalog definition provides the information to connect to a datasource and also gives a name to them to refer in PPL commands. - - -Definitions of catalog and connector -==================================== -* Connector is a component that adapts the query engine to a datasource. For example, Prometheus connector would adapt and help execute the queries to run on Prometheus data source. connector name is enough in the catalog definition json. -* Catalog is a construct to define how to connect to a datasource and which connector to adapt by query engine. - -Example Prometheus Catalog Definition :: - - [{ - "name" : "my_prometheus", - "connector": "prometheus", - "properties" : { - "prometheus.uri" : "http://localhost:8080", - "prometheus.auth.type" : "basicauth", - "prometheus.auth.username" : "admin", - "prometheus.auth.password" : "admin" - } - }] -Catalog configuration Restrictions. - -* ``name``, ``connector``, ``properties`` are required fields in the catalog configuration. -* All the catalog names should be unique and match the following regex[``[@*A-Za-z]+?[*a-zA-Z_\-0-9]*``]. -* Allowed Connectors. - * ``prometheus`` [More details: `Prometheus Connector `_] -* All the allowed config parameters in ``properties`` are defined in individual connector pages mentioned above. - -Configuring catalog in OpenSearch -==================================== - -* Catalogs are configured in opensearch keystore as secure settings under ``plugins.query.federation.catalog.config`` key as they contain credential info. -* A json file containing array of catalog configurations should be injected into keystore with the above mentioned key. sample json file can be seen in the above section. - - -[**To be run on all the nodes in the cluster**] Command to add catalog.json file to OpenSearch Keystore :: - - >> bin/opensearch-keystore add-file plugins.query.federation.catalog.config catalog.json - -Catalogs can be configured during opensearch start up or can be updated while the opensearch is running. -If we update catalog configuration during runtime, the following api should be triggered to update the query engine with the latest changes. - -[**Required only if we update keystore settings during runtime**] Secure Settings refresh api:: - - >> curl --request POST \ - --url http://{{opensearch-domain}}:9200/_nodes/reload_secure_settings \ - --data '{"secure_settings_password":"{{keystore-password}}"}' - - -Using a catalog in PPL command -==================================== -Catalog is referred in source command as show in the code block below. -Based on the abstraction designed by the connector, -one can refer the corresponding entity as table in the source command. -For example in prometheus connector, each metric is abstracted as a table. -so we can refer a metric and apply stats over it in the following way. - -Example source command with prometheus catalog :: - - >> source = my_prometheus.prometheus_http_requests_total | stats avg(@value) by job; - - -Limitations of catalog -==================================== -Catalog settings are global and users with PPL access are allowed to fetch data from all the defined catalogs. -PPL access can be controlled using roles.(More details: `Security Settings `_) \ No newline at end of file diff --git a/docs/user/ppl/admin/datasources.rst b/docs/user/ppl/admin/datasources.rst new file mode 100644 index 0000000000..2974ac20ce --- /dev/null +++ b/docs/user/ppl/admin/datasources.rst @@ -0,0 +1,83 @@ +.. highlight:: sh + +=================== +Datasource Settings +=================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + +Introduction +============ + +The concept of ``datasource`` is introduced to support the federation of SQL/PPL query engine to multiple data stores. +This helps PPL users to leverage data from multiple data stores and derive correlation and insights. +Datasource definition provides the information to connect to a data store and also gives a name to them to refer in PPL commands. + + +Definitions of datasource and connector +==================================== +* Connector is a component that adapts the query engine to a datastore. For example, Prometheus connector would adapt and help execute the queries to run on Prometheus datastore. connector name is enough in the datasource definition json. +* Datasource is a construct to define how to connect to a data store and which connector to adapt by query engine. + +Example Prometheus Datasource Definition :: + + [{ + "name" : "my_prometheus", + "connector": "prometheus", + "properties" : { + "prometheus.uri" : "http://localhost:8080", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "admin" + } + }] +Datasource configuration Restrictions. + +* ``name``, ``connector``, ``properties`` are required fields in the datasource configuration. +* All the datasource names should be unique and match the following regex[``[@*A-Za-z]+?[*a-zA-Z_\-0-9]*``]. +* Allowed Connectors. + * ``prometheus`` [More details: `Prometheus Connector `_] +* All the allowed config parameters in ``properties`` are defined in individual connector pages mentioned above. + +Configuring a datasource in OpenSearch +====================================== + +* Datasources are configured in opensearch keystore as secure settings under ``plugins.query.federation.datasources.config`` key as they contain credential info. +* A json file containing array of datasource configurations should be injected into keystore with the above mentioned key. sample json file can be seen in the above section. + + +[**To be run on all the nodes in the cluster**] Command to add datasources.json file to OpenSearch Keystore :: + + >> bin/opensearch-keystore add-file plugins.query.federation.datasource.config datasources.json + +Datasources can be configured during opensearch start up or can be updated while the opensearch is running. +If we update a datasource configuration during runtime, the following api should be triggered to update the query engine with the latest changes. + +[**Required only if we update keystore settings during runtime**] Secure Settings refresh api:: + + >> curl --request POST \ + --url http://{{opensearch-domain}}:9200/_nodes/reload_secure_settings \ + --data '{"secure_settings_password":"{{keystore-password}}"}' + + +Using a datasource in PPL command +==================================== +Datasource is referred in source command as show in the code block below. +Based on the abstraction designed by the connector, +one can refer the corresponding entity as table in the source command. +For example in prometheus connector, each metric is abstracted as a table. +so we can refer a metric and apply stats over it in the following way. + +Example source command with prometheus datasource :: + + >> source = my_prometheus.prometheus_http_requests_total | stats avg(@value) by job; + + +Limitations of datasource +==================================== +Datasource settings are global and users with PPL access are allowed to fetch data from all the defined datasources. +PPL access can be controlled using roles.(More details: `Security Settings `_) \ No newline at end of file diff --git a/docs/user/ppl/cmd/information_schema.rst b/docs/user/ppl/cmd/information_schema.rst index a756fb080e..26341d6972 100644 --- a/docs/user/ppl/cmd/information_schema.rst +++ b/docs/user/ppl/cmd/information_schema.rst @@ -11,19 +11,19 @@ Metadata queries using information_schema Description ============ -| Use ``information_schema`` in source command to query tables information under a catalog. +| Use ``information_schema`` in source command to query tables information under a datasource. In the current state, ``information_schema`` only support metadata of tables. This schema will be extended for views, columns and other metadata info in future. Syntax ============ -source = catalog.information_schema.tables; +source = datasource.information_schema.tables; -Example 1: Fetch tables in prometheus catalog. +Example 1: Fetch tables in prometheus datasource. ============================================== -The examples fetches tables in the prometheus catalog. +The examples fetches tables in the prometheus datasource. PPL query for fetching PROMETHEUS TABLES with where clause:: @@ -36,10 +36,10 @@ PPL query for fetching PROMETHEUS TABLES with where clause:: +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ -Example 2: Search tables in prometheus catalog. -=============================================== +Example 2: Search tables in prometheus datasource. +================================================= -The examples searches tables in the prometheus catalog. +The examples searches tables in the prometheus datasource. PPL query for searching PROMETHEUS TABLES:: diff --git a/docs/user/ppl/cmd/showcatalogs.rst b/docs/user/ppl/cmd/showcatalogs.rst deleted file mode 100644 index d304cba768..0000000000 --- a/docs/user/ppl/cmd/showcatalogs.rst +++ /dev/null @@ -1,36 +0,0 @@ -============= -show catalogs -============= - -.. rubric:: Table of contents - -.. contents:: - :local: - :depth: 2 - - -Description -============ -| Using ``show catalogs`` command to query catalogs configured in the PPL engine. ``show catalogs`` command could be only used as the first command in the PPL query. - - -Syntax -============ -show catalogs - - -Example 1: Fetch all PROMETHEUS catalogs -================================= - -The example fetches all the catalogs configured. - -PPL query for all PROMETHEUS CATALOGS:: - - os> show catalogs | where CONNECTOR_TYPE='PROMETHEUS'; - fetched rows / total rows = 1/1 - +----------------+------------------+ - | CATALOG_NAME | CONNECTOR_TYPE | - |----------------+------------------| - | my_prometheus | PROMETHEUS | - +----------------+------------------+ - diff --git a/docs/user/ppl/cmd/showdatasources.rst b/docs/user/ppl/cmd/showdatasources.rst new file mode 100644 index 0000000000..f7c6beb82f --- /dev/null +++ b/docs/user/ppl/cmd/showdatasources.rst @@ -0,0 +1,36 @@ +================ +show datasources +================ + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``show datasources`` command to query datasources configured in the PPL engine. ``show datasources`` command could be only used as the first command in the PPL query. + + +Syntax +============ +show datasources + + +Example 1: Fetch all PROMETHEUS datasources +=========================================== + +The example fetches all the datasources of type prometheus. + +PPL query for all PROMETHEUS DATASOURCES:: + + os> show datasources | where CONNECTOR_TYPE='PROMETHEUS'; + fetched rows / total rows = 1/1 + +-------------------+------------------+ + | DATASOURCE_NAME | CONNECTOR_TYPE | + |-------------------+------------------| + | my_prometheus | PROMETHEUS | + +-------------------+------------------+ + diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 223b3c5557..ca191e2426 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -291,13 +291,38 @@ Return type: DATE Example:: - >od source=people | eval `DATE('2020-08-26')` = DATE('2020-08-26'), `DATE(TIMESTAMP('2020-08-26 13:49:00'))` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | fields `DATE('2020-08-26')`, `DATE(TIMESTAMP('2020-08-26 13:49:00'))` + os> source=people | eval `DATE('2020-08-26')` = DATE('2020-08-26') | fields `DATE('2020-08-26')` fetched rows / total rows = 1/1 - +----------------------+------------------------------------------+ - | DATE('2020-08-26') | DATE(TIMESTAMP('2020-08-26 13:49:00')) | - |----------------------+------------------------------------------| - | DATE '2020-08-26' | DATE '2020-08-26' | - +----------------------+------------------------------------------+ + +----------------------+ + | DATE('2020-08-26') | + |----------------------| + | 2020-08-26 | + +----------------------+ + + os> source=people | eval `DATE(TIMESTAMP('2020-08-26 13:49:00'))` = DATE(TIMESTAMP('2020-08-26 13:49:00')) | fields `DATE(TIMESTAMP('2020-08-26 13:49:00'))` + fetched rows / total rows = 1/1 + +------------------------------------------+ + | DATE(TIMESTAMP('2020-08-26 13:49:00')) | + |------------------------------------------| + | 2020-08-26 | + +------------------------------------------+ + + os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` + fetched rows / total rows = 1/1 + +----------------------------+ + | DATE('2020-08-26 13:49') | + |----------------------------| + | 2020-08-26 | + +----------------------------+ + + os> source=people | eval `DATE('2020-08-26 13:49')` = DATE('2020-08-26 13:49') | fields `DATE('2020-08-26 13:49')` + fetched rows / total rows = 1/1 + +----------------------------+ + | DATE('2020-08-26 13:49') | + |----------------------------| + | 2020-08-26 | + +----------------------------+ + DATE_ADD @@ -1098,13 +1123,37 @@ Return type: TIME Example:: - >od source=people | eval `TIME('13:49:00')` = TIME('13:49:00'), `TIME(TIMESTAMP('2020-08-26 13:49:00'))` = TIME(TIMESTAMP('2020-08-26 13:49:00')) | fields `TIME('13:49:00')`, `TIME(TIMESTAMP('2020-08-26 13:49:00'))` + os> source=people | eval `TIME('13:49:00')` = TIME('13:49:00') | fields `TIME('13:49:00')` fetched rows / total rows = 1/1 - +--------------------+------------------------------------------+ - | TIME('13:49:00') | TIME(TIMESTAMP('2020-08-26 13:49:00')) | - |--------------------+------------------------------------------| - | TIME '13:49:00' | TIME '13:49:00' | - +--------------------+------------------------------------------+ + +--------------------+ + | TIME('13:49:00') | + |--------------------| + | 13:49:00 | + +--------------------+ + + os> source=people | eval `TIME('13:49')` = TIME('13:49') | fields `TIME('13:49')` + fetched rows / total rows = 1/1 + +-----------------+ + | TIME('13:49') | + |-----------------| + | 13:49:00 | + +-----------------+ + + os> source=people | eval `TIME('2020-08-26 13:49:00')` = TIME('2020-08-26 13:49:00') | fields `TIME('2020-08-26 13:49:00')` + fetched rows / total rows = 1/1 + +-------------------------------+ + | TIME('2020-08-26 13:49:00') | + |-------------------------------| + | 13:49:00 | + +-------------------------------+ + + os> source=people | eval `TIME('2020-08-26 13:49')` = TIME('2020-08-26 13:49') | fields `TIME('2020-08-26 13:49')` + fetched rows / total rows = 1/1 + +----------------------------+ + | TIME('2020-08-26 13:49') | + |----------------------------| + | 13:49:00 | + +----------------------------+ TIME_TO_SEC diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index e09315b1c3..a69136bb19 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -34,7 +34,7 @@ The query start with search command and then flowing a set of command delimited - `Monitoring `_ - - `Catalog Settings `_ + - `Datasource Settings `_ - `Prometheus Connector `_ @@ -48,7 +48,7 @@ The query start with search command and then flowing a set of command delimited - `describe command `_ - - `show catalogs command `_ + - `show datasources command `_ - `eval command `_ diff --git a/doctest/build.gradle b/doctest/build.gradle index c1d069e50b..62efd29b18 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -104,7 +104,7 @@ String mlCommonsPlugin = 'opensearch-ml' testClusters { docTestCluster { - keystore 'plugins.query.federation.catalog.config', new File("$projectDir/catalog", 'catalog.json') + keystore 'plugins.query.federation.datasources.config', new File("$projectDir/catalog", 'catalog.json') // Disable loading of `ML-commons` plugin, because it might be unavailable (not released yet). /* plugin(provider(new Callable(){ diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 7a2e5cd406..58618cd67c 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -115,7 +115,7 @@ testClusters.all { testClusters.integTest { plugin ":opensearch-sql-plugin" - keystore 'plugins.query.federation.catalog.config', new File("$projectDir/src/test/resources/catalog/", 'catalog.json') + keystore 'plugins.query.federation.datasources.config', new File("$projectDir/src/test/resources/catalog/", 'catalog.json') } task startPrometheus(type: SpawnProcessTask) { mustRunAfter ':doctest:doctest' diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java index 0859a03784..b1b6b24ada 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/MethodQueryIT.java @@ -38,7 +38,7 @@ public void queryTest() throws IOException { "select address from %s where query('address:880 Holmes Lane') limit 3", TestsConstants.TEST_INDEX_ACCOUNT)); Assert.assertThat(result, - containsString("query_string\":{\"query\":\"address:880 Holmes Lane")); + containsString("query_string\\\":{\\\"query\\\":\\\"address:880 Holmes Lane")); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java index 9e197bbb27..10c1e911ab 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java @@ -46,12 +46,12 @@ public void testSourceMetricCommand() { @SneakyThrows public void testMetricAvgAggregationCommand() { JSONObject response = - executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) by span(@timestamp, 15s), handler, job"); + executeQuery("source=`my_prometheus`.`prometheus_http_requests_total` | stats avg(@value) as `agg` by span(@timestamp, 15s), `handler`, `job`"); verifySchema(response, - schema("avg(@value)", "double"), + schema("agg", "double"), schema("span(@timestamp,15s)", "timestamp"), - schema("handler", "string"), - schema("job", "string")); + schema("`handler`", "string"), + schema("`job`", "string")); Assertions.assertTrue(response.getInt("size") > 0); Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); @@ -65,11 +65,11 @@ public void testMetricAvgAggregationCommand() { @SneakyThrows public void testMetricAvgAggregationCommandWithAlias() { JSONObject response = - executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) as agg by span(@timestamp, 15s), handler, job"); + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) as agg by span(@timestamp, 15s), `handler`, job"); verifySchema(response, schema("agg", "double"), schema("span(@timestamp,15s)", "timestamp"), - schema("handler", "string"), + schema("`handler`", "string"), schema("job", "string")); Assertions.assertTrue(response.getInt("size") > 0); Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java index 23418366be..e12aa040e3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java @@ -20,25 +20,25 @@ public class ShowCatalogsCommandIT extends PPLIntegTestCase { @Test public void testShowCatalogsCommands() throws IOException { - JSONObject result = executeQuery("show catalogs"); + JSONObject result = executeQuery("show datasources"); verifyDataRows(result, rows("my_prometheus", "PROMETHEUS"), rows("@opensearch", "OPENSEARCH")); verifyColumn( result, - columnName("CATALOG_NAME"), + columnName("DATASOURCE_NAME"), columnName("CONNECTOR_TYPE") ); } @Test public void testShowCatalogsCommandsWithWhereClause() throws IOException { - JSONObject result = executeQuery("show catalogs | where CONNECTOR_TYPE='PROMETHEUS'"); + JSONObject result = executeQuery("show datasources | where CONNECTOR_TYPE='PROMETHEUS'"); verifyDataRows(result, rows("my_prometheus", "PROMETHEUS")); verifyColumn( result, - columnName("CATALOG_NAME"), + columnName("DATASOURCE_NAME"), columnName("CONNECTOR_TYPE") ); } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java new file mode 100644 index 0000000000..e61593eb21 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryIT.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.sql.legacy.SQLIntegTestCase; + +public class QueryIT extends SQLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.BEER); + } + + @Test + public void all_fields_test() throws IOException { + String query = "SELECT * FROM " + + TEST_INDEX_BEER + " WHERE query('*:taste')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(16, result.getInt("total")); + } + + @Test + public void mandatory_params_test() throws IOException { + String query = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('Tags:taste OR Body:taste')"; + JSONObject result = executeJdbcRequest(query); + assertEquals(16, result.getInt("total")); + } + + @Test + public void all_params_test() throws IOException { + String query = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE query('Tags:taste', escape=false," + + "allow_leading_wildcard=true, enable_position_increments=true," + + "fuzziness= 1, fuzzy_rewrite='constant_score', max_determinized_states = 10000," + + "analyzer='standard', analyze_wildcard = false, quote_field_suffix = '.exact'," + + "auto_generate_synonyms_phrase_query=true, boost = 0.77," + + "quote_analyzer='standard', phrase_slop=0, rewrite='constant_score', type='best_fields'," + + "tie_breaker=0.3, time_zone='Canada/Pacific', default_operator='or'," + + "fuzzy_transpositions = false, lenient = true, fuzzy_max_expansions = 25," + + "minimum_should_match = '2<-25% 9<-3', fuzzy_prefix_length = 7);"; + JSONObject result = executeJdbcRequest(query); + assertEquals(8, result.getInt("total")); + } + + @Test + public void wildcard_test() throws IOException { + String query1 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('Tags:taste')"; + JSONObject result1 = executeJdbcRequest(query1); + String query2 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('*:taste')"; + JSONObject result2 = executeJdbcRequest(query2); + assertNotEquals(result2.getInt("total"), result1.getInt("total")); + + String query3 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE query('Tags:tas*');"; + JSONObject result3 = executeJdbcRequest(query3); + assertEquals(8, result3.getInt("total")); + + String query4 = "SELECT Id FROM " + TEST_INDEX_BEER + + " WHERE query('Tags:tas?e');"; + JSONObject result4 = executeJdbcRequest(query3); + assertEquals(8, result4.getInt("total")); + } + + @Test + public void query_string_and_query_return_the_same_results_test() throws IOException { + String query1 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query('Tags:taste')"; + JSONObject result1 = executeJdbcRequest(query1); + String query2 = "SELECT Id FROM " + + TEST_INDEX_BEER + " WHERE query_string(['Tags'],'taste')"; + JSONObject result2 = executeJdbcRequest(query2); + assertEquals(result2.getInt("total"), result1.getInt("total")); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index cf368c940d..8ef8a5e4a8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -55,7 +55,7 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( return buildHistogram( expr.getNameOrAlias(), spanExpr.getField().toString(), - spanExpr.getValue().valueOf(null).doubleValue(), + spanExpr.getValue().valueOf().doubleValue(), spanExpr.getUnit(), missingOrder); } else { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java index f4ff22dfbf..5e7d34abce 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java @@ -215,7 +215,7 @@ private Pair make(TopHitsAggregationBuilder bu MetricParser parser) { String fieldName = ((ReferenceExpression) expression).getAttr(); builder.fetchSource(fieldName, null); - builder.size(size.valueOf(null).integerValue()); + builder.size(size.valueOf().integerValue()); builder.from(0); if (condition != null) { return Pair.of( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java index 68f8ec8c66..ab8fb562da 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java @@ -34,6 +34,7 @@ import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchPhraseQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MatchQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.MultiMatchQuery; +import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryStringQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.SimpleQueryStringQuery; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -60,7 +61,7 @@ public class FilterQueryBuilder extends ExpressionNodeVisitor>builder() .put(BuiltinFunctionName.CAST_TO_STRING.getName(), expr -> { if (!expr.type().equals(ExprCoreType.STRING)) { - return new ExprStringValue(String.valueOf(expr.valueOf(null).value())); + return new ExprStringValue(String.valueOf(expr.valueOf().value())); } else { - return expr.valueOf(null); + return expr.valueOf(); } }) .put(BuiltinFunctionName.CAST_TO_BYTE.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprByteValue(expr.valueOf(null).byteValue()); + return new ExprByteValue(expr.valueOf().byteValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprByteValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprByteValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprByteValue(Byte.valueOf(expr.valueOf(null).stringValue())); + return new ExprByteValue(Byte.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_SHORT.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprShortValue(expr.valueOf(null).shortValue()); + return new ExprShortValue(expr.valueOf().shortValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprShortValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprShortValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprShortValue(Short.valueOf(expr.valueOf(null).stringValue())); + return new ExprShortValue(Short.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_INT.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprIntegerValue(expr.valueOf(null).integerValue()); + return new ExprIntegerValue(expr.valueOf().integerValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprIntegerValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprIntegerValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprIntegerValue(Integer.valueOf(expr.valueOf(null).stringValue())); + return new ExprIntegerValue(Integer.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_LONG.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprLongValue(expr.valueOf(null).longValue()); + return new ExprLongValue(expr.valueOf().longValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprLongValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprLongValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprLongValue(Long.valueOf(expr.valueOf(null).stringValue())); + return new ExprLongValue(Long.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_FLOAT.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprFloatValue(expr.valueOf(null).floatValue()); + return new ExprFloatValue(expr.valueOf().floatValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprFloatValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprFloatValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprFloatValue(Float.valueOf(expr.valueOf(null).stringValue())); + return new ExprFloatValue(Float.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_DOUBLE.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return new ExprDoubleValue(expr.valueOf(null).doubleValue()); + return new ExprDoubleValue(expr.valueOf().doubleValue()); } else if (expr.type().equals(ExprCoreType.BOOLEAN)) { - return new ExprDoubleValue(expr.valueOf(null).booleanValue() ? 1 : 0); + return new ExprDoubleValue(expr.valueOf().booleanValue() ? 1 : 0); } else { - return new ExprDoubleValue(Double.valueOf(expr.valueOf(null).stringValue())); + return new ExprDoubleValue(Double.valueOf(expr.valueOf().stringValue())); } }) .put(BuiltinFunctionName.CAST_TO_BOOLEAN.getName(), expr -> { if (ExprCoreType.numberTypes().contains(expr.type())) { - return expr.valueOf(null).doubleValue() == 1 + return expr.valueOf().doubleValue() != 0 ? ExprBooleanValue.of(true) : ExprBooleanValue.of(false); } else if (expr.type().equals(ExprCoreType.STRING)) { - return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf(null).stringValue())); + return ExprBooleanValue.of(Boolean.valueOf(expr.valueOf().stringValue())); } else { - return expr.valueOf(null); + return expr.valueOf(); } }) .put(BuiltinFunctionName.CAST_TO_DATE.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprDateValue(expr.valueOf(null).stringValue()); + return new ExprDateValue(expr.valueOf().stringValue()); } else { - return new ExprDateValue(expr.valueOf(null).dateValue()); + return new ExprDateValue(expr.valueOf().dateValue()); } }) .put(BuiltinFunctionName.CAST_TO_TIME.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimeValue(expr.valueOf(null).stringValue()); + return new ExprTimeValue(expr.valueOf().stringValue()); } else { - return new ExprTimeValue(expr.valueOf(null).timeValue()); + return new ExprTimeValue(expr.valueOf().timeValue()); } }) .put(BuiltinFunctionName.CAST_TO_DATETIME.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprDatetimeValue(expr.valueOf(null).stringValue()); + return new ExprDatetimeValue(expr.valueOf().stringValue()); } else { - return new ExprDatetimeValue(expr.valueOf(null).datetimeValue()); + return new ExprDatetimeValue(expr.valueOf().datetimeValue()); } }) .put(BuiltinFunctionName.CAST_TO_TIMESTAMP.getName(), expr -> { if (expr.type().equals(ExprCoreType.STRING)) { - return new ExprTimestampValue(expr.valueOf(null).stringValue()); + return new ExprTimestampValue(expr.valueOf().stringValue()); } else { - return new ExprTimestampValue(expr.valueOf(null).timestampValue()); + return new ExprTimestampValue(expr.valueOf().timestampValue()); } }) .build(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java index 8390b5ef44..9f37951072 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java @@ -37,13 +37,13 @@ public T createQueryBuilder(List arguments) { var fieldsAndWeights = fields .getValue() - .valueOf(null) + .valueOf() .tupleValue() .entrySet() .stream() .collect(ImmutableMap.toImmutableMap(e -> e.getKey(), e -> e.getValue().floatValue())); - return createBuilder(fieldsAndWeights, query.getValue().valueOf(null).stringValue()); + return createBuilder(fieldsAndWeights, query.getValue().valueOf().stringValue()); } protected abstract T createBuilder(ImmutableMap fields, String query); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java new file mode 100644 index 0000000000..1467cf8e4b --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQuery.java @@ -0,0 +1,72 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.NamedArgumentExpression; + +/** + * Base class to represent relevance queries that have no 'fields' array as an argument. + * + * @param The builder class for the OpenSearch query. + */ +abstract class NoFieldQuery extends RelevanceQuery { + public NoFieldQuery(Map> queryBuildActions) { + super(queryBuildActions); + } + + @Override + protected void ignoreArguments(List arguments) { + arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("query")); + } + + @Override + protected void checkValidArguments(String argNormalized, T queryBuilder) { + if (!getQueryBuildActions().containsKey(argNormalized)) { + throw new SemanticCheckException( + String.format("Parameter %s is invalid for %s function.", + argNormalized, getQueryName())); + } + } + /** + * Override build function because RelevanceQuery requires 2 fields, + * but NoFieldQuery must have no fields. + * + * @param func : Contains function name and passed in arguments. + * @return : QueryBuilder object + */ + + @Override + public QueryBuilder build(FunctionExpression func) { + var arguments = func.getArguments().stream().map( + a -> (NamedArgumentExpression) a).collect(Collectors.toList()); + if (arguments.size() < 1) { + throw new SyntaxCheckException(String.format( + "%s requires at least one parameter", func.getFunctionName())); + } + + return loadArguments(arguments); + } + + + @Override + public T createQueryBuilder(List arguments) { + // Extract 'query' + var query = arguments.stream().filter(a -> a.getArgName().equalsIgnoreCase("query")).findFirst() + .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); + + return createBuilder(query.getValue().valueOf().stringValue()); + } + + protected abstract T createBuilder(String query); +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java new file mode 100644 index 0000000000..35d5a43a41 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryQuery.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.QueryStringQueryBuilder; + +/** + * Class for Lucene query that builds the 'query' query. + */ +public class QueryQuery extends NoFieldQuery { + + private final String queryQueryName = "query"; + + /** + * Default constructor for QueryQuery configures how RelevanceQuery.build() handles + * named arguments by calling the constructor of QueryStringQuery. + */ + public QueryQuery() { + super(FunctionParameterRepository.QueryStringQueryBuildActions); + } + + /** + * Builds QueryBuilder with query value and other default parameter values set. + * + * @param query : Query value for query_string query + * @return : Builder for query query + */ + protected QueryStringQueryBuilder createBuilder(String query) { + return QueryBuilders.queryStringQuery(query); + } + + @Override + public String getQueryName() { + return queryQueryName; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java index 579f77d2cd..e07c0cd966 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java @@ -10,6 +10,7 @@ import java.util.Objects; import java.util.function.BiFunction; import java.util.stream.Collectors; +import lombok.Getter; import lombok.RequiredArgsConstructor; import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -24,17 +25,24 @@ */ @RequiredArgsConstructor public abstract class RelevanceQuery extends LuceneQuery { + @Getter private final Map> queryBuildActions; - @Override - public QueryBuilder build(FunctionExpression func) { - var arguments = func.getArguments().stream() - .map(a -> (NamedArgumentExpression)a).collect(Collectors.toList()); - if (arguments.size() < 2) { - throw new SyntaxCheckException( - String.format("%s requires at least two parameters", getQueryName())); + protected void ignoreArguments(List arguments) { + arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("field") + || a.getArgName().equalsIgnoreCase("fields") + || a.getArgName().equalsIgnoreCase("query")); + } + + protected void checkValidArguments(String argNormalized, T queryBuilder) { + if (!queryBuildActions.containsKey(argNormalized)) { + throw new SemanticCheckException( + String.format("Parameter %s is invalid for %s function.", + argNormalized, queryBuilder.getWriteableName())); } + } + protected T loadArguments(List arguments) throws SemanticCheckException { // Aggregate parameters by name, so getting a Map arguments.stream().collect(Collectors.groupingBy(a -> a.getArgName().toLowerCase())) .forEach((k, v) -> { @@ -46,32 +54,45 @@ public QueryBuilder build(FunctionExpression func) { T queryBuilder = createQueryBuilder(arguments); - arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("field") - || a.getArgName().equalsIgnoreCase("fields") - || a.getArgName().equalsIgnoreCase("query")); + ignoreArguments(arguments); var iterator = arguments.listIterator(); while (iterator.hasNext()) { NamedArgumentExpression arg = iterator.next(); String argNormalized = arg.getArgName().toLowerCase(); - if (!queryBuildActions.containsKey(argNormalized)) { - throw new SemanticCheckException( - String.format("Parameter %s is invalid for %s function.", - argNormalized, queryBuilder.getWriteableName())); - } + checkValidArguments(argNormalized, queryBuilder); + (Objects.requireNonNull( queryBuildActions .get(argNormalized))) - .apply(queryBuilder, arg.getValue().valueOf(null)); + .apply(queryBuilder, arg.getValue().valueOf()); } + return queryBuilder; } + @Override + public QueryBuilder build(FunctionExpression func) { + var arguments = func.getArguments().stream() + .map(a -> (NamedArgumentExpression)a).collect(Collectors.toList()); + if (arguments.size() < 2) { + throw new SyntaxCheckException( + String.format("%s requires at least two parameters", getQueryName())); + } + + return loadArguments(arguments); + + } + protected abstract T createQueryBuilder(List arguments); protected abstract String getQueryName(); + public Map> getQueryBuildActions() { + return queryBuildActions; + } + /** * Convenience interface for a function that updates a QueryBuilder * based on ExprValue. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java index 15eda7f483..a7d7584d4f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java @@ -36,8 +36,8 @@ protected T createQueryBuilder(List arguments) { .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); return createBuilder( - field.getValue().valueOf(null).stringValue(), - query.getValue().valueOf(null).stringValue()); + field.getValue().valueOf().stringValue(), + query.getValue().valueOf().stringValue()); } protected abstract T createBuilder(String field, String query); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java index 48121baad2..d7ec5bee5e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeRecognitionTest.java @@ -42,6 +42,6 @@ private static Stream types() { } private String typeofGetValue(ExprValue input) { - return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue(); + return dsl.typeof(DSL.literal(input)).valueOf().stringValue(); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index ff80f3bcc0..bae4028986 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -48,7 +48,6 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; @@ -64,18 +63,18 @@ class FilterQueryBuilderTest { private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); private static Stream numericCastSource() { - return Stream.of(literal((byte) 1), literal((short) 1), literal( - 1), literal(1L), literal(1F), literal(1D), literal(true), literal("1")); + return Stream.of(literal((byte) 1), literal((short) -1), literal( + 1), literal(21L), literal(3.14F), literal(3.1415D), literal(true), literal("1")); } private static Stream booleanTrueCastSource() { - return Stream.of(literal((byte) 1), literal((short) 1), literal( - 1), literal(1L), literal(1F), literal(1D), literal(true), literal("true")); + return Stream.of(literal((byte) 1), literal((short) -1), literal( + 1), literal(42L), literal(3.14F), literal(3.1415D), literal(true), literal("true")); } private static Stream booleanFalseCastSource() { return Stream.of(literal((byte) 0), literal((short) 0), literal( - 0), literal(0L), literal(0F), literal(0D), literal(false), literal("false")); + 0), literal(0L), literal(0.0F), literal(0.0D), literal(false), literal("false")); } @Mock @@ -411,6 +410,15 @@ void match_missing_field() { assertEquals("'field' parameter is missing.", msg); } + @Test + void match_missing_query() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("field1")), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'query' parameter is missing", msg); + } + @Test void should_build_match_phrase_query_with_default_parameters() { assertJsonEquals( @@ -626,6 +634,93 @@ void should_build_match_phrase_query_with_custom_parameters() { dsl.namedArgument("zero_terms_query", literal("ALL"))))); } + @Test + void query_invalid_parameter() { + FunctionExpression expr = dsl.query( + dsl.namedArgument("invalid_parameter", literal("invalid_value"))); + assertThrows(SemanticCheckException.class, () -> buildQuery(expr), + "Parameter invalid_parameter is invalid for query function."); + } + + @Test + void query_invalid_fields_parameter_exception_message() { + FunctionExpression expr = dsl.query( + dsl.namedArgument("fields", literal("field1")), + dsl.namedArgument("query", literal("search query"))); + + var exception = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)); + assertEquals("Parameter fields is invalid for query function.", exception.getMessage()); + } + + @Test + void should_build_query_query_with_default_parameters() { + var expected = "{\n" + + " \"query_string\" : {\n" + + " \"query\" : \"field1:query_value\",\n" + + " \"fields\" : [],\n" + + " \"type\" : \"best_fields\",\n" + + " \"default_operator\" : \"or\",\n" + + " \"max_determinized_states\" : 10000,\n" + + " \"enable_position_increments\" : true,\n" + + " \"fuzziness\" : \"AUTO\",\n" + + " \"fuzzy_prefix_length\" : 0,\n" + + " \"fuzzy_max_expansions\" : 50,\n" + + " \"phrase_slop\" : 0,\n" + + " \"escape\" : false,\n" + + " \"auto_generate_synonyms_phrase_query\" : true,\n" + + " \"fuzzy_transpositions\" : true,\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + + assertJsonEquals(expected, buildQuery(dsl.query( + dsl.namedArgument("query", literal("field1:query_value"))))); + } + + @Test + void should_build_query_query_with_custom_parameters() { + var expected = "{\n" + + " \"query_string\" : {\n" + + " \"query\" : \"field1:query_value\",\n" + + " \"fields\" : [],\n" + + " \"type\" : \"cross_fields\",\n" + + " \"tie_breaker\" : 1.3,\n" + + " \"default_operator\" : \"and\",\n" + + " \"analyzer\" : \"keyword\",\n" + + " \"max_determinized_states\" : 10000,\n" + + " \"enable_position_increments\" : true,\n" + + " \"fuzziness\" : \"AUTO\",\n" + + " \"fuzzy_prefix_length\" : 2,\n" + + " \"fuzzy_max_expansions\" : 10,\n" + + " \"phrase_slop\" : 0,\n" + + " \"analyze_wildcard\" : true,\n" + + " \"minimum_should_match\" : \"3\",\n" + + " \"lenient\" : false,\n" + + " \"escape\" : false,\n" + + " \"auto_generate_synonyms_phrase_query\" : false,\n" + + " \"fuzzy_transpositions\" : false,\n" + + " \"boost\" : 2.0,\n" + + " }\n" + + "}"; + var actual = buildQuery( + dsl.query( + dsl.namedArgument("query", literal("field1:query_value")), + dsl.namedArgument("analyze_wildcard", literal("true")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("auto_generate_synonyms_phrase_query", literal("false")), + dsl.namedArgument("default_operator", literal("AND")), + dsl.namedArgument("fuzzy_max_expansions", literal("10")), + dsl.namedArgument("fuzzy_prefix_length", literal("2")), + dsl.namedArgument("fuzzy_transpositions", literal("false")), + dsl.namedArgument("lenient", literal("false")), + dsl.namedArgument("minimum_should_match", literal("3")), + dsl.namedArgument("tie_breaker", literal("1.3")), + dsl.namedArgument("type", literal("cross_fields")), + dsl.namedArgument("boost", literal("2.0")))); + + assertJsonEquals(expected, actual); + } + @Test void query_string_invalid_parameter() { FunctionExpression expr = dsl.query_string( @@ -981,6 +1076,18 @@ void multi_match_missing_fields_even_with_struct() { assertEquals("'fields' parameter is missing.", msg); } + @Test + void multi_match_missing_query_even_with_struct() { + FunctionExpression expr = dsl.multi_match( + dsl.namedArgument("fields", DSL.literal( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "field1", ExprValueUtils.floatValue(1.F), + "field2", ExprValueUtils.floatValue(.3F)))))), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'query' parameter is missing", msg); + } + @Test void should_build_match_phrase_prefix_query_with_default_parameters() { assertJsonEquals( @@ -1042,93 +1149,126 @@ void cast_to_string_in_filter() { dsl.equal(ref("string_value", STRING), dsl.castString(literal("1"))))); } + private Float castToFloat(Object o) { + if (o instanceof Number) { + return ((Number)o).floatValue(); + } + if (o instanceof String) { + return Float.parseFloat((String) o); + } + if (o instanceof Boolean) { + return ((Boolean)o) ? 1F : 0F; + } + // unreachable code + throw new IllegalArgumentException(); + } + + private Integer castToInteger(Object o) { + if (o instanceof Number) { + return ((Number)o).intValue(); + } + if (o instanceof String) { + return Integer.parseInt((String) o); + } + if (o instanceof Boolean) { + return ((Boolean)o) ? 1 : 0; + } + // unreachable code + throw new IllegalArgumentException(); + } + @ParameterizedTest(name = "castByte({0})") @MethodSource({"numericCastSource"}) void cast_to_byte_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"byte_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("byte_value", BYTE), dsl.castByte(expr)))); } @ParameterizedTest(name = "castShort({0})") @MethodSource({"numericCastSource"}) void cast_to_short_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"short_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("short_value", SHORT), dsl.castShort(expr)))); } @ParameterizedTest(name = "castInt({0})") @MethodSource({"numericCastSource"}) void cast_to_int_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"integer_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("integer_value", INTEGER), dsl.castInt(expr)))); } @ParameterizedTest(name = "castLong({0})") @MethodSource({"numericCastSource"}) void cast_to_long_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"long_value\" : {\n" - + " \"value\" : 1,\n" + + " \"value\" : %d,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToInteger(expr.valueOf().value())), buildQuery(dsl.equal(ref("long_value", LONG), dsl.castLong(expr)))); } @ParameterizedTest(name = "castFloat({0})") @MethodSource({"numericCastSource"}) void cast_to_float_in_filter(LiteralExpression expr) { - assertJsonEquals( + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"float_value\" : {\n" - + " \"value\" : 1.0,\n" + + " \"value\" : %f,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", castToFloat(expr.valueOf().value())), buildQuery(dsl.equal(ref("float_value", FLOAT), dsl.castFloat(expr)))); } @ParameterizedTest(name = "castDouble({0})") @MethodSource({"numericCastSource"}) void cast_to_double_in_filter(LiteralExpression expr) { - assertJsonEquals( + // double values affected by floating point imprecision, so we can't compare them in json + // (Double)(Float)3.14 -> 3.14000010490417 + assertEquals(castToFloat(expr.valueOf().value()), + dsl.castDouble(expr).valueOf().doubleValue(), 0.00001); + + assertJsonEquals(String.format( "{\n" + " \"term\" : {\n" + " \"double_value\" : {\n" - + " \"value\" : 1.0,\n" + + " \"value\" : %2.20f,\n" + " \"boost\" : 1.0\n" + " }\n" + " }\n" - + "}", + + "}", dsl.castDouble(expr).valueOf().doubleValue()), buildQuery(dsl.equal(ref("double_value", DOUBLE), dsl.castDouble(expr)))); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java new file mode 100644 index 0000000000..e0681bceac --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryTest.java @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.common.antlr.SyntaxCheckException; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.NamedArgumentExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryQuery; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class QueryTest { + private static final DSL dsl = new ExpressionConfig() + .dsl(new ExpressionConfig().functionRepository()); + private final QueryQuery queryQuery = new QueryQuery(); + private final FunctionName queryFunc = FunctionName.of("query"); + private static final LiteralExpression query_value = DSL.literal("title:query_value"); + + static Stream> generateValidData() { + Expression query = dsl.namedArgument("query", query_value); + return List.of( + dsl.namedArgument("analyzer", DSL.literal("standard")), + dsl.namedArgument("analyze_wildcard", DSL.literal("true")), + dsl.namedArgument("allow_leading_wildcard", DSL.literal("true")), + dsl.namedArgument("auto_generate_synonyms_phrase_query", DSL.literal("true")), + dsl.namedArgument("boost", DSL.literal("1")), + dsl.namedArgument("default_operator", DSL.literal("AND")), + dsl.namedArgument("default_operator", DSL.literal("and")), + dsl.namedArgument("enable_position_increments", DSL.literal("true")), + dsl.namedArgument("escape", DSL.literal("false")), + dsl.namedArgument("fuzziness", DSL.literal("1")), + dsl.namedArgument("fuzzy_rewrite", DSL.literal("constant_score")), + dsl.namedArgument("fuzzy_max_expansions", DSL.literal("42")), + dsl.namedArgument("fuzzy_prefix_length", DSL.literal("42")), + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")), + dsl.namedArgument("lenient", DSL.literal("true")), + dsl.namedArgument("max_determinized_states", DSL.literal("10000")), + dsl.namedArgument("minimum_should_match", DSL.literal("4")), + dsl.namedArgument("quote_analyzer", DSL.literal("standard")), + dsl.namedArgument("phrase_slop", DSL.literal("0")), + dsl.namedArgument("quote_field_suffix", DSL.literal(".exact")), + dsl.namedArgument("rewrite", DSL.literal("constant_score")), + dsl.namedArgument("type", DSL.literal("best_fields")), + dsl.namedArgument("tie_breaker", DSL.literal("0.3")), + dsl.namedArgument("time_zone", DSL.literal("Canada/Pacific")), + dsl.namedArgument("ANALYZER", DSL.literal("standard")), + dsl.namedArgument("ANALYZE_wildcard", DSL.literal("true")), + dsl.namedArgument("Allow_Leading_wildcard", DSL.literal("true")), + dsl.namedArgument("Auto_Generate_Synonyms_Phrase_Query", DSL.literal("true")), + dsl.namedArgument("Boost", DSL.literal("1")) + ).stream().map(arg -> List.of(query, arg)); + } + + @ParameterizedTest + @MethodSource("generateValidData") + public void test_valid_parameters(List validArgs) { + Assertions.assertNotNull(queryQuery.build( + new QueryExpression(validArgs))); + } + + @Test + public void test_SyntaxCheckException_when_no_arguments() { + List arguments = List.of(); + assertThrows(SyntaxCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + @Test + public void test_SyntaxCheckException_when_field_argument() { + List arguments = List.of( + namedArgument("fields", "invalid argument"), + namedArgument("query", query_value)); + assertThrows(SemanticCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + @Test + public void test_SemanticCheckException_when_invalid_parameter() { + List arguments = List.of( + namedArgument("query", query_value), + namedArgument("unsupported", "unsupported_value")); + Assertions.assertThrows(SemanticCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + @Test + public void test_SemanticCheckException_when_sending_parameter_multiple_times() { + List arguments = List.of( + namedArgument("query", query_value), + namedArgument("allow_leading_wildcard", DSL.literal("true")), + namedArgument("allow_leading_wildcard", DSL.literal("true"))); + Assertions.assertThrows(SemanticCheckException.class, + () -> queryQuery.build(new QueryExpression(arguments))); + } + + private NamedArgumentExpression namedArgument(String name, String value) { + return dsl.namedArgument(name, DSL.literal(value)); + } + + private NamedArgumentExpression namedArgument(String name, LiteralExpression value) { + return dsl.namedArgument(name, value); + } + + private class QueryExpression extends FunctionExpression { + public QueryExpression(List arguments) { + super(QueryTest.this.queryFunc, arguments); + } + + @Override + public ExprValue valueOf(Environment valueEnv) { + throw new UnsupportedOperationException("Invalid function call, " + + "valueOf function need implementation only to support Expression interface"); + } + + @Override + public ExprType type() { + throw new UnsupportedOperationException("Invalid function call, " + + "type function need implementation only to support Expression interface"); + } + } + + @Test + public void test_can_get_query_name() { + List arguments = List.of(namedArgument("query", query_value)); + queryQuery.build(new QueryExpression(arguments)); + Assertions.assertEquals("query", + queryQuery.getQueryName()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java new file mode 100644 index 0000000000..e06dd7d32a --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/NoFieldQueryTest.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; + +class NoFieldQueryTest { + NoFieldQuery query; + private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + private final String testQueryName = "test_query"; + private final Map actionMap + = ImmutableMap.of("paramA", (o, v) -> o); + + @BeforeEach + void setUp() { + query = mock(NoFieldQuery.class, + Mockito.withSettings().useConstructor(actionMap) + .defaultAnswer(Mockito.CALLS_REAL_METHODS)); + when(query.getQueryName()).thenReturn(testQueryName); + } + + @Test + void createQueryBuilderTest() { + String sampleQuery = "field:query"; + + query.createQueryBuilder(List.of( + dsl.namedArgument("query", + new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); + + verify(query).createBuilder(eq(sampleQuery)); + } +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java index 20efce1b7a..558e7558ca 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogSettings.java @@ -12,6 +12,6 @@ public class CatalogSettings { public static final Setting CATALOG_CONFIG = SecureSetting.secureFile( - "plugins.query.federation.catalog.config", + "plugins.query.federation.datasources.config", null); } diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java index 07ee458e5c..cdbce55cb1 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java @@ -30,7 +30,7 @@ public class CatalogServiceImplTest { public static final String CATALOG_SETTING_METADATA_KEY = - "plugins.query.federation.catalog.config"; + "plugins.query.federation.datasources.config"; @Mock private StorageEngine storageEngine; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 79c812949f..75b99f2daf 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -43,7 +43,7 @@ SOURCE: 'SOURCE'; INDEX: 'INDEX'; D: 'D'; DESC: 'DESC'; -CATALOGS: 'CATALOGS'; +CATALOGS: 'DATASOURCES'; // CLAUSE KEYWORDS SORTBY: 'SORTBY'; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 658bf1d295..1e6cc5a7b9 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -800,7 +800,7 @@ public void test_batchRCFADCommand() { @Test public void testShowCatalogsCommand() { - assertEqual("show catalogs", + assertEqual("show datasources", relation(".CATALOGS")); } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java index 4a469c7bbb..512c014564 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java @@ -106,7 +106,8 @@ private JSONObject readResponse(Response response) throws IOException { } } else { throw new RuntimeException( - String.format("Request to Prometheus is Unsuccessful with : %s", response.message())); + String.format("Request to Prometheus is Unsuccessful with : %s", Objects.requireNonNull( + response.body(), "Response body can't be null").string())); } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java index bccb9c3bff..2d3710037a 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java @@ -83,7 +83,7 @@ private PrometheusQueryRequest buildQueryFromQueryRangeFunction(List arguments.forEach(arg -> { String argName = ((NamedArgumentExpression) arg).getArgName(); Expression argValue = ((NamedArgumentExpression) arg).getValue(); - ExprValue literalValue = argValue.valueOf(null); + ExprValue literalValue = argValue.valueOf(); switch (argName) { case QUERY: prometheusQueryRequest diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java index e26e006403..ef7f19ba2f 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java @@ -5,7 +5,6 @@ package org.opensearch.sql.prometheus.response; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; @@ -16,6 +15,7 @@ import java.util.LinkedHashMap; import java.util.List; import lombok.NonNull; +import org.apache.commons.lang3.StringUtils; import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.sql.data.model.ExprDoubleValue; @@ -26,6 +26,8 @@ import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; public class PrometheusResponse implements Iterable { @@ -100,7 +102,7 @@ public Iterator iterator() { private void insertLabels(LinkedHashMap linkedHashMap, JSONObject metric) { for (String key : metric.keySet()) { - linkedHashMap.put(key, new ExprStringValue(metric.getString(key))); + linkedHashMap.put(getKey(key), new ExprStringValue(metric.getString(key))); } } @@ -113,4 +115,18 @@ private ExprValue getValue(JSONArray jsonArray, Integer index, ExprType exprType return new ExprDoubleValue(jsonArray.getDouble(index)); } + private String getKey(String key) { + if (this.prometheusResponseFieldNames.getGroupByList() == null) { + return key; + } else { + return this.prometheusResponseFieldNames.getGroupByList().stream() + .filter(expression -> expression.getDelegated() instanceof ReferenceExpression) + .filter(expression + -> ((ReferenceExpression) expression.getDelegated()).getAttr().equals(key)) + .findFirst() + .map(NamedExpression::getName) + .orElse(key); + } + } + } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java index 071cd7ba8c..8cae250e5e 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java @@ -7,10 +7,6 @@ package org.opensearch.sql.prometheus.storage.implementor; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; - -import java.util.ArrayList; import java.util.List; import java.util.Optional; import lombok.RequiredArgsConstructor; @@ -18,14 +14,11 @@ import org.opensearch.sql.common.utils.StringUtils; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.NamedExpression; -import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.span.SpanExpression; import org.opensearch.sql.planner.DefaultImplementor; import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricAgg; import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricScan; import org.opensearch.sql.prometheus.storage.PrometheusMetricScan; @@ -130,6 +123,7 @@ private void setPrometheusResponseFieldNames(PrometheusLogicalMetricAgg node, prometheusResponseFieldNames.setValueFieldName(node.getAggregatorList().get(0).getName()); prometheusResponseFieldNames.setValueType(node.getAggregatorList().get(0).type()); prometheusResponseFieldNames.setTimestampFieldName(spanExpression.get().getNameOrAlias()); + prometheusResponseFieldNames.setGroupByList(node.getGroupByList()); context.setPrometheusResponseFieldNames(prometheusResponseFieldNames); } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java index 4276848aa2..d3a6ef184f 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java @@ -11,9 +11,11 @@ import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; +import java.util.List; import lombok.Getter; import lombok.Setter; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.NamedExpression; @Getter @@ -23,5 +25,6 @@ public class PrometheusResponseFieldNames { private String valueFieldName = VALUE; private ExprType valueType = DOUBLE; private String timestampFieldName = TIMESTAMP; + private List groupByList; } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java index 1aff9eca88..76c8c6872e 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java @@ -7,11 +7,14 @@ package org.opensearch.sql.prometheus.storage.querybuilder; +import java.sql.Ref; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.span.SpanExpression; @@ -63,7 +66,10 @@ public static String build(List namedAggregatorList, if (groupByList.size() > 0) { aggregateQuery.append("by("); aggregateQuery.append( - groupByList.stream().map(NamedExpression::getName).collect(Collectors.joining(", "))); + groupByList.stream() + .filter(expression -> expression.getDelegated() instanceof ReferenceExpression) + .map(expression -> ((ReferenceExpression) expression.getDelegated()).getAttr()) + .collect(Collectors.joining(", "))); aggregateQuery.append(")"); } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java index 6c338d61a6..810ed71379 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java @@ -57,7 +57,7 @@ public Void visitFunction(FunctionExpression func, Object context) { ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); Expression rightExpr = func.getArguments().get(1); if (ref.getAttr().equals("@timestamp")) { - ExprValue literalValue = rightExpr.valueOf(null); + ExprValue literalValue = rightExpr.valueOf(); if (func.getFunctionName().getFunctionName().contains(">")) { startTime = literalValue.timestampValue().toEpochMilli() / 1000; } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java index f2a54b7347..98195eecf7 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java @@ -48,7 +48,7 @@ void testValueOfAndTypeAndToString() { QueryRangeFunctionImplementation queryRangeFunctionImplementation = new QueryRangeFunctionImplementation(functionName, namedArgumentExpressionList, client); UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, - () -> queryRangeFunctionImplementation.valueOf(null)); + () -> queryRangeFunctionImplementation.valueOf()); assertEquals("Prometheus defined function [query_range] is only " + "supported in SOURCE clause with prometheus connector catalog", exception.getMessage()); assertEquals("query_range(query=\"http_latency\", starttime=12345, endtime=12345, step=14)", diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java index ac99a996af..cb70e9e064 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.prometheus.constants.TestConstants.ENDTIME; import static org.opensearch.sql.prometheus.constants.TestConstants.QUERY; import static org.opensearch.sql.prometheus.constants.TestConstants.STARTTIME; @@ -22,6 +23,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.Collections; import java.util.LinkedHashMap; import lombok.SneakyThrows; import org.json.JSONObject; @@ -36,6 +38,7 @@ import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.expression.DSL; import org.opensearch.sql.prometheus.client.PrometheusClient; import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; @@ -163,6 +166,49 @@ void testQueryResponseIteratorWithGivenPrometheusResponseWithLongInAggType() { Assertions.assertFalse(prometheusMetricScan.hasNext()); } + @Test + @SneakyThrows + void testQueryResponseIteratorWithGivenPrometheusResponseWithBackQuotedFieldNames() { + PrometheusResponseFieldNames prometheusResponseFieldNames + = new PrometheusResponseFieldNames(); + prometheusResponseFieldNames.setValueFieldName("testAgg"); + prometheusResponseFieldNames.setValueType(LONG); + prometheusResponseFieldNames.setTimestampFieldName(TIMESTAMP); + prometheusResponseFieldNames.setGroupByList( + Collections.singletonList(DSL.named("`instance`", DSL.ref("instance", STRING)))); + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.setPrometheusResponseFieldNames(prometheusResponseFieldNames); + prometheusMetricScan.getRequest().setPromQl(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("testAgg", new ExprLongValue(1)); + put("`instance`", new ExprStringValue("localhost:9090")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("prometheus")); + } + }); + assertEquals(firstRow, prometheusMetricScan.next()); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("testAgg", new ExprLongValue(0)); + put("`instance`", new ExprStringValue("localhost:9091")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("node")); + } + }); + assertEquals(secondRow, prometheusMetricScan.next()); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + @Test @SneakyThrows void testQueryResponseIteratorForQueryRangeFunction() { @@ -247,6 +293,7 @@ void testEmptyQueryWithException() { runtimeException.getMessage()); } + @Test @SneakyThrows void testExplain() { diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java index 3acae0c493..e106fa225b 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java @@ -750,4 +750,31 @@ void testUnsupportedOperation() { assertThrows(UnsupportedOperationException.class, () -> prometheusMetricTable.create(Collections.emptyMap())); } + + @Test + void testImplementPrometheusQueryWithBackQuotedFieldNamesInStatsQuery() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + // IndexScanAgg with Filter + PhysicalPlan plan = prometheusMetricTable.implement( + indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("`job`", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s"))))); + assertTrue(plan instanceof PrometheusMetricScan); + PrometheusQueryRequest prometheusQueryRequest = ((PrometheusMetricScan) plan).getRequest(); + assertEquals( + "avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + prometheusQueryRequest.getPromQl()); + + } } diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 4dedaaf396..c803f2b5c3 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -329,7 +329,11 @@ specificFunction ; relevanceFunction - : singleFieldRelevanceFunction | multiFieldRelevanceFunction + : noFieldRelevanceFunction | singleFieldRelevanceFunction | multiFieldRelevanceFunction + ; + +noFieldRelevanceFunction + : noFieldRelevanceFunctionName LR_BRACKET query=relevanceQuery (COMMA relevanceArg)* RR_BRACKET ; // Field is a single column @@ -412,6 +416,10 @@ flowControlFunctionName : IF | IFNULL | NULLIF | ISNULL ; +noFieldRelevanceFunctionName + : QUERY + ; + systemFunctionName : TYPEOF ; diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 0d551e3f38..131b6d9116 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -380,6 +380,14 @@ public UnresolvedExpression visitConvertedDataType( return AstDSL.stringLiteral(ctx.getText()); } + @Override + public UnresolvedExpression visitNoFieldRelevanceFunction( + OpenSearchSQLParser.NoFieldRelevanceFunctionContext ctx) { + return new Function( + ctx.noFieldRelevanceFunctionName().getText().toLowerCase(), + noFieldRelevanceArguments(ctx)); + } + @Override public UnresolvedExpression visitSingleFieldRelevanceFunction( SingleFieldRelevanceFunctionContext ctx) { @@ -429,6 +437,24 @@ private QualifiedName visitIdentifiers(List identifiers) { ); } + private void fillRelevanceArgs(List args, + ImmutableList.Builder builder) { + args.forEach(v -> builder.add(new UnresolvedArgument( + v.relevanceArgName().getText().toLowerCase(), new Literal(StringUtils.unquoteText( + v.relevanceArgValue().getText()), DataType.STRING)))); + } + + private List noFieldRelevanceArguments( + OpenSearchSQLParser.NoFieldRelevanceFunctionContext ctx) { + // all the arguments are defaulted to string values + // to skip environment resolving and function signature resolving + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(new UnresolvedArgument("query", + new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); + fillRelevanceArgs(ctx.relevanceArg(), builder); + return builder.build(); + } + private List singleFieldRelevanceArguments( OpenSearchSQLParser.SingleFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values @@ -438,12 +464,12 @@ private List singleFieldRelevanceArguments( new Literal(StringUtils.unquoteText(ctx.field.getText()), DataType.STRING))); builder.add(new UnresolvedArgument("query", new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); - ctx.relevanceArg().forEach(v -> builder.add(new UnresolvedArgument( - v.relevanceArgName().getText().toLowerCase(), new Literal(StringUtils.unquoteText( - v.relevanceArgValue().getText()), DataType.STRING)))); + fillRelevanceArgs(ctx.relevanceArg(), builder); return builder.build(); } + + private List multiFieldRelevanceArguments( OpenSearchSQLParser.MultiFieldRelevanceFunctionContext ctx) { // all the arguments are defaulted to string values @@ -458,9 +484,7 @@ private List multiFieldRelevanceArguments( builder.add(new UnresolvedArgument("fields", fields)); builder.add(new UnresolvedArgument("query", new Literal(StringUtils.unquoteText(ctx.query.getText()), DataType.STRING))); - ctx.relevanceArg().forEach(v -> builder.add(new UnresolvedArgument( - v.relevanceArgName().getText().toLowerCase(), new Literal(StringUtils.unquoteText( - v.relevanceArgValue().getText()), DataType.STRING)))); + fillRelevanceArgs(ctx.relevanceArg(), builder); return builder.build(); } } diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index af428bdc52..6b78376d45 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -325,6 +325,49 @@ public void can_parse_query_string_relevance_function() { + "operator='AND', tie_breaker=0.3, type = \"most_fields\", fuzziness = 4)")); } + + @Test + public void can_parse_query_relevance_function() { + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:query OR notes:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:query OR notes:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:query`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:query OR notes:query`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('*:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"*:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`*:query`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:*uery OR notes:?uery')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:*uery OR notes:?uery\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:*uery OR notes:?uery`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:qu*ry OR notes:qu?ry')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:qu*ry OR notes:qu?ry\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(`address:qu*ry OR notes:qu?ry`)")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query('address:query notes:query')")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE query(\"address:query notes:query\")")); + assertNotNull(parser.parse( + "SELECT id FROM test WHERE " + + "query(\"Body:\'taste beer\' Tags:\'taste beer\' Title:\'taste beer\'\")")); + } + + @Test public void can_parse_match_relevance_function() { assertNotNull(parser.parse("SELECT * FROM test WHERE match(column, \"this is a test\")")); diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java index ec0a0dd0d3..cb00ea2f18 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstExpressionBuilderTest.java @@ -543,6 +543,22 @@ public void relevanceQuery_string() { + "analyzer='keyword', time_zone='Canada/Pacific', tie_breaker='1.3')")); } + @Test + public void relevanceQuery() { + assertEquals(AstDSL.function("query", + unresolvedArg("query", stringLiteral("field1:query OR field2:query"))), + buildExprAst("query('field1:query OR field2:query')") + ); + + assertEquals(AstDSL.function("query", + unresolvedArg("query", stringLiteral("search query")), + unresolvedArg("analyzer", stringLiteral("keyword")), + unresolvedArg("time_zone", stringLiteral("Canada/Pacific")), + unresolvedArg("tie_breaker", stringLiteral("1.3"))), + buildExprAst("query('search query'," + + "analyzer='keyword', time_zone='Canada/Pacific', tie_breaker='1.3')")); + } + @Test public void canBuildInClause() { assertEquals(