From 526b9737a2c63cdd30275a5f6914936ea05ad4c9 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Tue, 11 Oct 2022 21:37:22 -0700 Subject: [PATCH 01/20] Fix workflow and add `name` option. Signed-off-by: Yury-Fridlyand --- .github/workflows/sql-test-workflow.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index 9a20f53b87..b5a0c4c852 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -2,6 +2,13 @@ name: SQL Plugin Tests on: workflow_dispatch: + inputs: + name: + required: false + type: string + +run-name: + ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} jobs: build: @@ -64,10 +71,10 @@ jobs: - name: Verify test results run: | - if [[ -e failures.log ]] + if [[ -e report.log ]] then echo "## FAILED TESTS :facepalm::warning::bangbang:" >> $GITHUB_STEP_SUMMARY - cat failures.log >> $GITHUB_STEP_SUMMARY + cat report.log >> $GITHUB_STEP_SUMMARY exit 1 fi From ba97742b548209ad881715ae3502fb6ef2d932cd Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Thu, 27 Oct 2022 11:14:29 -0700 Subject: [PATCH 02/20] Removed json-smart (#978) Removed json-smart in the jdbc driver as it introduces a CVE (CVE-2021-31684). Signed-off-by: Guian Gumpac --- sql-jdbc/build.gradle | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql-jdbc/build.gradle b/sql-jdbc/build.gradle index 7b3cc71317..1aa135af45 100644 --- a/sql-jdbc/build.gradle +++ b/sql-jdbc/build.gradle @@ -57,9 +57,8 @@ dependencies { testImplementation('org.junit-pioneer:junit-pioneer:0.3.0') testImplementation('org.eclipse.jetty:jetty-server:9.4.48.v20220622') - // Enforce wiremock to use latest guava and json-smart + // Enforce wiremock to use latest guava testImplementation('com.google.guava:guava:31.1-jre') - testImplementation('net.minidev:json-smart:2.4.8') testRuntimeOnly('org.slf4j:slf4j-simple:1.7.25') // capture WireMock logging } From f37736f541baffdfc58f6e1b91e6411577685ab0 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Thu, 27 Oct 2022 11:46:06 -0700 Subject: [PATCH 03/20] Add take() aggregation function in PPL (#949) Signed-off-by: Joshua Li --- .../sql/analysis/ExpressionAnalyzer.java | 10 ++- .../sql/ast/expression/AggregateFunction.java | 1 - .../org/opensearch/sql/expression/DSL.java | 4 + .../aggregation/AggregatorFunction.java | 14 +++ .../aggregation/TakeAggregator.java | 75 ++++++++++++++++ .../function/BuiltinFunctionName.java | 3 + .../sql/analysis/ExpressionAnalyzerTest.java | 8 ++ .../aggregation/TakeAggregatorTest.java | 86 +++++++++++++++++++ docs/user/ppl/cmd/stats.rst | 38 ++++++++ .../value/OpenSearchExprValueFactory.java | 10 ++- .../response/agg/TopHitsParser.java | 34 ++++++++ .../dsl/MetricAggregationBuilder.java | 43 ++++++++-- .../value/OpenSearchExprValueFactoryTest.java | 8 ++ .../response/AggregationResponseUtils.java | 4 + ...enSearchAggregationResponseParserTest.java | 46 ++++++++++ .../dsl/MetricAggregationBuilderTest.java | 68 ++++++++++++++- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 5 ++ .../sql/ppl/parser/AstExpressionBuilder.java | 17 +++- .../ppl/parser/AstExpressionBuilderTest.java | 27 ++++++ 20 files changed, 489 insertions(+), 13 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/aggregation/TakeAggregatorTest.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index b877fcf673..061c4b505f 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -6,6 +6,7 @@ package org.opensearch.sql.analysis; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; @@ -21,6 +22,7 @@ import org.opensearch.sql.ast.expression.AggregateFunction; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.And; +import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; @@ -151,9 +153,13 @@ public Expression visitAggregateFunction(AggregateFunction node, AnalysisContext Optional builtinFunctionName = BuiltinFunctionName.ofAggregation(node.getFuncName()); if (builtinFunctionName.isPresent()) { - Expression arg = node.getField().accept(this, context); + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(node.getField().accept(this, context)); + for (UnresolvedExpression arg : node.getArgList()) { + builder.add(arg.accept(this, context)); + } Aggregator aggregator = (Aggregator) repository.compile( - builtinFunctionName.get().getName(), Collections.singletonList(arg)); + builtinFunctionName.get().getName(), builder.build()); aggregator.distinct(node.getDistinct()); if (node.condition() != null) { aggregator.condition(analyze(node.condition(), context)); diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java b/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java index 4c7389b04e..e8f730d7e9 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/AggregateFunction.java @@ -8,7 +8,6 @@ import java.util.Collections; import java.util.List; -import javax.annotation.Nullable; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 09971cb981..961608d104 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -547,6 +547,10 @@ public Aggregator stddevPop(Expression... expressions) { return aggregate(BuiltinFunctionName.STDDEV_POP, expressions); } + public Aggregator take(Expression... expressions) { + return aggregate(BuiltinFunctionName.TAKE, expressions); + } + public RankingWindowFunction rowNumber() { return (RankingWindowFunction) repository.compile( BuiltinFunctionName.ROW_NUMBER.getName(), Collections.emptyList()); diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunction.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunction.java index 172e1ee778..9fbf1557aa 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/AggregatorFunction.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.aggregation; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; @@ -20,6 +21,7 @@ import static org.opensearch.sql.expression.aggregation.VarianceAggregator.variancePopulation; import static org.opensearch.sql.expression.aggregation.VarianceAggregator.varianceSample; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Collections; import java.util.stream.Collectors; @@ -57,6 +59,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(varPop()); repository.register(stddevSamp()); repository.register(stddevPop()); + repository.register(take()); } private static DefaultFunctionResolver avg() { @@ -192,4 +195,15 @@ private static DefaultFunctionResolver stddevPop() { .build() ); } + + private static DefaultFunctionResolver take() { + FunctionName functionName = BuiltinFunctionName.TAKE.getName(); + DefaultFunctionResolver functionResolver = new DefaultFunctionResolver(functionName, + new ImmutableMap.Builder() + .put(new FunctionSignature(functionName, ImmutableList.of(STRING, INTEGER)), + arguments -> new TakeAggregator(arguments, ARRAY)) + .build()); + return functionResolver; + } + } diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java new file mode 100644 index 0000000000..4ac0991979 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.aggregation; + +import static org.opensearch.sql.utils.ExpressionUtils.format; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.opensearch.sql.data.model.ExprCollectionValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.function.BuiltinFunctionName; + +/** + * The take aggregator keeps and returns the original values of a field. + * If the field value is NULL or MISSING, then it is skipped. + */ +public class TakeAggregator extends Aggregator { + + public TakeAggregator(List arguments, ExprCoreType returnType) { + super(BuiltinFunctionName.TAKE.getName(), arguments, returnType); + } + + @Override + public TakeState create() { + return new TakeState(getArguments().get(1).valueOf(null).integerValue()); + } + + @Override + protected TakeState iterate(ExprValue value, TakeState state) { + state.take(value); + return state; + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "take(%s)", format(getArguments())); + } + + /** + * Take State. + */ + protected static class TakeState implements AggregationState { + protected int index; + protected int size; + protected List hits; + + TakeState(int size) { + if (size <= 0) { + throw new IllegalArgumentException("size must be greater than 0"); + } + this.index = 0; + this.size = size; + this.hits = new ArrayList<>(); + } + + public void take(ExprValue value) { + if (index < size) { + hits.add(value); + } + index++; + } + + @Override + public ExprValue result() { + return new ExprCollectionValue(hits); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 093d66b01f..a7d7b74865 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -144,6 +144,8 @@ public enum BuiltinFunctionName { STDDEV_SAMP(FunctionName.of("stddev_samp")), // population standard deviation. STDDEV_POP(FunctionName.of("stddev_pop")), + // take top documents from aggregation bucket. + TAKE(FunctionName.of("take")), /** * Text Functions. @@ -244,6 +246,7 @@ public enum BuiltinFunctionName { .put("stddev", BuiltinFunctionName.STDDEV_POP) .put("stddev_pop", BuiltinFunctionName.STDDEV_POP) .put("stddev_samp", BuiltinFunctionName.STDDEV_SAMP) + .put("take", BuiltinFunctionName.TAKE) .build(); public static Optional of(String str) { diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index c76f449357..d1cef1edb3 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -315,6 +315,14 @@ public void filtered_distinct_count() { ); } + @Test + public void take_aggregation() { + assertAnalyzeEqual( + dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)), + AstDSL.aggregate("take", qualifiedName("string_value"), intLiteral(10)) + ); + } + @Test public void named_argument() { assertAnalyzeEqual( diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/TakeAggregatorTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/TakeAggregatorTest.java new file mode 100644 index 0000000000..900d0d1963 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/TakeAggregatorTest.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.aggregation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.expression.DSL; + +class TakeAggregatorTest extends AggregationTest { + + @Test + public void take_string_field_expression() { + ExprValue result = + aggregation(dsl.take(DSL.ref("string_value", STRING), DSL.literal(2)), tuples); + assertEquals(ImmutableList.of("m", "f"), result.value()); + } + + @Test + public void take_string_field_expression_with_large_size() { + ExprValue result = + aggregation(dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)), tuples); + assertEquals(ImmutableList.of("m", "f", "m", "n"), result.value()); + } + + @Test + public void filtered_take() { + ExprValue result = + aggregation(dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)) + .condition(dsl.equal(DSL.ref("string_value", STRING), DSL.literal("m"))), tuples); + assertEquals(ImmutableList.of("m", "m"), result.value()); + } + + @Test + public void test_take_null() { + ExprValue result = + aggregation(dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)), + tuples_with_null_and_missing); + assertEquals(ImmutableList.of("m", "f"), result.value()); + } + + @Test + public void test_take_missing() { + ExprValue result = + aggregation(dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)), + tuples_with_null_and_missing); + assertEquals(ImmutableList.of("m", "f"), result.value()); + } + + @Test + public void test_take_all_missing_or_null() { + ExprValue result = + aggregation(dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)), + tuples_with_all_null_or_missing); + assertEquals(ImmutableList.of(), result.value()); + } + + @Test + public void test_take_with_invalid_size() { + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> aggregation(dsl.take(DSL.ref("string_value", STRING), DSL.literal(0)), tuples)); + assertEquals("size must be greater than 0", exception.getMessage()); + } + + @Test + public void test_value_of() { + ExpressionEvaluationException exception = assertThrows(ExpressionEvaluationException.class, + () -> dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)).valueOf(valueEnv())); + assertEquals("can't evaluate on aggregator: take", exception.getMessage()); + } + + @Test + public void test_to_string() { + Aggregator takeAggregator = dsl.take(DSL.ref("string_value", STRING), DSL.literal(10)); + assertEquals("take(string_value,10)", takeAggregator.toString()); + } +} diff --git a/docs/user/ppl/cmd/stats.rst b/docs/user/ppl/cmd/stats.rst index dd7220d77f..3a34e68a7d 100644 --- a/docs/user/ppl/cmd/stats.rst +++ b/docs/user/ppl/cmd/stats.rst @@ -238,6 +238,27 @@ Example:: | 2.8613807855648994 | +--------------------+ +TAKE +---------- + +Description +>>>>>>>>>>> + +Usage: TAKE(field [, size]). Return original values of a field. It does not guarantee on the order of values. + +* field: mandatory. The field must be a text field. +* size: optional integer. The number of values should be returned. Default is 10. + +Example:: + + os> source=accounts | stats take(firstname); + fetched rows / total rows = 1/1 + +-----------------------------+ + | take(firstname) | + |-----------------------------| + | [Amber,Hattie,Nanette,Dale] | + +-----------------------------+ + Example 1: Calculate the count of events ======================================== @@ -381,3 +402,20 @@ PPL query:: | 2 | 30 | M | | 1 | 35 | M | +-------+------------+----------+ + +Example 10: Calculate the count and get email list by a gender and span +======================================================================= + +The example gets the count of age by the interval of 10 years and group by gender, additionally for each row get a list of at most 5 emails. + +PPL query:: + + os> source=accounts | stats count() as cnt, take(email, 5) by span(age, 5) as age_span, gender + fetched rows / total rows = 3/3 + +-------+--------------------------------------------+------------+----------+ + | cnt | take(email, 5) | age_span | gender | + |-------+--------------------------------------------+------------+----------| + | 1 | [] | 25 | F | + | 2 | [amberduke@pyrami.com,daleadams@boink.com] | 30 | M | + | 1 | [hattiebond@netagy.com] | 35 | M | + +-------+--------------------------------------------+------------+----------+ diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 008cbb7ec3..2536121e91 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -214,7 +214,15 @@ private ExprValue parseStruct(Content content, String prefix) { */ private ExprValue parseArray(Content content, String prefix) { List result = new ArrayList<>(); - content.array().forEachRemaining(v -> result.add(parse(v, prefix, Optional.of(STRUCT)))); + content.array().forEachRemaining(v -> { + // ExprCoreType.ARRAY does not indicate inner elements type. OpenSearch nested will be an + // array of structs, otherwise parseArray currently only supports array of strings. + if (v.isString()) { + result.add(parse(v, prefix, Optional.of(STRING))); + } else { + result.add(parse(v, prefix, Optional.of(STRUCT))); + } + }); return new ExprCollectionValue(result); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java new file mode 100644 index 0000000000..4a3a346a84 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.search.aggregations.Aggregation; +import org.opensearch.search.aggregations.metrics.TopHits; + +/** + * {@link TopHits} metric parser. + */ +@RequiredArgsConstructor +public class TopHitsParser implements MetricParser { + + @Getter + private final String name; + + @Override + public Map parse(Aggregation agg) { + return Collections.singletonMap( + agg.getName(), + Arrays.stream(((TopHits) agg).getHits().getHits()) + .flatMap(h -> h.getSourceAsMap().values().stream()).collect(Collectors.toList())); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java index 4c316a076e..f4ff22dfbf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java @@ -18,6 +18,7 @@ import org.opensearch.search.aggregations.bucket.filter.FilterAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStats; +import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; @@ -28,6 +29,7 @@ import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.response.agg.StatsParser; +import org.opensearch.sql.opensearch.response.agg.TopHitsParser; import org.opensearch.sql.opensearch.storage.script.filter.FilterQueryBuilder; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -132,28 +134,36 @@ public Pair visitNamedAggregator( expression, condition, name, - new StatsParser(ExtendedStats::getVarianceSampling,name)); + new StatsParser(ExtendedStats::getVarianceSampling, name)); case "var_pop": return make( AggregationBuilders.extendedStats(name), expression, condition, name, - new StatsParser(ExtendedStats::getVariancePopulation,name)); + new StatsParser(ExtendedStats::getVariancePopulation, name)); case "stddev_samp": return make( AggregationBuilders.extendedStats(name), expression, condition, name, - new StatsParser(ExtendedStats::getStdDeviationSampling,name)); + new StatsParser(ExtendedStats::getStdDeviationSampling, name)); case "stddev_pop": return make( AggregationBuilders.extendedStats(name), expression, condition, name, - new StatsParser(ExtendedStats::getStdDeviationPopulation,name)); + new StatsParser(ExtendedStats::getStdDeviationPopulation, name)); + case "take": + return make( + AggregationBuilders.topHits(name), + expression, + node.getArguments().get(1), + condition, + name, + new TopHitsParser(name)); default: throw new IllegalStateException( String.format("unsupported aggregator %s", node.getFunctionName().getFunctionName())); @@ -194,6 +204,27 @@ private Pair make(CardinalityAggregationBuilde return Pair.of(aggregationBuilder, parser); } + /** + * Make {@link TopHitsAggregationBuilder} for take aggregations. + */ + private Pair make(TopHitsAggregationBuilder builder, + Expression expression, + Expression size, + Expression condition, + String name, + MetricParser parser) { + String fieldName = ((ReferenceExpression) expression).getAttr(); + builder.fetchSource(fieldName, null); + builder.size(size.valueOf(null).integerValue()); + builder.from(0); + if (condition != null) { + return Pair.of( + makeFilterAggregation(builder, condition, name), + FilterParser.builder().name(name).metricsParser(parser).build()); + } + return Pair.of(builder, parser); + } + /** * Replace star or literal with OpenSearch metadata field "_index". Because: 1) Analyzer already * converts * to string literal, literal check here can handle both COUNT(*) and COUNT(1). 2) @@ -214,8 +245,8 @@ private Expression replaceStarOrLiteral(Expression countArg) { * Make builder to build FilterAggregation for aggregations with filter in the bucket. * * @param subAggBuilder AggregationBuilder instance which the filter is applied to. - * @param condition Condition expression in the filter. - * @param name Name of the FilterAggregation instance to build. + * @param condition Condition expression in the filter. + * @param name Name of the FilterAggregation instance to build. * @return {@link FilterAggregationBuilder}. */ private FilterAggregationBuilder makeFilterAggregation( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 259ea0ea5a..8d5552d6a8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -49,6 +49,7 @@ import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; +import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; @@ -247,6 +248,13 @@ public void constructArray() { ImmutableMap.of("info", "zz", "author", "au")))); } + @Test + public void constructArrayOfStrings() { + assertEquals(new ExprCollectionValue( + ImmutableList.of(new ExprStringValue("zz"), new ExprStringValue("au"))), + constructFromObject("arrayV", ImmutableList.of("zz", "au"))); + } + @Test public void constructStruct() { assertEquals( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java index 2bd0e10eb0..bb8ab11c15 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/AggregationResponseUtils.java @@ -43,8 +43,10 @@ import org.opensearch.search.aggregations.metrics.ParsedMax; import org.opensearch.search.aggregations.metrics.ParsedMin; import org.opensearch.search.aggregations.metrics.ParsedSum; +import org.opensearch.search.aggregations.metrics.ParsedTopHits; import org.opensearch.search.aggregations.metrics.ParsedValueCount; import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; +import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.opensearch.search.aggregations.pipeline.ParsedPercentilesBucket; import org.opensearch.search.aggregations.pipeline.PercentilesBucketPipelineAggregationBuilder; @@ -73,6 +75,8 @@ public class AggregationResponseUtils { (p, c) -> ParsedComposite.fromXContent(p, (String) c)) .put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)) + .put(TopHitsAggregationBuilder.NAME, + (p, c) -> ParsedTopHits.fromXContent(p, (String) c)) .build() .entrySet() .stream() diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java index 7a40f4f928..5108e03ad9 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.opensearch.response.AggregationResponseUtils.fromJson; import static org.opensearch.sql.opensearch.response.agg.Utils.handleNanValue; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.Map; @@ -28,6 +29,7 @@ import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; import org.opensearch.sql.opensearch.response.agg.StatsParser; +import org.opensearch.sql.opensearch.response.agg.TopHitsParser; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class OpenSearchAggregationResponseParserTest { @@ -270,6 +272,50 @@ void no_bucket_max_and_extended_stats() { contains(entry("esField", 93.71390409320287, "maxField", 360D))); } + @Test + void top_hits_aggregation_should_pass() { + String response = "{\n" + + " \"composite#composite_buckets\": {\n" + + " \"buckets\": [\n" + + " {\n" + + " \"key\": {\n" + + " \"type\": \"take\"\n" + + " },\n" + + " \"doc_count\": 2,\n" + + " \"top_hits#take\": {\n" + + " \"hits\": {\n" + + " \"total\": { \"value\": 2, \"relation\": \"eq\" },\n" + + " \"max_score\": 1.0,\n" + + " \"hits\": [\n" + + " {\n" + + " \"_index\": \"accounts\",\n" + + " \"_id\": \"1\",\n" + + " \"_score\": 1.0,\n" + + " \"_source\": {\n" + + " \"gender\": \"m\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"_index\": \"accounts\",\n" + + " \"_id\": \"2\",\n" + + " \"_score\": 1.0,\n" + + " \"_source\": {\n" + + " \"gender\": \"f\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + "}"; + OpenSearchAggregationResponseParser parser = + new CompositeAggregationParser(new TopHitsParser("take")); + assertThat(parse(parser, response), + contains(ImmutableMap.of("type", "take", "take", ImmutableList.of("m", "f")))); + } + public List> parse(OpenSearchAggregationResponseParser parser, String json) { return parser.parse(fromJson(json)); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 5161b35021..b2ad41d516 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.when; import static org.opensearch.sql.common.utils.StringUtils.format; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; @@ -21,6 +22,7 @@ import static org.opensearch.sql.expression.aggregation.VarianceAggregator.varianceSample; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -39,6 +41,7 @@ import org.opensearch.sql.expression.aggregation.MinAggregator; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.aggregation.SumAggregator; +import org.opensearch.sql.expression.aggregation.TakeAggregator; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.function.FunctionName; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @@ -288,6 +291,69 @@ void should_build_filtered_cardinality_aggregation() { .distinct(true))))); } + @Test + void should_build_top_hits_aggregation() { + assertEquals(format( + "{%n" + + " \"take(name, 10)\" : {%n" + + " \"top_hits\" : {%n" + + " \"from\" : 0,%n" + + " \"size\" : 10,%n" + + " \"version\" : false,%n" + + " \"seq_no_primary_term\" : false,%n" + + " \"explain\" : false,%n" + + " \"_source\" : {%n" + + " \"includes\" : [ \"name\" ],%n" + + " \"excludes\" : [ ]%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), + buildQuery( + Collections.singletonList(named("take(name, 10)", new TakeAggregator( + ImmutableList.of(ref("name", STRING), literal(10)), ARRAY))))); + } + + @Test + void should_build_filtered_top_hits_aggregation() { + assertEquals(format( + "{%n" + + " \"take(name, 10) filter(where age > 30)\" : {%n" + + " \"filter\" : {%n" + + " \"range\" : {%n" + + " \"age\" : {%n" + + " \"from\" : 30,%n" + + " \"to\" : null,%n" + + " \"include_lower\" : false,%n" + + " \"include_upper\" : true,%n" + + " \"boost\" : 1.0%n" + + " }%n" + + " }%n" + + " },%n" + + " \"aggregations\" : {%n" + + " \"take(name, 10) filter(where age > 30)\" : {%n" + + " \"top_hits\" : {%n" + + " \"from\" : 0,%n" + + " \"size\" : 10,%n" + + " \"version\" : false,%n" + + " \"seq_no_primary_term\" : false,%n" + + " \"explain\" : false,%n" + + " \"_source\" : {%n" + + " \"includes\" : [ \"name\" ],%n" + + " \"excludes\" : [ ]%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + " }%n" + + "}"), + buildQuery(Collections.singletonList(named( + "take(name, 10) filter(where age > 30)", + new TakeAggregator( + ImmutableList.of(ref("name", STRING), literal(10)), ARRAY) + .condition(dsl.greater(ref("age", INTEGER), literal(30))))))); + } + @Test void should_throw_exception_for_unsupported_distinct_aggregator() { assertThrows(IllegalStateException.class, @@ -322,7 +388,7 @@ void should_throw_exception_for_unsupported_exception() { private String buildQuery(List namedAggregatorList) { ObjectMapper objectMapper = new ObjectMapper(); return objectMapper.readTree( - aggregationBuilder.build(namedAggregatorList).getLeft().toString()) + aggregationBuilder.build(namedAggregatorList).getLeft().toString()) .toPrettyString(); } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 17e308cbc9..fb8194d65c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -180,6 +180,7 @@ VAR_POP: 'VAR_POP'; STDDEV_SAMP: 'STDDEV_SAMP'; STDDEV_POP: 'STDDEV_POP'; PERCENTILE: 'PERCENTILE'; +TAKE: 'TAKE'; FIRST: 'FIRST'; LAST: 'LAST'; LIST: 'LIST'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a1061b0020..4dfac10dc2 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -202,12 +202,17 @@ statsFunction | COUNT LT_PRTHS RT_PRTHS #countAllFunctionCall | (DISTINCT_COUNT | DC) LT_PRTHS valueExpression RT_PRTHS #distinctCountFunctionCall | percentileAggFunction #percentileAggFunctionCall + | takeAggFunction #takeAggFunctionCall ; statsFunctionName : AVG | COUNT | SUM | MIN | MAX | VAR_SAMP | VAR_POP | STDDEV_SAMP | STDDEV_POP ; +takeAggFunction + : TAKE LT_PRTHS fieldExpression (COMMA size=integerLiteral)? RT_PRTHS + ; + percentileAggFunction : PERCENTILE LESS value=integerLiteral GREATER LT_PRTHS aggField=fieldExpression RT_PRTHS ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 5df1c4ec56..d2e6131cbe 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -90,6 +90,8 @@ */ public class AstExpressionBuilder extends OpenSearchPPLParserBaseVisitor { + private static final int DEFAULT_TAKE_FUNCTION_SIZE_VALUE = 10; + /** * The function name mapping between fronted and core engine. */ @@ -214,6 +216,17 @@ public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionCont Collections.singletonList(new Argument("rank", (Literal) visit(ctx.value)))); } + @Override + public UnresolvedExpression visitTakeAggFunctionCall( + OpenSearchPPLParser.TakeAggFunctionCallContext ctx) { + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(new UnresolvedArgument("size", + ctx.takeAggFunction().size != null ? visit(ctx.takeAggFunction().size) : + AstDSL.intLiteral(DEFAULT_TAKE_FUNCTION_SIZE_VALUE))); + return new AggregateFunction("take", visit(ctx.takeAggFunction().fieldExpression()), + builder.build()); + } + /** * Eval function. */ @@ -259,8 +272,8 @@ private UnresolvedExpression visitConstantFunction(String functionName, FunctionArgsContext args) { return new ConstantFunction(functionName, args == null - ? Collections.emptyList() - : args.functionArg() + ? Collections.emptyList() + : args.functionArg() .stream() .map(this::visitFunctionArg) .collect(Collectors.toList())); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 1becf086ac..5073aa1459 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -477,6 +477,33 @@ public void testDistinctCount() { defaultStatsArgs())); } + @Test + public void testTakeAggregationNoArgsShouldPass() { + assertEqual("source=t | stats take(a)", + agg( + relation("t"), + exprList(alias("take(a)", + aggregate("take", field("a"), unresolvedArg("size", intLiteral(10))))), + emptyList(), + emptyList(), + defaultStatsArgs() + )); + } + + @Test + public void testTakeAggregationWithArgsShouldPass() { + assertEqual("source=t | stats take(a, 5)", + agg( + relation("t"), + exprList(alias("take(a, 5)", + aggregate("take", field("a"), unresolvedArg("size", intLiteral(5))))), + emptyList(), + emptyList(), + defaultStatsArgs() + )); + } + + @Test public void testEvalFuncCallExpr() { assertEqual("source=t | eval f=abs(a)", From b8436392b4bdac546d098d6b62a5ca862e11cbb7 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Thu, 27 Oct 2022 15:15:12 -0700 Subject: [PATCH 04/20] SHOW CATALOGS documentation and integ tests (#977) Signed-off-by: Vamsi Manohar --- docs/category.json | 1 + docs/user/ppl/cmd/showcatalogs.rst | 36 +++++++++++++++ docs/user/ppl/index.rst | 2 + doctest/build.gradle | 9 ++-- doctest/catalog/catalog.json | 2 +- integ-test/build.gradle | 30 ++++++++++++ .../sql/ppl/ShowCatalogsCommandIT.java | 46 +++++++++++++++++++ .../src/test/resources/catalog/catalog.json | 7 +++ 8 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 docs/user/ppl/cmd/showcatalogs.rst create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java create mode 100644 integ-test/src/test/resources/catalog/catalog.json diff --git a/docs/category.json b/docs/category.json index 3f5bdb0a23..203b641ec2 100644 --- a/docs/category.json +++ b/docs/category.json @@ -10,6 +10,7 @@ "user/ppl/cmd/ad.rst", "user/ppl/cmd/dedup.rst", "user/ppl/cmd/describe.rst", + "user/ppl/cmd/showcatalogs.rst", "user/ppl/cmd/eval.rst", "user/ppl/cmd/fields.rst", "user/ppl/cmd/grok.rst", diff --git a/docs/user/ppl/cmd/showcatalogs.rst b/docs/user/ppl/cmd/showcatalogs.rst new file mode 100644 index 0000000000..d304cba768 --- /dev/null +++ b/docs/user/ppl/cmd/showcatalogs.rst @@ -0,0 +1,36 @@ +============= +show catalogs +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Using ``show catalogs`` command to query catalogs configured in the PPL engine. ``show catalogs`` command could be only used as the first command in the PPL query. + + +Syntax +============ +show catalogs + + +Example 1: Fetch all PROMETHEUS catalogs +================================= + +The example fetches all the catalogs configured. + +PPL query for all PROMETHEUS CATALOGS:: + + os> show catalogs | where CONNECTOR_TYPE='PROMETHEUS'; + fetched rows / total rows = 1/1 + +----------------+------------------+ + | CATALOG_NAME | CONNECTOR_TYPE | + |----------------+------------------| + | my_prometheus | PROMETHEUS | + +----------------+------------------+ + diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 2ec974ea28..0af43c3b33 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -46,6 +46,8 @@ The query start with search command and then flowing a set of command delimited - `describe command `_ + - `show catalogs command `_ + - `eval command `_ - `fields command `_ diff --git a/doctest/build.gradle b/doctest/build.gradle index 69fac44d95..bf789e7434 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -62,9 +62,11 @@ task doctest(type: Exec, dependsOn: ['bootstrap']) { } } -task stopOpenSearch(type: KillProcessTask) { +task stopOpenSearch(type: KillProcessTask) - finalizedBy { +task stopPrometheus() { + + doLast { def pidFile = new File(path, ".prom.pid.lock") if(!pidFile.exists()) { logger.quiet "No Prometheus server running!" @@ -79,13 +81,14 @@ task stopOpenSearch(type: KillProcessTask) { } finally { pidFile.delete() } - println("Killed Prometheus") } } +stopPrometheus.mustRunAfter startPrometheus doctest.dependsOn startOpenSearch startOpenSearch.dependsOn startPrometheus doctest.finalizedBy stopOpenSearch +stopOpenSearch.finalizedBy stopPrometheus build.dependsOn doctest clean.dependsOn(cleanBootstrap) diff --git a/doctest/catalog/catalog.json b/doctest/catalog/catalog.json index 5d9a6be612..84038fe83b 100644 --- a/doctest/catalog/catalog.json +++ b/doctest/catalog/catalog.json @@ -1,6 +1,6 @@ [ { - "name" : "prometheus", + "name" : "my_prometheus", "connector": "prometheus", "uri" : "http://localhost:9090" } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 5e0a53bf1a..4ee35bac2e 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -28,11 +28,16 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask import java.util.concurrent.Callable +plugins { + id "de.undercouch.download" version "5.3.0" +} apply plugin: 'opensearch.build' apply plugin: 'opensearch.rest-test' apply plugin: 'java' apply plugin: 'io.freefair.lombok' +apply plugin: 'com.wiredforcode.spawn' + ext { projectSubstitutions = [:] @@ -101,11 +106,36 @@ testClusters.all { testClusters.integTest { plugin ":opensearch-sql-plugin" + keystore 'plugins.query.federation.catalog.config', new File("$projectDir/src/test/resources/catalog/", 'catalog.json') +} +task startPrometheus(type: SpawnProcessTask) { + doFirst { + download.run { + src 'https://github.com/prometheus/prometheus/releases/download/v2.39.1/prometheus-2.39.1.linux-amd64.tar.gz' + dest new File("$projectDir/bin", 'prometheus.tar.gz') + } + copy { + from tarTree("$projectDir/bin/prometheus.tar.gz") + into "$projectDir/bin" + } + copy { + from "$projectDir/bin/prometheus.yml" + into "$projectDir/bin/prometheus-2.39.1.linux-amd64/prometheus" + } + } + command "$projectDir/bin/prometheus-2.39.1.linux-amd64/prometheus --storage.tsdb.path=$projectDir/bin/prometheus-2.39.1.linux-amd64/data --config.file=$projectDir/bin/prometheus-2.39.1.linux-amd64/prometheus.yml" + ready 'TSDB started' } +task stopPrometheus(type: KillProcessTask) + +stopPrometheus.mustRunAfter startPrometheus + // Run PPL ITs and new, legacy and comparison SQL ITs with new SQL engine enabled integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' + dependsOn startPrometheus + finalizedBy stopPrometheus systemProperty 'tests.security.manager', 'false' systemProperty('project.root', project.projectDir.absolutePath) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java new file mode 100644 index 0000000000..11b94f8da6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java @@ -0,0 +1,46 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class ShowCatalogsCommandIT extends PPLIntegTestCase { + + @Test + public void testShowCatalogsCommands() throws IOException { + JSONObject result = executeQuery("show catalogs"); + verifyDataRows(result, + rows("my_prometheus", "PROMETHEUS"), + rows(".opensearch", "OPENSEARCH")); + verifyColumn( + result, + columnName("CATALOG_NAME"), + columnName("CONNECTOR_TYPE") + ); + } + + @Test + public void testShowCatalogsCommandsWithWhereClause() throws IOException { + JSONObject result = executeQuery("show catalogs | where CONNECTOR_TYPE='PROMETHEUS'"); + verifyDataRows(result, + rows("my_prometheus", "PROMETHEUS")); + verifyColumn( + result, + columnName("CATALOG_NAME"), + columnName("CONNECTOR_TYPE") + ); + } + +} diff --git a/integ-test/src/test/resources/catalog/catalog.json b/integ-test/src/test/resources/catalog/catalog.json new file mode 100644 index 0000000000..84038fe83b --- /dev/null +++ b/integ-test/src/test/resources/catalog/catalog.json @@ -0,0 +1,7 @@ +[ + { + "name" : "my_prometheus", + "connector": "prometheus", + "uri" : "http://localhost:9090" + } +] \ No newline at end of file From d46b389c9ec725b7d9baf2bf824ec8910ff1fccc Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 28 Oct 2022 10:42:13 -0700 Subject: [PATCH 05/20] Add functions `PERIOD_ADD` and `PERIOD_DIFF`. (#933) * Add functions `PERIOD_ADD` and `PERIOD_DIFF`. Signed-off-by: Yury-Fridlyand * Update signature - use `INT` instead of `LONG`. Signed-off-by: Yury-Fridlyand Signed-off-by: Yury-Fridlyand --- .../expression/datetime/DateTimeFunction.java | 235 ++++++++++++------ .../function/BuiltinFunctionName.java | 2 + .../expression/datetime/DateTimeTestBase.java | 103 +++++--- .../datetime/PeriodFunctionsTest.java | 103 ++++++++ docs/user/dql/functions.rst | 46 ++++ docs/user/ppl/functions/datetime.rst | 46 ++++ .../sql/ppl/DateTimeFunctionIT.java | 28 ++- .../sql/sql/DateTimeFunctionIT.java | 44 +++- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 4 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 9 +- sql/src/main/antlr/OpenSearchSQLLexer.g4 | 6 +- sql/src/main/antlr/OpenSearchSQLParser.g4 | 9 +- 12 files changed, 488 insertions(+), 147 deletions(-) create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 84716a425a..5b1af29590 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.datetime; +import static java.time.temporal.ChronoUnit.MONTHS; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; @@ -68,6 +69,7 @@ * 2) the implementation should rely on ExprValue. */ @UtilityClass +@SuppressWarnings("unchecked") public class DateTimeFunction { // The number of days from year zero to year 1970. private static final Long DAYS_0000_TO_1970 = (146097 * 5L) - (30L * 365L + 7L); @@ -84,6 +86,11 @@ public class DateTimeFunction { public void register(BuiltinFunctionRepository repository) { repository.register(adddate()); repository.register(convert_tz()); + repository.register(curtime()); + repository.register(curdate()); + repository.register(current_date()); + repository.register(current_time()); + repository.register(current_timestamp()); repository.register(date()); repository.register(datetime()); repository.register(date_add()); @@ -96,15 +103,21 @@ public void register(BuiltinFunctionRepository repository) { repository.register(from_days()); repository.register(from_unixtime()); repository.register(hour()); + repository.register(localtime()); + repository.register(localtimestamp()); repository.register(makedate()); repository.register(maketime()); repository.register(microsecond()); repository.register(minute()); repository.register(month()); repository.register(monthName()); + repository.register(now()); + repository.register(period_add()); + repository.register(period_diff()); repository.register(quarter()); repository.register(second()); repository.register(subdate()); + repository.register(sysdate()); repository.register(time()); repository.register(time_to_sec()); repository.register(timestamp()); @@ -113,84 +126,6 @@ public void register(BuiltinFunctionRepository repository) { repository.register(unix_timestamp()); repository.register(week()); repository.register(year()); - - repository.register(now()); - repository.register(current_timestamp()); - repository.register(localtimestamp()); - repository.register(localtime()); - repository.register(sysdate()); - repository.register(curtime()); - repository.register(current_time()); - repository.register(curdate()); - repository.register(current_date()); - } - - /** - * NOW() returns a constant time that indicates the time at which the statement began to execute. - * `fsp` argument support is removed until refactoring to avoid bug where `now()`, `now(x)` and - * `now(y) return different values. - */ - private FunctionResolver now(FunctionName functionName) { - return define(functionName, - impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME) - ); - } - - private FunctionResolver now() { - return now(BuiltinFunctionName.NOW.getName()); - } - - private FunctionResolver current_timestamp() { - return now(BuiltinFunctionName.CURRENT_TIMESTAMP.getName()); - } - - private FunctionResolver localtimestamp() { - return now(BuiltinFunctionName.LOCALTIMESTAMP.getName()); - } - - private FunctionResolver localtime() { - return now(BuiltinFunctionName.LOCALTIME.getName()); - } - - /** - * SYSDATE() returns the time at which it executes. - */ - private FunctionResolver sysdate() { - return define(BuiltinFunctionName.SYSDATE.getName(), - impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME), - impl((v) -> new ExprDatetimeValue(formatNow(v.integerValue())), DATETIME, INTEGER) - ); - } - - /** - * Synonym for @see `now`. - */ - private FunctionResolver curtime(FunctionName functionName) { - return define(functionName, - impl(() -> new ExprTimeValue(formatNow(null).toLocalTime()), TIME) - ); - } - - private FunctionResolver curtime() { - return curtime(BuiltinFunctionName.CURTIME.getName()); - } - - private FunctionResolver current_time() { - return curtime(BuiltinFunctionName.CURRENT_TIME.getName()); - } - - private FunctionResolver curdate(FunctionName functionName) { - return define(functionName, - impl(() -> new ExprDateValue(formatNow(null).toLocalDate()), DATE) - ); - } - - private FunctionResolver curdate() { - return curdate(BuiltinFunctionName.CURDATE.getName()); - } - - private FunctionResolver current_date() { - return curdate(BuiltinFunctionName.CURRENT_DATE.getName()); } /** @@ -236,6 +171,41 @@ private DefaultFunctionResolver convert_tz() { ); } + private FunctionResolver curdate(FunctionName functionName) { + return define(functionName, + impl(() -> new ExprDateValue(formatNow(null).toLocalDate()), DATE) + ); + } + + private FunctionResolver curdate() { + return curdate(BuiltinFunctionName.CURDATE.getName()); + } + + /** + * Synonym for @see `now`. + */ + private FunctionResolver curtime(FunctionName functionName) { + return define(functionName, + impl(() -> new ExprTimeValue(formatNow(null).toLocalTime()), TIME) + ); + } + + private FunctionResolver curtime() { + return curtime(BuiltinFunctionName.CURTIME.getName()); + } + + private FunctionResolver current_date() { + return curdate(BuiltinFunctionName.CURRENT_DATE.getName()); + } + + private FunctionResolver current_time() { + return curtime(BuiltinFunctionName.CURRENT_TIME.getName()); + } + + private FunctionResolver current_timestamp() { + return now(BuiltinFunctionName.CURRENT_TIMESTAMP.getName()); + } + /** * Extracts the date part of a date and time value. * Also to construct a date type. The supported signatures: @@ -386,6 +356,29 @@ private DefaultFunctionResolver hour() { ); } + private FunctionResolver localtime() { + return now(BuiltinFunctionName.LOCALTIME.getName()); + } + + private FunctionResolver localtimestamp() { + return now(BuiltinFunctionName.LOCALTIMESTAMP.getName()); + } + + /** + * NOW() returns a constant time that indicates the time at which the statement began to execute. + * `fsp` argument support is removed until refactoring to avoid bug where `now()`, `now(x)` and + * `now(y) return different values. + */ + private FunctionResolver now(FunctionName functionName) { + return define(functionName, + impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME) + ); + } + + private FunctionResolver now() { + return now(BuiltinFunctionName.NOW.getName()); + } + private FunctionResolver makedate() { return define(BuiltinFunctionName.MAKEDATE.getName(), impl(nullMissingHandling(DateTimeFunction::exprMakeDate), DATE, DOUBLE, DOUBLE)); @@ -444,6 +437,27 @@ private DefaultFunctionResolver monthName() { ); } + /** + * Add N months to period P (in the format YYMM or YYYYMM). Returns a value in the format YYYYMM. + * (INTEGER, INTEGER) -> INTEGER + */ + private DefaultFunctionResolver period_add() { + return define(BuiltinFunctionName.PERIOD_ADD.getName(), + impl(nullMissingHandling(DateTimeFunction::exprPeriodAdd), INTEGER, INTEGER, INTEGER) + ); + } + + /** + * Returns the number of months between periods P1 and P2. + * P1 and P2 should be in the format YYMM or YYYYMM. + * (INTEGER, INTEGER) -> INTEGER + */ + private DefaultFunctionResolver period_diff() { + return define(BuiltinFunctionName.PERIOD_DIFF.getName(), + impl(nullMissingHandling(DateTimeFunction::exprPeriodDiff), INTEGER, INTEGER, INTEGER) + ); + } + /** * QUARTER(STRING/DATE/DATETIME/TIMESTAMP). return the month for date (1-4). */ @@ -889,6 +903,61 @@ private ExprValue exprMonthName(ExprValue date) { date.dateValue().getMonth().getDisplayName(TextStyle.FULL, Locale.getDefault())); } + private LocalDate parseDatePeriod(Integer period) { + var input = period.toString(); + // MySQL undocumented: if year is not specified or has 1 digit - 2000/200x is assumed + if (input.length() <= 5) { + input = String.format("200%05d", period); + } + try { + return LocalDate.parse(input, DATE_FORMATTER_SHORT_YEAR); + } catch (DateTimeParseException ignored) { + // nothing to do, try another format + } + try { + return LocalDate.parse(input, DATE_FORMATTER_LONG_YEAR); + } catch (DateTimeParseException ignored) { + return null; + } + } + + /** + * Adds N months to period P (in the format YYMM or YYYYMM). + * Returns a value in the format YYYYMM. + * + * @param period Period in the format YYMM or YYYYMM. + * @param months Amount of months to add. + * @return ExprIntegerValue. + */ + private ExprValue exprPeriodAdd(ExprValue period, ExprValue months) { + // We should add a day to make string parsable and remove it afterwards + var input = period.integerValue() * 100 + 1; // adds 01 to end of the string + var parsedDate = parseDatePeriod(input); + if (parsedDate == null) { + return ExprNullValue.of(); + } + var res = DATE_FORMATTER_LONG_YEAR.format(parsedDate.plusMonths(months.integerValue())); + return new ExprIntegerValue(Integer.parseInt( + res.substring(0, res.length() - 2))); // Remove the day part, .eg. 20070101 -> 200701 + } + + /** + * Returns the number of months between periods P1 and P2. + * P1 and P2 should be in the format YYMM or YYYYMM. + * + * @param period1 Period in the format YYMM or YYYYMM. + * @param period2 Period in the format YYMM or YYYYMM. + * @return ExprIntegerValue. + */ + private ExprValue exprPeriodDiff(ExprValue period1, ExprValue period2) { + var parsedDate1 = parseDatePeriod(period1.integerValue() * 100 + 1); + var parsedDate2 = parseDatePeriod(period2.integerValue() * 100 + 1); + if (parsedDate1 == null || parsedDate2 == null) { + return ExprNullValue.of(); + } + return new ExprIntegerValue(MONTHS.between(parsedDate2, parsedDate1)); + } + /** * Quarter for date implementation for ExprValue. * @@ -936,6 +1005,16 @@ private ExprValue exprSubDateInterval(ExprValue date, ExprValue expr) { : exprValue); } + /** + * SYSDATE() returns the time at which it executes. + */ + private FunctionResolver sysdate() { + return define(BuiltinFunctionName.SYSDATE.getName(), + impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME), + impl((v) -> new ExprDatetimeValue(formatNow(v.integerValue())), DATETIME, INTEGER) + ); + } + /** * Time implementation for ExprValue. * diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index a7d7b74865..51d91eb372 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -77,6 +77,8 @@ public enum BuiltinFunctionName { MINUTE(FunctionName.of("minute")), MONTH(FunctionName.of("month")), MONTHNAME(FunctionName.of("monthname")), + PERIOD_ADD(FunctionName.of("period_add")), + PERIOD_DIFF(FunctionName.of("period_diff")), QUARTER(FunctionName.of("quarter")), SECOND(FunctionName.of("second")), SUBDATE(FunctionName.of("subdate")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java index 8811bf870b..91d1fc4b0f 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeTestBase.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.datetime; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.expression.function.BuiltinFunctionRepository.DEFAULT_NAMESPACE; import java.time.Instant; @@ -46,6 +47,51 @@ public class DateTimeTestBase extends ExpressionTestBase { @Autowired protected BuiltinFunctionRepository functionRepository; + protected ExprValue eval(Expression expression) { + return expression.valueOf(env); + } + + protected FunctionExpression fromUnixTime(Expression value) { + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("from_unixtime"), + List.of(value.type()))); + return (FunctionExpression)func.apply(List.of(value)); + } + + protected FunctionExpression fromUnixTime(Expression value, Expression format) { + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("from_unixtime"), + List.of(value.type(), format.type()))); + return (FunctionExpression)func.apply(List.of(value, format)); + } + + protected LocalDateTime fromUnixTime(Long value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + protected LocalDateTime fromUnixTime(Double value) { + return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); + } + + protected String fromUnixTime(Long value, String format) { + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + } + + protected String fromUnixTime(Double value, String format) { + return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); + } + + protected FunctionExpression makedate(Expression year, Expression dayOfYear) { + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("makedate"), + List.of(DOUBLE, DOUBLE))); + return (FunctionExpression)func.apply(List.of(year, dayOfYear)); + } + + protected LocalDate makedate(Double year, Double dayOfYear) { + return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); + } + protected FunctionExpression maketime(Expression hour, Expression minute, Expression second) { var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), new FunctionSignature(new FunctionName("maketime"), @@ -58,15 +104,26 @@ protected LocalTime maketime(Double hour, Double minute, Double second) { .valueOf(null).timeValue(); } - protected FunctionExpression makedate(Expression year, Expression dayOfYear) { + protected FunctionExpression period_add(Expression period, Expression months) { var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), - new FunctionSignature(new FunctionName("makedate"), - List.of(DOUBLE, DOUBLE))); - return (FunctionExpression)func.apply(List.of(year, dayOfYear)); + new FunctionSignature(new FunctionName("period_add"), + List.of(INTEGER, INTEGER))); + return (FunctionExpression)func.apply(List.of(period, months)); } - protected LocalDate makedate(Double year, Double dayOfYear) { - return makedate(DSL.literal(year), DSL.literal(dayOfYear)).valueOf(null).dateValue(); + protected Integer period_add(Integer period, Integer months) { + return period_add(DSL.literal(period), DSL.literal(months)).valueOf(null).integerValue(); + } + + protected FunctionExpression period_diff(Expression first, Expression second) { + var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), + new FunctionSignature(new FunctionName("period_diff"), + List.of(INTEGER, INTEGER))); + return (FunctionExpression)func.apply(List.of(first, second)); + } + + protected Integer period_diff(Integer first, Integer second) { + return period_diff(DSL.literal(first), DSL.literal(second)).valueOf(null).integerValue(); } protected FunctionExpression unixTimeStampExpr() { @@ -101,38 +158,4 @@ protected Double unixTimeStampOf(LocalDateTime value) { protected Double unixTimeStampOf(Instant value) { return unixTimeStampOf(DSL.literal(new ExprTimestampValue(value))).valueOf(null).doubleValue(); } - - protected FunctionExpression fromUnixTime(Expression value) { - var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), - new FunctionSignature(new FunctionName("from_unixtime"), - List.of(value.type()))); - return (FunctionExpression)func.apply(List.of(value)); - } - - protected FunctionExpression fromUnixTime(Expression value, Expression format) { - var func = functionRepository.resolve(Collections.singletonList(DEFAULT_NAMESPACE), - new FunctionSignature(new FunctionName("from_unixtime"), - List.of(value.type(), format.type()))); - return (FunctionExpression)func.apply(List.of(value, format)); - } - - protected LocalDateTime fromUnixTime(Long value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); - } - - protected LocalDateTime fromUnixTime(Double value) { - return fromUnixTime(DSL.literal(value)).valueOf(null).datetimeValue(); - } - - protected String fromUnixTime(Long value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); - } - - protected String fromUnixTime(Double value, String format) { - return fromUnixTime(DSL.literal(value), DSL.literal(format)).valueOf(null).stringValue(); - } - - protected ExprValue eval(Expression expression) { - return expression.valueOf(env); - } } diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java new file mode 100644 index 0000000000..ff63cb6f0f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/PeriodFunctionsTest.java @@ -0,0 +1,103 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.expression.DSL; + +public class PeriodFunctionsTest extends DateTimeTestBase { + + /** + * Generate sample data for `PERIOD_ADD` function. + * @return A data set. + */ + public static Stream getTestDataForPeriodAdd() { + // arguments are: first arg for `PERIOD_ADD`, second arg and expected result value. + return Stream.of( + Arguments.of(1, 3, 200004), // Jan 2000 + 3 + Arguments.of(3, -1, 200002), // Mar 2000 - 1 + Arguments.of(12, 0, 200012), // Dec 2000 + 0 + Arguments.of(6104, 100, 206908), // Apr 2061 + 100m (8y4m) + Arguments.of(201510, 14, 201612) + ); + } + + @ParameterizedTest + @MethodSource("getTestDataForPeriodAdd") + public void period_add_with_different_data(int period, int months, int expected) { + assertEquals(expected, period_add(period, months)); + } + + /** + * Generate sample data for `PERIOD_DIFF` function. + * @return A data set. + */ + public static Stream getTestDataForPeriodDiff() { + // arguments are: first arg for `PERIOD_DIFF`, second arg and expected result value. + return Stream.of( + Arguments.of(1, 3, -2), // Jan - Mar 2000 + Arguments.of(3, 1, 2), // Mar - Jan 2000 + Arguments.of(12, 111, -11), // Dec 2000 - Nov 2001 + Arguments.of(2212, 201105, 139), // Dec 2022 - May 2011 + Arguments.of(200505, 7505, 360), // May 2005 - May 1975 + Arguments.of(6104, 8509, 907), // Apr 2061 - Sep 1985 + Arguments.of(207707, 7707, 1200) // Jul 2077 - Jul 1977 + ); + } + + @ParameterizedTest + @MethodSource("getTestDataForPeriodDiff") + public void period_diff_with_different_data(int period1, int period2, int expected) { + assertEquals(expected, period_diff(period1, period2)); + } + + @ParameterizedTest + @MethodSource("getTestDataForPeriodDiff") + public void two_way_conversion(int period1, int period2, int expected) { + assertEquals(0, period_diff(period_add(period1, -expected), period2)); + } + + /** + * Generate invalid sample data for test. + * @return A data set. + */ + public static Stream getInvalidTestData() { + return Stream.of( + Arguments.of(0), + Arguments.of(123), + Arguments.of(100), + Arguments.of(1234), + Arguments.of(1000), + Arguments.of(2020), + Arguments.of(12345), + Arguments.of(123456), + Arguments.of(1234567), + Arguments.of(200213), + Arguments.of(200300), + Arguments.of(-1), + Arguments.of(-1234), + Arguments.of(-123401) + ); + } + + /** + * Check that `PERIOD_ADD` and `PERIOD_DIFF` return NULL on invalid input. + * @param period An invalid data. + */ + @ParameterizedTest + @MethodSource("getInvalidTestData") + public void period_add_returns_null_on_invalid_input(int period) { + assertNull(period_add(DSL.literal(period), DSL.literal(1)).valueOf(null).value()); + assertNull(period_diff(DSL.literal(period), DSL.literal(1)).valueOf(null).value()); + assertNull(period_diff(DSL.literal(1), DSL.literal(period)).valueOf(null).value()); + } +} diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 43cfba43e7..15f8a768bb 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1762,6 +1762,52 @@ Example:: +---------------------+---------------------+ +PERIOD_ADD +---------- + +Description +>>>>>>>>>>> + +Usage: period_add(P, N) add N months to period P (in the format YYMM or YYYYMM). Returns a value in the format YYYYMM. + +Argument type: INTEGER, INTEGER + +Return type: INTEGER + +Example:: + + os> SELECT PERIOD_ADD(200801, 2), PERIOD_ADD(200801, -12) + fetched rows / total rows = 1/1 + +-------------------------+---------------------------+ + | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | + |-------------------------+---------------------------| + | 200803 | 200701 | + +-------------------------+---------------------------+ + + +PERIOD_DIFF +----------- + +Description +>>>>>>>>>>> + +Usage: period_diff(P1, P2) returns the number of months between periods P1 and P2 given in the format YYMM or YYYYMM. + +Argument type: INTEGER, INTEGER + +Return type: INTEGER + +Example:: + + os> SELECT PERIOD_DIFF(200802, 200703), PERIOD_DIFF(200802, 201003) + fetched rows / total rows = 1/1 + +-------------------------------+-------------------------------+ + | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | + |-------------------------------+-------------------------------| + | 11 | -25 | + +-------------------------------+-------------------------------+ + + QUARTER ------- diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index e6ff88d2a8..7d783e914b 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -934,6 +934,52 @@ Example:: +---------------------+---------------------+ +PERIOD_ADD +---------- + +Description +>>>>>>>>>>> + +Usage: period_add(P, N) add N months to period P (in the format YYMM or YYYYMM). Returns a value in the format YYYYMM. + +Argument type: INTEGER, INTEGER + +Return type: INTEGER + +Example:: + + os> source=people | eval `PERIOD_ADD(200801, 2)` = PERIOD_ADD(200801, 2), `PERIOD_ADD(200801, -12)` = PERIOD_ADD(200801, -12) | fields `PERIOD_ADD(200801, 2)`, `PERIOD_ADD(200801, -12)` + fetched rows / total rows = 1/1 + +-------------------------+---------------------------+ + | PERIOD_ADD(200801, 2) | PERIOD_ADD(200801, -12) | + |-------------------------+---------------------------| + | 200803 | 200701 | + +-------------------------+---------------------------+ + + +PERIOD_DIFF +----------- + +Description +>>>>>>>>>>> + +Usage: period_diff(P1, P2) returns the number of months between periods P1 and P2 given in the format YYMM or YYYYMM. + +Argument type: INTEGER, INTEGER + +Return type: INTEGER + +Example:: + + os> source=people | eval `PERIOD_DIFF(200802, 200703)` = PERIOD_DIFF(200802, 200703), `PERIOD_DIFF(200802, 201003)` = PERIOD_DIFF(200802, 201003) | fields `PERIOD_DIFF(200802, 200703)`, `PERIOD_DIFF(200802, 201003)` + fetched rows / total rows = 1/1 + +-------------------------------+-------------------------------+ + | PERIOD_DIFF(200802, 200703) | PERIOD_DIFF(200802, 201003) | + |-------------------------------+-------------------------------| + | 11 | -25 | + +-------------------------------+-------------------------------+ + + QUARTER ------- diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index d35dcc566b..8c0661d6ff 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -13,12 +13,12 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import static org.opensearch.sql.util.MatcherUtils.verifySome; +import com.google.common.collect.ImmutableMap; import java.io.IOException; +import java.time.LocalDate; import java.time.LocalTime; import java.time.Duration; -import java.time.LocalDate; import java.time.LocalDateTime; -import java.time.LocalTime; import java.time.Period; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; @@ -30,13 +30,12 @@ import java.util.function.BiFunction; import java.util.function.Supplier; import java.util.stream.Collectors; -import com.google.common.collect.ImmutableMap; import org.json.JSONArray; -import java.time.LocalTime; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.common.utils.StringUtils; +@SuppressWarnings("unchecked") public class DateTimeFunctionIT extends PPLIntegTestCase { @Override @@ -644,7 +643,6 @@ public void testDateFormat() throws IOException { verifyDateFormat(date, "date", dateFormat, dateFormatted); } - @Test public void testDateFormatISO8601() throws IOException { String timestamp = "1998-01-31 13:14:15.012345"; @@ -661,7 +659,7 @@ public void testDateFormatISO8601() throws IOException { @Test public void testMakeTime() throws IOException { var result = executeQuery(String.format( - "source=%s | eval f1 = MAKETIME(20, 30, 40), f2 = MAKETIME(20.2, 49.5, 42.100502) | fields f1, f2", TEST_INDEX_DATE)); + "source=%s | eval f1 = MAKETIME(20, 30, 40), f2 = MAKETIME(20.2, 49.5, 42.100502) | fields f1, f2", TEST_INDEX_DATE)); verifySchema(result, schema("f1", null, "time"), schema("f2", null, "time")); verifySome(result.getJSONArray("datarows"), rows("20:30:40", "20:50:42.100502")); } @@ -669,7 +667,7 @@ public void testMakeTime() throws IOException { @Test public void testMakeDate() throws IOException { var result = executeQuery(String.format( - "source=%s | eval f1 = MAKEDATE(1945, 5.9), f2 = MAKEDATE(1984, 1984) | fields f1, f2", TEST_INDEX_DATE)); + "source=%s | eval f1 = MAKEDATE(1945, 5.9), f2 = MAKEDATE(1984, 1984) | fields f1, f2", TEST_INDEX_DATE)); verifySchema(result, schema("f1", null, "date"), schema("f2", null, "date")); verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); } @@ -862,4 +860,20 @@ public void testUnixTimeStamp() throws IOException { schema("f3", null, "double")); verifySome(result.getJSONArray("datarows"), rows(613094400d, 1072872000d, 3404817525d)); } + + @Test + public void testPeriodAdd() throws IOException { + var result = executeQuery(String.format( + "source=%s | eval f1 = PERIOD_ADD(200801, 2), f2 = PERIOD_ADD(200801, -12) | fields f1, f2", TEST_INDEX_DATE)); + verifySchema(result, schema("f1", null, "integer"), schema("f2", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(200803, 200701)); + } + + @Test + public void testPeriodDiff() throws IOException { + var result = executeQuery(String.format( + "source=%s | eval f1 = PERIOD_DIFF(200802, 200703), f2 = PERIOD_DIFF(200802, 201003) | fields f1, f2", TEST_INDEX_DATE)); + verifySchema(result, schema("f1", null, "integer"), schema("f2", null, "integer")); + verifySome(result.getJSONArray("datarows"), rows(11, -25)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 207c3beb7d..15dee748b2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -7,9 +7,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE2; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -18,6 +16,7 @@ import static org.opensearch.sql.util.MatcherUtils.verifySome; import static org.opensearch.sql.util.TestUtils.getResponseBody; +import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.time.Duration; import java.time.LocalDate; @@ -34,7 +33,6 @@ import java.util.TimeZone; import java.util.function.BiFunction; import java.util.function.Supplier; -import com.google.common.collect.ImmutableMap; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -475,18 +473,22 @@ public void testDateFormat() throws IOException { @Test public void testMakeTime() throws IOException { - var result = executeQuery(String.format( - "select MAKETIME(20, 30, 40) as f1, MAKETIME(20.2, 49.5, 42.100502) as f2", TEST_INDEX_DATE)); - verifySchema(result, schema("MAKETIME(20, 30, 40)", "f1", "time"), schema("MAKETIME(20.2, 49.5, 42.100502)", "f2", "time")); - verifySome(result.getJSONArray("datarows"), rows("20:30:40", "20:50:42.100502")); + var result = executeQuery( + "select MAKETIME(20, 30, 40) as f1, MAKETIME(20.2, 49.5, 42.100502) as f2"); + verifySchema(result, + schema("MAKETIME(20, 30, 40)", "f1", "time"), + schema("MAKETIME(20.2, 49.5, 42.100502)", "f2", "time")); + verifyDataRows(result, rows("20:30:40", "20:50:42.100502")); } @Test public void testMakeDate() throws IOException { - var result = executeQuery(String.format( - "select MAKEDATE(1945, 5.9) as f1, MAKEDATE(1984, 1984) as f2", TEST_INDEX_DATE)); - verifySchema(result, schema("MAKEDATE(1945, 5.9)", "f1", "date"), schema("MAKEDATE(1984, 1984)", "f2", "date")); - verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); + var result = executeQuery( + "select MAKEDATE(1945, 5.9) as f1, MAKEDATE(1984, 1984) as f2"); + verifySchema(result, + schema("MAKEDATE(1945, 5.9)", "f1", "date"), + schema("MAKEDATE(1984, 1984)", "f2", "date")); + verifyDataRows(result, rows("1945-01-06", "1989-06-06")); } private List> nowLikeFunctionsData() { @@ -674,6 +676,26 @@ public void testUnixTimeStamp() throws IOException { verifySome(result.getJSONArray("datarows"), rows(613094400d, 1072872000d, 3404817525d)); } + @Test + public void testPeriodAdd() throws IOException { + var result = executeQuery( + "select PERIOD_ADD(200801, 2) as f1, PERIOD_ADD(200801, -12) as f2"); + verifySchema(result, + schema("PERIOD_ADD(200801, 2)", "f1", "integer"), + schema("PERIOD_ADD(200801, -12)", "f2", "integer")); + verifyDataRows(result, rows(200803, 200701)); + } + + @Test + public void testPeriodDiff() throws IOException { + var result = executeQuery( + "select PERIOD_DIFF(200802, 200703) as f1, PERIOD_DIFF(200802, 201003) as f2"); + verifySchema(result, + schema("PERIOD_DIFF(200802, 200703)", "f1", "integer"), + schema("PERIOD_DIFF(200802, 201003)", "f2", "integer")); + verifyDataRows(result, rows(11, -25)); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index fb8194d65c..ffb9780faf 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -244,10 +244,10 @@ DATE: 'DATE'; DATE_ADD: 'DATE_ADD'; DATE_FORMAT: 'DATE_FORMAT'; DATE_SUB: 'DATE_SUB'; +DAYNAME: 'DAYNAME'; DAYOFMONTH: 'DAYOFMONTH'; DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; -DAYNAME: 'DAYNAME'; FROM_DAYS: 'FROM_DAYS'; LOCALTIME: 'LOCALTIME'; LOCALTIMESTAMP: 'LOCALTIMESTAMP'; @@ -256,6 +256,8 @@ MAKEDATE: 'MAKEDATE'; MAKETIME: 'MAKETIME'; MONTHNAME: 'MONTHNAME'; NOW: 'NOW'; +PERIOD_ADD: 'PERIOD_ADD'; +PERIOD_DIFF: 'PERIOD_DIFF'; SUBDATE: 'SUBDATE'; SYSDATE: 'SYSDATE'; TIME: 'TIME'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 4dfac10dc2..7a42c7fa2b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -413,10 +413,11 @@ trigonometricFunctionName ; dateAndTimeFunctionBase - : ADDDATE | CONVERT_TZ | DATE | DATETIME | DATE_ADD | DATE_FORMAT | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK - | DAYOFYEAR | FROM_DAYS | FROM_UNIXTIME | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE - | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | SYSDATE | TIME | TIMESTAMP | TIME_TO_SEC - | TO_DAYS | UNIX_TIMESTAMP | WEEK | YEAR + : ADDDATE | CONVERT_TZ | DATE | DATE_ADD | DATE_FORMAT | DATE_SUB + | DATETIME | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS | FROM_UNIXTIME + | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE | MONTH | MONTHNAME | PERIOD_ADD + | PERIOD_DIFF | QUARTER | SECOND | SUBDATE | SYSDATE | TIME | TIME_TO_SEC + | TIMESTAMP | TO_DAYS | UNIX_TIMESTAMP | WEEK | YEAR ; // Functions which value could be cached in scope of a single query diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 470ff5050f..9e0a409401 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -197,13 +197,13 @@ CURRENT_DATE: 'CURRENT_DATE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; DATE: 'DATE'; -DATE_FORMAT: 'DATE_FORMAT'; DATE_ADD: 'DATE_ADD'; +DATE_FORMAT: 'DATE_FORMAT'; DATE_SUB: 'DATE_SUB'; +DAYNAME: 'DAYNAME'; DAYOFMONTH: 'DAYOFMONTH'; DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; -DAYNAME: 'DAYNAME'; DEGREES: 'DEGREES'; E: 'E'; EXP: 'EXP'; @@ -231,6 +231,8 @@ MONTHNAME: 'MONTHNAME'; MULTIPLY: 'MULTIPLY'; NOW: 'NOW'; NULLIF: 'NULLIF'; +PERIOD_ADD: 'PERIOD_ADD'; +PERIOD_DIFF: 'PERIOD_DIFF'; PI: 'PI'; POW: 'POW'; POWER: 'POWER'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5bb56653f9..6470c9f546 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -395,10 +395,11 @@ trigonometricFunctionName ; dateTimeFunctionName - : ADDDATE | CONVERT_TZ | DATE | DATETIME | DATE_ADD | DATE_FORMAT | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK - | DAYOFYEAR | FROM_DAYS | FROM_UNIXTIME | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE - | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | SYSDATE | TIME | TIME_TO_SEC | TIMESTAMP - | TO_DAYS | UNIX_TIMESTAMP | WEEK | YEAR + : ADDDATE | CONVERT_TZ | DATE | DATE_ADD | DATE_FORMAT | DATE_SUB + | DATETIME | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS | FROM_UNIXTIME + | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE | MONTH | MONTHNAME | PERIOD_ADD + | PERIOD_DIFF | QUARTER | SECOND | SUBDATE | SYSDATE | TIME | TIME_TO_SEC + | TIMESTAMP | TO_DAYS | UNIX_TIMESTAMP | WEEK | YEAR ; // Functions which value could be cached in scope of a single query From 3fd53cac2f388149a8d2c7b8ab5fcfea453a5b80 Mon Sep 17 00:00:00 2001 From: Guian Gumpac Date: Fri, 28 Oct 2022 12:27:21 -0700 Subject: [PATCH 06/20] Enable ignored and disabled tests (#926) - Enabled ignored and disabled tests. - Created #922, #917, #912, #910 and attached them to disabled tests for traceability. - Removed irrelevant tests. --- .../org/opensearch/sql/analysis/AnalyzerTest.java | 3 ++- .../sql/analysis/SelectExpressionAnalyzerTest.java | 13 ------------- .../org/opensearch/sql/legacy/AggregationIT.java | 2 -- .../sql/legacy/PrettyFormatResponseIT.java | 6 +----- .../java/org/opensearch/sql/legacy/QueryIT.java | 2 -- .../java/org/opensearch/sql/legacy/SubqueryIT.java | 1 - .../org/opensearch/sql/ppl/FieldsCommandIT.java | 2 +- .../java/org/opensearch/sql/ppl/MatchPhraseIT.java | 10 ---------- .../org/opensearch/sql/ppl/QueryAnalysisIT.java | 1 - .../org/opensearch/sql/ppl/RenameCommandIT.java | 2 +- .../java/org/opensearch/sql/ppl/SortCommandIT.java | 7 +++---- .../org/opensearch/sql/sql/QueryValidationIT.java | 3 ++- .../logical/OpenSearchLogicOptimizerTest.java | 2 +- .../DefaultExpressionSerializerTest.java | 1 - .../sql/ppl/parser/AstExpressionBuilderTest.java | 2 -- .../sql/sql/parser/AstAggregationBuilderTest.java | 3 ++- 16 files changed, 13 insertions(+), 47 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index bd65e011c2..40c33e6d61 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -396,7 +396,8 @@ public void remove_source() { AstDSL.field("double_value"))); } - @Disabled("the project/remove command should shrink the type env") + @Disabled("the project/remove command should shrink the type env. Should be enabled once " + + "https://github.com/opensearch-project/sql/issues/917 is resolved") @Test public void project_source_change_type_env() { SemanticCheckException exception = diff --git a/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java index 7fbe5bdb84..8ad38f5322 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/SelectExpressionAnalyzerTest.java @@ -56,19 +56,6 @@ public void named_expression_with_alias() { ); } - @Disabled("we didn't define the aggregator symbol any more") - @Test - public void named_expression_with_delegated_expression_defined_in_symbol_table() { - analysisContext.push(); - analysisContext.peek().define(new Symbol(Namespace.FIELD_NAME, "AVG(integer_value)"), FLOAT); - - assertAnalyzeEqual( - DSL.named("AVG(integer_value)", DSL.ref("AVG(integer_value)", FLOAT)), - AstDSL.alias("AVG(integer_value)", - AstDSL.aggregate("AVG", AstDSL.qualifiedName("integer_value"))) - ); - } - @Test public void field_name_with_qualifier() { analysisContext.peek().define(new Symbol(Namespace.INDEX_NAME, "index_alias"), STRUCT); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java index 1d530efa5e..ba007f43f9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AggregationIT.java @@ -1307,8 +1307,6 @@ public void distinctWithOneField() { ); } - @Ignore("Skip this because it compares result of GROUP BY and DISTINCT and find difference in\n" - + "schema type (string and text). Remove this when DISTINCT supported in new engine.") @Test public void distinctWithMultipleFields() { Assert.assertEquals( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java index 067f72e986..226645ce85 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java @@ -75,7 +75,7 @@ protected Request getSqlRequest(String request, boolean explain) { return sqlRequest; } - @Ignore("Index type is removed in OpenSearch 7+") + // This is testing a deprecated feature @Test public void wrongIndexType() throws IOException { String type = "wrongType"; @@ -248,9 +248,6 @@ public void groupBySingleField() throws IOException { assertContainsData(getDataRows(response), fields); } - @Ignore("The semantic of this and previous are wrong. The correct semantic is that * will " - + "be expanded to all fields of the index. Error should be raise for both due to difference " - + "between columns in SELECT and GROUP BY.") @Test public void groupByMultipleFields() throws IOException { JSONObject response = executeQuery( @@ -398,7 +395,6 @@ public void aggregationFunctionInHaving() throws IOException { // public void nestedAggregationFunctionInSelect() { // String query = String.format(Locale.ROOT, "SELECT SUM(SQRT(age)) FROM age GROUP BY age", TEST_INDEX_ACCOUNT); // } - @Ignore("New engine returns string type") @Test public void fieldsWithAlias() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java index 96ce82a75e..eeff107f15 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/QueryIT.java @@ -235,7 +235,6 @@ private void assertResponseForSelectSpecificFields(JSONObject response, } } - @Ignore("Will fix this in issue https://github.com/opendistro-for-elasticsearch/sql/issues/121") @Test public void selectFieldWithSpace() throws IOException { String[] arr = new String[] {"test field"}; @@ -574,7 +573,6 @@ public void notBetweenTest() throws IOException { } } - @Ignore("Semantic analysis failed because 'age' doesn't exist.") @Test public void inTest() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java index c9beeb8747..7fbfb1ef1c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SubqueryIT.java @@ -348,7 +348,6 @@ public void selectFromSubqueryCountAndSum() throws IOException { assertThat(result.query("/aggregations/balance/value"), equalTo(25714837.0)); } - @Ignore("Skip to avoid breaking test due to inconsistency in JDBC schema") @Test public void selectFromSubqueryWithoutAliasShouldPass() throws IOException { JSONObject response = executeJdbcRequest( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java index 64adeb4f7f..4eb99e8b04 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FieldsCommandIT.java @@ -43,7 +43,7 @@ public void testFieldsWithMultiFields() throws IOException { verifyColumn(result, columnName("firstname"), columnName("lastname")); } - @Ignore("Cannot resolve wildcard yet") + @Ignore("Cannot resolve wildcard yet. Enable once https://github.com/opensearch-project/sql/issues/787 is resolved.") @Test public void testFieldsWildCard() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhraseIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhraseIT.java index 8003ec0c80..5b9fd07e31 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhraseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/MatchPhraseIT.java @@ -32,16 +32,6 @@ public void test_match_phrase_function() throws IOException { verifyDataRows(result, rows("quick fox"), rows("quick fox here")); } - @Test - @Ignore("Not supported actually in PPL") - public void test_matchphrase_legacy_function() throws IOException { - JSONObject result = - executeQuery( - String.format( - "source=%s | where matchphrase(phrase, 'quick fox') | fields phrase", TEST_INDEX_PHRASE)); - verifyDataRows(result, rows("quick fox"), rows("quick fox here")); - } - @Test public void test_match_phrase_with_slop() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryAnalysisIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryAnalysisIT.java index 8a4abe2415..dd2fcb84c8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/QueryAnalysisIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/QueryAnalysisIT.java @@ -41,7 +41,6 @@ public void fieldsCommandShouldPassSemanticCheck() { queryShouldPassSyntaxAndSemanticCheck(query); } - @Ignore("Can't resolve target field yet") @Test public void renameCommandShouldPassSemanticCheck() { String query = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java index 38904dc579..ad1add4e12 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/RenameCommandIT.java @@ -43,7 +43,7 @@ public void testRenameMultiField() throws IOException { verifyColumn(result, columnName("FIRSTNAME"), columnName("AGE")); } - @Ignore("Wildcard is unsupported yet") + @Ignore("Wildcard is unsupported yet. Enable once https://github.com/opensearch-project/sql/issues/787 is resolved.") @Test public void testRenameWildcardFields() throws IOException { JSONObject result = executeQuery("source=" + TEST_INDEX_ACCOUNT + " | rename %name as %NAME"); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java index a563ae60e0..0fd4e9ec86 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java @@ -33,7 +33,6 @@ public void testSortCommand() throws IOException { verifyOrder(result, rows(28), rows(32), rows(33), rows(34), rows(36), rows(36), rows(39)); } - @Ignore("Order with duplicated value") @Test public void testSortWithNullValue() throws IOException { JSONObject result = @@ -43,9 +42,9 @@ public void testSortWithNullValue() throws IOException { TEST_INDEX_BANK_WITH_NULL_VALUES)); verifyOrder( result, - rows("Hattie"), - rows("Elinor"), - rows("Virginia"), + rows("Hattie", null), + rows("Elinor", null), + rows("Virginia", null), rows("Dale", 4180), rows("Nanette", 32838), rows("Amber JOHnny", 39225), diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java index 8b41eb650b..62869be168 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/QueryValidationIT.java @@ -38,7 +38,8 @@ protected void init() throws Exception { loadIndex(Index.ACCOUNT); } - @Ignore("Will add this validation in analyzer later") + @Ignore("Will add this validation in analyzer later. This test should be enabled once " + + "https://github.com/opensearch-project/sql/issues/910 has been resolved") @Test public void testNonAggregatedSelectColumnMissingInGroupByClause() throws IOException { expectResponseException() diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/logical/OpenSearchLogicOptimizerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/logical/OpenSearchLogicOptimizerTest.java index 9ad37c6ef3..df42a2b201 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/logical/OpenSearchLogicOptimizerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/logical/OpenSearchLogicOptimizerTest.java @@ -129,7 +129,7 @@ void aggregation_merge_filter_relation() { ); } - @Disabled + @Disabled("This test should be enabled once https://github.com/opensearch-project/sql/issues/912 is fixed") @Test void aggregation_cant_merge_indexScan_with_project() { assertEquals( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serialization/DefaultExpressionSerializerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serialization/DefaultExpressionSerializerTest.java index 05f9b83856..1bec475e04 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serialization/DefaultExpressionSerializerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serialization/DefaultExpressionSerializerTest.java @@ -55,7 +55,6 @@ public void can_serialize_and_deserialize_predicates() { assertEquals(original, actual); } - @Disabled("Bypass until all functions become serializable") @Test public void can_serialize_and_deserialize_functions() { Expression original = dsl.abs(literal(30.0)); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 5073aa1459..565157dd06 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -528,7 +528,6 @@ public void testDataTypeFuncCall() { )); } - @Ignore("Nested field is not supported in backend yet") @Test public void testNestedFieldName() { assertEqual("source=t | fields field0.field1.field2", @@ -553,7 +552,6 @@ public void testFieldNameWithSpecialChars() { )); } - @Ignore("Nested field is not supported in backend yet") @Test public void testNestedFieldNameWithSpecialChars() { assertEqual("source=t | fields `field-0`.`field#1`.`field*2`", diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java index 9e10d70926..79896d9400 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstAggregationBuilderTest.java @@ -198,7 +198,8 @@ void should_report_error_for_non_integer_ordinal_in_group_by() { error.getMessage()); } - @Disabled("This validation is supposed to be in analyzing phase") + @Disabled("This validation is supposed to be in analyzing phase. This test should be enabled " + + "once https://github.com/opensearch-project/sql/issues/910 has been resolved") @Test void should_report_error_for_mismatch_between_select_and_group_by_items() { SemanticCheckException error1 = assertThrows(SemanticCheckException.class, () -> From 89b3e1a0f543d6e36b6e94d0f848629d7d3c5979 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 16 Sep 2022 19:28:21 -0700 Subject: [PATCH 07/20] Rework on error reporting to make it more verbose and human-friendly. (#116) Co-authored-by: MaxKsyunz Co-authored-by: forestmvey Signed-off-by: Yury-Fridlyand --- .../expression/datetime/DateTimeFunction.java | 36 +- .../sql/legacy/utils/QueryDataAnonymizer.java | 8 +- .../FunctionParameterRepository.java | 355 ++++++++++++++++++ .../relevance/MatchBoolPrefixQuery.java | 15 +- .../relevance/MatchPhrasePrefixQuery.java | 12 +- .../lucene/relevance/MatchPhraseQuery.java | 20 +- .../filter/lucene/relevance/MatchQuery.java | 20 +- .../lucene/relevance/MultiFieldQuery.java | 19 +- .../lucene/relevance/MultiMatchQuery.java | 22 +- .../lucene/relevance/QueryStringQuery.java | 48 +-- .../lucene/relevance/RelevanceQuery.java | 48 ++- .../relevance/SimpleQueryStringQuery.java | 26 +- .../lucene/relevance/SingleFieldQuery.java | 17 +- .../script/filter/FilterQueryBuilderTest.java | 155 ++++++-- .../script/filter/lucene/MultiMatchTest.java | 2 +- .../script/filter/lucene/QueryStringTest.java | 4 +- .../filter/lucene/SimpleQueryStringTest.java | 2 +- .../lucene/relevance/MultiFieldQueryTest.java | 5 +- .../relevance/RelevanceQueryBuildTest.java | 10 +- .../relevance/SingleFieldQueryTest.java | 5 +- 20 files changed, 583 insertions(+), 246 deletions(-) create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/FunctionParameterRepository.java diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 5b1af29590..43f5234d31 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -60,7 +60,6 @@ import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; import org.opensearch.sql.expression.function.FunctionName; -import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.utils.DateTimeUtils; /** @@ -135,7 +134,6 @@ public void register(BuiltinFunctionRepository repository) { * (DATE, LONG) -> DATE * (STRING/DATETIME/TIMESTAMP, LONG) -> DATETIME */ - private DefaultFunctionResolver add_date(FunctionName functionName) { return define(functionName, impl(nullMissingHandling(DateTimeFunction::exprAddDateInterval), @@ -171,38 +169,38 @@ private DefaultFunctionResolver convert_tz() { ); } - private FunctionResolver curdate(FunctionName functionName) { + private DefaultFunctionResolver curdate(FunctionName functionName) { return define(functionName, impl(() -> new ExprDateValue(formatNow(null).toLocalDate()), DATE) ); } - private FunctionResolver curdate() { + private DefaultFunctionResolver curdate() { return curdate(BuiltinFunctionName.CURDATE.getName()); } /** * Synonym for @see `now`. */ - private FunctionResolver curtime(FunctionName functionName) { + private DefaultFunctionResolver curtime(FunctionName functionName) { return define(functionName, impl(() -> new ExprTimeValue(formatNow(null).toLocalTime()), TIME) ); } - private FunctionResolver curtime() { + private DefaultFunctionResolver curtime() { return curtime(BuiltinFunctionName.CURTIME.getName()); } - private FunctionResolver current_date() { + private DefaultFunctionResolver current_date() { return curdate(BuiltinFunctionName.CURRENT_DATE.getName()); } - private FunctionResolver current_time() { + private DefaultFunctionResolver current_time() { return curtime(BuiltinFunctionName.CURRENT_TIME.getName()); } - private FunctionResolver current_timestamp() { + private DefaultFunctionResolver current_timestamp() { return now(BuiltinFunctionName.CURRENT_TIMESTAMP.getName()); } @@ -225,7 +223,7 @@ private DefaultFunctionResolver date() { * (STRING, STRING) -> DATETIME * (STRING) -> DATETIME */ - private FunctionResolver datetime() { + private DefaultFunctionResolver datetime() { return define(BuiltinFunctionName.DATETIME.getName(), impl(nullMissingHandling(DateTimeFunction::exprDateTime), DATETIME, STRING, STRING), @@ -337,7 +335,7 @@ private DefaultFunctionResolver from_days() { impl(nullMissingHandling(DateTimeFunction::exprFromDays), DATE, LONG)); } - private FunctionResolver from_unixtime() { + private DefaultFunctionResolver from_unixtime() { return define(BuiltinFunctionName.FROM_UNIXTIME.getName(), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE), impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat), @@ -356,11 +354,11 @@ private DefaultFunctionResolver hour() { ); } - private FunctionResolver localtime() { + private DefaultFunctionResolver localtime() { return now(BuiltinFunctionName.LOCALTIME.getName()); } - private FunctionResolver localtimestamp() { + private DefaultFunctionResolver localtimestamp() { return now(BuiltinFunctionName.LOCALTIMESTAMP.getName()); } @@ -369,22 +367,22 @@ private FunctionResolver localtimestamp() { * `fsp` argument support is removed until refactoring to avoid bug where `now()`, `now(x)` and * `now(y) return different values. */ - private FunctionResolver now(FunctionName functionName) { + private DefaultFunctionResolver now(FunctionName functionName) { return define(functionName, impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME) ); } - private FunctionResolver now() { + private DefaultFunctionResolver now() { return now(BuiltinFunctionName.NOW.getName()); } - private FunctionResolver makedate() { + private DefaultFunctionResolver makedate() { return define(BuiltinFunctionName.MAKEDATE.getName(), impl(nullMissingHandling(DateTimeFunction::exprMakeDate), DATE, DOUBLE, DOUBLE)); } - private FunctionResolver maketime() { + private DefaultFunctionResolver maketime() { return define(BuiltinFunctionName.MAKETIME.getName(), impl(nullMissingHandling(DateTimeFunction::exprMakeTime), TIME, DOUBLE, DOUBLE, DOUBLE)); } @@ -536,7 +534,7 @@ private DefaultFunctionResolver to_days() { impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME)); } - private FunctionResolver unix_timestamp() { + private DefaultFunctionResolver unix_timestamp() { return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(), impl(DateTimeFunction::unixTimeStamp, LONG), impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATE), @@ -1008,7 +1006,7 @@ private ExprValue exprSubDateInterval(ExprValue date, ExprValue expr) { /** * SYSDATE() returns the time at which it executes. */ - private FunctionResolver sysdate() { + private DefaultFunctionResolver sysdate() { return define(BuiltinFunctionName.SYSDATE.getName(), impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME), impl((v) -> new ExprDatetimeValue(formatNow(v.integerValue())), DATETIME, INTEGER) diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java b/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java index 91406333ae..b58691c022 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/utils/QueryDataAnonymizer.java @@ -26,7 +26,8 @@ public class QueryDataAnonymizer { * Sensitive data includes index names, column names etc., * which in druid parser are parsed to SQLIdentifierExpr instances * @param query entire sql query string - * @return sql query string with all identifiers replaced with "***" + * @return sql query string with all identifiers replaced with "***" on success + * and failure string otherwise to ensure no non-anonymized data is logged in production. */ public static String anonymizeData(String query) { String resultQuery; @@ -38,8 +39,9 @@ public static String anonymizeData(String query) { .replaceAll("false", "boolean_literal") .replaceAll("[\\n][\\t]+", " "); } catch (Exception e) { - LOG.warn("Caught an exception when anonymizing sensitive data"); - resultQuery = query; + LOG.warn("Caught an exception when anonymizing sensitive data."); + LOG.debug("String {} failed anonymization.", query); + resultQuery = "Failed to anonymize data."; } return resultQuery; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/FunctionParameterRepository.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/FunctionParameterRepository.java new file mode 100644 index 0000000000..373df4e5fc --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/FunctionParameterRepository.java @@ -0,0 +1,355 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; + +import com.google.common.collect.ImmutableMap; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; +import org.opensearch.common.unit.Fuzziness; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; +import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; +import org.opensearch.index.query.MatchPhraseQueryBuilder; +import org.opensearch.index.query.MatchQueryBuilder; +import org.opensearch.index.query.MultiMatchQueryBuilder; +import org.opensearch.index.query.Operator; +import org.opensearch.index.query.QueryStringQueryBuilder; +import org.opensearch.index.query.SimpleQueryStringBuilder; +import org.opensearch.index.query.SimpleQueryStringFlag; +import org.opensearch.index.query.support.QueryParsers; +import org.opensearch.index.search.MatchQuery; +import org.opensearch.sql.data.model.ExprValue; + +@UtilityClass +public class FunctionParameterRepository { + + public static final Map> + MatchBoolPrefixQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(checkRewrite(v, "fuzzy_rewrite"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("operator", (b, v) -> b.operator(convertOperator(v, "operator"))) + .put("prefix_length", (b, v) -> b.prefixLength(convertIntValue(v, "prefix_length"))) + .build(); + + public static final Map> + MatchPhrasePrefixQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("slop", (b, v) -> b.slop(convertIntValue(v, "slop"))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + public static final Map> + MatchPhraseQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("slop", (b, v) -> b.slop(convertIntValue(v, "slop"))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + public static final Map> + MatchQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(checkRewrite(v, "fuzzy_rewrite"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("operator", (b, v) -> b.operator(convertOperator(v, "operator"))) + .put("prefix_length", (b, v) -> b.prefixLength(convertIntValue(v, "prefix_length"))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + @SuppressWarnings("deprecation") // cutoffFrequency is deprecated + public static final Map> + MultiMatchQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("cutoff_frequency", (b, v) -> b.cutoffFrequency( + convertFloatValue(v, "cutoff_frequency"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("max_expansions", (b, v) -> b.maxExpansions(convertIntValue(v, "max_expansions"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("operator", (b, v) -> b.operator(convertOperator(v, "operator"))) + .put("prefix_length", (b, v) -> b.prefixLength(convertIntValue(v, "prefix_length"))) + .put("slop", (b, v) -> b.slop(convertIntValue(v, "slop"))) + .put("tie_breaker", (b, v) -> b.tieBreaker(convertFloatValue(v, "tie_breaker"))) + .put("type", (b, v) -> b.type(convertType(v))) + .put("zero_terms_query", (b, v) -> b.zeroTermsQuery(convertZeroTermsQuery(v))) + .build(); + + public static final Map> + QueryStringQueryBuildActions = ImmutableMap.>builder() + .put("allow_leading_wildcard", (b, v) -> b.allowLeadingWildcard( + convertBoolValue(v, "allow_leading_wildcard"))) + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("analyze_wildcard", (b, v) -> b.analyzeWildcard( + convertBoolValue(v, "analyze_wildcard"))) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("default_operator", (b, v) -> b.defaultOperator( + convertOperator(v, "default_operator"))) + .put("enable_position_increments", (b, v) -> b.enablePositionIncrements( + convertBoolValue(v, "enable_position_increments"))) + .put("escape", (b, v) -> b.escape(convertBoolValue(v, "escape"))) + .put("fuzziness", (b, v) -> b.fuzziness(convertFuzziness(v))) + .put("fuzzy_max_expansions", (b, v) -> b.fuzzyMaxExpansions( + convertIntValue(v, "fuzzy_max_expansions"))) + .put("fuzzy_prefix_length", (b, v) -> b.fuzzyPrefixLength( + convertIntValue(v, "fuzzy_prefix_length"))) + .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(checkRewrite(v, "fuzzy_rewrite"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("max_determinized_states", (b, v) -> b.maxDeterminizedStates( + convertIntValue(v, "max_determinized_states"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("phrase_slop", (b, v) -> b.phraseSlop(convertIntValue(v, "phrase_slop"))) + .put("quote_analyzer", (b, v) -> b.quoteAnalyzer(v.stringValue())) + .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) + .put("rewrite", (b, v) -> b.rewrite(checkRewrite(v, "rewrite"))) + .put("tie_breaker", (b, v) -> b.tieBreaker(convertFloatValue(v, "tie_breaker"))) + .put("time_zone", (b, v) -> b.timeZone(checkTimeZone(v))) + .put("type", (b, v) -> b.type(convertType(v))) + .build(); + + public static final Map> + SimpleQueryStringQueryBuildActions = ImmutableMap.>builder() + .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) + .put("analyze_wildcard", (b, v) -> b.analyzeWildcard( + convertBoolValue(v, "analyze_wildcard"))) + .put("auto_generate_synonyms_phrase_query", (b, v) -> b.autoGenerateSynonymsPhraseQuery( + convertBoolValue(v, "auto_generate_synonyms_phrase_query"))) + .put("boost", (b, v) -> b.boost(convertFloatValue(v, "boost"))) + .put("default_operator", (b, v) -> b.defaultOperator( + convertOperator(v, "default_operator"))) + .put("flags", (b, v) -> b.flags(convertFlags(v))) + .put("fuzzy_max_expansions", (b, v) -> b.fuzzyMaxExpansions( + convertIntValue(v, "fuzzy_max_expansions"))) + .put("fuzzy_prefix_length", (b, v) -> b.fuzzyPrefixLength( + convertIntValue(v, "fuzzy_prefix_length"))) + .put("fuzzy_transpositions", (b, v) -> b.fuzzyTranspositions( + convertBoolValue(v, "fuzzy_transpositions"))) + .put("lenient", (b, v) -> b.lenient(convertBoolValue(v, "lenient"))) + .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) + .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) + .build(); + + public static final Map ArgumentLimitations = + ImmutableMap.builder() + .put("boost", "Accepts only floating point values greater than 0.") + .put("tie_breaker", "Accepts only floating point values in range 0 to 1.") + .put("rewrite", "Available values are: constant_score, " + + "scoring_boolean, constant_score_boolean, top_terms_X, top_terms_boost_X, " + + "top_terms_blended_freqs_X, where X is an integer value.") + .put("flags", String.format( + "Available values are: %s and any combinations of these separated by '|'.", + Arrays.stream(SimpleQueryStringFlag.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("time_zone", "For more information, follow this link: " + + "https://docs.oracle.com/javase/8/docs/api/java/time/ZoneId.html#of-java.lang.String-") + .put("fuzziness", "Available values are: " + + "'AUTO', 'AUTO:x,y' or z, where x, y, z - integer values.") + .put("operator", String.format("Available values are: %s.", + Arrays.stream(Operator.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("type", String.format("Available values are: %s.", + Arrays.stream(MultiMatchQueryBuilder.Type.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("zero_terms_query", String.format("Available values are: %s.", + Arrays.stream(MatchQuery.ZeroTermsQuery.class.getEnumConstants()) + .map(Enum::toString).collect(Collectors.joining(", ")))) + .put("int", "Accepts only integer values.") + .put("float", "Accepts only floating point values.") + .put("bool", "Accepts only boolean values: 'true' or 'false'.") + .build(); + + + private static String formatErrorMessage(String name, String value) { + return formatErrorMessage(name, value, name); + } + + private static String formatErrorMessage(String name, String value, String limitationName) { + return String.format("Invalid %s value: '%s'. %s", + name, value, ArgumentLimitations.containsKey(name) ? ArgumentLimitations.get(name) + : ArgumentLimitations.getOrDefault(limitationName, "")); + } + + /** + * Check whether value is valid for 'rewrite' or 'fuzzy_rewrite'. + * @param value Value + * @param name Value name + * @return Converted + */ + public static String checkRewrite(ExprValue value, String name) { + try { + QueryParsers.parseRewriteMethod( + value.stringValue().toLowerCase(), null, LoggingDeprecationHandler.INSTANCE); + return value.stringValue(); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "rewrite")); + } + } + + /** + * Convert ExprValue to Flags. + * @param value Value + * @return Array of flags + */ + public static SimpleQueryStringFlag[] convertFlags(ExprValue value) { + try { + return Arrays.stream(value.stringValue().toUpperCase().split("\\|")) + .map(SimpleQueryStringFlag::valueOf) + .toArray(SimpleQueryStringFlag[]::new); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("flags", value.stringValue()), e); + } + } + + /** + * Check whether ExprValue could be converted to timezone object. + * @param value Value + * @return Converted to string + */ + public static String checkTimeZone(ExprValue value) { + try { + ZoneId.of(value.stringValue()); + return value.stringValue(); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("time_zone", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to Fuzziness object. + * @param value Value + * @return Fuzziness + */ + public static Fuzziness convertFuzziness(ExprValue value) { + try { + return Fuzziness.build(value.stringValue().toUpperCase()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("fuzziness", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to Operator object, could be used for 'operator' and 'default_operator'. + * @param value Value + * @param name Value name + * @return Operator + */ + public static Operator convertOperator(ExprValue value, String name) { + try { + return Operator.fromString(value.stringValue().toUpperCase()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "operator")); + } + } + + /** + * Convert ExprValue to Type object. + * @param value Value + * @return Type + */ + public static MultiMatchQueryBuilder.Type convertType(ExprValue value) { + try { + return MultiMatchQueryBuilder.Type.parse(value.stringValue().toLowerCase(), + LoggingDeprecationHandler.INSTANCE); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("type", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to ZeroTermsQuery object. + * @param value Value + * @return ZeroTermsQuery + */ + public static MatchQuery.ZeroTermsQuery convertZeroTermsQuery(ExprValue value) { + try { + return MatchQuery.ZeroTermsQuery.valueOf(value.stringValue().toUpperCase()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage("zero_terms_query", value.stringValue()), e); + } + } + + /** + * Convert ExprValue to int. + * @param value Value + * @param name Value name + * @return int + */ + public static int convertIntValue(ExprValue value, String name) { + try { + return Integer.parseInt(value.stringValue()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "int"), e); + } + } + + /** + * Convert ExprValue to float. + * @param value Value + * @param name Value name + * @return float + */ + public static float convertFloatValue(ExprValue value, String name) { + try { + return Float.parseFloat(value.stringValue()); + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "float"), e); + } + } + + /** + * Convert ExprValue to bool. + * @param value Value + * @param name Value name + * @return bool + */ + public static boolean convertBoolValue(ExprValue value, String name) { + try { + // Boolean.parseBoolean interprets integers or any other stuff as a valid value + Boolean res = Boolean.parseBoolean(value.stringValue()); + if (value.stringValue().equalsIgnoreCase(res.toString())) { + return res; + } else { + throw new Exception("Invalid boolean value"); + } + } catch (Exception e) { + throw new RuntimeException(formatErrorMessage(name, value.stringValue(), "bool"), e); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java index 33e357afe3..7044a56035 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchBoolPrefixQuery.java @@ -5,9 +5,7 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MatchBoolPrefixQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; /** @@ -20,18 +18,7 @@ public class MatchBoolPrefixQuery * with support of optional parameters. */ public MatchBoolPrefixQuery() { - super(ImmutableMap.>builder() - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("fuzziness", (b, v) -> b.fuzziness(v.stringValue())) - .put("prefix_length", (b, v) -> b.prefixLength(Integer.parseInt(v.stringValue()))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", - (b, v) -> b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(v.stringValue())) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("operator", (b,v) -> b.operator(Operator.fromString(v.stringValue()))) - .build()); + super(FunctionParameterRepository.MatchBoolPrefixQueryBuildActions); } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java index 6d181daa4c..8ee9ae299e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhrasePrefixQuery.java @@ -5,27 +5,19 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; import org.opensearch.index.query.QueryBuilders; /** * Lucene query that builds a match_phrase_prefix query. */ -public class MatchPhrasePrefixQuery extends SingleFieldQuery { +public class MatchPhrasePrefixQuery extends SingleFieldQuery { /** * Default constructor for MatchPhrasePrefixQuery configures how RelevanceQuery.build() handles * named arguments. */ public MatchPhrasePrefixQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .build()); + super(FunctionParameterRepository.MatchPhrasePrefixQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java index 6a7694f629..2afaca1a7a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchPhraseQuery.java @@ -5,20 +5,8 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.function.BiFunction; import org.opensearch.index.query.MatchPhraseQueryBuilder; -import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.NamedArgumentExpression; -import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery; /** * Lucene query that builds a match_phrase query. @@ -29,13 +17,7 @@ public class MatchPhraseQuery extends SingleFieldQuery * named arguments. */ public MatchPhraseQuery() { - super(ImmutableMap.>builder() - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .build()); + super(FunctionParameterRepository.MatchPhraseQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java index f6d88013e4..a4de1c0831 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MatchQuery.java @@ -5,9 +5,7 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MatchQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; /** @@ -19,23 +17,7 @@ public class MatchQuery extends SingleFieldQuery { * named arguments. */ public MatchQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("auto_generate_synonyms_phrase_query", - (b, v) -> b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("fuzziness", (b, v) -> b.fuzziness(valueOfToUpper(v))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("prefix_length", (b, v) -> b.prefixLength(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", - (b, v) -> b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(v.stringValue())) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("operator", (b, v) -> b.operator(Operator.fromString(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .build()); + super(FunctionParameterRepository.MatchQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java index b447f2ffe2..8390b5ef44 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQuery.java @@ -6,8 +6,10 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.NamedArgumentExpression; /** @@ -21,7 +23,18 @@ public MultiFieldQuery(Map> queryBuildActions) { } @Override - public T createQueryBuilder(NamedArgumentExpression fields, NamedArgumentExpression queryExpr) { + public T createQueryBuilder(List arguments) { + // Extract 'fields' and 'query' + var fields = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("fields")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'fields' parameter is missing.")); + + var query = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("query")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); + var fieldsAndWeights = fields .getValue() .valueOf(null) @@ -29,8 +42,8 @@ public T createQueryBuilder(NamedArgumentExpression fields, NamedArgumentExpress .entrySet() .stream() .collect(ImmutableMap.toImmutableMap(e -> e.getKey(), e -> e.getValue().floatValue())); - var query = queryExpr.getValue().valueOf(null).stringValue(); - return createBuilder(fieldsAndWeights, query); + + return createBuilder(fieldsAndWeights, query.getValue().valueOf(null).stringValue()); } protected abstract T createBuilder(ImmutableMap fields, String query); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java index 549f58cb19..a791bf756b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiMatchQuery.java @@ -7,7 +7,6 @@ import com.google.common.collect.ImmutableMap; import org.opensearch.index.query.MultiMatchQueryBuilder; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; public class MultiMatchQuery extends MultiFieldQuery { @@ -16,26 +15,7 @@ public class MultiMatchQuery extends MultiFieldQuery { * named arguments. */ public MultiMatchQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("auto_generate_synonyms_phrase_query", (b, v) -> - b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("cutoff_frequency", (b, v) -> b.cutoffFrequency(Float.parseFloat(v.stringValue()))) - .put("fuzziness", (b, v) -> b.fuzziness(v.stringValue())) - .put("fuzzy_transpositions", (b, v) -> - b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("max_expansions", (b, v) -> b.maxExpansions(Integer.parseInt(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("operator", (b, v) -> b.operator(Operator.fromString(v.stringValue()))) - .put("prefix_length", (b, v) -> b.prefixLength(Integer.parseInt(v.stringValue()))) - .put("tie_breaker", (b, v) -> b.tieBreaker(Float.parseFloat(v.stringValue()))) - .put("type", (b, v) -> b.type(v.stringValue())) - .put("slop", (b, v) -> b.slop(Integer.parseInt(v.stringValue()))) - .put("zero_terms_query", (b, v) -> b.zeroTermsQuery( - org.opensearch.index.search.MatchQuery.ZeroTermsQuery.valueOf(valueOfToUpper(v)))) - .build()); + super(FunctionParameterRepository.MultiMatchQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java index 21eb3f8837..43131baa3e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/QueryStringQuery.java @@ -6,19 +6,8 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; import com.google.common.collect.ImmutableMap; -import java.util.Iterator; -import java.util.Objects; -import org.opensearch.common.unit.Fuzziness; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.index.query.MultiMatchQueryBuilder; -import org.opensearch.index.query.Operator; -import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.QueryStringQueryBuilder; -import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.expression.Expression; -import org.opensearch.sql.expression.FunctionExpression; -import org.opensearch.sql.expression.NamedArgumentExpression; /** * Class for Lucene query that builds the query_string query. @@ -29,44 +18,9 @@ public class QueryStringQuery extends MultiFieldQuery { * named arguments. */ public QueryStringQuery() { - super(ImmutableMap.>builder() - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("allow_leading_wildcard", (b, v) -> - b.allowLeadingWildcard(Boolean.parseBoolean(v.stringValue()))) - .put("analyze_wildcard", (b, v) -> - b.analyzeWildcard(Boolean.parseBoolean(v.stringValue()))) - .put("auto_generate_synonyms_phrase_query", (b, v) -> - b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("default_operator", (b, v) -> - b.defaultOperator(Operator.fromString(v.stringValue()))) - .put("enable_position_increments", (b, v) -> - b.enablePositionIncrements(Boolean.parseBoolean(v.stringValue()))) - .put("fuzziness", (b, v) -> b.fuzziness(Fuzziness.build(v.stringValue()))) - .put("fuzzy_rewrite", (b, v) -> b.fuzzyRewrite(v.stringValue())) - .put("escape", (b, v) -> b.escape(Boolean.parseBoolean(v.stringValue()))) - .put("fuzzy_max_expansions", (b, v) -> - b.fuzzyMaxExpansions(Integer.parseInt(v.stringValue()))) - .put("fuzzy_prefix_length", (b, v) -> - b.fuzzyPrefixLength(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", (b, v) -> - b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("max_determinized_states", (b, v) -> - b.maxDeterminizedStates(Integer.parseInt(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("quote_analyzer", (b, v) -> b.quoteAnalyzer(v.stringValue())) - .put("phrase_slop", (b, v) -> b.phraseSlop(Integer.parseInt(v.stringValue()))) - .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) - .put("rewrite", (b, v) -> b.rewrite(v.stringValue())) - .put("type", (b, v) -> b.type(MultiMatchQueryBuilder.Type.parse(valueOfToLower(v), - LoggingDeprecationHandler.INSTANCE))) - .put("tie_breaker", (b, v) -> b.tieBreaker(Float.parseFloat(v.stringValue()))) - .put("time_zone", (b, v) -> b.timeZone(v.stringValue())) - .build()); + super(FunctionParameterRepository.QueryStringQueryBuildActions); } - /** * Builds QueryBuilder with query value and other default parameter values set. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java index 282c5478b4..579f77d2cd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQuery.java @@ -5,19 +5,16 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; -import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.function.BiFunction; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.opensearch.index.query.QueryBuilder; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.opensearch.storage.script.filter.lucene.LuceneQuery; @@ -31,26 +28,32 @@ public abstract class RelevanceQuery extends LuceneQuery @Override public QueryBuilder build(FunctionExpression func) { - List arguments = func.getArguments(); + var arguments = func.getArguments().stream() + .map(a -> (NamedArgumentExpression)a).collect(Collectors.toList()); if (arguments.size() < 2) { throw new SyntaxCheckException( String.format("%s requires at least two parameters", getQueryName())); } - NamedArgumentExpression field = (NamedArgumentExpression) arguments.get(0); - NamedArgumentExpression query = (NamedArgumentExpression) arguments.get(1); - T queryBuilder = createQueryBuilder(field, query); - Iterator iterator = arguments.listIterator(2); - Set visitedParms = new HashSet(); + // Aggregate parameters by name, so getting a Map + arguments.stream().collect(Collectors.groupingBy(a -> a.getArgName().toLowerCase())) + .forEach((k, v) -> { + if (v.size() > 1) { + throw new SemanticCheckException( + String.format("Parameter '%s' can only be specified once.", k)); + } + }); + + T queryBuilder = createQueryBuilder(arguments); + + arguments.removeIf(a -> a.getArgName().equalsIgnoreCase("field") + || a.getArgName().equalsIgnoreCase("fields") + || a.getArgName().equalsIgnoreCase("query")); + + var iterator = arguments.listIterator(); while (iterator.hasNext()) { - NamedArgumentExpression arg = (NamedArgumentExpression) iterator.next(); + NamedArgumentExpression arg = iterator.next(); String argNormalized = arg.getArgName().toLowerCase(); - if (visitedParms.contains(argNormalized)) { - throw new SemanticCheckException(String.format("Parameter '%s' can only be specified once.", - argNormalized)); - } else { - visitedParms.add(argNormalized); - } if (!queryBuildActions.containsKey(argNormalized)) { throw new SemanticCheckException( @@ -65,8 +68,7 @@ public QueryBuilder build(FunctionExpression func) { return queryBuilder; } - protected abstract T createQueryBuilder(NamedArgumentExpression field, - NamedArgumentExpression query); + protected abstract T createQueryBuilder(List arguments); protected abstract String getQueryName(); @@ -79,12 +81,4 @@ protected abstract T createQueryBuilder(NamedArgumentExpression field, protected interface QueryBuilderStep extends BiFunction { } - - public static String valueOfToUpper(ExprValue v) { - return v.stringValue().toUpperCase(); - } - - public static String valueOfToLower(ExprValue v) { - return v.stringValue().toLowerCase(); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java index 1b7c18cb2c..157921572a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SimpleQueryStringQuery.java @@ -6,13 +6,8 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; import com.google.common.collect.ImmutableMap; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Objects; -import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.SimpleQueryStringBuilder; -import org.opensearch.index.query.SimpleQueryStringFlag; public class SimpleQueryStringQuery extends MultiFieldQuery { /** @@ -20,26 +15,7 @@ public class SimpleQueryStringQuery extends MultiFieldQuery>builder() - .put("analyze_wildcard", (b, v) -> b.analyzeWildcard(Boolean.parseBoolean(v.stringValue()))) - .put("analyzer", (b, v) -> b.analyzer(v.stringValue())) - .put("auto_generate_synonyms_phrase_query", (b, v) -> - b.autoGenerateSynonymsPhraseQuery(Boolean.parseBoolean(v.stringValue()))) - .put("boost", (b, v) -> b.boost(Float.parseFloat(v.stringValue()))) - .put("default_operator", (b, v) -> b.defaultOperator(Operator.fromString(v.stringValue()))) - .put("flags", (b, v) -> b.flags(Arrays.stream(valueOfToUpper(v).split("\\|")) - .map(SimpleQueryStringFlag::valueOf) - .toArray(SimpleQueryStringFlag[]::new))) - .put("fuzzy_max_expansions", (b, v) -> - b.fuzzyMaxExpansions(Integer.parseInt(v.stringValue()))) - .put("fuzzy_prefix_length", (b, v) -> - b.fuzzyPrefixLength(Integer.parseInt(v.stringValue()))) - .put("fuzzy_transpositions", (b, v) -> - b.fuzzyTranspositions(Boolean.parseBoolean(v.stringValue()))) - .put("lenient", (b, v) -> b.lenient(Boolean.parseBoolean(v.stringValue()))) - .put("minimum_should_match", (b, v) -> b.minimumShouldMatch(v.stringValue())) - .put("quote_field_suffix", (b, v) -> b.quoteFieldSuffix(v.stringValue())) - .build()); + super(FunctionParameterRepository.SimpleQueryStringQueryBuildActions); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java index 9876c62cce..15eda7f483 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQuery.java @@ -5,8 +5,10 @@ package org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance; +import java.util.List; import java.util.Map; import org.opensearch.index.query.QueryBuilder; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.NamedArgumentExpression; /** @@ -21,9 +23,20 @@ public SingleFieldQuery(Map> queryBuildActions) { } @Override - protected T createQueryBuilder(NamedArgumentExpression fields, NamedArgumentExpression query) { + protected T createQueryBuilder(List arguments) { + // Extract 'field' and 'query' + var field = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("field")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'field' parameter is missing.")); + + var query = arguments.stream() + .filter(a -> a.getArgName().equalsIgnoreCase("query")) + .findFirst() + .orElseThrow(() -> new SemanticCheckException("'query' parameter is missing")); + return createBuilder( - fields.getValue().valueOf(null).stringValue(), + field.getValue().valueOf(null).stringValue(), query.getValue().valueOf(null).stringValue()); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java index 75ddd1dd93..ff80f3bcc0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilderTest.java @@ -332,7 +332,7 @@ void should_build_match_query_with_custom_parameters() { + " \"prefix_length\" : 0,\n" + " \"max_expansions\" : 50,\n" + " \"minimum_should_match\" : \"3\"," - + " \"fuzzy_rewrite\" : \"top_terms_N\"," + + " \"fuzzy_rewrite\" : \"top_terms_1\"," + " \"fuzzy_transpositions\" : false,\n" + " \"lenient\" : false,\n" + " \"zero_terms_query\" : \"ALL\",\n" @@ -352,7 +352,7 @@ void should_build_match_query_with_custom_parameters() { dsl.namedArgument("max_expansions", literal("50")), dsl.namedArgument("prefix_length", literal("0")), dsl.namedArgument("fuzzy_transpositions", literal("false")), - dsl.namedArgument("fuzzy_rewrite", literal("top_terms_N")), + dsl.namedArgument("fuzzy_rewrite", literal("top_terms_1")), dsl.namedArgument("lenient", literal("false")), dsl.namedArgument("minimum_should_match", literal("3")), dsl.namedArgument("zero_terms_query", literal("ALL")), @@ -366,7 +366,49 @@ void match_invalid_parameter() { dsl.namedArgument("query", literal("search query")), dsl.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("Parameter invalid_parameter is invalid for match function.", msg); + assertTrue(msg.startsWith("Parameter invalid_parameter is invalid for match function.")); + } + + @Test + void match_disallow_duplicate_parameter() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("message")), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("AnalYzer", literal("english"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("Parameter 'analyzer' can only be specified once.", msg); + } + + @Test + void match_disallow_duplicate_query() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("message")), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("QUERY", literal("something"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("Parameter 'query' can only be specified once.", msg); + } + + @Test + void match_disallow_duplicate_field() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("field", literal("message")), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword")), + dsl.namedArgument("Field", literal("something"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("Parameter 'field' can only be specified once.", msg); + } + + @Test + void match_missing_field() { + FunctionExpression expr = dsl.match( + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'field' parameter is missing.", msg); } @Test @@ -570,12 +612,13 @@ void should_build_match_phrase_query_with_custom_parameters() { + " \"analyzer\" : \"keyword\"," + " \"slop\" : 2,\n" + " \"zero_terms_query\" : \"ALL\",\n" - + " \"boost\" : 1.0\n" + + " \"boost\" : 1.2\n" + " }\n" + " }\n" + "}", buildQuery( dsl.match_phrase( + dsl.namedArgument("boost", literal("1.2")), dsl.namedArgument("field", literal("message")), dsl.namedArgument("query", literal("search query")), dsl.namedArgument("analyzer", literal("keyword")), @@ -831,32 +874,72 @@ void match_phrase_invalid_parameter() { dsl.namedArgument("query", literal("search query")), dsl.namedArgument("invalid_parameter", literal("invalid_value"))); var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("Parameter invalid_parameter is invalid for match_phrase function.", msg); + assertTrue(msg.startsWith("Parameter invalid_parameter is invalid for match_phrase function.")); } @Test - void match_phrase_invalid_value_slop() { - FunctionExpression expr = dsl.match_phrase( - dsl.namedArgument("field", literal("message")), - dsl.namedArgument("query", literal("search query")), + void relevancy_func_invalid_arg_values() { + final var field = dsl.namedArgument("field", literal("message")); + final var fields = dsl.namedArgument("fields", DSL.literal( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "field1", ExprValueUtils.floatValue(1.F), + "field2", ExprValueUtils.floatValue(.3F)))))); + final var query = dsl.namedArgument("query", literal("search query")); + + var slopTest = dsl.match_phrase(field, query, dsl.namedArgument("slop", literal("1.5"))); - var msg = assertThrows(NumberFormatException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("For input string: \"1.5\"", msg); - } + var msg = assertThrows(RuntimeException.class, () -> buildQuery(slopTest)).getMessage(); + assertEquals("Invalid slop value: '1.5'. Accepts only integer values.", msg); - @Test - void match_phrase_invalid_value_ztq() { - FunctionExpression expr = dsl.match_phrase( - dsl.namedArgument("field", literal("message")), - dsl.namedArgument("query", literal("search query")), + var ztqTest = dsl.match_phrase(field, query, dsl.namedArgument("zero_terms_query", literal("meow"))); - var msg = assertThrows(IllegalArgumentException.class, () -> buildQuery(expr)).getMessage(); - assertEquals("No enum constant org.opensearch.index.search.MatchQuery.ZeroTermsQuery.MEOW", - msg); + msg = assertThrows(RuntimeException.class, () -> buildQuery(ztqTest)).getMessage(); + assertEquals( + "Invalid zero_terms_query value: 'meow'. Available values are: NONE, ALL, NULL.", msg); + + var boostTest = dsl.match(field, query, + dsl.namedArgument("boost", literal("pewpew"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(boostTest)).getMessage(); + assertEquals( + "Invalid boost value: 'pewpew'. Accepts only floating point values greater than 0.", msg); + + var boolTest = dsl.query_string(fields, query, + dsl.namedArgument("escape", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(boolTest)).getMessage(); + assertEquals( + "Invalid escape value: '42'. Accepts only boolean values: 'true' or 'false'.", msg); + + var typeTest = dsl.multi_match(fields, query, + dsl.namedArgument("type", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(typeTest)).getMessage(); + assertTrue(msg.startsWith("Invalid type value: '42'. Available values are:")); + + var operatorTest = dsl.simple_query_string(fields, query, + dsl.namedArgument("default_operator", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(operatorTest)).getMessage(); + assertTrue(msg.startsWith("Invalid default_operator value: '42'. Available values are:")); + + var flagsTest = dsl.simple_query_string(fields, query, + dsl.namedArgument("flags", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(flagsTest)).getMessage(); + assertTrue(msg.startsWith("Invalid flags value: '42'. Available values are:")); + + var fuzzinessTest = dsl.match_bool_prefix(field, query, + dsl.namedArgument("fuzziness", literal("AUTO:"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(fuzzinessTest)).getMessage(); + assertTrue(msg.startsWith("Invalid fuzziness value: 'AUTO:'. Available values are:")); + + var rewriteTest = dsl.match_bool_prefix(field, query, + dsl.namedArgument("fuzzy_rewrite", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(rewriteTest)).getMessage(); + assertTrue(msg.startsWith("Invalid fuzzy_rewrite value: '42'. Available values are:")); + + var timezoneTest = dsl.query_string(fields, query, + dsl.namedArgument("time_zone", literal("42"))); + msg = assertThrows(RuntimeException.class, () -> buildQuery(timezoneTest)).getMessage(); + assertTrue(msg.startsWith("Invalid time_zone value: '42'.")); } - - @Test void should_build_match_bool_prefix_query_with_default_parameters() { assertJsonEquals( @@ -878,6 +961,26 @@ void should_build_match_bool_prefix_query_with_default_parameters() { dsl.namedArgument("query", literal("search query"))))); } + @Test + void multi_match_missing_fields() { + var msg = assertThrows(SemanticCheckException.class, () -> + dsl.multi_match( + dsl.namedArgument("query", literal("search query")))).getMessage(); + assertEquals("Expected type STRUCT instead of STRING for parameter #1", msg); + } + + @Test + void multi_match_missing_fields_even_with_struct() { + FunctionExpression expr = dsl.multi_match( + dsl.namedArgument("something-but-not-fields", DSL.literal( + new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of( + "pewpew", ExprValueUtils.integerValue(42)))))), + dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("analyzer", literal("keyword"))); + var msg = assertThrows(SemanticCheckException.class, () -> buildQuery(expr)).getMessage(); + assertEquals("'fields' parameter is missing.", msg); + } + @Test void should_build_match_phrase_prefix_query_with_default_parameters() { assertJsonEquals( @@ -899,7 +1002,7 @@ void should_build_match_phrase_prefix_query_with_default_parameters() { } @Test - void should_build_match_phrase_prefix_query_with_analyzer() { + void should_build_match_phrase_prefix_query_with_non_default_parameters() { assertJsonEquals( "{\n" + " \"match_phrase_prefix\" : {\n" @@ -907,8 +1010,8 @@ void should_build_match_phrase_prefix_query_with_analyzer() { + " \"query\" : \"search query\",\n" + " \"slop\" : 0,\n" + " \"zero_terms_query\" : \"NONE\",\n" - + " \"max_expansions\" : 50,\n" - + " \"boost\" : 1.0,\n" + + " \"max_expansions\" : 42,\n" + + " \"boost\" : 1.2,\n" + " \"analyzer\": english\n" + " }\n" + " }\n" @@ -917,6 +1020,8 @@ void should_build_match_phrase_prefix_query_with_analyzer() { dsl.match_phrase_prefix( dsl.namedArgument("field", literal("message")), dsl.namedArgument("query", literal("search query")), + dsl.namedArgument("boost", literal("1.2")), + dsl.namedArgument("max_expansions", literal("42")), dsl.namedArgument("analyzer", literal("english"))))); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java index 261870ca17..748384f4c8 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/MultiMatchTest.java @@ -80,7 +80,7 @@ static Stream> generateValidData() { List.of( dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), - dsl.namedArgument("fuzzy_transpositions", DSL.literal("42")) + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")) ), List.of( dsl.namedArgument("fields", fields_value), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java index 21b03abab0..4692f046db 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/QueryStringTest.java @@ -46,7 +46,7 @@ class QueryStringTest { private static final LiteralExpression query_value = DSL.literal("query_value"); static Stream> generateValidData() { - Expression field = dsl.namedArgument("field", fields_value); + Expression field = dsl.namedArgument("fields", fields_value); Expression query = dsl.namedArgument("query", query_value); return List.of( dsl.namedArgument("analyzer", DSL.literal("standard")), @@ -62,7 +62,7 @@ static Stream> generateValidData() { dsl.namedArgument("fuzzy_rewrite", DSL.literal("constant_score")), dsl.namedArgument("fuzzy_max_expansions", DSL.literal("42")), dsl.namedArgument("fuzzy_prefix_length", DSL.literal("42")), - dsl.namedArgument("fuzzy_transpositions", DSL.literal("42")), + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")), dsl.namedArgument("lenient", DSL.literal("true")), dsl.namedArgument("max_determinized_states", DSL.literal("10000")), dsl.namedArgument("minimum_should_match", DSL.literal("4")), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java index 8f06f48727..de8576e9d4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/SimpleQueryStringTest.java @@ -105,7 +105,7 @@ static Stream> generateValidData() { List.of( dsl.namedArgument("fields", fields_value), dsl.namedArgument("query", query_value), - dsl.namedArgument("fuzzy_transpositions", DSL.literal("42")) + dsl.namedArgument("fuzzy_transpositions", DSL.literal("true")) ), List.of( dsl.namedArgument("fields", fields_value), diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java index 7e4c6ea011..c50f2efb0d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/MultiFieldQueryTest.java @@ -12,6 +12,7 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,10 +49,10 @@ void createQueryBuilderTest() { var fieldSpec = ImmutableMap.builder().put(sampleField, ExprValueUtils.floatValue(sampleValue)).build(); - query.createQueryBuilder(dsl.namedArgument("fields", + query.createQueryBuilder(List.of(dsl.namedArgument("fields", new LiteralExpression(ExprTupleValue.fromExprValueMap(fieldSpec))), dsl.namedArgument("query", - new LiteralExpression(ExprValueUtils.stringValue(sampleQuery)))); + new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); verify(query).createBuilder(argThat( (ArgumentMatcher>) map -> map.size() == 1 diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java index fa6a43474a..5406f4cb58 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/RelevanceQueryBuildTest.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableMap; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import org.apache.commons.lang3.NotImplementedException; import org.junit.jupiter.api.BeforeEach; @@ -45,15 +46,16 @@ class RelevanceQueryBuildTest { public static final NamedArgumentExpression QUERY_ARG = namedArgument("query", "find me"); private RelevanceQuery query; private QueryBuilder queryBuilder; + private final Map> queryBuildActions = + ImmutableMap.>builder() + .put("boost", (k, v) -> k.boost(Float.parseFloat(v.stringValue()))).build(); @BeforeEach public void setUp() { - query = mock(RelevanceQuery.class, withSettings().useConstructor( - ImmutableMap.>builder() - .put("boost", (k, v) -> k.boost(Float.parseFloat(v.stringValue()))).build()) + query = mock(RelevanceQuery.class, withSettings().useConstructor(queryBuildActions) .defaultAnswer(Mockito.CALLS_REAL_METHODS)); queryBuilder = mock(QueryBuilder.class); - when(query.createQueryBuilder(any(), any())).thenReturn(queryBuilder); + when(query.createQueryBuilder(any())).thenReturn(queryBuilder); String queryName = "mock_query"; when(queryBuilder.queryName()).thenReturn(queryName); when(queryBuilder.getWriteableName()).thenReturn(queryName); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java index 5d35327116..d6f178b1d6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/relevance/SingleFieldQueryTest.java @@ -11,6 +11,7 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,10 +41,10 @@ void createQueryBuilderTest() { String sampleQuery = "sample query"; String sampleField = "fieldA"; - query.createQueryBuilder(dsl.namedArgument("field", + query.createQueryBuilder(List.of(dsl.namedArgument("field", new LiteralExpression(ExprValueUtils.stringValue(sampleField))), dsl.namedArgument("query", - new LiteralExpression(ExprValueUtils.stringValue(sampleQuery)))); + new LiteralExpression(ExprValueUtils.stringValue(sampleQuery))))); verify(query).createBuilder(eq(sampleField), eq(sampleQuery)); From e3fe086e2d8a04309c9ac8103da90df59b279672 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Fri, 28 Oct 2022 14:22:37 -0700 Subject: [PATCH 08/20] List tables/metrics using information_schema in source command. (#914) Signed-off-by: Vamsi Manohar Signed-off-by: Vamsi Manohar --- .../org/opensearch/sql/CatalogSchemaName.java | 21 ++++ .../org/opensearch/sql/analysis/Analyzer.java | 26 ++-- .../CatalogSchemaIdentifierNameResolver.java | 78 ++++++++++++ .../sql/analysis/model/CatalogName.java | 36 ------ .../model/CatalogSchemaIdentifierName.java | 49 -------- .../sql/analysis/model/SchemaName.java | 37 ------ .../opensearch/sql/storage/StorageEngine.java | 3 +- .../sql/utils/SystemIndexUtils.java | 32 ++--- .../opensearch/sql/analysis/AnalyzerTest.java | 35 ++++-- .../sql/analysis/AnalyzerTestBase.java | 8 +- ...talogSchemaIdentifierNameResolverTest.java | 30 +++++ .../CatalogSchemaIdentifierNameTest.java | 27 ----- .../org/opensearch/sql/config/TestConfig.java | 3 +- .../opensearch/sql/planner/PlannerTest.java | 9 +- .../sql/storage/StorageEngineTest.java | 7 +- .../sql/utils/SystemIndexUtilsTest.java | 21 +++- docs/category.json | 1 + docs/user/ppl/cmd/information_schema.rst | 57 +++++++++ docs/user/ppl/index.rst | 2 + integ-test/build.gradle | 1 + .../sql/ppl/InformationSchemaCommandIT.java | 74 ++++++++++++ .../sql/ppl/ShowCatalogsCommandIT.java | 2 +- .../storage/OpenSearchStorageEngine.java | 3 +- .../storage/OpenSearchStorageEngineTest.java | 13 +- .../plugin/catalog/CatalogServiceImpl.java | 7 +- .../catalog/CatalogServiceImplTest.java | 8 +- .../sql/ppl/config/PPLServiceConfig.java | 1 - .../opensearch/sql/ppl/PPLServiceTest.java | 2 +- .../prometheus/client/PrometheusClient.java | 6 +- .../client/PrometheusClientImpl.java | 41 ++++--- .../constants/PrometheusFieldConstants.java | 2 +- .../PrometheusDescribeMetricRequest.java | 77 ------------ .../PrometheusDescribeMetricRequest.java | 112 ++++++++++++++++++ .../system/PrometheusListMetricsRequest.java | 71 +++++++++++ .../system/PrometheusSystemRequest.java | 25 ++++ .../request/system/model/MetricMetadata.java | 27 +++++ .../response/PrometheusResponse.java | 4 +- .../PrometheusMetricDefaultSchema.java | 34 ++++++ .../storage/PrometheusMetricTable.java | 15 ++- .../storage/PrometheusStorageEngine.java | 38 ++++-- .../PrometheusDefaultImplementor.java | 4 +- .../storage/system/PrometheusSystemTable.java | 78 ++++++++++++ .../system/PrometheusSystemTableScan.java | 48 ++++++++ .../system/PrometheusSystemTableSchema.java | 39 ++++++ .../client/PrometheusClientImplTest.java | 34 +++++- .../PrometheusDescribeMetricRequestTest.java | 56 +++++++-- .../PrometheusListMetricsRequestTest.java | 89 ++++++++++++++ .../storage/PrometheusMetricScanTest.java | 15 ++- .../storage/PrometheusMetricTableTest.java | 12 +- .../storage/PrometheusStorageEngineTest.java | 35 +++++- .../system/PrometheusSystemTableScanTest.java | 44 +++++++ .../system/PrometheusSystemTableTest.java | 85 +++++++++++++ .../test/resources/all_metrics_response.json | 19 +++ 53 files changed, 1247 insertions(+), 356 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/CatalogSchemaName.java create mode 100644 core/src/main/java/org/opensearch/sql/analysis/CatalogSchemaIdentifierNameResolver.java delete mode 100644 core/src/main/java/org/opensearch/sql/analysis/model/CatalogName.java delete mode 100644 core/src/main/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierName.java delete mode 100644 core/src/main/java/org/opensearch/sql/analysis/model/SchemaName.java create mode 100644 core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java delete mode 100644 core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameTest.java create mode 100644 docs/user/ppl/cmd/information_schema.rst create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java delete mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequest.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusSystemRequest.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/model/MetricMetadata.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScan.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableSchema.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScanTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java create mode 100644 prometheus/src/test/resources/all_metrics_response.json diff --git a/core/src/main/java/org/opensearch/sql/CatalogSchemaName.java b/core/src/main/java/org/opensearch/sql/CatalogSchemaName.java new file mode 100644 index 0000000000..8dde03ca3d --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/CatalogSchemaName.java @@ -0,0 +1,21 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class CatalogSchemaName { + + private final String catalogName; + + private final String schemaName; + +} diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index e3a8ab1fe4..e050908a87 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -14,7 +14,6 @@ import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; -import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIMESTAMP; import static org.opensearch.sql.utils.MLCommonsConstants.TIME_FIELD; import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; @@ -30,7 +29,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.analysis.model.CatalogSchemaIdentifierName; +import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -137,9 +136,9 @@ public LogicalPlan visitRelation(Relation node, AnalysisContext context) { .stream() .map(Catalog::getName) .collect(Collectors.toSet()); - CatalogSchemaIdentifierName catalogSchemaIdentifierName - = new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames); - String tableName = catalogSchemaIdentifierName.getIdentifierName(); + CatalogSchemaIdentifierNameResolver catalogSchemaIdentifierNameResolver + = new CatalogSchemaIdentifierNameResolver(qualifiedName.getParts(), allowedCatalogNames); + String tableName = catalogSchemaIdentifierNameResolver.getIdentifierName(); context.push(); TypeEnvironment curEnv = context.peek(); Table table; @@ -147,9 +146,11 @@ public LogicalPlan visitRelation(Relation node, AnalysisContext context) { table = new CatalogTable(catalogService); } else { table = catalogService - .getCatalog(catalogSchemaIdentifierName.getCatalogName()) + .getCatalog(catalogSchemaIdentifierNameResolver.getCatalogName()) .getStorageEngine() - .getTable(tableName); + .getTable(new CatalogSchemaName(catalogSchemaIdentifierNameResolver.getCatalogName(), + catalogSchemaIdentifierNameResolver.getSchemaName()), + catalogSchemaIdentifierNameResolver.getIdentifierName()); } table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); @@ -181,17 +182,18 @@ public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext contex .stream() .map(Catalog::getName) .collect(Collectors.toSet()); - CatalogSchemaIdentifierName catalogSchemaIdentifierName - = new CatalogSchemaIdentifierName(qualifiedName.getParts(), allowedCatalogNames); + CatalogSchemaIdentifierNameResolver catalogSchemaIdentifierNameResolver + = new CatalogSchemaIdentifierNameResolver(qualifiedName.getParts(), allowedCatalogNames); - FunctionName functionName = FunctionName.of(catalogSchemaIdentifierName.getIdentifierName()); + FunctionName functionName + = FunctionName.of(catalogSchemaIdentifierNameResolver.getIdentifierName()); List arguments = node.getArguments().stream() .map(unresolvedExpression -> this.expressionAnalyzer.analyze(unresolvedExpression, context)) .collect(Collectors.toList()); TableFunctionImplementation tableFunctionImplementation = (TableFunctionImplementation) repository.compile( - catalogSchemaIdentifierName.getCatalogName(), functionName, arguments); - return new LogicalRelation(catalogSchemaIdentifierName.getIdentifierName(), + catalogSchemaIdentifierNameResolver.getCatalogName(), functionName, arguments); + return new LogicalRelation(catalogSchemaIdentifierNameResolver.getIdentifierName(), tableFunctionImplementation.applyArguments()); } diff --git a/core/src/main/java/org/opensearch/sql/analysis/CatalogSchemaIdentifierNameResolver.java b/core/src/main/java/org/opensearch/sql/analysis/CatalogSchemaIdentifierNameResolver.java new file mode 100644 index 0000000000..7e0d2af028 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/analysis/CatalogSchemaIdentifierNameResolver.java @@ -0,0 +1,78 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.analysis; + +import java.util.List; +import java.util.Set; + +public class CatalogSchemaIdentifierNameResolver { + + public static final String DEFAULT_CATALOG_NAME = "@opensearch"; + public static final String DEFAULT_SCHEMA_NAME = "default"; + public static final String INFORMATION_SCHEMA_NAME = "information_schema"; + + private String catalogName = DEFAULT_CATALOG_NAME; + private String schemaName = DEFAULT_SCHEMA_NAME; + private String identifierName; + + private static final String DOT = "."; + + /** + * Data model for capturing catalog, schema and identifier from + * fully qualifiedName. In the current state, it is used to capture + * CatalogSchemaTable name and CatalogSchemaFunction in case of table + * functions. + * + * @param parts parts of qualifiedName. + * @param allowedCatalogs allowedCatalogs. + */ + public CatalogSchemaIdentifierNameResolver(List parts, Set allowedCatalogs) { + List remainingParts = captureSchemaName(captureCatalogName(parts, allowedCatalogs)); + identifierName = String.join(DOT, remainingParts); + } + + public String getIdentifierName() { + return identifierName; + } + + public String getCatalogName() { + return catalogName; + } + + public String getSchemaName() { + return schemaName; + } + + + // Capture catalog name and return remaining parts(schema name and table name) + // from the fully qualified name. + private List captureCatalogName(List parts, Set allowedCatalogs) { + if (parts.size() > 1 && allowedCatalogs.contains(parts.get(0)) + || DEFAULT_CATALOG_NAME.equals(parts.get(0))) { + catalogName = parts.get(0); + return parts.subList(1, parts.size()); + } else { + return parts; + } + } + + // Capture schema name and return the remaining parts(table name ) + // in the fully qualified name. + private List captureSchemaName(List parts) { + if (parts.size() > 1 + && (DEFAULT_SCHEMA_NAME.equals(parts.get(0)) + || INFORMATION_SCHEMA_NAME.contains(parts.get(0)))) { + schemaName = parts.get(0); + return parts.subList(1, parts.size()); + } else { + return parts; + } + } + + +} diff --git a/core/src/main/java/org/opensearch/sql/analysis/model/CatalogName.java b/core/src/main/java/org/opensearch/sql/analysis/model/CatalogName.java deleted file mode 100644 index edd1c8d6ef..0000000000 --- a/core/src/main/java/org/opensearch/sql/analysis/model/CatalogName.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.opensearch.sql.analysis.model; - -import java.util.List; -import java.util.Set; -import lombok.Getter; - -@Getter -public class CatalogName { - - public static final String DEFAULT_CATALOG_NAME = ".opensearch"; - private String name = DEFAULT_CATALOG_NAME; - - /** - * Capture only if there are more parts in the name. - * - * @param parts parts. - * @param allowedCatalogs allowedCatalogs. - * @return remaining parts. - */ - List capture(List parts, Set allowedCatalogs) { - if (parts.size() > 1 && allowedCatalogs.contains(parts.get(0))) { - name = parts.get(0); - return parts.subList(1, parts.size()); - } else { - return parts; - } - } - -} diff --git a/core/src/main/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierName.java b/core/src/main/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierName.java deleted file mode 100644 index 5ce50ef567..0000000000 --- a/core/src/main/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierName.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.opensearch.sql.analysis.model; - -import java.util.List; -import java.util.Set; -import lombok.EqualsAndHashCode; - -public class CatalogSchemaIdentifierName { - private final CatalogName catalogName; - private final SchemaName schemaName; - private final String identifierName; - - private static final String DOT = "."; - - /** - * Data model for capturing catalog, schema and identifier from - * fully qualifiedName. In the current state, it is used to capture - * CatalogSchemaTable name and CatalogSchemaFunction in case of table - * functions. - * - * @param parts parts of qualifiedName. - * @param allowedCatalogs allowedCatalogs. - */ - public CatalogSchemaIdentifierName(List parts, Set allowedCatalogs) { - catalogName = new CatalogName(); - schemaName = new SchemaName(); - List remainingParts = schemaName.capture(catalogName.capture(parts, allowedCatalogs)); - identifierName = String.join(DOT, remainingParts); - } - - public String getIdentifierName() { - return identifierName; - } - - public String getCatalogName() { - return catalogName.getName(); - } - - public String getSchemaName() { - return schemaName.getName(); - } - -} diff --git a/core/src/main/java/org/opensearch/sql/analysis/model/SchemaName.java b/core/src/main/java/org/opensearch/sql/analysis/model/SchemaName.java deleted file mode 100644 index 4ea2e89588..0000000000 --- a/core/src/main/java/org/opensearch/sql/analysis/model/SchemaName.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.opensearch.sql.analysis.model; - -import java.util.List; -import lombok.Getter; - -@Getter -public class SchemaName { - - public static final String DEFAULT_SCHEMA_NAME = "default"; - public static final String INFORMATION_SCHEMA_NAME = "information_schema"; - private String name = DEFAULT_SCHEMA_NAME; - - /** - * Capture only if there are more parts in the name. - * - * @param parts parts. - * @return remaining parts. - */ - List capture(List parts) { - if (parts.size() > 1 - && (DEFAULT_SCHEMA_NAME.equals(parts.get(0)) - || INFORMATION_SCHEMA_NAME.contains(parts.get(0)))) { - name = parts.get(0); - return parts.subList(1, parts.size()); - } else { - return parts; - } - } - -} diff --git a/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java b/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java index 202b7409a9..609949578c 100644 --- a/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java +++ b/core/src/main/java/org/opensearch/sql/storage/StorageEngine.java @@ -8,6 +8,7 @@ import java.util.Collection; import java.util.Collections; +import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.expression.function.FunctionResolver; /** @@ -18,7 +19,7 @@ public interface StorageEngine { /** * Get {@link Table} from storage engine. */ - Table getTable(String name); + Table getTable(CatalogSchemaName catalogSchemaName, String tableName); /** * Get list of catalog related functions. diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 3c4f5cdf39..9ba3a67847 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -16,31 +16,33 @@ */ @UtilityClass public class SystemIndexUtils { + + public static final String TABLE_NAME_FOR_TABLES_INFO = "tables"; /** - * The prefix of all the system tables. + * The suffix of all the system tables. */ - private static final String SYS_TABLES_PREFIX = "_ODFE_SYS_TABLE"; + private static final String SYS_TABLES_SUFFIX = "ODFE_SYS_TABLE"; /** - * The prefix of all the meta tables. + * The suffix of all the meta tables. */ - private static final String SYS_META_PREFIX = SYS_TABLES_PREFIX + "_META"; + private static final String SYS_META_SUFFIX = "META_" + SYS_TABLES_SUFFIX; /** - * The prefix of all the table mappings. + * The suffix of all the table mappings. */ - private static final String SYS_MAPPINGS_PREFIX = SYS_TABLES_PREFIX + "_MAPPINGS"; + private static final String SYS_MAPPINGS_SUFFIX = "MAPPINGS_" + SYS_TABLES_SUFFIX; /** - * The _ODFE_SYS_TABLE_META.ALL contain all the table info. + * The ALL.META_ODFE_SYS_TABLE contain all the table info. */ - public static final String TABLE_INFO = SYS_META_PREFIX + ".ALL"; + public static final String TABLE_INFO = "ALL." + SYS_META_SUFFIX; public static final String CATALOGS_TABLE_NAME = ".CATALOGS"; public static Boolean isSystemIndex(String indexName) { - return indexName.startsWith(SYS_TABLES_PREFIX); + return indexName.endsWith(SYS_TABLES_SUFFIX); } /** @@ -49,7 +51,7 @@ public static Boolean isSystemIndex(String indexName) { * @return system mapping table. */ public static String mappingTable(String indexName) { - return String.join(".", SYS_MAPPINGS_PREFIX, indexName); + return String.join(".", indexName, SYS_MAPPINGS_SUFFIX); } /** @@ -58,14 +60,14 @@ public static String mappingTable(String indexName) { * @return {@link SystemTable} */ public static SystemTable systemTable(String indexName) { - final int lastDot = indexName.indexOf("."); - String prefix = indexName.substring(0, lastDot); - String tableName = indexName.substring(lastDot + 1) + final int lastDot = indexName.lastIndexOf("."); + String suffix = indexName.substring(lastDot + 1); + String tableName = indexName.substring(0, lastDot) .replace("%", "*"); - if (prefix.equalsIgnoreCase(SYS_META_PREFIX)) { + if (suffix.equalsIgnoreCase(SYS_META_SUFFIX)) { return new SystemInfoTable(tableName); - } else if (prefix.equalsIgnoreCase(SYS_MAPPINGS_PREFIX)) { + } else if (suffix.equalsIgnoreCase(SYS_MAPPINGS_SUFFIX)) { return new MetaInfoTable(tableName); } else { throw new IllegalStateException("Invalid system index name: " + indexName); diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 40c33e6d61..33e5985bd7 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -6,11 +6,11 @@ package org.opensearch.sql.analysis; -import static java.lang.Boolean.TRUE; import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; import static org.opensearch.sql.ast.dsl.AstDSL.argument; @@ -31,18 +31,15 @@ import static org.opensearch.sql.ast.tree.Sort.SortOption.DEFAULT_ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; -import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -67,7 +64,6 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.HighlightExpression; -import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalAD; @@ -133,13 +129,36 @@ public void filter_relation_with_escaped_catalog() { } @Test - public void filter_relation_with_information_schema_and_catalog() { + public void filter_relation_with_information_schema_and_prom_catalog() { assertAnalyzeEqual( LogicalPlanDSL.filter( LogicalPlanDSL.relation("tables", table), dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), AstDSL.filter( - AstDSL.relation(AstDSL.qualifiedName("prometheus","default","tables")), + AstDSL.relation(AstDSL.qualifiedName("prometheus", "information_schema", "tables")), + AstDSL.equalTo(AstDSL.field("integer_value"), AstDSL.intLiteral(1)))); + } + + @Test + public void filter_relation_with_default_schema_and_prom_catalog() { + assertAnalyzeEqual( + LogicalPlanDSL.filter( + LogicalPlanDSL.relation("tables", table), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), + AstDSL.filter( + AstDSL.relation(AstDSL.qualifiedName("prometheus", "default", "tables")), + AstDSL.equalTo(AstDSL.field("integer_value"), AstDSL.intLiteral(1)))); + } + + @Test + public void filter_relation_with_information_schema_and_os_catalog() { + assertAnalyzeEqual( + LogicalPlanDSL.filter( + LogicalPlanDSL.relation("tables", table), + dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), + AstDSL.filter( + AstDSL.relation( + AstDSL.qualifiedName(DEFAULT_CATALOG_NAME, "information_schema", "tables")), AstDSL.equalTo(AstDSL.field("integer_value"), AstDSL.intLiteral(1)))); } @@ -150,7 +169,7 @@ public void filter_relation_with_information_schema() { LogicalPlanDSL.relation("tables.test", table), dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(integerValue(1)))), AstDSL.filter( - AstDSL.relation(AstDSL.qualifiedName("information_schema","tables", "test")), + AstDSL.relation(AstDSL.qualifiedName("information_schema", "tables", "test")), AstDSL.equalTo(AstDSL.field("integer_value"), AstDSL.intLiteral(1)))); } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java index d66a6f7004..3523fed98e 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTestBase.java @@ -15,6 +15,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.analysis.symbol.SymbolTable; @@ -51,12 +52,7 @@ protected Map typeMapping() { @Bean protected StorageEngine storageEngine() { - return new StorageEngine() { - @Override - public Table getTable(String name) { - return table; - } - }; + return (catalogSchemaName, tableName) -> table; } @Bean diff --git a/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java b/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java new file mode 100644 index 0000000000..069a1d814f --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameResolverTest.java @@ -0,0 +1,30 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.analysis.model; + + +import java.util.Arrays; +import java.util.Collections; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver; + +public class CatalogSchemaIdentifierNameResolverTest { + + @Test + void testFullyQualifiedName() { + CatalogSchemaIdentifierNameResolver + catalogSchemaIdentifierNameResolver = new CatalogSchemaIdentifierNameResolver( + Arrays.asList("prom", "information_schema", "tables"), Collections.singleton("prom")); + Assertions.assertEquals("information_schema", + catalogSchemaIdentifierNameResolver.getSchemaName()); + Assertions.assertEquals("prom", catalogSchemaIdentifierNameResolver.getCatalogName()); + Assertions.assertEquals("tables", catalogSchemaIdentifierNameResolver.getIdentifierName()); + } + +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameTest.java b/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameTest.java deleted file mode 100644 index e29d3c9778..0000000000 --- a/core/src/test/java/org/opensearch/sql/analysis/model/CatalogSchemaIdentifierNameTest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.opensearch.sql.analysis.model; - - -import java.util.Arrays; -import java.util.Collections; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class CatalogSchemaIdentifierNameTest { - - @Test - void testFullyQualifiedName() { - CatalogSchemaIdentifierName catalogSchemaIdentifierName = new CatalogSchemaIdentifierName( - Arrays.asList("prom", "information_schema", "tables"), Collections.singleton("prom")); - Assertions.assertEquals("information_schema", catalogSchemaIdentifierName.getSchemaName()); - Assertions.assertEquals("prom", catalogSchemaIdentifierName.getCatalogName()); - Assertions.assertEquals("tables", catalogSchemaIdentifierName.getIdentifierName()); - } - -} diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index 3112dc9234..c41bff91f2 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -8,6 +8,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.analysis.symbol.Namespace; import org.opensearch.sql.analysis.symbol.Symbol; import org.opensearch.sql.analysis.symbol.SymbolTable; @@ -62,7 +63,7 @@ public class TestConfig { protected StorageEngine storageEngine() { return new StorageEngine() { @Override - public Table getTable(String name) { + public Table getTable(CatalogSchemaName catalogSchemaName, String name) { return new Table() { @Override public Map getFieldTypes() { diff --git a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java index 32e9d1b45b..91f4e7ba45 100644 --- a/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/PlannerTest.java @@ -12,6 +12,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; +import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -24,6 +25,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.planner.logical.LogicalAggregation; @@ -56,7 +58,7 @@ public class PlannerTest extends PhysicalPlanTestBase { @BeforeEach public void setUp() { - when(storageEngine.getTable(any())).thenReturn(new MockTable()); + when(storageEngine.getTable(any(), any())).thenReturn(new MockTable()); } @Test @@ -77,7 +79,10 @@ public void planner_test() { LogicalPlanDSL.rename( LogicalPlanDSL.aggregation( LogicalPlanDSL.filter( - LogicalPlanDSL.relation("schema", storageEngine.getTable("schema")), + LogicalPlanDSL.relation("schema", + storageEngine.getTable( + new CatalogSchemaName(DEFAULT_CATALOG_NAME, "default"), + "schema")), dsl.equal(DSL.ref("response", INTEGER), DSL.literal(10)) ), ImmutableList.of(DSL.named("avg(response)", dsl.avg(DSL.ref("response", INTEGER)))), diff --git a/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java index bdebe33fd4..b0da30212d 100644 --- a/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java +++ b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java @@ -16,12 +16,7 @@ public class StorageEngineTest { @Test void testFunctionsMethod() { - StorageEngine k = new StorageEngine() { - @Override - public Table getTable(String name) { - return null; - } - }; + StorageEngine k = (catalogSchemaName, tableName) -> null; Assertions.assertEquals(Collections.emptyList(), k.getFunctions()); } diff --git a/core/src/test/java/org/opensearch/sql/utils/SystemIndexUtilsTest.java b/core/src/test/java/org/opensearch/sql/utils/SystemIndexUtilsTest.java index f1b94a409b..81d28f40db 100644 --- a/core/src/test/java/org/opensearch/sql/utils/SystemIndexUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/utils/SystemIndexUtilsTest.java @@ -20,18 +20,18 @@ class SystemIndexUtilsTest { @Test void test_system_index() { - assertTrue(isSystemIndex("_ODFE_SYS_TABLE_META.ALL")); + assertTrue(isSystemIndex("ALL.META_ODFE_SYS_TABLE")); assertFalse(isSystemIndex(".opensearch_dashboards")); } @Test void test_compose_mapping_table() { - assertEquals("_ODFE_SYS_TABLE_MAPPINGS.employee", mappingTable("employee")); + assertEquals("employee.MAPPINGS_ODFE_SYS_TABLE", mappingTable("employee")); } @Test void test_system_info_table() { - final SystemIndexUtils.SystemTable table = systemTable("_ODFE_SYS_TABLE_META.ALL"); + final SystemIndexUtils.SystemTable table = systemTable("ALL.META_ODFE_SYS_TABLE"); assertTrue(table.isSystemInfoTable()); assertFalse(table.isMetaInfoTable()); @@ -40,17 +40,26 @@ void test_system_info_table() { @Test void test_mapping_info_table() { - final SystemIndexUtils.SystemTable table = systemTable("_ODFE_SYS_TABLE_MAPPINGS.employee"); + final SystemIndexUtils.SystemTable table = systemTable("employee.MAPPINGS_ODFE_SYS_TABLE"); assertTrue(table.isMetaInfoTable()); assertFalse(table.isSystemInfoTable()); assertEquals("employee", table.getTableName()); } + @Test + void test_mapping_info_table_with_special_index_name() { + final SystemIndexUtils.SystemTable table + = systemTable("logs-2021.01.11.MAPPINGS_ODFE_SYS_TABLE"); + assertTrue(table.isMetaInfoTable()); + assertFalse(table.isSystemInfoTable()); + assertEquals("logs-2021.01.11", table.getTableName()); + } + @Test void throw_exception_for_invalid_index() { final IllegalStateException exception = - assertThrows(IllegalStateException.class, () -> systemTable("_ODFE_SYS_TABLE.employee")); - assertEquals("Invalid system index name: _ODFE_SYS_TABLE.employee", exception.getMessage()); + assertThrows(IllegalStateException.class, () -> systemTable("employee._ODFE_SYS_TABLE")); + assertEquals("Invalid system index name: employee._ODFE_SYS_TABLE", exception.getMessage()); } } diff --git a/docs/category.json b/docs/category.json index 203b641ec2..0b11209111 100644 --- a/docs/category.json +++ b/docs/category.json @@ -11,6 +11,7 @@ "user/ppl/cmd/dedup.rst", "user/ppl/cmd/describe.rst", "user/ppl/cmd/showcatalogs.rst", + "user/ppl/cmd/information_schema.rst", "user/ppl/cmd/eval.rst", "user/ppl/cmd/fields.rst", "user/ppl/cmd/grok.rst", diff --git a/docs/user/ppl/cmd/information_schema.rst b/docs/user/ppl/cmd/information_schema.rst new file mode 100644 index 0000000000..a756fb080e --- /dev/null +++ b/docs/user/ppl/cmd/information_schema.rst @@ -0,0 +1,57 @@ +========================================= +Metadata queries using information_schema +========================================= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| Use ``information_schema`` in source command to query tables information under a catalog. +In the current state, ``information_schema`` only support metadata of tables. +This schema will be extended for views, columns and other metadata info in future. + + +Syntax +============ +source = catalog.information_schema.tables; + +Example 1: Fetch tables in prometheus catalog. +============================================== + +The examples fetches tables in the prometheus catalog. + +PPL query for fetching PROMETHEUS TABLES with where clause:: + + os> source = my_prometheus.information_schema.tables | where TABLE_NAME='prometheus_http_requests_total' + fetched rows / total rows = 1/1 + +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | + |-----------------+----------------+--------------------------------+--------------+--------+---------------------------| + | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | + +-----------------+----------------+--------------------------------+--------------+--------+---------------------------+ + + +Example 2: Search tables in prometheus catalog. +=============================================== + +The examples searches tables in the prometheus catalog. + +PPL query for searching PROMETHEUS TABLES:: + + os> source = my_prometheus.information_schema.tables | where LIKE(TABLE_NAME, "%http%"); + fetched rows / total rows = 6/6 + +-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | + |-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------| + | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | + | my_prometheus | default | promhttp_metric_handler_requests_in_flight | gauge | | Current number of scrapes being served. | + | my_prometheus | default | prometheus_http_request_duration_seconds | histogram | | Histogram of latencies for HTTP requests. | + | my_prometheus | default | prometheus_sd_http_failures_total | counter | | Number of HTTP service discovery refresh failures. | + | my_prometheus | default | promhttp_metric_handler_requests_total | counter | | Total number of scrapes by HTTP status code. | + | my_prometheus | default | prometheus_http_response_size_bytes | histogram | | Histogram of response size for HTTP requests. | + +-----------------+----------------+--------------------------------------------+--------------+--------+----------------------------------------------------+ diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 0af43c3b33..3165569908 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -76,6 +76,8 @@ The query start with search command and then flowing a set of command delimited - `top command `_ + - `metadata commands `_ + * **Functions** - `Expressions `_ diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 4ee35bac2e..5c3e124f2a 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -109,6 +109,7 @@ testClusters.integTest { keystore 'plugins.query.federation.catalog.config', new File("$projectDir/src/test/resources/catalog/", 'catalog.json') } task startPrometheus(type: SpawnProcessTask) { + mustRunAfter ':doctest:doctest' doFirst { download.run { src 'https://github.com/prometheus/prometheus/releases/download/v2.39.1/prometheus-2.39.1.linux-amd64.tar.gz' diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java new file mode 100644 index 0000000000..3f9191c9c9 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java @@ -0,0 +1,74 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyColumn; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class InformationSchemaCommandIT extends PPLIntegTestCase { + + @Test + public void testSearchTablesFromPrometheusCatalog() throws IOException { + JSONObject result = + executeQuery("source=my_prometheus.information_schema.tables " + + "| where LIKE(TABLE_NAME, '%http%')"); + this.logger.error(result.toString()); + verifyColumn( + result, + columnName("TABLE_CATALOG"), + columnName("TABLE_SCHEMA"), + columnName("TABLE_NAME"), + columnName("TABLE_TYPE"), + columnName("UNIT"), + columnName("REMARKS") + ); + verifyDataRows(result, + rows("my_prometheus", "default", "promhttp_metric_handler_requests_in_flight", + "gauge", "", "Current number of scrapes being served."), + rows("my_prometheus", "default", "prometheus_sd_http_failures_total", + "counter", "", "Number of HTTP service discovery refresh failures."), + rows("my_prometheus", "default", "promhttp_metric_handler_requests_total", + "counter", "", "Total number of scrapes by HTTP status code."), + rows("my_prometheus", "default", "prometheus_http_request_duration_seconds", + "histogram", "", "Histogram of latencies for HTTP requests."), + rows("my_prometheus", "default", "prometheus_http_requests_total", + "counter", "", "Counter of HTTP requests."), + rows("my_prometheus", "default", "prometheus_http_response_size_bytes", + "histogram", "", "Histogram of response size for HTTP requests.")); + } + + + @Test + public void testTablesFromPrometheusCatalog() throws IOException { + JSONObject result = + executeQuery( + "source = my_prometheus.information_schema.tables " + + "| where TABLE_NAME='prometheus_http_requests_total'"); + this.logger.error(result.toString()); + verifyColumn( + result, + columnName("TABLE_CATALOG"), + columnName("TABLE_SCHEMA"), + columnName("TABLE_NAME"), + columnName("TABLE_TYPE"), + columnName("UNIT"), + columnName("REMARKS") + ); + verifyDataRows(result, + rows("my_prometheus", + "default", "prometheus_http_requests_total", + "counter", "", "Counter of HTTP requests.")); + } + +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java index 11b94f8da6..23418366be 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowCatalogsCommandIT.java @@ -23,7 +23,7 @@ public void testShowCatalogsCommands() throws IOException { JSONObject result = executeQuery("show catalogs"); verifyDataRows(result, rows("my_prometheus", "PROMETHEUS"), - rows(".opensearch", "OPENSEARCH")); + rows("@opensearch", "OPENSEARCH")); verifyColumn( result, columnName("CATALOG_NAME"), diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index b0b1290381..9a9847dd8c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -9,6 +9,7 @@ import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import lombok.RequiredArgsConstructor; +import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; @@ -25,7 +26,7 @@ public class OpenSearchStorageEngine implements StorageEngine { private final Settings settings; @Override - public Table getTable(String name) { + public Table getTable(CatalogSchemaName catalogSchemaName, String name) { if (isSystemIndex(name)) { return new OpenSearchSystemIndex(client, name); } else { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index d2cb4460e7..dd660d54a1 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -8,12 +8,14 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.CatalogSchemaName; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; @@ -22,21 +24,24 @@ @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { - @Mock private OpenSearchClient client; + @Mock + private OpenSearchClient client; - @Mock private Settings settings; + @Mock + private Settings settings; @Test public void getTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - Table table = engine.getTable("test"); + Table table = engine.getTable(new CatalogSchemaName(DEFAULT_CATALOG_NAME, "default"), "test"); assertNotNull(table); } @Test public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - Table table = engine.getTable(TABLE_INFO); + Table table = engine.getTable(new CatalogSchemaName(DEFAULT_CATALOG_NAME, "default"), + TABLE_INFO); assertNotNull(table); assertTrue(table instanceof OpenSearchSystemIndex); } diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java index 6edf7e998b..c462c5a596 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java @@ -5,6 +5,8 @@ package org.opensearch.sql.plugin.catalog; +import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -23,7 +25,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.settings.Settings; -import org.opensearch.sql.analysis.model.CatalogName; import org.opensearch.sql.catalog.CatalogService; import org.opensearch.sql.catalog.model.Catalog; import org.opensearch.sql.catalog.model.CatalogMetadata; @@ -103,8 +104,8 @@ public void registerDefaultOpenSearchCatalog(StorageEngine storageEngine) { if (storageEngine == null) { throw new IllegalArgumentException("Default storage engine can't be null"); } - catalogMap.put(CatalogName.DEFAULT_CATALOG_NAME, - new Catalog(CatalogName.DEFAULT_CATALOG_NAME, ConnectorType.OPENSEARCH, storageEngine)); + catalogMap.put(DEFAULT_CATALOG_NAME, + new Catalog(DEFAULT_CATALOG_NAME, ConnectorType.OPENSEARCH, storageEngine)); } private T doPrivileged(PrivilegedExceptionAction action) { diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java index 624467b981..4d339f384b 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java @@ -5,6 +5,8 @@ package org.opensearch.sql.plugin.catalog; +import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.DEFAULT_CATALOG_NAME; + import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; @@ -92,13 +94,13 @@ public void testGetStorageEngineAfterGetCatalogs() { CatalogServiceImpl.getInstance().loadConnectors(settings); CatalogServiceImpl.getInstance().registerDefaultOpenSearchCatalog(storageEngine); Set expected = new HashSet<>(); - expected.add(new Catalog(".opensearch", ConnectorType.OPENSEARCH, storageEngine)); + expected.add(new Catalog(DEFAULT_CATALOG_NAME, ConnectorType.OPENSEARCH, storageEngine)); Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); Assert.assertEquals(storageEngine, - CatalogServiceImpl.getInstance().getCatalog(".opensearch").getStorageEngine()); + CatalogServiceImpl.getInstance().getCatalog(DEFAULT_CATALOG_NAME).getStorageEngine()); Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); Assert.assertEquals(storageEngine, - CatalogServiceImpl.getInstance().getCatalog(".opensearch").getStorageEngine()); + CatalogServiceImpl.getInstance().getCatalog(DEFAULT_CATALOG_NAME).getStorageEngine()); IllegalArgumentException illegalArgumentException = Assert.assertThrows(IllegalArgumentException.class, () -> CatalogServiceImpl.getInstance().getCatalog("test")); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java index 4410f8e33f..fb20fa2713 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/config/PPLServiceConfig.java @@ -7,7 +7,6 @@ package org.opensearch.sql.ppl.config; import org.opensearch.sql.catalog.CatalogService; -import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java index 0ecebf160d..f53f333b17 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/PPLServiceTest.java @@ -78,7 +78,7 @@ public class PPLServiceTest { public void setUp() { when(table.getFieldTypes()).thenReturn(ImmutableMap.of("a", ExprCoreType.INTEGER)); when(table.implement(any())).thenReturn(plan); - when(storageEngine.getTable(any())).thenReturn(table); + when(storageEngine.getTable(any(), any())).thenReturn(table); when(catalogService.getCatalogs()) .thenReturn(Set.of(new Catalog("prometheus", ConnectorType.PROMETHEUS, storageEngine))); when(catalogService.getCatalog(any())) diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClient.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClient.java index c00e846866..ebc3e2dd39 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClient.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClient.java @@ -7,11 +7,15 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import org.json.JSONObject; +import org.opensearch.sql.prometheus.request.system.model.MetricMetadata; public interface PrometheusClient { JSONObject queryRange(String query, Long start, Long end, String step) throws IOException; List getLabels(String metricName) throws IOException; -} \ No newline at end of file + + Map> getAllMetrics() throws IOException; +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java index e41896f143..cd9aed4d18 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java @@ -5,10 +5,14 @@ package org.opensearch.sql.prometheus.client; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.net.URI; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -18,6 +22,7 @@ import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; +import org.opensearch.sql.prometheus.request.system.model.MetricMetadata; public class PrometheusClientImpl implements PrometheusClient { @@ -35,21 +40,11 @@ public PrometheusClientImpl(OkHttpClient okHttpClient, URI uri) { @Override public JSONObject queryRange(String query, Long start, Long end, String step) throws IOException { - HttpUrl httpUrl = new HttpUrl.Builder() - .scheme(uri.getScheme()) - .host(uri.getHost()) - .port(uri.getPort()) - .addPathSegment("api") - .addPathSegment("v1") - .addPathSegment("query_range") - .addQueryParameter("query", query) - .addQueryParameter("start", Long.toString(start)) - .addQueryParameter("end", Long.toString(end)) - .addQueryParameter("step", step) - .build(); - logger.debug("queryUrl: " + httpUrl); + String queryUrl = String.format("%s/api/v1/query_range?query=%s&start=%s&end=%s&step=%s", + uri.toString().replaceAll("/$", ""), query, start, end, step); + logger.debug("queryUrl: " + queryUrl); Request request = new Request.Builder() - .url(httpUrl) + .url(queryUrl) .build(); Response response = this.okHttpClient.newCall(request).execute(); JSONObject jsonObject = readResponse(response); @@ -58,7 +53,8 @@ public JSONObject queryRange(String query, Long start, Long end, String step) th @Override public List getLabels(String metricName) throws IOException { - String queryUrl = String.format("%sapi/v1/labels?match[]=%s", uri.toString(), metricName); + String queryUrl = String.format("%s/api/v1/labels?match[]=%s", + uri.toString().replaceAll("/$", ""), metricName); logger.debug("queryUrl: " + queryUrl); Request request = new Request.Builder() .url(queryUrl) @@ -68,6 +64,21 @@ public List getLabels(String metricName) throws IOException { return toListOfStrings(jsonObject.getJSONArray("data")); } + @Override + public Map> getAllMetrics() throws IOException { + String queryUrl = String.format("%s/api/v1/metadata", + uri.toString().replaceAll("/$", "")); + logger.debug("queryUrl: " + queryUrl); + Request request = new Request.Builder() + .url(queryUrl) + .build(); + Response response = this.okHttpClient.newCall(request).execute(); + JSONObject jsonObject = readResponse(response); + TypeReference>> typeRef + = new TypeReference<>() {}; + return new ObjectMapper().readValue(jsonObject.getJSONObject("data").toString(), typeRef); + } + private List toListOfStrings(JSONArray array) { List result = new ArrayList<>(); for (int i = 0; i < array.length(); i++) { diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/data/constants/PrometheusFieldConstants.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/data/constants/PrometheusFieldConstants.java index 65b1a39c23..1afab200b3 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/data/constants/PrometheusFieldConstants.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/data/constants/PrometheusFieldConstants.java @@ -8,5 +8,5 @@ public class PrometheusFieldConstants { public static final String TIMESTAMP = "@timestamp"; public static final String VALUE = "@value"; - public static final String METRIC = "metric"; + public static final String LABELS = "@labels"; } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequest.java deleted file mode 100644 index 3846f567c8..0000000000 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - - -package org.opensearch.sql.prometheus.request; - -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; - -import java.io.IOException; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import lombok.ToString; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.prometheus.client.PrometheusClient; - -/** - * Describe Metric metadata request. - * This is triggered in case of both query range table function and relation. - * In case of table function metric name is null. - */ -@ToString(onlyExplicitlyIncluded = true) -public class PrometheusDescribeMetricRequest { - - private final PrometheusClient prometheusClient; - - @ToString.Include - private final Optional metricName; - - private static final Logger LOG = LogManager.getLogger(); - - - public PrometheusDescribeMetricRequest(PrometheusClient prometheusClient, - String metricName) { - this.prometheusClient = prometheusClient; - this.metricName = Optional.ofNullable(metricName); - } - - - /** - * Get the mapping of field and type. - * - * @return mapping of field and type. - */ - public Map getFieldTypes() { - Map fieldTypes = new HashMap<>(); - AccessController.doPrivileged((PrivilegedAction>) () -> { - if (metricName.isPresent()) { - try { - prometheusClient.getLabels(metricName.get()) - .forEach(label -> fieldTypes.put(label, ExprCoreType.STRING)); - } catch (IOException e) { - LOG.error("Error while fetching labels for {} from prometheus: {}", - metricName, e.getMessage()); - throw new RuntimeException(String.format("Error while fetching labels " - + "for %s from prometheus: %s", metricName.get(), e.getMessage())); - } - } - return null; - }); - fieldTypes.put(VALUE, ExprCoreType.DOUBLE); - fieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); - fieldTypes.put(METRIC, ExprCoreType.STRING); - return fieldTypes; - } - -} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java new file mode 100644 index 0000000000..9b76cbff27 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusDescribeMetricRequest.java @@ -0,0 +1,112 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + + +package org.opensearch.sql.prometheus.request.system; + +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.NonNull; +import lombok.ToString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.storage.PrometheusMetricDefaultSchema; + +/** + * Describe Metric metadata request. + * This is triggered in case of both query range table function and relation. + * In case of table function metric name is null. + */ +@ToString(onlyExplicitlyIncluded = true) +public class PrometheusDescribeMetricRequest implements PrometheusSystemRequest { + + private final PrometheusClient prometheusClient; + + @ToString.Include + private final String metricName; + + private final CatalogSchemaName catalogSchemaName; + + private static final Logger LOG = LogManager.getLogger(); + + /** + * Constructor for Prometheus Describe Metric Request. + * In case of pass through queries like query_range function, + * metric names are optional. + * + * @param prometheusClient prometheusClient. + * @param catalogSchemaName catalogSchemaName. + * @param metricName metricName. + */ + public PrometheusDescribeMetricRequest(PrometheusClient prometheusClient, + CatalogSchemaName catalogSchemaName, + @NonNull String metricName) { + this.prometheusClient = prometheusClient; + this.metricName = metricName; + this.catalogSchemaName = catalogSchemaName; + } + + + /** + * Get the mapping of field and type. + * Returns labels and default schema fields. + * + * @return mapping of field and type. + */ + public Map getFieldTypes() { + Map fieldTypes = new HashMap<>(); + AccessController.doPrivileged((PrivilegedAction>) () -> { + try { + prometheusClient.getLabels(metricName) + .forEach(label -> fieldTypes.put(label, ExprCoreType.STRING)); + } catch (IOException e) { + LOG.error("Error while fetching labels for {} from prometheus: {}", + metricName, e.getMessage()); + throw new RuntimeException(String.format("Error while fetching labels " + + "for %s from prometheus: %s", metricName, e.getMessage())); + } + return null; + }); + fieldTypes.putAll(PrometheusMetricDefaultSchema.DEFAULT_MAPPING.getMapping()); + return fieldTypes; + } + + @Override + public List search() { + List results = new ArrayList<>(); + for (Map.Entry entry : getFieldTypes().entrySet()) { + results.add(row(entry.getKey(), entry.getValue().legacyTypeName().toLowerCase(), + catalogSchemaName)); + } + return results; + } + + private ExprTupleValue row(String fieldName, String fieldType, + CatalogSchemaName catalogSchemaName) { + LinkedHashMap valueMap = new LinkedHashMap<>(); + valueMap.put("TABLE_CATALOG", stringValue(catalogSchemaName.getCatalogName())); + valueMap.put("TABLE_SCHEMA", stringValue(catalogSchemaName.getSchemaName())); + valueMap.put("TABLE_NAME", stringValue(metricName)); + valueMap.put("COLUMN_NAME", stringValue(fieldName)); + valueMap.put("DATA_TYPE", stringValue(fieldType)); + return new ExprTupleValue(valueMap); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java new file mode 100644 index 0000000000..c4dbbebde1 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusListMetricsRequest.java @@ -0,0 +1,71 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.request.system; + +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; + +import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.request.system.model.MetricMetadata; + +@RequiredArgsConstructor +public class PrometheusListMetricsRequest implements PrometheusSystemRequest { + + private final PrometheusClient prometheusClient; + + private final CatalogSchemaName catalogSchemaName; + + private static final Logger LOG = LogManager.getLogger(); + + + @Override + public List search() { + return AccessController.doPrivileged((PrivilegedAction>) () -> { + try { + Map> result = prometheusClient.getAllMetrics(); + return result.keySet() + .stream() + .map(x -> { + MetricMetadata metricMetadata = result.get(x).get(0); + return row(x, metricMetadata.getType(), + metricMetadata.getUnit(), metricMetadata.getHelp()); + }) + .collect(Collectors.toList()); + } catch (IOException e) { + LOG.error("Error while fetching metric list for from prometheus: {}", + e.getMessage()); + throw new RuntimeException(String.format("Error while fetching metric list " + + "for from prometheus: %s", e.getMessage())); + } + }); + + } + + private ExprTupleValue row(String metricName, String tableType, String unit, String help) { + LinkedHashMap valueMap = new LinkedHashMap<>(); + valueMap.put("TABLE_CATALOG", stringValue(catalogSchemaName.getCatalogName())); + valueMap.put("TABLE_SCHEMA", stringValue("default")); + valueMap.put("TABLE_NAME", stringValue(metricName)); + valueMap.put("TABLE_TYPE", stringValue(tableType)); + valueMap.put("UNIT", stringValue(unit)); + valueMap.put("REMARKS", stringValue(help)); + return new ExprTupleValue(valueMap); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusSystemRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusSystemRequest.java new file mode 100644 index 0000000000..e68ad22c30 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/PrometheusSystemRequest.java @@ -0,0 +1,25 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.request.system; + +import java.util.List; +import org.opensearch.sql.data.model.ExprValue; + +/** + * Prometheus system request query to get metadata Info. + */ +public interface PrometheusSystemRequest { + + /** + * Search. + * + * @return list of ExprValue. + */ + List search(); + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/model/MetricMetadata.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/model/MetricMetadata.java new file mode 100644 index 0000000000..195d56f405 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/system/model/MetricMetadata.java @@ -0,0 +1,27 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.request.system.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode +@JsonIgnoreProperties(ignoreUnknown = true) +public class MetricMetadata { + private String type; + private String help; + private String unit; +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java index 6b98425491..98a54a7731 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java @@ -5,7 +5,7 @@ package org.opensearch.sql.prometheus.response; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; @@ -47,7 +47,7 @@ public Iterator iterator() { linkedHashMap.put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli((long) (val.getDouble(0) * 1000)))); linkedHashMap.put(VALUE, new ExprDoubleValue(val.getDouble(1))); - linkedHashMap.put(METRIC, new ExprStringValue(metric.toString())); + linkedHashMap.put(LABELS, new ExprStringValue(metric.toString())); result.add(new ExprTupleValue(linkedHashMap)); } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java new file mode 100644 index 0000000000..5fc7edfc73 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java @@ -0,0 +1,34 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage; + +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; + +@Getter +@RequiredArgsConstructor +public enum PrometheusMetricDefaultSchema { + + DEFAULT_MAPPING(new ImmutableMap.Builder() + .put(TIMESTAMP, ExprCoreType.TIMESTAMP) + .put(VALUE, ExprCoreType.DOUBLE) + .put(LABELS, ExprCoreType.STRING) + .build()); + + private final Map mapping; + + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java index c2a637a85a..3d8ef69b17 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java @@ -6,7 +6,6 @@ package org.opensearch.sql.prometheus.storage; -import com.google.common.annotations.VisibleForTesting; import java.util.Map; import java.util.Optional; import javax.annotation.Nonnull; @@ -15,13 +14,15 @@ import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.prometheus.client.PrometheusClient; -import org.opensearch.sql.prometheus.request.PrometheusDescribeMetricRequest; import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; +import org.opensearch.sql.prometheus.request.system.PrometheusDescribeMetricRequest; import org.opensearch.sql.prometheus.storage.implementor.PrometheusDefaultImplementor; import org.opensearch.sql.storage.Table; /** * Prometheus table (metric) implementation. + * This can be constructed from a metric Name + * or from PrometheusQueryRequest In case of query_range table function. */ public class PrometheusMetricTable implements Table { @@ -61,9 +62,13 @@ public PrometheusMetricTable(PrometheusClient prometheusService, @Override public Map getFieldTypes() { if (cachedFieldTypes == null) { - cachedFieldTypes = - new PrometheusDescribeMetricRequest(prometheusClient, - metricName.orElse(null)).getFieldTypes(); + if (metricName.isPresent()) { + cachedFieldTypes = + new PrometheusDescribeMetricRequest(prometheusClient, null, + metricName.orElse(null)).getFieldTypes(); + } else { + cachedFieldTypes = PrometheusMetricDefaultSchema.DEFAULT_MAPPING.getMapping(); + } } return cachedFieldTypes; } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java index 948cbabc44..f18ff1bba8 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java @@ -7,34 +7,58 @@ package org.opensearch.sql.prometheus.storage; +import static org.opensearch.sql.analysis.CatalogSchemaIdentifierNameResolver.INFORMATION_SCHEMA_NAME; +import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; + import java.util.Collection; import java.util.Collections; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.prometheus.client.PrometheusClient; import org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver; +import org.opensearch.sql.prometheus.storage.system.PrometheusSystemTable; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; /** * Prometheus storage engine implementation. */ +@RequiredArgsConstructor public class PrometheusStorageEngine implements StorageEngine { private final PrometheusClient prometheusClient; - public PrometheusStorageEngine(PrometheusClient prometheusClient) { - this.prometheusClient = prometheusClient; + @Override + public Collection getFunctions() { + return Collections.singletonList( + new QueryRangeTableFunctionResolver(prometheusClient)); } @Override - public Table getTable(String name) { - return null; + public Table getTable(CatalogSchemaName catalogSchemaName, String tableName) { + if (isSystemIndex(tableName)) { + return new PrometheusSystemTable(prometheusClient, catalogSchemaName, tableName); + } else if (INFORMATION_SCHEMA_NAME.equals(catalogSchemaName.getSchemaName())) { + return resolveInformationSchemaTable(catalogSchemaName, tableName); + } else { + return null; + } } - @Override - public Collection getFunctions() { - return Collections.singletonList(new QueryRangeTableFunctionResolver(prometheusClient)); + private Table resolveInformationSchemaTable(CatalogSchemaName catalogSchemaName, + String tableName) { + if (SystemIndexUtils.TABLE_NAME_FOR_TABLES_INFO.equals(tableName)) { + return new PrometheusSystemTable(prometheusClient, + catalogSchemaName, SystemIndexUtils.TABLE_INFO); + } else { + throw new SemanticCheckException( + String.format("Information Schema doesn't contain %s table", tableName)); + } } + } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java index f6b8b56e63..6235ecc14b 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java @@ -7,7 +7,7 @@ package org.opensearch.sql.prometheus.storage.implementor; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; @@ -43,7 +43,7 @@ public PhysicalPlan visitRelation(LogicalRelation node, public PhysicalPlan visitProject(LogicalProject node, PrometheusMetricScan context) { List finalProjectList = new ArrayList<>(); finalProjectList.add( - new NamedExpression(METRIC, new ReferenceExpression(METRIC, ExprCoreType.STRING))); + new NamedExpression(LABELS, new ReferenceExpression(LABELS, ExprCoreType.STRING))); finalProjectList.add( new NamedExpression(TIMESTAMP, new ReferenceExpression(TIMESTAMP, ExprCoreType.TIMESTAMP))); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java new file mode 100644 index 0000000000..2d185d4a5d --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTable.java @@ -0,0 +1,78 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.prometheus.storage.system; + + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.request.system.PrometheusDescribeMetricRequest; +import org.opensearch.sql.prometheus.request.system.PrometheusListMetricsRequest; +import org.opensearch.sql.prometheus.request.system.PrometheusSystemRequest; +import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** + * Prometheus System Table Implementation. + */ +public class PrometheusSystemTable implements Table { + /** + * System Index Name. + */ + private final Pair systemIndexBundle; + + private final CatalogSchemaName catalogSchemaName; + + public PrometheusSystemTable( + PrometheusClient client, CatalogSchemaName catalogSchemaName, String indexName) { + this.catalogSchemaName = catalogSchemaName; + this.systemIndexBundle = buildIndexBundle(client, indexName); + } + + @Override + public Map getFieldTypes() { + return systemIndexBundle.getLeft().getMapping(); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return plan.accept(new PrometheusSystemTableDefaultImplementor(), null); + } + + @VisibleForTesting + @RequiredArgsConstructor + public class PrometheusSystemTableDefaultImplementor + extends DefaultImplementor { + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new PrometheusSystemTableScan(systemIndexBundle.getRight()); + } + } + + private Pair buildIndexBundle( + PrometheusClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of(PrometheusSystemTableSchema.SYS_TABLE_TABLES, + new PrometheusListMetricsRequest(client, catalogSchemaName)); + } else { + return Pair.of(PrometheusSystemTableSchema.SYS_TABLE_MAPPINGS, + new PrometheusDescribeMetricRequest(client, + catalogSchemaName, systemTable.getTableName())); + } + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScan.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScan.java new file mode 100644 index 0000000000..5c0bc656fe --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScan.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.prometheus.storage.system; + +import java.util.Iterator; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.prometheus.request.system.PrometheusSystemRequest; +import org.opensearch.sql.storage.TableScanOperator; + +/** + * Prometheus table scan operator. + */ +@RequiredArgsConstructor +@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = false) +@ToString(onlyExplicitlyIncluded = true) +public class PrometheusSystemTableScan extends TableScanOperator { + + @EqualsAndHashCode.Include + private final PrometheusSystemRequest request; + + private Iterator iterator; + + @Override + public void open() { + iterator = request.search().iterator(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + + @Override + public String explain() { + return request.toString(); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableSchema.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableSchema.java new file mode 100644 index 0000000000..668a208c79 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableSchema.java @@ -0,0 +1,39 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.system; + +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.data.type.ExprType; + +@Getter +@RequiredArgsConstructor +public enum PrometheusSystemTableSchema { + + SYS_TABLE_TABLES(new ImmutableMap.Builder() + .put("TABLE_CATALOG", STRING) + .put("TABLE_SCHEMA", STRING) + .put("TABLE_NAME", STRING) + .put("TABLE_TYPE", STRING) + .put("UNIT", STRING) + .put("REMARKS", STRING) + .build()), + SYS_TABLE_MAPPINGS(new ImmutableMap.Builder() + .put("TABLE_CATALOG", STRING) + .put("TABLE_SCHEMA", STRING) + .put("TABLE_NAME", STRING) + .put("COLUMN_NAME", STRING) + .put("DATA_TYPE", STRING) + .build()); + + private final Map mapping; +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java index 6a9c8aaf84..5f4715fca7 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java @@ -18,8 +18,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Map; import lombok.SneakyThrows; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -33,6 +35,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.prometheus.request.system.model.MetricMetadata; @ExtendWith(MockitoExtension.class) public class PrometheusClientImplTest { @@ -46,7 +49,7 @@ void setUp() throws IOException { this.mockWebServer = new MockWebServer(); this.mockWebServer.start(); this.prometheusClient = - new PrometheusClientImpl(new OkHttpClient(), mockWebServer.url("/").uri()); + new PrometheusClientImpl(new OkHttpClient(), mockWebServer.url("").uri().normalize()); } @@ -112,6 +115,26 @@ void testGetLabel() { verifyGetLabelsCall(recordedRequest); } + @Test + @SneakyThrows + void testGetAllMetrics() { + MockResponse mockResponse = new MockResponse() + .addHeader("Content-Type", "application/json; charset=utf-8") + .setBody(getJson("all_metrics_response.json")); + mockWebServer.enqueue(mockResponse); + Map> response = prometheusClient.getAllMetrics(); + Map> expected = new HashMap<>(); + expected.put("go_gc_duration_seconds", + Collections.singletonList(new MetricMetadata("summary", + "A summary of the pause duration of garbage collection cycles.", ""))); + expected.put("go_goroutines", + Collections.singletonList(new MetricMetadata("gauge", + "Number of goroutines that currently exist.", ""))); + assertEquals(expected, response); + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + verifyGetAllMetricsCall(recordedRequest); + } + @AfterEach void tearDown() throws IOException { mockWebServer.shutdown(); @@ -136,4 +159,11 @@ private void verifyGetLabelsCall(RecordedRequest recordedRequest) { assertEquals(METRIC_NAME, httpUrl.queryParameter("match[]")); } + private void verifyGetAllMetricsCall(RecordedRequest recordedRequest) { + HttpUrl httpUrl = recordedRequest.getRequestUrl(); + assertEquals("GET", recordedRequest.getMethod()); + assertNotNull(httpUrl); + assertEquals("/api/v1/metadata", httpUrl.encodedPath()); + } + } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java index cf2dc7439b..fdd9ce6392 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java @@ -11,23 +11,30 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.prometheus.constants.TestConstants.METRIC_NAME; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import lombok.SneakyThrows; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.request.system.PrometheusDescribeMetricRequest; @ExtendWith(MockitoExtension.class) public class PrometheusDescribeMetricRequestTest { @@ -50,10 +57,11 @@ void testGetFieldTypes() { put("code", ExprCoreType.STRING); put(VALUE, ExprCoreType.DOUBLE); put(TIMESTAMP, ExprCoreType.TIMESTAMP); - put(METRIC, ExprCoreType.STRING); + put(LABELS, ExprCoreType.STRING); }}; PrometheusDescribeMetricRequest prometheusDescribeMetricRequest - = new PrometheusDescribeMetricRequest(prometheusClient, METRIC_NAME); + = new PrometheusDescribeMetricRequest(prometheusClient, + new CatalogSchemaName("prometheus", "default"), METRIC_NAME); assertEquals(expected, prometheusDescribeMetricRequest.getFieldTypes()); verify(prometheusClient, times(1)).getLabels(METRIC_NAME); } @@ -65,12 +73,12 @@ void testGetFieldTypesWithEmptyMetricName() { Map expected = new HashMap<>() {{ put(VALUE, ExprCoreType.DOUBLE); put(TIMESTAMP, ExprCoreType.TIMESTAMP); - put(METRIC, ExprCoreType.STRING); + put(LABELS, ExprCoreType.STRING); }}; - PrometheusDescribeMetricRequest prometheusDescribeMetricRequest - = new PrometheusDescribeMetricRequest(prometheusClient, null); - assertEquals(expected, prometheusDescribeMetricRequest.getFieldTypes()); - verifyNoInteractions(prometheusClient); + assertThrows(NullPointerException.class, + () -> new PrometheusDescribeMetricRequest(prometheusClient, + new CatalogSchemaName("prometheus", "default"), + null)); } @@ -79,7 +87,8 @@ void testGetFieldTypesWithEmptyMetricName() { void testGetFieldTypesWhenException() { when(prometheusClient.getLabels(METRIC_NAME)).thenThrow(new RuntimeException("ERROR Message")); PrometheusDescribeMetricRequest prometheusDescribeMetricRequest - = new PrometheusDescribeMetricRequest(prometheusClient, METRIC_NAME); + = new PrometheusDescribeMetricRequest(prometheusClient, + new CatalogSchemaName("prometheus", "default"), METRIC_NAME); RuntimeException exception = assertThrows(RuntimeException.class, prometheusDescribeMetricRequest::getFieldTypes); verify(prometheusClient, times(1)).getLabels(METRIC_NAME); @@ -91,7 +100,8 @@ void testGetFieldTypesWhenException() { void testGetFieldTypesWhenIOException() { when(prometheusClient.getLabels(METRIC_NAME)).thenThrow(new IOException("ERROR Message")); PrometheusDescribeMetricRequest prometheusDescribeMetricRequest - = new PrometheusDescribeMetricRequest(prometheusClient, METRIC_NAME); + = new PrometheusDescribeMetricRequest(prometheusClient, + new CatalogSchemaName("prometheus", "default"), METRIC_NAME); RuntimeException exception = assertThrows(RuntimeException.class, prometheusDescribeMetricRequest::getFieldTypes); assertEquals("Error while fetching labels for http_requests_total" @@ -99,4 +109,30 @@ void testGetFieldTypesWhenIOException() { verify(prometheusClient, times(1)).getLabels(METRIC_NAME); } + @Test + @SneakyThrows + void testSearch() { + when(prometheusClient.getLabels(METRIC_NAME)).thenReturn(new ArrayList() {{ + add("call"); + } + }); + PrometheusDescribeMetricRequest prometheusDescribeMetricRequest + = new PrometheusDescribeMetricRequest(prometheusClient, + new CatalogSchemaName("test", "default"), METRIC_NAME); + List result = prometheusDescribeMetricRequest.search(); + assertEquals(4, result.size()); + assertEquals(expectedRow(), result.get(0)); + verify(prometheusClient, times(1)).getLabels(METRIC_NAME); + } + + private ExprValue expectedRow() { + LinkedHashMap valueMap = new LinkedHashMap<>(); + valueMap.put("TABLE_CATALOG", stringValue("test")); + valueMap.put("TABLE_SCHEMA", stringValue("default")); + valueMap.put("TABLE_NAME", stringValue(METRIC_NAME)); + valueMap.put("COLUMN_NAME", stringValue("call")); + valueMap.put("DATA_TYPE", stringValue(ExprCoreType.STRING.legacyTypeName().toLowerCase())); + return new ExprTupleValue(valueMap); + } + } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java new file mode 100644 index 0000000000..2d0bf3f1e9 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusListMetricsRequestTest.java @@ -0,0 +1,89 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.request; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.request.system.PrometheusListMetricsRequest; +import org.opensearch.sql.prometheus.request.system.model.MetricMetadata; + +@ExtendWith(MockitoExtension.class) +public class PrometheusListMetricsRequestTest { + + @Mock + private PrometheusClient prometheusClient; + + @Test + @SneakyThrows + void testSearch() { + Map> metricsResult = new HashMap<>(); + metricsResult.put("go_gc_duration_seconds", + Collections.singletonList(new MetricMetadata("summary", + "A summary of the pause duration of garbage collection cycles.", ""))); + metricsResult.put("go_goroutines", + Collections.singletonList(new MetricMetadata("gauge", + "Number of goroutines that currently exist.", ""))); + when(prometheusClient.getAllMetrics()).thenReturn(metricsResult); + PrometheusListMetricsRequest prometheusListMetricsRequest + = new PrometheusListMetricsRequest(prometheusClient, + new CatalogSchemaName("prometheus", "information_schema")); + List result = prometheusListMetricsRequest.search(); + assertEquals(expectedRow(), result.get(0)); + assertEquals(2, result.size()); + verify(prometheusClient, times(1)).getAllMetrics(); + } + + + @Test + @SneakyThrows + void testSearchWhenIOException() { + when(prometheusClient.getAllMetrics()).thenThrow(new IOException("ERROR Message")); + PrometheusListMetricsRequest prometheusListMetricsRequest + = new PrometheusListMetricsRequest(prometheusClient, + new CatalogSchemaName("prometheus", "information_schema")); + RuntimeException exception = assertThrows(RuntimeException.class, + prometheusListMetricsRequest::search); + assertEquals("Error while fetching metric list for from prometheus: ERROR Message", + exception.getMessage()); + verify(prometheusClient, times(1)).getAllMetrics(); + } + + + private ExprTupleValue expectedRow() { + LinkedHashMap valueMap = new LinkedHashMap<>(); + valueMap.put("TABLE_CATALOG", stringValue("prometheus")); + valueMap.put("TABLE_SCHEMA", stringValue("default")); + valueMap.put("TABLE_NAME", stringValue("go_gc_duration_seconds")); + valueMap.put("TABLE_TYPE", stringValue("summary")); + valueMap.put("UNIT", stringValue("")); + valueMap.put("REMARKS", + stringValue("A summary of the pause duration of garbage collection cycles.")); + return new ExprTupleValue(valueMap); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java index 7fd0f295bd..16a2a7be0b 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java @@ -13,6 +13,9 @@ import static org.opensearch.sql.prometheus.constants.TestConstants.QUERY; import static org.opensearch.sql.prometheus.constants.TestConstants.STARTTIME; import static org.opensearch.sql.prometheus.constants.TestConstants.STEP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; import static org.opensearch.sql.prometheus.utils.TestUtils.getJson; import java.io.IOException; @@ -51,18 +54,18 @@ void testQueryResponseIterator() { prometheusMetricScan.open(); Assertions.assertTrue(prometheusMetricScan.hasNext()); ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ - put("@timestamp", new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); - put("@value", new ExprDoubleValue(1)); - put("metric", new ExprStringValue( + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put(VALUE, new ExprDoubleValue(1)); + put(LABELS, new ExprStringValue( "{\"instance\":\"localhost:9090\",\"__name__\":\"up\",\"job\":\"prometheus\"}")); } }); assertEquals(firstRow, prometheusMetricScan.next()); Assertions.assertTrue(prometheusMetricScan.hasNext()); ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ - put("@timestamp", new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); - put("@value", new ExprDoubleValue(0)); - put("metric", new ExprStringValue( + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put(VALUE, new ExprDoubleValue(0)); + put(LABELS, new ExprStringValue( "{\"instance\":\"localhost:9091\",\"__name__\":\"up\",\"job\":\"node\"}")); } }); diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java index 1a048a7b8e..116ff72a6a 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java @@ -14,7 +14,7 @@ import static org.mockito.Mockito.when; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.METRIC; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; @@ -56,7 +56,7 @@ void getFieldTypesFromMetric() { expectedFieldTypes.put("label2", ExprCoreType.STRING); expectedFieldTypes.put(VALUE, ExprCoreType.DOUBLE); expectedFieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); - expectedFieldTypes.put(METRIC, ExprCoreType.STRING); + expectedFieldTypes.put(LABELS, ExprCoreType.STRING); Map fieldTypes = prometheusMetricTable.getFieldTypes(); @@ -76,7 +76,7 @@ void getFieldTypesFromPrometheusQueryRequest() { Map expectedFieldTypes = new HashMap<>(); expectedFieldTypes.put(VALUE, ExprCoreType.DOUBLE); expectedFieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); - expectedFieldTypes.put(METRIC, ExprCoreType.STRING); + expectedFieldTypes.put(LABELS, ExprCoreType.STRING); Map fieldTypes = prometheusMetricTable.getFieldTypes(); @@ -93,7 +93,7 @@ void testImplement() { new PrometheusMetricTable(client, prometheusQueryRequest); List finalProjectList = new ArrayList<>(); finalProjectList.add( - new NamedExpression(METRIC, new ReferenceExpression(METRIC, ExprCoreType.STRING))); + new NamedExpression(LABELS, new ReferenceExpression(LABELS, ExprCoreType.STRING))); PhysicalPlan plan = prometheusMetricTable.implement( project(relation("query_range", prometheusMetricTable), finalProjectList, null)); @@ -103,7 +103,7 @@ void testImplement() { List projectList = ((ProjectOperator) plan).getProjectList(); List outputFields = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); - assertEquals(List.of(METRIC, TIMESTAMP, VALUE), outputFields); + assertEquals(List.of(LABELS, TIMESTAMP, VALUE), outputFields); assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); PrometheusMetricScan prometheusMetricScan = (PrometheusMetricScan) ((ProjectOperator) plan).getInput(); @@ -117,7 +117,7 @@ void testOptimize() { new PrometheusMetricTable(client, prometheusQueryRequest); List finalProjectList = new ArrayList<>(); finalProjectList.add( - new NamedExpression(METRIC, new ReferenceExpression(METRIC, ExprCoreType.STRING))); + new NamedExpression(LABELS, new ReferenceExpression(LABELS, ExprCoreType.STRING))); LogicalPlan inputPlan = project(relation("query_range", prometheusMetricTable), finalProjectList, null); LogicalPlan optimizedPlan = prometheusMetricTable.optimize( diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java index 412abc99e0..992b17b549 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java @@ -8,16 +8,21 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.prometheus.client.PrometheusClient; import org.opensearch.sql.prometheus.functions.resolver.QueryRangeTableFunctionResolver; +import org.opensearch.sql.prometheus.storage.system.PrometheusSystemTable; import org.opensearch.sql.storage.Table; @ExtendWith(MockitoExtension.class) @@ -29,18 +34,44 @@ class PrometheusStorageEngineTest { @Test public void getTable() { PrometheusStorageEngine engine = new PrometheusStorageEngine(client); - Table table = engine.getTable("test"); + Table table = engine.getTable(new CatalogSchemaName("prometheus", "default"), "test"); assertNull(table); } @Test public void getFunctions() { PrometheusStorageEngine engine = new PrometheusStorageEngine(client); - Collection functionResolverCollection = engine.getFunctions(); + Collection functionResolverCollection + = engine.getFunctions(); assertNotNull(functionResolverCollection); assertEquals(1, functionResolverCollection.size()); assertTrue( functionResolverCollection.iterator().next() instanceof QueryRangeTableFunctionResolver); } + @Test + public void getSystemTable() { + PrometheusStorageEngine engine = new PrometheusStorageEngine(client); + Table table = engine.getTable(new CatalogSchemaName("prometheus", "default"), TABLE_INFO); + assertNotNull(table); + assertTrue(table instanceof PrometheusSystemTable); + } + + @Test + public void getSystemTableForAllTablesInfo() { + PrometheusStorageEngine engine = new PrometheusStorageEngine(client); + Table table + = engine.getTable(new CatalogSchemaName("prometheus", "information_schema"), "tables"); + assertNotNull(table); + assertTrue(table instanceof PrometheusSystemTable); + } + + @Test + public void getSystemTableWithWrongInformationSchemaTable() { + PrometheusStorageEngine engine = new PrometheusStorageEngine(client); + SemanticCheckException exception = assertThrows(SemanticCheckException.class, + () -> engine.getTable(new CatalogSchemaName("prometheus", "information_schema"), "test")); + assertEquals("Information Schema doesn't contain test table", exception.getMessage()); + } + } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScanTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScanTest.java new file mode 100644 index 0000000000..0d7ec4e2cc --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableScanTest.java @@ -0,0 +1,44 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.system; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; + +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.prometheus.request.system.PrometheusSystemRequest; + +@ExtendWith(MockitoExtension.class) +public class PrometheusSystemTableScanTest { + + @Mock + private PrometheusSystemRequest request; + + @Test + public void queryData() { + when(request.search()).thenReturn(Collections.singletonList(stringValue("text"))); + final PrometheusSystemTableScan systemIndexScan = new PrometheusSystemTableScan(request); + + systemIndexScan.open(); + assertTrue(systemIndexScan.hasNext()); + assertEquals(stringValue("text"), systemIndexScan.next()); + } + + @Test + public void explain() { + when(request.toString()).thenReturn("request"); + final PrometheusSystemTableScan systemIndexScan = new PrometheusSystemTableScan(request); + assertEquals("request", systemIndexScan.explain()); + } +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java new file mode 100644 index 0000000000..960b5b1319 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/system/PrometheusSystemTableTest.java @@ -0,0 +1,85 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.system; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.hasEntry; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.expression.DSL.named; +import static org.opensearch.sql.expression.DSL.ref; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; +import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; +import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; + +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.CatalogSchemaName; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.ProjectOperator; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.storage.Table; + +@ExtendWith(MockitoExtension.class) +public class PrometheusSystemTableTest { + + @Mock + private PrometheusClient client; + + @Mock + private Table table; + + @Test + void testGetFieldTypesOfMetaTable() { + PrometheusSystemTable systemIndex = new PrometheusSystemTable(client, + new CatalogSchemaName("prometheus", "information_schema"), TABLE_INFO); + final Map fieldTypes = systemIndex.getFieldTypes(); + assertThat(fieldTypes, anyOf( + hasEntry("TABLE_CATALOG", STRING) + )); + assertThat(fieldTypes, anyOf( + hasEntry("UNIT", STRING) + )); + } + + @Test + void testGetFieldTypesOfMappingTable() { + PrometheusSystemTable systemIndex = new PrometheusSystemTable(client, + new CatalogSchemaName("prometheus", "information_schema"), mappingTable( + "test_metric")); + final Map fieldTypes = systemIndex.getFieldTypes(); + assertThat(fieldTypes, anyOf( + hasEntry("COLUMN_NAME", STRING) + )); + } + + + + @Test + void implement() { + PrometheusSystemTable systemIndex = new PrometheusSystemTable(client, + new CatalogSchemaName("prometheus", "information_schema"), TABLE_INFO); + NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); + + final PhysicalPlan plan = systemIndex.implement( + project( + relation(TABLE_INFO, table), + projectExpr + )); + assertTrue(plan instanceof ProjectOperator); + assertTrue(plan.getChild().get(0) instanceof PrometheusSystemTableScan); + } + +} diff --git a/prometheus/src/test/resources/all_metrics_response.json b/prometheus/src/test/resources/all_metrics_response.json new file mode 100644 index 0000000000..94cc7782d9 --- /dev/null +++ b/prometheus/src/test/resources/all_metrics_response.json @@ -0,0 +1,19 @@ +{ + "status": "success", + "data": { + "go_gc_duration_seconds": [ + { + "type": "summary", + "help": "A summary of the pause duration of garbage collection cycles.", + "unit": "" + } + ], + "go_goroutines": [ + { + "type": "gauge", + "help": "Number of goroutines that currently exist.", + "unit": "" + } + ] + } +} From 99cebe349710c6e2f3fecef8ddd1d9be488029c0 Mon Sep 17 00:00:00 2001 From: Forest Vey <36905077+forestmvey@users.noreply.github.com> Date: Fri, 28 Oct 2022 15:33:26 -0700 Subject: [PATCH 09/20] Update MacOS version to supported level. Updated suitable dependencies to be included through vcpkg. Fixed compilation errors with newer compilers on libraries required to build from source. (#987) Signed-off-by: forestmvey --- .github/workflows/sql-odbc-main.yml | 11 ++-- sql-odbc/aws_sdk_cpp_setup.sh | 5 -- sql-odbc/build_mac_debug64.sh | 12 ++-- sql-odbc/build_mac_release64.sh | 12 ++-- sql-odbc/build_win_debug32.ps1 | 4 ++ sql-odbc/build_win_debug64.ps1 | 4 ++ sql-odbc/build_win_release32.ps1 | 4 ++ sql-odbc/build_win_release64.ps1 | 4 ++ sql-odbc/libraries/rabbit/include/rabbit.hpp | 4 -- sql-odbc/scripts/build_aws-sdk-cpp.ps1 | 45 --------------- sql-odbc/scripts/build_driver.ps1 | 8 ++- sql-odbc/scripts/build_installer.ps1 | 4 ++ sql-odbc/scripts/build_windows.ps1 | 30 ++-------- sql-odbc/src/CMakeLists.txt | 26 +++------ .../ITODBCAwsAuth/CMakeLists.txt | 5 +- .../ITODBCCatalog/CMakeLists.txt | 5 +- .../ITODBCCatalog/test_odbc_catalog.cpp | 1 + .../ITODBCConnection/CMakeLists.txt | 5 +- .../ITODBCDescriptors/CMakeLists.txt | 5 +- .../ITODBCExecution/CMakeLists.txt | 5 +- .../ITODBCHelper/CMakeLists.txt | 5 +- .../ITODBCInfo/CMakeLists.txt | 5 +- .../ITODBCPagination/CMakeLists.txt | 5 +- .../ITODBCResults/CMakeLists.txt | 5 +- .../ITODBCTableauQueries/CMakeLists.txt | 5 +- .../PTODBCInfo/CMakeLists.txt | 5 +- .../PTODBCResults/CMakeLists.txt | 5 +- .../performance_odbc_results.cpp | 5 +- .../src/UnitTests/UTAwsSdkCpp/CMakeLists.txt | 5 +- sql-odbc/src/UnitTests/UTConn/CMakeLists.txt | 5 +- .../UTCriticalSection/CMakeLists.txt | 5 +- .../src/UnitTests/UTHelper/CMakeLists.txt | 5 +- .../src/UnitTests/UTRabbit/CMakeLists.txt | 5 +- sql-odbc/src/gtest/googletest-download.cmake | 20 ------- sql-odbc/src/gtest/googletest.cmake | 32 ----------- sql-odbc/src/installer/CMakeLists.txt | 17 ------ sql-odbc/src/sqlodbc/info.c | 3 - sql-odbc/src/sqlodbc/odbcapiw.c | 22 ------- .../src/sqlodbc/opensearch_communication.cpp | 6 +- sql-odbc/src/sqlodbc/options.c | 5 -- sql-odbc/src/sqlodbc/results.c | 7 --- sql-odbc/src/sqlodbc/unicode_support.h | 1 - sql-odbc/src/sqlodbc/win_unicode.c | 57 ------------------- sql-odbc/src/vcpkg.json | 12 ++++ 44 files changed, 144 insertions(+), 302 deletions(-) delete mode 100755 sql-odbc/aws_sdk_cpp_setup.sh delete mode 100644 sql-odbc/scripts/build_aws-sdk-cpp.ps1 delete mode 100644 sql-odbc/src/gtest/googletest-download.cmake delete mode 100644 sql-odbc/src/gtest/googletest.cmake create mode 100644 sql-odbc/src/vcpkg.json diff --git a/.github/workflows/sql-odbc-main.yml b/.github/workflows/sql-odbc-main.yml index 6e01be6bc3..5ebfcf4dc3 100644 --- a/.github/workflows/sql-odbc-main.yml +++ b/.github/workflows/sql-odbc-main.yml @@ -13,14 +13,15 @@ env: CI_OUTPUT_PATH: "sql-odbc/ci-output" ODBC_LIB_PATH: "./build/odbc/lib" ODBC_BIN_PATH: "./build/odbc/bin" - ODBC_BUILD_PATH: "./build/odbc/build" - AWS_SDK_INSTALL_PATH: "./build/aws-sdk/install" + ODBC_BUILD_PATH: "./build/odbc/cmake" + VCPKG_X64_INSTALL_PATH: ".\\src\\vcpkg_installed\\x64-windows" + VCPKG_X86_INSTALL_PATH: ".\\src\\vcpkg_installed\\x86-windows" # Tests are disabled (commented out) in all jobs because they are fail and/or outdated # Keeping them for the brighten future when we can re-activate them jobs: build-mac: - runs-on: macos-10.15 + runs-on: macos-12 defaults: run: working-directory: sql-odbc @@ -103,7 +104,7 @@ jobs: - name: build-installer if: success() run: | - .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH + .\scripts\build_installer.ps1 Release Win32 .\src $Env:ODBC_BUILD_PATH $Env:VCPKG_X86_INSTALL_PATH #- name: test # run: | # cp .\\libraries\\VisualLeakDetector\\bin32\\*.* .\\bin32\\Release @@ -148,7 +149,7 @@ jobs: - name: build-installer if: success() run: | - .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:AWS_SDK_INSTALL_PATH + .\scripts\build_installer.ps1 Release x64 .\src $Env:ODBC_BUILD_PATH $Env:VCPKG_X64_INSTALL_PATH #- name: test # run: | # cp .\\libraries\\VisualLeakDetector\\bin64\\*.* .\\bin64\\Release diff --git a/sql-odbc/aws_sdk_cpp_setup.sh b/sql-odbc/aws_sdk_cpp_setup.sh deleted file mode 100755 index b75cc01aa7..0000000000 --- a/sql-odbc/aws_sdk_cpp_setup.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cd src -git clone -b "1.7.329" "https://github.com/aws/aws-sdk-cpp.git" -cd .. diff --git a/sql-odbc/build_mac_debug64.sh b/sql-odbc/build_mac_debug64.sh index 3522137921..784aeade81 100755 --- a/sql-odbc/build_mac_debug64.sh +++ b/sql-odbc/build_mac_debug64.sh @@ -1,14 +1,18 @@ -# Build AWS SDK -# $BITNESS=64 +#!/bin/bash cd src -git clone -b "1.7.329" "https://github.com/aws/aws-sdk-cpp.git" +vcpkg install cd .. +vcpkg_installed_dir='x64-osx' +if [[ $MACHTYPE == 'arm64-apple-darwin'* ]]; then + vcpkg_installed_dir='arm64-osx' +fi + PREFIX_PATH=$(pwd) mkdir cmake-build64 cd cmake-build64 -cmake ../src -DCMAKE_INSTALL_PREFIX=${PREFIX_PATH}/AWSSDK/ -DCMAKE_BUILD_TYPE=Debug -DBUILD_ONLY="core" -DCUSTOM_MEMORY_MANAGEMENT="OFF" -DENABLE_RTTI="OFF" -DENABLE_TESTING="OFF" +cmake ../src -DCMAKE_INSTALL_PREFIX=${PREFIX_PATH}/src/vcpkg_installed/${vcpkg_installed_dir}/ -DCMAKE_BUILD_TYPE=Debug -DBUILD_ONLY="core" -DCUSTOM_MEMORY_MANAGEMENT="OFF" -DENABLE_RTTI="OFF" -DENABLE_TESTING="OFF" cd .. cmake --build cmake-build64 -- -j 4 diff --git a/sql-odbc/build_mac_release64.sh b/sql-odbc/build_mac_release64.sh index 707a0ee53f..8fab73efc3 100755 --- a/sql-odbc/build_mac_release64.sh +++ b/sql-odbc/build_mac_release64.sh @@ -1,14 +1,18 @@ -# Build AWS SDK -# $BITNESS=64 +#!/bin/bash cd src -git clone -b "1.7.329" "https://github.com/aws/aws-sdk-cpp.git" +vcpkg install cd .. +vcpkg_installed_dir='x64-osx' +if [[ $MACHTYPE == 'arm64-apple-darwin'* ]]; then + vcpkg_installed_dir='arm64-osx' +fi + PREFIX_PATH=$(pwd) mkdir cmake-build64 cd cmake-build64 -cmake ../src -DCMAKE_INSTALL_PREFIX=${PREFIX_PATH}/AWSSDK/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="core" -DCUSTOM_MEMORY_MANAGEMENT="OFF" -DENABLE_RTTI="OFF" -DENABLE_TESTING="OFF" +cmake ../src -DCMAKE_INSTALL_PREFIX=${PREFIX_PATH}/src/vcpkg_installed/${vcpkg_installed_dir}/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="core" -DCUSTOM_MEMORY_MANAGEMENT="OFF" -DENABLE_RTTI="OFF" -DENABLE_TESTING="OFF" cd .. cmake --build cmake-build64 -- -j 4 diff --git a/sql-odbc/build_win_debug32.ps1 b/sql-odbc/build_win_debug32.ps1 index 7e23ada173..2717064b14 100644 --- a/sql-odbc/build_win_debug32.ps1 +++ b/sql-odbc/build_win_debug32.ps1 @@ -1,2 +1,6 @@ $WORKING_DIR = (Get-Location).Path +$env:VCPKG_DEFAULT_TRIPLET = 'x86-windows' +cd src +vcpkg install +cd .. .\scripts\build_windows.ps1 $WORKING_DIR Debug 32 diff --git a/sql-odbc/build_win_debug64.ps1 b/sql-odbc/build_win_debug64.ps1 index ea7084bada..98a9a24ff1 100644 --- a/sql-odbc/build_win_debug64.ps1 +++ b/sql-odbc/build_win_debug64.ps1 @@ -1,2 +1,6 @@ $WORKING_DIR = (Get-Location).Path +$env:VCPKG_DEFAULT_TRIPLET = 'x64-windows' +cd src +vcpkg install +cd .. .\scripts\build_windows.ps1 $WORKING_DIR Debug 64 diff --git a/sql-odbc/build_win_release32.ps1 b/sql-odbc/build_win_release32.ps1 index 4bcf4bd48e..c7e41da659 100644 --- a/sql-odbc/build_win_release32.ps1 +++ b/sql-odbc/build_win_release32.ps1 @@ -1,2 +1,6 @@ $WORKING_DIR = (Get-Location).Path +$env:VCPKG_DEFAULT_TRIPLET = 'x86-windows' +cd src +vcpkg install +cd .. .\scripts\build_windows.ps1 $WORKING_DIR Release 32 diff --git a/sql-odbc/build_win_release64.ps1 b/sql-odbc/build_win_release64.ps1 index 82b1199b33..a17f4d63f6 100644 --- a/sql-odbc/build_win_release64.ps1 +++ b/sql-odbc/build_win_release64.ps1 @@ -1,2 +1,6 @@ $WORKING_DIR = (Get-Location).Path +$env:VCPKG_DEFAULT_TRIPLET = 'x64-windows' +cd src +vcpkg install +cd .. .\scripts\build_windows.ps1 $WORKING_DIR Release 64 diff --git a/sql-odbc/libraries/rabbit/include/rabbit.hpp b/sql-odbc/libraries/rabbit/include/rabbit.hpp index ea4cddebc8..736de5e1b5 100644 --- a/sql-odbc/libraries/rabbit/include/rabbit.hpp +++ b/sql-odbc/libraries/rabbit/include/rabbit.hpp @@ -1150,10 +1150,6 @@ class basic_array : public basic_value : base_type(alloc) {} - basic_array(const basic_array& other) - : base_type(other) - {} - template basic_array(const basic_value_ref& other) : base_type(other) diff --git a/sql-odbc/scripts/build_aws-sdk-cpp.ps1 b/sql-odbc/scripts/build_aws-sdk-cpp.ps1 deleted file mode 100644 index 999d12f5bf..0000000000 --- a/sql-odbc/scripts/build_aws-sdk-cpp.ps1 +++ /dev/null @@ -1,45 +0,0 @@ -$CONFIGURATION = $args[0] -$WIN_ARCH = $args[1] -$SRC_DIR = $args[2] -$BUILD_DIR = $args[3] -$INSTALL_DIR = $args[4] -$VCPKG_DIR = $args[5] -$LIBCURL_WIN_ARCH = $args[6] - -Write-Host $args - -# Clone the AWS SDK CPP repo -$SDK_VER = "1.9.199" - -git clone ` - --branch ` - $SDK_VER ` - --single-branch ` - "https://github.com/aws/aws-sdk-cpp.git" ` - --recurse-submodules ` - $SRC_DIR - -# Make and move to build directory -New-Item -Path $BUILD_DIR -ItemType Directory -Force | Out-Null -Set-Location $BUILD_DIR - -# Configure and build -cmake $SRC_DIR ` - -A $WIN_ARCH ` - -D CMAKE_VERBOSE_MAKEFILE=ON ` - -D CMAKE_INSTALL_PREFIX=$INSTALL_DIR ` - -D CMAKE_BUILD_TYPE=$CONFIGURATION ` - -D BUILD_ONLY="core" ` - -D ENABLE_UNITY_BUILD="ON" ` - -D CUSTOM_MEMORY_MANAGEMENT="OFF" ` - -D ENABLE_RTTI="OFF" ` - -D ENABLE_TESTING="OFF" ` - -D FORCE_CURL="ON" ` - -D ENABLE_CURL_CLIENT="ON" ` - -DCMAKE_TOOLCHAIN_FILE="${VCPKG_DIR}/scripts/buildsystems/vcpkg.cmake" ` - -D CURL_LIBRARY="${VCPKG_DIR}/packages/curl_${LIBCURL_WIN_ARCH}-windows/lib" ` - -D CURL_INCLUDE_DIR="${VCPKG_DIR}/packages/curl_${LIBCURL_WIN_ARCH}-windows/include/" - -# Build AWS SDK and install to $INSTALL_DIR -msbuild ALL_BUILD.vcxproj /m /p:Configuration=$CONFIGURATION -msbuild INSTALL.vcxproj /m /p:Configuration=$CONFIGURATION diff --git a/sql-odbc/scripts/build_driver.ps1 b/sql-odbc/scripts/build_driver.ps1 index 1c0f8a799c..7f514a08c3 100644 --- a/sql-odbc/scripts/build_driver.ps1 +++ b/sql-odbc/scripts/build_driver.ps1 @@ -2,13 +2,17 @@ $CONFIGURATION = $args[0] $WIN_ARCH = $args[1] $SRC_DIR = $args[2] $BUILD_DIR = $args[3] -$INSTALL_DIR = $args[4] +$VCPKG_INSTALLED_DIR = $args[4] + +# aws-sdk-cpp fails compilation with warning: +# "Various members of std::allocator are deprecated in C++17" +$env:CL='-D_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING' cmake -S $SRC_DIR ` -B $BUILD_DIR ` -A $WIN_ARCH ` -D CMAKE_BUILD_TYPE=$CONFIGURATION ` - -D CMAKE_INSTALL_PREFIX=$INSTALL_DIR ` + -D CMAKE_INSTALL_PREFIX=$VCPKG_INSTALLED_DIR ` -D BUILD_WITH_TESTS=ON # # Build Project diff --git a/sql-odbc/scripts/build_installer.ps1 b/sql-odbc/scripts/build_installer.ps1 index b6a8e6edc0..63857b4705 100644 --- a/sql-odbc/scripts/build_installer.ps1 +++ b/sql-odbc/scripts/build_installer.ps1 @@ -6,6 +6,10 @@ $INSTALL_DIR = $args[4] Write-Host $args +# aws-sdk-cpp fails compilation with warning: +# "Various members of std::allocator are deprecated in C++17" +$env:CL='-D_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING' + cmake -S $SRC_DIR ` -B $BUILD_DIR ` -A $WIN_ARCH ` diff --git a/sql-odbc/scripts/build_windows.ps1 b/sql-odbc/scripts/build_windows.ps1 index 49b857ed8d..f2090df541 100644 --- a/sql-odbc/scripts/build_windows.ps1 +++ b/sql-odbc/scripts/build_windows.ps1 @@ -9,50 +9,28 @@ if ($BITNESS -eq "64") { else { $WIN_ARCH = "Win32" } -if ($BITNESS -eq "64") { - $LIBCURL_WIN_ARCH = "x64" -} -else { - $LIBCURL_WIN_ARCH = "x86" -} # Create build directory; remove if exists $BUILD_DIR = "${WORKING_DIR}\build" -# $BUILD_DIR = "${WORKING_DIR}\build\${CONFIGURATION}${BITNESS}" New-Item -Path $BUILD_DIR -ItemType Directory -Force | Out-Null -$VCPKG_DIR = $Env:VCPKG_ROOT -vcpkg.exe install curl[tool]:${LIBCURL_WIN_ARCH}-windows - -Set-Location $CURRENT_DIR - -# Build AWS SDK CPP -$SDK_SOURCE_DIR = "${WORKING_DIR}\src\aws-sdk-cpp" -$SDK_BUILD_DIR = "${BUILD_DIR}\aws-sdk\build" -$SDK_INSTALL_DIR = "${BUILD_DIR}\aws-sdk\install" - -.\scripts\build_aws-sdk-cpp.ps1 ` - $CONFIGURATION $WIN_ARCH ` - $SDK_SOURCE_DIR $SDK_BUILD_DIR $SDK_INSTALL_DIR $VCPKG_DIR ` - $LIBCURL_WIN_ARCH - Set-Location $CURRENT_DIR # Build driver $DRIVER_SOURCE_DIR = "${WORKING_DIR}\src" $DRIVER_BUILD_DIR = "${BUILD_DIR}\odbc\cmake" +$VCPKG_INSTALLED_DIR = "${DRIVER_SOURCE_DIR}\vcpkg_installed\$env:VCPKG_DEFAULT_TRIPLET" .\scripts\build_driver.ps1 ` $CONFIGURATION $WIN_ARCH ` - $DRIVER_SOURCE_DIR $DRIVER_BUILD_DIR $SDK_INSTALL_DIR + $DRIVER_SOURCE_DIR $DRIVER_BUILD_DIR $VCPKG_INSTALLED_DIR Set-Location $CURRENT_DIR # Move driver dependencies to bin directory for testing -$DRIVER_BIN_DIR = "$DRIVER_BUILD_DIR\..\bin\$CONFIGURATION" +$DRIVER_BIN_DIR = "${BUILD_DIR}\odbc\bin\$CONFIGURATION" New-Item -Path $DRIVER_BIN_DIR -ItemType Directory -Force | Out-Null -Copy-Item $SDK_BUILD_DIR\bin\$CONFIGURATION\* $DRIVER_BIN_DIR -Copy-Item $DRIVER_BUILD_DIR\bin\$CONFIGURATION\* $DRIVER_BIN_DIR +Copy-Item $VCPKG_INSTALLED_DIR\bin\* $DRIVER_BIN_DIR if ($BITNESS -eq "32") { # Strip bitness from 32bit VLD DLL dir name $BITNESS = $null diff --git a/sql-odbc/src/CMakeLists.txt b/sql-odbc/src/CMakeLists.txt index e2555ba9e5..93838c7cab 100644 --- a/sql-odbc/src/CMakeLists.txt +++ b/sql-odbc/src/CMakeLists.txt @@ -74,21 +74,17 @@ set(PERFORMANCE_TESTS "${CMAKE_CURRENT_SOURCE_DIR}/PerformanceTests") set(UT_HELPER "${UNIT_TESTS}/UTHelper") set(IT_HELPER "${INTEGRATION_TESTS}/ITODBCHelper") set(RABBIT_SRC ${LIBRARY_DIRECTORY}/rabbit/include) -set(RAPIDJSON_SRC ${LIBRARY_DIRECTORY}/rapidjson/include) set(VLD_SRC ${LIBRARY_DIRECTORY}/VisualLeakDetector/include) - +if(WIN32) + set(RAPIDJSON_SRC ${LIBRARY_DIRECTORY}/rapidjson/include) +endif () # Without this symbols will be exporting to Unix but not Windows set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) -# Set path for AWS SDK -set(aws-cpp-sdk-base "${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp") -set(aws-cpp-sdk-core_DIR "${PROJECT_ROOT}/sdk-build${BITNESS}/AWSSDK/lib/cmake/aws-cpp-sdk-core") -set(aws-c-event-stream_DIR "${PROJECT_ROOT}/sdk-build${BITNESS}/AWSSDK/lib/aws-c-event-stream/cmake") -set(aws-c-common_DIR "${PROJECT_ROOT}/sdk-build${BITNESS}/AWSSDK/lib/aws-c-common/cmake") -set(aws-checksums_DIR "${PROJECT_ROOT}/sdk-build${BITNESS}/AWSSDK/lib/aws-checksums/cmake") - -if (WIN32) - find_package(AWSSDK REQUIRED core) +find_package(aws-cpp-sdk-core CONFIG REQUIRED) +if(APPLE) + find_package(ZLIB REQUIRED) + find_package(RapidJSON CONFIG REQUIRED) endif() # General compiler definitions @@ -134,18 +130,10 @@ endif() if(BUILD_WITH_TESTS) # GTest import - include(gtest/googletest.cmake) - fetch_googletest( - ${PROJECT_SOURCE_DIR}/gtest - ${PROJECT_BINARY_DIR}/googletest - ) enable_testing() endif() # Projects to build -if (APPLE) - add_subdirectory(${aws-cpp-sdk-base}) -endif() add_subdirectory(${OPENSEARCHODBC_SRC}) add_subdirectory(${OPENSEARCHENLIST_SRC}) add_subdirectory(${INSTALL_SRC}) diff --git a/sql-odbc/src/IntegrationTests/ITODBCAwsAuth/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCAwsAuth/CMakeLists.txt index 760ae5d732..adf520e073 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCAwsAuth/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCAwsAuth/CMakeLists.txt @@ -12,6 +12,9 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_aws_auth ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(itodbc_aws_auth sqlodbc itodbc_helper ut_helper gtest_main aws-cpp-sdk-core) +target_link_libraries(itodbc_aws_auth sqlodbc itodbc_helper ut_helper GTest::gtest_main aws-cpp-sdk-core) target_compile_definitions(itodbc_aws_auth PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCCatalog/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCCatalog/CMakeLists.txt index 657f32bf6c..f27abdbcfd 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCCatalog/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCCatalog/CMakeLists.txt @@ -10,7 +10,10 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_catalog ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies target_code_coverage(itodbc_catalog PUBLIC AUTO ALL) -target_link_libraries(itodbc_catalog sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_catalog sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_catalog PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp b/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp index c1f12c94f6..68c672a388 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp +++ b/sql-odbc/src/IntegrationTests/ITODBCCatalog/test_odbc_catalog.cpp @@ -3,6 +3,7 @@ #include "pch.h" #include "unit_test_helper.h" #include "it_odbc_helper.h" +#include // clang-format on // General test constants and structures diff --git a/sql-odbc/src/IntegrationTests/ITODBCConnection/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCConnection/CMakeLists.txt index d336a9b304..81a028b266 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCConnection/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCConnection/CMakeLists.txt @@ -10,7 +10,10 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_connection ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies target_code_coverage(itodbc_connection PUBLIC AUTO ALL) -target_link_libraries(itodbc_connection sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_connection sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_connection PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCDescriptors/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCDescriptors/CMakeLists.txt index 37e6d116df..8e83be637b 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCDescriptors/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCDescriptors/CMakeLists.txt @@ -10,7 +10,10 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_descriptors ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies target_code_coverage(itodbc_descriptors PUBLIC AUTO ALL) -target_link_libraries(itodbc_descriptors sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_descriptors sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_descriptors PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCExecution/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCExecution/CMakeLists.txt index 4fa5d5e187..0f3eeec15f 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCExecution/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCExecution/CMakeLists.txt @@ -10,6 +10,9 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_execution ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(itodbc_execution sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_execution sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_execution PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCHelper/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCHelper/CMakeLists.txt index 56f0bc2cd8..61c97e5df9 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCHelper/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCHelper/CMakeLists.txt @@ -12,6 +12,9 @@ include_directories( # Generate dll (SHARED) add_library(itodbc_helper SHARED ${SOURCE_FILES} ${HEADER_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(itodbc_helper sqlodbc ut_helper gtest_main) +target_link_libraries(itodbc_helper sqlodbc ut_helper GTest::gtest_main) target_compile_definitions(itodbc_helper PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCInfo/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCInfo/CMakeLists.txt index 9b9698650f..833df41cb2 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCInfo/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCInfo/CMakeLists.txt @@ -10,7 +10,10 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_info ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies target_code_coverage(itodbc_info PUBLIC AUTO ALL) -target_link_libraries(itodbc_info sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_info sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_info PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCPagination/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCPagination/CMakeLists.txt index 7adfd4d13f..ce452d5131 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCPagination/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCPagination/CMakeLists.txt @@ -10,7 +10,10 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_pagination ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies target_code_coverage(itodbc_pagination PUBLIC AUTO ALL) -target_link_libraries(itodbc_pagination sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_pagination sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_pagination PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCResults/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCResults/CMakeLists.txt index 1107fbf4f9..9ea86199b4 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCResults/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCResults/CMakeLists.txt @@ -10,7 +10,10 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_results ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies target_code_coverage(itodbc_results PUBLIC AUTO ALL) -target_link_libraries(itodbc_results sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_results sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_results PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/IntegrationTests/ITODBCTableauQueries/CMakeLists.txt b/sql-odbc/src/IntegrationTests/ITODBCTableauQueries/CMakeLists.txt index 74f8c6d509..e8c13d1770 100644 --- a/sql-odbc/src/IntegrationTests/ITODBCTableauQueries/CMakeLists.txt +++ b/sql-odbc/src/IntegrationTests/ITODBCTableauQueries/CMakeLists.txt @@ -10,7 +10,10 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(itodbc_tableau_queries ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies target_code_coverage(itodbc_tableau_queries PUBLIC AUTO ALL) -target_link_libraries(itodbc_tableau_queries sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(itodbc_tableau_queries sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(itodbc_tableau_queries PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/PerformanceTests/PTODBCInfo/CMakeLists.txt b/sql-odbc/src/PerformanceTests/PTODBCInfo/CMakeLists.txt index 71550382ac..319dae7984 100644 --- a/sql-odbc/src/PerformanceTests/PTODBCInfo/CMakeLists.txt +++ b/sql-odbc/src/PerformanceTests/PTODBCInfo/CMakeLists.txt @@ -12,6 +12,9 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(performance_info ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(performance_info sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(performance_info sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(performance_info PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/PerformanceTests/PTODBCResults/CMakeLists.txt b/sql-odbc/src/PerformanceTests/PTODBCResults/CMakeLists.txt index e04f6b736e..589e4eac72 100644 --- a/sql-odbc/src/PerformanceTests/PTODBCResults/CMakeLists.txt +++ b/sql-odbc/src/PerformanceTests/PTODBCResults/CMakeLists.txt @@ -10,6 +10,9 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(performance_results ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(performance_results sqlodbc itodbc_helper ut_helper gtest_main) +target_link_libraries(performance_results sqlodbc itodbc_helper ut_helper GTest::gtest_main) target_compile_definitions(performance_results PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/PerformanceTests/PTODBCResults/performance_odbc_results.cpp b/sql-odbc/src/PerformanceTests/PTODBCResults/performance_odbc_results.cpp index ed116d0d11..95184f8907 100644 --- a/sql-odbc/src/PerformanceTests/PTODBCResults/performance_odbc_results.cpp +++ b/sql-odbc/src/PerformanceTests/PTODBCResults/performance_odbc_results.cpp @@ -3,6 +3,7 @@ #include "unit_test_helper.h" #include "it_odbc_helper.h" #include "chrono" +#include #include #include // clang-format on @@ -149,7 +150,6 @@ TEST_F(TestPerformance, Time_BindColumn_FetchSingleRow) { } TEST_F(TestPerformance, Time_BindColumn_Fetch5Rows) { - SQLROWSETSIZE row_count = 0; SQLSMALLINT total_columns = 0; SQLROWSETSIZE rows_fetched = 0; SQLUSMALLINT row_status[ROWSET_SIZE_5]; @@ -176,7 +176,6 @@ TEST_F(TestPerformance, Time_BindColumn_Fetch5Rows) { while (SQLExtendedFetch(m_hstmt, SQL_FETCH_NEXT, 0, &rows_fetched, row_status) == SQL_SUCCESS) { - row_count += rows_fetched; if (rows_fetched < ROWSET_SIZE_5) break; } @@ -190,7 +189,6 @@ TEST_F(TestPerformance, Time_BindColumn_Fetch5Rows) { } TEST_F(TestPerformance, Time_BindColumn_Fetch50Rows) { - SQLROWSETSIZE row_count = 0; SQLSMALLINT total_columns = 0; SQLROWSETSIZE rows_fetched = 0; SQLUSMALLINT row_status[ROWSET_SIZE_50]; @@ -217,7 +215,6 @@ TEST_F(TestPerformance, Time_BindColumn_Fetch50Rows) { while (SQLExtendedFetch(m_hstmt, SQL_FETCH_NEXT, 0, &rows_fetched, row_status) == SQL_SUCCESS) { - row_count += rows_fetched; if (rows_fetched < ROWSET_SIZE_50) break; } diff --git a/sql-odbc/src/UnitTests/UTAwsSdkCpp/CMakeLists.txt b/sql-odbc/src/UnitTests/UTAwsSdkCpp/CMakeLists.txt index 97d7df09fb..05f665aabf 100644 --- a/sql-odbc/src/UnitTests/UTAwsSdkCpp/CMakeLists.txt +++ b/sql-odbc/src/UnitTests/UTAwsSdkCpp/CMakeLists.txt @@ -9,6 +9,9 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(ut_aws_sdk_cpp ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(ut_aws_sdk_cpp ut_helper gtest_main aws-cpp-sdk-core ${VLD}) +target_link_libraries(ut_aws_sdk_cpp ut_helper GTest::gtest_main aws-cpp-sdk-core ${VLD}) target_compile_definitions(ut_aws_sdk_cpp PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/UnitTests/UTConn/CMakeLists.txt b/sql-odbc/src/UnitTests/UTConn/CMakeLists.txt index 5d985af2cc..0e90d8f4d6 100644 --- a/sql-odbc/src/UnitTests/UTConn/CMakeLists.txt +++ b/sql-odbc/src/UnitTests/UTConn/CMakeLists.txt @@ -12,6 +12,9 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(ut_conn ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(ut_conn sqlodbc ut_helper gtest_main) +target_link_libraries(ut_conn sqlodbc ut_helper GTest::gtest_main) target_compile_definitions(ut_conn PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/UnitTests/UTCriticalSection/CMakeLists.txt b/sql-odbc/src/UnitTests/UTCriticalSection/CMakeLists.txt index 43526f0ec1..e806ef9f4b 100644 --- a/sql-odbc/src/UnitTests/UTCriticalSection/CMakeLists.txt +++ b/sql-odbc/src/UnitTests/UTCriticalSection/CMakeLists.txt @@ -11,6 +11,9 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(ut_critical_section ${SOURCE_FILES}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(ut_critical_section sqlodbc ut_helper gtest_main) +target_link_libraries(ut_critical_section sqlodbc ut_helper GTest::gtest_main) target_compile_definitions(ut_critical_section PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/UnitTests/UTHelper/CMakeLists.txt b/sql-odbc/src/UnitTests/UTHelper/CMakeLists.txt index f9764d31cb..1bfad42485 100644 --- a/sql-odbc/src/UnitTests/UTHelper/CMakeLists.txt +++ b/sql-odbc/src/UnitTests/UTHelper/CMakeLists.txt @@ -22,6 +22,9 @@ find_library( VLD target_link_libraries(ut_helper ${VLD}) endif() +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + # Library dependencies -target_link_libraries(ut_helper sqlodbc gtest_main) +target_link_libraries(ut_helper sqlodbc GTest::gtest_main) target_compile_definitions(ut_helper PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/UnitTests/UTRabbit/CMakeLists.txt b/sql-odbc/src/UnitTests/UTRabbit/CMakeLists.txt index a5fec96f5e..b5f638fe01 100644 --- a/sql-odbc/src/UnitTests/UTRabbit/CMakeLists.txt +++ b/sql-odbc/src/UnitTests/UTRabbit/CMakeLists.txt @@ -10,5 +10,8 @@ include_directories( ${UT_HELPER} # Generate executable add_executable(ut_rabbit ${SOURCE_FILES}) -target_link_libraries(ut_rabbit ut_helper gtest_main ${VLD}) +# Find packages from vcpkg +find_package(GTest CONFIG REQUIRED) + +target_link_libraries(ut_rabbit ut_helper GTest::gtest_main ${VLD}) target_compile_definitions(ut_rabbit PUBLIC _UNICODE UNICODE) diff --git a/sql-odbc/src/gtest/googletest-download.cmake b/sql-odbc/src/gtest/googletest-download.cmake deleted file mode 100644 index 0ec4d55866..0000000000 --- a/sql-odbc/src/gtest/googletest-download.cmake +++ /dev/null @@ -1,20 +0,0 @@ -# code copied from https://crascit.com/2015/07/25/cmake-gtest/ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - -project(googletest-download NONE) - -include(ExternalProject) - -ExternalProject_Add( - googletest - SOURCE_DIR "@GOOGLETEST_DOWNLOAD_ROOT@/googletest-src" - BINARY_DIR "@GOOGLETEST_DOWNLOAD_ROOT@/googletest-build" - GIT_REPOSITORY - https://github.com/google/googletest.git - GIT_TAG - release-1.10.0 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - ) diff --git a/sql-odbc/src/gtest/googletest.cmake b/sql-odbc/src/gtest/googletest.cmake deleted file mode 100644 index 5ca7090877..0000000000 --- a/sql-odbc/src/gtest/googletest.cmake +++ /dev/null @@ -1,32 +0,0 @@ -# the following code to fetch googletest -# is inspired by and adapted after https://crascit.com/2015/07/25/cmake-gtest/ -# download and unpack googletest at configure time - -macro(fetch_googletest _download_module_path _download_root) - set(GOOGLETEST_DOWNLOAD_ROOT ${_download_root}) - configure_file( - ${_download_module_path}/googletest-download.cmake - ${_download_root}/CMakeLists.txt - @ONLY - ) - unset(GOOGLETEST_DOWNLOAD_ROOT) - - execute_process( - COMMAND - "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY - ${_download_root} - ) - execute_process( - COMMAND - "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY - ${_download_root} - ) - - # adds the targers: gtest, gtest_main, gmock, gmock_main - add_subdirectory( - ${_download_root}/googletest-src - ${_download_root}/googletest-build - ) -endmacro() diff --git a/sql-odbc/src/installer/CMakeLists.txt b/sql-odbc/src/installer/CMakeLists.txt index 726b8f6e6d..d172b91ac0 100644 --- a/sql-odbc/src/installer/CMakeLists.txt +++ b/sql-odbc/src/installer/CMakeLists.txt @@ -90,21 +90,4 @@ install(FILES "${PROJECT_ROOT}/THIRD-PARTY" DESTINATION doc COMPONENT "Docs") # Install resource files install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/Resources/opensearch_sql_odbc.tdc" DESTINATION resources COMPONENT "Resources") -# Install AWS dependencies -if(WIN32) - set(AWS_SDK_BIN_DIR "${PROJECT_ROOT}/build/aws-sdk/install/bin") - install(DIRECTORY ${AWS_SDK_BIN_DIR} DESTINATION . COMPONENT "Driver") -endif() - -if(WIN32) - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - # We actually never build the installer for Debug - install(FILES "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/Debug/libcurl-d.dll" DESTINATION bin COMPONENT "Driver") - install(FILES "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/Debug/zlibd1.dll" DESTINATION bin COMPONENT "Driver") - else() # release - install(FILES "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/Release/libcurl.dll" DESTINATION bin COMPONENT "Driver") - install(FILES "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/Release/zlib1.dll" DESTINATION bin COMPONENT "Driver") - endif() -endif() - include(CPack) diff --git a/sql-odbc/src/sqlodbc/info.c b/sql-odbc/src/sqlodbc/info.c index 6925f490ef..c40f783a55 100644 --- a/sql-odbc/src/sqlodbc/info.c +++ b/sql-odbc/src/sqlodbc/info.c @@ -33,7 +33,6 @@ RETCODE SQL_API OPENSEARCHAPI_GetInfo(HDBC hdbc, SQLUSMALLINT fInfoType, SQLSMALLINT *pcbInfoValue) { CSTR func = "OPENSEARCHAPI_GetInfo"; ConnectionClass *conn = (ConnectionClass *)hdbc; - ConnInfo *ci; const char *p = NULL; char tmp[MAX_INFO_STRING]; SQLULEN len = 0, value = 0; @@ -47,8 +46,6 @@ RETCODE SQL_API OPENSEARCHAPI_GetInfo(HDBC hdbc, SQLUSMALLINT fInfoType, return SQL_INVALID_HANDLE; } - ci = &(conn->connInfo); - switch (fInfoType) { case SQL_ACCESSIBLE_PROCEDURES: /* ODBC 1.0 */ p = "N"; diff --git a/sql-odbc/src/sqlodbc/odbcapiw.c b/sql-odbc/src/sqlodbc/odbcapiw.c index 6b22948f90..7577e0577c 100644 --- a/sql-odbc/src/sqlodbc/odbcapiw.c +++ b/sql-odbc/src/sqlodbc/odbcapiw.c @@ -19,17 +19,13 @@ RETCODE SQL_API SQLColumnsW(HSTMT StatementHandle, SQLWCHAR *CatalogName, char *ctName, *scName, *tbName, *clName; SQLLEN nmlen1, nmlen2, nmlen3, nmlen4; StatementClass *stmt = (StatementClass *)StatementHandle; - ConnectionClass *conn; BOOL lower_id; UWORD flag = PODBC_SEARCH_PUBLIC_SCHEMA; - ConnInfo *ci; MYLOG(OPENSEARCH_TRACE, "entering\n"); if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); - ci = &(conn->connInfo); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(CatalogName, NameLength1, &nmlen1, lower_id); scName = ucs2_to_utf8(SchemaName, NameLength2, &nmlen2, lower_id); @@ -426,14 +422,12 @@ RETCODE SQL_API SQLSpecialColumnsW( char *ctName, *scName, *tbName; SQLLEN nmlen1, nmlen2, nmlen3; StatementClass *stmt = (StatementClass *)StatementHandle; - ConnectionClass *conn; BOOL lower_id; MYLOG(OPENSEARCH_TRACE, "entering\n"); if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(CatalogName, NameLength1, &nmlen1, lower_id); scName = ucs2_to_utf8(SchemaName, NameLength2, &nmlen2, lower_id); @@ -467,14 +461,12 @@ RETCODE SQL_API SQLStatisticsW(HSTMT StatementHandle, SQLWCHAR *CatalogName, char *ctName, *scName, *tbName; SQLLEN nmlen1, nmlen2, nmlen3; StatementClass *stmt = (StatementClass *)StatementHandle; - ConnectionClass *conn; BOOL lower_id; MYLOG(OPENSEARCH_TRACE, "entering\n"); if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(CatalogName, NameLength1, &nmlen1, lower_id); scName = ucs2_to_utf8(SchemaName, NameLength2, &nmlen2, lower_id); @@ -508,7 +500,6 @@ RETCODE SQL_API SQLTablesW(HSTMT StatementHandle, SQLWCHAR *CatalogName, char *ctName, *scName, *tbName, *tbType; SQLLEN nmlen1, nmlen2, nmlen3, nmlen4; StatementClass *stmt = (StatementClass *)StatementHandle; - ConnectionClass *conn; BOOL lower_id; UWORD flag = 0; @@ -516,7 +507,6 @@ RETCODE SQL_API SQLTablesW(HSTMT StatementHandle, SQLWCHAR *CatalogName, if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(CatalogName, NameLength1, &nmlen1, lower_id); scName = ucs2_to_utf8(SchemaName, NameLength2, &nmlen2, lower_id); @@ -554,7 +544,6 @@ RETCODE SQL_API SQLColumnPrivilegesW( char *ctName, *scName, *tbName, *clName; SQLLEN nmlen1, nmlen2, nmlen3, nmlen4; StatementClass *stmt = (StatementClass *)hstmt; - ConnectionClass *conn; BOOL lower_id; UWORD flag = 0; @@ -562,7 +551,6 @@ RETCODE SQL_API SQLColumnPrivilegesW( if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(szCatalogName, cbCatalogName, &nmlen1, lower_id); scName = ucs2_to_utf8(szSchemaName, cbSchemaName, &nmlen2, lower_id); @@ -603,14 +591,12 @@ RETCODE SQL_API SQLForeignKeysW( char *ctName, *scName, *tbName, *fkctName, *fkscName, *fktbName; SQLLEN nmlen1, nmlen2, nmlen3, nmlen4, nmlen5, nmlen6; StatementClass *stmt = (StatementClass *)hstmt; - ConnectionClass *conn; BOOL lower_id; MYLOG(OPENSEARCH_TRACE, "entering\n"); if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(szPkCatalogName, cbPkCatalogName, &nmlen1, lower_id); scName = ucs2_to_utf8(szPkSchemaName, cbPkSchemaName, &nmlen2, lower_id); @@ -705,14 +691,12 @@ RETCODE SQL_API SQLPrimaryKeysW(HSTMT hstmt, SQLWCHAR *szCatalogName, char *ctName, *scName, *tbName; SQLLEN nmlen1, nmlen2, nmlen3; StatementClass *stmt = (StatementClass *)hstmt; - ConnectionClass *conn; BOOL lower_id; MYLOG(OPENSEARCH_TRACE, "entering\n"); if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(szCatalogName, cbCatalogName, &nmlen1, lower_id); scName = ucs2_to_utf8(szSchemaName, cbSchemaName, &nmlen2, lower_id); @@ -744,12 +728,10 @@ RETCODE SQL_API SQLProcedureColumnsW( char *ctName, *scName, *prName, *clName; SQLLEN nmlen1, nmlen2, nmlen3, nmlen4; StatementClass *stmt = (StatementClass *)hstmt; - ConnectionClass *conn; BOOL lower_id; UWORD flag = 0; MYLOG(OPENSEARCH_TRACE, "entering\n"); - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(szCatalogName, cbCatalogName, &nmlen1, lower_id); scName = ucs2_to_utf8(szSchemaName, cbSchemaName, &nmlen2, lower_id); @@ -787,7 +769,6 @@ RETCODE SQL_API SQLProceduresW(HSTMT hstmt, SQLWCHAR *szCatalogName, char *ctName, *scName, *prName; SQLLEN nmlen1, nmlen2, nmlen3; StatementClass *stmt = (StatementClass *)hstmt; - ConnectionClass *conn; BOOL lower_id; UWORD flag = 0; @@ -795,7 +776,6 @@ RETCODE SQL_API SQLProceduresW(HSTMT hstmt, SQLWCHAR *szCatalogName, if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(szCatalogName, cbCatalogName, &nmlen1, lower_id); scName = ucs2_to_utf8(szSchemaName, cbSchemaName, &nmlen2, lower_id); @@ -831,7 +811,6 @@ RETCODE SQL_API SQLTablePrivilegesW(HSTMT hstmt, SQLWCHAR *szCatalogName, char *ctName, *scName, *tbName; SQLLEN nmlen1, nmlen2, nmlen3; StatementClass *stmt = (StatementClass *)hstmt; - ConnectionClass *conn; BOOL lower_id; UWORD flag = 0; @@ -839,7 +818,6 @@ RETCODE SQL_API SQLTablePrivilegesW(HSTMT hstmt, SQLWCHAR *szCatalogName, if (SC_connection_lost_check(stmt, __FUNCTION__)) return SQL_ERROR; - conn = SC_get_conn(stmt); lower_id = DEFAULT_LOWERCASEIDENTIFIER; ctName = ucs2_to_utf8(szCatalogName, cbCatalogName, &nmlen1, lower_id); scName = ucs2_to_utf8(szSchemaName, cbSchemaName, &nmlen2, lower_id); diff --git a/sql-odbc/src/sqlodbc/opensearch_communication.cpp b/sql-odbc/src/sqlodbc/opensearch_communication.cpp index e3bdd73c54..dab46cb1fa 100644 --- a/sql-odbc/src/sqlodbc/opensearch_communication.cpp +++ b/sql-odbc/src/sqlodbc/opensearch_communication.cpp @@ -398,7 +398,7 @@ OpenSearchCommunication::IssueRequest( // Set header type if (!content_type.empty()) - request->SetHeaderValue(Aws::Http::CONTENT_TYPE_HEADER, ctype); + request->SetHeaderValue(Aws::Http::CONTENT_TYPE_HEADER, Aws::String(ctype.c_str(), ctype.size())); // Set body if (!query.empty() || !cursor.empty()) { @@ -414,7 +414,7 @@ OpenSearchCommunication::IssueRequest( Aws::MakeShared< Aws::StringStream >("RabbitStream"); *aws_ss << std::string(body.str()); request->AddContentBody(aws_ss); - request->SetContentLength(std::to_string(body.str().size())); + request->SetContentLength(Aws::Utils::StringUtils::to_string(body.str().size())); } // Handle authentication @@ -424,7 +424,7 @@ OpenSearchCommunication::IssueRequest( Aws::Utils::Array< unsigned char > userpw_arr( reinterpret_cast< const unsigned char* >(userpw_str.c_str()), userpw_str.length()); - std::string hashed_userpw = + Aws::String hashed_userpw = Aws::Utils::HashingUtils::Base64Encode(userpw_arr); request->SetAuthorization("Basic " + hashed_userpw); } else if (m_rt_opts.auth.auth_type == AUTHTYPE_IAM) { diff --git a/sql-odbc/src/sqlodbc/options.c b/sql-odbc/src/sqlodbc/options.c index 243b9385ae..fd100c0ea7 100644 --- a/sql-odbc/src/sqlodbc/options.c +++ b/sql-odbc/src/sqlodbc/options.c @@ -13,13 +13,8 @@ static RETCODE set_statement_option(ConnectionClass *conn, StatementClass *stmt, SQLUSMALLINT fOption, SQLULEN vParam) { CSTR func = "set_statement_option"; char changed = FALSE; - ConnInfo *ci = NULL; SQLULEN setval; - if (conn) - ci = &(conn->connInfo); - else - ci = &(SC_get_conn(stmt)->connInfo); switch (fOption) { case SQL_ASYNC_ENABLE: /* ignored */ break; diff --git a/sql-odbc/src/sqlodbc/results.c b/sql-odbc/src/sqlodbc/results.c index 7420062382..18f1cd1a72 100644 --- a/sql-odbc/src/sqlodbc/results.c +++ b/sql-odbc/src/sqlodbc/results.c @@ -111,7 +111,6 @@ RETCODE SQL_API OPENSEARCHAPI_DescribeCol(HSTMT hstmt, SQLUSMALLINT icol, SQLLEN column_size = 0; int unknown_sizes; SQLINTEGER decimal_digits = 0; - ConnInfo *ci; FIELD_INFO *fi; char buf[255]; int len = 0; @@ -125,7 +124,6 @@ RETCODE SQL_API OPENSEARCHAPI_DescribeCol(HSTMT hstmt, SQLUSMALLINT icol, } conn = SC_get_conn(stmt); - ci = &(conn->connInfo); unknown_sizes = DEFAULT_UNKNOWNSIZES; SC_clear_error(stmt); @@ -321,7 +319,6 @@ RETCODE SQL_API OPENSEARCHAPI_ColAttributes(HSTMT hstmt, SQLUSMALLINT icol, OID field_type = 0; Int2 col_idx; ConnectionClass *conn; - ConnInfo *ci; int column_size, unknown_sizes; int cols = 0; RETCODE result; @@ -349,7 +346,6 @@ RETCODE SQL_API OPENSEARCHAPI_ColAttributes(HSTMT hstmt, SQLUSMALLINT icol, *pcbDesc = 0; irdflds = SC_get_IRDF(stmt); conn = SC_get_conn(stmt); - ci = &(conn->connInfo); /* * Dont check for bookmark column. This is the responsibility of the @@ -415,8 +411,6 @@ RETCODE SQL_API OPENSEARCHAPI_ColAttributes(HSTMT hstmt, SQLUSMALLINT icol, if (FI_is_applicable(fi)) field_type = getEffectiveOid(conn, fi); else { - BOOL build_fi = FALSE; - fi = NULL; switch (fDescType) { case SQL_COLUMN_OWNER_NAME: @@ -429,7 +423,6 @@ RETCODE SQL_API OPENSEARCHAPI_ColAttributes(HSTMT hstmt, SQLUSMALLINT icol, case SQL_DESC_BASE_COLUMN_NAME: case SQL_COLUMN_UPDATABLE: case 1212: /* SQL_CA_SS_COLUMN_KEY ? */ - build_fi = TRUE; break; } diff --git a/sql-odbc/src/sqlodbc/unicode_support.h b/sql-odbc/src/sqlodbc/unicode_support.h index 2a481b0c06..e2b2a63521 100644 --- a/sql-odbc/src/sqlodbc/unicode_support.h +++ b/sql-odbc/src/sqlodbc/unicode_support.h @@ -20,7 +20,6 @@ SQLLEN bindcol_hybrid_exec(SQLWCHAR *utf16, const char *ldt, size_t n, SQLLEN bindcol_localize_estimate(const char *utf8dt, BOOL lf_conv, char **wcsbuf); SQLLEN bindcol_localize_exec(char *ldt, size_t n, BOOL lf_conv, char **wcsbuf); -SQLLEN bindpara_msg_to_utf8(const char *ldt, char **wcsbuf, SQLLEN used); SQLLEN bindpara_wchar_to_msg(const SQLWCHAR *utf16, char **wcsbuf, SQLLEN used); SQLLEN locale_to_sqlwchar(SQLWCHAR *utf16, const char *ldt, size_t n, diff --git a/sql-odbc/src/sqlodbc/win_unicode.c b/sql-odbc/src/sqlodbc/win_unicode.c index 5bc03b64fd..706e86e53c 100644 --- a/sql-odbc/src/sqlodbc/win_unicode.c +++ b/sql-odbc/src/sqlodbc/win_unicode.c @@ -841,63 +841,6 @@ static SQLLEN c16tombs(char *c8dt, const char16_t *c16dt, size_t n) { } #endif /* __CHAR16_UTF_16__ */ -// -// SQLBindParameter SQL_C_CHAR to UTF-8 case -// the current locale => UTF-8 -// -SQLLEN bindpara_msg_to_utf8(const char *ldt, char **wcsbuf, SQLLEN used) { - SQLLEN l = (-2); - char *utf8 = NULL, *ldt_nts, *alloc_nts = NULL, ntsbuf[128]; - int count; - - if (SQL_NTS == used) { - count = (int)strlen(ldt); - ldt_nts = (char *)ldt; - } else if (used < 0) { - return -1; - } else { - count = (int)used; - if (used < (SQLLEN)sizeof(ntsbuf)) - ldt_nts = ntsbuf; - else { - if (NULL == (alloc_nts = malloc(used + 1))) - return l; - ldt_nts = alloc_nts; - } - memcpy(ldt_nts, ldt, used); - ldt_nts[used] = '\0'; - } - - get_convtype(); - MYLOG(OPENSEARCH_DEBUG, " \n"); -#if defined(__WCS_ISO10646__) - if (use_wcs) { - wchar_t *wcsdt = (wchar_t *)malloc((count + 1) * sizeof(wchar_t)); - - if ((l = msgtowstr(ldt_nts, (wchar_t *)wcsdt, count + 1)) >= 0) - utf8 = wcs_to_utf8(wcsdt, -1, &l, FALSE); - free(wcsdt); - } -#endif /* __WCS_ISO10646__ */ -#ifdef __CHAR16_UTF_16__ - if (use_c16) { - SQLWCHAR *utf16 = (SQLWCHAR *)malloc((count + 1) * sizeof(SQLWCHAR)); - - if ((l = mbstoc16_lf((char16_t *)utf16, ldt_nts, count + 1, FALSE)) - >= 0) - utf8 = ucs2_to_utf8(utf16, -1, &l, FALSE); - free(utf16); - } -#endif /* __CHAR16_UTF_16__ */ - if (l < 0 && NULL != utf8) - free(utf8); - else - *wcsbuf = (char *)utf8; - - if (NULL != alloc_nts) - free(alloc_nts); - return l; -} // // SQLBindParameter hybrid case diff --git a/sql-odbc/src/vcpkg.json b/sql-odbc/src/vcpkg.json new file mode 100644 index 0000000000..a5903fbb85 --- /dev/null +++ b/sql-odbc/src/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "sql-odbc", + "version-string": "1.1.0.1", + "dependencies": ["aws-sdk-cpp", "rapidjson", "zlib", "gtest", "curl"], + "builtin-baseline": "6ca56aeb457f033d344a7106cb3f9f1abf8f4e98", + "overrides": [ + { "name": "aws-sdk-cpp", "version": "1.8.83#2" }, + { "name": "rapidjson", "version": "2022-06-28#1" }, + { "name": "zlib", "version": "1.2.12#1" }, + { "name": "gtest", "version": "1.11.0" } + ] +} From 634e2ffa1ee62cb6a48301a06c8edc82d2de40c2 Mon Sep 17 00:00:00 2001 From: Margarit Hakobyan Date: Mon, 31 Oct 2022 10:45:09 -0700 Subject: [PATCH 10/20] Update Jackson Databind version to 2.13.4.2 (#148) (#992) Signed-off-by: Margarit Hakobyan Signed-off-by: Margarit Hakobyan --- build.gradle | 1 + core/build.gradle | 2 +- integ-test/build.gradle | 2 +- opensearch/build.gradle | 2 +- plugin/build.gradle | 4 ++-- prometheus/build.gradle | 2 +- protocol/build.gradle | 4 ++-- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index 4cc38bca5a..b25c36f687 100644 --- a/build.gradle +++ b/build.gradle @@ -9,6 +9,7 @@ buildscript { opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") spring_version = "5.3.22" jackson_version = "2.13.4" + jackson_databind_version = "2.13.4.2" isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/core/build.gradle b/core/build.gradle index 2926eb0614..eb70f110d1 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -46,7 +46,7 @@ dependencies { api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' api "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - api "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" + api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" api project(':common') diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 5c3e124f2a..bb217b463c 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -60,7 +60,7 @@ configurations.all { resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" } diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 8b5f917dff..7ad7d63546 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -33,7 +33,7 @@ dependencies { api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" implementation group: 'org.json', name: 'json', version:'20180813' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" diff --git a/plugin/build.gradle b/plugin/build.gradle index d170b72a95..9de6a474cf 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -88,7 +88,7 @@ configurations.all { resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" } @@ -103,7 +103,7 @@ compileTestJava { dependencies { api group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" api "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - api "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" + api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" api project(":ppl") diff --git a/prometheus/build.gradle b/prometheus/build.gradle index 50e0b444ea..c936e63785 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -14,7 +14,7 @@ dependencies { api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' diff --git a/protocol/build.gradle b/protocol/build.gradle index fc35b94d34..9c41fbf101 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -31,7 +31,7 @@ plugins { dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" implementation 'com.google.code.gson:gson:2.8.9' implementation project(':core') @@ -44,7 +44,7 @@ dependencies { } configurations.all { - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" } test { From c6b234c1d733905e21b1787ce2be1a9422b07064 Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Mon, 31 Oct 2022 15:34:26 -0700 Subject: [PATCH 11/20] A Generic ML Command in PPL (#971) * Add generic ml command in ppl. Signed-off-by: Jing Zhang * Recover ml client dependency. Signed-off-by: Jing Zhang * Address the comments I. Signed-off-by: Jing Zhang Signed-off-by: Jing Zhang --- .../org/opensearch/sql/analysis/Analyzer.java | 23 +++ .../sql/analysis/TypeEnvironment.java | 14 +- .../sql/ast/AbstractNodeVisitor.java | 5 + .../java/org/opensearch/sql/ast/tree/ML.java | 135 ++++++++++++++ .../sql/planner/logical/LogicalML.java | 33 ++++ .../logical/LogicalPlanNodeVisitor.java | 4 + .../physical/PhysicalPlanNodeVisitor.java | 4 + .../sql/utils/MLCommonsConstants.java | 16 ++ .../opensearch/sql/analysis/AnalyzerTest.java | 131 ++++++++++++++ .../logical/LogicalPlanNodeVisitorTest.java | 12 ++ .../physical/PhysicalPlanNodeVisitorTest.java | 8 + .../OpenSearchExecutionProtector.java | 11 ++ .../physical/MLCommonsOperatorActions.java | 65 +++++++ .../planner/physical/MLOperator.java | 112 ++++++++++++ .../opensearch/storage/OpenSearchIndex.java | 8 + .../OpenSearchExecutionProtectorTest.java | 21 +++ .../planner/physical/MLOperatorTest.java | 171 ++++++++++++++++++ .../OpenSearchDefaultImplementorTest.java | 11 ++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 10 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 15 ++ .../sql/ppl/parser/AstBuilderTest.java | 15 ++ 22 files changed, 822 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/ML.java create mode 100644 core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java create mode 100644 opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLOperator.java create mode 100644 opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index e050908a87..188de5aa64 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -11,10 +11,18 @@ import static org.opensearch.sql.ast.tree.Sort.SortOrder.ASC; import static org.opensearch.sql.ast.tree.Sort.SortOrder.DESC; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.utils.MLCommonsConstants.ACTION; +import static org.opensearch.sql.utils.MLCommonsConstants.MODELID; +import static org.opensearch.sql.utils.MLCommonsConstants.PREDICT; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIMESTAMP; +import static org.opensearch.sql.utils.MLCommonsConstants.STATUS; +import static org.opensearch.sql.utils.MLCommonsConstants.TASKID; import static org.opensearch.sql.utils.MLCommonsConstants.TIME_FIELD; +import static org.opensearch.sql.utils.MLCommonsConstants.TRAIN; +import static org.opensearch.sql.utils.MLCommonsConstants.TRAINANDPREDICT; import static org.opensearch.sql.utils.SystemIndexUtils.CATALOGS_TABLE_NAME; import com.google.common.collect.ImmutableList; @@ -49,6 +57,7 @@ import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; @@ -82,6 +91,7 @@ import org.opensearch.sql.planner.logical.LogicalEval; import org.opensearch.sql.planner.logical.LogicalFilter; import org.opensearch.sql.planner.logical.LogicalLimit; +import org.opensearch.sql.planner.logical.LogicalML; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalProject; @@ -505,6 +515,19 @@ public LogicalPlan visitAD(AD node, AnalysisContext context) { return new LogicalAD(child, options); } + /** + * Build {@link LogicalML} for ml command. + */ + @Override + public LogicalPlan visitML(ML node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + TypeEnvironment currentEnv = context.peek(); + node.getOutputSchema(currentEnv).entrySet().stream() + .forEach(v -> currentEnv.define(new Symbol(Namespace.FIELD_NAME, v.getKey()), v.getValue())); + + return new LogicalML(child, node.getArguments()); + } + /** * The first argument is always "asc", others are optional. * Given nullFirst argument, use its value. Otherwise just use DEFAULT_ASC/DESC. diff --git a/core/src/main/java/org/opensearch/sql/analysis/TypeEnvironment.java b/core/src/main/java/org/opensearch/sql/analysis/TypeEnvironment.java index 1be195e056..c86d8109ad 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/TypeEnvironment.java +++ b/core/src/main/java/org/opensearch/sql/analysis/TypeEnvironment.java @@ -6,6 +6,8 @@ package org.opensearch.sql.analysis; +import static org.opensearch.sql.analysis.symbol.Namespace.FIELD_NAME; + import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -82,7 +84,7 @@ public void define(Symbol symbol, ExprType type) { * @param ref {@link ReferenceExpression} */ public void define(ReferenceExpression ref) { - define(new Symbol(Namespace.FIELD_NAME, ref.getAttr()), ref.type()); + define(new Symbol(FIELD_NAME, ref.getAttr()), ref.type()); } public void remove(Symbol symbol) { @@ -93,6 +95,14 @@ public void remove(Symbol symbol) { * Remove ref. */ public void remove(ReferenceExpression ref) { - remove(new Symbol(Namespace.FIELD_NAME, ref.getAttr())); + remove(new Symbol(FIELD_NAME, ref.getAttr())); + } + + /** + * Clear all fields in the current environment. + */ + public void clearAllFields() { + lookupAllFields(FIELD_NAME).keySet().stream() + .forEach(v -> remove(new Symbol(Namespace.FIELD_NAME, v))); } } diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 295db7680f..60e7d6f06e 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -43,6 +43,7 @@ import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; @@ -266,6 +267,10 @@ public T visitAD(AD node, C context) { return visitChildren(node, context); } + public T visitML(ML node, C context) { + return visitChildren(node, context); + } + public T visitHighlightFunction(HighlightFunction node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/ML.java b/core/src/main/java/org/opensearch/sql/ast/tree/ML.java new file mode 100644 index 0000000000..2f83a993b7 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/ML.java @@ -0,0 +1,135 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ast.tree; + +import static org.opensearch.sql.utils.MLCommonsConstants.ACTION; +import static org.opensearch.sql.utils.MLCommonsConstants.ALGO; +import static org.opensearch.sql.utils.MLCommonsConstants.ASYNC; +import static org.opensearch.sql.utils.MLCommonsConstants.CLUSTERID; +import static org.opensearch.sql.utils.MLCommonsConstants.KMEANS; +import static org.opensearch.sql.utils.MLCommonsConstants.MODELID; +import static org.opensearch.sql.utils.MLCommonsConstants.PREDICT; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIME_FIELD; +import static org.opensearch.sql.utils.MLCommonsConstants.STATUS; +import static org.opensearch.sql.utils.MLCommonsConstants.TASKID; +import static org.opensearch.sql.utils.MLCommonsConstants.TRAIN; +import static org.opensearch.sql.utils.MLCommonsConstants.TRAINANDPREDICT; + +import com.google.common.collect.ImmutableList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.analysis.TypeEnvironment; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.data.type.ExprCoreType; + +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = true) +@RequiredArgsConstructor +@AllArgsConstructor +public class ML extends UnresolvedPlan { + private UnresolvedPlan child; + + private final Map arguments; + + @Override + public UnresolvedPlan attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitML(this, context); + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + private String getAction() { + return (String) arguments.get(ACTION).getValue(); + } + + /** + * Generate the ml output schema. + * + * @param env the current environment + * @return the schema + */ + public Map getOutputSchema(TypeEnvironment env) { + switch (getAction()) { + case TRAIN: + env.clearAllFields(); + return getTrainOutputSchema(); + case PREDICT: + case TRAINANDPREDICT: + return getPredictOutputSchema(); + default: + throw new IllegalArgumentException( + "Action error. Please indicate train, predict or trainandpredict."); + } + } + + /** + * Generate the ml predict output schema. + * + * @return the schema + */ + public Map getPredictOutputSchema() { + HashMap res = new HashMap<>(); + String algo = arguments.containsKey(ALGO) ? (String) arguments.get(ALGO).getValue() : null; + switch (algo) { + case KMEANS: + res.put(CLUSTERID, ExprCoreType.INTEGER); + break; + case RCF: + res.put(RCF_SCORE, ExprCoreType.DOUBLE); + if (arguments.containsKey(RCF_TIME_FIELD)) { + res.put(RCF_ANOMALY_GRADE, ExprCoreType.DOUBLE); + res.put((String) arguments.get(RCF_TIME_FIELD).getValue(), ExprCoreType.TIMESTAMP); + } else { + res.put(RCF_ANOMALOUS, ExprCoreType.BOOLEAN); + } + break; + default: + throw new IllegalArgumentException("Unsupported algorithm: " + algo); + } + return res; + } + + /** + * Generate the ml train output schema. + * + * @return the schema + */ + public Map getTrainOutputSchema() { + boolean isAsync = arguments.containsKey(ASYNC) + ? (boolean) arguments.get(ASYNC).getValue() : false; + Map res = new HashMap<>(Map.of(STATUS, ExprCoreType.STRING)); + if (isAsync) { + res.put(TASKID, ExprCoreType.STRING); + } else { + res.put(MODELID, ExprCoreType.STRING); + } + return res; + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java new file mode 100644 index 0000000000..c54ee92e08 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java @@ -0,0 +1,33 @@ +package org.opensearch.sql.planner.logical; + +import java.util.Collections; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.ast.expression.Literal; + +/** + * ML logical plan. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class LogicalML extends LogicalPlan { + private final Map arguments; + + /** + * Constructor of LogicalML. + * @param child child logical plan + * @param arguments arguments of the algorithm + */ + public LogicalML(LogicalPlan child, Map arguments) { + super(Collections.singletonList(child)); + this.arguments = arguments; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitML(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java index df23b9cd20..28539562e7 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -78,6 +78,10 @@ public R visitMLCommons(LogicalMLCommons plan, C context) { return visitNode(plan, context); } + public R visitML(LogicalML plan, C context) { + return visitNode(plan, context); + } + public R visitAD(LogicalAD plan, C context) { return visitNode(plan, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 646aae8220..63dd05cc6b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -79,4 +79,8 @@ public R visitMLCommons(PhysicalPlan node, C context) { public R visitAD(PhysicalPlan node, C context) { return visitNode(node, context); } + + public R visitML(PhysicalPlan node, C context) { + return visitNode(node, context); + } } diff --git a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java index 883d012d2f..90bca8fe8a 100644 --- a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java +++ b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java @@ -25,4 +25,20 @@ public class MLCommonsConstants { public static final String CENTROIDS = "centroids"; public static final String ITERATIONS = "iterations"; public static final String DISTANCE_TYPE = "distance_type"; + + public static final String ACTION = "action"; + public static final String TRAIN = "train"; + public static final String PREDICT = "predict"; + public static final String TRAINANDPREDICT = "trainandpredict"; + public static final String ASYNC = "async"; + public static final String ALGO = "algorithm"; + public static final String KMEANS = "kmeans"; + public static final String CLUSTERID = "ClusterID"; + public static final String RCF = "rcf"; + public static final String RCF_TIME_FIELD = "timeField"; + public static final String MODELID = "model_id"; + public static final String TASKID = "task_id"; + public static final String STATUS = "status"; + public static final String LIR = "linear_regression"; + public static final String LIR_TARGET = "target"; } diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 33e5985bd7..97c560d505 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -37,6 +37,21 @@ import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.utils.MLCommonsConstants.ACTION; +import static org.opensearch.sql.utils.MLCommonsConstants.ALGO; +import static org.opensearch.sql.utils.MLCommonsConstants.ASYNC; +import static org.opensearch.sql.utils.MLCommonsConstants.CLUSTERID; +import static org.opensearch.sql.utils.MLCommonsConstants.KMEANS; +import static org.opensearch.sql.utils.MLCommonsConstants.MODELID; +import static org.opensearch.sql.utils.MLCommonsConstants.PREDICT; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALOUS; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_ANOMALY_GRADE; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_SCORE; +import static org.opensearch.sql.utils.MLCommonsConstants.RCF_TIME_FIELD; +import static org.opensearch.sql.utils.MLCommonsConstants.STATUS; +import static org.opensearch.sql.utils.MLCommonsConstants.TASKID; +import static org.opensearch.sql.utils.MLCommonsConstants.TRAIN; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -59,6 +74,7 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Kmeans; +import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; @@ -1067,4 +1083,119 @@ public void show_catalogs() { } + @Test + public void ml_relation_unsupported_action() { + Map argumentMap = new HashMap<>() {{ + put(ACTION, new Literal("unsupported", DataType.STRING)); + put(ALGO, new Literal(KMEANS, DataType.STRING)); + }}; + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> analyze(AstDSL.project( + new ML(AstDSL.relation("schema"), argumentMap), AstDSL.allFields()))); + assertEquals( + "Action error. Please indicate train, predict or trainandpredict.", + exception.getMessage()); + } + + @Test + public void ml_relation_unsupported_algorithm() { + Map argumentMap = new HashMap<>() {{ + put(ACTION, new Literal(PREDICT, DataType.STRING)); + put(ALGO, new Literal("unsupported", DataType.STRING)); + }}; + + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> analyze(AstDSL.project( + new ML(AstDSL.relation("schema"), argumentMap), AstDSL.allFields()))); + assertEquals( + "Unsupported algorithm: unsupported", + exception.getMessage()); + } + + @Test + public void ml_relation_train_sync() { + Map argumentMap = new HashMap<>() {{ + put(ACTION, new Literal(TRAIN, DataType.STRING)); + put(ALGO, new Literal(KMEANS, DataType.STRING)); + }}; + + LogicalPlan actual = analyze(AstDSL.project( + new ML(AstDSL.relation("schema"), argumentMap), AstDSL.allFields())); + assertTrue(((LogicalProject) actual).getProjectList().size() >= 2); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(STATUS, DSL.ref(STATUS, STRING)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(MODELID, DSL.ref(MODELID, STRING)))); + } + + @Test + public void ml_relation_train_async() { + Map argumentMap = new HashMap<>() {{ + put(ACTION, new Literal(TRAIN, DataType.STRING)); + put(ALGO, new Literal(KMEANS, DataType.STRING)); + put(ASYNC, new Literal(true, DataType.BOOLEAN)); + }}; + + LogicalPlan actual = analyze(AstDSL.project( + new ML(AstDSL.relation("schema"), argumentMap), AstDSL.allFields())); + assertTrue(((LogicalProject) actual).getProjectList().size() >= 2); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(STATUS, DSL.ref(STATUS, STRING)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(TASKID, DSL.ref(TASKID, STRING)))); + } + + @Test + public void ml_relation_predict_kmeans() { + Map argumentMap = new HashMap<>() {{ + put(ACTION, new Literal(PREDICT, DataType.STRING)); + put(ALGO, new Literal(KMEANS, DataType.STRING)); + }}; + + LogicalPlan actual = analyze(AstDSL.project( + new ML(AstDSL.relation("schema"), argumentMap), AstDSL.allFields())); + assertTrue(((LogicalProject) actual).getProjectList().size() >= 1); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(CLUSTERID, DSL.ref(CLUSTERID, INTEGER)))); + } + + @Test + public void ml_relation_predict_rcf_with_time_field() { + Map argumentMap = new HashMap<>() {{ + put(ACTION, new Literal(PREDICT, DataType.STRING)); + put(ALGO, new Literal(RCF, DataType.STRING)); + put(RCF_TIME_FIELD, new Literal("ts", DataType.STRING)); + }}; + + LogicalPlan actual = analyze(AstDSL.project( + new ML(AstDSL.relation("schema"), argumentMap), AstDSL.allFields())); + assertTrue(((LogicalProject) actual).getProjectList().size() >= 3); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(RCF_SCORE, DSL.ref(RCF_SCORE, DOUBLE)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(RCF_ANOMALY_GRADE, DSL.ref(RCF_ANOMALY_GRADE, DOUBLE)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named("ts", DSL.ref("ts", TIMESTAMP)))); + } + + @Test + public void ml_relation_predict_rcf_without_time_field() { + Map argumentMap = new HashMap<>() {{ + put(ACTION, new Literal(PREDICT, DataType.STRING)); + put(ALGO, new Literal(RCF, DataType.STRING)); + }}; + + LogicalPlan actual = analyze(AstDSL.project( + new ML(AstDSL.relation("schema"), argumentMap), AstDSL.allFields())); + assertTrue(((LogicalProject) actual).getProjectList().size() >= 2); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(RCF_SCORE, DSL.ref(RCF_SCORE, DOUBLE)))); + assertTrue(((LogicalProject) actual).getProjectList() + .contains(DSL.named(RCF_ANOMALOUS, DSL.ref(RCF_ANOMALOUS, BOOLEAN)))); + } } diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index 329708b7d8..03eeb9c626 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -143,6 +143,18 @@ public void testAbstractPlanNodeVisitorShouldReturnNull() { }); assertNull(ad.accept(new LogicalPlanNodeVisitor() { }, null)); + + LogicalPlan ml = new LogicalML(LogicalPlanDSL.relation("schema", table), + new HashMap() {{ + put("action", new Literal("train", DataType.STRING)); + put("algorithm", new Literal("rcf", DataType.STRING)); + put("shingle_size", new Literal(8, DataType.INTEGER)); + put("time_decay", new Literal(0.0001, DataType.DOUBLE)); + put("time_field", new Literal(null, DataType.STRING)); + } + }); + assertNull(ml.accept(new LogicalPlanNodeVisitor() { + }, null)); } private static class NodesCount extends LogicalPlanNodeVisitor { diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java index cd561f3c09..8780177c88 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -150,6 +150,14 @@ public void test_visitAD() { assertNull(physicalPlanNodeVisitor.visitAD(plan, null)); } + @Test + public void test_visitML() { + PhysicalPlanNodeVisitor physicalPlanNodeVisitor = + new PhysicalPlanNodeVisitor() {}; + + assertNull(physicalPlanNodeVisitor.visitML(plan, null)); + } + public static class PhysicalPlanPrinter extends PhysicalPlanNodeVisitor { public String print(PhysicalPlan node) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java index 45d2b12620..f06ecb8576 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java @@ -10,6 +10,7 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.planner.physical.ADOperator; import org.opensearch.sql.opensearch.planner.physical.MLCommonsOperator; +import org.opensearch.sql.opensearch.planner.physical.MLOperator; import org.opensearch.sql.planner.physical.AggregationOperator; import org.opensearch.sql.planner.physical.DedupeOperator; import org.opensearch.sql.planner.physical.EvalOperator; @@ -150,6 +151,16 @@ public PhysicalPlan visitAD(PhysicalPlan node, Object context) { ); } + @Override + public PhysicalPlan visitML(PhysicalPlan node, Object context) { + MLOperator mlOperator = (MLOperator) node; + return doProtect( + new MLOperator(visitInput(mlOperator.getInput(), context), + mlOperator.getArguments(), + mlOperator.getNodeClient()) + ); + } + PhysicalPlan visitInput(PhysicalPlan node, Object context) { if (null == node) { return node; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java index 9003d2ec47..e1f12fb8a7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLCommonsOperatorActions.java @@ -6,6 +6,10 @@ package org.opensearch.sql.opensearch.planner.physical; +import static org.opensearch.sql.utils.MLCommonsConstants.MODELID; +import static org.opensearch.sql.utils.MLCommonsConstants.STATUS; +import static org.opensearch.sql.utils.MLCommonsConstants.TASKID; + import com.google.common.collect.ImmutableMap; import java.util.HashMap; import java.util.Iterator; @@ -28,7 +32,10 @@ import org.opensearch.ml.common.dataset.DataFrameInputDataset; import org.opensearch.ml.common.input.MLInput; import org.opensearch.ml.common.input.parameter.MLAlgoParams; +import org.opensearch.ml.common.input.parameter.sample.SampleAlgoParams; +import org.opensearch.ml.common.output.MLOutput; import org.opensearch.ml.common.output.MLPredictionOutput; +import org.opensearch.ml.common.output.MLTrainingOutput; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprDoubleValue; import org.opensearch.sql.data.model.ExprFloatValue; @@ -216,6 +223,64 @@ protected MLPredictionOutput getMLPredictionResult(FunctionName functionName, .actionGet(30, TimeUnit.SECONDS); } + /** + * get ml-commons train, predict and trainandpredict result. + * @param inputDataFrame input data frame + * @param arguments ml parameters + * @param nodeClient node client + * @return ml-commons result + */ + protected MLOutput getMLOutput(DataFrame inputDataFrame, + Map arguments, + NodeClient nodeClient) { + MLInput mlinput = MLInput.builder() + .inputDataset(new DataFrameInputDataset(inputDataFrame)) + //Just the placeholders for algorithm and parameters which must be initialized. + //They will be overridden in ml client. + .algorithm(FunctionName.SAMPLE_ALGO) + .parameters(new SampleAlgoParams(0)) + .build(); + + MachineLearningNodeClient machineLearningClient = + MLClient.getMLClient(nodeClient); + + return machineLearningClient + .run(mlinput, arguments) + .actionGet(30, TimeUnit.SECONDS); + } + + /** + * iterate result and built it into ExprTupleValue. + * @param inputRowIter input row iterator + * @param inputDataFrame input data frame + * @param mlResult train/predict result + * @param resultRowIter predict result iterator + * @return result in ExprTupleValue format + */ + protected ExprTupleValue buildPPLResult(boolean isPredict, + Iterator inputRowIter, + DataFrame inputDataFrame, + MLOutput mlResult, + Iterator resultRowIter) { + if (isPredict) { + return buildResult(inputRowIter, + inputDataFrame, + (MLPredictionOutput) mlResult, + resultRowIter); + } else { + return buildTrainResult((MLTrainingOutput) mlResult); + } + } + + protected ExprTupleValue buildTrainResult(MLTrainingOutput trainResult) { + ImmutableMap.Builder resultBuilder = new ImmutableMap.Builder<>(); + resultBuilder.put(MODELID, new ExprStringValue(trainResult.getModelId())); + resultBuilder.put(TASKID, new ExprStringValue(trainResult.getTaskId())); + resultBuilder.put(STATUS, new ExprStringValue(trainResult.getStatus())); + + return ExprTupleValue.fromExprValueMap(resultBuilder.build()); + } + private static class MLInputRows extends LinkedList> { /** * Add tuple value to input map, skip if any value is null. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLOperator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLOperator.java new file mode 100644 index 0000000000..938ff60157 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/MLOperator.java @@ -0,0 +1,112 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.physical; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.client.node.NodeClient; +import org.opensearch.ml.common.dataframe.DataFrame; +import org.opensearch.ml.common.dataframe.Row; +import org.opensearch.ml.common.output.MLOutput; +import org.opensearch.ml.common.output.MLPredictionOutput; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlanNodeVisitor; + +/** + * ml-commons Physical operator to call machine learning interface to get results for + * algorithm execution. + */ +@RequiredArgsConstructor +@EqualsAndHashCode(callSuper = false) +public class MLOperator extends MLCommonsOperatorActions { + @Getter + private final PhysicalPlan input; + + @Getter + private final Map arguments; + + @Getter + private final NodeClient nodeClient; + + @EqualsAndHashCode.Exclude + private Iterator iterator; + + @Override + public void open() { + super.open(); + DataFrame inputDataFrame = generateInputDataset(input); + Map args = processArgs(arguments); + + MLOutput mlOutput = getMLOutput(inputDataFrame, args, nodeClient); + final Iterator inputRowIter = inputDataFrame.iterator(); + // Only need to check train here, as action should be already checked in ml client. + final boolean isPrediction = ((String) args.get("action")).equals("train") ? false : true; + //For train, only one row to return. + final Iterator trainIter = new ArrayList() { + { + add("train"); + } + }.iterator(); + final Iterator resultRowIter = isPrediction + ? ((MLPredictionOutput) mlOutput).getPredictionResult().iterator() + : null; + iterator = new Iterator() { + @Override + public boolean hasNext() { + if (isPrediction) { + return inputRowIter.hasNext(); + } else { + boolean res = trainIter.hasNext(); + if (res) { + trainIter.next(); + } + return res; + } + } + + @Override + public ExprValue next() { + return buildPPLResult(isPrediction, inputRowIter, inputDataFrame, mlOutput, resultRowIter); + } + }; + } + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitML(this, context); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public ExprValue next() { + return iterator.next(); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + protected Map processArgs(Map arguments) { + Map res = new HashMap<>(); + arguments.forEach((k, v) -> res.put(k, v.getValue())); + return res; + } +} + diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index 2849fbbec9..9ebdc12ba2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -24,6 +24,7 @@ import org.opensearch.sql.opensearch.planner.logical.OpenSearchLogicalPlanOptimizerFactory; import org.opensearch.sql.opensearch.planner.physical.ADOperator; import org.opensearch.sql.opensearch.planner.physical.MLCommonsOperator; +import org.opensearch.sql.opensearch.planner.physical.MLOperator; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -34,6 +35,7 @@ import org.opensearch.sql.planner.DefaultImplementor; import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalHighlight; +import org.opensearch.sql.planner.logical.LogicalML; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalRelation; @@ -205,6 +207,12 @@ public PhysicalPlan visitAD(LogicalAD node, OpenSearchIndexScan context) { node.getArguments(), client.getNodeClient()); } + @Override + public PhysicalPlan visitML(LogicalML node, OpenSearchIndexScan context) { + return new MLOperator(visitChild(node, context), + node.getArguments(), client.getNodeClient()); + } + @Override public PhysicalPlan visitHighlight(LogicalHighlight node, OpenSearchIndexScan context) { context.getRequestBuilder().pushDownHighlight( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java index fded7848b6..857ff601e1 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtectorTest.java @@ -56,6 +56,7 @@ import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; import org.opensearch.sql.opensearch.planner.physical.ADOperator; import org.opensearch.sql.opensearch.planner.physical.MLCommonsOperator; +import org.opensearch.sql.opensearch.planner.physical.MLOperator; import org.opensearch.sql.opensearch.setting.OpenSearchSettings; import org.opensearch.sql.opensearch.storage.OpenSearchIndexScan; import org.opensearch.sql.planner.physical.PhysicalPlan; @@ -293,6 +294,26 @@ public void testVisitAD() { executionProtector.visitAD(adOperator, null)); } + @Test + public void testVisitML() { + NodeClient nodeClient = mock(NodeClient.class); + MLOperator mlOperator = + new MLOperator( + values(emptyList()), + new HashMap() {{ + put("action", new Literal("train", DataType.STRING)); + put("algorithm", new Literal("rcf", DataType.STRING)); + put("shingle_size", new Literal(8, DataType.INTEGER)); + put("time_decay", new Literal(0.0001, DataType.DOUBLE)); + put("time_field", new Literal(null, DataType.STRING)); + }}, + nodeClient + ); + + assertEquals(executionProtector.doProtect(mlOperator), + executionProtector.visitML(mlOperator, null)); + } + PhysicalPlan resourceMonitor(PhysicalPlan input) { return new ResourceMonitorPlan(input, resourceMonitor); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java new file mode 100644 index 0000000000..7a73468391 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/planner/physical/MLOperatorTest.java @@ -0,0 +1,171 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.physical; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.utils.MLCommonsConstants.ACTION; +import static org.opensearch.sql.utils.MLCommonsConstants.ALGO; +import static org.opensearch.sql.utils.MLCommonsConstants.KMEANS; +import static org.opensearch.sql.utils.MLCommonsConstants.PREDICT; +import static org.opensearch.sql.utils.MLCommonsConstants.TRAIN; + +import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.opensearch.action.support.PlainActionFuture; +import org.opensearch.client.node.NodeClient; +import org.opensearch.ml.client.MachineLearningNodeClient; +import org.opensearch.ml.common.dataframe.DataFrame; +import org.opensearch.ml.common.dataframe.DataFrameBuilder; +import org.opensearch.ml.common.input.MLInput; +import org.opensearch.ml.common.output.MLOutput; +import org.opensearch.ml.common.output.MLPredictionOutput; +import org.opensearch.ml.common.output.MLTrainingOutput; +import org.opensearch.sql.ast.dsl.AstDSL; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.opensearch.client.MLClient; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlanNodeVisitor; + +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +@RunWith(MockitoJUnitRunner.Silent.class) +public class MLOperatorTest { + @Mock + private PhysicalPlan input; + + @Mock + PlainActionFuture actionFuture; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private NodeClient nodeClient; + + private MLOperator mlOperator; + Map arguments = new HashMap<>(); + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private MachineLearningNodeClient machineLearningNodeClient; + + void setUp(boolean isPredict) { + arguments.put("k1",AstDSL.intLiteral(3)); + arguments.put("k2",AstDSL.stringLiteral("v1")); + arguments.put("k3",AstDSL.booleanLiteral(true)); + arguments.put("k4",AstDSL.doubleLiteral(2.0D)); + arguments.put("k5",AstDSL.shortLiteral((short)2)); + arguments.put("k6",AstDSL.longLiteral(2L)); + arguments.put("k7",AstDSL.floatLiteral(2F)); + + mlOperator = new MLOperator(input, arguments, nodeClient); + when(input.hasNext()).thenReturn(true).thenReturn(false); + ImmutableMap.Builder resultBuilder = new ImmutableMap.Builder<>(); + resultBuilder.put("k1", new ExprIntegerValue(2)); + when(input.next()).thenReturn(ExprTupleValue.fromExprValueMap(resultBuilder.build())); + + DataFrame dataFrame = DataFrameBuilder + .load(Collections.singletonList( + ImmutableMap.builder().put("result-k1", 2D) + .put("result-k2", 1) + .put("result-k3", "v3") + .put("result-k4", true) + .put("result-k5", (short)2) + .put("result-k6", 2L) + .put("result-k7", 2F) + .build()) + ); + + MLOutput mlOutput; + if (isPredict) { + mlOutput = MLPredictionOutput.builder() + .taskId("test_task_id") + .status("test_status") + .predictionResult(dataFrame) + .build(); + } else { + mlOutput = MLTrainingOutput.builder() + .taskId("test_task_id") + .status("test_status") + .modelId("test_model_id") + .build(); + } + + when(actionFuture.actionGet(anyLong(), eq(TimeUnit.SECONDS))) + .thenReturn(mlOutput); + when(machineLearningNodeClient.run(any(MLInput.class), any())) + .thenReturn(actionFuture); + } + + void setUpPredict() { + arguments.put(ACTION,AstDSL.stringLiteral(PREDICT)); + arguments.put(ALGO,AstDSL.stringLiteral(KMEANS)); + arguments.put("modelid",AstDSL.stringLiteral("dummyID")); + setUp(true); + } + + void setUpTrain() { + arguments.put(ACTION,AstDSL.stringLiteral(TRAIN)); + arguments.put(ALGO,AstDSL.stringLiteral(KMEANS)); + setUp(false); + } + + @Test + public void testOpenPredict() { + setUpPredict(); + try (MockedStatic mlClientMockedStatic = Mockito.mockStatic(MLClient.class)) { + when(MLClient.getMLClient(any(NodeClient.class))).thenReturn(machineLearningNodeClient); + mlOperator.open(); + assertTrue(mlOperator.hasNext()); + assertNotNull(mlOperator.next()); + assertFalse(mlOperator.hasNext()); + } + } + + @Test + public void testOpenTrain() { + setUpTrain(); + try (MockedStatic mlClientMockedStatic = Mockito.mockStatic(MLClient.class)) { + when(MLClient.getMLClient(any(NodeClient.class))).thenReturn(machineLearningNodeClient); + mlOperator.open(); + assertTrue(mlOperator.hasNext()); + assertNotNull(mlOperator.next()); + assertFalse(mlOperator.hasNext()); + } + } + + @Test + public void testAccept() { + setUpPredict(); + try (MockedStatic mlClientMockedStatic = Mockito.mockStatic(MLClient.class)) { + when(MLClient.getMLClient(any(NodeClient.class))).thenReturn(machineLearningNodeClient); + PhysicalPlanNodeVisitor physicalPlanNodeVisitor + = new PhysicalPlanNodeVisitor() {}; + assertNull(mlOperator.accept(physicalPlanNodeVisitor, null)); + } + } + +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java index ced87a7d31..a74c5fcbd4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchDefaultImplementorTest.java @@ -23,6 +23,7 @@ import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.planner.logical.LogicalAD; import org.opensearch.sql.planner.logical.LogicalHighlight; +import org.opensearch.sql.planner.logical.LogicalML; import org.opensearch.sql.planner.logical.LogicalMLCommons; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.storage.Table; @@ -77,6 +78,16 @@ public void visitAD() { assertNotNull(implementor.visitAD(node, indexScan)); } + @Test + public void visitML() { + LogicalML node = Mockito.mock(LogicalML.class, + Answers.RETURNS_DEEP_STUBS); + Mockito.when(node.getChild().get(0)).thenReturn(Mockito.mock(LogicalPlan.class)); + OpenSearchIndex.OpenSearchDefaultImplementor implementor = + new OpenSearchIndex.OpenSearchDefaultImplementor(indexScan, client); + assertNotNull(implementor.visitML(node, indexScan)); + } + @Test public void visitHighlight() { LogicalHighlight node = Mockito.mock(LogicalHighlight.class, diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ffb9780faf..79c812949f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -34,6 +34,7 @@ PATTERNS: 'PATTERNS'; NEW_FIELD: 'NEW_FIELD'; KMEANS: 'KMEANS'; AD: 'AD'; +ML: 'ML'; // COMMAND ASSIST KEYWORDS AS: 'AS'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 7a42c7fa2b..11e1c4c71f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -26,7 +26,7 @@ pplCommands commands : whereCommand | fieldsCommand | renameCommand | statsCommand | dedupCommand | sortCommand | evalCommand | headCommand - | topCommand | rareCommand | grokCommand | parseCommand | patternsCommand | kmeansCommand | adCommand; + | topCommand | rareCommand | grokCommand | parseCommand | patternsCommand | kmeansCommand | adCommand | mlCommand; searchCommand : (SEARCH)? fromClause #searchFrom @@ -149,6 +149,14 @@ adParameter | (ANOMALY_SCORE_THRESHOLD EQUAL anomaly_score_threshold=decimalLiteral) ; +mlCommand + : ML (mlArg)* + ; + +mlArg + : (argName=ident EQUAL argValue=literalValue) + ; + /** clauses */ fromClause : SOURCE EQUAL tableSourceClause diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index b0d17940a4..13d8f8cddf 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -53,6 +53,7 @@ import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Kmeans; +import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; @@ -411,6 +412,20 @@ public UnresolvedPlan visitAdCommand(AdCommandContext ctx) { return new AD(builder.build()); } + /** + * ml command. + */ + @Override + public UnresolvedPlan visitMlCommand(OpenSearchPPLParser.MlCommandContext ctx) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + ctx.mlArg() + .forEach(x -> { + builder.put(x.argName.getText(), + (Literal) internalVisitExpression(x.argValue)); + }); + return new ML(builder.build()); + } + /** * Get original text in query. */ diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9bcbe66330..eb2f651ac7 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -55,6 +55,7 @@ import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.tree.AD; import org.opensearch.sql.ast.tree.Kmeans; +import org.opensearch.sql.ast.tree.ML; import org.opensearch.sql.ast.tree.RareTopN.CommandType; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; @@ -712,6 +713,20 @@ public void testKmeansCommandWithoutParameter() { new Kmeans(relation("t"), ImmutableMap.of())); } + @Test + public void testMLCommand() { + assertEqual("source=t | ml action='trainandpredict' " + + "algorithm='kmeans' centroid=3 iteration=2 dist_type='l1'", + new ML(relation("t"), ImmutableMap.builder() + .put("action", new Literal("trainandpredict", DataType.STRING)) + .put("algorithm", new Literal("kmeans", DataType.STRING)) + .put("centroid", new Literal(3, DataType.INTEGER)) + .put("iteration", new Literal(2, DataType.INTEGER)) + .put("dist_type", new Literal("l1", DataType.STRING)) + .build() + )); + } + @Test public void testDescribeCommand() { assertEqual("describe t", From 40d8d9f27304fd87ca55e20199762558423367d5 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Tue, 1 Nov 2022 10:25:28 -0700 Subject: [PATCH 12/20] Describe Table with catalog name. (#989) Signed-off-by: Vamsi Manohar Signed-off-by: Vamsi Manohar --- docs/user/general/identifiers.rst | 86 +++++++++++++++++++ docs/user/ppl/cmd/describe.rst | 28 +++++- .../opensearch/sql/ppl/DescribeCommandIT.java | 23 +++++ .../opensearch/sql/ppl/parser/AstBuilder.java | 12 ++- .../sql/ppl/parser/AstBuilderTest.java | 8 ++ .../client/PrometheusClientImpl.java | 7 +- .../client/PrometheusClientImplTest.java | 1 - 7 files changed, 159 insertions(+), 6 deletions(-) diff --git a/docs/user/general/identifiers.rst b/docs/user/general/identifiers.rst index b211930884..affd381d41 100644 --- a/docs/user/general/identifiers.rst +++ b/docs/user/general/identifiers.rst @@ -176,3 +176,89 @@ Query delimited multiple indices seperated by ``,``:: |-------| | 5 | +-------+ + + + +Fully Qualified Table Names +=========================== + +Description +----------- +With the introduction of different datasource catalogs along with Opensearch, support for fully qualified table names became compulsory to resolve tables to a catalog. + +Format for fully qualified table name. +``..`` + +* catalogName:[Mandatory] Catalog information is mandatory when querying over tables from catalogs other than opensearch connector. + +* schemaName:[Optional] Schema is a logical abstraction for a group of tables. In the current state, we only support ``default`` and ``information_schema``. Any schema mentioned in the fully qualified name other than these two will be resolved to be part of tableName. + +* tableName:[Mandatory] tableName is mandatory. + +The current resolution algorithm works in such a way, the old queries on opensearch work without specifying any catalog name. +So queries on opensearch indices doesn't need a fully qualified table name. + +Table Name Resolution Algorithm. +-------------------------------- + +Fully qualified Name is divided into parts based on ``.`` character. + +TableName resolution algorithm works in the following manner. + +1. Take the first part of the qualified name and resolve it to a catalog from the list of catalogs configured. +If it doesn't resolve to any of the catalog names configured, catalog name will default to ``@opensearch`` catalog. + +2. Take the first part of the remaining qualified name after capturing the catalog name. +If this part represents any of the supported schemas under catalog, it will resolve to the same otherwise schema name will resolve to ``default`` schema. +Currently ``default`` and ``information_schema`` are the only schemas supported. + +3. Rest of the parts are combined to resolve tablename. + +** Only table name identifiers are supported with fully qualified names, identifiers used for columns and other attributes doesn't require prefixing with catalog and schema information.** + +Examples +-------- +Assume [my_prometheus] is the only catalog configured other than default opensearch engine. + +1. ``my_prometheus.default.http_requests_total`` + +catalogName = ``my_prometheus`` [Is in the list of catalogs configured]. + +schemaName = ``default`` [Is in the list of schemas supported]. + +tableName = ``http_requests_total``. + +2. ``logs.12.13.1`` + + +catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] + +schemaName = ``default`` [No supported schema found, so default to `default`]. + +tableName = ``logs.12.13.1``. + + +3. ``my_prometheus.http_requests_total`` + + +catalogName = ```my_prometheus`` [Is in the list of catalogs configured]. + +schemaName = ``default`` [No supported schema found, so default to `default`]. + +tableName = ``http_requests_total``. + +4. ``prometheus.http_requests_total`` + +catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] + +schemaName = ``default`` [No supported schema found, so default to `default`]. + +tableName = ``prometheus.http_requests_total``. + +5. ``prometheus.default.http_requests_total.1.2.3`` + +catalogName = ``@opensearch`` [Resolves to default @opensearch connector since [my_prometheus] is the only catalog configured name.] + +schemaName = ``default`` [No supported schema found, so default to `default`]. + +tableName = ``prometheus.default.http_requests_total.1.2.3``. diff --git a/docs/user/ppl/cmd/describe.rst b/docs/user/ppl/cmd/describe.rst index 0abd569684..c683aa1781 100644 --- a/docs/user/ppl/cmd/describe.rst +++ b/docs/user/ppl/cmd/describe.rst @@ -16,9 +16,12 @@ Description Syntax ============ -describe +describe .. + +* catalog: optional. If catalog is not provided, it resolves to opensearch catalog. +* schema: optional. If schema is not provided, it resolves to default schema. +* tablename: mandatory. describe command must specify which tablename to query from. -* index: mandatory. describe command must specify which index to query from. Example 1: Fetch all the metadata @@ -63,3 +66,24 @@ PPL query:: | age | +----------------+ + +Example 3: Fetch metadata for table in prometheus catalog +========================================================= + +The example retrieves table info for ``prometheus_http_requests_total`` metric in prometheus catalog. + +PPL query:: + + os> describe my_prometheus.prometheus_http_requests_total; + fetched rows / total rows = 7/7 + +-----------------+----------------+--------------------------------+---------------+-------------+ + | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | + |-----------------+----------------+--------------------------------+---------------+-------------| + | my_prometheus | default | prometheus_http_requests_total | @labels | keyword | + | my_prometheus | default | prometheus_http_requests_total | handler | keyword | + | my_prometheus | default | prometheus_http_requests_total | code | keyword | + | my_prometheus | default | prometheus_http_requests_total | instance | keyword | + | my_prometheus | default | prometheus_http_requests_total | @timestamp | timestamp | + | my_prometheus | default | prometheus_http_requests_total | @value | double | + | my_prometheus | default | prometheus_http_requests_total | job | keyword | + +-----------------+----------------+--------------------------------+---------------+-------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java index c06ef3bc21..88ef028fbe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG; import static org.opensearch.sql.util.MatcherUtils.columnName; +import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.verifyColumn; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -87,4 +88,26 @@ public void describeCommandWithoutIndexShouldFailToParse() throws IOException { assertTrue(e.getMessage().contains("Failed to parse query due to offending symbol")); } } + + @Test + public void testDescribeCommandWithPrometheusCatalog() throws IOException { + JSONObject result = executeQuery("describe my_prometheus.prometheus_http_requests_total"); + verifyColumn( + result, + columnName("TABLE_CATALOG"), + columnName("TABLE_SCHEMA"), + columnName("TABLE_NAME"), + columnName("COLUMN_NAME"), + columnName("DATA_TYPE") + ); + verifyDataRows(result, + rows("my_prometheus", "default", "prometheus_http_requests_total", "@labels", "keyword"), + rows("my_prometheus", "default", "prometheus_http_requests_total", "handler", "keyword"), + rows("my_prometheus", "default", "prometheus_http_requests_total", "code", "keyword"), + rows("my_prometheus", "default", "prometheus_http_requests_total", "instance", "keyword"), + rows("my_prometheus", "default", "prometheus_http_requests_total", "@value", "double"), + rows("my_prometheus", "default", "prometheus_http_requests_total", "@timestamp", + "timestamp"), + rows("my_prometheus", "default", "prometheus_http_requests_total", "job", "keyword")); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 13d8f8cddf..699d671bb2 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -44,6 +45,7 @@ import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Map; import org.opensearch.sql.ast.expression.ParseMethod; +import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.ast.expression.UnresolvedArgument; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.tree.AD; @@ -118,11 +120,19 @@ public UnresolvedPlan visitSearchFilterFrom(SearchFilterFromContext ctx) { /** * Describe command. + * Current logic separates table and metadata info about table by adding + * MAPPING_ODFE_SYS_TABLE as suffix. + * Even with the introduction of catalog and schema name in fully qualified table name, + * we do the same thing by appending MAPPING_ODFE_SYS_TABLE as syffix to the last part + * of qualified name. */ @Override public UnresolvedPlan visitDescribeCommand(DescribeCommandContext ctx) { final Relation table = (Relation) visitTableSourceClause(ctx.tableSourceClause()); - return new Relation(qualifiedName(mappingTable(table.getTableName()))); + QualifiedName tableQualifiedName = table.getTableQualifiedName(); + ArrayList parts = new ArrayList<>(tableQualifiedName.getParts()); + parts.set(parts.size() - 1, mappingTable(parts.get(parts.size() - 1))); + return new Relation(new QualifiedName(parts)); } /** diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index eb2f651ac7..26e44b674d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -739,6 +739,14 @@ public void testDescribeCommandWithMultipleIndices() { relation(mappingTable("t,u"))); } + @Test + public void testDescribeCommandWithFullyQualifiedTableName() { + assertEqual("describe prometheus.http_metric", + relation(qualifiedName("prometheus", mappingTable("http_metric")))); + assertEqual("describe prometheus.schema.http_metric", + relation(qualifiedName("prometheus", "schema", mappingTable("http_metric")))); + } + @Test public void test_fitRCFADCommand_withoutDataFormat() { assertEqual("source=t | AD shingle_size=10 time_decay=0.0001 time_field='timestamp' " diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java index cd9aed4d18..1c427cd3a7 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java @@ -14,7 +14,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; @@ -82,7 +81,11 @@ public Map> getAllMetrics() throws IOException { private List toListOfStrings(JSONArray array) { List result = new ArrayList<>(); for (int i = 0; i < array.length(); i++) { - result.add(array.optString(i)); + //__name__ is internal label in prometheus representing the metric name. + //Exempting this from labels list as it is not required in any of the operations. + if (!"__name__".equals(array.optString(i))) { + result.add(array.optString(i)); + } } return result; } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java index 5f4715fca7..f49672a882 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java @@ -106,7 +106,6 @@ void testGetLabel() { mockWebServer.enqueue(mockResponse); List response = prometheusClient.getLabels(METRIC_NAME); assertEquals(new ArrayList() {{ - add("__name__"); add("call"); add("code"); } From c2305321d636beb96b2a4d8149ad75fcfdf644c8 Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Tue, 1 Nov 2022 11:04:13 -0700 Subject: [PATCH 13/20] Included Authenticators (#988) Signed-off-by: vamsi-amazon --- .../model/AbstractAuthenticationData.java | 32 ----- .../sql/catalog/model/AuthenticationType.java | 10 -- .../model/BasicAuthenticationData.java | 25 ---- .../sql/catalog/model/CatalogMetadata.java | 7 +- .../model/auth/AuthenticationType.java | 42 ++++++ .../sql/storage/StorageEngineFactory.java | 19 +++ docs/user/ppl/admin/catalog.rst | 31 ++-- docs/user/ppl/admin/prometheus_connector.rst | 74 ++++++++++ docs/user/ppl/index.rst | 2 + doctest/catalog/catalog.json | 4 +- integ-test/build.gradle | 5 + .../src/test/resources/catalog/catalog.json | 4 +- plugin/build.gradle | 3 + .../plugin/catalog/CatalogServiceImpl.java | 72 +++++----- .../plugin-metadata/plugin-security.policy | 1 + .../catalog/CatalogServiceImplTest.java | 26 ++-- .../test/resources/catalog_missing_name.json | 10 +- plugin/src/test/resources/catalogs.json | 10 +- .../resources/duplicate_catalog_names.json | 20 +-- .../test/resources/illegal_catalog_name.json | 10 +- .../src/test/resources/multiple_catalogs.json | 21 +-- prometheus/build.gradle | 12 +- .../AwsSigningInterceptor.java | 59 ++++++++ .../BasicAuthenticationInterceptor.java | 34 +++++ .../client/PrometheusClientImpl.java | 7 +- .../request/PrometheusQueryRequest.java | 3 - .../storage/PrometheusStorageFactory.java | 95 ++++++++++++ .../AwsSigningInterceptorTest.java | 61 ++++++++ .../BasicAuthenticationInterceptorTest.java | 61 ++++++++ .../client/PrometheusClientImplTest.java | 3 +- .../storage/PrometheusStorageFactoryTest.java | 135 ++++++++++++++++++ 31 files changed, 708 insertions(+), 190 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/catalog/model/AbstractAuthenticationData.java delete mode 100644 core/src/main/java/org/opensearch/sql/catalog/model/AuthenticationType.java delete mode 100644 core/src/main/java/org/opensearch/sql/catalog/model/BasicAuthenticationData.java create mode 100644 core/src/main/java/org/opensearch/sql/catalog/model/auth/AuthenticationType.java create mode 100644 core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java create mode 100644 docs/user/ppl/admin/prometheus_connector.rst create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/AbstractAuthenticationData.java b/core/src/main/java/org/opensearch/sql/catalog/model/AbstractAuthenticationData.java deleted file mode 100644 index e6a0dfa538..0000000000 --- a/core/src/main/java/org/opensearch/sql/catalog/model/AbstractAuthenticationData.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.catalog.model; - -import com.fasterxml.jackson.annotation.JsonFormat; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import lombok.Getter; -import lombok.Setter; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.EXISTING_PROPERTY, - property = "type", - defaultImpl = AbstractAuthenticationData.class, - visible = true) -@JsonSubTypes({ - @JsonSubTypes.Type(value = BasicAuthenticationData.class, name = "basicauth"), -}) -@Getter -@Setter -public abstract class AbstractAuthenticationData { - - @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) - private AuthenticationType type; - -} diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/AuthenticationType.java b/core/src/main/java/org/opensearch/sql/catalog/model/AuthenticationType.java deleted file mode 100644 index 3e602c7f62..0000000000 --- a/core/src/main/java/org/opensearch/sql/catalog/model/AuthenticationType.java +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.catalog.model; - -public enum AuthenticationType { - BASICAUTH,NO -} diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/BasicAuthenticationData.java b/core/src/main/java/org/opensearch/sql/catalog/model/BasicAuthenticationData.java deleted file mode 100644 index 5ac8a72085..0000000000 --- a/core/src/main/java/org/opensearch/sql/catalog/model/BasicAuthenticationData.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.catalog.model; - - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Getter; -import lombok.Setter; - -@Getter -@Setter -@JsonIgnoreProperties(ignoreUnknown = true) -public class BasicAuthenticationData extends AbstractAuthenticationData { - - @JsonProperty(required = true) - private String username; - - @JsonProperty(required = true) - private String password; - -} diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/CatalogMetadata.java b/core/src/main/java/org/opensearch/sql/catalog/model/CatalogMetadata.java index 46c1894f6c..a859090a5d 100644 --- a/core/src/main/java/org/opensearch/sql/catalog/model/CatalogMetadata.java +++ b/core/src/main/java/org/opensearch/sql/catalog/model/CatalogMetadata.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Map; import lombok.Getter; import lombok.Setter; @@ -19,13 +20,11 @@ public class CatalogMetadata { @JsonProperty(required = true) private String name; - @JsonProperty(required = true) - private String uri; - @JsonProperty(required = true) @JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) private ConnectorType connector; - private AbstractAuthenticationData authentication; + @JsonProperty(required = true) + private Map properties; } diff --git a/core/src/main/java/org/opensearch/sql/catalog/model/auth/AuthenticationType.java b/core/src/main/java/org/opensearch/sql/catalog/model/auth/AuthenticationType.java new file mode 100644 index 0000000000..1157d8e497 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/catalog/model/auth/AuthenticationType.java @@ -0,0 +1,42 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.catalog.model.auth; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public enum AuthenticationType { + + BASICAUTH("basicauth"), AWSSIGV4AUTH("awssigv4"); + + private String name; + + private static final Map ENUM_MAP; + + AuthenticationType(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + static { + Map map = new HashMap<>(); + for (AuthenticationType instance : AuthenticationType.values()) { + map.put(instance.getName().toLowerCase(), instance); + } + ENUM_MAP = Collections.unmodifiableMap(map); + } + + public static AuthenticationType get(String name) { + return ENUM_MAP.get(name.toLowerCase()); + } +} diff --git a/core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java b/core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java new file mode 100644 index 0000000000..4cc27f6fa0 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/storage/StorageEngineFactory.java @@ -0,0 +1,19 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.storage; + +import java.util.Map; +import org.opensearch.sql.catalog.model.ConnectorType; + +public interface StorageEngineFactory { + + ConnectorType getConnectorType(); + + StorageEngine getStorageEngine(String catalogName, Map requiredConfig); + +} diff --git a/docs/user/ppl/admin/catalog.rst b/docs/user/ppl/admin/catalog.rst index 7b0a08f307..ccaab342a5 100644 --- a/docs/user/ppl/admin/catalog.rst +++ b/docs/user/ppl/admin/catalog.rst @@ -26,21 +26,22 @@ Definitions of catalog and connector Example Prometheus Catalog Definition :: [{ - "name" : "prometheus", + "name" : "my_prometheus", "connector": "prometheus", - "uri" : "http://localhost:9090", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "admin" + "properties" : { + "prometheus.uri" : "http://localhost:8080", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "admin" } }] Catalog configuration Restrictions. -* ``name``, ``uri``, ``connector`` are required fields in the catalog configuration. -* All the catalog names should be unique. -* Catalog names should match with the regex of an identifier[``[@*A-Za-z]+?[*a-zA-Z_\-0-9]*``]. -* ``prometheus`` is the only connector allowed. +* ``name``, ``connector``, ``properties`` are required fields in the catalog configuration. +* All the catalog names should be unique and match the following regex[``[@*A-Za-z]+?[*a-zA-Z_\-0-9]*``]. +* Allowed Connectors. + * ``prometheus`` [More details: `Prometheus Connector `_] +* All the allowed config parameters in ``properties`` are defined in individual connector pages mentioned above. Configuring catalog in OpenSearch ==================================== @@ -73,14 +74,10 @@ so we can refer a metric and apply stats over it in the following way. Example source command with prometheus catalog :: - >> source = prometheus.prometheus_http_requests_total | stats avg(@value) by job; + >> source = my_prometheus.prometheus_http_requests_total | stats avg(@value) by job; Limitations of catalog ==================================== -* Catalog settings are global and all PPL users are allowed to fetch data from all the defined catalogs. -* In each catalog, PPL users can access all the data available with the credentials provided in the catalog definition. -* With the current release, Basic and AWSSigV4 are the only authentication mechanisms supported with the underlying data sources. - - - +Catalog settings are global and users with PPL access are allowed to fetch data from all the defined catalogs. +PPL access can be controlled using roles.(More details: `Security Settings `_) \ No newline at end of file diff --git a/docs/user/ppl/admin/prometheus_connector.rst b/docs/user/ppl/admin/prometheus_connector.rst new file mode 100644 index 0000000000..53c862dfae --- /dev/null +++ b/docs/user/ppl/admin/prometheus_connector.rst @@ -0,0 +1,74 @@ +.. highlight:: sh + +==================== +Prometheus Connector +==================== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 1 + + +Introduction +============ + +This page covers prometheus connector properties for catalog configuration +and the nuances associated with prometheus connector. + + +Prometheus Connector Properties in Catalog Configuration +======================================================== +Prometheus Connector Properties. + +* ``prometheus.uri`` [Required]. + * This parameters provides the URI information to connect to a prometheus instance. +* ``prometheus.auth.type`` [Optional] + * This parameters provides the authentication type information. + * Prometheus connector currently supports ``basicauth`` and ``awssigv4`` authentication mechanisms. + * If prometheus.auth.type is basicauth, following are required parameters. + * ``prometheus.auth.username`` and ``prometheus.auth.password``. + * If prometheus.auth.type is awssigv4, following are required parameters. + * ``prometheus.auth.region``, ``prometheus.auth.access_key`` and ``prometheus.auth.secret_key`` + +Example prometheus catalog configuration with different authentications +======================================================================= + +No Auth :: + + [{ + "name" : "my_prometheus", + "connector": "prometheus", + "properties" : { + "prometheus.uri" : "http://localhost:9090" + } + }] + +Basic Auth :: + + [{ + "name" : "my_prometheus", + "connector": "prometheus", + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "admin" + } + }] + +AWSSigV4 Auth:: + + [{ + "name" : "my_prometheus", + "connector": "prometheus", + "properties" : { + "prometheus.uri" : "http://localhost:8080", + "prometheus.auth.type" : "awssigv4", + "prometheus.auth.region" : "us-east-1", + "prometheus.auth.access_key" : "{{accessKey}}" + "prometheus.auth.secret_key" : "{{secretKey}}" + } + }] + diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 3165569908..a4f620173c 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -36,6 +36,8 @@ The query start with search command and then flowing a set of command delimited - `Catalog Settings `_ + - `Prometheus Connector `_ + * **Commands** - `Syntax `_ diff --git a/doctest/catalog/catalog.json b/doctest/catalog/catalog.json index 84038fe83b..5f195747ae 100644 --- a/doctest/catalog/catalog.json +++ b/doctest/catalog/catalog.json @@ -2,6 +2,8 @@ { "name" : "my_prometheus", "connector": "prometheus", - "uri" : "http://localhost:9090" + "properties" : { + "prometheus.uri" : "http://localhost:9090" + } } ] \ No newline at end of file diff --git a/integ-test/build.gradle b/integ-test/build.gradle index bb217b463c..11ba5542fd 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -38,6 +38,10 @@ apply plugin: 'java' apply plugin: 'io.freefair.lombok' apply plugin: 'com.wiredforcode.spawn' +repositories { + mavenCentral() + maven { url 'https://jitpack.io' } +} ext { projectSubstitutions = [:] @@ -63,6 +67,7 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" + resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" } dependencies { diff --git a/integ-test/src/test/resources/catalog/catalog.json b/integ-test/src/test/resources/catalog/catalog.json index 84038fe83b..5f195747ae 100644 --- a/integ-test/src/test/resources/catalog/catalog.json +++ b/integ-test/src/test/resources/catalog/catalog.json @@ -2,6 +2,8 @@ { "name" : "my_prometheus", "connector": "prometheus", - "uri" : "http://localhost:9090" + "properties" : { + "prometheus.uri" : "http://localhost:9090" + } } ] \ No newline at end of file diff --git a/plugin/build.gradle b/plugin/build.gradle index 9de6a474cf..6a0900c3cc 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -39,6 +39,7 @@ ext { repositories { mavenCentral() + maven { url 'https://jitpack.io' } } opensearchplugin { @@ -91,6 +92,7 @@ configurations.all { resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" + resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" } compileJava { options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor']) @@ -106,6 +108,7 @@ dependencies { api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" + api project(":ppl") api project(':legacy') api project(':opensearch') diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java index c462c5a596..4f2f5e0f57 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java @@ -12,15 +12,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; -import okhttp3.OkHttpClient; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -30,10 +28,9 @@ import org.opensearch.sql.catalog.model.CatalogMetadata; import org.opensearch.sql.catalog.model.ConnectorType; import org.opensearch.sql.opensearch.security.SecurityAccess; -import org.opensearch.sql.prometheus.client.PrometheusClient; -import org.opensearch.sql.prometheus.client.PrometheusClientImpl; -import org.opensearch.sql.prometheus.storage.PrometheusStorageEngine; +import org.opensearch.sql.prometheus.storage.PrometheusStorageFactory; import org.opensearch.sql.storage.StorageEngine; +import org.opensearch.sql.storage.StorageEngineFactory; /** * This class manages catalogs and responsible for creating connectors to these catalogs. @@ -48,11 +45,17 @@ public class CatalogServiceImpl implements CatalogService { private Map catalogMap = new HashMap<>(); + private final Map connectorTypeStorageEngineFactoryMap; + public static CatalogServiceImpl getInstance() { return INSTANCE; } private CatalogServiceImpl() { + connectorTypeStorageEngineFactoryMap = new HashMap<>(); + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + connectorTypeStorageEngineFactoryMap.put(prometheusStorageFactory.getConnectorType(), + prometheusStorageFactory); } /** @@ -71,13 +74,12 @@ public void loadConnectors(Settings settings) { List catalogs = objectMapper.readValue(inputStream, new TypeReference<>() { }); - LOG.info(catalogs.toString()); validateCatalogs(catalogs); constructConnectors(catalogs); } catch (IOException e) { - LOG.error("Catalog Configuration File uploaded is malformed. Verify and re-upload."); - throw new IllegalArgumentException( - "Malformed Catalog Configuration Json" + e.getMessage()); + LOG.error("Catalog Configuration File uploaded is malformed. Verify and re-upload.", e); + } catch (Throwable e) { + LOG.error("Catalog constructed failed.", e); } } return null; @@ -116,35 +118,31 @@ private T doPrivileged(PrivilegedExceptionAction action) { } } - private StorageEngine createStorageEngine(CatalogMetadata catalog) throws URISyntaxException { - StorageEngine storageEngine; + private StorageEngine createStorageEngine(CatalogMetadata catalog) { ConnectorType connector = catalog.getConnector(); switch (connector) { case PROMETHEUS: - PrometheusClient - prometheusClient = - new PrometheusClientImpl(new OkHttpClient(), - new URI(catalog.getUri())); - storageEngine = new PrometheusStorageEngine(prometheusClient); - break; + return connectorTypeStorageEngineFactoryMap + .get(catalog.getConnector()) + .getStorageEngine(catalog.getName(), catalog.getProperties()); default: - LOG.info( - "Unknown connector \"{}\". " - + "Please re-upload catalog configuration with a supported connector.", - connector); throw new IllegalStateException( - "Unknown connector. Connector doesn't exist in the list of supported."); + String.format("Unsupported Connector: %s", connector.name())); } - return storageEngine; } - private void constructConnectors(List catalogs) throws URISyntaxException { + private void constructConnectors(List catalogs) { catalogMap = new HashMap<>(); for (CatalogMetadata catalog : catalogs) { - String catalogName = catalog.getName(); - StorageEngine storageEngine = createStorageEngine(catalog); - catalogMap.put(catalogName, - new Catalog(catalog.getName(), catalog.getConnector(), storageEngine)); + try { + String catalogName = catalog.getName(); + StorageEngine storageEngine = createStorageEngine(catalog); + catalogMap.put(catalogName, + new Catalog(catalog.getName(), catalog.getConnector(), storageEngine)); + } catch (Throwable e) { + LOG.error("Catalog : {} storage engine creation failed with the following message: {}", + catalog.getName(), e.getMessage(), e); + } } } @@ -160,32 +158,28 @@ private void validateCatalogs(List catalogs) { for (CatalogMetadata catalog : catalogs) { if (StringUtils.isEmpty(catalog.getName())) { - LOG.error("Found a catalog with no name. {}", catalog.toString()); throw new IllegalArgumentException( "Missing Name Field from a catalog. Name is a required parameter."); } if (!catalog.getName().matches(CATALOG_NAME_REGEX)) { - LOG.error(String.format("Catalog Name: %s contains illegal characters." - + " Allowed characters: a-zA-Z0-9_-*@ ", catalog.getName())); throw new IllegalArgumentException( String.format("Catalog Name: %s contains illegal characters." + " Allowed characters: a-zA-Z0-9_-*@ ", catalog.getName())); } - if (StringUtils.isEmpty(catalog.getUri())) { - LOG.error("Found a catalog with no uri. {}", catalog.toString()); - throw new IllegalArgumentException( - "Missing URI Field from a catalog. URI is a required parameter."); - } - String catalogName = catalog.getName(); if (reviewedCatalogs.contains(catalogName)) { - LOG.error("Found duplicate catalog names"); throw new IllegalArgumentException("Catalogs with same name are not allowed."); } else { reviewedCatalogs.add(catalogName); } + + if (Objects.isNull(catalog.getProperties())) { + throw new IllegalArgumentException("Missing properties field in catalog configuration. " + + "Properties are required parameters"); + } + } } diff --git a/plugin/src/main/plugin-metadata/plugin-security.policy b/plugin/src/main/plugin-metadata/plugin-security.policy index 2dda426dc1..aec517aa84 100644 --- a/plugin/src/main/plugin-metadata/plugin-security.policy +++ b/plugin/src/main/plugin-metadata/plugin-security.policy @@ -9,6 +9,7 @@ grant { permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "defineClass"; permission java.lang.RuntimePermission "getClassLoader"; + permission java.lang.RuntimePermission "accessUserInformation"; permission java.net.NetPermission "getProxySelector"; permission java.net.SocketPermission "*", "accept,connect,resolve"; diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java index 4d339f384b..07ee458e5c 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/catalog/CatalogServiceImplTest.java @@ -63,28 +63,27 @@ public void testLoadConnectorsWithMultipleCatalogs() { @Test public void testLoadConnectorsWithMissingName() { Settings settings = getCatalogSettings("catalog_missing_name.json"); - IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, - () -> CatalogServiceImpl.getInstance().loadConnectors(settings)); - Assert.assertEquals("Missing Name Field from a catalog. Name is a required parameter.", - exception.getMessage()); + Set expected = CatalogServiceImpl.getInstance().getCatalogs(); + CatalogServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } @SneakyThrows @Test public void testLoadConnectorsWithDuplicateCatalogNames() { Settings settings = getCatalogSettings("duplicate_catalog_names.json"); - IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, - () -> CatalogServiceImpl.getInstance().loadConnectors(settings)); - Assert.assertEquals("Catalogs with same name are not allowed.", - exception.getMessage()); + Set expected = CatalogServiceImpl.getInstance().getCatalogs(); + CatalogServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } @SneakyThrows @Test public void testLoadConnectorsWithMalformedJson() { Settings settings = getCatalogSettings("malformed_catalogs.json"); - Assert.assertThrows(IllegalArgumentException.class, - () -> CatalogServiceImpl.getInstance().loadConnectors(settings)); + Set expected = CatalogServiceImpl.getInstance().getCatalogs(); + CatalogServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } @SneakyThrows @@ -124,10 +123,9 @@ public void testGetStorageEngineAfterLoadingConnectors() { @Test public void testLoadConnectorsWithIllegalCatalogNames() { Settings settings = getCatalogSettings("illegal_catalog_name.json"); - IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, - () -> CatalogServiceImpl.getInstance().loadConnectors(settings)); - Assert.assertEquals("Catalog Name: prometheus.test contains illegal characters." - + " Allowed characters: a-zA-Z0-9_-*@ ", exception.getMessage()); + Set expected = CatalogServiceImpl.getInstance().getCatalogs(); + CatalogServiceImpl.getInstance().loadConnectors(settings); + Assert.assertEquals(expected, CatalogServiceImpl.getInstance().getCatalogs()); } private Settings getCatalogSettings(String filename) throws URISyntaxException, IOException { diff --git a/plugin/src/test/resources/catalog_missing_name.json b/plugin/src/test/resources/catalog_missing_name.json index 86dc752cf0..4491ebb0db 100644 --- a/plugin/src/test/resources/catalog_missing_name.json +++ b/plugin/src/test/resources/catalog_missing_name.json @@ -1,11 +1,11 @@ [ { "connector": "prometheus", - "uri" : "http://localhost:9090", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "password" + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "type" } } ] \ No newline at end of file diff --git a/plugin/src/test/resources/catalogs.json b/plugin/src/test/resources/catalogs.json index aae3403462..5756b05094 100644 --- a/plugin/src/test/resources/catalogs.json +++ b/plugin/src/test/resources/catalogs.json @@ -2,11 +2,11 @@ { "name" : "prometheus", "connector": "prometheus", - "uri" : "http://localhost:9090", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "password" + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "type" } } ] \ No newline at end of file diff --git a/plugin/src/test/resources/duplicate_catalog_names.json b/plugin/src/test/resources/duplicate_catalog_names.json index b2f3694e5c..eefc56b6ef 100644 --- a/plugin/src/test/resources/duplicate_catalog_names.json +++ b/plugin/src/test/resources/duplicate_catalog_names.json @@ -2,21 +2,21 @@ { "name" : "prometheus", "connector": "prometheus", - "uri" : "http://localhost:9090", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "password" + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "type" } }, { "name" : "prometheus", "connector": "prometheus", - "uri" : "http://localhost:9219", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "password" + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "type" } } ] \ No newline at end of file diff --git a/plugin/src/test/resources/illegal_catalog_name.json b/plugin/src/test/resources/illegal_catalog_name.json index 359bbcd712..212ca6ec93 100644 --- a/plugin/src/test/resources/illegal_catalog_name.json +++ b/plugin/src/test/resources/illegal_catalog_name.json @@ -2,11 +2,11 @@ { "name" : "prometheus.test", "connector": "prometheus", - "uri" : "http://localhost:9090", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "password" + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "type" } } ] \ No newline at end of file diff --git a/plugin/src/test/resources/multiple_catalogs.json b/plugin/src/test/resources/multiple_catalogs.json index 112ecad858..4dae501561 100644 --- a/plugin/src/test/resources/multiple_catalogs.json +++ b/plugin/src/test/resources/multiple_catalogs.json @@ -2,21 +2,22 @@ { "name" : "prometheus", "connector": "prometheus", - "uri" : "http://localhost:9090", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "password" + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "basicauth", + "prometheus.auth.username" : "admin", + "prometheus.auth.password" : "type" } }, { "name" : "prometheus-1", "connector": "prometheus", - "uri" : "http://localhost:9090", - "authentication" : { - "type" : "basicauth", - "username" : "admin", - "password" : "password" + "properties" : { + "prometheus.uri" : "http://localhost:9090", + "prometheus.auth.type" : "awssigv4", + "prometheus.auth.region" : "us-east-1", + "prometheus.auth.access_key" : "accessKey", + "prometheus.auth.secret_key" : "secretKey" } } ] \ No newline at end of file diff --git a/prometheus/build.gradle b/prometheus/build.gradle index c936e63785..45a3a4a8ed 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -9,23 +9,25 @@ plugins { id 'jacoco' } +repositories { + mavenCentral() + maven { url 'https://jitpack.io' } +} + dependencies { api project(':core') - api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" - compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" - api group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' + implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' + implementation 'com.github.babbel:okhttp-aws-signer:1.0.2' implementation group: 'org.json', name: 'json', version: '20180813' testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') testImplementation group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1' testImplementation group: 'org.mockito', name: 'mockito-core', version: '3.12.4' testImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '3.12.4' - testImplementation group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" - testImplementation group: 'org.opensearch.test', name: 'framework', version: "${opensearch_version}" testImplementation group: 'com.squareup.okhttp3', name: 'mockwebserver', version: '4.9.3' } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java new file mode 100644 index 0000000000..f3d91c55a2 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptor.java @@ -0,0 +1,59 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.authinterceptors; + +import com.babbel.mobile.android.commons.okhttpawssigner.OkHttpAwsV4Signer; +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import lombok.NonNull; +import okhttp3.Interceptor; +import okhttp3.Request; +import okhttp3.Response; + +public class AwsSigningInterceptor implements Interceptor { + + private OkHttpAwsV4Signer okHttpAwsV4Signer; + + private String accessKey; + + private String secretKey; + + /** + * AwsSigningInterceptor which intercepts http requests + * and adds required headers for sigv4 authentication. + * + * @param accessKey accessKey. + * @param secretKey secretKey. + * @param region region. + * @param serviceName serviceName. + */ + public AwsSigningInterceptor(@NonNull String accessKey, @NonNull String secretKey, + @NonNull String region, @NonNull String serviceName) { + this.okHttpAwsV4Signer = new OkHttpAwsV4Signer(region, serviceName); + this.accessKey = accessKey; + this.secretKey = secretKey; + } + + @Override + public Response intercept(Interceptor.Chain chain) throws IOException { + Request request = chain.request(); + + DateTimeFormatter timestampFormat = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'") + .withZone(ZoneId.of("GMT")); + + Request newRequest = request.newBuilder() + .addHeader("x-amz-date", timestampFormat.format(ZonedDateTime.now())) + .addHeader("host", request.url().host()) + .build(); + Request signed = okHttpAwsV4Signer.sign(newRequest, accessKey, secretKey); + return chain.proceed(signed); + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java new file mode 100644 index 0000000000..6151018567 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptor.java @@ -0,0 +1,34 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.authinterceptors; + +import java.io.IOException; +import lombok.NonNull; +import okhttp3.Credentials; +import okhttp3.Interceptor; +import okhttp3.Request; +import okhttp3.Response; + +public class BasicAuthenticationInterceptor implements Interceptor { + + private String credentials; + + public BasicAuthenticationInterceptor(@NonNull String username, @NonNull String password) { + this.credentials = Credentials.basic(username, password); + } + + + @Override + public Response intercept(Interceptor.Chain chain) throws IOException { + Request request = chain.request(); + Request authenticatedRequest = request.newBuilder() + .header("Authorization", credentials).build(); + return chain.proceed(authenticatedRequest); + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java index 1c427cd3a7..7068b848ca 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java @@ -9,6 +9,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -40,7 +42,8 @@ public PrometheusClientImpl(OkHttpClient okHttpClient, URI uri) { @Override public JSONObject queryRange(String query, Long start, Long end, String step) throws IOException { String queryUrl = String.format("%s/api/v1/query_range?query=%s&start=%s&end=%s&step=%s", - uri.toString().replaceAll("/$", ""), query, start, end, step); + uri.toString().replaceAll("/$", ""), URLEncoder.encode(query, StandardCharsets.UTF_8), + start, end, step); logger.debug("queryUrl: " + queryUrl); Request request = new Request.Builder() .url(queryUrl) @@ -106,4 +109,4 @@ private JSONObject readResponse(Response response) throws IOException { } -} +} \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java index 3deb41569e..5d4bf2ae7c 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java @@ -11,7 +11,6 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; -import org.opensearch.common.unit.TimeValue; /** * Prometheus metric query request. @@ -22,8 +21,6 @@ @AllArgsConstructor public class PrometheusQueryRequest { - public static final TimeValue DEFAULT_QUERY_TIMEOUT = TimeValue.timeValueMinutes(1L); - /** * PromQL. */ diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java new file mode 100644 index 0000000000..41cbf3748f --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactory.java @@ -0,0 +1,95 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import okhttp3.OkHttpClient; +import org.opensearch.sql.catalog.model.ConnectorType; +import org.opensearch.sql.catalog.model.auth.AuthenticationType; +import org.opensearch.sql.prometheus.authinterceptors.AwsSigningInterceptor; +import org.opensearch.sql.prometheus.authinterceptors.BasicAuthenticationInterceptor; +import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.client.PrometheusClientImpl; +import org.opensearch.sql.storage.StorageEngine; +import org.opensearch.sql.storage.StorageEngineFactory; + +public class PrometheusStorageFactory implements StorageEngineFactory { + + public static final String URI = "prometheus.uri"; + public static final String AUTH_TYPE = "prometheus.auth.type"; + public static final String USERNAME = "prometheus.auth.username"; + public static final String PASSWORD = "prometheus.auth.password"; + public static final String REGION = "prometheus.auth.region"; + public static final String ACCESS_KEY = "prometheus.auth.access_key"; + public static final String SECRET_KEY = "prometheus.auth.secret_key"; + + + @Override + public ConnectorType getConnectorType() { + return ConnectorType.PROMETHEUS; + } + + @Override + public StorageEngine getStorageEngine(String catalogName, Map requiredConfig) { + validateFieldsInConfig(requiredConfig, Set.of(URI)); + PrometheusClient prometheusClient; + try { + prometheusClient = new PrometheusClientImpl(getHttpClient(requiredConfig), + new URI(requiredConfig.get(URI))); + } catch (URISyntaxException e) { + throw new RuntimeException( + String.format("Prometheus Client creation failed due to: %s", e.getMessage())); + } + return new PrometheusStorageEngine(prometheusClient); + } + + + private OkHttpClient getHttpClient(Map config) { + OkHttpClient.Builder okHttpClient = new OkHttpClient.Builder(); + okHttpClient.callTimeout(1, TimeUnit.MINUTES); + okHttpClient.connectTimeout(30, TimeUnit.SECONDS); + if (config.get(AUTH_TYPE) != null) { + AuthenticationType authenticationType = AuthenticationType.get(config.get(AUTH_TYPE)); + if (AuthenticationType.BASICAUTH.equals(authenticationType)) { + validateFieldsInConfig(config, Set.of(USERNAME, PASSWORD)); + okHttpClient.addInterceptor(new BasicAuthenticationInterceptor(config.get(USERNAME), + config.get(PASSWORD))); + } else if (AuthenticationType.AWSSIGV4AUTH.equals(authenticationType)) { + validateFieldsInConfig(config, Set.of(REGION, ACCESS_KEY, SECRET_KEY)); + okHttpClient.addInterceptor(new AwsSigningInterceptor( + config.get(ACCESS_KEY), config.get(SECRET_KEY), + config.get(REGION), "aps")); + } else { + throw new IllegalArgumentException( + String.format("AUTH Type : %s is not supported with Prometheus Connector", + config.get(AUTH_TYPE))); + } + } + return okHttpClient.build(); + } + + private void validateFieldsInConfig(Map config, Set fields) { + Set missingFields = new HashSet<>(); + for (String field : fields) { + if (!config.containsKey(field)) { + missingFields.add(field); + } + } + if (missingFields.size() > 0) { + throw new IllegalArgumentException(String.format( + "Missing %s fields in the Prometheus connector properties.", missingFields)); + } + } + + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java new file mode 100644 index 0000000000..a9224bf80f --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/AwsSigningInterceptorTest.java @@ -0,0 +1,61 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.authinterceptors; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import lombok.SneakyThrows; +import okhttp3.Interceptor; +import okhttp3.Request; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class AwsSigningInterceptorTest { + + @Mock + private Interceptor.Chain chain; + + @Captor + ArgumentCaptor requestArgumentCaptor; + + @Test + void testConstructors() { + Assertions.assertThrows(NullPointerException.class, () -> + new AwsSigningInterceptor(null, "secretKey", "us-east-1", "aps")); + Assertions.assertThrows(NullPointerException.class, () -> + new AwsSigningInterceptor("accessKey", null, "us-east-1", "aps")); + Assertions.assertThrows(NullPointerException.class, () -> + new AwsSigningInterceptor("accessKey", "secretKey", null, "aps")); + Assertions.assertThrows(NullPointerException.class, () -> + new AwsSigningInterceptor("accessKey", "secretKey", "us-east-1", null)); + } + + @Test + @SneakyThrows + void testIntercept() { + when(chain.request()).thenReturn(new Request.Builder() + .url("http://localhost:9090") + .build()); + AwsSigningInterceptor awsSigningInterceptor + = new AwsSigningInterceptor("testAccessKey", "testSecretKey", "us-east-1", "aps"); + awsSigningInterceptor.intercept(chain); + verify(chain).proceed(requestArgumentCaptor.capture()); + Request request = requestArgumentCaptor.getValue(); + Assertions.assertNotNull(request.headers("Authorization")); + Assertions.assertNotNull(request.headers("x-amz-date")); + Assertions.assertNotNull(request.headers("host")); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java new file mode 100644 index 0000000000..b5b5acd457 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/authinterceptors/BasicAuthenticationInterceptorTest.java @@ -0,0 +1,61 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.authinterceptors; + +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import lombok.SneakyThrows; +import okhttp3.Credentials; +import okhttp3.Interceptor; +import okhttp3.Request; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.internal.matchers.Null; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class BasicAuthenticationInterceptorTest { + + @Mock + private Interceptor.Chain chain; + + @Captor + ArgumentCaptor requestArgumentCaptor; + + @Test + void testConstructors() { + Assertions.assertThrows(NullPointerException.class, () -> + new BasicAuthenticationInterceptor(null, "test")); + Assertions.assertThrows(NullPointerException.class, () -> + new BasicAuthenticationInterceptor("testAdmin", null)); + } + + + @Test + @SneakyThrows + void testIntercept() { + when(chain.request()).thenReturn(new Request.Builder() + .url("http://localhost:9090") + .build()); + BasicAuthenticationInterceptor basicAuthenticationInterceptor + = new BasicAuthenticationInterceptor("testAdmin", "testPassword"); + basicAuthenticationInterceptor.intercept(chain); + verify(chain).proceed(requestArgumentCaptor.capture()); + Request request = requestArgumentCaptor.getValue(); + Assertions.assertEquals( + Collections.singletonList(Credentials.basic("testAdmin", "testPassword")), + request.headers("Authorization")); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java index f49672a882..76abb05751 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/client/PrometheusClientImplTest.java @@ -28,7 +28,6 @@ import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; -import org.apache.http.HttpStatus; import org.json.JSONObject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -86,7 +85,7 @@ void testQueryRangeWith2xxStatusAndError() { void testQueryRangeWithNon2xxError() { MockResponse mockResponse = new MockResponse() .addHeader("Content-Type", "application/json; charset=utf-8") - .setResponseCode(HttpStatus.SC_BAD_REQUEST); + .setResponseCode(400); mockWebServer.enqueue(mockResponse); RuntimeException runtimeException = assertThrows(RuntimeException.class, diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java new file mode 100644 index 0000000000..1b54cde5d9 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageFactoryTest.java @@ -0,0 +1,135 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage; + +import java.util.HashMap; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.catalog.model.ConnectorType; +import org.opensearch.sql.storage.StorageEngine; + +@ExtendWith(MockitoExtension.class) +public class PrometheusStorageFactoryTest { + + @Test + void testGetConnectorType() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + Assertions.assertEquals(ConnectorType.PROMETHEUS, prometheusStorageFactory.getConnectorType()); + } + + @Test + @SneakyThrows + void testGetStorageEngineWithBasicAuth() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://dummyprometheus:9090"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "admin"); + properties.put("prometheus.auth.password", "admin"); + StorageEngine storageEngine + = prometheusStorageFactory.getStorageEngine("my_prometheus", properties); + Assertions.assertTrue(storageEngine instanceof PrometheusStorageEngine); + } + + @Test + @SneakyThrows + void testGetStorageEngineWithAWSSigV4Auth() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://dummyprometheus:9090"); + properties.put("prometheus.auth.type", "awssigv4"); + properties.put("prometheus.auth.region", "us-east-1"); + properties.put("prometheus.auth.secret_key", "accessKey"); + properties.put("prometheus.auth.access_key", "secretKey"); + StorageEngine storageEngine + = prometheusStorageFactory.getStorageEngine("my_prometheus", properties); + Assertions.assertTrue(storageEngine instanceof PrometheusStorageEngine); + } + + + @Test + @SneakyThrows + void testGetStorageEngineWithMissingURI() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + HashMap properties = new HashMap<>(); + properties.put("prometheus.auth.type", "awssigv4"); + properties.put("prometheus.auth.region", "us-east-1"); + properties.put("prometheus.auth.secret_key", "accessKey"); + properties.put("prometheus.auth.access_key", "secretKey"); + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + Assertions.assertEquals("Missing [prometheus.uri] fields " + + "in the Prometheus connector properties.", + exception.getMessage()); + } + + @Test + @SneakyThrows + void testGetStorageEngineWithMissingRegionInAWS() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://dummyprometheus:9090"); + properties.put("prometheus.auth.type", "awssigv4"); + properties.put("prometheus.auth.secret_key", "accessKey"); + properties.put("prometheus.auth.access_key", "secretKey"); + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + Assertions.assertEquals("Missing [prometheus.auth.region] fields in the " + + "Prometheus connector properties.", + exception.getMessage()); + } + + @Test + @SneakyThrows + void testGetStorageEngineWithWrongAuthType() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "https://test.com"); + properties.put("prometheus.auth.type", "random"); + properties.put("prometheus.auth.region", "us-east-1"); + properties.put("prometheus.auth.secret_key", "accessKey"); + properties.put("prometheus.auth.access_key", "secretKey"); + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + Assertions.assertEquals("AUTH Type : random is not supported with Prometheus Connector", + exception.getMessage()); + } + + + @Test + @SneakyThrows + void testGetStorageEngineWithNONEAuthType() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "https://test.com"); + StorageEngine storageEngine + = prometheusStorageFactory.getStorageEngine("my_prometheus", properties); + Assertions.assertTrue(storageEngine instanceof PrometheusStorageEngine); + } + + @Test + @SneakyThrows + void testGetStorageEngineWithInvalidURISyntax() { + PrometheusStorageFactory prometheusStorageFactory = new PrometheusStorageFactory(); + HashMap properties = new HashMap<>(); + properties.put("prometheus.uri", "http://dummyprometheus:9090? param"); + properties.put("prometheus.auth.type", "basicauth"); + properties.put("prometheus.auth.username", "admin"); + properties.put("prometheus.auth.password", "admin"); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> prometheusStorageFactory.getStorageEngine("my_prometheus", properties)); + Assertions.assertTrue( + exception.getMessage().contains("Prometheus Client creation failed due to:")); + } + + +} + From 4e418c7bf770f13cb80026ea3f270ccf7852a52a Mon Sep 17 00:00:00 2001 From: Jing Zhang Date: Wed, 2 Nov 2022 21:19:03 -0700 Subject: [PATCH 14/20] Add document for ml command. (#1001) * Add document for ml command. Signed-off-by: Jing Zhang * Update the index. Signed-off-by: Jing Zhang * Add deprecation for AD and KMEANS documents. Signed-off-by: Jing Zhang Signed-off-by: Jing Zhang --- docs/user/ppl/cmd/ad.rst | 2 +- docs/user/ppl/cmd/kmeans.rst | 2 +- docs/user/ppl/cmd/ml.rst | 136 +++++++++++++++++++++++++++++++++++ docs/user/ppl/index.rst | 2 + 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 docs/user/ppl/cmd/ml.rst diff --git a/docs/user/ppl/cmd/ad.rst b/docs/user/ppl/cmd/ad.rst index 9f61a8dd86..2a655e376e 100644 --- a/docs/user/ppl/cmd/ad.rst +++ b/docs/user/ppl/cmd/ad.rst @@ -1,5 +1,5 @@ ============= -ad +ad (deprecated by ml command) ============= .. rubric:: Table of contents diff --git a/docs/user/ppl/cmd/kmeans.rst b/docs/user/ppl/cmd/kmeans.rst index 4608473c2c..ce991984d3 100644 --- a/docs/user/ppl/cmd/kmeans.rst +++ b/docs/user/ppl/cmd/kmeans.rst @@ -1,5 +1,5 @@ ============= -kmeans +kmeans (deprecated by ml command) ============= .. rubric:: Table of contents diff --git a/docs/user/ppl/cmd/ml.rst b/docs/user/ppl/cmd/ml.rst new file mode 100644 index 0000000000..2e04674c1e --- /dev/null +++ b/docs/user/ppl/cmd/ml.rst @@ -0,0 +1,136 @@ +============= +ml +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| The ``ml`` command is to train/predict/trainandpredict on any algorithm in the ml-commons plugin on the search result returned by a PPL command. + + +List of algorithms supported +============ +AD(RCF) +KMEANS + + +AD - Fixed In Time RCF For Time-series Data Command Syntax +===================================================== +ml action='train' algorithm='rcf' + +* number_of_trees(integer): optional. Number of trees in the forest. The default value is 30. +* shingle_size(integer): optional. A shingle is a consecutive sequence of the most recent records. The default value is 8. +* sample_size(integer): optional. The sample size used by stream samplers in this forest. The default value is 256. +* output_after(integer): optional. The number of points required by stream samplers before results are returned. The default value is 32. +* time_decay(double): optional. The decay factor used by stream samplers in this forest. The default value is 0.0001. +* anomaly_rate(double): optional. The anomaly rate. The default value is 0.005. +* time_field(string): mandatory. It specifies the time field for RCF to use as time-series data. +* date_format(string): optional. It's used for formatting time_field field. The default formatting is "yyyy-MM-dd HH:mm:ss". +* time_zone(string): optional. It's used for setting time zone for time_field filed. The default time zone is UTC. +* category_field(string): optional. It specifies the category field used to group inputs. Each category will be independently predicted. + + +AD - Batch RCF for Non-time-series Data Command Syntax +================================================= +ml action='train' algorithm='rcf' + +* number_of_trees(integer): optional. Number of trees in the forest. The default value is 30. +* sample_size(integer): optional. Number of random samples given to each tree from the training data set. The default value is 256. +* output_after(integer): optional. The number of points required by stream samplers before results are returned. The default value is 32. +* training_data_size(integer): optional. The default value is the size of your training data set. +* anomaly_score_threshold(double): optional. The threshold of anomaly score. The default value is 1.0. +* category_field(string): optional. It specifies the category field used to group inputs. Each category will be independently predicted. + +Example 1: Detecting events in New York City from taxi ridership data with time-series data +=========================================================================================== + +The example trains an RCF model and uses the model to detect anomalies in the time-series ridership data. + +PPL query:: + + os> source=nyc_taxi | fields value, timestamp | ml action='train' algorithm='rcf' time_field='timestamp' | where value=10844.0 + fetched rows / total rows = 1/1 + +---------+---------------------+---------+-----------------+ + | value | timestamp | score | anomaly_grade | + |---------+---------------------+---------+-----------------| + | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + +---------+---------------------+---------+-----------------+ + +Example 2: Detecting events in New York City from taxi ridership data with time-series data independently with each category +============================================================================================================================ + +The example trains an RCF model and uses the model to detect anomalies in the time-series ridership data with multiple category values. + +PPL query:: + + os> source=nyc_taxi | fields category, value, timestamp | ml action='train' algorithm='rcf' time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 + fetched rows / total rows = 2/2 + +------------+---------+---------------------+---------+-----------------+ + | category | value | timestamp | score | anomaly_grade | + |------------+---------+---------------------+---------+-----------------| + | night | 10844.0 | 2014-07-01 00:00:00 | 0.0 | 0.0 | + | day | 6526.0 | 2014-07-01 06:00:00 | 0.0 | 0.0 | + +------------+---------+---------------------+---------+-----------------+ + + +Example 3: Detecting events in New York City from taxi ridership data with non-time-series data +=============================================================================================== + +The example trains an RCF model and uses the model to detect anomalies in the non-time-series ridership data. + +PPL query:: + + os> source=nyc_taxi | fields value | ml action='train' algorithm='rcf' | where value=10844.0 + fetched rows / total rows = 1/1 + +---------+---------+-------------+ + | value | score | anomalous | + |---------+---------+-------------| + | 10844.0 | 0.0 | False | + +---------+---------+-------------+ + +Example 4: Detecting events in New York City from taxi ridership data with non-time-series data independently with each category +================================================================================================================================ + +The example trains an RCF model and uses the model to detect anomalies in the non-time-series ridership data with multiple category values. + +PPL query:: + + os> source=nyc_taxi | fields category, value | ml action='train' algorithm='rcf' category_field='category' | where value=10844.0 or value=6526.0 + fetched rows / total rows = 2/2 + +------------+---------+---------+-------------+ + | category | value | score | anomalous | + |------------+---------+---------+-------------| + | night | 10844.0 | 0.0 | False | + | day | 6526.0 | 0.0 | False | + +------------+---------+---------+-------------+ + +KMEANS +====== +ml action='train' algorithm='kmeans' + +* centroids: optional. The number of clusters you want to group your data points into. The default value is 2. +* iterations: optional. Number of iterations. The default value is 10. +* distance_type: optional. The distance type can be COSINE, L1, or EUCLIDEAN, The default type is EUCLIDEAN. + + +Example: Clustering of Iris Dataset +=================================== + +The example shows how to classify three Iris species (Iris setosa, Iris virginica and Iris versicolor) based on the combination of four features measured from each sample: the length and the width of the sepals and petals. + +PPL query:: + + os> source=iris_data | fields sepal_length_in_cm, sepal_width_in_cm, petal_length_in_cm, petal_width_in_cm | ml action='train' algorithm='kmeans' centroids=3 + +--------------------+-------------------+--------------------+-------------------+-----------+ + | sepal_length_in_cm | sepal_width_in_cm | petal_length_in_cm | petal_width_in_cm | ClusterID | + |--------------------+-------------------+--------------------+-------------------+-----------| + | 5.1 | 3.5 | 1.4 | 0.2 | 1 | + | 5.6 | 3.0 | 4.1 | 1.3 | 0 | + | 6.7 | 2.5 | 5.8 | 1.8 | 2 | + +--------------------+-------------------+--------------------+-------------------+-----------+ diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index a4f620173c..e09315b1c3 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -58,6 +58,8 @@ The query start with search command and then flowing a set of command delimited - `kmeans command `_ + - `ml command `_ + - `parse command `_ - `patterns command `_ From 293889096ae5a4ac6662f57a60baec73179d1fdc Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 2 Nov 2022 22:32:09 -0700 Subject: [PATCH 15/20] Add function name as identifier in antlr (#1015) Signed-off-by: Peng Huo --- docs/user/dql/functions.rst | 68 +++++++++--------- docs/user/ppl/functions/datetime.rst | 70 +++++++++---------- .../sql/ppl/DateTimeFunctionIT.java | 10 +-- .../sql/sql/DateTimeFunctionIT.java | 10 +-- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 12 ++-- .../sql/ppl/parser/AstExpressionBuilder.java | 17 ++--- .../ppl/parser/AstExpressionBuilderTest.java | 54 +++++++++++--- .../ppl/parser/AstNowLikeFunctionTest.java | 10 +-- .../antlr/OpenSearchSQLIdentifierParser.g4 | 2 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 8 +-- .../sql/sql/parser/AstExpressionBuilder.java | 17 ++--- .../sql/sql/parser/AstBuilderTest.java | 10 +-- .../parser/AstQualifiedNameBuilderTest.java | 25 +++++++ 13 files changed, 175 insertions(+), 138 deletions(-) diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 15f8a768bb..a04f01446e 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -987,17 +987,17 @@ CURRENT_DATE Description >>>>>>>>>>> -`CURRENT_DATE` and `CURRENT_DATE()` are synonyms for `CURDATE() <#curdate>`_. +`CURRENT_DATE()` are synonyms for `CURDATE() <#curdate>`_. Example:: - > SELECT CURRENT_DATE(), CURRENT_DATE; + > SELECT CURRENT_DATE(); fetched rows / total rows = 1/1 - +------------------+----------------+ - | CURRENT_DATE() | CURRENT_DATE | - |------------------+----------------| - | 2022-08-02 | 2022-08-02 | - +------------------+----------------+ + +------------------+ + | CURRENT_DATE() | + |------------------+ + | 2022-08-02 | + +------------------+ CURRENT_TIME @@ -1006,17 +1006,17 @@ CURRENT_TIME Description >>>>>>>>>>> -`CURRENT_TIME` and `CURRENT_TIME()` are synonyms for `CURTIME() <#curtime>`_. +`CURRENT_TIME()` are synonyms for `CURTIME() <#curtime>`_. Example:: - > SELECT CURRENT_TIME(), CURRENT_TIME; + > SELECT CURRENT_TIME(); fetched rows / total rows = 1/1 - +-----------------+----------------+ - | CURRENT_TIME() | CURRENT_TIME | - |-----------------+----------------| - | 15:39:05 | 15:39:05 | - +-----------------+----------------+ + +-----------------+ + | CURRENT_TIME() | + |-----------------+ + | 15:39:05 | + +-----------------+ CURRENT_TIMESTAMP @@ -1025,17 +1025,17 @@ CURRENT_TIMESTAMP Description >>>>>>>>>>> -`CURRENT_TIMESTAMP` and `CURRENT_TIMESTAMP()` are synonyms for `NOW() <#now>`_. +`CURRENT_TIMESTAMP()` are synonyms for `NOW() <#now>`_. Example:: - > SELECT CURRENT_TIMESTAMP(), CURRENT_TIMESTAMP; + > SELECT CURRENT_TIMESTAMP(); fetched rows / total rows = 1/1 - +-----------------------+---------------------+ - | CURRENT_TIMESTAMP() | CURRENT_TIMESTAMP | - |-----------------------+---------------------| - | 2022-08-02 15:54:19 | 2022-08-02 15:54:19 | - +-----------------------+---------------------+ + +-----------------------+ + | CURRENT_TIMESTAMP() | + |-----------------------+ + | 2022-08-02 15:54:19 | + +-----------------------+ CURTIME @@ -1548,17 +1548,17 @@ LOCALTIMESTAMP Description >>>>>>>>>>> -`LOCALTIMESTAMP` and `LOCALTIMESTAMP()` are synonyms for `NOW() <#now>`_. +`LOCALTIMESTAMP()` are synonyms for `NOW() <#now>`_. Example:: - > SELECT LOCALTIMESTAMP(), LOCALTIMESTAMP; + > SELECT LOCALTIMESTAMP(); fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LOCALTIMESTAMP() | LOCALTIMESTAMP | - |---------------------+---------------------| - | 2022-08-02 15:54:19 | 2022-08-02 15:54:19 | - +---------------------+---------------------+ + +---------------------+ + | LOCALTIMESTAMP() | + |---------------------+ + | 2022-08-02 15:54:19 | + +---------------------+ LOCALTIME @@ -1567,17 +1567,17 @@ LOCALTIME Description >>>>>>>>>>> -`LOCALTIME` and `LOCALTIME()` are synonyms for `NOW() <#now>`_. +`LOCALTIME()` are synonyms for `NOW() <#now>`_. Example:: > SELECT LOCALTIME(), LOCALTIME; fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LOCALTIME() | LOCALTIME | - |---------------------+---------------------| - | 2022-08-02 15:54:19 | 2022-08-02 15:54:19 | - +---------------------+---------------------+ + +---------------------+ + | LOCALTIME() | + |---------------------+ + | 2022-08-02 15:54:19 | + +---------------------+ MAKEDATE diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 7d783e914b..223b3c5557 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -202,17 +202,17 @@ CURRENT_DATE Description >>>>>>>>>>> -`CURRENT_DATE` and `CURRENT_DATE()` are synonyms for `CURDATE() <#curdate>`_. +`CURRENT_DATE()` are synonyms for `CURDATE() <#curdate>`_. Example:: - > source=people | eval `CURRENT_DATE()` = CURRENT_DATE(), `CURRENT_DATE` = CURRENT_DATE | fields `CURRENT_DATE()`, `CURRENT_DATE` + > source=people | eval `CURRENT_DATE()` = CURRENT_DATE() | fields `CURRENT_DATE()` fetched rows / total rows = 1/1 - +------------------+----------------+ - | CURRENT_DATE() | CURRENT_DATE | - |------------------+----------------| - | 2022-08-02 | 2022-08-02 | - +------------------+----------------+ + +------------------+ + | CURRENT_DATE() | + |------------------+ + | 2022-08-02 | + +------------------+ CURRENT_TIME @@ -221,17 +221,17 @@ CURRENT_TIME Description >>>>>>>>>>> -`CURRENT_TIME` and `CURRENT_TIME()` are synonyms for `CURTIME() <#curtime>`_. +`CURRENT_TIME()` are synonyms for `CURTIME() <#curtime>`_. Example:: - > source=people | eval `CURRENT_TIME()` = CURRENT_TIME(), `CURRENT_TIME` = CURRENT_TIME | fields `CURRENT_TIME()`, `CURRENT_TIME` + > source=people | eval `CURRENT_TIME()` = CURRENT_TIME() | fields `CURRENT_TIME()` fetched rows / total rows = 1/1 - +------------------+----------------+ - | CURRENT_TIME() | CURRENT_TIME | - |------------------+----------------| - | 15:39:05 | 15:39:05 | - +------------------+----------------+ + +------------------+ + | CURRENT_TIME() | + |------------------+ + | 15:39:05 | + +------------------+ CURRENT_TIMESTAMP @@ -240,17 +240,17 @@ CURRENT_TIMESTAMP Description >>>>>>>>>>> -`CURRENT_TIMESTAMP` and `CURRENT_TIMESTAMP()` are synonyms for `NOW() <#now>`_. +`CURRENT_TIMESTAMP()` are synonyms for `NOW() <#now>`_. Example:: - > source=people | eval `CURRENT_TIMESTAMP()` = CURRENT_TIMESTAMP(), `CURRENT_TIMESTAMP` = CURRENT_TIMESTAMP | fields `CURRENT_TIMESTAMP()`, `CURRENT_TIMESTAMP` + > source=people | eval `CURRENT_TIMESTAMP()` = CURRENT_TIMESTAMP() | fields `CURRENT_TIMESTAMP()` fetched rows / total rows = 1/1 - +-----------------------+---------------------+ - | CURRENT_TIMESTAMP() | CURRENT_TIMESTAMP | - |-----------------------+---------------------| - | 2022-08-02 15:54:19 | 2022-08-02 15:54:19 | - +-----------------------+---------------------+ + +-----------------------+ + | CURRENT_TIMESTAMP() | + |-----------------------+ + | 2022-08-02 15:54:19 | + +-----------------------+ CURTIME @@ -720,17 +720,17 @@ LOCALTIMESTAMP Description >>>>>>>>>>> -`LOCALTIMESTAMP` and `LOCALTIMESTAMP()` are synonyms for `NOW() <#now>`_. +`LOCALTIMESTAMP()` are synonyms for `NOW() <#now>`_. Example:: - > source=people | eval `LOCALTIMESTAMP()` = LOCALTIMESTAMP(), `LOCALTIMESTAMP` = LOCALTIMESTAMP | fields `LOCALTIMESTAMP()`, `LOCALTIMESTAMP` + > source=people | eval `LOCALTIMESTAMP()` = LOCALTIMESTAMP() | fields `LOCALTIMESTAMP()` fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LOCALTIMESTAMP() | LOCALTIMESTAMP | - |---------------------+---------------------| - | 2022-08-02 15:54:19 | 2022-08-02 15:54:19 | - +---------------------+---------------------+ + +---------------------+ + | LOCALTIMESTAMP() | + |---------------------+ + | 2022-08-02 15:54:19 | + +---------------------+ LOCALTIME @@ -739,17 +739,17 @@ LOCALTIME Description >>>>>>>>>>> -`LOCALTIME` and `LOCALTIME()` are synonyms for `NOW() <#now>`_. +`LOCALTIME()` are synonyms for `NOW() <#now>`_. Example:: - > source=people | eval `LOCALTIME()` = LOCALTIME(), `LOCALTIME` = LOCALTIME | fields `LOCALTIME()`, `LOCALTIME` + > source=people | eval `LOCALTIME()` = LOCALTIME() | fields `LOCALTIME()` fetched rows / total rows = 1/1 - +---------------------+---------------------+ - | LOCALTIME() | LOCALTIME | - |---------------------+---------------------| - | 2022-08-02 15:54:19 | 2022-08-02 15:54:19 | - +---------------------+---------------------+ + +---------------------+ + | LOCALTIME() | + |---------------------+ + | 2022-08-02 15:54:19 | + +---------------------+ MAKEDATE diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 8c0661d6ff..afabc241fe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -686,7 +686,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "current_timestamp") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", true) .put("referenceGetter", (Supplier) LocalDateTime::now) .put("parser", (BiFunction) LocalDateTime::parse) @@ -695,7 +695,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "localtimestamp") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", true) .put("referenceGetter", (Supplier) LocalDateTime::now) .put("parser", (BiFunction) LocalDateTime::parse) @@ -704,7 +704,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "localtime") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", true) .put("referenceGetter", (Supplier) LocalDateTime::now) .put("parser", (BiFunction) LocalDateTime::parse) @@ -731,7 +731,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "current_time") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", false) .put("referenceGetter", (Supplier) LocalTime::now) .put("parser", (BiFunction) LocalTime::parse) @@ -749,7 +749,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "current_date") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", false) .put("referenceGetter", (Supplier) LocalDate::now) .put("parser", (BiFunction) LocalDate::parse) diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 15dee748b2..8c47966e52 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -505,7 +505,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "current_timestamp") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", true) .put("referenceGetter", (Supplier) LocalDateTime::now) .put("parser", (BiFunction) LocalDateTime::parse) @@ -514,7 +514,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "localtimestamp") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", true) .put("referenceGetter", (Supplier) LocalDateTime::now) .put("parser", (BiFunction) LocalDateTime::parse) @@ -523,7 +523,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "localtime") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", true) .put("referenceGetter", (Supplier) LocalDateTime::now) .put("parser", (BiFunction) LocalDateTime::parse) @@ -550,7 +550,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "current_time") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", false) .put("referenceGetter", (Supplier) LocalTime::now) .put("parser", (BiFunction) LocalTime::parse) @@ -568,7 +568,7 @@ private List> nowLikeFunctionsData() { ImmutableMap.builder() .put("name", "current_date") .put("hasFsp", false) - .put("hasShortcut", true) + .put("hasShortcut", false) .put("constValue", false) .put("referenceGetter", (Supplier) LocalDate::now) .put("parser", (BiFunction) LocalDate::parse) diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 11e1c4c71f..99ff931f39 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -430,7 +430,7 @@ dateAndTimeFunctionBase // Functions which value could be cached in scope of a single query constantFunctionName - : datetimeConstantLiteral + : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME | CURDATE | CURTIME | NOW ; @@ -507,7 +507,6 @@ datetimeLiteral : dateLiteral | timeLiteral | timestampLiteral - | datetimeConstantLiteral ; dateLiteral @@ -522,11 +521,6 @@ timestampLiteral : TIMESTAMP timestamp=stringLiteral ; -// Actually, these constants are shortcuts to the corresponding functions -datetimeConstantLiteral - : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME - ; - intervalUnit : MICROSECOND | SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR | SECOND_MICROSECOND | MINUTE_MICROSECOND | MINUTE_SECOND | HOUR_MICROSECOND | HOUR_SECOND | HOUR_MINUTE | DAY_MICROSECOND @@ -571,4 +565,8 @@ keywordsCanBeId | TIMESTAMP | DATE | TIME | FIRST | LAST | timespanUnit | SPAN + | constantFunctionName + | dateAndTimeFunctionBase + | textFunctionBase + | mathematicalFunctionBase ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index d2e6131cbe..4430820081 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -18,7 +18,6 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ConvertedDataTypeContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CountAllFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DataTypeFunctionCallContext; -import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DatetimeConstantLiteralContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DecimalLiteralContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DistinctCountFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalClauseContext; @@ -258,11 +257,6 @@ public UnresolvedExpression visitConvertedDataType(ConvertedDataTypeContext ctx) return AstDSL.stringLiteral(ctx.getText()); } - @Override - public UnresolvedExpression visitDatetimeConstantLiteral(DatetimeConstantLiteralContext ctx) { - return visitConstantFunction(ctx.getText(), null); - } - public UnresolvedExpression visitConstantFunction(ConstantFunctionContext ctx) { return visitConstantFunction(ctx.constantFunctionName().getText(), ctx.functionArgs()); @@ -270,13 +264,10 @@ public UnresolvedExpression visitConstantFunction(ConstantFunctionContext ctx) { private UnresolvedExpression visitConstantFunction(String functionName, FunctionArgsContext args) { - return new ConstantFunction(functionName, - args == null - ? Collections.emptyList() - : args.functionArg() - .stream() - .map(this::visitFunctionArg) - .collect(Collectors.toList())); + return new ConstantFunction(functionName, args.functionArg() + .stream() + .map(this::visitFunctionArg) + .collect(Collectors.toList())); } private Function visitFunction(String functionName, FunctionArgsContext args) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 565157dd06..e4048c5fe1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -7,7 +7,7 @@ package org.opensearch.sql.ppl.parser; import static java.util.Collections.emptyList; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; @@ -45,16 +45,14 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; -import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; import org.junit.Ignore; import org.junit.Test; -import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.expression.AllFields; -import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.DataType; -import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.RelevanceFieldList; -import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; public class AstExpressionBuilderTest extends AstBuilderTest { @@ -756,8 +754,46 @@ public void canBuildQuery_stringRelevanceFunctionWithArguments() { ); } - private Node buildExprAst(String query) { - AstBuilder astBuilder = new AstBuilder(new AstExpressionBuilder(), query); - return astBuilder.visit(new PPLSyntaxParser().parse(query)); + @Test + public void functionNameCanBeUsedAsIdentifier() { + assertFunctionNameCouldBeId( + "AVG | COUNT | SUM | MIN | MAX | VAR_SAMP | VAR_POP | STDDEV_SAMP | STDDEV_POP"); + assertFunctionNameCouldBeId( + "CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | " + + "UTC_TIMESTAMP | UTC_DATE | UTC_TIME | CURDATE | CURTIME | NOW"); + assertFunctionNameCouldBeId( + "ADDDATE | CONVERT_TZ | DATE | DATE_ADD | DATE_FORMAT | DATE_SUB " + + "| DATETIME | DAY | DAYNAME | DAYOFMONTH " + + "| DAYOFWEEK | DAYOFYEAR | FROM_DAYS | FROM_UNIXTIME | HOUR | MAKEDATE | MAKETIME " + + "| MICROSECOND | MINUTE | MONTH | MONTHNAME " + + "| PERIOD_ADD | PERIOD_DIFF | QUARTER | SECOND | SUBDATE | SYSDATE | TIME " + + "| TIME_TO_SEC | TIMESTAMP | TO_DAYS | UNIX_TIMESTAMP | WEEK | YEAR"); + assertFunctionNameCouldBeId( + "SUBSTR | SUBSTRING | TRIM | LTRIM | RTRIM | LOWER | UPPER | CONCAT | CONCAT_WS | LENGTH " + + "| STRCMP | RIGHT | LEFT | ASCII | LOCATE | REPLACE" + ); + assertFunctionNameCouldBeId( + "ABS | CEIL | CEILING | CONV | CRC32 | E | EXP | FLOOR | LN | LOG" + + " | LOG10 | LOG2 | MOD | PI |POW | POWER | RAND | ROUND | SIGN | SQRT | TRUNCATE " + + "| ACOS | ASIN | ATAN | ATAN2 | COS | COT | DEGREES | RADIANS | SIN | TAN"); + } + + void assertFunctionNameCouldBeId(String antlrFunctionName) { + List functionList = + Arrays.stream(antlrFunctionName.split("\\|")).map(String::stripLeading) + .map(String::stripTrailing).collect( + Collectors.toList()); + + assertFalse(functionList.isEmpty()); + for (String functionName : functionList) { + assertEqual(String.format(Locale.ROOT, "source=t | fields %s", functionName), + projectWithArg( + relation("t"), + defaultFieldsArgs(), + field( + qualifiedName(functionName) + ) + )); + } } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java index 6c6233a17f..1350305391 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java @@ -52,14 +52,14 @@ public AstNowLikeFunctionTest(String name, Boolean hasFsp, Boolean hasShortcut, public static Iterable functionNames() { return List.of(new Object[][]{ {"now", false, false, true}, - {"current_timestamp", false, true, true}, - {"localtimestamp", false, true, true}, - {"localtime", false, true, true}, + {"current_timestamp", false, false, true}, + {"localtimestamp", false, false, true}, + {"localtime", false, false, true}, {"sysdate", true, false, false}, {"curtime", false, false, true}, - {"current_time", false, true, true}, + {"current_time", false, false, true}, {"curdate", false, false, true}, - {"current_date", false, true, true} + {"current_date", false, false, true} }); } diff --git a/sql/src/main/antlr/OpenSearchSQLIdentifierParser.g4 b/sql/src/main/antlr/OpenSearchSQLIdentifierParser.g4 index 665d48c97c..cd65e5066c 100644 --- a/sql/src/main/antlr/OpenSearchSQLIdentifierParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLIdentifierParser.g4 @@ -63,4 +63,6 @@ keywordsCanBeId | COUNT | SUM | AVG | MAX | MIN | TIMESTAMP | DATE | TIME | DAYOFWEEK | FIRST | LAST + | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME + | CURDATE | CURTIME | NOW ; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 6470c9f546..4dedaaf396 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -225,7 +225,6 @@ datetimeLiteral : dateLiteral | timeLiteral | timestampLiteral - | datetimeConstantLiteral ; dateLiteral @@ -240,11 +239,6 @@ timestampLiteral : TIMESTAMP timestamp=stringLiteral ; -// Actually, these constants are shortcuts to the corresponding functions -datetimeConstantLiteral - : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME - ; - intervalLiteral : INTERVAL expression intervalUnit ; @@ -404,7 +398,7 @@ dateTimeFunctionName // Functions which value could be cached in scope of a single query constantFunctionName - : datetimeConstantLiteral + : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME | CURDATE | CURTIME | NOW ; diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index ebfafeec23..0d551e3f38 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -23,7 +23,6 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CountStarFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DataTypeFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DateLiteralContext; -import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DatetimeConstantLiteralContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DistinctCountFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IsNullPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.LikePredicateContext; @@ -407,11 +406,6 @@ private Function visitFunction(String functionName, FunctionArgsContext args) { ); } - @Override - public UnresolvedExpression visitDatetimeConstantLiteral(DatetimeConstantLiteralContext ctx) { - return visitConstantFunction(ctx.getText(), null); - } - @Override public UnresolvedExpression visitConstantFunction(ConstantFunctionContext ctx) { return visitConstantFunction(ctx.constantFunctionName().getText(), @@ -420,13 +414,10 @@ public UnresolvedExpression visitConstantFunction(ConstantFunctionContext ctx) { private UnresolvedExpression visitConstantFunction(String functionName, FunctionArgsContext args) { - return new ConstantFunction(functionName, - args == null - ? Collections.emptyList() - : args.functionArg() - .stream() - .map(this::visitFunctionArg) - .collect(Collectors.toList())); + return new ConstantFunction(functionName, args.functionArg() + .stream() + .map(this::visitFunctionArg) + .collect(Collectors.toList())); } private QualifiedName visitIdentifiers(List identifiers) { diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 0c06754261..a955399c4d 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -682,14 +682,14 @@ public void can_build_limit_clause_with_offset() { private static Stream nowLikeFunctionsData() { return Stream.of( Arguments.of("now", false, false, true), - Arguments.of("current_timestamp", false, true, true), - Arguments.of("localtimestamp", false, true, true), - Arguments.of("localtime", false, true, true), + Arguments.of("current_timestamp", false, false, true), + Arguments.of("localtimestamp", false, false, true), + Arguments.of("localtime", false, false, true), Arguments.of("sysdate", true, false, false), Arguments.of("curtime", false, false, true), - Arguments.of("current_time", false, true, true), + Arguments.of("current_time", false, false, true), Arguments.of("curdate", false, false, true), - Arguments.of("current_date", false, true, true) + Arguments.of("current_date", false, false, true) ); } diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java index fdd4f2f58c..92b535144f 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstQualifiedNameBuilderTest.java @@ -7,8 +7,12 @@ package org.opensearch.sql.sql.parser; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import java.util.Arrays; +import java.util.List; import java.util.function.Function; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.tree.RuleNode; @@ -47,6 +51,27 @@ public void canBuildQualifiedIdentifier() { buildFromQualifiers("account.location.city").expectQualifiedName("account", "location", "city"); } + + @Test + public void functionNameCanBeUsedAsIdentifier() { + assertFunctionNameCouldBeId("AVG | COUNT | SUM | MIN | MAX"); + assertFunctionNameCouldBeId( + "CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP |" + + " UTC_TIMESTAMP | UTC_DATE | UTC_TIME | CURDATE | CURTIME | NOW"); + } + + void assertFunctionNameCouldBeId(String antlrFunctionName) { + List functionList = + Arrays.stream(antlrFunctionName.split("\\|")).map(String::stripLeading) + .map(String::stripTrailing).collect( + Collectors.toList()); + + assertFalse(functionList.isEmpty()); + for (String functionName : functionList) { + buildFromQualifiers(functionName).expectQualifiedName(functionName); + } + } + private AstExpressionBuilderAssertion buildFromIdentifier(String expr) { return new AstExpressionBuilderAssertion(OpenSearchSQLParser::ident, expr); } From 1a525112a86f7025852d282a06c5022ec51f633f Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 3 Nov 2022 09:39:10 -0700 Subject: [PATCH 16/20] Disable loading `ML-commons` plugin for doctests and corresponding tests. (#1011) Signed-off-by: Yury-Fridlyand Signed-off-by: Yury-Fridlyand --- docs/user/ppl/cmd/ad.rst | 8 ++++---- docs/user/ppl/cmd/kmeans.rst | 2 +- doctest/build.gradle | 3 +++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/user/ppl/cmd/ad.rst b/docs/user/ppl/cmd/ad.rst index 2a655e376e..103c7f7483 100644 --- a/docs/user/ppl/cmd/ad.rst +++ b/docs/user/ppl/cmd/ad.rst @@ -48,7 +48,7 @@ The example trains an RCF model and uses the model to detect anomalies in the ti PPL query:: - os> source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0 + > source=nyc_taxi | fields value, timestamp | AD time_field='timestamp' | where value=10844.0 fetched rows / total rows = 1/1 +---------+---------------------+---------+-----------------+ | value | timestamp | score | anomaly_grade | @@ -63,7 +63,7 @@ The example trains an RCF model and uses the model to detect anomalies in the ti PPL query:: - os> source=nyc_taxi | fields category, value, timestamp | AD time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 + > source=nyc_taxi | fields category, value, timestamp | AD time_field='timestamp' category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 +------------+---------+---------------------+---------+-----------------+ | category | value | timestamp | score | anomaly_grade | @@ -80,7 +80,7 @@ The example trains an RCF model and uses the model to detect anomalies in the no PPL query:: - os> source=nyc_taxi | fields value | AD | where value=10844.0 + > source=nyc_taxi | fields value | AD | where value=10844.0 fetched rows / total rows = 1/1 +---------+---------+-------------+ | value | score | anomalous | @@ -95,7 +95,7 @@ The example trains an RCF model and uses the model to detect anomalies in the no PPL query:: - os> source=nyc_taxi | fields category, value | AD category_field='category' | where value=10844.0 or value=6526.0 + > source=nyc_taxi | fields category, value | AD category_field='category' | where value=10844.0 or value=6526.0 fetched rows / total rows = 2/2 +------------+---------+---------+-------------+ | category | value | score | anomalous | diff --git a/docs/user/ppl/cmd/kmeans.rst b/docs/user/ppl/cmd/kmeans.rst index ce991984d3..faf29d078b 100644 --- a/docs/user/ppl/cmd/kmeans.rst +++ b/docs/user/ppl/cmd/kmeans.rst @@ -30,7 +30,7 @@ The example shows how to classify three Iris species (Iris setosa, Iris virginic PPL query:: - os> source=iris_data | fields sepal_length_in_cm, sepal_width_in_cm, petal_length_in_cm, petal_width_in_cm | kmeans centroids=3 + > source=iris_data | fields sepal_length_in_cm, sepal_width_in_cm, petal_length_in_cm, petal_width_in_cm | kmeans centroids=3 +--------------------+-------------------+--------------------+-------------------+-----------+ | sepal_length_in_cm | sepal_width_in_cm | petal_length_in_cm | petal_width_in_cm | ClusterID | |--------------------+-------------------+--------------------+-------------------+-----------| diff --git a/doctest/build.gradle b/doctest/build.gradle index bf789e7434..8378d5ec00 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -105,6 +105,8 @@ String mlCommonsPlugin = 'opensearch-ml' testClusters { docTestCluster { keystore 'plugins.query.federation.catalog.config', new File("$projectDir/catalog", 'catalog.json') + // Disable loading of `ML-commons` plugin, because it might be unavailable (not released yet). + /* plugin(provider(new Callable(){ @Override RegularFile call() throws Exception { @@ -124,6 +126,7 @@ testClusters { } } })) + */ plugin ':opensearch-sql-plugin' testDistribution = 'integ_test' } From be4512e0a5b5466b12defa88afd6d5ec6476ea6f Mon Sep 17 00:00:00 2001 From: vamsi-amazon Date: Thu, 3 Nov 2022 09:51:23 -0700 Subject: [PATCH 17/20] Prometheus select metric and stats queries. (#956) * Prometheus select metric and stats queries. Signed-off-by: reddyvam-amazon --- .../org/opensearch/sql/analysis/Analyzer.java | 6 + docs/user/ppl/admin/prometheus_connector.rst | 113 +++ docs/user/ppl/cmd/describe.rst | 3 +- .../opensearch/sql/ppl/DescribeCommandIT.java | 1 - .../sql/ppl/PrometheusCatalogCommandsIT.java | 158 +++++ .../plugin/catalog/CatalogServiceImpl.java | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 - .../opensearch/sql/ppl/parser/AstBuilder.java | 19 +- .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 10 +- .../sql/ppl/parser/AstBuilderTest.java | 2 + .../ppl/utils/PPLQueryDataAnonymizerTest.java | 2 + .../client/PrometheusClientImpl.java | 10 +- .../QueryRangeFunctionImplementation.java | 2 +- .../logical/PrometheusLogicalMetricAgg.java | 76 +++ .../logical/PrometheusLogicalMetricScan.java | 54 ++ ...PrometheusLogicalPlanOptimizerFactory.java | 32 + .../logical/rules/MergeAggAndIndexScan.java | 56 ++ .../logical/rules/MergeAggAndRelation.java | 54 ++ .../logical/rules/MergeFilterAndRelation.java | 53 ++ .../request/PrometheusQueryRequest.java | 16 +- .../response/PrometheusResponse.java | 67 +- .../PrometheusMetricDefaultSchema.java | 3 - .../storage/PrometheusMetricScan.java | 20 +- .../storage/PrometheusMetricTable.java | 33 +- .../storage/PrometheusStorageEngine.java | 2 +- .../PrometheusDefaultImplementor.java | 124 +++- .../model/PrometheusResponseFieldNames.java | 27 + .../storage/model/QueryRangeParameters.java | 25 + .../querybuilder/AggregationQueryBuilder.java | 78 +++ .../SeriesSelectionQueryBuilder.java | 70 ++ .../querybuilder/StepParameterResolver.java | 63 ++ .../TimeRangeParametersResolver.java | 76 +++ .../QueryRangeFunctionImplementationTest.java | 8 +- .../QueryRangeTableFunctionResolverTest.java | 22 +- .../logical/PrometheusLogicOptimizerTest.java | 121 ++++ .../PrometheusDescribeMetricRequestTest.java | 13 +- .../storage/PrometheusMetricScanTest.java | 136 +++- .../storage/PrometheusMetricTableTest.java | 646 +++++++++++++++++- .../storage/PrometheusStorageEngineTest.java | 3 +- .../StepParameterResolverTest.java | 26 + .../prometheus/utils/LogicalPlanUtils.java | 77 +++ 41 files changed, 2181 insertions(+), 130 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricAgg.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricScan.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalPlanOptimizerFactory.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndIndexScan.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndRelation.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeFilterAndRelation.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/QueryRangeParameters.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/StepParameterResolver.java create mode 100644 prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicOptimizerTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/storage/querybuilders/StepParameterResolverTest.java create mode 100644 prometheus/src/test/java/org/opensearch/sql/prometheus/utils/LogicalPlanUtils.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 188de5aa64..7d0a452e1b 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -203,6 +203,12 @@ public LogicalPlan visitTableFunction(TableFunction node, AnalysisContext contex TableFunctionImplementation tableFunctionImplementation = (TableFunctionImplementation) repository.compile( catalogSchemaIdentifierNameResolver.getCatalogName(), functionName, arguments); + context.push(); + TypeEnvironment curEnv = context.peek(); + Table table = tableFunctionImplementation.applyArguments(); + table.getFieldTypes().forEach((k, v) -> curEnv.define(new Symbol(Namespace.FIELD_NAME, k), v)); + curEnv.define(new Symbol(Namespace.INDEX_NAME, + catalogSchemaIdentifierNameResolver.getIdentifierName()), STRUCT); return new LogicalRelation(catalogSchemaIdentifierNameResolver.getIdentifierName(), tableFunctionImplementation.applyArguments()); } diff --git a/docs/user/ppl/admin/prometheus_connector.rst b/docs/user/ppl/admin/prometheus_connector.rst index 53c862dfae..aced79cbdb 100644 --- a/docs/user/ppl/admin/prometheus_connector.rst +++ b/docs/user/ppl/admin/prometheus_connector.rst @@ -72,3 +72,116 @@ AWSSigV4 Auth:: } }] +PPL Query support for prometheus connector +========================================== + +Metric as a Table +--------------------------- +Each connector has to abstract the underlying datasource constructs into a table as part of the interface contract with the PPL query engine. +Prometheus connector abstracts each metric as a table and the columns of this table are ``@value``, ``@timestamp``, ``label1``, ``label2``---. +``@value`` represents metric measurement and ``@timestamp`` represents the timestamp at which the metric is collected. labels are tags associated with metric queried. +For eg: ``handler``, ``code``, ``instance``, ``code`` are the labels associated with ``prometheus_http_requests_total`` metric. With this abstraction, we can query prometheus +data using PPL syntax similar to opensearch indices. + +Sample Example:: + + > source = my_prometheus.prometheus_http_requests_total; + + +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + | @value | @timestamp | handler | code | instance | job | + |------------+------------------------+--------------------------------+---------------+-------------+-------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | + +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + + + +Default time range and resolution +--------------------------------- +Since time range and resolution are required parameters for query apis and these parameters are determined in the following manner from the PPL commands. +* Time range is determined through filter clause on ``@timestamp``. If there is no such filter clause, time range will be set to 1h with endtime set to now(). +* In case of stats, resolution is determined by ``span(@timestamp,15s)`` expression. For normal select queries, resolution is auto determined from the time range set. + +Prometheus Connector Limitations +-------------------------------- +* Only one aggregation is supported in stats command. +* Span Expression is compulsory in stats command. +* AVG, MAX, MIN, SUM, COUNT are the only aggregations supported in prometheus connector. + +Example queries +--------------- + +1. Metric Selection Query:: + + > source = my_prometheus.prometheus_http_requests_total + +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + | @value | @timestamp | handler | code | instance | job | + |------------+------------------------+--------------------------------+---------------+-------------+-------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" |"/-/metrics" | 500 | 192.15.2.1 | prometheus | + +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +2. Metric Selecting Query with specific dimensions:: + + > source = my_prometheus.prometheus_http_requests_total | where handler='/-/ready' and code='200' + +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + | @value | @timestamp | handler | code | instance | job | + |------------+------------------------+--------------------------------+---------------+-------------+-------------| + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | 192.15.1.1 | prometheus | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + | 9 | "2022-11-03 07:18:54" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + | 11 | "2022-11-03 07:18:64" | "/-/ready" | 200 | 192.15.2.1 | prometheus | + +------------+------------------------+--------------------------------+---------------+-------------+-------------+ + +3. Average aggregation on a metric:: + + > source = my_prometheus.prometheus_http_requests_total | stats avg(@value) by span(@timestamp,15s) + +------------+------------------------+ + | avg(@value)| span(@timestamp,15s) | + |------------+------------------------+ + | 5 | "2022-11-03 07:18:14" | + | 3 | "2022-11-03 07:18:24" | + | 7 | "2022-11-03 07:18:34" | + | 2 | "2022-11-03 07:18:44" | + | 9 | "2022-11-03 07:18:54" | + | 11 | "2022-11-03 07:18:64" | + +------------+------------------------+ + +4. Average aggregation grouped by dimensions:: + + > source = my_prometheus.prometheus_http_requests_total | stats avg(@value) by span(@timestamp,15s), handler, code + +------------+------------------------+--------------------------------+---------------+ + | avg(@value)| span(@timestamp,15s) | handler | code | + |------------+------------------------+--------------------------------+---------------+ + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | + +------------+------------------------+--------------------------------+---------------+ + +5. Count aggregation query:: + + > source = my_prometheus.prometheus_http_requests_total | stats count() by span(@timestamp,15s), handler, code + +------------+------------------------+--------------------------------+---------------+ + | count() | span(@timestamp,15s) | handler | code | + |------------+------------------------+--------------------------------+---------------+ + | 5 | "2022-11-03 07:18:14" | "/-/ready" | 200 | + | 3 | "2022-11-03 07:18:24" | "/-/ready" | 200 | + | 7 | "2022-11-03 07:18:34" | "/-/ready" | 200 | + | 2 | "2022-11-03 07:18:44" | "/-/ready" | 400 | + | 9 | "2022-11-03 07:18:54" | "/-/promql" | 400 | + | 11 | "2022-11-03 07:18:64" | "/-/metrics" | 500 | + +------------+------------------------+--------------------------------+---------------+ + diff --git a/docs/user/ppl/cmd/describe.rst b/docs/user/ppl/cmd/describe.rst index c683aa1781..12fcf35ded 100644 --- a/docs/user/ppl/cmd/describe.rst +++ b/docs/user/ppl/cmd/describe.rst @@ -75,11 +75,10 @@ The example retrieves table info for ``prometheus_http_requests_total`` metric i PPL query:: os> describe my_prometheus.prometheus_http_requests_total; - fetched rows / total rows = 7/7 + fetched rows / total rows = 6/6 +-----------------+----------------+--------------------------------+---------------+-------------+ | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | COLUMN_NAME | DATA_TYPE | |-----------------+----------------+--------------------------------+---------------+-------------| - | my_prometheus | default | prometheus_http_requests_total | @labels | keyword | | my_prometheus | default | prometheus_http_requests_total | handler | keyword | | my_prometheus | default | prometheus_http_requests_total | code | keyword | | my_prometheus | default | prometheus_http_requests_total | instance | keyword | diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java index 88ef028fbe..77fd910f35 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DescribeCommandIT.java @@ -101,7 +101,6 @@ public void testDescribeCommandWithPrometheusCatalog() throws IOException { columnName("DATA_TYPE") ); verifyDataRows(result, - rows("my_prometheus", "default", "prometheus_http_requests_total", "@labels", "keyword"), rows("my_prometheus", "default", "prometheus_http_requests_total", "handler", "keyword"), rows("my_prometheus", "default", "prometheus_http_requests_total", "code", "keyword"), rows("my_prometheus", "default", "prometheus_http_requests_total", "instance", "keyword"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java new file mode 100644 index 0000000000..9e197bbb27 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusCatalogCommandsIT.java @@ -0,0 +1,158 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +import lombok.SneakyThrows; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PrometheusCatalogCommandsIT extends PPLIntegTestCase { + + @Test + @SneakyThrows + public void testSourceMetricCommand() { + JSONObject response = + executeQuery("source=my_prometheus.prometheus_http_requests_total"); + verifySchema(response, + schema(VALUE, "double"), + schema(TIMESTAMP, "timestamp"), + schema("handler", "string"), + schema("code", "string"), + schema("instance", "string"), + schema("job", "string")); + Assertions.assertTrue(response.getInt("size") > 0); + Assertions.assertEquals(6, response.getJSONArray("datarows").getJSONArray(0).length()); + JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); + for (int i = 0; i < firstRow.length(); i++) { + Assertions.assertNotNull(firstRow.get(i)); + Assertions.assertTrue(StringUtils.isNotEmpty(firstRow.get(i).toString())); + } + } + + @Test + @SneakyThrows + public void testMetricAvgAggregationCommand() { + JSONObject response = + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) by span(@timestamp, 15s), handler, job"); + verifySchema(response, + schema("avg(@value)", "double"), + schema("span(@timestamp,15s)", "timestamp"), + schema("handler", "string"), + schema("job", "string")); + Assertions.assertTrue(response.getInt("size") > 0); + Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); + JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); + for (int i = 0; i < firstRow.length(); i++) { + Assertions.assertNotNull(firstRow.get(i)); + Assertions.assertTrue(StringUtils.isNotEmpty(firstRow.get(i).toString())); + } + } + + @Test + @SneakyThrows + public void testMetricAvgAggregationCommandWithAlias() { + JSONObject response = + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats avg(@value) as agg by span(@timestamp, 15s), handler, job"); + verifySchema(response, + schema("agg", "double"), + schema("span(@timestamp,15s)", "timestamp"), + schema("handler", "string"), + schema("job", "string")); + Assertions.assertTrue(response.getInt("size") > 0); + Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); + JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); + for (int i = 0; i < firstRow.length(); i++) { + Assertions.assertNotNull(firstRow.get(i)); + Assertions.assertTrue(StringUtils.isNotEmpty(firstRow.get(i).toString())); + } + } + + + @Test + @SneakyThrows + public void testMetricMaxAggregationCommand() { + JSONObject response = + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats max(@value) by span(@timestamp, 15s)"); + verifySchema(response, + schema("max(@value)", "double"), + schema("span(@timestamp,15s)", "timestamp")); + Assertions.assertTrue(response.getInt("size") > 0); + Assertions.assertEquals(2, response.getJSONArray("datarows").getJSONArray(0).length()); + JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); + for (int i = 0; i < firstRow.length(); i++) { + Assertions.assertNotNull(firstRow.get(i)); + Assertions.assertTrue(StringUtils.isNotEmpty(firstRow.get(i).toString())); + } + } + + + @Test + @SneakyThrows + public void testMetricMinAggregationCommand() { + JSONObject response = + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats min(@value) by span(@timestamp, 15s), handler"); + verifySchema(response, + schema("min(@value)", "double"), + schema("span(@timestamp,15s)", "timestamp"), + schema("handler", "string")); + Assertions.assertTrue(response.getInt("size") > 0); + Assertions.assertEquals(3, response.getJSONArray("datarows").getJSONArray(0).length()); + JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); + for (int i = 0; i < firstRow.length(); i++) { + Assertions.assertNotNull(firstRow.get(i)); + Assertions.assertTrue(StringUtils.isNotEmpty(firstRow.get(i).toString())); + } + } + + @Test + @SneakyThrows + public void testMetricCountAggregationCommand() { + JSONObject response = + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats count() by span(@timestamp, 15s), handler, job"); + verifySchema(response, + schema("count()", "integer"), + schema("span(@timestamp,15s)", "timestamp"), + schema("handler", "string"), + schema("job", "string")); + Assertions.assertTrue(response.getInt("size") > 0); + Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); + JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); + for (int i = 0; i < firstRow.length(); i++) { + Assertions.assertNotNull(firstRow.get(i)); + Assertions.assertTrue(StringUtils.isNotEmpty(firstRow.get(i).toString())); + } + } + + @Test + @SneakyThrows + public void testMetricSumAggregationCommand() { + JSONObject response = + executeQuery("source=my_prometheus.prometheus_http_requests_total | stats sum(@value) by span(@timestamp, 15s), handler, job"); + verifySchema(response, + schema("sum(@value)", "double"), + schema("span(@timestamp,15s)", "timestamp"), + schema("handler", "string"), + schema("job", "string")); + Assertions.assertTrue(response.getInt("size") > 0); + Assertions.assertEquals(4, response.getJSONArray("datarows").getJSONArray(0).length()); + JSONArray firstRow = response.getJSONArray("datarows").getJSONArray(0); + for (int i = 0; i < firstRow.length(); i++) { + Assertions.assertNotNull(firstRow.get(i)); + Assertions.assertTrue(StringUtils.isNotEmpty(firstRow.get(i).toString())); + } + } + +} diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java index 4f2f5e0f57..2e88a49f35 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/catalog/CatalogServiceImpl.java @@ -79,7 +79,7 @@ public void loadConnectors(Settings settings) { } catch (IOException e) { LOG.error("Catalog Configuration File uploaded is malformed. Verify and re-upload.", e); } catch (Throwable e) { - LOG.error("Catalog constructed failed.", e); + LOG.error("Catalog construction failed.", e); } } return null; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 99ff931f39..14ebbe717a 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -161,8 +161,6 @@ mlArg fromClause : SOURCE EQUAL tableSourceClause | INDEX EQUAL tableSourceClause - | SOURCE EQUAL tableFunction - | INDEX EQUAL tableFunction ; tableSourceClause diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 699d671bb2..2638fc9a42 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import lombok.Generated; import lombok.RequiredArgsConstructor; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; @@ -348,11 +349,7 @@ public UnresolvedPlan visitTopCommand(TopCommandContext ctx) { */ @Override public UnresolvedPlan visitFromClause(FromClauseContext ctx) { - if (ctx.tableFunction() != null) { - return visitTableFunction(ctx.tableFunction()); - } else { - return visitTableSourceClause(ctx.tableSourceClause()); - } + return visitTableSourceClause(ctx.tableSourceClause()); } @Override @@ -363,16 +360,10 @@ public UnresolvedPlan visitTableSourceClause(TableSourceClauseContext ctx) { } @Override + @Generated //To exclude from jacoco..will remove https://github.com/opensearch-project/sql/issues/1019 public UnresolvedPlan visitTableFunction(OpenSearchPPLParser.TableFunctionContext ctx) { - ImmutableList.Builder builder = ImmutableList.builder(); - ctx.functionArgs().functionArg().forEach(arg - -> { - String argName = (arg.ident() != null) ? arg.ident().getText() : null; - builder.add( - new UnresolvedArgument(argName, - this.internalVisitExpression(arg.valueExpression()))); - }); - return new TableFunction(this.internalVisitExpression(ctx.qualifiedName()), builder.build()); + // + return null; } /** diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 314d97009c..be81fcb06c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -11,6 +11,7 @@ import com.google.common.collect.ImmutableMap; import java.util.List; import java.util.stream.Collectors; +import lombok.Generated; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.ast.AbstractNodeVisitor; @@ -80,13 +81,10 @@ public String visitRelation(Relation node, String context) { } @Override + @Generated //To exclude from jacoco..will remove https://github.com/opensearch-project/sql/issues/1019 public String visitTableFunction(TableFunction node, String context) { - String arguments = - node.getArguments().stream() - .map(unresolvedExpression - -> this.expressionAnalyzer.analyze(unresolvedExpression, context)) - .collect(Collectors.joining(",")); - return StringUtils.format("source=%s(%s)", node.getFunctionName().toString(), arguments); + // + return null; } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 26e44b674d..658bf1d295 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -97,6 +97,7 @@ public void testSearchCommandWithDotInIndexName() { ); } + @Ignore @Test public void testSearchWithPrometheusQueryRangeWithPositionedArguments() { assertEqual("search source = prometheus.query_range(\"test{code='200'}\",1234, 12345, 3)", @@ -108,6 +109,7 @@ public void testSearchWithPrometheusQueryRangeWithPositionedArguments() { )); } + @Ignore @Test public void testSearchWithPrometheusQueryRangeWithNamedArguments() { assertEqual("search source = prometheus.query_range(query = \"test{code='200'}\", " diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 52f2f18b72..4a4c0a5f22 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableSet; import java.util.Collections; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -39,6 +40,7 @@ public void testSearchCommand() { } @Test + @Ignore public void testTableFunctionCommand() { assertEquals("source=prometheus.query_range(***,***,***,***)", anonymize("source=prometheus.query_range('afsd',123,123,3)") diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java index 7068b848ca..4a469c7bbb 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/client/PrometheusClientImpl.java @@ -55,15 +55,17 @@ public JSONObject queryRange(String query, Long start, Long end, String step) th @Override public List getLabels(String metricName) throws IOException { - String queryUrl = String.format("%s/api/v1/labels?match[]=%s", - uri.toString().replaceAll("/$", ""), metricName); + String queryUrl = String.format("%s/api/v1/labels?%s=%s", + uri.toString().replaceAll("/$", ""), + URLEncoder.encode("match[]", StandardCharsets.UTF_8), + URLEncoder.encode(metricName, StandardCharsets.UTF_8)); logger.debug("queryUrl: " + queryUrl); Request request = new Request.Builder() .url(queryUrl) .build(); Response response = this.okHttpClient.newCall(request).execute(); JSONObject jsonObject = readResponse(response); - return toListOfStrings(jsonObject.getJSONArray("data")); + return toListOfLabels(jsonObject.getJSONArray("data")); } @Override @@ -81,7 +83,7 @@ public Map> getAllMetrics() throws IOException { return new ObjectMapper().readValue(jsonObject.getJSONObject("data").toString(), typeRef); } - private List toListOfStrings(JSONArray array) { + private List toListOfLabels(JSONArray array) { List result = new ArrayList<>(); for (int i = 0; i < array.length(); i++) { //__name__ is internal label in prometheus representing the metric name. diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java index 8238a3a4e0..bccb9c3bff 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/functions/implementation/QueryRangeFunctionImplementation.java @@ -87,7 +87,7 @@ private PrometheusQueryRequest buildQueryFromQueryRangeFunction(List switch (argName) { case QUERY: prometheusQueryRequest - .getPromQl().append((String) literalValue.value()); + .setPromQl((String) literalValue.value()); break; case STARTTIME: prometheusQueryRequest.setStartTime(((Number) literalValue.value()).longValue()); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricAgg.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricAgg.java new file mode 100644 index 0000000000..f348c699a1 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricAgg.java @@ -0,0 +1,76 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.planner.logical; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.aggregation.NamedAggregator; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalPlanNodeVisitor; + + +/** + * Logical Metric Scan along with aggregation Operation. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +public class PrometheusLogicalMetricAgg extends LogicalPlan { + + private final String metricName; + + /** + * Filter Condition. + */ + @Setter + private Expression filter; + + /** + * Aggregation List. + */ + @Setter + private List aggregatorList; + + /** + * Group List. + */ + @Setter + private List groupByList; + + /** + * Constructor for LogicalMetricAgg Logical Plan. + * + * @param metricName metricName + * @param filter filter + * @param aggregatorList aggregatorList + * @param groupByList groupByList. + */ + @Builder + public PrometheusLogicalMetricAgg(String metricName, + Expression filter, + List aggregatorList, + List groupByList) { + super(ImmutableList.of()); + this.metricName = metricName; + this.filter = filter; + this.aggregatorList = aggregatorList; + this.groupByList = groupByList; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitNode(this, context); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricScan.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricScan.java new file mode 100644 index 0000000000..5e07d6899f --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalMetricScan.java @@ -0,0 +1,54 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.planner.logical; + +import com.google.common.collect.ImmutableList; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalPlanNodeVisitor; + +/** + * Prometheus Logical Metric Scan Operation. + * In an optimized plan this node represents both Relation and Filter Operation. + */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) +public class PrometheusLogicalMetricScan extends LogicalPlan { + + private final String metricName; + + /** + * Filter Condition. + */ + private final Expression filter; + + /** + * PrometheusLogicalMetricScan constructor. + * + * @param metricName metricName. + * @param filter filter. + */ + @Builder + public PrometheusLogicalMetricScan(String metricName, + Expression filter) { + super(ImmutableList.of()); + this.metricName = metricName; + this.filter = filter; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitNode(this, context); + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalPlanOptimizerFactory.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalPlanOptimizerFactory.java new file mode 100644 index 0000000000..8a365b2786 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicalPlanOptimizerFactory.java @@ -0,0 +1,32 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.prometheus.planner.logical; + + +import java.util.Arrays; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; +import org.opensearch.sql.prometheus.planner.logical.rules.MergeAggAndIndexScan; +import org.opensearch.sql.prometheus.planner.logical.rules.MergeAggAndRelation; +import org.opensearch.sql.prometheus.planner.logical.rules.MergeFilterAndRelation; + +/** + * Prometheus storage engine specified logical plan optimizer. + */ +@UtilityClass +public class PrometheusLogicalPlanOptimizerFactory { + + /** + * Create Prometheus storage specified logical plan optimizer. + */ + public static LogicalPlanOptimizer create() { + return new LogicalPlanOptimizer(Arrays.asList( + new MergeFilterAndRelation(), + new MergeAggAndIndexScan(), + new MergeAggAndRelation() + )); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndIndexScan.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndIndexScan.java new file mode 100644 index 0000000000..76bc6cc840 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndIndexScan.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.prometheus.planner.logical.rules; + +import static com.facebook.presto.matching.Pattern.typeOf; +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.source; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import lombok.Getter; +import lombok.experimental.Accessors; +import org.opensearch.sql.planner.logical.LogicalAggregation; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.optimizer.Rule; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricAgg; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricScan; + +/** + * Merge Aggregation -- Relation to MetricScanAggregation. + */ +public class MergeAggAndIndexScan implements Rule { + + private final Capture capture; + + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + /** + * Constructor of MergeAggAndIndexScan. + */ + public MergeAggAndIndexScan() { + this.capture = Capture.newCapture(); + this.pattern = typeOf(LogicalAggregation.class) + .with(source().matching(typeOf(PrometheusLogicalMetricScan.class) + .capturedAs(capture))); + } + + @Override + public LogicalPlan apply(LogicalAggregation aggregation, + Captures captures) { + PrometheusLogicalMetricScan indexScan = captures.get(capture); + return PrometheusLogicalMetricAgg + .builder() + .metricName(indexScan.getMetricName()) + .filter(indexScan.getFilter()) + .aggregatorList(aggregation.getAggregatorList()) + .groupByList(aggregation.getGroupByList()) + .build(); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndRelation.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndRelation.java new file mode 100644 index 0000000000..fa9b0c7206 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeAggAndRelation.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.prometheus.planner.logical.rules; + +import static com.facebook.presto.matching.Pattern.typeOf; +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.source; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import lombok.Getter; +import lombok.experimental.Accessors; +import org.opensearch.sql.planner.logical.LogicalAggregation; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.optimizer.Rule; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricAgg; + +/** + * Merge Aggregation -- Relation to IndexScanAggregation. + */ +public class MergeAggAndRelation implements Rule { + + private final Capture relationCapture; + + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + /** + * Constructor of MergeAggAndRelation. + */ + public MergeAggAndRelation() { + this.relationCapture = Capture.newCapture(); + this.pattern = typeOf(LogicalAggregation.class) + .with(source().matching(typeOf(LogicalRelation.class).capturedAs(relationCapture))); + } + + @Override + public LogicalPlan apply(LogicalAggregation aggregation, + Captures captures) { + LogicalRelation relation = captures.get(relationCapture); + return PrometheusLogicalMetricAgg + .builder() + .metricName(relation.getRelationName()) + .aggregatorList(aggregation.getAggregatorList()) + .groupByList(aggregation.getGroupByList()) + .build(); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeFilterAndRelation.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeFilterAndRelation.java new file mode 100644 index 0000000000..a99eb695be --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/planner/logical/rules/MergeFilterAndRelation.java @@ -0,0 +1,53 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.prometheus.planner.logical.rules; + +import static com.facebook.presto.matching.Pattern.typeOf; +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.source; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import org.opensearch.sql.planner.logical.LogicalFilter; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.optimizer.Rule; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricScan; + +/** + * Merge Filter -- Relation to LogicalMetricScan. + */ +public class MergeFilterAndRelation implements Rule { + + private final Capture relationCapture; + private final Pattern pattern; + + /** + * Constructor of MergeFilterAndRelation. + */ + public MergeFilterAndRelation() { + this.relationCapture = Capture.newCapture(); + this.pattern = typeOf(LogicalFilter.class) + .with(source().matching(typeOf(LogicalRelation.class).capturedAs(relationCapture))); + } + + @Override + public Pattern pattern() { + return pattern; + } + + @Override + public LogicalPlan apply(LogicalFilter filter, + Captures captures) { + LogicalRelation relation = captures.get(relationCapture); + return PrometheusLogicalMetricScan + .builder() + .metricName(relation.getRelationName()) + .filter(filter.getCondition()) + .build(); + } +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java index 5d4bf2ae7c..176a52a1d9 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/request/PrometheusQueryRequest.java @@ -7,8 +7,10 @@ package org.opensearch.sql.prometheus.request; import lombok.AllArgsConstructor; +import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; @@ -16,38 +18,30 @@ * Prometheus metric query request. */ @EqualsAndHashCode -@Getter +@Data @ToString @AllArgsConstructor +@NoArgsConstructor public class PrometheusQueryRequest { /** * PromQL. */ - private final StringBuilder promQl; + private String promQl; /** * startTime of the query. */ - @Setter private Long startTime; /** * endTime of the query. */ - @Setter private Long endTime; /** * step is the resolution required between startTime and endTime. */ - @Setter private String step; - /** - * Constructor of PrometheusQueryRequest. - */ - public PrometheusQueryRequest() { - this.promQl = new StringBuilder(); - } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java index 98a54a7731..e26e006403 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/response/PrometheusResponse.java @@ -5,9 +5,10 @@ package org.opensearch.sql.prometheus.response; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; import java.time.Instant; import java.util.ArrayList; @@ -18,17 +19,37 @@ import org.json.JSONArray; import org.json.JSONObject; import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; public class PrometheusResponse implements Iterable { private final JSONObject responseObject; - public PrometheusResponse(JSONObject responseObject) { + private final PrometheusResponseFieldNames prometheusResponseFieldNames; + + private final Boolean isQueryRangeFunctionScan; + + /** + * Constructor. + * + * @param responseObject Prometheus responseObject. + * @param prometheusResponseFieldNames data model which + * contains field names for the metric measurement + * and timestamp fieldName. + */ + public PrometheusResponse(JSONObject responseObject, + PrometheusResponseFieldNames prometheusResponseFieldNames, + Boolean isQueryRangeFunctionScan) { this.responseObject = responseObject; + this.prometheusResponseFieldNames = prometheusResponseFieldNames; + this.isQueryRangeFunctionScan = isQueryRangeFunctionScan; } @NonNull @@ -44,10 +65,28 @@ public Iterator iterator() { for (int j = 0; j < values.length(); j++) { LinkedHashMap linkedHashMap = new LinkedHashMap<>(); JSONArray val = values.getJSONArray(j); - linkedHashMap.put(TIMESTAMP, + linkedHashMap.put(prometheusResponseFieldNames.getTimestampFieldName(), new ExprTimestampValue(Instant.ofEpochMilli((long) (val.getDouble(0) * 1000)))); - linkedHashMap.put(VALUE, new ExprDoubleValue(val.getDouble(1))); - linkedHashMap.put(LABELS, new ExprStringValue(metric.toString())); + linkedHashMap.put(prometheusResponseFieldNames.getValueFieldName(), getValue(val, 1, + prometheusResponseFieldNames.getValueType())); + // Concept: + // {\"instance\":\"localhost:9090\",\"__name__\":\"up\",\"job\":\"prometheus\"}" + // This is the label string in the prometheus response. + // Q: how do we map this to columns in a table. + // For queries like source = prometheus.metric_name | .... + // we can get the labels list in prior as we know which metric we are working on. + // In case of commands like source = prometheus.query_range('promQL'); + // Any arbitrary command can be written and we don't know the labels + // in the prometheus response in prior. + // So for PPL like commands...output structure is @value, @timestamp + // and each label is treated as a separate column where as in case of query_range + // function irrespective of promQL, the output structure is + // @value, @timestamp, @labels [jsonfied string of all the labels for a data point] + if (isQueryRangeFunctionScan) { + linkedHashMap.put(LABELS, new ExprStringValue(metric.toString())); + } else { + insertLabels(linkedHashMap, metric); + } result.add(new ExprTupleValue(linkedHashMap)); } } @@ -58,4 +97,20 @@ public Iterator iterator() { } return result.iterator(); } + + private void insertLabels(LinkedHashMap linkedHashMap, JSONObject metric) { + for (String key : metric.keySet()) { + linkedHashMap.put(key, new ExprStringValue(metric.getString(key))); + } + } + + private ExprValue getValue(JSONArray jsonArray, Integer index, ExprType exprType) { + if (INTEGER.equals(exprType)) { + return new ExprIntegerValue(jsonArray.getInt(index)); + } else if (LONG.equals(exprType)) { + return new ExprLongValue(jsonArray.getLong(index)); + } + return new ExprDoubleValue(jsonArray.getDouble(index)); + } + } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java index 5fc7edfc73..790189d903 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricDefaultSchema.java @@ -7,7 +7,6 @@ package org.opensearch.sql.prometheus.storage; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; @@ -25,10 +24,8 @@ public enum PrometheusMetricDefaultSchema { DEFAULT_MAPPING(new ImmutableMap.Builder() .put(TIMESTAMP, ExprCoreType.TIMESTAMP) .put(VALUE, ExprCoreType.DOUBLE) - .put(LABELS, ExprCoreType.STRING) .build()); private final Map mapping; - } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java index d8ab97709b..8611ae04f1 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScan.java @@ -20,6 +20,7 @@ import org.opensearch.sql.prometheus.client.PrometheusClient; import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; import org.opensearch.sql.prometheus.response.PrometheusResponse; +import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; import org.opensearch.sql.storage.TableScanOperator; /** @@ -39,11 +40,25 @@ public class PrometheusMetricScan extends TableScanOperator { private Iterator iterator; + @Setter + @Getter + private Boolean isQueryRangeFunctionScan = Boolean.FALSE; + + @Setter + private PrometheusResponseFieldNames prometheusResponseFieldNames; + + private static final Logger LOG = LogManager.getLogger(); + /** + * Constructor. + * + * @param prometheusClient prometheusClient. + */ public PrometheusMetricScan(PrometheusClient prometheusClient) { this.prometheusClient = prometheusClient; this.request = new PrometheusQueryRequest(); + this.prometheusResponseFieldNames = new PrometheusResponseFieldNames(); } @Override @@ -52,9 +67,10 @@ public void open() { this.iterator = AccessController.doPrivileged((PrivilegedAction>) () -> { try { JSONObject responseObject = prometheusClient.queryRange( - request.getPromQl().toString(), + request.getPromQl(), request.getStartTime(), request.getEndTime(), request.getStep()); - return new PrometheusResponse(responseObject).iterator(); + return new PrometheusResponse(responseObject, prometheusResponseFieldNames, + isQueryRangeFunctionScan).iterator(); } catch (IOException e) { LOG.error(e.getMessage()); throw new RuntimeException("Error fetching data from prometheus server. " + e.getMessage()); diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java index 3d8ef69b17..b81314d936 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTable.java @@ -6,14 +6,18 @@ package org.opensearch.sql.prometheus.storage; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; + +import java.util.HashMap; import java.util.Map; -import java.util.Optional; import javax.annotation.Nonnull; import lombok.Getter; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalPlanOptimizerFactory; import org.opensearch.sql.prometheus.request.PrometheusQueryRequest; import org.opensearch.sql.prometheus.request.system.PrometheusDescribeMetricRequest; import org.opensearch.sql.prometheus.storage.implementor.PrometheusDefaultImplementor; @@ -29,10 +33,10 @@ public class PrometheusMetricTable implements Table { private final PrometheusClient prometheusClient; @Getter - private final Optional metricName; + private final String metricName; @Getter - private final Optional prometheusQueryRequest; + private final PrometheusQueryRequest prometheusQueryRequest; /** @@ -45,8 +49,8 @@ public class PrometheusMetricTable implements Table { */ public PrometheusMetricTable(PrometheusClient prometheusService, @Nonnull String metricName) { this.prometheusClient = prometheusService; - this.metricName = Optional.of(metricName); - this.prometheusQueryRequest = Optional.empty(); + this.metricName = metricName; + this.prometheusQueryRequest = null; } /** @@ -55,19 +59,21 @@ public PrometheusMetricTable(PrometheusClient prometheusService, @Nonnull String public PrometheusMetricTable(PrometheusClient prometheusService, @Nonnull PrometheusQueryRequest prometheusQueryRequest) { this.prometheusClient = prometheusService; - this.metricName = Optional.empty(); - this.prometheusQueryRequest = Optional.of(prometheusQueryRequest); + this.metricName = null; + this.prometheusQueryRequest = prometheusQueryRequest; } @Override public Map getFieldTypes() { if (cachedFieldTypes == null) { - if (metricName.isPresent()) { + if (metricName != null) { cachedFieldTypes = new PrometheusDescribeMetricRequest(prometheusClient, null, - metricName.orElse(null)).getFieldTypes(); + metricName).getFieldTypes(); } else { - cachedFieldTypes = PrometheusMetricDefaultSchema.DEFAULT_MAPPING.getMapping(); + cachedFieldTypes = new HashMap<>(PrometheusMetricDefaultSchema.DEFAULT_MAPPING + .getMapping()); + cachedFieldTypes.put(LABELS, ExprCoreType.STRING); } } return cachedFieldTypes; @@ -77,13 +83,16 @@ public Map getFieldTypes() { public PhysicalPlan implement(LogicalPlan plan) { PrometheusMetricScan metricScan = new PrometheusMetricScan(prometheusClient); - prometheusQueryRequest.ifPresent(metricScan::setRequest); + if (prometheusQueryRequest != null) { + metricScan.setRequest(prometheusQueryRequest); + metricScan.setIsQueryRangeFunctionScan(Boolean.TRUE); + } return plan.accept(new PrometheusDefaultImplementor(), metricScan); } @Override public LogicalPlan optimize(LogicalPlan plan) { - return plan; + return PrometheusLogicalPlanOptimizerFactory.create().optimize(plan); } } \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java index f18ff1bba8..f8ae0936ee 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngine.java @@ -45,7 +45,7 @@ public Table getTable(CatalogSchemaName catalogSchemaName, String tableName) { } else if (INFORMATION_SCHEMA_NAME.equals(catalogSchemaName.getSchemaName())) { return resolveInformationSchemaTable(catalogSchemaName, tableName); } else { - return null; + return new PrometheusMetricTable(prometheusClient, tableName); } } diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java index 6235ecc14b..071cd7ba8c 100644 --- a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/implementor/PrometheusDefaultImplementor.java @@ -7,22 +7,34 @@ package org.opensearch.sql.prometheus.storage.implementor; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import lombok.RequiredArgsConstructor; -import org.opensearch.sql.data.type.ExprCoreType; +import org.apache.commons.math3.util.Pair; +import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.NamedExpression; import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.expression.span.SpanExpression; import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.ProjectOperator; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricAgg; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricScan; import org.opensearch.sql.prometheus.storage.PrometheusMetricScan; +import org.opensearch.sql.prometheus.storage.PrometheusMetricTable; +import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; +import org.opensearch.sql.prometheus.storage.querybuilder.AggregationQueryBuilder; +import org.opensearch.sql.prometheus.storage.querybuilder.SeriesSelectionQueryBuilder; +import org.opensearch.sql.prometheus.storage.querybuilder.StepParameterResolver; +import org.opensearch.sql.prometheus.storage.querybuilder.TimeRangeParametersResolver; /** * Default Implementor of Logical plan for prometheus. @@ -31,26 +43,104 @@ public class PrometheusDefaultImplementor extends DefaultImplementor { + + @Override + public PhysicalPlan visitNode(LogicalPlan plan, PrometheusMetricScan context) { + if (plan instanceof PrometheusLogicalMetricScan) { + return visitIndexScan((PrometheusLogicalMetricScan) plan, context); + } else if (plan instanceof PrometheusLogicalMetricAgg) { + return visitIndexAggregation((PrometheusLogicalMetricAgg) plan, context); + } else { + throw new IllegalStateException(StringUtils.format("unexpected plan node type %s", + plan.getClass())); + } + } + + /** + * Implement PrometheusLogicalMetricScan. + */ + public PhysicalPlan visitIndexScan(PrometheusLogicalMetricScan node, + PrometheusMetricScan context) { + String query = SeriesSelectionQueryBuilder.build(node.getMetricName(), node.getFilter()); + + context.getRequest().setPromQl(query); + setTimeRangeParameters(node.getFilter(), context); + context.getRequest() + .setStep(StepParameterResolver.resolve(context.getRequest().getStartTime(), + context.getRequest().getEndTime(), null)); + return context; + } + + /** + * Implement PrometheusLogicalMetricAgg. + */ + public PhysicalPlan visitIndexAggregation(PrometheusLogicalMetricAgg node, + PrometheusMetricScan context) { + setTimeRangeParameters(node.getFilter(), context); + context.getRequest() + .setStep(StepParameterResolver.resolve(context.getRequest().getStartTime(), + context.getRequest().getEndTime(), node.getGroupByList())); + String step = context.getRequest().getStep(); + String seriesSelectionQuery + = SeriesSelectionQueryBuilder.build(node.getMetricName(), node.getFilter()); + + String aggregateQuery + = AggregationQueryBuilder.build(node.getAggregatorList(), + node.getGroupByList()); + + String finalQuery = String.format(aggregateQuery, seriesSelectionQuery + "[" + step + "]"); + context.getRequest().setPromQl(finalQuery); + + //Since prometheus response doesn't have any fieldNames in its output. + //the field names are sent to PrometheusResponse constructor via context. + setPrometheusResponseFieldNames(node, context); + return context; + } + @Override public PhysicalPlan visitRelation(LogicalRelation node, PrometheusMetricScan context) { + PrometheusMetricTable prometheusMetricTable = (PrometheusMetricTable) node.getTable(); + if (prometheusMetricTable.getMetricName() != null) { + String query = SeriesSelectionQueryBuilder.build(node.getRelationName(), null); + context.getRequest().setPromQl(query); + setTimeRangeParameters(null, context); + context.getRequest() + .setStep(StepParameterResolver.resolve(context.getRequest().getStartTime(), + context.getRequest().getEndTime(), null)); + } return context; } - // Since getFieldTypes include labels - // we are explicitly specifying the output column names; - @Override - public PhysicalPlan visitProject(LogicalProject node, PrometheusMetricScan context) { - List finalProjectList = new ArrayList<>(); - finalProjectList.add( - new NamedExpression(LABELS, new ReferenceExpression(LABELS, ExprCoreType.STRING))); - finalProjectList.add( - new NamedExpression(TIMESTAMP, - new ReferenceExpression(TIMESTAMP, ExprCoreType.TIMESTAMP))); - finalProjectList.add( - new NamedExpression(VALUE, new ReferenceExpression(VALUE, ExprCoreType.DOUBLE))); - return new ProjectOperator(visitChild(node, context), finalProjectList, - node.getNamedParseExpressions()); + private void setTimeRangeParameters(Expression filter, PrometheusMetricScan context) { + TimeRangeParametersResolver timeRangeParametersResolver = new TimeRangeParametersResolver(); + Pair timeRange = timeRangeParametersResolver.resolve(filter); + context.getRequest().setStartTime(timeRange.getFirst()); + context.getRequest().setEndTime(timeRange.getSecond()); } + private void setPrometheusResponseFieldNames(PrometheusLogicalMetricAgg node, + PrometheusMetricScan context) { + Optional spanExpression = getSpanExpression(node.getGroupByList()); + if (spanExpression.isEmpty()) { + throw new RuntimeException( + "Prometheus Catalog doesn't support aggregations without span expression"); + } + PrometheusResponseFieldNames prometheusResponseFieldNames = new PrometheusResponseFieldNames(); + prometheusResponseFieldNames.setValueFieldName(node.getAggregatorList().get(0).getName()); + prometheusResponseFieldNames.setValueType(node.getAggregatorList().get(0).type()); + prometheusResponseFieldNames.setTimestampFieldName(spanExpression.get().getNameOrAlias()); + context.setPrometheusResponseFieldNames(prometheusResponseFieldNames); + } + + private Optional getSpanExpression(List namedExpressionList) { + if (namedExpressionList == null) { + return Optional.empty(); + } + return namedExpressionList.stream() + .filter(expression -> expression.getDelegated() instanceof SpanExpression) + .findFirst(); + } + + } \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java new file mode 100644 index 0000000000..4276848aa2 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/PrometheusResponseFieldNames.java @@ -0,0 +1,27 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.model; + +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; + +import lombok.Getter; +import lombok.Setter; +import org.opensearch.sql.data.type.ExprType; + + +@Getter +@Setter +public class PrometheusResponseFieldNames { + + private String valueFieldName = VALUE; + private ExprType valueType = DOUBLE; + private String timestampFieldName = TIMESTAMP; + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/QueryRangeParameters.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/QueryRangeParameters.java new file mode 100644 index 0000000000..86ca99cea8 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/model/QueryRangeParameters.java @@ -0,0 +1,25 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class QueryRangeParameters { + + private Long start; + private Long end; + private String step; + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java new file mode 100644 index 0000000000..1aff9eca88 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/AggregationQueryBuilder.java @@ -0,0 +1,78 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.querybuilder; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import lombok.NoArgsConstructor; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.aggregation.NamedAggregator; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.span.SpanExpression; + +/** + * This class builds aggregation query for the given stats commands. + * In the generated query a placeholder(%s) is added in place of metric selection query + * and later replaced by metric selection query. + */ +@NoArgsConstructor +public class AggregationQueryBuilder { + + private static final Set allowedStatsFunctions = Set.of( + BuiltinFunctionName.MAX.getName().getFunctionName(), + BuiltinFunctionName.MIN.getName().getFunctionName(), + BuiltinFunctionName.COUNT.getName().getFunctionName(), + BuiltinFunctionName.SUM.getName().getFunctionName(), + BuiltinFunctionName.AVG.getName().getFunctionName() + ); + + + /** + * Build Aggregation query from series selector query from expression. + * + * @return query string. + */ + public static String build(List namedAggregatorList, + List groupByList) { + + if (namedAggregatorList.size() > 1) { + throw new RuntimeException( + "Prometheus Catalog doesn't multiple aggregations in stats command"); + } + + if (!allowedStatsFunctions + .contains(namedAggregatorList.get(0).getFunctionName().getFunctionName())) { + throw new RuntimeException(String.format( + "Prometheus Catalog only supports %s aggregations.", allowedStatsFunctions)); + } + + StringBuilder aggregateQuery = new StringBuilder(); + aggregateQuery.append(namedAggregatorList.get(0).getFunctionName().getFunctionName()) + .append(" "); + + if (groupByList != null && !groupByList.isEmpty()) { + groupByList = groupByList.stream() + .filter(expression -> !(expression.getDelegated() instanceof SpanExpression)) + .collect(Collectors.toList()); + if (groupByList.size() > 0) { + aggregateQuery.append("by("); + aggregateQuery.append( + groupByList.stream().map(NamedExpression::getName).collect(Collectors.joining(", "))); + aggregateQuery.append(")"); + } + } + aggregateQuery + .append(" (") + .append(namedAggregatorList.get(0).getFunctionName().getFunctionName()) + .append("_over_time") + .append("(%s))"); + return aggregateQuery.toString(); + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java new file mode 100644 index 0000000000..baa235aa89 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/SeriesSelectionQueryBuilder.java @@ -0,0 +1,70 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.querybuilder; + + +import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; + +import java.util.stream.Collectors; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionNodeVisitor; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.ReferenceExpression; + +/** + * This class builds metric selection query from the filter condition + * and metric name. + */ +@NoArgsConstructor +public class SeriesSelectionQueryBuilder { + + + /** + * Build Prometheus series selector query from expression. + * + * @param filterCondition expression. + * @return query string + */ + public static String build(String metricName, Expression filterCondition) { + if (filterCondition != null) { + SeriesSelectionExpressionNodeVisitor seriesSelectionExpressionNodeVisitor + = new SeriesSelectionExpressionNodeVisitor(); + String selectorQuery = filterCondition.accept(seriesSelectionExpressionNodeVisitor, null); + return metricName + "{" + selectorQuery + "}"; + } + return metricName; + } + + static class SeriesSelectionExpressionNodeVisitor extends ExpressionNodeVisitor { + @Override + public String visitFunction(FunctionExpression func, Object context) { + if (func.getFunctionName().getFunctionName().equals("and")) { + return func.getArguments().stream() + .map(arg -> visitFunction((FunctionExpression) arg, context)) + .filter(StringUtils::isNotEmpty) + .collect(Collectors.joining(" , ")); + } else if (func.getFunctionName().getFunctionName().contains("=")) { + ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); + if (!ref.getAttr().equals(TIMESTAMP)) { + return func.getArguments().get(0) + + func.getFunctionName().getFunctionName() + + func.getArguments().get(1); + } else { + return null; + } + } else { + throw new RuntimeException( + String.format("Prometheus Catalog doesn't support %s in where command.", + func.getFunctionName().getFunctionName())); + } + } + } + +} diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/StepParameterResolver.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/StepParameterResolver.java new file mode 100644 index 0000000000..54315bb792 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/StepParameterResolver.java @@ -0,0 +1,63 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.querybuilder; + +import java.util.List; +import java.util.Optional; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import org.apache.commons.lang3.StringUtils; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.span.SpanExpression; + +/** + * This class resolves step parameter required for + * query_range api of prometheus. + */ +@NoArgsConstructor +public class StepParameterResolver { + + /** + * Extract step from groupByList or apply heuristic arithmetic + * on endTime and startTime. + * + * + * @param startTime startTime. + * @param endTime endTime. + * @param groupByList groupByList. + * @return Step String. + */ + public static String resolve(@NonNull Long startTime, @NonNull Long endTime, + List groupByList) { + Optional spanExpression = getSpanExpression(groupByList); + if (spanExpression.isPresent()) { + if (StringUtils.isEmpty(spanExpression.get().getUnit().getName())) { + throw new RuntimeException("Missing TimeUnit in the span expression"); + } else { + return spanExpression.get().getValue().toString() + + spanExpression.get().getUnit().getName(); + } + } else { + return Math.max((endTime - startTime) / 250, 1) + "s"; + } + } + + private static Optional getSpanExpression( + List namedExpressionList) { + if (namedExpressionList == null) { + return Optional.empty(); + } + return namedExpressionList.stream() + .filter(expression -> expression.getDelegated() instanceof SpanExpression) + .map(expression -> (SpanExpression) expression.getDelegated()) + .findFirst(); + } + + + +} \ No newline at end of file diff --git a/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java new file mode 100644 index 0000000000..6c338d61a6 --- /dev/null +++ b/prometheus/src/main/java/org/opensearch/sql/prometheus/storage/querybuilder/TimeRangeParametersResolver.java @@ -0,0 +1,76 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.querybuilder; + +import java.util.Date; +import lombok.NoArgsConstructor; +import org.apache.commons.math3.util.Pair; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionNodeVisitor; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.ReferenceExpression; + +@NoArgsConstructor +public class TimeRangeParametersResolver extends ExpressionNodeVisitor { + + + private Long startTime; + private Long endTime; + + /** + * Build Range Query Parameters from filter expression. + * If the filter condition consists of @timestamp, startTime and + * endTime are derived. or else it will be defaulted to now() and now()-1hr. + * If one of starttime and endtime are provided, the other will be derived from them + * by fixing the time range duration to 1hr. + * + * @param filterCondition expression. + * @return query string + */ + public Pair resolve(Expression filterCondition) { + if (filterCondition == null) { + long endTime = new Date().getTime() / 1000; + return Pair.create(endTime - 3600, endTime); + } + filterCondition.accept(this, null); + if (startTime == null && endTime == null) { + long endTime = new Date().getTime() / 1000; + return Pair.create(endTime - 3600, endTime); + } else if (startTime == null) { + return Pair.create(endTime - 3600, endTime); + } else if (endTime == null) { + return Pair.create(startTime, startTime + 3600); + } else { + return Pair.create(startTime, endTime); + } + } + + @Override + public Void visitFunction(FunctionExpression func, Object context) { + if (func.getFunctionName().getFunctionName().contains("=")) { + ReferenceExpression ref = (ReferenceExpression) func.getArguments().get(0); + Expression rightExpr = func.getArguments().get(1); + if (ref.getAttr().equals("@timestamp")) { + ExprValue literalValue = rightExpr.valueOf(null); + if (func.getFunctionName().getFunctionName().contains(">")) { + startTime = literalValue.timestampValue().toEpochMilli() / 1000; + } + if (func.getFunctionName().getFunctionName().contains("<")) { + endTime = literalValue.timestampValue().toEpochMilli() / 1000; + } + } + } else { + func.getArguments() + .forEach(arg -> visitFunction((FunctionExpression) arg, context)); + } + return null; + } + + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java index b20ee6f7d6..f2a54b7347 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeFunctionImplementationTest.java @@ -9,6 +9,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -66,10 +68,10 @@ void testApplyArguments() { = new QueryRangeFunctionImplementation(functionName, namedArgumentExpressionList, client); PrometheusMetricTable prometheusMetricTable = (PrometheusMetricTable) queryRangeFunctionImplementation.applyArguments(); - assertFalse(prometheusMetricTable.getMetricName().isPresent()); - assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + assertNull(prometheusMetricTable.getMetricName()); + assertNotNull(prometheusMetricTable.getPrometheusQueryRequest()); PrometheusQueryRequest prometheusQueryRequest - = prometheusMetricTable.getPrometheusQueryRequest().get(); + = prometheusMetricTable.getPrometheusQueryRequest(); assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); assertEquals(12345, prometheusQueryRequest.getStartTime()); assertEquals(1234, prometheusQueryRequest.getEndTime()); diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java index 9cc6231eb3..caca48f834 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/functions/QueryRangeTableFunctionResolverTest.java @@ -8,6 +8,7 @@ package org.opensearch.sql.prometheus.functions; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.type.ExprCoreType.LONG; @@ -62,11 +63,10 @@ void testResolve() { assertTrue(functionImplementation instanceof QueryRangeFunctionImplementation); PrometheusMetricTable prometheusMetricTable = (PrometheusMetricTable) functionImplementation.applyArguments(); - assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + assertNotNull(prometheusMetricTable.getPrometheusQueryRequest()); PrometheusQueryRequest prometheusQueryRequest = - prometheusMetricTable.getPrometheusQueryRequest() - .get(); - assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); + prometheusMetricTable.getPrometheusQueryRequest(); + assertEquals("http_latency", prometheusQueryRequest.getPromQl()); assertEquals(12345L, prometheusQueryRequest.getStartTime()); assertEquals(12345L, prometheusQueryRequest.getEndTime()); assertEquals("14", prometheusQueryRequest.getStep()); @@ -97,11 +97,10 @@ void testArgumentsPassedByPosition() { assertTrue(functionImplementation instanceof QueryRangeFunctionImplementation); PrometheusMetricTable prometheusMetricTable = (PrometheusMetricTable) functionImplementation.applyArguments(); - assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + assertNotNull(prometheusMetricTable.getPrometheusQueryRequest()); PrometheusQueryRequest prometheusQueryRequest = - prometheusMetricTable.getPrometheusQueryRequest() - .get(); - assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); + prometheusMetricTable.getPrometheusQueryRequest(); + assertEquals("http_latency", prometheusQueryRequest.getPromQl()); assertEquals(12345L, prometheusQueryRequest.getStartTime()); assertEquals(12345L, prometheusQueryRequest.getEndTime()); assertEquals("14", prometheusQueryRequest.getStep()); @@ -133,11 +132,10 @@ void testArgumentsPassedByNameWithDifferentOrder() { assertTrue(functionImplementation instanceof QueryRangeFunctionImplementation); PrometheusMetricTable prometheusMetricTable = (PrometheusMetricTable) functionImplementation.applyArguments(); - assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); + assertNotNull(prometheusMetricTable.getPrometheusQueryRequest()); PrometheusQueryRequest prometheusQueryRequest = - prometheusMetricTable.getPrometheusQueryRequest() - .get(); - assertEquals("http_latency", prometheusQueryRequest.getPromQl().toString()); + prometheusMetricTable.getPrometheusQueryRequest(); + assertEquals("http_latency", prometheusQueryRequest.getPromQl()); assertEquals(12345L, prometheusQueryRequest.getStartTime()); assertEquals(12345L, prometheusQueryRequest.getEndTime()); assertEquals("14", prometheusQueryRequest.getStep()); diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicOptimizerTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicOptimizerTest.java new file mode 100644 index 0000000000..7d6d3bed28 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/planner/logical/PrometheusLogicOptimizerTest.java @@ -0,0 +1,121 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.planner.logical; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.aggregation; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.filter; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; +import static org.opensearch.sql.prometheus.utils.LogicalPlanUtils.indexScan; +import static org.opensearch.sql.prometheus.utils.LogicalPlanUtils.indexScanAgg; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.config.ExpressionConfig; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.optimizer.LogicalPlanOptimizer; +import org.opensearch.sql.storage.Table; + +@ExtendWith(MockitoExtension.class) +public class PrometheusLogicOptimizerTest { + + private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + + @Mock + private Table table; + + @Test + void project_filter_merge_with_relation() { + assertEquals( + project( + indexScan("prometheus_http_total_requests", + dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200")))) + ), + optimize( + project( + filter( + relation("prometheus_http_total_requests", table), + dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))) + )) + ) + ); + } + + @Test + void aggregation_merge_relation() { + assertEquals( + project( + indexScanAgg("prometheus_http_total_requests", ImmutableList + .of(DSL.named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(DSL.named("code", DSL.ref("code", STRING)))), + DSL.named("AVG(intV)", DSL.ref("AVG(intV)", DOUBLE))), + optimize( + project( + aggregation( + relation("prometheus_http_total_requests", table), + ImmutableList + .of(DSL.named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(DSL.named("code", + DSL.ref("code", STRING)))), + DSL.named("AVG(intV)", DSL.ref("AVG(intV)", DOUBLE))) + ) + ); + } + + + @Test + void aggregation_merge_filter_relation() { + assertEquals( + project( + indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))), + ImmutableList + .of(DSL.named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(DSL.named("job", DSL.ref("job", STRING)))), + DSL.named("AVG(@value)", DSL.ref("AVG(@value)", DOUBLE))), + optimize( + project( + aggregation( + filter( + relation("prometheus_http_total_requests", table), + dsl.and( + dsl.equal(DSL.ref("code", STRING), + DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), + DSL.literal(stringValue("/ready/")))) + ), + ImmutableList + .of(DSL.named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(DSL.named("job", + DSL.ref("job", STRING)))), + DSL.named("AVG(@value)", DSL.ref("AVG(@value)", DOUBLE))) + ) + ); + } + + + private LogicalPlan optimize(LogicalPlan plan) { + final LogicalPlanOptimizer optimizer = PrometheusLogicalPlanOptimizerFactory.create(); + return optimizer.optimize(plan); + } + +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java index fdd9ce6392..a190abb6a1 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/request/PrometheusDescribeMetricRequestTest.java @@ -9,11 +9,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; import static org.opensearch.sql.prometheus.constants.TestConstants.METRIC_NAME; -import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; @@ -46,18 +44,15 @@ public class PrometheusDescribeMetricRequestTest { @SneakyThrows void testGetFieldTypes() { when(prometheusClient.getLabels(METRIC_NAME)).thenReturn(new ArrayList() {{ - add("__name__"); add("call"); add("code"); } }); Map expected = new HashMap<>() {{ - put("__name__", ExprCoreType.STRING); put("call", ExprCoreType.STRING); put("code", ExprCoreType.STRING); put(VALUE, ExprCoreType.DOUBLE); put(TIMESTAMP, ExprCoreType.TIMESTAMP); - put(LABELS, ExprCoreType.STRING); }}; PrometheusDescribeMetricRequest prometheusDescribeMetricRequest = new PrometheusDescribeMetricRequest(prometheusClient, @@ -73,7 +68,6 @@ void testGetFieldTypesWithEmptyMetricName() { Map expected = new HashMap<>() {{ put(VALUE, ExprCoreType.DOUBLE); put(TIMESTAMP, ExprCoreType.TIMESTAMP); - put(LABELS, ExprCoreType.STRING); }}; assertThrows(NullPointerException.class, () -> new PrometheusDescribeMetricRequest(prometheusClient, @@ -112,15 +106,16 @@ void testGetFieldTypesWhenIOException() { @Test @SneakyThrows void testSearch() { - when(prometheusClient.getLabels(METRIC_NAME)).thenReturn(new ArrayList() {{ + when(prometheusClient.getLabels(METRIC_NAME)).thenReturn(new ArrayList<>() { + { add("call"); } - }); + }); PrometheusDescribeMetricRequest prometheusDescribeMetricRequest = new PrometheusDescribeMetricRequest(prometheusClient, new CatalogSchemaName("test", "default"), METRIC_NAME); List result = prometheusDescribeMetricRequest.search(); - assertEquals(4, result.size()); + assertEquals(3, result.size()); assertEquals(expectedRow(), result.get(0)); verify(prometheusClient, times(1)).getLabels(METRIC_NAME); } diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java index 16a2a7be0b..ac99a996af 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricScanTest.java @@ -9,6 +9,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.prometheus.constants.TestConstants.ENDTIME; import static org.opensearch.sql.prometheus.constants.TestConstants.QUERY; import static org.opensearch.sql.prometheus.constants.TestConstants.STARTTIME; @@ -29,10 +31,13 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.model.ExprDoubleValue; +import org.opensearch.sql.data.model.ExprIntegerValue; +import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.prometheus.client.PrometheusClient; +import org.opensearch.sql.prometheus.storage.model.PrometheusResponseFieldNames; @ExtendWith(MockitoExtension.class) public class PrometheusMetricScanTest { @@ -44,7 +49,7 @@ public class PrometheusMetricScanTest { @SneakyThrows void testQueryResponseIterator() { PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); - prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setPromQl(QUERY); prometheusMetricScan.getRequest().setStartTime(STARTTIME); prometheusMetricScan.getRequest().setEndTime(ENDTIME); prometheusMetricScan.getRequest().setStep(STEP); @@ -56,6 +61,125 @@ void testQueryResponseIterator() { ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); put(VALUE, new ExprDoubleValue(1)); + put("instance", new ExprStringValue("localhost:9090")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("prometheus")); + } + }); + assertEquals(firstRow, prometheusMetricScan.next()); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put("@timestamp", new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("@value", new ExprDoubleValue(0)); + put("instance", new ExprStringValue("localhost:9091")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("node")); + } + }); + assertEquals(secondRow, prometheusMetricScan.next()); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + + @Test + @SneakyThrows + void testQueryResponseIteratorWithGivenPrometheusResponseFieldNames() { + PrometheusResponseFieldNames prometheusResponseFieldNames + = new PrometheusResponseFieldNames(); + prometheusResponseFieldNames.setValueFieldName("count()"); + prometheusResponseFieldNames.setValueType(INTEGER); + prometheusResponseFieldNames.setTimestampFieldName(TIMESTAMP); + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.setPrometheusResponseFieldNames(prometheusResponseFieldNames); + prometheusMetricScan.getRequest().setPromQl(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("count()", new ExprIntegerValue(1)); + put("instance", new ExprStringValue("localhost:9090")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("prometheus")); + } + }); + assertEquals(firstRow, prometheusMetricScan.next()); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("count()", new ExprIntegerValue(0)); + put("instance", new ExprStringValue("localhost:9091")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("node")); + } + }); + assertEquals(secondRow, prometheusMetricScan.next()); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + + + @Test + @SneakyThrows + void testQueryResponseIteratorWithGivenPrometheusResponseWithLongInAggType() { + PrometheusResponseFieldNames prometheusResponseFieldNames + = new PrometheusResponseFieldNames(); + prometheusResponseFieldNames.setValueFieldName("testAgg"); + prometheusResponseFieldNames.setValueType(LONG); + prometheusResponseFieldNames.setTimestampFieldName(TIMESTAMP); + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.setPrometheusResponseFieldNames(prometheusResponseFieldNames); + prometheusMetricScan.getRequest().setPromQl(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("testAgg", new ExprLongValue(1)); + put("instance", new ExprStringValue("localhost:9090")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("prometheus")); + } + }); + assertEquals(firstRow, prometheusMetricScan.next()); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put("testAgg", new ExprLongValue(0)); + put("instance", new ExprStringValue("localhost:9091")); + put("__name__", new ExprStringValue("up")); + put("job", new ExprStringValue("node")); + } + }); + assertEquals(secondRow, prometheusMetricScan.next()); + Assertions.assertFalse(prometheusMetricScan.hasNext()); + } + + @Test + @SneakyThrows + void testQueryResponseIteratorForQueryRangeFunction() { + PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); + prometheusMetricScan.setIsQueryRangeFunctionScan(Boolean.TRUE); + prometheusMetricScan.getRequest().setPromQl(QUERY); + prometheusMetricScan.getRequest().setStartTime(STARTTIME); + prometheusMetricScan.getRequest().setEndTime(ENDTIME); + prometheusMetricScan.getRequest().setStep(STEP); + + when(prometheusClient.queryRange(any(), any(), any(), any())) + .thenReturn(new JSONObject(getJson("query_range_result.json"))); + prometheusMetricScan.open(); + Assertions.assertTrue(prometheusMetricScan.hasNext()); + ExprTupleValue firstRow = new ExprTupleValue(new LinkedHashMap<>() {{ + put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); + put(VALUE, new ExprLongValue(1)); put(LABELS, new ExprStringValue( "{\"instance\":\"localhost:9090\",\"__name__\":\"up\",\"job\":\"prometheus\"}")); } @@ -64,7 +188,7 @@ void testQueryResponseIterator() { Assertions.assertTrue(prometheusMetricScan.hasNext()); ExprTupleValue secondRow = new ExprTupleValue(new LinkedHashMap<>() {{ put(TIMESTAMP, new ExprTimestampValue(Instant.ofEpochMilli(1435781430781L))); - put(VALUE, new ExprDoubleValue(0)); + put(VALUE, new ExprLongValue(0)); put(LABELS, new ExprStringValue( "{\"instance\":\"localhost:9091\",\"__name__\":\"up\",\"job\":\"node\"}")); } @@ -77,7 +201,7 @@ void testQueryResponseIterator() { @SneakyThrows void testEmptyQueryResponseIterator() { PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); - prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setPromQl(QUERY); prometheusMetricScan.getRequest().setStartTime(STARTTIME); prometheusMetricScan.getRequest().setEndTime(ENDTIME); prometheusMetricScan.getRequest().setStep(STEP); @@ -92,7 +216,7 @@ void testEmptyQueryResponseIterator() { @SneakyThrows void testEmptyQueryWithNoMatrixKeyInResultJson() { PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); - prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setPromQl(QUERY); prometheusMetricScan.getRequest().setStartTime(STARTTIME); prometheusMetricScan.getRequest().setEndTime(ENDTIME); prometheusMetricScan.getRequest().setStep(STEP); @@ -110,7 +234,7 @@ void testEmptyQueryWithNoMatrixKeyInResultJson() { @SneakyThrows void testEmptyQueryWithException() { PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); - prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setPromQl(QUERY); prometheusMetricScan.getRequest().setStartTime(STARTTIME); prometheusMetricScan.getRequest().setEndTime(ENDTIME); prometheusMetricScan.getRequest().setStep(STEP); @@ -127,7 +251,7 @@ void testEmptyQueryWithException() { @SneakyThrows void testExplain() { PrometheusMetricScan prometheusMetricScan = new PrometheusMetricScan(prometheusClient); - prometheusMetricScan.getRequest().getPromQl().append(QUERY); + prometheusMetricScan.getRequest().setPromQl(QUERY); prometheusMetricScan.getRequest().setStartTime(STARTTIME); prometheusMetricScan.getRequest().setEndTime(ENDTIME); prometheusMetricScan.getRequest().setStep(STEP); diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java index 116ff72a6a..ff5ae5dcf5 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusMetricTableTest.java @@ -6,32 +6,50 @@ package org.opensearch.sql.prometheus.storage; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import static org.opensearch.sql.data.model.ExprValueUtils.fromObjectValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.expression.DSL.named; +import static org.opensearch.sql.expression.DSL.ref; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.filter; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.LABELS; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.TIMESTAMP; import static org.opensearch.sql.prometheus.data.constants.PrometheusFieldConstants.VALUE; +import static org.opensearch.sql.prometheus.utils.LogicalPlanUtils.indexScan; +import static org.opensearch.sql.prometheus.utils.LogicalPlanUtils.indexScanAgg; +import static org.opensearch.sql.prometheus.utils.LogicalPlanUtils.testLogicalPlanNode; +import com.google.common.collect.ImmutableList; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import lombok.SneakyThrows; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.NamedExpression; -import org.opensearch.sql.expression.ReferenceExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.planner.logical.LogicalPlan; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.planner.physical.ProjectOperator; @@ -45,9 +63,11 @@ class PrometheusMetricTableTest { @Mock private PrometheusClient client; + private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository()); + @Test @SneakyThrows - void getFieldTypesFromMetric() { + void testGetFieldTypesFromMetric() { when(client.getLabels(TestConstants.METRIC_NAME)).thenReturn(List.of("label1", "label2")); PrometheusMetricTable prometheusMetricTable = new PrometheusMetricTable(client, TestConstants.METRIC_NAME); @@ -56,44 +76,51 @@ void getFieldTypesFromMetric() { expectedFieldTypes.put("label2", ExprCoreType.STRING); expectedFieldTypes.put(VALUE, ExprCoreType.DOUBLE); expectedFieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); - expectedFieldTypes.put(LABELS, ExprCoreType.STRING); Map fieldTypes = prometheusMetricTable.getFieldTypes(); assertEquals(expectedFieldTypes, fieldTypes); verify(client, times(1)).getLabels(TestConstants.METRIC_NAME); - assertFalse(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); - assertTrue(prometheusMetricTable.getMetricName().isPresent()); + assertNull(prometheusMetricTable.getPrometheusQueryRequest()); + assertNotNull(prometheusMetricTable.getMetricName()); + + //testing Caching fieldTypes = prometheusMetricTable.getFieldTypes(); + + assertEquals(expectedFieldTypes, fieldTypes); verifyNoMoreInteractions(client); + assertNull(prometheusMetricTable.getPrometheusQueryRequest()); + assertNotNull(prometheusMetricTable.getMetricName()); } @Test @SneakyThrows - void getFieldTypesFromPrometheusQueryRequest() { + void testGetFieldTypesFromPrometheusQueryRequest() { PrometheusMetricTable prometheusMetricTable = new PrometheusMetricTable(client, new PrometheusQueryRequest()); Map expectedFieldTypes = new HashMap<>(); expectedFieldTypes.put(VALUE, ExprCoreType.DOUBLE); expectedFieldTypes.put(TIMESTAMP, ExprCoreType.TIMESTAMP); - expectedFieldTypes.put(LABELS, ExprCoreType.STRING); + expectedFieldTypes.put(LABELS, STRING); Map fieldTypes = prometheusMetricTable.getFieldTypes(); assertEquals(expectedFieldTypes, fieldTypes); verifyNoMoreInteractions(client); - assertTrue(prometheusMetricTable.getPrometheusQueryRequest().isPresent()); - assertFalse(prometheusMetricTable.getMetricName().isPresent()); + assertNotNull(prometheusMetricTable.getPrometheusQueryRequest()); + assertNull(prometheusMetricTable.getMetricName()); } @Test - void testImplement() { + void testImplementWithQueryRangeFunction() { PrometheusQueryRequest prometheusQueryRequest = new PrometheusQueryRequest(); + prometheusQueryRequest.setPromQl("test"); + prometheusQueryRequest.setStep("15m"); PrometheusMetricTable prometheusMetricTable = new PrometheusMetricTable(client, prometheusQueryRequest); List finalProjectList = new ArrayList<>(); - finalProjectList.add( - new NamedExpression(LABELS, new ReferenceExpression(LABELS, ExprCoreType.STRING))); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); PhysicalPlan plan = prometheusMetricTable.implement( project(relation("query_range", prometheusMetricTable), finalProjectList, null)); @@ -103,21 +130,608 @@ void testImplement() { List projectList = ((ProjectOperator) plan).getProjectList(); List outputFields = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); - assertEquals(List.of(LABELS, TIMESTAMP, VALUE), outputFields); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); PrometheusMetricScan prometheusMetricScan = (PrometheusMetricScan) ((ProjectOperator) plan).getInput(); assertEquals(prometheusQueryRequest, prometheusMetricScan.getRequest()); } + @Test + void testImplementWithBasicMetricQuery() { + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_requests_total"); + List finalProjectList = new ArrayList<>(); + finalProjectList.add(named("@value", ref("@value", ExprCoreType.DOUBLE))); + PhysicalPlan plan = prometheusMetricTable.implement( + project(relation("prometheus_http_requests_total", prometheusMetricTable), + finalProjectList, null)); + + assertTrue(plan instanceof ProjectOperator); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE), outputFields); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusMetricScan prometheusMetricScan = + (PrometheusMetricScan) ((ProjectOperator) plan).getInput(); + assertEquals("prometheus_http_requests_total", prometheusMetricScan.getRequest().getPromQl()); + assertEquals(3600 / 250 + "s", prometheusMetricScan.getRequest().getStep()); + } + + + @Test + void testImplementPrometheusQueryWithStatsQueryAndNoFilter() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + // IndexScanAgg without Filter + PhysicalPlan plan = prometheusMetricTable.implement( + filter( + indexScanAgg("prometheus_http_total_requests", ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("code", DSL.ref("code", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))))); + + assertTrue(plan.getChild().get(0) instanceof PrometheusMetricScan); + PrometheusQueryRequest prometheusQueryRequest = + ((PrometheusMetricScan) plan.getChild().get(0)).getRequest(); + assertEquals( + "avg by(code) (avg_over_time(prometheus_http_total_requests[40s]))", + prometheusQueryRequest.getPromQl()); + } + + @Test + void testImplementPrometheusQueryWithStatsQueryAndFilter() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + // IndexScanAgg with Filter + PhysicalPlan plan = prometheusMetricTable.implement( + indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s"))))); + assertTrue(plan instanceof PrometheusMetricScan); + PrometheusQueryRequest prometheusQueryRequest = ((PrometheusMetricScan) plan).getRequest(); + assertEquals( + "avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + prometheusQueryRequest.getPromQl()); + + } + + + @Test + void testImplementPrometheusQueryWithStatsQueryAndFilterAndProject() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + // IndexScanAgg with Filter and Project + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + PhysicalPlan plan = prometheusMetricTable.implement( + project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))), + ImmutableList + .of(DSL.named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(DSL.named("job", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + finalProjectList, null)); + assertTrue(plan instanceof ProjectOperator); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) plan).getInput()).getRequest(); + assertEquals(request.getStep(), "40s"); + assertEquals("avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + request.getPromQl()); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + + + @Test + void testTimeRangeResolver() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + //Both endTime and startTime are set. + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + PhysicalPlan plan = prometheusMetricTable.implement( + project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.and(dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP)))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + finalProjectList, null)); + assertTrue(plan instanceof ProjectOperator); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) plan).getInput()).getRequest(); + assertEquals("40s", request.getStep()); + assertEquals("avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + request.getPromQl()); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + + @Test + void testTimeRangeResolverWithOutEndTimeInFilter() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + //Both endTime and startTime are set. + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + PhysicalPlan plan = prometheusMetricTable.implement( + project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + finalProjectList, null)); + assertTrue(plan instanceof ProjectOperator); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) plan).getInput()).getRequest(); + assertEquals("40s", request.getStep()); + assertEquals("avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + request.getPromQl()); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + + @Test + void testTimeRangeResolverWithOutStartTimeInFilter() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + //Both endTime and startTime are set. + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + Long endTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + PhysicalPlan plan = prometheusMetricTable.implement( + project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + finalProjectList, null)); + assertTrue(plan instanceof ProjectOperator); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) plan).getInput()).getRequest(); + assertEquals("40s", request.getStep()); + assertEquals("avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + request.getPromQl()); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + + + @Test + void testSpanResolverWithoutSpanExpression() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + LogicalPlan plan = project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.and(dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP)))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + null), + finalProjectList, null); + RuntimeException runtimeException + = Assertions.assertThrows(RuntimeException.class, + () -> prometheusMetricTable.implement(plan)); + Assertions.assertEquals("Prometheus Catalog doesn't support " + + "aggregations without span expression", + runtimeException.getMessage()); + } + + @Test + void testSpanResolverWithEmptyGroupByList() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + LogicalPlan plan = project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.and(dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP)))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of()), + finalProjectList, null); + RuntimeException runtimeException + = Assertions.assertThrows(RuntimeException.class, + () -> prometheusMetricTable.implement(plan)); + Assertions.assertEquals("Prometheus Catalog doesn't support " + + "aggregations without span expression", + runtimeException.getMessage()); + } + + @Test + void testSpanResolverWithSpanExpression() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + PhysicalPlan plan = prometheusMetricTable.implement( + project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.and(dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP)))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + finalProjectList, null)); + assertTrue(plan instanceof ProjectOperator); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) plan).getInput()).getRequest(); + assertEquals("40s", request.getStep()); + assertEquals("avg by(job) (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + request.getPromQl()); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + + @Test + void testExpressionWithMissingTimeUnitInSpanExpression() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + LogicalPlan logicalPlan = project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.and(dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP)))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING)), + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "")))), + finalProjectList, null); + RuntimeException exception = + Assertions.assertThrows(RuntimeException.class, + () -> prometheusMetricTable.implement(logicalPlan)); + assertEquals("Missing TimeUnit in the span expression", exception.getMessage()); + } + + @Test + void testPrometheusQueryWithOnlySpanExpressionInGroupByList() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + PhysicalPlan plan = prometheusMetricTable.implement( + project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.and(dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP)))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of( + named("span", DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + finalProjectList, null)); + assertTrue(plan instanceof ProjectOperator); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) plan).getInput()).getRequest(); + assertEquals("40s", request.getStep()); + assertEquals("avg (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + request.getPromQl()); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + + @Test + void testStatsWithNoGroupByList() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + + + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + Long endTime = new Date(System.currentTimeMillis()).getTime(); + Long startTime = new Date(System.currentTimeMillis() - 4800 * 1000).getTime(); + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + PhysicalPlan plan = prometheusMetricTable.implement( + project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.and( + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))), + dsl.and(dsl.gte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(startTime)), + ExprCoreType.TIMESTAMP))), + dsl.lte(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal( + fromObjectValue(dateFormat.format(new Date(endTime)), + ExprCoreType.TIMESTAMP)))))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("span", + DSL.span(DSL.ref("@timestamp", ExprCoreType.TIMESTAMP), + DSL.literal(40), "s")))), + finalProjectList, null)); + assertTrue(plan instanceof ProjectOperator); + assertTrue(((ProjectOperator) plan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) plan).getInput()).getRequest(); + assertEquals("40s", request.getStep()); + assertEquals("avg (avg_over_time" + + "(prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}[40s]))", + request.getPromQl()); + List projectList = ((ProjectOperator) plan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + + @Test + void testImplementWithUnexpectedLogicalNode() { + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + LogicalPlan plan = project(testLogicalPlanNode()); + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> prometheusMetricTable.implement(plan)); + assertEquals("unexpected plan node type class" + + " org.opensearch.sql.prometheus.utils.LogicalPlanUtils$TestLogicalPlan", + runtimeException.getMessage()); + } + + @Test + void testMultipleAggregationsThrowsRuntimeException() { + + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + LogicalPlan plan = project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))), + ImmutableList + .of(named("AVG(@value)", + dsl.avg(DSL.ref("@value", INTEGER))), + named("SUM(@value)", + dsl.avg(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING))))); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> prometheusMetricTable.implement(plan)); + assertEquals("Prometheus Catalog doesn't multiple aggregations in stats command", + runtimeException.getMessage()); + } + + + @Test + void testUnSupportedAggregation() { + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + LogicalPlan plan = project(indexScanAgg("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/")))), + ImmutableList + .of(named("VAR_SAMP(@value)", + dsl.varSamp(DSL.ref("@value", INTEGER)))), + ImmutableList.of(named("job", DSL.ref("job", STRING))))); + + RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, + () -> prometheusMetricTable.implement(plan)); + assertTrue(runtimeException.getMessage().contains("Prometheus Catalog only supports")); + } + + @Test + void testImplementWithORConditionInWhereClause() { + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + LogicalPlan plan = indexScan("prometheus_http_total_requests", + dsl.or(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))))); + RuntimeException exception + = assertThrows(RuntimeException.class, () -> prometheusMetricTable.implement(plan)); + assertEquals("Prometheus Catalog doesn't support or in where command.", exception.getMessage()); + } + + @Test + void testImplementWithRelationAndFilter() { + List finalProjectList = new ArrayList<>(); + finalProjectList.add(DSL.named(VALUE, DSL.ref(VALUE, STRING))); + finalProjectList.add(DSL.named(TIMESTAMP, DSL.ref(TIMESTAMP, ExprCoreType.TIMESTAMP))); + PrometheusMetricTable prometheusMetricTable = + new PrometheusMetricTable(client, "prometheus_http_total_requests"); + LogicalPlan logicalPlan = project(indexScan("prometheus_http_total_requests", + dsl.and(dsl.equal(DSL.ref("code", STRING), DSL.literal(stringValue("200"))), + dsl.equal(DSL.ref("handler", STRING), DSL.literal(stringValue("/ready/"))))), + finalProjectList, null); + PhysicalPlan physicalPlan = prometheusMetricTable.implement(logicalPlan); + assertTrue(physicalPlan instanceof ProjectOperator); + assertTrue(((ProjectOperator) physicalPlan).getInput() instanceof PrometheusMetricScan); + PrometheusQueryRequest request + = ((PrometheusMetricScan) ((ProjectOperator) physicalPlan).getInput()).getRequest(); + assertEquals((3600 / 250) + "s", request.getStep()); + assertEquals("prometheus_http_total_requests{code=\"200\" , handler=\"/ready/\"}", + request.getPromQl()); + List projectList = ((ProjectOperator) physicalPlan).getProjectList(); + List outputFields + = projectList.stream().map(NamedExpression::getName).collect(Collectors.toList()); + assertEquals(List.of(VALUE, TIMESTAMP), outputFields); + } + @Test void testOptimize() { PrometheusQueryRequest prometheusQueryRequest = new PrometheusQueryRequest(); PrometheusMetricTable prometheusMetricTable = new PrometheusMetricTable(client, prometheusQueryRequest); List finalProjectList = new ArrayList<>(); - finalProjectList.add( - new NamedExpression(LABELS, new ReferenceExpression(LABELS, ExprCoreType.STRING))); LogicalPlan inputPlan = project(relation("query_range", prometheusMetricTable), finalProjectList, null); LogicalPlan optimizedPlan = prometheusMetricTable.optimize( diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java index 992b17b549..fadd061072 100644 --- a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/PrometheusStorageEngineTest.java @@ -35,7 +35,8 @@ class PrometheusStorageEngineTest { public void getTable() { PrometheusStorageEngine engine = new PrometheusStorageEngine(client); Table table = engine.getTable(new CatalogSchemaName("prometheus", "default"), "test"); - assertNull(table); + assertNotNull(table); + assertTrue(table instanceof PrometheusMetricTable); } @Test diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/querybuilders/StepParameterResolverTest.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/querybuilders/StepParameterResolverTest.java new file mode 100644 index 0000000000..37e24a56b5 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/storage/querybuilders/StepParameterResolverTest.java @@ -0,0 +1,26 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.storage.querybuilders; + +import java.util.Collections; +import java.util.Date; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.prometheus.storage.querybuilder.StepParameterResolver; + +public class StepParameterResolverTest { + + @Test + void testNullChecks() { + StepParameterResolver stepParameterResolver = new StepParameterResolver(); + Assertions.assertThrows(NullPointerException.class, + () -> stepParameterResolver.resolve(null, new Date().getTime(), Collections.emptyList())); + Assertions.assertThrows(NullPointerException.class, + () -> stepParameterResolver.resolve(new Date().getTime(), null, Collections.emptyList())); + } +} diff --git a/prometheus/src/test/java/org/opensearch/sql/prometheus/utils/LogicalPlanUtils.java b/prometheus/src/test/java/org/opensearch/sql/prometheus/utils/LogicalPlanUtils.java new file mode 100644 index 0000000000..5fcebf52e6 --- /dev/null +++ b/prometheus/src/test/java/org/opensearch/sql/prometheus/utils/LogicalPlanUtils.java @@ -0,0 +1,77 @@ +/* + * + * * Copyright OpenSearch Contributors + * * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.opensearch.sql.prometheus.utils; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.aggregation.NamedAggregator; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalPlanNodeVisitor; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricAgg; +import org.opensearch.sql.prometheus.planner.logical.PrometheusLogicalMetricScan; + +public class LogicalPlanUtils { + + /** + * Build PrometheusLogicalMetricScan. + */ + public static LogicalPlan indexScan(String metricName, Expression filter) { + return PrometheusLogicalMetricScan.builder().metricName(metricName) + .filter(filter) + .build(); + } + + /** + * Build PrometheusLogicalMetricAgg. + */ + public static LogicalPlan indexScanAgg(String metricName, Expression filter, + List aggregators, + List groupByList) { + return PrometheusLogicalMetricAgg.builder().metricName(metricName) + .filter(filter) + .aggregatorList(aggregators) + .groupByList(groupByList) + .build(); + } + + /** + * Build PrometheusLogicalMetricAgg. + */ + public static LogicalPlan indexScanAgg(String metricName, + List aggregators, + List groupByList) { + return PrometheusLogicalMetricAgg.builder().metricName(metricName) + .aggregatorList(aggregators) + .groupByList(groupByList) + .build(); + } + + /** + * Build PrometheusLogicalMetricAgg. + */ + public static LogicalPlan testLogicalPlanNode() { + return new TestLogicalPlan(); + } + + static class TestLogicalPlan extends LogicalPlan { + + public TestLogicalPlan() { + super(ImmutableList.of()); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitNode(this, null); + } + } + + + +} From 4282450878005c88126d25333cad662baa1ee2ed Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 3 Nov 2022 13:29:39 -0700 Subject: [PATCH 18/20] Fix incorrect results returned by `min`, `max` and `avg` (#1000) * Fix incorrect results returned by `min`, `max` and `avg` aggregations on null/missing values. Signed-off-by: Yury-Fridlyand * Fix indentation. Signed-off-by: Yury-Fridlyand * Activate and fix integration tests. Signed-off-by: Yury-Fridlyand * Add more tests. Signed-off-by: Yury-Fridlyand Signed-off-by: Yury-Fridlyand --- .../sql/legacy/SQLIntegTestCase.java | 11 +- .../opensearch/sql/legacy/TestsConstants.java | 2 + .../org/opensearch/sql/sql/AggregationIT.java | 206 +++++++++++++++++- integ-test/src/test/resources/calcs.json | 34 +++ .../calcs_index_mappings.json | 94 ++++++++ .../null_missing_index_mapping.json | 37 ++++ .../src/test/resources/null_missing.json | 18 ++ .../response/agg/SingleValueParser.java | 4 +- .../opensearch/response/agg/StatsParser.java | 4 +- .../sql/opensearch/response/agg/Utils.java | 8 +- ...enSearchAggregationResponseParserTest.java | 6 +- 11 files changed, 409 insertions(+), 15 deletions(-) create mode 100644 integ-test/src/test/resources/calcs.json create mode 100644 integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json create mode 100644 integ-test/src/test/resources/indexDefinitions/null_missing_index_mapping.json create mode 100644 integ-test/src/test/resources/null_missing.json diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 5c339cc7bb..f03acbbbfd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -48,6 +48,7 @@ import static org.opensearch.sql.legacy.TestUtils.getGameOfThronesIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getJoinTypeIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getLocationIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getMappingFile; import static org.opensearch.sql.legacy.TestUtils.getNestedSimpleIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getNestedTypeIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getOdbcIndexMapping; @@ -575,7 +576,15 @@ public enum Index { BEER(TestsConstants.TEST_INDEX_BEER, "beer", null, - "src/test/resources/beer.stackexchange.json"),; + "src/test/resources/beer.stackexchange.json"), + NULL_MISSING(TestsConstants.TEST_INDEX_NULL_MISSING, + "null_missing", + getMappingFile("null_missing_index_mapping.json"), + "src/test/resources/null_missing.json"), + CALCS(TestsConstants.TEST_INDEX_CALCS, + "calcs", + getMappingFile("calcs_index_mappings.json"), + "src/test/resources/calcs.json"),; private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index f54f079bb6..a9f81c68fe 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -51,6 +51,8 @@ public class TestsConstants { public final static String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; public final static String TEST_INDEX_DATATYPE_NONNUMERIC = TEST_INDEX + "_datatypes_nonnumeric"; public final static String TEST_INDEX_BEER = TEST_INDEX + "_beer"; + public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing"; + public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java index edd78b9506..95f5b5e3e4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/AggregationIT.java @@ -6,36 +6,234 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NULL_MISSING; +import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import static org.opensearch.sql.util.MatcherUtils.verifySome; +import static org.opensearch.sql.util.TestUtils.getResponseBody; import java.io.IOException; +import java.util.List; +import java.util.Locale; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.RequestOptions; +import org.opensearch.client.Response; import org.opensearch.sql.legacy.SQLIntegTestCase; public class AggregationIT extends SQLIntegTestCase { @Override protected void init() throws Exception { + super.init(); loadIndex(Index.BANK); + loadIndex(Index.NULL_MISSING); + loadIndex(Index.CALCS); } @Test - void filteredAggregatePushedDown() throws IOException { + public void testFilteredAggregatePushDown() throws IOException { JSONObject response = executeQuery( "SELECT COUNT(*) FILTER(WHERE age > 35) FROM " + TEST_INDEX_BANK); - verifySchema(response, schema("COUNT(*)", null, "integer")); + verifySchema(response, schema("COUNT(*) FILTER(WHERE age > 35)", null, "integer")); verifyDataRows(response, rows(3)); } @Test - void filteredAggregateNotPushedDown() throws IOException { + public void testFilteredAggregateNotPushDown() throws IOException { JSONObject response = executeQuery( "SELECT COUNT(*) FILTER(WHERE age > 35) FROM (SELECT * FROM " + TEST_INDEX_BANK + ") AS a"); - verifySchema(response, schema("COUNT(*)", null, "integer")); + verifySchema(response, schema("COUNT(*) FILTER(WHERE age > 35)", null, "integer")); verifyDataRows(response, rows(3)); } + + @Test + public void testPushDownAggregationOnNullValues() throws IOException { + // OpenSearch aggregation query (MetricAggregation) + var response = executeQuery(String.format( + "SELECT min(`int`), max(`int`), avg(`int`), min(`dbl`), max(`dbl`), avg(`dbl`) " + + "FROM %s WHERE `key` = 'null'", TEST_INDEX_NULL_MISSING)); + verifySchema(response, + schema("min(`int`)", null, "integer"), schema("max(`int`)", null, "integer"), + schema("avg(`int`)", null, "double"), schema("min(`dbl`)", null, "double"), + schema("max(`dbl`)", null, "double"), schema("avg(`dbl`)", null, "double")); + verifyDataRows(response, rows(null, null, null, null, null, null)); + } + + @Test + public void testPushDownAggregationOnMissingValues() throws IOException { + // OpenSearch aggregation query (MetricAggregation) + var response = executeQuery(String.format( + "SELECT min(`int`), max(`int`), avg(`int`), min(`dbl`), max(`dbl`), avg(`dbl`) " + + "FROM %s WHERE `key` = 'null'", TEST_INDEX_NULL_MISSING)); + verifySchema(response, + schema("min(`int`)", null, "integer"), schema("max(`int`)", null, "integer"), + schema("avg(`int`)", null, "double"), schema("min(`dbl`)", null, "double"), + schema("max(`dbl`)", null, "double"), schema("avg(`dbl`)", null, "double")); + verifyDataRows(response, rows(null, null, null, null, null, null)); + } + + @Test + public void testInMemoryAggregationOnNullValues() throws IOException { + // In-memory aggregation performed by the plugin + var response = executeQuery(String.format("SELECT" + + " min(`int`) over (PARTITION BY `key`), max(`int`) over (PARTITION BY `key`)," + + " avg(`int`) over (PARTITION BY `key`), min(`dbl`) over (PARTITION BY `key`)," + + " max(`dbl`) over (PARTITION BY `key`), avg(`dbl`) over (PARTITION BY `key`)" + + " FROM %s WHERE `key` = 'null'", TEST_INDEX_NULL_MISSING)); + verifySchema(response, + schema("min(`int`) over (PARTITION BY `key`)", null, "integer"), + schema("max(`int`) over (PARTITION BY `key`)", null, "integer"), + schema("avg(`int`) over (PARTITION BY `key`)", null, "double"), + schema("min(`dbl`) over (PARTITION BY `key`)", null, "double"), + schema("max(`dbl`) over (PARTITION BY `key`)", null, "double"), + schema("avg(`dbl`) over (PARTITION BY `key`)", null, "double")); + verifyDataRows(response, // 4 rows with null values + rows(null, null, null, null, null, null), + rows(null, null, null, null, null, null), + rows(null, null, null, null, null, null), + rows(null, null, null, null, null, null)); + } + + @Test + public void testInMemoryAggregationOnMissingValues() throws IOException { + // In-memory aggregation performed by the plugin + var response = executeQuery(String.format("SELECT" + + " min(`int`) over (PARTITION BY `key`), max(`int`) over (PARTITION BY `key`)," + + " avg(`int`) over (PARTITION BY `key`), min(`dbl`) over (PARTITION BY `key`)," + + " max(`dbl`) over (PARTITION BY `key`), avg(`dbl`) over (PARTITION BY `key`)" + + " FROM %s WHERE `key` = 'missing'", TEST_INDEX_NULL_MISSING)); + verifySchema(response, + schema("min(`int`) over (PARTITION BY `key`)", null, "integer"), + schema("max(`int`) over (PARTITION BY `key`)", null, "integer"), + schema("avg(`int`) over (PARTITION BY `key`)", null, "double"), + schema("min(`dbl`) over (PARTITION BY `key`)", null, "double"), + schema("max(`dbl`) over (PARTITION BY `key`)", null, "double"), + schema("avg(`dbl`) over (PARTITION BY `key`)", null, "double")); + verifyDataRows(response, // 4 rows with null values + rows(null, null, null, null, null, null), + rows(null, null, null, null, null, null), + rows(null, null, null, null, null, null), + rows(null, null, null, null, null, null)); + } + + @Test + public void testInMemoryAggregationOnNullValuesReturnsNull() throws IOException { + var response = executeQuery(String.format("SELECT " + + " max(int0) over (PARTITION BY `datetime1`)," + + " min(int0) over (PARTITION BY `datetime1`)," + + " avg(int0) over (PARTITION BY `datetime1`)" + + "from %s where int0 IS NULL;", TEST_INDEX_CALCS)); + verifySchema(response, + schema("max(int0) over (PARTITION BY `datetime1`)", null, "integer"), + schema("min(int0) over (PARTITION BY `datetime1`)", null, "integer"), + schema("avg(int0) over (PARTITION BY `datetime1`)", null, "double")); + verifySome(response.getJSONArray("datarows"), rows(null, null, null)); + } + + @Test + public void testInMemoryAggregationOnAllValuesAndOnNotNullReturnsSameResult() throws IOException { + var responseNotNulls = executeQuery(String.format("SELECT " + + " max(int0) over (PARTITION BY `datetime1`)," + + " min(int0) over (PARTITION BY `datetime1`)," + + " avg(int0) over (PARTITION BY `datetime1`)" + + "from %s where int0 IS NOT NULL;", TEST_INDEX_CALCS)); + var responseAllValues = executeQuery(String.format("SELECT " + + " max(int0) over (PARTITION BY `datetime1`)," + + " min(int0) over (PARTITION BY `datetime1`)," + + " avg(int0) over (PARTITION BY `datetime1`)" + + "from %s;", TEST_INDEX_CALCS)); + verifySchema(responseNotNulls, + schema("max(int0) over (PARTITION BY `datetime1`)", null, "integer"), + schema("min(int0) over (PARTITION BY `datetime1`)", null, "integer"), + schema("avg(int0) over (PARTITION BY `datetime1`)", null, "double")); + verifySchema(responseAllValues, + schema("max(int0) over (PARTITION BY `datetime1`)", null, "integer"), + schema("min(int0) over (PARTITION BY `datetime1`)", null, "integer"), + schema("avg(int0) over (PARTITION BY `datetime1`)", null, "double")); + assertEquals(responseNotNulls.query("/datarows/0/0"), responseAllValues.query("/datarows/0/0")); + assertEquals(responseNotNulls.query("/datarows/0/1"), responseAllValues.query("/datarows/0/1")); + assertEquals(responseNotNulls.query("/datarows/0/2"), responseAllValues.query("/datarows/0/2")); + } + + @Test + public void testPushDownAggregationOnNullValuesReturnsNull() throws IOException { + var response = executeQuery(String.format("SELECT " + + "max(int0), min(int0), avg(int0) from %s where int0 IS NULL;", TEST_INDEX_CALCS)); + verifySchema(response, + schema("max(int0)", null, "integer"), + schema("min(int0)", null, "integer"), + schema("avg(int0)", null, "double")); + verifyDataRows(response, rows(null, null, null)); + } + + @Test + public void testPushDownAggregationOnAllValuesAndOnNotNullReturnsSameResult() throws IOException { + var responseNotNulls = executeQuery(String.format("SELECT " + + "max(int0), min(int0), avg(int0) from %s where int0 IS NOT NULL;", TEST_INDEX_CALCS)); + var responseAllValues = executeQuery(String.format("SELECT " + + "max(int0), min(int0), avg(int0) from %s;", TEST_INDEX_CALCS)); + verifySchema(responseNotNulls, + schema("max(int0)", null, "integer"), + schema("min(int0)", null, "integer"), + schema("avg(int0)", null, "double")); + verifySchema(responseAllValues, + schema("max(int0)", null, "integer"), + schema("min(int0)", null, "integer"), + schema("avg(int0)", null, "double")); + assertEquals(responseNotNulls.query("/datarows/0/0"), responseAllValues.query("/datarows/0/0")); + assertEquals(responseNotNulls.query("/datarows/0/1"), responseAllValues.query("/datarows/0/1")); + assertEquals(responseNotNulls.query("/datarows/0/2"), responseAllValues.query("/datarows/0/2")); + } + + @Test + public void testPushDownAndInMemoryAggregationReturnTheSameResult() throws IOException { + // Playing with 'over (PARTITION BY `datetime1`)' - `datetime1` column has the same value for all rows + // so partitioning by this column has no sense and doesn't (shouldn't) affect the results + // Aggregations with `OVER` clause are executed in memory (in SQL plugin memory), + // Aggregations without it are performed the OpenSearch node itself (pushed down to opensearch) + // Going to compare results of `min`, `max` and `avg` aggregation on all numeric columns in `calcs` + var columns = List.of("int0", "int1", "int2", "int3", "num0", "num1", "num2", "num3", "num4"); + var aggregations = List.of("min", "max", "avg"); + var inMemoryAggregQuery = new StringBuilder("SELECT "); + var pushDownAggregQuery = new StringBuilder("SELECT "); + for (var col : columns) { + for (var aggreg : aggregations) { + inMemoryAggregQuery.append(String.format(" %s(%s) over (PARTITION BY `datetime1`),", aggreg, col)); + pushDownAggregQuery.append(String.format(" %s(%s),", aggreg, col)); + } + } + // delete last comma + inMemoryAggregQuery.deleteCharAt(inMemoryAggregQuery.length() - 1); + pushDownAggregQuery.deleteCharAt(pushDownAggregQuery.length() - 1); + + var responseInMemory = executeQuery( + inMemoryAggregQuery.append("from " + TEST_INDEX_CALCS).toString()); + var responsePushDown = executeQuery( + pushDownAggregQuery.append("from " + TEST_INDEX_CALCS).toString()); + + for (int i = 0; i < columns.size() * aggregations.size(); i++) { + assertEquals( + ((Number)responseInMemory.query("/datarows/0/" + i)).doubleValue(), + ((Number)responsePushDown.query("/datarows/0/" + i)).doubleValue(), + 0.0000001); // a minor delta is affordable + } + } + + protected JSONObject executeQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + + RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); + restOptionsBuilder.addHeader("Content-Type", "application/json"); + request.setOptions(restOptionsBuilder); + + Response response = client().performRequest(request); + return new JSONObject(getResponseBody(response)); + } } diff --git a/integ-test/src/test/resources/calcs.json b/integ-test/src/test/resources/calcs.json new file mode 100644 index 0000000000..c310cc8d0f --- /dev/null +++ b/integ-test/src/test/resources/calcs.json @@ -0,0 +1,34 @@ +{"index": {}} +{"key": "key00", "num0": 12.3, "num1": 8.42, "num2": 17.86, "num3": -11.52, "num4": null, "str0": "FURNITURE", "str1": "CLAMP ON LAMPS", "str2": "one", "str3": "e", "int0": 1, "int1": -3, "int2": 5, "int3": 8, "bool0": true, "bool1": true, "bool2": false, "bool3": true, "date0": "2004-04-15", "date1": "2004-04-01", "date2": "1977-04-20", "date3": "1986-03-20", "time0": "1899-12-30T21:07:32Z", "time1": "19:36:22", "datetime0": "2004-07-09T10:17:35Z", "datetime1": null, "zzz": "a"} +{"index": {}} +{"key": "key01", "num0": -12.3, "num1": 6.71, "num2": 16.73, "num3": -9.31, "num4": 10.85, "str0": "FURNITURE", "str1": "CLOCKS", "str2": "two", "str3": "e", "int0": null, "int1": -6, "int2": -4, "int3": 13, "bool0": false, "bool1": true, "bool2": false, "bool3": null, "date0": "1972-07-04", "date1": "2004-04-02", "date2": "1995-09-03", "date3": null, "time0": "1900-01-01T13:48:48Z", "time1": "02:05:25", "datetime0": "2004-07-26T12:30:34Z", "datetime1": null, "zzz": "b"} +{"index": {}} +{"key": "key02", "num0": 15.7, "num1": 9.78, "num2": null, "num3": -12.17, "num4": -13.47, "str0": "OFFICE SUPPLIES", "str1": "AIR PURIFIERS", "str2": "three", "str3": "e", "int0": null, "int1": null, "int2": 5, "int3": 2, "bool0": null, "bool1": true, "bool2": false, "bool3": null, "date0": "1975-11-12", "date1": "2004-04-03", "date2": "1997-09-19", "date3": "1997-02-02", "time0": "1900-01-01T18:21:08Z", "time1": "09:33:31", "datetime0": "2004-08-02T07:59:23Z", "datetime1": null, "zzz": "c"} +{"index": {}} +{"key": "key03", "num0": -15.7, "num1": 7.43, "num2": 8.51, "num3": -7.25, "num4": -6.05, "str0": "OFFICE SUPPLIES", "str1": "BINDER ACCESSORIES", "str2": null, "str3": "e", "int0": null, "int1": -4, "int2": -5, "int3": 5, "bool0": true, "bool1": false, "bool2": false, "bool3": null, "date0": "2004-06-04", "date1": "2004-04-04", "date2": "1980-07-26", "date3": null, "time0": "1900-01-01T18:51:48Z", "time1": "22:50:16", "datetime0": "2004-07-05T13:14:20Z", "datetime1": null, "zzz": "d"} +{"index": {}} +{"key": "key04", "num0": 3.5, "num1": 9.05, "num2": 6.46, "num3": 12.93, "num4": 8.32, "str0": "OFFICE SUPPLIES", "str1": "BINDER CLIPS", "str2": "five", "str3": null, "int0": 7, "int1": null, "int2": 3, "int3": 9, "bool0": false, "bool1": false, "bool2": true, "bool3": true, "date0": "2004-06-19", "date1": "2004-04-05", "date2": "1997-05-30", "date3": "1996-03-07", "time0": "1900-01-01T15:01:19Z", "time1": null, "datetime0": "2004-07-28T23:30:22Z", "datetime1": null, "zzz": "e"} +{"index": {}} +{"key": "key05", "num0": -3.5, "num1": 9.38, "num2": 8.98, "num3": -19.96, "num4": 10.71, "str0": "OFFICE SUPPLIES", "str1": "BINDING MACHINES", "str2": "six", "str3": null, "int0": 3, "int1": null, "int2": 2, "int3": 7, "bool0": null, "bool1": false, "bool2": true, "bool3": false, "date0": null, "date1": "2004-04-06", "date2": "1980-11-07", "date3": "1979-04-01", "time0": "1900-01-01T08:59:39Z", "time1": "19:57:33", "datetime0": "2004-07-22T00:30:23Z", "datetime1": null, "zzz": "f"} +{"index": {}} +{"key": "key06", "num0": 0, "num1": 16.42, "num2": 11.69, "num3": 10.93, "num4": null, "str0": "OFFICE SUPPLIES", "str1": "BINDING SUPPLIES", "str2": null, "str3": "e", "int0": 8, "int1": null, "int2": 9, "int3": 18, "bool0": true, "bool1": null, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-07", "date2": "1977-02-08", "date3": null, "time0": "1900-01-01T07:37:48Z", "time1": null, "datetime0": "2004-07-28T06:54:50Z", "datetime1": null, "zzz": "g"} +{"index": {}} +{"key": "key07", "num0": null, "num1": 11.38, "num2": 17.25, "num3": 3.64, "num4": -10.24, "str0": "OFFICE SUPPLIES", "str1": "BUSINESS ENVELOPES", "str2": "eight", "str3": "e", "int0": null, "int1": 2, "int2": 0, "int3": 3, "bool0": false, "bool1": null, "bool2": true, "bool3": false, "date0": null, "date1": "2004-04-08", "date2": "1974-05-03", "date3": null, "time0": "1900-01-01T19:45:54Z", "time1": "19:48:23", "datetime0": "2004-07-12T17:30:16Z", "datetime1": null, "zzz": "h"} +{"index": {}} +{"key": "key08", "num0": 10, "num1": 9.47, "num2": null, "num3": -13.38, "num4": 4.77, "str0": "TECHNOLOGY", "str1": "ANSWERING MACHINES", "str2": "nine", "str3": null, "int0": null, "int1": 3, "int2": -6, "int3": 17, "bool0": null, "bool1": null, "bool2": false, "bool3": false, "date0": null, "date1": "2004-04-09", "date2": "1976-09-09", "date3": "1983-05-22", "time0": "1900-01-01T09:00:59Z", "time1": "22:20:14", "datetime0": "2004-07-04T22:49:28Z", "datetime1": null, "zzz": "i"} +{"index": {}} +{"key": "key09", "num0": null, "num1": 12.4, "num2": 11.5, "num3": -10.56, "num4": null, "str0": "TECHNOLOGY", "str1": "BUSINESS COPIERS", "str2": "ten", "str3": "e", "int0": 8, "int1": 3, "int2": -9, "int3": 2, "bool0": null, "bool1": true, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-10", "date2": "1998-08-12", "date3": null, "time0": "1900-01-01T20:36:00Z", "time1": null, "datetime0": "2004-07-23T21:13:37Z", "datetime1": null, "zzz": "j"} +{"index": {}} +{"key": "key10", "num0": null, "num1": 10.32, "num2": 6.8, "num3": -4.79, "num4": 19.39, "str0": "TECHNOLOGY", "str1": "CD-R MEDIA", "str2": "eleven", "str3": "e", "int0": 4, "int1": null, "int2": -3, "int3": 11, "bool0": true, "bool1": true, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-11", "date2": "1974-03-17", "date3": "1999-08-20", "time0": "1900-01-01T01:31:32Z", "time1": "00:05:57", "datetime0": "2004-07-14T08:16:44Z", "datetime1": null, "zzz": "k"} +{"index": {}} +{"key": "key11", "num0": null, "num1": 2.47, "num2": 3.79, "num3": -10.81, "num4": 3.82, "str0": "TECHNOLOGY", "str1": "CONFERENCE PHONES", "str2": "twelve", "str3": null, "int0": 10, "int1": -8, "int2": -4, "int3": 2, "bool0": false, "bool1": true, "bool2": true, "bool3": null, "date0": null, "date1": "2004-04-12", "date2": "1994-04-20", "date3": null, "time0": "1899-12-30T22:15:40Z", "time1": "04:40:49", "datetime0": "2004-07-25T15:22:26Z", "datetime1": null, "zzz": "l"} +{"index": {}} +{"key": "key12", "num0": null, "num1": 12.05, "num2": null, "num3": -6.62, "num4": 3.38, "str0": "TECHNOLOGY", "str1": "CORDED KEYBOARDS", "str2": null, "str3": null, "int0": null, "int1": null, "int2": 0, "int3": 11, "bool0": null, "bool1": false, "bool2": true, "bool3": true, "date0": null, "date1": "2004-04-13", "date2": "2001-02-04", "date3": null, "time0": "1900-01-01T13:53:46Z", "time1": "04:48:07", "datetime0": "2004-07-17T14:01:56Z", "datetime1": null, "zzz": "m"} +{"index": {}} +{"key": "key13", "num0": null, "num1": 10.37, "num2": 13.04, "num3": -18.43, "num4": null, "str0": "TECHNOLOGY", "str1": "CORDLESS KEYBOARDS", "str2": "fourteen", "str3": null, "int0": 4, "int1": null, "int2": 4, "int3": 18, "bool0": null, "bool1": false, "bool2": true, "bool3": true, "date0": null, "date1": "2004-04-14", "date2": "1988-01-05", "date3": "1996-05-13", "time0": "1900-01-01T04:57:51Z", "time1": null, "datetime0": "2004-07-19T22:21:31Z", "datetime1": null, "zzz": "n"} +{"index": {}} +{"key": "key14", "num0": null, "num1": 7.1, "num2": null, "num3": 6.84, "num4": -14.21, "str0": "TECHNOLOGY", "str1": "DOT MATRIX PRINTERS", "str2": "fifteen", "str3": "e", "int0": 11, "int1": null, "int2": -8, "int3": 18, "bool0": true, "bool1": false, "bool2": true, "bool3": null, "date0": null, "date1": "2004-04-15", "date2": "1972-07-12", "date3": "1986-11-08", "time0": "1899-12-30T22:42:43Z", "time1": "18:58:41", "datetime0": "2004-07-31T11:57:52Z", "datetime1": null, "zzz": "o"} +{"index": {}} +{"key": "key15", "num0": null, "num1": 16.81, "num2": 10.98, "num3": -10.98, "num4": 6.75, "str0": "TECHNOLOGY", "str1": "DVD", "str2": "sixteen", "str3": "e", "int0": 4, "int1": null, "int2": -9, "int3": 11, "bool0": false, "bool1": null, "bool2": false, "bool3": true, "date0": null, "date1": "2004-04-16", "date2": "1995-06-04", "date3": null, "time0": "1899-12-30T22:24:08Z", "time1": null, "datetime0": "2004-07-14T07:43:00Z", "datetime1": null, "zzz": "p"} +{"index": {}} +{"key": "key16", "num0": null, "num1": 7.12, "num2": 7.87, "num3": -2.6, "num4": null, "str0": "TECHNOLOGY", "str1": "ERICSSON", "str2": null, "str3": null, "int0": 8, "int1": -9, "int2": 6, "int3": 0, "bool0": null, "bool1": null, "bool2": false, "bool3": null, "date0": null, "date1": "2004-04-17", "date2": "2002-04-27", "date3": "1992-01-18", "time0": "1900-01-01T11:58:29Z", "time1": "12:33:57", "datetime0": "2004-07-28T12:34:28Z", "datetime1": null, "zzz": "q"} diff --git a/integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json new file mode 100644 index 0000000000..08a88a9d32 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json @@ -0,0 +1,94 @@ +{ + "mappings" : { + "properties" : { + "key" : { + "type" : "keyword" + }, + "num0" : { + "type" : "double" + }, + "num1" : { + "type" : "double" + }, + "num2" : { + "type" : "double" + }, + "num3" : { + "type" : "double" + }, + "num4" : { + "type" : "double" + }, + "str0" : { + "type" : "keyword" + }, + "str1" : { + "type" : "keyword" + }, + "str2" : { + "type" : "keyword" + }, + "str3" : { + "type" : "keyword" + }, + "int0" : { + "type" : "integer" + }, + "int1" : { + "type" : "integer" + }, + "int2" : { + "type" : "integer" + }, + "int3" : { + "type" : "integer" + }, + "bool0" : { + "type" : "boolean" + }, + "bool1" : { + "type" : "boolean" + }, + "bool2" : { + "type" : "boolean" + }, + "bool3" : { + "type" : "boolean" + }, + "date0" : { + "type" : "date", + "format": "yyyy-MM-dd" + }, + "date1" : { + "type" : "date", + "format": "yyyy-MM-dd" + }, + "date2" : { + "type" : "date", + "format": "yyyy-MM-dd" + }, + "date3" : { + "type" : "date", + "format": "yyyy-MM-dd" + }, + "time0" : { + "type" : "date", + "format": "date_time_no_millis" + }, + "time1" : { + "type" : "date", + "format": "hour_minute_second" + }, + "datetime0" : { + "type" : "date", + "format": "date_time_no_millis" + }, + "datetime1" : { + "type" : "date" + }, + "zzz" : { + "type" : "keyword" + } + } + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/null_missing_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/null_missing_index_mapping.json new file mode 100644 index 0000000000..52faafae93 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/null_missing_index_mapping.json @@ -0,0 +1,37 @@ +{ + "mappings" : { + "properties" : { + "key" : { + "type" : "keyword" + }, + "int" : { + "type" : "integer" + }, + "dbl" : { + "type" : "double" + }, + "bool" : { + "type" : "boolean" + }, + "str" : { + "type" : "text" + }, + "date" : { + "type" : "date", + "format": "yyyy-MM-dd" + }, + "time" : { + "type" : "date", + "format": "HH:mm:ss" + }, + "datetime" : { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss" + }, + "timestamp" : { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss" + } + } + } +} diff --git a/integ-test/src/test/resources/null_missing.json b/integ-test/src/test/resources/null_missing.json new file mode 100644 index 0000000000..40c7d4a131 --- /dev/null +++ b/integ-test/src/test/resources/null_missing.json @@ -0,0 +1,18 @@ +{"index":{}} +{"key" : "values", "int" : 42, "dbl" : 3.1415, "bool" : true, "str" : "pewpew", "date" : "1984-05-22", "time" : "22:13:37", "datetime" : "1984-05-22 22:13:37", "timestamp" : "2000-01-02 03:04:05" } +{"index":{}} +{"key" : "missing"} +{"index":{}} +{"key" : "missing"} +{"index":{}} +{"key" : "missing"} +{"index":{}} +{"key" : "missing"} +{"index":{}} +{"key" : "null", "int" : null, "dbl" : null, "bool" : null, "str" : null, "date" : null, "time" : null, "datetime" : null, "timestamp" : null} +{"index":{}} +{"key" : "null", "int" : null, "dbl" : null, "bool" : null, "str" : null, "date" : null, "time" : null, "datetime" : null, "timestamp" : null} +{"index":{}} +{"key" : "null", "int" : null, "dbl" : null, "bool" : null, "str" : null, "date" : null, "time" : null, "datetime" : null, "timestamp" : null} +{"index":{}} +{"key" : "null", "int" : null, "dbl" : null, "bool" : null, "str" : null, "date" : null, "time" : null, "datetime" : null, "timestamp" : null} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java index 7536a24661..88d9604137 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java @@ -13,7 +13,7 @@ package org.opensearch.sql.opensearch.response.agg; -import static org.opensearch.sql.opensearch.response.agg.Utils.handleNanValue; +import static org.opensearch.sql.opensearch.response.agg.Utils.handleNanInfValue; import java.util.Collections; import java.util.Map; @@ -34,6 +34,6 @@ public class SingleValueParser implements MetricParser { public Map parse(Aggregation agg) { return Collections.singletonMap( agg.getName(), - handleNanValue(((NumericMetricsAggregation.SingleValue) agg).value())); + handleNanInfValue(((NumericMetricsAggregation.SingleValue) agg).value())); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java index 6cac2fbdc9..5928b7efc9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java @@ -13,7 +13,7 @@ package org.opensearch.sql.opensearch.response.agg; -import static org.opensearch.sql.opensearch.response.agg.Utils.handleNanValue; +import static org.opensearch.sql.opensearch.response.agg.Utils.handleNanInfValue; import java.util.Collections; import java.util.Map; @@ -36,6 +36,6 @@ public class StatsParser implements MetricParser { @Override public Map parse(Aggregation agg) { return Collections.singletonMap( - agg.getName(), handleNanValue(valueExtractor.apply((ExtendedStats) agg))); + agg.getName(), handleNanInfValue(valueExtractor.apply((ExtendedStats) agg))); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java index 28b9d41e83..953f4d19b4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java @@ -18,10 +18,10 @@ @UtilityClass public class Utils { /** - * Utils to handle Nan Value. - * @return null if is Nan. + * Utils to handle Nan/Infinite Value. + * @return null if is Nan or is +-Infinity. */ - public static Object handleNanValue(double value) { - return Double.isNaN(value) ? null : value; + public static Object handleNanInfValue(double value) { + return Double.isNaN(value) || Double.isInfinite(value) ? null : value; } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java index 5108e03ad9..318110bdde 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java @@ -13,7 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.opensearch.sql.opensearch.response.AggregationResponseUtils.fromJson; -import static org.opensearch.sql.opensearch.response.agg.Utils.handleNanValue; +import static org.opensearch.sql.opensearch.response.agg.Utils.handleNanInfValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -163,7 +163,9 @@ void unsupported_aggregation_should_fail() { @Test void nan_value_should_return_null() { - assertNull(handleNanValue(Double.NaN)); + assertNull(handleNanInfValue(Double.NaN)); + assertNull(handleNanInfValue(Double.NEGATIVE_INFINITY)); + assertNull(handleNanInfValue(Double.POSITIVE_INFINITY)); } @Test From b76c8f2664a9509935d06a59e1dbda19c3182f16 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Thu, 3 Nov 2022 15:35:28 -0700 Subject: [PATCH 19/20] Bump sql-cli version to 1.1.0 (#1023) Signed-off-by: Peng Huo --- sql-cli/src/opensearch_sql_cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql-cli/src/opensearch_sql_cli/__init__.py b/sql-cli/src/opensearch_sql_cli/__init__.py index a9c06a7f3a..770a0c7f59 100644 --- a/sql-cli/src/opensearch_sql_cli/__init__.py +++ b/sql-cli/src/opensearch_sql_cli/__init__.py @@ -3,4 +3,4 @@ SPDX-License-Identifier: Apache-2.0 """ -__version__ = "1.0.0" +__version__ = "1.1.0" From b56edc7b693594b87fe7b0ffd635eff79148e179 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:47:24 -0700 Subject: [PATCH 20/20] Add release note (#1033) (#1034) Signed-off-by: Chen Dai Signed-off-by: Chen Dai (cherry picked from commit 9944f2e9a263ee06e7da99d72cc88f7eb267633f) Co-authored-by: Chen Dai --- .../opensearch-sql.release-notes-2.4.0.0.md | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 release-notes/opensearch-sql.release-notes-2.4.0.0.md diff --git a/release-notes/opensearch-sql.release-notes-2.4.0.0.md b/release-notes/opensearch-sql.release-notes-2.4.0.0.md new file mode 100644 index 0000000000..95b5b3a100 --- /dev/null +++ b/release-notes/opensearch-sql.release-notes-2.4.0.0.md @@ -0,0 +1,82 @@ +Compatible with OpenSearch and OpenSearch Dashboards Version 2.4.0 + +### Features + +#### Data Source Management + +* Catalog Implementation ([#819](https://github.com/opensearch-project/sql/pull/819)) +* Catalog to Datasource changes ([#1027](https://github.com/opensearch-project/sql/pull/1027)) + +#### Prometheus Support + +* Prometheus Connector Initial Code ([#878](https://github.com/opensearch-project/sql/pull/878)) +* Restricted catalog name to [a-zA-Z0-9_-] characters ([#876](https://github.com/opensearch-project/sql/pull/876)) +* Table function for supporting prometheus query_range function ([#875](https://github.com/opensearch-project/sql/pull/875)) +* List tables/metrics using information_schema in source command. ([#914](https://github.com/opensearch-project/sql/pull/914)) +* [Backport 2.4] Prometheus select metric and stats queries. ([#1020](https://github.com/opensearch-project/sql/pull/1020)) + +#### Log Pattern Command + +* Add patterns and grok command ([#813](https://github.com/opensearch-project/sql/pull/813)) + +#### ML Command + +* Add category_field to AD command in PPL ([#952](https://github.com/opensearch-project/sql/pull/952)) +* A Generic ML Command in PPL ([#971](https://github.com/opensearch-project/sql/pull/971)) + +### Enhancements + +* Add datetime functions `FROM_UNIXTIME` and `UNIX_TIMESTAMP` ([#835](https://github.com/opensearch-project/sql/pull/835)) +* Adding `CONVERT_TZ` and `DATETIME` functions to SQL and PPL ([#848](https://github.com/opensearch-project/sql/pull/848)) +* Add Support for Highlight Wildcard in SQL ([#827](https://github.com/opensearch-project/sql/pull/827)) +* Update SQL CLI to use AWS session token. ([#918](https://github.com/opensearch-project/sql/pull/918)) +* Add `typeof` function. ([#867](https://github.com/opensearch-project/sql/pull/867)) +* Show catalogs ([#925](https://github.com/opensearch-project/sql/pull/925)) +* Add functions `PERIOD_ADD` and `PERIOD_DIFF`. ([#933](https://github.com/opensearch-project/sql/pull/933)) +* Add take() aggregation function in PPL ([#949](https://github.com/opensearch-project/sql/pull/949)) +* Describe Table with catalog name. ([#989](https://github.com/opensearch-project/sql/pull/989)) +* Catalog Enhancements ([#988](https://github.com/opensearch-project/sql/pull/988)) +* Rework on error reporting to make it more verbose and human-friendly. ([#839](https://github.com/opensearch-project/sql/pull/839)) + +### Bug Fixes + +* Fix EqualsAndHashCode Annotation Warning Messages ([#847](https://github.com/opensearch-project/sql/pull/847)) +* Remove duplicated png file ([#865](https://github.com/opensearch-project/sql/pull/865)) +* Fix NPE with multiple queries containing DOT(.) in index name. ([#870](https://github.com/opensearch-project/sql/pull/870)) +* Update JDBC driver version ([#941](https://github.com/opensearch-project/sql/pull/941)) +* Fix result order of parse with other run time fields ([#934](https://github.com/opensearch-project/sql/pull/934)) +* AD timefield name issue ([#919](https://github.com/opensearch-project/sql/pull/919)) +* [Backport 2.4] Add function name as identifier in antlr ([#1018](https://github.com/opensearch-project/sql/pull/1018)) +* [Backport 2.4] Fix incorrect results returned by `min`, `max` and `avg` ([#1022](https://github.com/opensearch-project/sql/pull/1022)) + +### Infrastructure + +* Fix failing ODBC workflow ([#828](https://github.com/opensearch-project/sql/pull/828)) +* Reorganize GitHub workflows. ([#837](https://github.com/opensearch-project/sql/pull/837)) +* Update com.fasterxml.jackson to 2.13.4 to match opensearch repo. ([#858](https://github.com/opensearch-project/sql/pull/858)) +* Trigger build on pull request synchronize action. ([#873](https://github.com/opensearch-project/sql/pull/873)) +* Update Jetty Dependency ([#872](https://github.com/opensearch-project/sql/pull/872)) +* Fix manual CI workflow and add `name` option. ([#904](https://github.com/opensearch-project/sql/pull/904)) +* add groupId to pluginzip publication ([#906](https://github.com/opensearch-project/sql/pull/906)) +* Enable ci for windows and macos ([#907](https://github.com/opensearch-project/sql/pull/907)) +* Update group to groupId ([#908](https://github.com/opensearch-project/sql/pull/908)) +* Enable ignored and disabled tests ([#926](https://github.com/opensearch-project/sql/pull/926)) +* Update version of `jackson-databind` for `sql-jdbc` only ([#943](https://github.com/opensearch-project/sql/pull/943)) +* Add security policy for ml-commons library ([#945](https://github.com/opensearch-project/sql/pull/945)) +* Change condition to always upload coverage for linux workbench ([#967](https://github.com/opensearch-project/sql/pull/967)) +* Bump ansi-regex for workbench ([#975](https://github.com/opensearch-project/sql/pull/975)) +* Removed json-smart in the JDBC driver ([#978](https://github.com/opensearch-project/sql/pull/978)) +* Update MacOS Version for ODBC Driver ([#987](https://github.com/opensearch-project/sql/pull/987)) +* Update Jackson Databind version to 2.13.4.2 ([#992](https://github.com/opensearch-project/sql/pull/992)) +* [Backport 2.4] Bump sql-cli version to 1.1.0 ([#1024](https://github.com/opensearch-project/sql/pull/1024)) + +### Documentation + +* Add Forum link in SQL plugin README.md ([#809](https://github.com/opensearch-project/sql/pull/809)) +* Fix indentation of patterns example ([#880](https://github.com/opensearch-project/sql/pull/880)) +* Update docs - missing changes for #754. ([#884](https://github.com/opensearch-project/sql/pull/884)) +* Fix broken links ([#911](https://github.com/opensearch-project/sql/pull/911)) +* Adding docs related to catalog. ([#963](https://github.com/opensearch-project/sql/pull/963)) +* SHOW CATALOGS documentation and integ tests ([#977](https://github.com/opensearch-project/sql/pull/977)) +* [Backport 2.4] Add document for ml command. ([#1017](https://github.com/opensearch-project/sql/pull/1017)) +