From 0075c1fb1e82c51c747729924ce67efef6daadeb Mon Sep 17 00:00:00 2001 From: Fang Xing <155562079+fang-xing-esql@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:20:12 -0400 Subject: [PATCH] [ES|QL] String literal implicit casting (#106932) * string literal casting for scalar functions and arithmetic operations. --- .../functions/kibana/definition/bucket.json | 240 ------------------ .../esql/functions/types/bucket.asciidoc | 8 - .../src/main/resources/date.csv-spec | 44 ++++ .../src/main/resources/math.csv-spec | 65 +++++ .../src/main/resources/meta.csv-spec | 4 +- .../src/main/resources/string.csv-spec | 33 +++ .../xpack/esql/analysis/Analyzer.java | 124 ++++++++- .../function/EsqlFunctionRegistry.java | 88 ++++++- .../function/scalar/date/DateTrunc.java | 45 ++-- .../function/scalar/math/Bucket.java | 4 +- .../function/scalar/multivalue/MvSlice.java | 15 +- .../function/scalar/multivalue/MvZip.java | 15 +- .../arithmetic/EsqlArithmeticOperation.java | 2 +- .../xpack/esql/plugin/EsqlFeatures.java | 8 +- .../esql/type/EsqlDataTypeConverter.java | 53 +++- .../xpack/esql/analysis/VerifierTests.java | 4 +- .../function/AbstractFunctionTestCase.java | 8 +- .../function/scalar/math/BucketTests.java | 2 +- .../expression/function/FunctionRegistry.java | 1 - 19 files changed, 446 insertions(+), 317 deletions(-) diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index dda3f384424b4..050c334ac7e6e 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -34,246 +34,6 @@ "variadic" : false, "returnType" : "datetime" }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "keyword", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "text", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "keyword", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "datetime", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "keyword", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "keyword", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "keyword", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "text", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "text", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "datetime", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "text", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "keyword", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, - { - "params" : [ - { - "name" : "field", - "type" : "datetime", - "optional" : false, - "description" : "" - }, - { - "name" : "buckets", - "type" : "integer", - "optional" : false, - "description" : "" - }, - { - "name" : "from", - "type" : "text", - "optional" : false, - "description" : "" - }, - { - "name" : "to", - "type" : "text", - "optional" : false, - "description" : "" - } - ], - "variadic" : false, - "returnType" : "datetime" - }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/bucket.asciidoc b/docs/reference/esql/functions/types/bucket.asciidoc index cfe74ae25c3d0..c4b997d0e124d 100644 --- a/docs/reference/esql/functions/types/bucket.asciidoc +++ b/docs/reference/esql/functions/types/bucket.asciidoc @@ -6,14 +6,6 @@ |=== field | buckets | from | to | result datetime | integer | datetime | datetime | datetime -datetime | integer | datetime | keyword | datetime -datetime | integer | datetime | text | datetime -datetime | integer | keyword | datetime | datetime -datetime | integer | keyword | keyword | datetime -datetime | integer | keyword | text | datetime -datetime | integer | text | datetime | datetime -datetime | integer | text | keyword | datetime -datetime | integer | text | text | datetime double | integer | double | double | double double | integer | double | integer | double double | integer | double | long | double diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 5b3b6235ccb8b..57cda24d15fa2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -510,6 +510,17 @@ date1:date | date2:date | dd_ms:integer // end::docsDateDiff-result[] ; +evalDateDiffString +required_feature: esql.string_literal_auto_casting + +ROW date1 = TO_DATETIME("2023-12-02T11:00:00.000Z") +| EVAL dd_ms = DATE_DIFF("microseconds", date1, "2023-12-02T11:00:00.001Z") +; + +date1:date | dd_ms:integer +2023-12-02T11:00:00.000Z | 1000 +; + evalDateParseWithSimpleDate row a = "2023-02-01" | eval b = date_parse("yyyy-MM-dd", a) | keep b; @@ -1085,6 +1096,17 @@ date:date | year:long // end::dateExtract-result[] ; +dateExtractString +required_feature: esql.string_literal_auto_casting + +ROW date = DATE_PARSE("yyyy-MM-dd", "2022-05-06") +| EVAL year = DATE_EXTRACT("year", "2022-05-06") +; + +date:date | year:long +2022-05-06T00:00:00.000Z | 2022 +; + docsDateExtractBusinessHours // tag::docsDateExtractBusinessHours[] FROM sample_data @@ -1115,6 +1137,17 @@ Anneke |Preusig |1989-06-02T00:00:00.000Z|1989-06-02 // end::docsDateFormat-result[] ; +evalDateFormatString +required_feature: esql.string_literal_auto_casting + +ROW a = 1 +| EVAL df = DATE_FORMAT("YYYY-MM-dd", "1989-06-02T00:00:00.000Z") +; + +a:integer | df:keyword +1 | 1989-06-02 +; + docsDateTrunc // tag::docsDateTrunc[] FROM employees @@ -1133,6 +1166,17 @@ Anneke |Preusig |1989-06-02T00:00:00.000Z|1989-01-01T00:00:00.000 // end::docsDateTrunc-result[] ; +evalDateTruncString +required_feature: esql.string_literal_auto_casting + +ROW a = 1 +| EVAL year_hired = DATE_TRUNC(1 year, "1991-06-26T00:00:00.000Z") +; + +a:integer | year_hired:date +1 | 1991-01-01T00:00:00.000Z +; + docsDateTruncHistogram // tag::docsDateTruncHistogram[] FROM employees diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec index 6caeade1af58c..905eac30a3012 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/math.csv-spec @@ -1427,3 +1427,68 @@ Amabile |Gomatam |2.09 |2.09 Anneke |Preusig |1.56 |1.56 //end::abs-employees-result[] ; + +evalAbsString +required_feature: esql.string_literal_auto_casting + +ROW number = -1.0 +| EVAL abs_number = ABS("10.0") +; + +number:double | abs_number:double +-1.0 | 10.0 +; + +arithmeticOperationWithString +required_feature: esql.string_literal_auto_casting + +from employees +| eval s1 = salary + "10000", s2 = height * "2", s3 = avg_worked_seconds / "2", s4 = languages - "1" +| sort emp_no +| keep emp_no, salary, s1, height, s2, avg_worked_seconds, s3, languages, s4 +| limit 2; + +emp_no:integer | salary:integer | s1:integer | height:double | s2:double | avg_worked_seconds:long | s3:long | languages:integer | s4:integer +10001 | 57305 | 67305 | 2.03 | 4.06 | 268728049 | 134364024 | 2 | 1 +10002 | 56371 | 66371 | 2.08 | 4.16 | 328922887 | 164461443 | 5 | 4 +; + +arithmeticOperationNestedWithString +required_feature: esql.string_literal_auto_casting + +from employees +| eval x = languages + "1", y = x * 2 +| sort emp_no +| keep emp_no, languages, x, y +| limit 2; + +emp_no: integer | languages:integer | x:integer | y:integer +10001 | 2 | 3 | 6 +10002 | 5 | 6 | 12 +; + +functionUnderArithmeticOperationAggString +required_feature: esql.string_literal_auto_casting + +ROW a = 1 +| eval x = date_trunc(1 month, "2024-11-22") + 2 days, y = x + 3 days +| stats count() by y +; + +count():long | y:date +1 | 2024-11-06T00:00:00.000Z +; + +functionUnderArithmeticOperationString +required_feature: esql.string_literal_auto_casting + +from employees +| eval x = date_trunc(1 month, "2024-11-22") + 2 days, y = x + 3 days +| sort emp_no +| keep emp_no, x, y +| limit 2; + +emp_no: integer | x:date | y:date +10001 | 2024-11-03 | 2024-11-06 +10002 | 2024-11-03 | 2024-11-06 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 1a154bc6a61fa..cb448dca55596 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -8,7 +8,7 @@ synopsis:keyword "double atan(number:double|integer|long|unsigned_long)" "double atan2(y_coordinate:double|integer|long|unsigned_long, x_coordinate:double|integer|long|unsigned_long)" "double avg(number:double|integer|long)" -"double|date bucket(field:integer|long|double|date, buckets:integer, from:integer|long|double|date|keyword|text, to:integer|long|double|date|keyword|text)" +"double|date bucket(field:integer|long|double|date, buckets:integer, from:integer|long|double|date, to:integer|long|double|date)" "boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version case(condition:boolean, trueValue...:boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version)" "double|integer|long|unsigned_long ceil(number:double|integer|long|unsigned_long)" "boolean cidr_match(ip:ip, blockX...:keyword|text)" @@ -118,7 +118,7 @@ asin |number |"double|integer|long|unsigne atan |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. atan2 |[y_coordinate, x_coordinate] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |[y coordinate. If `null`\, the function returns `null`., x coordinate. If `null`\, the function returns `null`.] avg |number |"double|integer|long" |[""] -bucket |[field, buckets, from, to] |["integer|long|double|date", integer, "integer|long|double|date|keyword|text", "integer|long|double|date|keyword|text"] |["", "", "", ""] +bucket |[field, buckets, from, to] |["integer|long|double|date", integer, "integer|long|double|date", "integer|long|double|date"] |["", "", "", ""] case |[condition, trueValue] |[boolean, "boolean|cartesian_point|date|double|geo_point|integer|ip|keyword|long|text|unsigned_long|version"] |["", ""] ceil |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. cidr_match |[ip, blockX] |[ip, "keyword|text"] |[, CIDR block to test the IP against.] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index 5a81a05cee143..7de8f36e48b01 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -70,6 +70,29 @@ emp_no:integer | last_name:keyword | gender:keyword | f_l:boolean 10010 | Piveteau | null | null ; +stringCast +required_feature: esql.string_literal_auto_casting + +ROW a = 1 | eval ss = substring("abcd", "2"), l = left("abcd", "2"), r = right("abcd", "2"); + +a:integer | ss:keyword | l:keyword | r:keyword +1 | bcd | ab | cd +; + +stringCastEmp +required_feature: esql.string_literal_auto_casting + +from employees +| eval ss = substring(first_name, "2") +| sort emp_no +| keep emp_no, first_name, ss +| limit 2; + +emp_no: integer | first_name:keyword | ss:keyword +10001 | Georgi | eorgi +10002 | Bezalel | ezalel +; + substring from employees | where emp_no <= 10010 | eval f_l = substring(last_name, 3) | keep emp_no, last_name, f_l; ignoreOrder:true @@ -748,6 +771,16 @@ emp_no:integer | job_positions:keyword 10005 | null | null | null ; +mvSliceCast +required_feature: esql.string_literal_auto_casting + +ROW a = ["1", "2", "3", "4"] +| eval a1 = mv_slice(a, "0", "1"); + +a:keyword | a1:keyword +["1", "2", "3", "4"] | ["1", "2"] +; + mvSliceEmp required_feature: esql.mv_sort diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 005dd8081a9e8..13e088b81c95f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -8,12 +8,16 @@ package org.elasticsearch.xpack.esql.analysis; import org.elasticsearch.common.logging.HeaderWarning; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.expression.NamedExpressions; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; +import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.esql.plan.logical.Drop; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsqlAggregate; @@ -24,6 +28,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.stats.FeatureMetric; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules; import org.elasticsearch.xpack.ql.analyzer.AnalyzerRules.BaseAnalyzerRule; @@ -46,6 +51,7 @@ import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.plan.TableIdentifier; @@ -110,7 +116,14 @@ public class Analyzer extends ParameterizedRuleExecutor> rules; static { - var resolution = new Batch<>("Resolution", new ResolveTable(), new ResolveEnrich(), new ResolveFunctions(), new ResolveRefs()); + var resolution = new Batch<>( + "Resolution", + new ResolveTable(), + new ResolveEnrich(), + new ResolveFunctions(), + new ResolveRefs(), + new ImplicitCasting() + ); var finish = new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new PromoteStringsInDateComparisons()); rules = List.of(resolution, finish); } @@ -780,13 +793,13 @@ private static Expression promote(BinaryComparison cmp) { var right = cmp.right(); boolean modified = false; if (left.dataType() == DATETIME) { - if (right.dataType() == KEYWORD && right.foldable()) { + if (right.dataType() == KEYWORD && right.foldable() && ((right instanceof EsqlScalarFunction) == false)) { right = stringToDate(right); modified = true; } } else { if (right.dataType() == DATETIME) { - if (left.dataType() == KEYWORD && left.foldable()) { + if (left.dataType() == KEYWORD && left.foldable() && ((left instanceof EsqlScalarFunction) == false)) { left = stringToDate(left); modified = true; } @@ -825,4 +838,109 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { plan.forEachDown(p -> FeatureMetric.set(p, b)); return b; } + + private static class ImplicitCasting extends ParameterizedRule { + @Override + public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + return plan.transformExpressionsUp( + ScalarFunction.class, + e -> ImplicitCasting.cast(e, (EsqlFunctionRegistry) context.functionRegistry()) + ); + } + + private static Expression cast(ScalarFunction f, EsqlFunctionRegistry registry) { + if (f instanceof EsqlScalarFunction esf) { + return processScalarFunction(esf, registry); + } + + if (f instanceof EsqlArithmeticOperation eao) { + return processArithmeticOperation(eao); + } + + return f; + } + + private static Expression processScalarFunction(EsqlScalarFunction f, EsqlFunctionRegistry registry) { + List args = f.arguments(); + List targetDataTypes = registry.getDataTypeForStringLiteralConversion(f.getClass()); + if (targetDataTypes == null || targetDataTypes.isEmpty()) { + return f; + } + List newChildren = new ArrayList<>(args.size()); + boolean childrenChanged = false; + DataType targetDataType = DataTypes.NULL; + Expression arg; + for (int i = 0; i < args.size(); i++) { + arg = args.get(i); + if (arg.resolved() && arg.dataType() == KEYWORD && arg.foldable() && ((arg instanceof EsqlScalarFunction) == false)) { + if (i < targetDataTypes.size()) { + targetDataType = targetDataTypes.get(i); + } + if (targetDataType != DataTypes.NULL && targetDataType != DataTypes.UNSUPPORTED) { + Expression e = castStringLiteral(arg, targetDataType); + childrenChanged = true; + newChildren.add(e); + continue; + } + } + newChildren.add(args.get(i)); + } + return childrenChanged ? f.replaceChildren(newChildren) : f; + } + + private static Expression processArithmeticOperation(EsqlArithmeticOperation o) { + Expression left = o.left(); + Expression right = o.right(); + if (left.resolved() == false || right.resolved() == false) { + return o; + } + List newChildren = new ArrayList<>(2); + boolean childrenChanged = false; + DataType targetDataType = DataTypes.NULL; + Expression from = Literal.NULL; + + if (left.dataType() == KEYWORD + && left.foldable() + && (right.dataType().isNumeric() || right.dataType() == DATETIME) + && ((left instanceof EsqlScalarFunction) == false)) { + targetDataType = right.dataType(); + from = left; + } + if (right.dataType() == KEYWORD + && right.foldable() + && (left.dataType().isNumeric() || left.dataType() == DATETIME) + && ((right instanceof EsqlScalarFunction) == false)) { + targetDataType = left.dataType(); + from = right; + } + if (from != Literal.NULL) { + Expression e = castStringLiteral(from, targetDataType); + newChildren.add(from == left ? e : left); + newChildren.add(from == right ? e : right); + childrenChanged = true; + } + return childrenChanged ? o.replaceChildren(newChildren) : o; + } + + public static Expression castStringLiteral(Expression from, DataType target) { + assert from.foldable(); + try { + Object to = EsqlDataTypeConverter.convert(from.fold(), target); + return new Literal(from.source(), to, target); + } catch (Exception e) { + String message = LoggerMessageFormat.format( + "Cannot convert string [{}] to [{}], error [{}]", + from.fold(), + target, + e.getMessage() + ); + return new UnsupportedAttribute( + from.source(), + String.valueOf(from.fold()), + new UnsupportedEsField(String.valueOf(from.fold()), from.dataType().typeName()), + message + ); + } + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 037b76801ca75..9fce2c3ddadd3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -102,20 +102,70 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim; import org.elasticsearch.xpack.esql.plan.logical.meta.MetaFunctions; +import org.elasticsearch.xpack.ql.expression.function.Function; import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.ql.session.Configuration; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_POINT; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.CARTESIAN_SHAPE; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_POINT; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.GEO_SHAPE; +import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN; +import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; +import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; +import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; +import static org.elasticsearch.xpack.ql.type.DataTypes.IP; +import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED; +import static org.elasticsearch.xpack.ql.type.DataTypes.VERSION; public final class EsqlFunctionRegistry extends FunctionRegistry { + private static final Map, List> dataTypesForStringLiteralConversion = new LinkedHashMap<>(); + + private static final Map dataTypeCastingPriority; + + static { + List typePriorityList = Arrays.asList( + DATETIME, + DOUBLE, + LONG, + INTEGER, + IP, + VERSION, + GEO_POINT, + GEO_SHAPE, + CARTESIAN_POINT, + CARTESIAN_SHAPE, + BOOLEAN, + UNSIGNED_LONG, + UNSUPPORTED + ); + dataTypeCastingPriority = new HashMap<>(); + for (int i = 0; i < typePriorityList.size(); i++) { + dataTypeCastingPriority.put(typePriorityList.get(i), i); + } + } + public EsqlFunctionRegistry() { register(functions()); + buildDataTypesForStringLiteralConversion(functions()); } EsqlFunctionRegistry(FunctionDefinition... functions) { @@ -246,7 +296,7 @@ public static String normalizeName(String name) { return name.toLowerCase(Locale.ROOT); } - public record ArgSignature(String name, String[] type, String description, boolean optional) { + public record ArgSignature(String name, String[] type, String description, boolean optional, DataType targetDataType) { @Override public String toString() { return "ArgSignature{" @@ -258,6 +308,8 @@ public String toString() { + description + "', optional=" + optional + + ", targetDataType=" + + targetDataType + '}'; } } @@ -310,6 +362,20 @@ public List argDescriptions() { } } + public static DataType getTargetType(String[] names) { + List types = new ArrayList<>(); + for (String name : names) { + types.add(DataTypes.fromEs(name)); + } + if (types.contains(KEYWORD) || types.contains(TEXT)) { + return UNSUPPORTED; + } + + return types.stream() + .min((dt1, dt2) -> dataTypeCastingPriority.get(dt1).compareTo(dataTypeCastingPriority.get(dt2))) + .orElse(UNSUPPORTED); + } + public static FunctionDescription description(FunctionDefinition def) { var constructors = def.clazz().getConstructors(); if (constructors.length == 0) { @@ -332,8 +398,8 @@ public static FunctionDescription description(FunctionDefinition def) { String[] type = paramInfo == null ? new String[] { "?" } : paramInfo.type(); String desc = paramInfo == null ? "" : paramInfo.description().replace('\n', ' '); boolean optional = paramInfo == null ? false : paramInfo.optional(); - - args.add(new EsqlFunctionRegistry.ArgSignature(name, type, desc, optional)); + DataType targetDataType = getTargetType(type); + args.add(new EsqlFunctionRegistry.ArgSignature(name, type, desc, optional, targetDataType)); } } return new FunctionDescription(def.name(), args, returnType, functionDescription, variadic, isAggregation); @@ -347,4 +413,20 @@ public static FunctionInfo functionInfo(FunctionDefinition def) { Constructor constructor = constructors[0]; return constructor.getAnnotation(FunctionInfo.class); } + + private void buildDataTypesForStringLiteralConversion(FunctionDefinition[]... groupFunctions) { + for (FunctionDefinition[] group : groupFunctions) { + for (FunctionDefinition def : group) { + FunctionDescription signature = description(def); + dataTypesForStringLiteralConversion.put( + def.clazz(), + signature.args().stream().map(EsqlFunctionRegistry.ArgSignature::targetDataType).collect(Collectors.toList()) + ); + } + } + } + + public List getDataTypeForStringLiteralConversion(Class clazz) { + return dataTypesForStringLiteralConversion.get(clazz); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java index 39ad0351b199f..e2b55fe8a677b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java @@ -12,19 +12,22 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.Duration; import java.time.Period; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -33,7 +36,10 @@ import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isDate; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isType; -public class DateTrunc extends BinaryDateTimeFunction implements EvaluatorMapper { +public class DateTrunc extends EsqlScalarFunction { + private final Expression interval; + private final Expression timestampField; + protected static final ZoneId DEFAULT_TZ = ZoneOffset.UTC; @FunctionInfo( returnType = "date", @@ -59,7 +65,9 @@ public DateTrunc( ) Expression interval, @Param(name = "date", type = { "date" }, description = "Date expression") Expression field ) { - super(source, interval, field); + super(source, List.of(interval, field)); + this.interval = interval; + this.timestampField = field; } @Override @@ -68,14 +76,13 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - return isType(interval(), EsqlDataTypes::isTemporalAmount, sourceText(), FIRST, "dateperiod", "timeduration").and( - isDate(timestampField(), sourceText(), SECOND) + return isType(interval, EsqlDataTypes::isTemporalAmount, sourceText(), FIRST, "dateperiod", "timeduration").and( + isDate(timestampField, sourceText(), SECOND) ); } - @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public DataType dataType() { + return DataTypes.DATETIME; } @Evaluator @@ -84,17 +91,18 @@ static long process(long fieldVal, @Fixed Rounding.Prepared rounding) { } @Override - protected BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight) { - return new DateTrunc(source(), newLeft, newRight); + public Expression replaceChildren(List newChildren) { + return new DateTrunc(source(), newChildren.get(0), newChildren.get(1)); } @Override protected NodeInfo info() { - return NodeInfo.create(this, DateTrunc::new, interval(), timestampField()); + return NodeInfo.create(this, DateTrunc::new, children().get(0), children().get(1)); } - public Expression interval() { - return left(); + @Override + public boolean foldable() { + return interval.foldable() && timestampField.foldable(); } static Rounding.Prepared createRounding(final Object interval) { @@ -159,10 +167,9 @@ private static Rounding.Prepared createRounding(final Duration duration, final Z @Override public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { - var fieldEvaluator = toEvaluator.apply(timestampField()); - Expression interval = interval(); + var fieldEvaluator = toEvaluator.apply(timestampField); if (interval.foldable() == false) { - throw new IllegalArgumentException("Function [" + sourceText() + "] has invalid interval [" + interval().sourceText() + "]."); + throw new IllegalArgumentException("Function [" + sourceText() + "] has invalid interval [" + interval.sourceText() + "]."); } Object foldedInterval; try { @@ -172,10 +179,10 @@ public ExpressionEvaluator.Factory toEvaluator(Function newChildren) { return new MvSlice(source(), newChildren.get(0), newChildren.get(1), newChildren.size() > 2 ? newChildren.get(2) : null); @@ -190,11 +184,6 @@ public DataType dataType() { return field.dataType(); } - @Override - public ScriptTemplate asScript() { - throw new UnsupportedOperationException("functions do not support scripting"); - } - @Override public int hashCode() { return Objects.hash(field, start, end); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java index 88e006b1dfd8d..a15b0a3d6eab0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java @@ -15,11 +15,10 @@ import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.function.OptionalArgument; -import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; -import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -38,7 +37,7 @@ /** * Combines the values from two multivalued fields with a delimiter that joins them together. */ -public class MvZip extends ScalarFunction implements OptionalArgument, EvaluatorMapper { +public class MvZip extends EsqlScalarFunction implements OptionalArgument, EvaluatorMapper { private final Expression mvLeft, mvRight, delim; private static final Literal COMMA = new Literal(Source.EMPTY, ",", DataTypes.TEXT); @@ -96,11 +95,6 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator( return new MvZipEvaluator.Factory(source(), toEvaluator.apply(mvLeft), toEvaluator.apply(mvRight), toEvaluator.apply(delim)); } - @Override - public Object fold() { - return EvaluatorMapper.super.fold(); - } - @Override public Expression replaceChildren(List newChildren) { return new MvZip(source(), newChildren.get(0), newChildren.get(1), newChildren.size() > 2 ? newChildren.get(2) : null); @@ -116,11 +110,6 @@ public DataType dataType() { return DataTypes.KEYWORD; } - @Override - public ScriptTemplate asScript() { - throw new UnsupportedOperationException("functions do not support scripting"); - } - @Override public int hashCode() { return Objects.hash(mvLeft, mvRight, delim); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 6e44a571cfe83..22f5798e5b1c4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -29,7 +29,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; -abstract class EsqlArithmeticOperation extends ArithmeticOperation implements EvaluatorMapper { +public abstract class EsqlArithmeticOperation extends ArithmeticOperation implements EvaluatorMapper { /** * The only role of this enum is to fit the super constructor that expects a BinaryOperation which is diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java index 192c011c4494b..93c0a5946cdf5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlFeatures.java @@ -100,6 +100,11 @@ public class EsqlFeatures implements FeatureSpecification { */ public static final NodeFeature FROM_OPTIONS = new NodeFeature("esql.from_options"); + /** + * Cast string literals to a desired data type. + */ + public static final NodeFeature STRING_LITERAL_AUTO_CASTING = new NodeFeature("esql.string_literal_auto_casting"); + @Override public Set getFeatures() { return Set.of( @@ -114,7 +119,8 @@ public Set getFeatures() { ST_CENTROID_AGG, ST_INTERSECTS, ST_CONTAINS_WITHIN, - ST_DISJOINT + ST_DISJOINT, + STRING_LITERAL_AUTO_CASTING ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 82e7fc2e9fc88..386dbd50dc9ba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.ql.type.Converter; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypeConverter; +import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.NumericUtils; import org.elasticsearch.xpack.ql.util.StringUtils; import org.elasticsearch.xpack.versionfield.Version; @@ -69,16 +70,42 @@ public static boolean canConvert(DataType from, DataType to) { public static Converter converterFor(DataType from, DataType to) { // TODO move EXPRESSION_TO_LONG here if there is no regression + if (isString(from)) { + if (to == DataTypes.DATETIME) { + return EsqlConverter.STRING_TO_DATETIME; + } + if (to == DataTypes.IP) { + return EsqlConverter.STRING_TO_IP; + } + if (to == DataTypes.VERSION) { + return EsqlConverter.STRING_TO_VERSION; + } + if (to == DataTypes.DOUBLE) { + return EsqlConverter.STRING_TO_DOUBLE; + } + if (to == DataTypes.LONG) { + return EsqlConverter.STRING_TO_LONG; + } + if (to == DataTypes.INTEGER) { + return EsqlConverter.STRING_TO_INT; + } + if (to == DataTypes.BOOLEAN) { + return EsqlConverter.STRING_TO_BOOLEAN; + } + if (EsqlDataTypes.isSpatial(to)) { + return EsqlConverter.STRING_TO_SPATIAL; + } + if (to == EsqlDataTypes.TIME_DURATION) { + return EsqlConverter.STRING_TO_TIME_DURATION; + } + if (to == EsqlDataTypes.DATE_PERIOD) { + return EsqlConverter.STRING_TO_DATE_PERIOD; + } + } Converter converter = DataTypeConverter.converterFor(from, to); if (converter != null) { return converter; } - if (isString(from) && to == EsqlDataTypes.TIME_DURATION) { - return EsqlConverter.STRING_TO_TIME_DURATION; - } - if (isString(from) && to == EsqlDataTypes.DATE_PERIOD) { - return EsqlConverter.STRING_TO_DATE_PERIOD; - } return null; } @@ -215,6 +242,10 @@ public static BytesRef stringToVersion(BytesRef field) { return new Version(field.utf8ToString()).toBytesRef(); } + public static Version stringToVersion(String field) { + return new Version(field); + } + public static String versionToString(BytesRef field) { return new Version(field).toString(); } @@ -349,7 +380,15 @@ public enum EsqlConverter implements Converter { STRING_TO_DATE_PERIOD(x -> EsqlDataTypeConverter.parseTemporalAmount(x, EsqlDataTypes.DATE_PERIOD)), STRING_TO_TIME_DURATION(x -> EsqlDataTypeConverter.parseTemporalAmount(x, EsqlDataTypes.TIME_DURATION)), - STRING_TO_CHRONO_FIELD(EsqlDataTypeConverter::stringToChrono); + STRING_TO_CHRONO_FIELD(EsqlDataTypeConverter::stringToChrono), + STRING_TO_DATETIME(x -> EsqlDataTypeConverter.dateTimeToLong((String) x)), + STRING_TO_IP(x -> EsqlDataTypeConverter.stringToIP((String) x)), + STRING_TO_VERSION(x -> EsqlDataTypeConverter.stringToVersion((String) x)), + STRING_TO_DOUBLE(x -> EsqlDataTypeConverter.stringToDouble((String) x)), + STRING_TO_LONG(x -> EsqlDataTypeConverter.stringToLong((String) x)), + STRING_TO_INT(x -> EsqlDataTypeConverter.stringToInt((String) x)), + STRING_TO_BOOLEAN(x -> EsqlDataTypeConverter.stringToBoolean((String) x)), + STRING_TO_SPATIAL(x -> EsqlDataTypeConverter.stringToSpatial((String) x)); private static final String NAME = "esql-converter"; private final Function converter; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index d5d82207a770e..e558dbe615642 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -57,8 +57,8 @@ public void testRoundFunctionInvalidInputs() { error("row a = 1, b = \"c\" | eval x = round(a, 3.5)") ); assertEquals( - "1:9: second argument of [round(123.45, \"1\")] must be [integer], found value [\"1\"] type [keyword]", - error("row a = round(123.45, \"1\")") + "1:23: Cannot convert string [c] to [INTEGER], error [Cannot parse number [c]]", + error("row a = round(123.45, \"c\")") ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index bc7a67d9eaefa..4450773ef6139 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -1073,7 +1073,13 @@ public static void renderDocs() throws IOException { args = List.of( args.get(0), falseValue, - new EsqlFunctionRegistry.ArgSignature("falseValue", falseValue.type(), falseValue.description(), true) + new EsqlFunctionRegistry.ArgSignature( + "falseValue", + falseValue.type(), + falseValue.description(), + true, + EsqlFunctionRegistry.getTargetType(falseValue.type()) + ) ); } renderKibanaFunctionDefinition(name, info, args, description.variadic()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java index 23122863b95f3..dbb178e08bce5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/BucketTests.java @@ -53,7 +53,7 @@ public static Iterable parameters() { } // TODO once we cast above the functions we can drop these - private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataTypes.DATETIME, DataTypes.KEYWORD, DataTypes.TEXT }; + private static final DataType[] DATE_BOUNDS_TYPE = new DataType[] { DataTypes.DATETIME }; private static void dateCases(List suppliers, String name, LongSupplier date) { for (DataType fromType : DATE_BOUNDS_TYPE) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java index 676681aa2a626..eb664fdc70e83 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java @@ -460,5 +460,4 @@ protected static Boolean asBool(Object[] extras) { } return (Boolean) extras[0]; } - }