ESQL: allow DATE_PARSE to read the timezones (#118603) (#119119)

This just removes fixing a formatter to a timezone (UTC), allowing
`DATE_PARSE` to correctly read timezones.

Fixes #117680.

docs/changelog/118603.yaml

@@ -0,0 +1,6 @@
+pr: 118603
+summary: Allow DATE_PARSE to read the timezones
+area: ES|QL
+type: bug
+issues:
+ - 117680

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec

@@ -495,6 +495,60 @@ b:datetime
 null
 ;
 
+evalDateParseWithTimezone
+required_capability: date_parse_tz
+row s = "12/Jul/2022:10:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
+
+s:keyword                  | d:datetime
+12/Jul/2022:10:24:10 +0900 | 2022-07-12T01:24:10.000Z
+;
+
+evalDateParseWithTimezoneCrossingDayBoundary
+required_capability: date_parse_tz
+row s = "12/Jul/2022:08:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);
+
+s:keyword                  | d:datetime
+12/Jul/2022:08:24:10 +0900 | 2022-07-11T23:24:10.000Z
+;
+
+evalDateParseWithTimezone2
+required_capability: date_parse_tz
+row s1 = "12/Jul/2022:10:24:10 +0900", s2 = "2022/12/07 09:24:10 +0800"
+| eval d1 = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s1), d2 = date_parse("yyyy/dd/MM HH:mm:ss Z", s2)
+| eval eq = d1 == d2
+| keep d1, eq
+;
+
+d1:datetime              | eq:boolean
+2022-07-12T01:24:10.000Z | true
+;
+
+evalDateParseWithAndWithoutTimezone
+required_capability: date_parse_tz
+row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
+| eval no_tz = date_parse(format, s)
+| eval with_tz = date_parse(concat(format, " Z"), concat(s, " +0900"))
+| keep s, no_tz, with_tz
+;
+
+s:keyword           | no_tz:datetime           | with_tz:datetime
+2022/12/07 09:24:10 | 2022-07-12T09:24:10.000Z | 2022-07-12T00:24:10.000Z
+;
+
+evalDateParseWithOtherTimezoneSpecifiers
+required_capability: date_parse_tz
+row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
+| eval with_tz1 = date_parse(concat(format, " Z"), concat(s, " +0900"))
+| eval with_tz2 = date_parse(concat(format, " x"), concat(s, " +09"))
+| eval with_tz3 = date_parse(concat(format, " X"), concat(s, " +0900"))
+| eval with_tz4 = date_parse(concat(format, " O"), concat(s, " GMT+9"))
+| keep s, with_tz*
+;
+
+s:keyword           | with_tz1:datetime        | with_tz2:datetime        | with_tz3:datetime        | with_tz4:datetime        
+2022/12/07 09:24:10 | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z
+;
+
 evalDateParseDynamic
 from employees | where emp_no == 10039 or emp_no == 10040 | sort emp_no 
 | eval birth_date_string = date_format("yyyy-MM-dd", birth_date)

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -307,6 +307,11 @@ public enum Cap {
          */
         TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
 
+        /**
+         * DATE_PARSE supports reading timezones
+         */
+        DATE_PARSE_TZ(),
+
         /**
          * Support for datetime in least and greatest functions
          */

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java

@@ -28,14 +28,12 @@
 import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
 
 import java.io.IOException;
-import java.time.ZoneId;
 import java.util.List;
 
 import static org.elasticsearch.common.time.DateFormatter.forPattern;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
 import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
-import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC;
 import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
@@ -130,13 +128,12 @@ public static long process(BytesRef val, @Fixed DateFormatter formatter) throws
     }
 
     @Evaluator(warnExceptions = { IllegalArgumentException.class })
-    static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) throws IllegalArgumentException {
-        return dateTimeToLong(val.utf8ToString(), toFormatter(formatter, zoneId));
+    static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentException {
+        return dateTimeToLong(val.utf8ToString(), toFormatter(formatter));
     }
 
     @Override
     public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
-        ZoneId zone = UTC; // TODO session timezone?
         ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field);
         if (format == null) {
             return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER);
@@ -146,18 +143,18 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
         }
         if (format.foldable()) {
             try {
-                DateFormatter formatter = toFormatter(format.fold(), zone);
+                DateFormatter formatter = toFormatter(format.fold());
                 return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
             } catch (IllegalArgumentException e) {
                 throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage());
             }
         }
         ExpressionEvaluator.Factory formatEvaluator = toEvaluator.apply(format);
-        return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, zone);
+        return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator);
     }
 
-    private static DateFormatter toFormatter(Object format, ZoneId zone) {
-        return forPattern(((BytesRef) format).utf8ToString()).withZone(zone);
+    private static DateFormatter toFormatter(Object format) {
+        return forPattern(((BytesRef) format).utf8ToString());
     }
 
     @Override

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -46,11 +47,26 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
                 ),
+                new TestCaseSupplier("Timezoned Case", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
+                    long ts_sec = 1657585450L; // 2022-07-12T00:24:10Z
+                    int hours = randomIntBetween(0, 23);
+                    String date = String.format(Locale.ROOT, "12/Jul/2022:%02d:24:10 +0900", hours);
+                    long expected_ts = (ts_sec + (hours - 9) * 3600L) * 1000L;
+                    return new TestCaseSupplier.TestCase(
+                        List.of(
+                            new TestCaseSupplier.TypedData(new BytesRef("dd/MMM/yyyy:HH:mm:ss Z"), DataType.KEYWORD, "first"),
+                            new TestCaseSupplier.TypedData(new BytesRef(date), DataType.KEYWORD, "second")
+                        ),
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
+                        DataType.DATETIME,
+                        equalTo(expected_ts)
+                    );
+                }),
                 new TestCaseSupplier(
                     "With Text",
                     List.of(DataType.KEYWORD, DataType.TEXT),
@@ -59,7 +75,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -72,7 +88,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -85,7 +101,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         equalTo(1683244800000L)
                     )
@@ -98,7 +114,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
 
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         is(nullValue())
                     ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
@@ -118,7 +134,7 @@ public static Iterable<Object[]> parameters() {
                             new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second")
 
                         ),
-                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
+                        "DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
                         DataType.DATETIME,
                         is(nullValue())
                     ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")