Skip to content

Commit

Permalink
ESQL: allow DATE_PARSE to read the timezones (elastic#118603)
Browse files Browse the repository at this point in the history 
This just removes fixing a formatter to a timezone (UTC), allowing
`DATE_PARSE` to correctly read timezones.

Fixes elastic#117680.
  • Loading branch information
@bpintea
bpintea committed Dec 19, 2024
1 parent b7dc70d commit 324f97a
Show file tree
Hide file tree
Showing 6 changed files with 100 additions and 29 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/118603.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 118603
summary: Allow DATE_PARSE to read the timezones
area: ES|QL
type: bug
issues:
- 117680
54 changes: 54 additions & 0 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec
View file
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,60 @@ b:datetime
null
;

evalDateParseWithTimezone
required_capability: date_parse_tz
row s = "12/Jul/2022:10:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);

s:keyword | d:datetime
12/Jul/2022:10:24:10 +0900 | 2022-07-12T01:24:10.000Z
;

evalDateParseWithTimezoneCrossingDayBoundary
required_capability: date_parse_tz
row s = "12/Jul/2022:08:24:10 +0900" | eval d = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s);

s:keyword | d:datetime
12/Jul/2022:08:24:10 +0900 | 2022-07-11T23:24:10.000Z
;

evalDateParseWithTimezone2
required_capability: date_parse_tz
row s1 = "12/Jul/2022:10:24:10 +0900", s2 = "2022/12/07 09:24:10 +0800"
| eval d1 = date_parse("dd/MMM/yyyy:HH:mm:ss Z", s1), d2 = date_parse("yyyy/dd/MM HH:mm:ss Z", s2)
| eval eq = d1 == d2
| keep d1, eq
;

d1:datetime | eq:boolean
2022-07-12T01:24:10.000Z | true
;

evalDateParseWithAndWithoutTimezone
required_capability: date_parse_tz
row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
| eval no_tz = date_parse(format, s)
| eval with_tz = date_parse(concat(format, " Z"), concat(s, " +0900"))
| keep s, no_tz, with_tz
;

s:keyword | no_tz:datetime | with_tz:datetime
2022/12/07 09:24:10 | 2022-07-12T09:24:10.000Z | 2022-07-12T00:24:10.000Z
;

evalDateParseWithOtherTimezoneSpecifiers
required_capability: date_parse_tz
row s = "2022/12/07 09:24:10", format="yyyy/dd/MM HH:mm:ss"
| eval with_tz1 = date_parse(concat(format, " Z"), concat(s, " +0900"))
| eval with_tz2 = date_parse(concat(format, " x"), concat(s, " +09"))
| eval with_tz3 = date_parse(concat(format, " X"), concat(s, " +0900"))
| eval with_tz4 = date_parse(concat(format, " O"), concat(s, " GMT+9"))
| keep s, with_tz*
;

s:keyword | with_tz1:datetime | with_tz2:datetime | with_tz3:datetime | with_tz4:datetime
2022/12/07 09:24:10 | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z | 2022-07-12T00:24:10.000Z
;

evalDateParseDynamic
from employees | where emp_no == 10039 or emp_no == 10040 | sort emp_no
| eval birth_date_string = date_format("yyyy-MM-dd", birth_date)
Expand Down
21 changes: 7 additions & 14 deletions ...ated/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseEvaluator.java
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,11 @@ public enum Cap {
*/
DATE_NANOS_AGGREGATIONS(),

/**
* DATE_PARSE supports reading timezones
*/
DATE_PARSE_TZ(),

/**
* Support for datetime in least and greatest functions
*/
Expand Down
15 changes: 6 additions & 9 deletions ...src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,12 @@
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;

import java.io.IOException;
import java.time.ZoneId;
import java.util.List;

import static org.elasticsearch.common.time.DateFormatter.forPattern;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
import static org.elasticsearch.xpack.esql.core.util.DateUtils.UTC;
import static org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions.isStringAndExact;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong;
Expand Down Expand Up @@ -130,13 +128,12 @@ public static long process(BytesRef val, @Fixed DateFormatter formatter) throws
}

@Evaluator(warnExceptions = { IllegalArgumentException.class })
static long process(BytesRef val, BytesRef formatter, @Fixed ZoneId zoneId) throws IllegalArgumentException {
return dateTimeToLong(val.utf8ToString(), toFormatter(formatter, zoneId));
static long process(BytesRef val, BytesRef formatter) throws IllegalArgumentException {
return dateTimeToLong(val.utf8ToString(), toFormatter(formatter));
}

@Override
public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
ZoneId zone = UTC; // TODO session timezone?
ExpressionEvaluator.Factory fieldEvaluator = toEvaluator.apply(field);
if (format == null) {
return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, DEFAULT_DATE_TIME_FORMATTER);
Expand All @@ -146,18 +143,18 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
}
if (format.foldable()) {
try {
DateFormatter formatter = toFormatter(format.fold(), zone);
DateFormatter formatter = toFormatter(format.fold());
return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter);
} catch (IllegalArgumentException e) {
throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage());
}
}
ExpressionEvaluator.Factory formatEvaluator = toEvaluator.apply(format);
return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator, zone);
return new DateParseEvaluator.Factory(source(), fieldEvaluator, formatEvaluator);
}

private static DateFormatter toFormatter(Object format, ZoneId zone) {
return forPattern(((BytesRef) format).utf8ToString()).withZone(zone);
private static DateFormatter toFormatter(Object format) {
return forPattern(((BytesRef) format).utf8ToString());
}

@Override
Expand Down
28 changes: 22 additions & 6 deletions ...est/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;

import java.util.List;
import java.util.Locale;
import java.util.function.Supplier;

import static org.hamcrest.Matchers.equalTo;
Expand All @@ -46,11 +47,26 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
),
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME,
equalTo(1683244800000L)
)
),
new TestCaseSupplier("Timezoned Case", List.of(DataType.KEYWORD, DataType.KEYWORD), () -> {
long ts_sec = 1657585450L; // 2022-07-12T00:24:10Z
int hours = randomIntBetween(0, 23);
String date = String.format(Locale.ROOT, "12/Jul/2022:%02d:24:10 +0900", hours);
long expected_ts = (ts_sec + (hours - 9) * 3600L) * 1000L;
return new TestCaseSupplier.TestCase(
List.of(
new TestCaseSupplier.TypedData(new BytesRef("dd/MMM/yyyy:HH:mm:ss Z"), DataType.KEYWORD, "first"),
new TestCaseSupplier.TypedData(new BytesRef(date), DataType.KEYWORD, "second")
),
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME,
equalTo(expected_ts)
);
}),
new TestCaseSupplier(
"With Text",
List.of(DataType.KEYWORD, DataType.TEXT),
Expand All @@ -59,7 +75,7 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.KEYWORD, "first"),
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
),
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME,
equalTo(1683244800000L)
)
Expand All @@ -72,7 +88,7 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.TEXT, "second")
),
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME,
equalTo(1683244800000L)
)
Expand All @@ -85,7 +101,7 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier.TypedData(new BytesRef("yyyy-MM-dd"), DataType.TEXT, "first"),
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")
),
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME,
equalTo(1683244800000L)
)
Expand All @@ -98,7 +114,7 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier.TypedData(new BytesRef("2023-05-05"), DataType.KEYWORD, "second")

),
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME,
is(nullValue())
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
Expand All @@ -118,7 +134,7 @@ public static Iterable<Object[]> parameters() {
new TestCaseSupplier.TypedData(new BytesRef("not a date"), DataType.KEYWORD, "second")

),
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], zoneId=Z]",
"DateParseEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0]]",
DataType.DATETIME,
is(nullValue())
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
Expand Down

0 comments on commit 324f97a

Please sign in to comment.