Skip to content

Commit

Permalink
Add datetime functions FROM_UNIXTIME and UNIX_TIMESTAMP (opensear…
Browse files Browse the repository at this point in the history 
…ch-project#835)

* Add datetime functions `FROM_UNIXTIME` and `UNIX_TIMESTAMP` (#114)

* Add implementation for `FROM_UNIXTIME` and `UNIX_TIMESTAMP` functions, UT and IT.

Signed-off-by: Yury-Fridlyand <[email protected]>

* Collent all DateTime formatters into one place.

Signed-off-by: Yury-Fridlyand <[email protected]>

* Rename `DateFormatters` -> `DateTimeFormatters`.

Signed-off-by: Yury-Fridlyand <[email protected]>

Signed-off-by: Yury-Fridlyand <[email protected]>
  • Loading branch information
@Yury-Fridlyand
Yury-Fridlyand authored Sep 27, 2022
1 parent 8245943 commit e04d6f8
Show file tree
Hide file tree
Showing 21 changed files with 1,064 additions and 154 deletions.
21 changes: 3 additions & 18 deletions core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@

package org.opensearch.sql.data.model;

import static org.opensearch.sql.utils.DateTimeFormatters.TIME_FORMATTER_VARIABLE_NANOS;

import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import org.opensearch.sql.data.type.ExprCoreType;
Expand All @@ -24,27 +24,12 @@
public class ExprTimeValue extends AbstractExprValue {
private final LocalTime time;

private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 9;

static {
FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder()
.appendPattern("HH:mm:ss")
.appendFraction(
ChronoField.NANO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
.toFormatter();
}

/**
* Constructor.
*/
public ExprTimeValue(String time) {
try {
this.time = LocalTime.parse(time, FORMATTER_VARIABLE_NANOS);
this.time = LocalTime.parse(time, TIME_FORMATTER_VARIABLE_NANOS);
} catch (DateTimeParseException e) {
throw new SemanticCheckException(String.format("time:%s in unsupported format, please use "
+ "HH:mm:ss[.SSSSSSSSS]", time));
Expand Down
33 changes: 7 additions & 26 deletions core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@

package org.opensearch.sql.data.model;

import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS;
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_WITHOUT_NANO;

import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
Expand All @@ -31,34 +31,15 @@ public class ExprTimestampValue extends AbstractExprValue {
* todo. only support UTC now.
*/
private static final ZoneId ZONE = ZoneId.of("UTC");
/**
* todo. only support timestamp in format yyyy-MM-dd HH:mm:ss.
*/
private static final DateTimeFormatter FORMATTER_WITHOUT_NANO = DateTimeFormatter
.ofPattern("yyyy-MM-dd HH:mm:ss");
private final Instant timestamp;

private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 9;

static {
FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(
ChronoField.NANO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
.toFormatter();
}
private final Instant timestamp;

/**
* Constructor.
*/
public ExprTimestampValue(String timestamp) {
try {
this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_NANOS)
this.timestamp = LocalDateTime.parse(timestamp, DATE_TIME_FORMATTER_VARIABLE_NANOS)
.atZone(ZONE)
.toInstant();
} catch (DateTimeParseException e) {
Expand All @@ -70,9 +51,9 @@ public ExprTimestampValue(String timestamp) {

@Override
public String value() {
return timestamp.getNano() == 0 ? FORMATTER_WITHOUT_NANO.withZone(ZONE)
return timestamp.getNano() == 0 ? DATE_TIME_FORMATTER_WITHOUT_NANO.withZone(ZONE)
.format(timestamp.truncatedTo(ChronoUnit.SECONDS))
: FORMATTER_VARIABLE_NANOS.withZone(ZONE).format(timestamp);
: DATE_TIME_FORMATTER_VARIABLE_NANOS.withZone(ZONE).format(timestamp);
}

@Override
Expand Down
136 changes: 136 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,38 @@
import static org.opensearch.sql.expression.function.FunctionDSL.define;
import static org.opensearch.sql.expression.function.FunctionDSL.impl;
import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_LONG_YEAR;
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_FORMATTER_SHORT_YEAR;
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_LONG_YEAR;
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_SHORT_YEAR;

import java.math.BigDecimal;
import java.math.RoundingMode;
import java.text.DecimalFormat;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.format.TextStyle;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import lombok.experimental.UtilityClass;
import org.opensearch.sql.data.model.ExprDateValue;
import org.opensearch.sql.data.model.ExprDatetimeValue;
import org.opensearch.sql.data.model.ExprDoubleValue;
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprLongValue;
import org.opensearch.sql.data.model.ExprNullValue;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprTimeValue;
import org.opensearch.sql.data.model.ExprTimestampValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
import org.opensearch.sql.expression.function.DefaultFunctionResolver;
Expand All @@ -56,6 +67,10 @@ public class DateTimeFunction {
// The number of days from year zero to year 1970.
private static final Long DAYS_0000_TO_1970 = (146097 * 5L) - (30L * 365L + 7L);

// MySQL doesn't process any datetime/timestamp values which are greater than
// 32536771199.999999, or equivalent '3001-01-18 23:59:59.999999' UTC
private static final Double MYSQL_MAX_TIMESTAMP = 32536771200d;

/**
* Register Date and Time Functions.
*
Expand All @@ -72,6 +87,7 @@ public void register(BuiltinFunctionRepository repository) {
repository.register(dayOfWeek());
repository.register(dayOfYear());
repository.register(from_days());
repository.register(from_unixtime());
repository.register(hour());
repository.register(makedate());
repository.register(maketime());
Expand All @@ -87,6 +103,7 @@ public void register(BuiltinFunctionRepository repository) {
repository.register(timestamp());
repository.register(date_format());
repository.register(to_days());
repository.register(unix_timestamp());
repository.register(week());
repository.register(year());

Expand Down Expand Up @@ -313,6 +330,13 @@ private DefaultFunctionResolver from_days() {
impl(nullMissingHandling(DateTimeFunction::exprFromDays), DATE, LONG));
}

private FunctionResolver from_unixtime() {
return define(BuiltinFunctionName.FROM_UNIXTIME.getName(),
impl(nullMissingHandling(DateTimeFunction::exprFromUnixTime), DATETIME, DOUBLE),
impl(nullMissingHandling(DateTimeFunction::exprFromUnixTimeFormat),
STRING, DOUBLE, STRING));
}

/**
* HOUR(STRING/TIME/DATETIME/TIMESTAMP). return the hour value for time.
*/
Expand Down Expand Up @@ -461,6 +485,16 @@ private DefaultFunctionResolver to_days() {
impl(nullMissingHandling(DateTimeFunction::exprToDays), LONG, DATETIME));
}

private FunctionResolver unix_timestamp() {
return define(BuiltinFunctionName.UNIX_TIMESTAMP.getName(),
impl(DateTimeFunction::unixTimeStamp, LONG),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATE),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DATETIME),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, TIMESTAMP),
impl(nullMissingHandling(DateTimeFunction::unixTimeStampOf), DOUBLE, DOUBLE)
);
}

/**
* WEEK(DATE[,mode]). return the week number for date.
*/
Expand Down Expand Up @@ -601,6 +635,35 @@ private ExprValue exprFromDays(ExprValue exprValue) {
return new ExprDateValue(LocalDate.ofEpochDay(exprValue.longValue() - DAYS_0000_TO_1970));
}

private ExprValue exprFromUnixTime(ExprValue time) {
if (0 > time.doubleValue()) {
return ExprNullValue.of();
}
// According to MySQL documentation:
// effective maximum is 32536771199.999999, which returns '3001-01-18 23:59:59.999999' UTC.
// Regardless of platform or version, a greater value for first argument than the effective
// maximum returns 0.
if (MYSQL_MAX_TIMESTAMP <= time.doubleValue()) {
return ExprNullValue.of();
}
return new ExprDatetimeValue(exprFromUnixTimeImpl(time));
}

private LocalDateTime exprFromUnixTimeImpl(ExprValue time) {
return LocalDateTime.ofInstant(
Instant.ofEpochSecond((long)Math.floor(time.doubleValue())),
ZoneId.of("UTC"))
.withNano((int)((time.doubleValue() % 1) * 1E9));
}

private ExprValue exprFromUnixTimeFormat(ExprValue time, ExprValue format) {
var value = exprFromUnixTime(time);
if (value.equals(ExprNullValue.of())) {
return ExprNullValue.of();
}
return DateTimeFormatterUtil.getFormattedDate(value, format);
}

/**
* Hour implementation for ExprValue.
*
Expand Down Expand Up @@ -803,6 +866,79 @@ private ExprValue exprWeek(ExprValue date, ExprValue mode) {
CalendarLookup.getWeekNumber(mode.integerValue(), date.dateValue()));
}

private ExprValue unixTimeStamp() {
return new ExprLongValue(Instant.now().getEpochSecond());
}

private ExprValue unixTimeStampOf(ExprValue value) {
var res = unixTimeStampOfImpl(value);
if (res == null) {
return ExprNullValue.of();
}
if (res < 0) {
// According to MySQL returns 0 if year < 1970, don't return negative values as java does.
return new ExprDoubleValue(0);
}
if (res >= MYSQL_MAX_TIMESTAMP) {
// Return 0 also for dates > '3001-01-19 03:14:07.999999' UTC (32536771199.999999 sec)
return new ExprDoubleValue(0);
}
return new ExprDoubleValue(res);
}

private Double unixTimeStampOfImpl(ExprValue value) {
// Also, according to MySQL documentation:
// The date argument may be a DATE, DATETIME, or TIMESTAMP ...
switch ((ExprCoreType)value.type()) {
case DATE: return value.dateValue().toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
case DATETIME: return value.datetimeValue().toEpochSecond(ZoneOffset.UTC)
+ value.datetimeValue().getNano() / 1E9;
case TIMESTAMP: return value.timestampValue().getEpochSecond()
+ value.timestampValue().getNano() / 1E9;
default:
// ... or a number in YYMMDD, YYMMDDhhmmss, YYYYMMDD, or YYYYMMDDhhmmss format.
// If the argument includes a time part, it may optionally include a fractional
// seconds part.

var format = new DecimalFormat("0.#");
format.setMinimumFractionDigits(0);
format.setMaximumFractionDigits(6);
String input = format.format(value.doubleValue());
double fraction = 0;
if (input.contains(".")) {
// Keeping fraction second part and adding it to the result, don't parse it
// Because `toEpochSecond` returns only `long`
// input = 12345.6789 becomes input = 12345 and fraction = 0.6789
fraction = value.doubleValue() - Math.round(Math.ceil(value.doubleValue()));
input = input.substring(0, input.indexOf('.'));
}
try {
var res = LocalDateTime.parse(input, DATE_TIME_FORMATTER_SHORT_YEAR);
return res.toEpochSecond(ZoneOffset.UTC) + fraction;
} catch (DateTimeParseException ignored) {
// nothing to do, try another format
}
try {
var res = LocalDateTime.parse(input, DATE_TIME_FORMATTER_LONG_YEAR);
return res.toEpochSecond(ZoneOffset.UTC) + fraction;
} catch (DateTimeParseException ignored) {
// nothing to do, try another format
}
try {
var res = LocalDate.parse(input, DATE_FORMATTER_SHORT_YEAR);
return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
} catch (DateTimeParseException ignored) {
// nothing to do, try another format
}
try {
var res = LocalDate.parse(input, DATE_FORMATTER_LONG_YEAR);
return res.toEpochSecond(LocalTime.MIN, ZoneOffset.UTC) + 0d;
} catch (DateTimeParseException ignored) {
return null;
}
}
}

/**
* Week for date implementation for ExprValue.
* When mode is not specified default value mode 0 is used for default_week_format.
Expand Down
2 changes: 2 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ public enum BuiltinFunctionName {
DAYOFWEEK(FunctionName.of("dayofweek")),
DAYOFYEAR(FunctionName.of("dayofyear")),
FROM_DAYS(FunctionName.of("from_days")),
FROM_UNIXTIME(FunctionName.of("from_unixtime")),
HOUR(FunctionName.of("hour")),
MAKEDATE(FunctionName.of("makedate")),
MAKETIME(FunctionName.of("maketime")),
Expand All @@ -82,6 +83,7 @@ public enum BuiltinFunctionName {
TIMESTAMP(FunctionName.of("timestamp")),
DATE_FORMAT(FunctionName.of("date_format")),
TO_DAYS(FunctionName.of("to_days")),
UNIX_TIMESTAMP(FunctionName.of("unix_timestamp")),
WEEK(FunctionName.of("week")),
YEAR(FunctionName.of("year")),
// `now`-like functions
Expand Down
Loading

0 comments on commit e04d6f8

Please sign in to comment.