Skip to content

Commit

Permalink
Support date_nanos OpenSearch field type (#360)
Browse files Browse the repository at this point in the history 
* Convert date_nanos to timestamp

Signed-off-by: Joshua Li <[email protected]>

* Use nano seconds precision for timestamp

Signed-off-by: Joshua Li <[email protected]>

* Update unit tests

Signed-off-by: Joshua Li <[email protected]>

* Update docs for date_nanos

Signed-off-by: Joshua Li <[email protected]>
  • Loading branch information
@joshuali925
joshuali925 authored Jan 5, 2022
1 parent 3ffa1ca commit 6ec7b9d
Show file tree
Hide file tree
Showing 8 changed files with 89 additions and 84 deletions.
12 changes: 6 additions & 6 deletions core/src/main/java/org/opensearch/sql/data/model/ExprDatetimeValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@
public class ExprDatetimeValue extends AbstractExprValue {
private final LocalDateTime datetime;

private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 6;
private static final int MAX_FRACTION_SECONDS = 9;

static {
FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(
ChronoField.MICRO_OF_SECOND,
ChronoField.NANO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
Expand All @@ -47,10 +47,10 @@ public class ExprDatetimeValue extends AbstractExprValue {
*/
public ExprDatetimeValue(String datetime) {
try {
this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_MICROS);
this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_NANOS);
} catch (DateTimeParseException e) {
throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please "
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime));
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", datetime));
}
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/opensearch/sql/data/model/ExprStringValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public LocalDateTime datetimeValue() {
.datetimeValue();
} catch (SemanticCheckException exception) {
throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please "
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", value));
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", value));
}
}
}
Expand Down
12 changes: 6 additions & 6 deletions core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@
public class ExprTimeValue extends AbstractExprValue {
private final LocalTime time;

private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 6;
private static final int MAX_FRACTION_SECONDS = 9;

static {
FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder()
.appendPattern("HH:mm:ss")
.appendFraction(
ChronoField.MICRO_OF_SECOND,
ChronoField.NANO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
Expand All @@ -44,10 +44,10 @@ public class ExprTimeValue extends AbstractExprValue {
*/
public ExprTimeValue(String time) {
try {
this.time = LocalTime.parse(time, FORMATTER_VARIABLE_MICROS);
this.time = LocalTime.parse(time, FORMATTER_VARIABLE_NANOS);
} catch (DateTimeParseException e) {
throw new SemanticCheckException(String.format("time:%s in unsupported format, please use "
+ "HH:mm:ss[.SSSSSS]", time));
+ "HH:mm:ss[.SSSSSSSSS]", time));
}
}

Expand Down
14 changes: 7 additions & 7 deletions core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@ public class ExprTimestampValue extends AbstractExprValue {
.ofPattern("yyyy-MM-dd HH:mm:ss");
private final Instant timestamp;

private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
private static final DateTimeFormatter FORMATTER_VARIABLE_NANOS;
private static final int MIN_FRACTION_SECONDS = 0;
private static final int MAX_FRACTION_SECONDS = 6;
private static final int MAX_FRACTION_SECONDS = 9;

static {
FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
FORMATTER_VARIABLE_NANOS = new DateTimeFormatterBuilder()
.appendPattern("yyyy-MM-dd HH:mm:ss")
.appendFraction(
ChronoField.MICRO_OF_SECOND,
ChronoField.NANO_OF_SECOND,
MIN_FRACTION_SECONDS,
MAX_FRACTION_SECONDS,
true)
Expand All @@ -58,12 +58,12 @@ public class ExprTimestampValue extends AbstractExprValue {
*/
public ExprTimestampValue(String timestamp) {
try {
this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS)
this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_NANOS)
.atZone(ZONE)
.toInstant();
} catch (DateTimeParseException e) {
throw new SemanticCheckException(String.format("timestamp:%s in unsupported format, please "
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", timestamp));
+ "use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]", timestamp));
}

}
Expand All @@ -72,7 +72,7 @@ public ExprTimestampValue(String timestamp) {
public String value() {
return timestamp.getNano() == 0 ? FORMATTER_WITHOUT_NANO.withZone(ZONE)
.format(timestamp.truncatedTo(ChronoUnit.SECONDS))
: FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp);
: FORMATTER_VARIABLE_NANOS.withZone(ZONE).format(timestamp);
}

@Override
Expand Down
92 changes: 46 additions & 46 deletions core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

public class DateTimeValueTest {

private static final int MICROS_PRECISION_MAX = 6;
private static final int NANOS_PRECISION_MAX = 9;

@Test
public void timeValueInterfaceTest() {
Expand Down Expand Up @@ -94,7 +94,7 @@ public void dateInUnsupportedFormat() {
public void timeInUnsupportedFormat() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class, () -> new ExprTimeValue("01:01:0"));
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSS]",
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}

Expand All @@ -105,7 +105,7 @@ public void timestampInUnsupportedFormat() {
() -> new ExprTimestampValue("2020-07-07T01:01:01Z"));
assertEquals(
"timestamp:2020-07-07T01:01:01Z in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}

Expand All @@ -116,7 +116,7 @@ public void datetimeInUnsupportedFormat() {
() -> new ExprDatetimeValue("2020-07-07T01:01:01Z"));
assertEquals(
"datetime:2020-07-07T01:01:01Z in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}

Expand All @@ -134,7 +134,7 @@ public void stringDateTimeValue() {
() -> new ExprStringValue("2020-07-07T01:01:01Z").datetimeValue());
assertEquals(
"datetime:2020-07-07T01:01:01Z in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}

Expand Down Expand Up @@ -163,96 +163,96 @@ public void stringTimeValue() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprStringValue("01:01:0").timeValue());
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSS]",
assertEquals("time:01:01:0 in unsupported format, please use HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}

@Test
public void timeWithVariableMicroPrecision() {
String timeWithMicrosFormat = "10:11:12.%s";
public void timeWithVariableNanoPrecision() {
String timeWithNanosFormat = "10:11:12.%s";

// Check all lengths of microsecond precision, up to max precision accepted
StringBuilder micros = new StringBuilder();
for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) {
micros.append(microPrecision);
String timeWithMicros = String.format(timeWithMicrosFormat, micros);
// Check all lengths of nanosecond precision, up to max precision accepted
StringBuilder nanos = new StringBuilder();
for (int nanosPrecision = 1; nanosPrecision <= NANOS_PRECISION_MAX; nanosPrecision++) {
nanos.append(nanosPrecision);
String timeWithNanos = String.format(timeWithNanosFormat, nanos);

ExprValue timeValue = new ExprTimeValue(timeWithMicros);
assertEquals(LocalTime.parse(timeWithMicros), timeValue.timeValue());
ExprValue timeValue = new ExprTimeValue(timeWithNanos);
assertEquals(LocalTime.parse(timeWithNanos), timeValue.timeValue());
}
}

@Test
public void timestampWithVariableMicroPrecision() {
public void timestampWithVariableNanoPrecision() {
String dateValue = "2020-08-17";
String timeWithMicrosFormat = "10:11:12.%s";
String timeWithNanosFormat = "10:11:12.%s";

// Check all lengths of microsecond precision, up to max precision accepted
StringBuilder micros = new StringBuilder();
for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) {
micros.append(microPrecision);
String timeWithMicros = String.format(timeWithMicrosFormat, micros);
// Check all lengths of nanosecond precision, up to max precision accepted
StringBuilder nanos = new StringBuilder();
for (int nanoPrecision = 1; nanoPrecision <= NANOS_PRECISION_MAX; nanoPrecision++) {
nanos.append(nanoPrecision);
String timeWithNanos = String.format(timeWithNanosFormat, nanos);

String timestampString = String.format("%s %s", dateValue, timeWithMicros);
String timestampString = String.format("%s %s", dateValue, timeWithNanos);
ExprValue timestampValue = new ExprTimestampValue(timestampString);

assertEquals(LocalDate.parse(dateValue), timestampValue.dateValue());
assertEquals(LocalTime.parse(timeWithMicros), timestampValue.timeValue());
String localDateTime = String.format("%sT%s", dateValue, timeWithMicros);
assertEquals(LocalTime.parse(timeWithNanos), timestampValue.timeValue());
String localDateTime = String.format("%sT%s", dateValue, timeWithNanos);
assertEquals(LocalDateTime.parse(localDateTime), timestampValue.datetimeValue());
}
}

@Test
public void datetimeWithVariableMicroPrecision() {
public void datetimeWithVariableNanoPrecision() {
String dateValue = "2020-08-17";
String timeWithMicrosFormat = "10:11:12.%s";
String timeWithNanosFormat = "10:11:12.%s";

// Check all lengths of microsecond precision, up to max precision accepted
StringBuilder micros = new StringBuilder();
for (int microPrecision = 1; microPrecision <= MICROS_PRECISION_MAX; microPrecision++) {
micros.append(microPrecision);
String timeWithMicros = String.format(timeWithMicrosFormat, micros);
// Check all lengths of nanosecond precision, up to max precision accepted
StringBuilder nanos = new StringBuilder();
for (int nanoPrecision = 1; nanoPrecision <= NANOS_PRECISION_MAX; nanoPrecision++) {
nanos.append(nanoPrecision);
String timeWithNanos = String.format(timeWithNanosFormat, nanos);

String datetimeString = String.format("%s %s", dateValue, timeWithMicros);
String datetimeString = String.format("%s %s", dateValue, timeWithNanos);
ExprValue datetimeValue = new ExprDatetimeValue(datetimeString);

assertEquals(LocalDate.parse(dateValue), datetimeValue.dateValue());
assertEquals(LocalTime.parse(timeWithMicros), datetimeValue.timeValue());
String localDateTime = String.format("%sT%s", dateValue, timeWithMicros);
assertEquals(LocalTime.parse(timeWithNanos), datetimeValue.timeValue());
String localDateTime = String.format("%sT%s", dateValue, timeWithNanos);
assertEquals(LocalDateTime.parse(localDateTime), datetimeValue.datetimeValue());
}
}

@Test
public void timestampOverMaxMicroPrecision() {
public void timestampOverMaxNanoPrecision() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprTimestampValue("2020-07-07 01:01:01.1234567"));
() -> new ExprTimestampValue("2020-07-07 01:01:01.1234567890"));
assertEquals(
"timestamp:2020-07-07 01:01:01.1234567 in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
"timestamp:2020-07-07 01:01:01.1234567890 in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}

@Test
public void datetimeOverMaxMicroPrecision() {
public void datetimeOverMaxNanoPrecision() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567"));
() -> new ExprDatetimeValue("2020-07-07 01:01:01.1234567890"));
assertEquals(
"datetime:2020-07-07 01:01:01.1234567 in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSS]",
"datetime:2020-07-07 01:01:01.1234567890 in unsupported format, "
+ "please use yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}

@Test
public void timeOverMaxMicroPrecision() {
public void timeOverMaxNanoPrecision() {
SemanticCheckException exception =
assertThrows(SemanticCheckException.class,
() -> new ExprTimeValue("01:01:01.1234567"));
() -> new ExprTimeValue("01:01:01.1234567890"));
assertEquals(
"time:01:01:01.1234567 in unsupported format, please use HH:mm:ss[.SSSSSS]",
"time:01:01:01.1234567890 in unsupported format, please use HH:mm:ss[.SSSSSSSSS]",
exception.getMessage());
}
}
36 changes: 19 additions & 17 deletions docs/user/general/datatypes.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ The OpenSearch SQL Engine support the following data types.
+---------------------+
| date |
+---------------------+
| date_nanos |
+---------------------+
| time |
+---------------------+
| interval |
Expand Down Expand Up @@ -91,9 +93,9 @@ The table below list the mapping between OpenSearch Data Type, OpenSearch SQL Da
+-----------------+---------------------+-----------+
| date | timestamp | TIMESTAMP |
+-----------------+---------------------+-----------+
| ip | ip | VARCHAR |
| date_nanos | timestamp | TIMESTAMP |
+-----------------+---------------------+-----------+
| date | timestamp | TIMESTAMP |
| ip | ip | VARCHAR |
+-----------------+---------------------+-----------+
| binary | binary | VARBINARY |
+-----------------+---------------------+-----------+
Expand Down Expand Up @@ -250,23 +252,23 @@ Time

Time represents the time on the clock or watch with no regard for which timezone it might be related with. Time type data does not have date information.

+------+-----------------------+----------------------------------------+
| Type | Syntax | Range |
+======+=======================+========================================+
| Time | 'hh:mm:ss[.fraction]' | '00:00:00.000000' to '23:59:59.999999' |
+------+-----------------------+----------------------------------------+
+------+-----------------------+----------------------------------------------+
| Type | Syntax | Range |
+======+=======================+==============================================+
| Time | 'hh:mm:ss[.fraction]' | '00:00:00.000000000' to '23:59:59.999999999' |
+------+-----------------------+----------------------------------------------+


Datetime
--------

Datetime type is the combination of date and time. The conversion rule of date or time to datetime is described in `Conversion between date and time types`_. Datetime type does not contain timezone information. For an absolute time point that contains both date time and timezone information, see `Timestamp`_.

+----------+----------------------------------+--------------------------------------------------------------+
| Type | Syntax | Range |
+==========+==================================+==============================================================+
| Datetime | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:00.000000' to '9999-12-31 23:59:59.999999' |
+----------+----------------------------------+--------------------------------------------------------------+
+----------+----------------------------------+--------------------------------------------------------------------+
| Type | Syntax | Range |
+==========+==================================+====================================================================+
| Datetime | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:00.000000000' to '9999-12-31 23:59:59.999999999' |
+----------+----------------------------------+--------------------------------------------------------------------+



Expand All @@ -275,11 +277,11 @@ Timestamp

A timestamp instance is an absolute instant independent of timezone or convention. For example, for a given point of time, if we set the timestamp of this time point into another timezone, the value should also be different accordingly. Besides, the storage of timestamp type is also different from the other types. The timestamp is converted from the current timezone to UTC for storage, and is converted back to the set timezone from UTC when retrieving.

+-----------+----------------------------------+------------------------------------------------------------------+
| Type | Syntax | Range |
+===========+==================================+==================================================================+
| Timestamp | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:01.000000' UTC to '9999-12-31 23:59:59.999999' |
+-----------+----------------------------------+------------------------------------------------------------------+
+-----------+----------------------------------+------------------------------------------------------------------------+
| Type | Syntax | Range |
+===========+==================================+========================================================================+
| Timestamp | 'yyyy-MM-dd hh:mm:ss[.fraction]' | '0001-01-01 00:00:01.000000000' UTC to '9999-12-31 23:59:59.999999999' |
+-----------+----------------------------------+------------------------------------------------------------------------+


Interval
Expand Down
4 changes: 3 additions & 1 deletion ...ch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,10 @@ private ExprType type(String field) {
}

/**
* Only default strict_date_optional_time||epoch_millis is supported.
* Only default strict_date_optional_time||epoch_millis is supported,
* strict_date_optional_time_nanos||epoch_millis if field is date_nanos.
* https://www.elastic.co/guide/en/elasticsearch/reference/current/date.html
* https://www.elastic.co/guide/en/elasticsearch/reference/current/date_nanos.html
* The customized date_format is not supported.
*/
private ExprValue constructTimestamp(String value) {
Expand Down
1 change: 1 addition & 0 deletions ...ain/java/org/opensearch/sql/opensearch/request/system/OpenSearchDescribeIndexRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ public class OpenSearchDescribeIndexRequest implements OpenSearchSystemRequest {
.put("nested", ExprCoreType.ARRAY)
.put("object", ExprCoreType.STRUCT)
.put("date", ExprCoreType.TIMESTAMP)
.put("date_nanos", ExprCoreType.TIMESTAMP)
.put("ip", OpenSearchDataType.OPENSEARCH_IP)
.put("geo_point", OpenSearchDataType.OPENSEARCH_GEO_POINT)
.put("binary", OpenSearchDataType.OPENSEARCH_BINARY)
Expand Down

0 comments on commit 6ec7b9d

Please sign in to comment.