[Backport 2.x] Add support for date_nanos and tests. (#337)

integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java

@@ -49,6 +49,7 @@ public void test_nonnumeric_data_types() throws IOException {
         schema("text_value", "string"),
         schema("binary_value", "binary"),
         schema("date_value", "timestamp"),
+        schema("date_nanos_value", "timestamp"),
         schema("ip_value", "ip"),
         schema("object_value", "struct"),
         schema("nested_value", "array"),

integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java

@@ -67,19 +67,28 @@ public void typeof_opensearch_types() throws IOException {
     response =
         executeQuery(
             String.format(
-                "source=%s | eval "
-                    + "`text` = typeof(text_value), `date` = typeof(date_value),"
-                    + "`boolean` = typeof(boolean_value), `object` = typeof(object_value),"
-                    + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value),"
-                    + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)"
+                "source=%s | eval `text` = typeof(text_value), `date` = typeof(date_value),"
+                    + " `date_nanos` = typeof(date_nanos_value),`boolean` = typeof(boolean_value),"
+                    + " `object` = typeof(object_value),`keyword` = typeof(keyword_value), `ip` ="
+                    + " typeof(ip_value),`binary` = typeof(binary_value), `geo_point` ="
+                    + " typeof(geo_point_value)"
                     // TODO activate this test once `ARRAY` type supported, see
                     // ExpressionAnalyzer::isTypeNotSupported
                     // + ", `nested` = typeof(nested_value)"
-                    + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`,"
-                    + " `geo_point`",
+                    + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`,"
+                    + " `ip`, `binary`, `geo_point`",
                 TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(
         response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT"));
+        rows(
+            "TEXT",
+            "TIMESTAMP",
+            "TIMESTAMP",
+            "BOOLEAN",
+            "OBJECT",
+            "KEYWORD",
+            "IP",
+            "BINARY",
+            "GEO_POINT"));
   }
 }

integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java

@@ -5,6 +5,7 @@
 
 package org.opensearch.sql.sql;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS;
 import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
 import static org.opensearch.sql.util.MatcherUtils.rows;
@@ -29,6 +30,7 @@ public class DateTimeFormatsIT extends SQLIntegTestCase {
   public void init() throws Exception {
     super.init();
     loadIndex(Index.DATE_FORMATS);
+    loadIndex(Index.DATA_TYPE_NONNUMERIC);
   }
 
   @Test
@@ -139,6 +141,94 @@ public void testNumericFormats() {
         rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5"));
   }
 
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithFormats() {
+    String query =
+        String.format("SELECT hour_minute_second_OR_t_time" + " FROM %s", TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("09:07:42"), rows("07:07:42.123456789"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithFunctions() {
+    // in memory funcs
+    String query =
+        String.format(
+            "SELECT"
+                + " hour_minute_second_OR_t_time > TIME '08:07:00',"
+                + " hour_minute_second_OR_t_time < TIME '08:07:00',"
+                + " hour_minute_second_OR_t_time = t_time_no_millis,"
+                + " hour_minute_second_OR_t_time <> strict_t_time,"
+                + " hour_minute_second_OR_t_time >= t_time"
+                + " FROM %s",
+            TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(
+        result,
+        schema("hour_minute_second_OR_t_time > TIME '08:07:00'", null, "boolean"),
+        schema("hour_minute_second_OR_t_time < TIME '08:07:00'", null, "boolean"),
+        schema("hour_minute_second_OR_t_time = t_time_no_millis", null, "boolean"),
+        schema("hour_minute_second_OR_t_time <> strict_t_time", null, "boolean"),
+        schema("hour_minute_second_OR_t_time >= t_time", null, "boolean"));
+    verifyDataRows(
+        result, rows(true, false, true, false, true), rows(false, true, false, true, false));
+    // push down
+    query =
+        String.format(
+            "SELECT hour_minute_second_OR_t_time"
+                + " FROM %s WHERE hour_minute_second_OR_t_time > TIME '08:07:00'",
+            TEST_INDEX_DATE_FORMATS);
+    result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("09:07:42"));
+    query =
+        String.format(
+            "SELECT hour_minute_second_OR_t_time"
+                + " FROM %s WHERE hour_minute_second_OR_t_time < TIME '08:07:00'",
+            TEST_INDEX_DATE_FORMATS);
+    result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("07:07:42.123456789"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosOrderBy() {
+    String query =
+        String.format(
+            "SELECT hour_minute_second_OR_t_time"
+                + " FROM %s ORDER BY hour_minute_second_OR_t_time ASC",
+            TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time"));
+    verifyDataRows(result, rows("07:07:42.123456789"), rows("09:07:42"));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosGroupBy() {
+    String query =
+        String.format(
+            "SELECT count(*)" + " FROM %s GROUP BY hour_minute_second_OR_t_time",
+            TEST_INDEX_DATE_FORMATS);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("count(*)", null, "integer"));
+    verifyDataRows(result, rows(1), rows(1));
+  }
+
+  @Test
+  @SneakyThrows
+  public void testDateNanosWithNanos() {
+    String query =
+        String.format("SELECT date_nanos_value" + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC);
+    JSONObject result = executeQuery(query);
+    verifySchema(result, schema("date_nanos_value", null, "timestamp"));
+    verifyDataRows(result, rows("2019-03-24 01:34:46.123456789"));
+  }
+
   protected JSONObject executeQuery(String query) throws IOException {
     Request request = new Request("POST", QUERY_API_ENDPOINT);
     request.setJsonEntity(String.format(Locale.ROOT, "{\n" + "  \"query\": \"%s\"\n" + "}", query));

integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java

@@ -55,16 +55,26 @@ public void typeof_opensearch_types() {
     response =
         executeJdbcRequest(
             String.format(
-                "SELECT typeof(text_value),typeof(date_value), typeof(boolean_value),"
-                    + " typeof(object_value), typeof(keyword_value),typeof(ip_value),"
-                    + " typeof(binary_value), typeof(geo_point_value)"
+                "SELECT typeof(text_value),typeof(date_value), typeof(date_nanos_value),"
+                    + " typeof(boolean_value), typeof(object_value),"
+                    + " typeof(keyword_value),typeof(ip_value), typeof(binary_value),"
+                    + " typeof(geo_point_value)"
                     // TODO activate this test once `ARRAY` type supported, see
                     // ExpressionAnalyzer::isTypeNotSupported
                     // + ", typeof(nested_value)"
                     + " from %s;",
                 TEST_INDEX_DATATYPE_NONNUMERIC));
     verifyDataRows(
         response,
-        rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT"));
+        rows(
+            "TEXT",
+            "TIMESTAMP",
+            "TIMESTAMP",
+            "BOOLEAN",
+            "OBJECT",
+            "KEYWORD",
+            "IP",
+            "BINARY",
+            "GEO_POINT"));
   }
 }

integ-test/src/test/resources/datatypes.json

@@ -1,2 +1,2 @@
 {"index":{"_id":"1"}}
-{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}
+{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" :  "Smith"}, {"first" : "Alice", "last" :  "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }}

integ-test/src/test/resources/date_formats.json

@@ -1,4 +1,4 @@
 {"index": {}}
 {"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time":  "1961-04-12", "custom_time_parser_check":  "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incomplete_custom_date": 1999, "incomplete_custom_time" : 10, "incorrect" : null, "epoch_sec" : 42, "epoch_milli" : 42, "custom_no_delimiter_date" : "19841020", "custom_no_delimiter_time" : "102030", "custom_no_delimiter_ts" : "19841020153548"}
 {"index": {}}
-{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time":  "09:07:00", "custom_time_parser_check":  "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"}
+{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T07:07:42.123456789Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time":  "09:07:00", "custom_time_parser_check":  "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"}

integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json

@@ -13,10 +13,13 @@
       "binary_value": {
         "type": "binary"
       },
-      "date_value":  {
+      "date_value": {
         "type" : "date",
         "format": "yyyy-MM-dd HH:mm:ss"
       },
+      "date_nanos_value": {
+        "type" : "date_nanos"
+      },
       "ip_value": {
         "type": "ip"
       },