diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java index 200c300f3b..3f8c2c971a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java @@ -27,6 +27,7 @@ import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; +import org.junit.jupiter.api.Disabled; import org.opensearch.client.Request; /** diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 35ae5d3675..901f198b14 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -634,6 +634,10 @@ public enum Index { "calcs", getMappingFile("calcs_index_mappings.json"), "src/test/resources/calcs.json"), + DATE_FORMATS(TestsConstants.TEST_INDEX_DATE_FORMATS, + "date_formats", + getMappingFile("date_formats_index_mapping.json"), + "src/test/resources/date_formats.json"), WILDCARD(TestsConstants.TEST_INDEX_WILDCARD, "wildcard", getMappingFile("wildcard_index_mappings.json"), diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index c3af98b794..338be25a0c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -56,6 +56,7 @@ public class TestsConstants { public final static String TEST_INDEX_BEER = TEST_INDEX + "_beer"; public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing"; public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs"; + public final static String TEST_INDEX_DATE_FORMATS = TEST_INDEX + "_date_formats"; public final static String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard"; public final static String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public final static String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index b5677b04a7..c70ca1dc2c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -40,7 +41,7 @@ public void init() throws Exception { loadIndex(Index.BANK); loadIndex(Index.CALCS); loadIndex(Index.PEOPLE2); - loadIndex(Index.CALCS); + loadIndex(Index.DATE_FORMATS); } // Integration test framework sets for OpenSearch instance a random timezone. @@ -1278,6 +1279,99 @@ public void testTimeFormat() throws IOException { verifyTimeFormat(timestamp, "timestamp", timestampFormat, timestampFormatted); } + @Test + public void testReadingDateFormats() throws IOException { + String query = String.format("SELECT * FROM %s LIMIT 1", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifyDataRows(result, + rows("1984-04-12 00:00:00", // date + "1984-04-12 09:07:42", // date_optional_time + "1984-04-12 09:07:42", // basic_week_date_time + "1970-01-01 09:07:42", // basic_ordinal_date_time TODO: fix bug + "1970-01-01 09:07:42", // strict_time_no_millis + "1984-04-12 09:07:42", // date_time + "1984-04-12 09:07:42", // strict_basic_week_date_time + "1984-04-12 00:00:00", // strict_basic_week_date + "1984-04-12 00:00:00", // strict_ordinal_date + "1970-01-01 09:07:42", // basic_time_no_millis + "1984-04-12 09:07:42.000123456", // strict_date_optional_time_nanos + "1984-04-12 00:00:00", // year_month_day + "1970-01-01 09:00:00", // strict_hour + "1984-04-12 00:00:00", // basic_week_date + "1984-04-12 00:00:00", // strict_week_date + "1970-01-01 09:07:42", // basic_t_time_no_millis + "1970-01-01 09:07:00", // strict_date_hour_minute + "1984-04-12 09:07:42", // strict_week_date_time_no_millis + "1984-04-12 09:07:42", // basic_date_time_no_millis + "1984-04-12 09:07:42", // strict_week_date_time + "1984-04-12 09:07:42.000123456", // epoch_second + "1984-04-12 09:07:42", // basic_ordinal_date_time_no_millis + "1984-04-12 09:07:42.000123456", // ordinal_date_time + "1984-04-12 09:07:42", // strict_date_optional_time + "1970-01-01 09:07:42", // hour_minute_second + "1970-01-01 09:07:42", // basic_time + "1970-01-01 09:07:42", // HH:mm:ss + "1970-01-01 09:07:42", // time_no_millis + "1970-01-01 09:07:42", // hour_minute_second_fraction + "1970-01-01 09:07:42", // strict_date_hour_minute_second_fraction + "1970-01-01 09:07:42", // strict_hour_minute_second_fraction + "1970-01-01 09:07:42", // strict_time + "1970-01-01 09:07:42", // strict_t_time_no_millis + "1970-01-01 09:07:00", // date_hour_minute + "1984-04-12 09:07:42.000123456", // strict_ordinal_date_time + "1970-01-01 09:07:42", // strict_t_time + "1984-04-12 09:07:42", // week_date_time + "1970-01-01 09:07:42", // date_hour_minute_second + "1984-04-12 09:07:42", // date_time_no_millis + "1970-01-01 09:00:00", // hour + "1970-01-01 09:07:42", // strict_date_hour_minute_second_millis TODO: fix bug + "1984-04-12 00:00:00", // yyyy-MM-dd_OR_epoch_millis + "1984-04-12 00:00:00", // strict_date + "1984-04-12 09:07:42", // week_date_time_no_millis + "1970-01-01 09:07:42", // t_time + "1970-01-01 09:07:42", // hour_minute_second_millis + "1984-04-12 00:00:00", // basic_ordinal_date + "1970-01-01 09:07:42", // date_hour_minute_second_millis TODO: fix bug + "1984-04-12 09:07:42", // strict_ordinal_date_time_no_millis + "1970-01-01 09:07:42", // hour_minute_second_OR_t_time + "1984-04-12 00:00:00", // strict_year_month_day + "1984-04-12 09:07:42", // ordinal_date_time_no_millis + "1984-04-12 00:00:00", // week_date + "1984-04-12 09:07:42.000123456", // epoch_millis + "1970-01-01 09:07:00", // strict_hour_minute + "1984-04-12 00:00:00", // basic_date + "1970-01-01 09:07:42", // t_time_no_millis + "1984-04-12 00:00:00", // weekyear_week_day + "1984-04-12 00:00:00", // ordinal_date + "1970-01-01 09:07:42", // strict_hour_minute_second_millis + "1984-04-12 09:07:42", // basic_date_time + "1970-01-01 09:00:00", // date_hour TODO: fix bug + "1970-01-01 09:00:00", // strict_date_hour TODO: fix bug + "1984-04-12 00:00:00", // strict_weekyear_week_day + "1970-01-01 09:07:42", // date_hour_minute_second_fraction TODO: fix bug + "1984-04-12 09:07:42", // strict_date_time_no_millis + "1970-01-01 09:07:42", // basic_t_time + "1970-01-01 09:07:00", // hour_minute + "1984-04-12 09:07:42", // basic_week_date_time_no_millis + "1984-04-12 00:00:00", // yyyy-MM-dd + "1984-04-12 09:07:42.000123456", // strict_date_time + "1970-01-01 09:07:42", // time + "1970-01-01 09:07:42", // strict_date_hour_minute_second TODO: fix bug + "1970-01-01 09:07:42", // strict_hour_minute_second + "1984-04-12 09:07:42" // strict_basic_week_date_time_no_millis + )); + } + + @Test + public void testDateFormatsWithOr() throws IOException { + String query = String.format("SELECT yyyy-MM-dd_OR_epoch_millis, hour_minute_second_OR_t_time" + + " FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifyDataRows(result, + rows("1984-04-12 00:00:00", "1970-01-01 09:07:42"), + rows("1984-04-12 09:07:42.000123456", "1970-01-01 11:27:25")); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json new file mode 100644 index 0000000000..a3a4affc0b --- /dev/null +++ b/integ-test/src/test/resources/date_formats.json @@ -0,0 +1,4 @@ +{"index": {}} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42"} +{"index": {}} +{"epoch_millis": "450608862000", "epoch_second": "450608862", "date_optional_time": "2023-05-03T11:27:25.000Z", "strict_date_optional_time": "2023-05-03T11:27:25.000Z", "strict_date_optional_time_nanos": "2023-05-03T11:27:25.000123456Z", "basic_date": "20230503", "basic_date_time": "20230503T112725.000Z", "basic_date_time_no_millis": "20230503T112725Z", "basic_ordinal_date": "2023123", "basic_ordinal_date_time": "2023123T112725.000Z", "basic_ordinal_date_time_no_millis": "2023123T112725Z", "basic_time": "112725.000Z", "basic_time_no_millis": "112725Z", "basic_t_time": "T112725.000Z", "basic_t_time_no_millis": "T112725Z", "basic_week_date": "2023W183", "strict_basic_week_date": "2023W183", "basic_week_date_time": "2023W183T112725.000Z", "strict_basic_week_date_time": "2023W183T112725.000Z", "basic_week_date_time_no_millis": "2023W183T112725Z", "strict_basic_week_date_time_no_millis": "2023W183T112725Z", "date": "2023-05-03", "strict_date": "2023-05-03", "date_hour": "2023-05-03T11", "strict_date_hour": "2023-05-03T11", "date_hour_minute": "2023-05-03T11:27", "strict_date_hour_minute": "2023-05-03T11:27", "date_hour_minute_second": "2023-05-03T11:27:25", "strict_date_hour_minute_second": "2023-05-03T11:27:25", "date_hour_minute_second_fraction": "2023-05-03T11:27:25.000", "strict_date_hour_minute_second_fraction": "2023-05-03T11:27:25.000", "date_hour_minute_second_millis": "2023-05-03T11:27:25.000", "strict_date_hour_minute_second_millis": "2023-05-03T11:27:25.000", "date_time": "2023-05-03T11:27:25.000Z", "strict_date_time": "2023-05-03T11:27:25.000123456Z", "date_time_no_millis": "2023-05-03T11:27:25Z", "strict_date_time_no_millis": "2023-05-03T11:27:25Z", "hour": "11", "strict_hour": "11", "hour_minute": "11:27", "strict_hour_minute": "11:27", "hour_minute_second": "11:27:25", "strict_hour_minute_second": "11:27:25", "hour_minute_second_fraction": "11:27:25.000", "strict_hour_minute_second_fraction": "11:27:25.000", "hour_minute_second_millis": "11:27:25.000", "strict_hour_minute_second_millis": "11:27:25.000", "ordinal_date": "2023-123", "strict_ordinal_date": "2023-123", "ordinal_date_time": "2023-123T11:27:25.000123456Z", "strict_ordinal_date_time": "2023-123T11:27:25.000123456Z", "ordinal_date_time_no_millis": "2023-123T11:27:25Z", "strict_ordinal_date_time_no_millis": "2023-123T11:27:25Z", "time": "11:27:25.000Z", "strict_time": "11:27:25.000Z", "time_no_millis": "11:27:25Z", "strict_time_no_millis": "11:27:25Z", "t_time": "T11:27:25.000Z", "strict_t_time": "T11:27:25.000Z", "t_time_no_millis": "T11:27:25Z", "strict_t_time_no_millis": "T11:27:25Z", "week_date": "2023-W18-3", "strict_week_date": "2023-W18-3", "week_date_time": "2023-W18-3T11:27:25.000Z", "strict_week_date_time": "2023-W18-3T11:27:25.000Z", "week_date_time_no_millis": "2023-W18-3T11:27:25Z", "strict_week_date_time_no_millis": "2023-W18-3T11:27:25Z", "weekyear_week_day": "2023-W18-3", "strict_weekyear_week_day": "2023-W18-3", "year_month_day": "2023-05-03", "strict_year_month_day": "2023-05-03", "yyyy-MM-dd": "2023-05-03", "HH:mm:ss": "11:27:25", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T11:27:25.000Z"} diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json new file mode 100644 index 0000000000..2c46555653 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -0,0 +1,306 @@ +{ + "mappings" : { + "properties" : { + "epoch_millis" : { + "type" : "date", + "format" : "epoch_millis" + }, + "epoch_second" : { + "type" : "date", + "format" : "epoch_second" + }, + "date_optional_time" : { + "type" : "date", + "format" : "date_optional_time" + }, + "strict_date_optional_time" : { + "type" : "date", + "format" : "strict_date_optional_time" + }, + "strict_date_optional_time_nanos" : { + "type" : "date", + "format" : "strict_date_optional_time_nanos" + }, + "basic_date" : { + "type" : "date", + "format" : "basic_date" + }, + "basic_date_time" : { + "type" : "date", + "format" : "basic_date_time" + }, + "basic_date_time_no_millis" : { + "type" : "date", + "format" : "basic_date_time_no_millis" + }, + "basic_ordinal_date" : { + "type" : "date", + "format" : "basic_ordinal_date" + }, + "basic_ordinal_date_time" : { + "type" : "date", + "format" : "basic_ordinal_date_time" + }, + "basic_ordinal_date_time_no_millis" : { + "type" : "date", + "format" : "basic_ordinal_date_time_no_millis" + }, + "basic_time" : { + "type" : "date", + "format" : "basic_time" + }, + "basic_time_no_millis" : { + "type" : "date", + "format" : "basic_time_no_millis" + }, + "basic_t_time" : { + "type" : "date", + "format" : "basic_t_time" + }, + "basic_t_time_no_millis" : { + "type" : "date", + "format" : "basic_t_time_no_millis" + }, + "basic_week_date" : { + "type" : "date", + "format" : "basic_week_date" + }, + "strict_basic_week_date" : { + "type" : "date", + "format" : "strict_basic_week_date" + }, + "basic_week_date_time" : { + "type" : "date", + "format" : "basic_week_date_time" + }, + "strict_basic_week_date_time" : { + "type" : "date", + "format" : "strict_basic_week_date_time" + }, + "basic_week_date_time_no_millis" : { + "type" : "date", + "format" : "basic_week_date_time_no_millis" + }, + "strict_basic_week_date_time_no_millis" : { + "type" : "date", + "format" : "strict_basic_week_date_time_no_millis" + }, + "date" : { + "type" : "date", + "format" : "date" + }, + "strict_date" : { + "type" : "date", + "format" : "strict_date" + }, + "date_hour" : { + "type" : "date", + "format" : "date_hour" + }, + "strict_date_hour" : { + "type" : "date", + "format" : "strict_date_hour" + }, + "date_hour_minute" : { + "type" : "date", + "format" : "date_hour_minute" + }, + "strict_date_hour_minute" : { + "type" : "date", + "format" : "strict_date_hour_minute" + }, + "date_hour_minute_second" : { + "type" : "date", + "format" : "date_hour_minute_second" + }, + "strict_date_hour_minute_second" : { + "type" : "date", + "format" : "strict_date_hour_minute_second" + }, + "date_hour_minute_second_fraction" : { + "type" : "date", + "format" : "date_hour_minute_second_fraction" + }, + "strict_date_hour_minute_second_fraction" : { + "type" : "date", + "format" : "strict_date_hour_minute_second_fraction" + }, + "date_hour_minute_second_millis" : { + "type" : "date", + "format" : "date_hour_minute_second_millis" + }, + "strict_date_hour_minute_second_millis" : { + "type" : "date", + "format" : "strict_date_hour_minute_second_millis" + }, + "date_time" : { + "type" : "date", + "format" : "date_time" + }, + "strict_date_time" : { + "type" : "date", + "format" : "strict_date_time" + }, + "date_time_no_millis" : { + "type" : "date", + "format" : "date_time_no_millis" + }, + "strict_date_time_no_millis" : { + "type" : "date", + "format" : "strict_date_time_no_millis" + }, + "hour" : { + "type" : "date", + "format" : "hour" + }, + "strict_hour" : { + "type" : "date", + "format" : "strict_hour" + }, + "hour_minute" : { + "type" : "date", + "format" : "hour_minute" + }, + "strict_hour_minute" : { + "type" : "date", + "format" : "strict_hour_minute" + }, + "hour_minute_second" : { + "type" : "date", + "format" : "hour_minute_second" + }, + "strict_hour_minute_second" : { + "type" : "date", + "format" : "strict_hour_minute_second" + }, + "hour_minute_second_fraction" : { + "type" : "date", + "format" : "hour_minute_second_fraction" + }, + "strict_hour_minute_second_fraction" : { + "type" : "date", + "format" : "strict_hour_minute_second_fraction" + }, + "hour_minute_second_millis" : { + "type" : "date", + "format" : "hour_minute_second_millis" + }, + "strict_hour_minute_second_millis" : { + "type" : "date", + "format" : "strict_hour_minute_second_millis" + }, + "ordinal_date" : { + "type" : "date", + "format" : "ordinal_date" + }, + "strict_ordinal_date" : { + "type" : "date", + "format" : "strict_ordinal_date" + }, + "ordinal_date_time" : { + "type" : "date", + "format" : "ordinal_date_time" + }, + "strict_ordinal_date_time" : { + "type" : "date", + "format" : "strict_ordinal_date_time" + }, + "ordinal_date_time_no_millis" : { + "type" : "date", + "format" : "ordinal_date_time_no_millis" + }, + "strict_ordinal_date_time_no_millis" : { + "type" : "date", + "format" : "strict_ordinal_date_time_no_millis" + }, + "time" : { + "type" : "date", + "format" : "time" + }, + "strict_time" : { + "type" : "date", + "format" : "strict_time" + }, + "time_no_millis" : { + "type" : "date", + "format" : "time_no_millis" + }, + "strict_time_no_millis" : { + "type" : "date", + "format" : "strict_time_no_millis" + }, + "t_time" : { + "type" : "date", + "format" : "t_time" + }, + "strict_t_time" : { + "type" : "date", + "format" : "strict_t_time" + }, + "t_time_no_millis" : { + "type" : "date", + "format" : "t_time_no_millis" + }, + "strict_t_time_no_millis" : { + "type" : "date", + "format" : "strict_t_time_no_millis" + }, + "week_date" : { + "type" : "date", + "format" : "week_date" + }, + "strict_week_date" : { + "type" : "date", + "format" : "strict_week_date" + }, + "week_date_time" : { + "type" : "date", + "format" : "week_date_time" + }, + "strict_week_date_time" : { + "type" : "date", + "format" : "strict_week_date_time" + }, + "week_date_time_no_millis" : { + "type" : "date", + "format" : "week_date_time_no_millis" + }, + "strict_week_date_time_no_millis" : { + "type" : "date", + "format" : "strict_week_date_time_no_millis" + }, + "weekyear_week_day" : { + "type" : "date", + "format" : "weekyear_week_day" + }, + "strict_weekyear_week_day" : { + "type" : "date", + "format" : "strict_weekyear_week_day" + }, + "year_month_day" : { + "type" : "date", + "format" : "year_month_day" + }, + "strict_year_month_day" : { + "type" : "date", + "format" : "strict_year_month_day" + }, + "yyyy-MM-dd" : { + "type" : "date", + "format": "yyyy-MM-dd" + }, + "HH:mm:ss" : { + "type" : "date", + "format": "HH:mm:ss" + }, + "yyyy-MM-dd_OR_epoch_millis" : { + "type" : "date", + "format": "yyyy-MM-dd||epoch_millis" + }, + "hour_minute_second_OR_t_time" : { + "type" : "date", + "format": "hour_minute_second||t_time" + } + } + } +} diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java index ebd9648d5e..1adb0e745e 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java @@ -6,16 +6,13 @@ package org.opensearch.sql.legacy.executor; -import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import java.io.IOException; import java.util.List; -import java.util.Map; import org.opensearch.action.admin.indices.get.GetIndexRequest; import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; @@ -25,7 +22,6 @@ import org.opensearch.rest.RestStatus; import org.opensearch.rest.action.RestBuilderListener; import org.opensearch.sql.legacy.antlr.semantic.SemanticAnalysisException; -import org.opensearch.sql.legacy.domain.Field; /** * Created by Eliran on 6/10/2015. diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java index 44baa8050b..595b6987a7 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java @@ -37,10 +37,8 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; -import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 1c1498feb5..b26680b3ba 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -112,11 +112,11 @@ public Map getIndexMaxResultWindows(String... indexExpression) GetSettingsResponse settingsResponse = client.admin().indices().prepareGetSettings(indexExpression).setLocal(true).get(); ImmutableMap.Builder result = ImmutableMap.builder(); - for (ObjectObjectCursor indexToSetting : - settingsResponse.getIndexToSettings()) { - Settings settings = indexToSetting.value; + for (Map.Entry indexToSetting : + settingsResponse.getIndexToSettings().entrySet()) { + Settings settings = indexToSetting.getValue(); result.put( - indexToSetting.key, + indexToSetting.getKey(), settings.getAsInt( IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), IndexSettings.MAX_RESULT_WINDOW_SETTING.getDefault(settings))); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index d9f9dbbe5d..66cc067541 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -30,7 +30,6 @@ import org.opensearch.client.indices.GetMappingsResponse; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.AliasMetadata; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.settings.Settings; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; @@ -85,21 +84,21 @@ public Map getIndexMaxResultWindows(String... indexExpression) .indices(indexExpression).includeDefaults(true); try { GetSettingsResponse response = client.indices().getSettings(request, RequestOptions.DEFAULT); - ImmutableOpenMap settings = response.getIndexToSettings(); - ImmutableOpenMap defaultSettings = response.getIndexToDefaultSettings(); + Map settings = response.getIndexToSettings(); + Map defaultSettings = response.getIndexToDefaultSettings(); Map result = new HashMap<>(); - defaultSettings.forEach(entry -> { - Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null); + defaultSettings.forEach((key, value) -> { + Integer maxResultWindow = value.getAsInt("index.max_result_window", null); if (maxResultWindow != null) { - result.put(entry.key, maxResultWindow); + result.put(key, maxResultWindow); } }); - settings.forEach(entry -> { - Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null); + settings.forEach((key, value) -> { + Integer maxResultWindow = value.getAsInt("index.max_result_window", null); if (maxResultWindow != null) { - result.put(entry.key, maxResultWindow); + result.put(key, maxResultWindow); } }); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index 2fda12a567..242f351f81 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -65,6 +65,7 @@ public String toString() { } @EqualsAndHashCode.Exclude + @Getter protected MappingType mappingType; // resolved ExprCoreType @@ -97,56 +98,81 @@ public ExprType getExprType() { instances.put(t.toString(), OpenSearchDataType.of(t))); } + /** + * Parses index mapping and maps it to a Data type in the SQL plugin. + * @param indexMapping An input with keys and objects that need to be mapped to a data type. + * @return The mapping. + */ + public static Map parseMapping(Map indexMapping) { + Map result = new LinkedHashMap<>(); + if (indexMapping != null) { + indexMapping.forEach((k, v) -> { + var innerMap = (Map)v; + // by default, the type is treated as an Object if "type" is not provided + var type = ((String) innerMap + .getOrDefault( + "type", + "object")) + .replace("_", ""); + if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) { + // unknown type, e.g. `alias` + // TODO resolve alias reference + return; + } + // create OpenSearchDataType + result.put(k, OpenSearchDataType.of( + EnumUtils.getEnumIgnoreCase(OpenSearchDataType.MappingType.class, type), + innerMap) + ); + }); + } + return result; + } + /** * A constructor function which builds proper `OpenSearchDataType` for given mapping `Type`. * @param mappingType A mapping type. * @return An instance or inheritor of `OpenSearchDataType`. */ - public static OpenSearchDataType of(MappingType mappingType) { - var res = instances.getOrDefault(mappingType.toString(), null); - if (res != null) { - return res; - } - ExprCoreType exprCoreType = mappingType.getExprCoreType(); - if (exprCoreType == ExprCoreType.UNKNOWN) { - switch (mappingType) { + public static OpenSearchDataType of(MappingType mappingType, Map innerMap) { + OpenSearchDataType res = instances.getOrDefault(mappingType.toString(), + new OpenSearchDataType(mappingType) + ); + switch (mappingType) { + case Object: + case Nested: + if (innerMap.isEmpty()) { + return res; + } + Map properties = + parseMapping((Map) innerMap.getOrDefault("properties", Map.of())); + OpenSearchDataType objectDataType = res.cloneEmpty(); + objectDataType.properties = properties; + return objectDataType; + case Text: // TODO update these 2 below #1038 https://github.com/opensearch-project/sql/issues/1038 - case Text: return OpenSearchTextType.of(); - case GeoPoint: return OpenSearchGeoPointType.of(); - case Binary: return OpenSearchBinaryType.of(); - case Ip: return OpenSearchIpType.of(); - default: - throw new IllegalArgumentException(mappingType.toString()); - } + Map fields = + parseMapping((Map) innerMap.getOrDefault("fields", Map.of())); + return (!fields.isEmpty()) ? OpenSearchTextType.of(fields) : OpenSearchTextType.of(); + case GeoPoint: return OpenSearchGeoPointType.of(); + case Binary: return OpenSearchBinaryType.of(); + case Ip: return OpenSearchIpType.of(); + case Date: + return OpenSearchDateType.create( + (String) innerMap.getOrDefault("format", "")); + default: + return res; } - res = new OpenSearchDataType(mappingType); - res.exprCoreType = exprCoreType; - return res; } /** * A constructor function which builds proper `OpenSearchDataType` for given mapping `Type`. * Designed to be called by the mapping parser only (and tests). * @param mappingType A mapping type. - * @param properties Properties to set. - * @param fields Fields to set. * @return An instance or inheritor of `OpenSearchDataType`. */ - public static OpenSearchDataType of(MappingType mappingType, - Map properties, - Map fields) { - var res = of(mappingType); - if (!properties.isEmpty() || !fields.isEmpty()) { - // Clone to avoid changing the singleton instance. - res = res.cloneEmpty(); - res.properties = ImmutableMap.copyOf(properties); - res.fields = ImmutableMap.copyOf(fields); - } - return res; - } - - protected OpenSearchDataType(MappingType mappingType) { - this.mappingType = mappingType; + public static OpenSearchDataType of(MappingType mappingType) { + return of(mappingType, Map.of()); } /** @@ -165,11 +191,13 @@ public static OpenSearchDataType of(ExprType type) { return new OpenSearchDataType((ExprCoreType) type); } - protected OpenSearchDataType(ExprCoreType type) { - this.exprCoreType = type; + protected OpenSearchDataType(MappingType mappingType) { + this.mappingType = mappingType; + this.exprCoreType = mappingType.getExprCoreType(); } - protected OpenSearchDataType() { + protected OpenSearchDataType(ExprCoreType type) { + this.exprCoreType = type; } // For datatypes with properties (example: object and nested types) @@ -178,11 +206,6 @@ protected OpenSearchDataType() { @EqualsAndHashCode.Exclude Map properties = ImmutableMap.of(); - // text could have fields - // a read-only collection - @EqualsAndHashCode.Exclude - Map fields = ImmutableMap.of(); - @Override // Called when building TypeEnvironment and when serializing PPL response public String typeName() { @@ -209,16 +232,16 @@ public String legacyTypeName() { * @return A cloned object. */ protected OpenSearchDataType cloneEmpty() { - var copy = new OpenSearchDataType(); - copy.mappingType = mappingType; - copy.exprCoreType = exprCoreType; - return copy; + if (this.mappingType == null) { + return new OpenSearchDataType(this.exprCoreType); + } + return new OpenSearchDataType(this.mappingType); } /** * Flattens mapping tree into a single layer list of objects (pairs of name-types actually), * which don't have nested types. - * See {@link OpenSearchDataTypeTest#traverseAndFlatten() test} for example. + * See OpenSearchDataTypeTest#traverseAndFlatten() test for example. * @param tree A list of `OpenSearchDataType`s - map between field name and its type. * @return A list of all `OpenSearchDataType`s from given map on the same nesting level (1). * Nested object names are prefixed by names of their host. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java new file mode 100644 index 0000000000..0379d63275 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java @@ -0,0 +1,89 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.data.type; + +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.sql.data.type.ExprType; + +/** + * Date type with support for predefined and custom formats read from the index mapping. + */ +@EqualsAndHashCode(callSuper = true) +public class OpenSearchDateType extends OpenSearchDataType { + + private static final OpenSearchDateType instance = new OpenSearchDateType(); + + private static final String FORMAT_DELIMITER = "\\|\\|"; + + @EqualsAndHashCode.Exclude + String formatString; + + private OpenSearchDateType() { + super(MappingType.Date); + this.formatString = ""; + } + + private OpenSearchDateType(String formatStringArg) { + super(MappingType.Date); + this.formatString = formatStringArg; + } + + /** + * Retrieves and splits a user defined format string from the mapping into a list of formats. + * @return A list of format names and user defined formats. + */ + public List getFormatList() { + return Arrays.stream(formatString.split(FORMAT_DELIMITER)) + .map(String::trim) + .collect(Collectors.toList()); + } + + + /** + * Retrieves named formatters defined by OpenSearch. + * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp. + */ + public List getNamedFormatters() { + return getFormatList().stream().filter(f -> { + try { + DateTimeFormatter.ofPattern(f); + //TODO: Filter off of a constant list of formats + return false; + } catch (IllegalArgumentException e) { + return true; + } + }).map(DateFormatter::forPattern).collect(Collectors.toList()); + } + + /** + * Create a Date type which has a LinkedHashMap defining all formats. + * @return A new type object. + */ + public static OpenSearchDateType create(String format) { + return new OpenSearchDateType(format); + } + + public static OpenSearchDateType of() { + return OpenSearchDateType.instance; + } + + @Override + public boolean shouldCast(ExprType other) { + return false; + } + + @Override + protected OpenSearchDataType cloneEmpty() { + return OpenSearchDateType.create(this.formatString); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java index 1098662e65..b27193757d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java @@ -23,19 +23,24 @@ public class OpenSearchTextType extends OpenSearchDataType { private static final OpenSearchTextType instance = new OpenSearchTextType(); + // text could have fields + // a read-only collection + @EqualsAndHashCode.Exclude + Map fields = ImmutableMap.of(); + private OpenSearchTextType() { super(MappingType.Text); exprCoreType = UNKNOWN; } /** - * Create a Text type which has fields. - * @param fields Fields to set for the new type. - * @return A new type object. + * Constructs a Text Type using the passed in fields argument. + * @param fields The fields to be used to construct the text type. + * @return A new OpenSeachTextTypeObject */ public static OpenSearchTextType of(Map fields) { var res = new OpenSearchTextType(); - res.fields = ImmutableMap.copyOf(fields); + res.fields = fields; return res; } @@ -59,7 +64,7 @@ public Map getFields() { @Override protected OpenSearchDataType cloneEmpty() { - return OpenSearchTextType.of(fields); + return OpenSearchTextType.of(ImmutableMap.copyOf(this.fields)); } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 0c4548a368..8cdfd0b2ac 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -6,12 +6,8 @@ package org.opensearch.sql.opensearch.data.value; -import static org.opensearch.sql.data.type.ExprCoreType.DATE; -import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; -import static org.opensearch.sql.data.type.ExprCoreType.TIME; -import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER; import com.fasterxml.jackson.core.JsonProcessingException; @@ -19,16 +15,23 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; +import java.time.DateTimeException; import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; +import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; +import org.apache.logging.log4j.LogManager; +import org.opensearch.common.time.DateFormatter; import org.opensearch.common.time.DateFormatters; import org.opensearch.sql.data.model.ExprBooleanValue; import org.opensearch.sql.data.model.ExprByteValue; @@ -46,8 +49,11 @@ import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.expression.function.FunctionProperties; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.utils.Content; import org.opensearch.sql.opensearch.data.utils.ObjectContent; import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent; @@ -85,40 +91,44 @@ public void extendTypeMapping(Map typeMapping) { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final Map> typeActionMap = - new ImmutableMap.Builder>() + private final Map> typeActionMap = + new ImmutableMap.Builder>() .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Integer), - c -> new ExprIntegerValue(c.intValue())) + (c, dt) -> new ExprIntegerValue(c.intValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Long), - c -> new ExprLongValue(c.longValue())) + (c, dt) -> new ExprLongValue(c.longValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Short), - c -> new ExprShortValue(c.shortValue())) + (c, dt) -> new ExprShortValue(c.shortValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Byte), - c -> new ExprByteValue(c.byteValue())) + (c, dt) -> new ExprByteValue(c.byteValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Float), - c -> new ExprFloatValue(c.floatValue())) + (c, dt) -> new ExprFloatValue(c.floatValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Double), - c -> new ExprDoubleValue(c.doubleValue())) + (c, dt) -> new ExprDoubleValue(c.doubleValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Text), - c -> new OpenSearchExprTextValue(c.stringValue())) + (c, dt) -> new OpenSearchExprTextValue(c.stringValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword), - c -> new ExprStringValue(c.stringValue())) + (c, dt) -> new ExprStringValue(c.stringValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Boolean), - c -> ExprBooleanValue.of(c.booleanValue())) - .put(OpenSearchDataType.of(TIMESTAMP), this::parseTimestamp) - .put(OpenSearchDataType.of(DATE), - c -> new ExprDateValue(parseTimestamp(c).dateValue().toString())) - .put(OpenSearchDataType.of(TIME), - c -> new ExprTimeValue(parseTimestamp(c).timeValue().toString())) - .put(OpenSearchDataType.of(DATETIME), - c -> new ExprDatetimeValue(parseTimestamp(c).datetimeValue())) + (c, dt) -> ExprBooleanValue.of(c.booleanValue())) + //Handles the creation of DATE, TIME, TIMESTAMP + .put(OpenSearchDataType.of(ExprCoreType.TIMESTAMP), + (c, dt) -> parseTimestamp(c, dt)) + .put(OpenSearchDataType.of(ExprCoreType.TIME), + (c, dt) -> parseTimestamp(c, dt)) + .put(OpenSearchDataType.of(ExprCoreType.DATETIME), + (c, dt) -> parseTimestamp(c, dt)) + .put(OpenSearchDataType.of(ExprCoreType.DATE), + (c, dt) -> parseTimestamp(c, dt)) + .put(OpenSearchDateType.create(""), + (c, dt) -> parseTimestamp(c, dt)) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip), - c -> new OpenSearchExprIpValue(c.stringValue())) + (c, dt) -> new OpenSearchExprIpValue(c.stringValue())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint), - c -> new OpenSearchExprGeoPointValue(c.geoValue().getLeft(), + (c, dt) -> new OpenSearchExprGeoPointValue(c.geoValue().getLeft(), c.geoValue().getRight())) .put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary), - c -> new OpenSearchExprBinaryValue(c.stringValue())) + (c, dt) -> new OpenSearchExprBinaryValue(c.stringValue())) .build(); /** @@ -173,7 +183,7 @@ private ExprValue parse(Content content, String field, Optional fieldT return parseArray(content, field); } else { if (typeActionMap.containsKey(type)) { - return typeActionMap.get(type).apply(content); + return typeActionMap.get(type).apply(content, type); } else { throw new IllegalStateException( String.format( @@ -210,14 +220,88 @@ private ExprValue constructTimestamp(String value) { } } - private ExprValue parseTimestamp(Content value) { - if (value.isNumber()) { - return new ExprTimestampValue(Instant.ofEpochMilli(value.longValue())); - } else if (value.isString()) { - return constructTimestamp(value.stringValue()); + private TemporalAccessor parseTimestampString(String value, OpenSearchDateType dt) { + for (DateFormatter formatter : dt.getNamedFormatters()) { + try { + return formatter.parse(value); + } catch (IllegalArgumentException ignored) { + // nothing to do, try another format + } + } + return null; + } + + private ExprValue formatReturn(ExprType formatType, ExprTimestampValue unformatted) { + if (formatType.equals(ExprCoreType.DATE)) { + return new ExprDateValue(unformatted.dateValue()); + } + if (formatType.equals(ExprCoreType.DATETIME)) { + return new ExprDatetimeValue(unformatted.datetimeValue()); + } + if (formatType.equals(ExprCoreType.TIME)) { + return new ExprTimeValue(unformatted.timeValue().toString()); + } + return unformatted; + } + + private ExprValue parseTimestamp(Content value, ExprType type) { + OpenSearchDateType dt; + ExprType returnFormat; + if (type instanceof OpenSearchDateType) { + //Case when an OpenSearchDateType is passed in + dt = (OpenSearchDateType) type; + returnFormat = dt.getExprType(); } else { - return new ExprTimestampValue((Instant) value.objectValue()); + //Case when an OpenSearchDataType.of() is passed in + dt = OpenSearchDateType.of(); + returnFormat = ((OpenSearchDataType) type).getExprType(); + } + + if (value.isNumber()) { + return formatReturn( + returnFormat, + new ExprTimestampValue(Instant.ofEpochMilli(value.longValue()))); + } + + if (value.isString()) { + TemporalAccessor parsed = parseTimestampString(value.stringValue(),dt); + if (parsed == null) { // failed to parse or no formats given + return formatReturn( + returnFormat, + (ExprTimestampValue)constructTimestamp(value.stringValue())); + } + // Try Timestamp + try { + return formatReturn(returnFormat, new ExprTimestampValue(Instant.from(parsed))); + } catch (DateTimeException ignored) { + // nothing to do, try another type + } + //Try Time + try { + return formatReturn( + returnFormat, + new ExprTimestampValue( + new ExprTimeValue(LocalTime.from(parsed)) + .timestampValue(new FunctionProperties(Instant.EPOCH, ZoneOffset.UTC)))); + } catch (DateTimeException ignored) { + // nothing to do, try another type + } + //Try Date + try { + return formatReturn( + returnFormat, + new ExprTimestampValue(new ExprDateValue(LocalDate.from(parsed)).timestampValue())); + } catch (DateTimeException ignored) { + LogManager.getLogger(OpenSearchExprValueFactory.class).error( + String.format("Can't recognize parsed value: %s, %s", parsed, parsed.getClass())); + throw new IllegalStateException( + String.format( + "Construct ExprTimestampValue from \"%s\" failed, unsupported date format.", + value.stringValue()), + ignored); + } } + return formatReturn(returnFormat, new ExprTimestampValue((Instant) value.objectValue())); } private ExprValue parseStruct(Content content, String prefix) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java index 4fdcf0c637..0185ca95b6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java @@ -27,10 +27,15 @@ public class IndexMapping { @Getter private final Map fieldMappings; + /** + * Maps each column in the index definition to an OpenSearchSQL datatype. + * @param metaData The metadata retrieved from the index mapping defined by the user. + */ @SuppressWarnings("unchecked") public IndexMapping(MappingMetadata metaData) { - this.fieldMappings = parseMapping((Map) metaData.getSourceAsMap() - .getOrDefault("properties", null)); + this.fieldMappings = OpenSearchDataType.parseMapping( + (Map) metaData.getSourceAsMap().getOrDefault("properties", null) + ); } /** @@ -41,28 +46,4 @@ public IndexMapping(MappingMetadata metaData) { public int size() { return fieldMappings.size(); } - - @SuppressWarnings("unchecked") - private Map parseMapping(Map indexMapping) { - Map result = new LinkedHashMap<>(); - if (indexMapping != null) { - indexMapping.forEach((k, v) -> { - var innerMap = (Map)v; - // TODO: confirm that only `object` mappings can omit `type` field. - var type = ((String) innerMap.getOrDefault("type", "object")).replace("_", ""); - if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) { - // unknown type, e.g. `alias` - // TODO resolve alias reference - return; - } - // TODO read formats for date type - result.put(k, OpenSearchDataType.of( - EnumUtils.getEnumIgnoreCase(OpenSearchDataType.MappingType.class, type), - parseMapping((Map) innerMap.getOrDefault("properties", null)), - parseMapping((Map) innerMap.getOrDefault("fields", null)) - )); - }); - } - return result; - } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 352635bfc3..8af9a4bbfa 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -55,7 +55,6 @@ import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.common.xcontent.XContentType; @@ -425,10 +424,8 @@ private void mockNodeClientSettings(String indexName, String indexMetadata) GetSettingsResponse mockResponse = mock(GetSettingsResponse.class); when(nodeClient.admin().indices().prepareGetSettings(any()).setLocal(anyBoolean()).get()) .thenReturn(mockResponse); - ImmutableOpenMap metadata = - new ImmutableOpenMap.Builder() - .fPut(indexName, IndexMetadata.fromXContent(createParser(indexMetadata)).getSettings()) - .build(); + Map metadata = Map.of(indexName, + IndexMetadata.fromXContent(createParser(indexMetadata)).getSettings()); when(mockResponse.getIndexToSettings()).thenReturn(metadata); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index dd5bfd4e6f..141e21c38a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -49,7 +49,6 @@ import org.opensearch.client.indices.GetMappingsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; -import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; @@ -232,9 +231,9 @@ void getIndexMaxResultWindowsSettings() throws IOException { .put("index.max_result_window", maxResultWindow) .build(); Settings emptySettings = Settings.builder().build(); - ImmutableOpenMap indexToSettings = + Map indexToSettings = mockSettings(indexName, maxResultWindowSettings); - ImmutableOpenMap indexToDefaultSettings = + Map indexToDefaultSettings = mockSettings(indexName, emptySettings); when(response.getIndexToSettings()).thenReturn(indexToSettings); when(response.getIndexToDefaultSettings()).thenReturn(indexToDefaultSettings); @@ -256,9 +255,9 @@ void getIndexMaxResultWindowsDefaultSettings() throws IOException { .put("index.max_result_window", maxResultWindow) .build(); Settings emptySettings = Settings.builder().build(); - ImmutableOpenMap indexToSettings = + Map indexToSettings = mockSettings(indexName, emptySettings); - ImmutableOpenMap indexToDefaultSettings = + Map indexToDefaultSettings = mockSettings(indexName, maxResultWindowSettings); when(response.getIndexToSettings()).thenReturn(indexToSettings); when(response.getIndexToDefaultSettings()).thenReturn(indexToDefaultSettings); @@ -427,10 +426,8 @@ private Map mockFieldMappings(String indexName, String return ImmutableMap.of(indexName, IndexMetadata.fromXContent(createParser(mappings)).mapping()); } - private ImmutableOpenMap mockSettings(String indexName, Settings settings) { - ImmutableOpenMap.Builder indexToSettingsBuilder = ImmutableOpenMap.builder(); - indexToSettingsBuilder.put(indexName, settings); - return indexToSettingsBuilder.build(); + private Map mockSettings(String indexName, Settings settings) { + return Map.of(indexName, settings); } private XContentParser createParser(String mappings) throws IOException { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index 5cd76b1962..fe2865ec21 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -9,6 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -18,6 +19,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; @@ -29,6 +31,8 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; +import java.time.format.DateTimeFormatter; +import java.util.List; import java.util.Map; import java.util.stream.Stream; import org.apache.commons.lang3.reflect.FieldUtils; @@ -39,6 +43,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.common.time.DateFormatter; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -49,6 +54,10 @@ class OpenSearchDataTypeTest { private static final OpenSearchDataType textKeywordType = OpenSearchTextType.of(Map.of("words", OpenSearchTextType.of(MappingType.Keyword))); + private static final String formatString = "epoch_millis || yyyyMMDD"; + + private static final OpenSearchDateType dateType = OpenSearchDateType.create(formatString); + @Test public void isCompatible() { assertTrue(STRING.isCompatible(textType)); @@ -157,15 +166,20 @@ public void of_OpenSearchDataType_from_MappingType(OpenSearchDataType.MappingTyp @Test // All types without `fields` and `properties` are singletones unless cloned. - public void types_but_clones_are_singletones_and_cached() { + public void types_but_clones_are_singletons_and_cached() { var type = OpenSearchDataType.of(MappingType.Object); var alsoType = OpenSearchDataType.of(MappingType.Object); - var typeWithProperties = OpenSearchDataType.of(MappingType.Object, - Map.of("subfield", OpenSearchDataType.of(INTEGER)), Map.of()); - var typeWithFields = OpenSearchDataType.of(MappingType.Text, - Map.of(), Map.of("subfield", OpenSearchDataType.of(INTEGER))); - + Map properties = Map.of( + "properties", + Map.of("number", Map.of("type", "integer"))); + var typeWithProperties = OpenSearchDataType.of( + MappingType.Object, + properties); + var typeWithFields = OpenSearchDataType.of( + MappingType.Text, + Map.of()); var cloneType = type.cloneEmpty(); + assertAll( () -> assertSame(type, alsoType), () -> assertNotSame(type, cloneType), @@ -173,6 +187,7 @@ public void types_but_clones_are_singletones_and_cached() { () -> assertNotSame(type, typeWithFields), () -> assertNotSame(typeWithProperties, typeWithProperties.cloneEmpty()), () -> assertNotSame(typeWithFields, typeWithFields.cloneEmpty()), + () -> assertNotSame(dateType, dateType.cloneEmpty()), () -> assertSame(OpenSearchDataType.of(MappingType.Text), OpenSearchTextType.of()), () -> assertSame(OpenSearchDataType.of(MappingType.Binary), @@ -182,7 +197,7 @@ public void types_but_clones_are_singletones_and_cached() { () -> assertSame(OpenSearchDataType.of(MappingType.Ip), OpenSearchIpType.of()), () -> assertNotSame(OpenSearchTextType.of(), - OpenSearchTextType.of(Map.of("subfield", OpenSearchDataType.of(INTEGER)))), + OpenSearchTextType.of(Map.of("properties", OpenSearchDataType.of(INTEGER)))), () -> assertSame(OpenSearchDataType.of(INTEGER), OpenSearchDataType.of(INTEGER)), () -> assertSame(OpenSearchDataType.of(STRING), OpenSearchDataType.of(STRING)), () -> assertSame(OpenSearchDataType.of(STRUCT), OpenSearchDataType.of(STRUCT)), @@ -213,26 +228,23 @@ public void fields_and_properties_are_readonly() { @Test // Test and type added for coverage only public void of_null_MappingType() { - assertThrows(IllegalArgumentException.class, () -> OpenSearchDataType.of(MappingType.Invalid)); + assertNotNull(OpenSearchDataType.of(MappingType.Invalid)); } @Test // cloneEmpty doesn't clone properties and fields. // Fields are cloned by OpenSearchTextType::cloneEmpty, because it is used in that type only. public void cloneEmpty() { - var type = OpenSearchDataType.of(MappingType.Object, - Map.of("val", OpenSearchDataType.of(INTEGER)), - Map.of("words", OpenSearchDataType.of(STRING))); + var type = OpenSearchDataType.of( + MappingType.Object, + Map.of("val", OpenSearchDataType.of(INTEGER)) + ); var clone = type.cloneEmpty(); var textClone = textKeywordType.cloneEmpty(); assertAll( // can compare because `properties` and `fields` are marked as @EqualsAndHashCode.Exclude () -> assertEquals(type, clone), - // read private field `fields` - () -> assertTrue( - ((Map) FieldUtils.readField(clone, "fields", true)) - .isEmpty()), () -> assertTrue(clone.getProperties().isEmpty()), () -> assertEquals(textKeywordType, textClone), () -> assertEquals(FieldUtils.readField(textKeywordType, "fields", true), @@ -261,17 +273,17 @@ public void cloneEmpty() { // ================= // as // ================= - // type : Object - // type.subtype : Object - // type.subtype.subsubtype : Object - // type.subtype.subsubtype.textWithKeywordType : Text + // mapping : Object + // mapping.submapping : Object + // mapping.submapping.subsubmapping : Object + // mapping.submapping.subsubmapping.textWithKeywordType : Text // |- keyword : Keyword - // type.subtype.subsubtype.INTEGER : INTEGER - // type.subtype.geo_point : GeoPoint - // type.subtype.textWithFieldsType: Text + // mapping.submapping.subsubmapping.INTEGER : INTEGER + // mapping.submapping.geo_point : GeoPoint + // mapping.submapping.textWithFieldsType: Text // |- words : Keyword - // type.text : Text - // type.keyword : Keyword + // mapping.text : Text + // mapping.keyword : Keyword // ================== // Objects are flattened by OpenSearch, but Texts aren't // TODO Arrays @@ -281,28 +293,27 @@ public void traverseAndFlatten() { var objectType = OpenSearchDataType.of(MappingType.Object); assertAll( () -> assertEquals(9, flattened.size()), - () -> assertTrue(flattened.get("type").getProperties().isEmpty()), - () -> assertTrue(flattened.get("type.subtype").getProperties().isEmpty()), - () -> assertTrue(flattened.get("type.subtype.subsubtype").getProperties().isEmpty()), + () -> assertTrue(flattened.get("mapping").getProperties().isEmpty()), + () -> assertTrue(flattened.get("mapping.submapping").getProperties().isEmpty()), + () -> assertTrue( + flattened.get("mapping.submapping.subsubmapping").getProperties().isEmpty()), - () -> assertEquals(objectType, flattened.get("type")), - () -> assertEquals(objectType, flattened.get("type.subtype")), - () -> assertEquals(objectType, flattened.get("type.subtype.subsubtype")), + () -> assertEquals(objectType, flattened.get("mapping")), + () -> assertEquals(objectType, flattened.get("mapping.submapping")), + () -> assertEquals(objectType, flattened.get("mapping.submapping.subsubmapping")), () -> assertEquals(OpenSearchDataType.of(MappingType.Keyword), - flattened.get("type.keyword")), + flattened.get("mapping.keyword")), () -> assertEquals(OpenSearchDataType.of(MappingType.Text), - flattened.get("type.text")), - + flattened.get("mapping.text")), () -> assertEquals(OpenSearchGeoPointType.of(), - flattened.get("type.subtype.geo_point")), + flattened.get("mapping.submapping.geo_point")), () -> assertEquals(OpenSearchTextType.of(), - flattened.get("type.subtype.textWithFieldsType")), - + flattened.get("mapping.submapping.textWithFieldsType")), () -> assertEquals(OpenSearchTextType.of(), - flattened.get("type.subtype.subsubtype.textWithKeywordType")), + flattened.get("mapping.submapping.subsubmapping.texttype")), () -> assertEquals(OpenSearchDataType.of(INTEGER), - flattened.get("type.subtype.subsubtype.INTEGER")) + flattened.get("mapping.submapping.subsubmapping.INTEGER")) ); } @@ -313,13 +324,13 @@ public void resolve() { assertAll( () -> assertNull(OpenSearchDataType.resolve(mapping, "incorrect")), () -> assertEquals(OpenSearchDataType.of(MappingType.Object), - OpenSearchDataType.resolve(mapping, "type")), + OpenSearchDataType.resolve(mapping, "mapping")), () -> assertEquals(OpenSearchDataType.of(MappingType.Object), - OpenSearchDataType.resolve(mapping, "subtype")), + OpenSearchDataType.resolve(mapping, "submapping")), () -> assertEquals(OpenSearchDataType.of(MappingType.Object), - OpenSearchDataType.resolve(mapping, "subsubtype")), + OpenSearchDataType.resolve(mapping, "subsubmapping")), () -> assertEquals(OpenSearchDataType.of(MappingType.Text), - OpenSearchDataType.resolve(mapping, "textWithKeywordType")), + OpenSearchDataType.resolve(mapping, "texttype")), () -> assertEquals(OpenSearchDataType.of(MappingType.Text), OpenSearchDataType.resolve(mapping, "textWithFieldsType")), () -> assertEquals(OpenSearchDataType.of(MappingType.Text), @@ -358,28 +369,31 @@ public void text_type_with_fields_ctor() { } private Map getSampleMapping() { - var textWithKeywordType = OpenSearchTextType.of(Map.of("keyword", - OpenSearchDataType.of(MappingType.Keyword))); + Map subsubmapping = Map.of( + "properties", Map.of( + "texttype", Map.of("type", "text"), + "INTEGER", Map.of("type", "integer") + ) + ); + + Map submapping = Map.of( + "properties", Map.of( + "subsubmapping", subsubmapping, + "textWithFieldsType", Map.of("type", "text", "fieldsType", true), + "geo_point", Map.of("type", "geo_point") + ) + ); - var subsubsubtypes = Map.of( - "textWithKeywordType", textWithKeywordType, - "INTEGER", OpenSearchDataType.of(INTEGER)); - - var subsubtypes = Map.of( - "subsubtype", OpenSearchDataType.of(MappingType.Object, - subsubsubtypes, Map.of()), - "textWithFieldsType", OpenSearchDataType.of(MappingType.Text, Map.of(), - Map.of("words", OpenSearchDataType.of(MappingType.Keyword))), - "geo_point", OpenSearchGeoPointType.of()); - - var subtypes = Map.of( - "subtype", OpenSearchDataType.of(MappingType.Object, - subsubtypes, Map.of()), - "keyword", OpenSearchDataType.of(MappingType.Keyword), - "text", OpenSearchDataType.of(MappingType.Text)); - - var type = OpenSearchDataType.of(MappingType.Object, subtypes, Map.of()); - return Map.of("type", type); + Map types = Map.of( + "properties", Map.of( + "submapping", submapping, + "keyword", Map.of("type", "keyword"), + "text", Map.of("type", "text") + ) + ); + + var mapping = OpenSearchDataType.of(MappingType.Object, types); + return Map.of("mapping", mapping); } @Test @@ -392,4 +406,15 @@ public void test_getExprType() { assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType()); assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType()); } + + @Test + public void test_getRegularFormatters() { + List definedFormatters = dateType.getNamedFormatters(); + assertEquals(definedFormatters.get(0), DateFormatter.forPattern("epoch_millis")); + } + + @Test + public void test_shouldCastFunction() { + assertFalse(dateType.shouldCast(DATE)); + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java index 2dfa5de93a..b60402e746 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java @@ -55,9 +55,9 @@ void non_text_types_arent_converted() { @Test void non_text_types_with_nested_objects_arent_converted() { var objectType = OpenSearchDataType.of(OpenSearchDataType.MappingType.Object, - Map.of("subfield", OpenSearchDataType.of(STRING)), Map.of()); + Map.of("subfield", OpenSearchDataType.of(STRING))); var arrayType = OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested, - Map.of("subfield", OpenSearchDataType.of(STRING)), Map.of()); + Map.of("subfield", OpenSearchDataType.of(STRING))); assertAll( () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", objectType)), () -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", arrayType)) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 8f2c954f65..fb23394ff6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -53,6 +53,7 @@ import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent; @@ -71,6 +72,12 @@ class OpenSearchExprValueFactoryTest { .put("datetimeV", OpenSearchDataType.of(DATETIME)) .put("timeV", OpenSearchDataType.of(TIME)) .put("timestampV", OpenSearchDataType.of(TIMESTAMP)) + .put("hourV", OpenSearchDateType.create("hour")) + .put("epochSecondV", OpenSearchDateType.create("epoch_second")) + .put("dateStringV", OpenSearchDateType.create("date")) + .put("epochMillisV", OpenSearchDateType.create("epoch_millis")) + .put("dateOrEpochMillisV", OpenSearchDateType.create("date_time_no_millis||epoch_millis")) + .put("badDateFormatV", OpenSearchDateType.create("MM,DD")) .put("boolV", OpenSearchDataType.of(BOOLEAN)) .put("structV", OpenSearchDataType.of(STRUCT)) .put("structV.id", OpenSearchDataType.of(INTEGER)) @@ -218,6 +225,9 @@ public void constructDate() { assertEquals( new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), constructFromObject("timestampV", Instant.ofEpochMilli(1420070400001L))); + assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("epochMillisV", "1420070400001")); assertEquals( new ExprTimestampValue("2015-01-01 12:10:30"), constructFromObject("timestampV", "2015-01-01 12:10:30")); @@ -230,6 +240,15 @@ public void constructDate() { assertEquals( new ExprDatetimeValue("2015-01-01 12:10:30"), constructFromObject("datetimeV", "2015-01-01 12:10:30")); + assertEquals( + new ExprDatetimeValue("1970-01-01 09:00:00"), + constructFromObject("hourV", "09")); + assertEquals( + new ExprDatetimeValue("1984-04-12 00:00:00"), + constructFromObject("dateStringV", "1984-04-12")); + assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + constructFromObject("dateOrEpochMillisV", "1420070400001")); } @Test @@ -241,6 +260,19 @@ public void constructDateFromUnsupportedFormatThrowException() { "Construct ExprTimestampValue from \"2015-01-01 12:10\" failed, " + "unsupported date format.", exception.getMessage()); + + exception = assertThrows( + IllegalStateException.class, () -> tupleValue("{\"badDateFormatV\":\"11,22\"}")); + assertEquals( + "Construct ExprTimestampValue from \"11,22\" failed, " + + "unsupported date format.", + exception.getMessage()); + exception = assertThrows( + IllegalStateException.class, () -> tupleValue("{\"dateStringV\":\"2023-11\"}")); + assertEquals( + "Construct ExprTimestampValue from \"2023-11\" failed, " + + "unsupported date format.", + exception.getMessage()); } @Test @@ -431,7 +463,7 @@ private ExprValue constructFromObject(String fieldName, Object value) { private static class TestType extends OpenSearchDataType { public TestType() { - mappingType = null; + super(MappingType.Invalid); } @Override