Added New DateTime parser implementation (opensearch-project#11465)

Signed-off-by: Prabhat Sharma <[email protected]> Co-authored-by: Prabhat Sharma <[email protected]> Signed-off-by: Shivansh Arora <[email protected]>
Gaurav614 · Apr 25, 2024 · 079b9c9 · 079b9c9
1 parent 3bea438
commit 079b9c9
Show file tree

Hide file tree

Showing 10 changed files with 796 additions and 37 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -133,6 +133,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump OpenTelemetry from 1.32.0 to 1.34.1 ([#11891](https://github.com/opensearch-project/OpenSearch/pull/11891))
 - Support index level allocation filtering for searchable snapshot index ([#11563](https://github.com/opensearch-project/OpenSearch/pull/11563))
 - Add `org.opensearch.rest.MethodHandlers` and `RestController#getAllHandlers` ([11876](https://github.com/opensearch-project/OpenSearch/pull/11876))
+- New DateTime format for RFC3339 compatible date fields ([#11465](https://github.com/opensearch-project/OpenSearch/pull/11465))
 
 ### Dependencies
 - Bumps jetty version to 9.4.52.v20230823 to fix GMS-2023-1857 ([#9822](https://github.com/opensearch-project/OpenSearch/pull/9822))

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -10,3 +10,6 @@ Foundation (http://www.apache.org/).
 
 This product includes software developed by
 Joda.org (http://www.joda.org/).
+
+This product includes software developed by
+Morten Haraldsen (ethlo) (https://github.com/ethlo) under the Apache License, version 2.0.
diff --git a/server/src/main/java/org/opensearch/common/time/DateFormatters.java b/server/src/main/java/org/opensearch/common/time/DateFormatters.java
@@ -1299,6 +1299,41 @@ public class DateFormatters {
             .withResolverStyle(ResolverStyle.STRICT)
     );
 
+    /**
+     * Returns RFC 3339 a popular ISO 8601 profile compatible date time formatter and parser.
+     * This is not fully compatible to the existing spec, its more linient and closely follows w3c note on datetime
+     */
+
+    public static final DateFormatter RFC3339_LENIENT_DATE_FORMATTER = new JavaDateFormatter(
+        "rfc3339_lenient",
+        new OpenSearchDateTimeFormatter(STRICT_DATE_OPTIONAL_TIME_PRINTER),
+        new RFC3339CompatibleDateTimeFormatter(
+            new DateTimeFormatterBuilder().append(DATE_FORMATTER)
+                .optionalStart()
+                .appendLiteral('T')
+                .appendValue(HOUR_OF_DAY, 1, 2, SignStyle.NOT_NEGATIVE)
+                .appendLiteral(':')
+                .appendValue(MINUTE_OF_HOUR, 1, 2, SignStyle.NOT_NEGATIVE)
+                .optionalStart()
+                .appendLiteral(':')
+                .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
+                .optionalStart()
+                .appendFraction(NANO_OF_SECOND, 1, 9, true)
+                .optionalEnd()
+                .optionalStart()
+                .appendLiteral(',')
+                .appendFraction(NANO_OF_SECOND, 1, 9, false)
+                .optionalEnd()
+                .optionalStart()
+                .appendOffsetId()
+                .optionalEnd()
+                .optionalEnd()
+                .optionalEnd()
+                .toFormatter(Locale.ROOT)
+                .withResolverStyle(ResolverStyle.STRICT)
+        )
+    );
+
     private static final DateTimeFormatter HOUR_MINUTE_SECOND_FORMATTER = new DateTimeFormatterBuilder().append(HOUR_MINUTE_FORMATTER)
         .appendLiteral(":")
         .appendValue(SECOND_OF_MINUTE, 1, 2, SignStyle.NOT_NEGATIVE)
@@ -2152,6 +2187,8 @@ static DateFormatter forPattern(String input) {
             return STRICT_YEAR_MONTH;
         } else if (FormatNames.STRICT_YEAR_MONTH_DAY.matches(input)) {
             return STRICT_YEAR_MONTH_DAY;
+        } else if (FormatNames.RFC3339_LENIENT.matches(input)) {
+            return RFC3339_LENIENT_DATE_FORMATTER;
         } else {
             try {
                 return new JavaDateFormatter(

diff --git a/server/src/main/java/org/opensearch/common/time/FormatNames.java b/server/src/main/java/org/opensearch/common/time/FormatNames.java
@@ -44,6 +44,7 @@
  */
 public enum FormatNames {
     ISO8601(null, "iso8601"),
+    RFC3339_LENIENT(null, "rfc3339_lenient"),
     BASIC_DATE("basicDate", "basic_date"),
     BASIC_DATE_TIME("basicDateTime", "basic_date_time"),
     BASIC_DATE_TIME_NO_MILLIS("basicDateTimeNoMillis", "basic_date_time_no_millis"),

diff --git a/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java b/server/src/main/java/org/opensearch/common/time/JavaDateFormatter.java
@@ -36,6 +36,7 @@
 import org.opensearch.core.common.Strings;
 
 import java.text.ParsePosition;
+import java.time.DateTimeException;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeFormatterBuilder;
@@ -52,7 +53,6 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
@@ -70,11 +70,11 @@ class JavaDateFormatter implements DateFormatter {
 
     private final String format;
     private final String printFormat;
-    private final DateTimeFormatter printer;
-    private final List<DateTimeFormatter> parsers;
+    private final OpenSearchDateTimePrinter printer;
+    private final List<OpenSearchDateTimeFormatter> parsers;
     private final JavaDateFormatter roundupParser;
     private final Boolean canCacheLastParsedFormatter;
-    private volatile DateTimeFormatter lastParsedformatter = null;
+    private volatile OpenSearchDateTimeFormatter lastParsedformatter = null;
 
     /**
      * A round up formatter
@@ -83,11 +83,11 @@ class JavaDateFormatter implements DateFormatter {
      */
     static class RoundUpFormatter extends JavaDateFormatter {
 
-        RoundUpFormatter(String format, List<DateTimeFormatter> roundUpParsers) {
+        RoundUpFormatter(String format, List<OpenSearchDateTimeFormatter> roundUpParsers) {
             super(format, firstFrom(roundUpParsers), null, roundUpParsers);
         }
 
-        private static DateTimeFormatter firstFrom(List<DateTimeFormatter> roundUpParsers) {
+        private static OpenSearchDateTimeFormatter firstFrom(List<OpenSearchDateTimeFormatter> roundUpParsers) {
             return roundUpParsers.get(0);
         }
 
@@ -101,14 +101,18 @@ JavaDateFormatter getRoundupParser() {
     JavaDateFormatter(
         String format,
         String printFormat,
-        DateTimeFormatter printer,
+        OpenSearchDateTimePrinter printer,
         Boolean canCacheLastParsedFormatter,
-        DateTimeFormatter... parsers
+        OpenSearchDateTimeFormatter... parsers
     ) {
         this(format, printFormat, printer, ROUND_UP_BASE_FIELDS, canCacheLastParsedFormatter, parsers);
     }
 
     JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter... parsers) {
+        this(format, format, wrapFormatter(printer), false, wrapAllFormatters(parsers));
+    }
+
+    JavaDateFormatter(String format, OpenSearchDateTimePrinter printer, OpenSearchDateTimeFormatter... parsers) {
         this(format, format, printer, false, parsers);
     }
 
@@ -127,19 +131,19 @@ JavaDateFormatter getRoundupParser() {
     JavaDateFormatter(
         String format,
         String printFormat,
-        DateTimeFormatter printer,
+        OpenSearchDateTimePrinter printer,
         BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer,
         Boolean canCacheLastParsedFormatter,
-        DateTimeFormatter... parsers
+        OpenSearchDateTimeFormatter... parsers
     ) {
         if (printer == null) {
             throw new IllegalArgumentException("printer may not be null");
         }
-        long distinctZones = Arrays.stream(parsers).map(DateTimeFormatter::getZone).distinct().count();
+        long distinctZones = Arrays.stream(parsers).map(OpenSearchDateTimeFormatter::getZone).distinct().count();
         if (distinctZones > 1) {
             throw new IllegalArgumentException("formatters must have the same time zone");
         }
-        long distinctLocales = Arrays.stream(parsers).map(DateTimeFormatter::getLocale).distinct().count();
+        long distinctLocales = Arrays.stream(parsers).map(OpenSearchDateTimeFormatter::getLocale).distinct().count();
         if (distinctLocales > 1) {
             throw new IllegalArgumentException("formatters must have the same locale");
         }
@@ -149,12 +153,12 @@ JavaDateFormatter getRoundupParser() {
         this.canCacheLastParsedFormatter = canCacheLastParsedFormatter;
 
         if (parsers.length == 0) {
-            this.parsers = Collections.singletonList(printer);
+            this.parsers = Collections.singletonList((OpenSearchDateTimeFormatter) printer);
         } else {
             this.parsers = Arrays.asList(parsers);
         }
         List<DateTimeFormatter> roundUp = createRoundUpParser(format, roundupParserConsumer);
-        this.roundupParser = new RoundUpFormatter(format, roundUp);
+        this.roundupParser = new RoundUpFormatter(format, wrapAllFormatters(roundUp));
     }
 
     JavaDateFormatter(
@@ -163,7 +167,7 @@ JavaDateFormatter getRoundupParser() {
         BiConsumer<DateTimeFormatterBuilder, DateTimeFormatter> roundupParserConsumer,
         DateTimeFormatter... parsers
     ) {
-        this(format, format, printer, roundupParserConsumer, false, parsers);
+        this(format, format, wrapFormatter(printer), roundupParserConsumer, false, wrapAllFormatters(parsers));
     }
 
     /**
@@ -181,7 +185,8 @@ private List<DateTimeFormatter> createRoundUpParser(
     ) {
         if (format.contains("||") == false) {
             List<DateTimeFormatter> roundUpParsers = new ArrayList<>();
-            for (DateTimeFormatter parser : this.parsers) {
+            for (OpenSearchDateTimeFormatter customparser : this.parsers) {
+                DateTimeFormatter parser = customparser.getFormatter();
                 DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
                 builder.append(parser);
                 roundupParserConsumer.accept(builder, parser);
@@ -201,12 +206,12 @@ public static DateFormatter combined(
         assert formatters.size() > 0;
         assert printFormatter != null;
 
-        List<DateTimeFormatter> parsers = new ArrayList<>(formatters.size());
-        List<DateTimeFormatter> roundUpParsers = new ArrayList<>(formatters.size());
+        List<OpenSearchDateTimeFormatter> parsers = new ArrayList<>(formatters.size());
+        List<OpenSearchDateTimeFormatter> roundUpParsers = new ArrayList<>(formatters.size());
 
         assert printFormatter instanceof JavaDateFormatter;
         JavaDateFormatter javaPrintFormatter = (JavaDateFormatter) printFormatter;
-        DateTimeFormatter printer = javaPrintFormatter.getPrinter();
+        OpenSearchDateTimePrinter printer = javaPrintFormatter.getPrinter();
         for (DateFormatter formatter : formatters) {
             assert formatter instanceof JavaDateFormatter;
             JavaDateFormatter javaDateFormatter = (JavaDateFormatter) formatter;
@@ -227,9 +232,9 @@ public static DateFormatter combined(
     private JavaDateFormatter(
         String format,
         String printFormat,
-        DateTimeFormatter printer,
-        List<DateTimeFormatter> roundUpParsers,
-        List<DateTimeFormatter> parsers,
+        OpenSearchDateTimePrinter printer,
+        List<OpenSearchDateTimeFormatter> roundUpParsers,
+        List<OpenSearchDateTimeFormatter> parsers,
         Boolean canCacheLastParsedFormatter
     ) {
         this.format = format;
@@ -245,6 +250,15 @@ private JavaDateFormatter(
         DateTimeFormatter printer,
         List<DateTimeFormatter> roundUpParsers,
         List<DateTimeFormatter> parsers
+    ) {
+        this(format, format, wrapFormatter(printer), wrapAllFormatters(roundUpParsers), wrapAllFormatters(parsers), false);
+    }
+
+    private JavaDateFormatter(
+        String format,
+        OpenSearchDateTimePrinter printer,
+        List<OpenSearchDateTimeFormatter> roundUpParsers,
+        List<OpenSearchDateTimeFormatter> parsers
     ) {
         this(format, format, printer, roundUpParsers, parsers, false);
     }
@@ -253,7 +267,7 @@ JavaDateFormatter getRoundupParser() {
         return roundupParser;
     }
 
-    DateTimeFormatter getPrinter() {
+    OpenSearchDateTimePrinter getPrinter() {
         return printer;
     }
 
@@ -265,7 +279,7 @@ public TemporalAccessor parse(String input) {
 
         try {
             return doParse(input);
-        } catch (DateTimeParseException e) {
+        } catch (DateTimeException e) {
             throw new IllegalArgumentException("failed to parse date field [" + input + "] with format [" + format + "]", e);
         }
     }
@@ -289,14 +303,14 @@ private TemporalAccessor doParse(String input) {
             Object object = null;
             if (canCacheLastParsedFormatter && lastParsedformatter != null) {
                 ParsePosition pos = new ParsePosition(0);
-                object = lastParsedformatter.toFormat().parseObject(input, pos);
+                object = lastParsedformatter.parseObject(input, pos);
                 if (parsingSucceeded(object, input, pos)) {
                     return (TemporalAccessor) object;
                 }
             }
-            for (DateTimeFormatter formatter : parsers) {
+            for (OpenSearchDateTimeFormatter formatter : parsers) {
                 ParsePosition pos = new ParsePosition(0);
-                object = formatter.toFormat().parseObject(input, pos);
+                object = formatter.parseObject(input, pos);
                 if (parsingSucceeded(object, input, pos)) {
                     lastParsedformatter = formatter;
                     return (TemporalAccessor) object;
@@ -312,16 +326,28 @@ private boolean parsingSucceeded(Object object, String input, ParsePosition pos)
         return object != null && pos.getIndex() == input.length();
     }
 
+    private static OpenSearchDateTimeFormatter wrapFormatter(DateTimeFormatter formatter) {
+        return new OpenSearchDateTimeFormatter(formatter);
+    }
+
+    private static OpenSearchDateTimeFormatter[] wrapAllFormatters(DateTimeFormatter... formatters) {
+        return Arrays.stream(formatters).map(JavaDateFormatter::wrapFormatter).toArray(OpenSearchDateTimeFormatter[]::new);
+    }
+
+    private static List<OpenSearchDateTimeFormatter> wrapAllFormatters(List<DateTimeFormatter> formatters) {
+        return formatters.stream().map(JavaDateFormatter::wrapFormatter).collect(Collectors.toList());
+    }
+
     @Override
     public DateFormatter withZone(ZoneId zoneId) {
         // shortcurt to not create new objects unnecessarily
         if (zoneId.equals(zone())) {
             return this;
         }
-        List<DateTimeFormatter> parsers = new CopyOnWriteArrayList<>(
+        List<OpenSearchDateTimeFormatter> parsers = new ArrayList<>(
             this.parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList())
         );
-        List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
+        List<OpenSearchDateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
             .stream()
             .map(p -> p.withZone(zoneId))
             .collect(Collectors.toList());
@@ -334,10 +360,10 @@ public DateFormatter withLocale(Locale locale) {
         if (locale.equals(locale())) {
             return this;
         }
-        List<DateTimeFormatter> parsers = new CopyOnWriteArrayList<>(
+        List<OpenSearchDateTimeFormatter> parsers = new ArrayList<>(
             this.parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList())
         );
-        List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
+        List<OpenSearchDateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
             .stream()
             .map(p -> p.withLocale(locale))
             .collect(Collectors.toList());
@@ -396,7 +422,7 @@ public String toString() {
         return String.format(Locale.ROOT, "format[%s] locale[%s]", format, locale());
     }
 
-    Collection<DateTimeFormatter> getParsers() {
+    Collection<OpenSearchDateTimeFormatter> getParsers() {
         return parsers;
     }
 }