[ML] Remove "8" prefixes from file structure finder timestamp formats (…

…elastic#38016) In 7.x Java timestamp formats are the default timestamp format and there is no need to prefix them with "8". (The "8" prefix was used in 6.7 to distinguish Java timestamp formats from Joda timestamp formats.) This change removes the "8" prefixes from timestamp formats in the output of the ML file structure finder.
jaymode · Feb 1, 2019 · 1fa413a · 1fa413a
1 parent 5c58c25
commit 1fa413a
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 29 deletions.
diff --git a/docs/reference/ml/apis/find-file-structure.asciidoc b/docs/reference/ml/apis/find-file-structure.asciidoc
@@ -606,11 +606,11 @@ If the request does not encounter errors, you receive the following result:
     },
     "tpep_dropoff_datetime" : {
       "type" : "date",
-      "format" : "8yyyy-MM-dd HH:mm:ss"
+      "format" : "yyyy-MM-dd HH:mm:ss"
     },
     "tpep_pickup_datetime" : {
       "type" : "date",
-      "format" : "8yyyy-MM-dd HH:mm:ss"
+      "format" : "yyyy-MM-dd HH:mm:ss"
     },
     "trip_distance" : {
       "type" : "double"
@@ -624,7 +624,7 @@ If the request does not encounter errors, you receive the following result:
           "field" : "tpep_pickup_datetime",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd HH:mm:ss"
+            "yyyy-MM-dd HH:mm:ss"
           ]
         }
       }
@@ -1398,7 +1398,7 @@ this:
           "field" : "timestamp",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd'T'HH:mm:ss,SSS"
+            "yyyy-MM-dd'T'HH:mm:ss,SSS"
           ]
         }
       },
@@ -1558,7 +1558,7 @@ this:
           "field" : "timestamp",
           "timezone" : "{{ beat.timezone }}",
           "formats" : [
-            "8yyyy-MM-dd'T'HH:mm:ss,SSS"
+            "yyyy-MM-dd'T'HH:mm:ss,SSS"
           ]
         }
       },

diff --git a/...n/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java b/...n/ml/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtils.java
@@ -353,7 +353,7 @@ public static Map<String, Object> makeIngestPipelineDefinition(String grokPatter
             if (needClientTimezone) {
                 dateProcessorSettings.put("timezone", "{{ " + BEAT_TIMEZONE_FIELD + " }}");
             }
-            dateProcessorSettings.put("formats", jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats));
+            dateProcessorSettings.put("formats", timestampFormats);
             processors.add(Collections.singletonMap("date", dateProcessorSettings));
         }
 
@@ -365,19 +365,4 @@ public static Map<String, Object> makeIngestPipelineDefinition(String grokPatter
         pipeline.put(Pipeline.PROCESSORS_KEY, processors);
         return pipeline;
     }
-
-    // TODO: remove this method when Java time formats are the default
-    static List<String> jodaBwcJavaTimestampFormatsForIngestPipeline(List<String> javaTimestampFormats) {
-        return javaTimestampFormats.stream().map(format -> {
-            switch (format) {
-                case "ISO8601":
-                case "UNIX_MS":
-                case "UNIX":
-                case "TAI64N":
-                    return format;
-                default:
-                    return "8" + format;
-            }
-        }).collect(Collectors.toList());
-    }
 }
diff --git a/...l/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java b/...l/src/main/java/org/elasticsearch/xpack/ml/filestructurefinder/TimestampFormatFinder.java
@@ -472,8 +472,7 @@ public Map<String, String> getEsDateMappingTypeWithFormat() {
                     case "UNIX":
                         return Stream.of("epoch_second");
                     default:
-                        // TODO: remove the "8" prefix when Java time formats are the default
-                        return Stream.of("8" + format);
+                        return Stream.of(format);
                 }
             }).collect(Collectors.joining("||"));
             if (formats.isEmpty() == false) {

diff --git a/...src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java b/...src/test/java/org/elasticsearch/xpack/ml/filestructurefinder/FileStructureUtilsTests.java
@@ -331,8 +331,7 @@ public void testGuessMappingsAndCalculateFieldStats() {
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("foo"));
         Map<String, String> expectedTimeMapping = new HashMap<>();
         expectedTimeMapping.put(FileStructureUtils.MAPPING_TYPE_SETTING, "date");
-        // TODO: remove the "8" prefix when Java time formats are the default
-        expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "8" + "yyyy-MM-dd HH:mm:ss,SSS");
+        expectedTimeMapping.put(FileStructureUtils.MAPPING_FORMAT_SETTING, "yyyy-MM-dd HH:mm:ss,SSS");
         assertEquals(expectedTimeMapping, mappings.get("time"));
         assertEquals(Collections.singletonMap(FileStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("bar"));
         assertNull(mappings.get("nothing"));
@@ -372,8 +371,7 @@ public void testMakeIngestPipelineDefinitionGivenStructuredWithTimestamp() {
         assertNotNull(dateProcessor);
         assertEquals(timestampField, dateProcessor.get("field"));
         assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
-        // TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
-        assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
+        assertEquals(timestampFormats, dateProcessor.get("formats"));
 
         // After removing the two expected fields there should be nothing left in the pipeline
         assertEquals(Collections.emptyMap(), pipeline);
@@ -406,8 +404,7 @@ public void testMakeIngestPipelineDefinitionGivenSemiStructured() {
         assertNotNull(dateProcessor);
         assertEquals(timestampField, dateProcessor.get("field"));
         assertEquals(needClientTimezone, dateProcessor.containsKey("timezone"));
-        // TODO: remove the call to jodaBwcJavaTimestampFormatsForIngestPipeline() when Java time formats are the default
-        assertEquals(FileStructureUtils.jodaBwcJavaTimestampFormatsForIngestPipeline(timestampFormats), dateProcessor.get("formats"));
+        assertEquals(timestampFormats, dateProcessor.get("formats"));
 
         Map<String, Object> removeProcessor = (Map<String, Object>) processors.get(2).get("remove");
         assertNotNull(removeProcessor);