Enabled specifying the format once the transform date_histogram is us…

src/main/kotlin/org/opensearch/indexmanagement/common/model/dimension/DateHistogram.kt

@@ -28,7 +28,8 @@ data class DateHistogram(
     override val targetField: String = sourceField,
     val fixedInterval: String? = null,
     val calendarInterval: String? = null,
-    val timezone: ZoneId = ZoneId.of(UTC)
+    val timezone: ZoneId = ZoneId.of(UTC),
+    val format: String? = null
 ) : Dimension(Type.DATE_HISTOGRAM, sourceField, targetField) {
 
     init {
@@ -54,6 +55,7 @@ data class DateHistogram(
         return builder.field(DIMENSION_SOURCE_FIELD_FIELD, sourceField)
             .field(DIMENSION_TARGET_FIELD_FIELD, targetField)
             .field(DATE_HISTOGRAM_TIMEZONE_FIELD, timezone.id)
+            .field(FORMAT, format)
             .endObject()
             .endObject()
     }
@@ -69,6 +71,7 @@ data class DateHistogram(
     override fun toSourceBuilder(appendType: Boolean): CompositeValuesSourceBuilder<*> {
         val calendarInterval = this.calendarInterval
         val fixedInterval = this.fixedInterval
+        val format = this.format
         val name = if (appendType) "${this.targetField}.${Type.DATE_HISTOGRAM.type}" else this.targetField
 
         return DateHistogramValuesSourceBuilder(name)
@@ -82,6 +85,9 @@ data class DateHistogram(
                 fixedInterval?.let {
                     this.fixedInterval(DateHistogramInterval(it))
                 }
+                format?.let {
+                    this.format(it)
+                }
             }
     }
 
@@ -128,6 +134,7 @@ data class DateHistogram(
         const val FIXED_INTERVAL_FIELD = "fixed_interval"
         const val CALENDAR_INTERVAL_FIELD = "calendar_interval"
         const val DATE_HISTOGRAM_TIMEZONE_FIELD = "timezone"
+        const val FORMAT = "format"
 
         @Suppress("ComplexMethod", "LongMethod")
         @JvmStatic
@@ -138,6 +145,7 @@ data class DateHistogram(
             var fixedInterval: String? = null
             var calendarInterval: String? = null
             var timezone = ZoneId.of(UTC)
+            var format: String? = null
 
             ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp)
             while (xcp.nextToken() != Token.END_OBJECT) {
@@ -150,6 +158,7 @@ data class DateHistogram(
                     DATE_HISTOGRAM_TIMEZONE_FIELD -> timezone = ZoneId.of(xcp.text())
                     DIMENSION_SOURCE_FIELD_FIELD -> sourceField = xcp.text()
                     DIMENSION_TARGET_FIELD_FIELD -> targetField = xcp.text()
+                    FORMAT -> format = xcp.textOrNull()
                     else -> throw IllegalArgumentException("Invalid field [$fieldName] found in date histogram")
                 }
             }
@@ -159,7 +168,8 @@ data class DateHistogram(
                 targetField = requireNotNull(targetField) { "Target field must not be null" },
                 fixedInterval = fixedInterval,
                 calendarInterval = calendarInterval,
-                timezone = timezone
+                timezone = timezone,
+                format = format
             )
         }
 

src/main/kotlin/org/opensearch/indexmanagement/transform/TransformSearchService.kt

@@ -31,6 +31,7 @@ import org.opensearch.index.query.RangeQueryBuilder
 import org.opensearch.index.seqno.SequenceNumbers
 import org.opensearch.index.shard.ShardId
 import org.opensearch.indexmanagement.common.model.dimension.Dimension
+import org.opensearch.indexmanagement.opensearchapi.convertToMap
 import org.opensearch.indexmanagement.opensearchapi.retry
 import org.opensearch.indexmanagement.opensearchapi.suspendUntil
 import org.opensearch.indexmanagement.transform.exceptions.TransformSearchServiceException
@@ -374,7 +375,13 @@ class TransformSearchService(
             return when (aggregation) {
                 is InternalSum, is InternalMin, is InternalMax, is InternalAvg, is InternalValueCount -> {
                     val agg = aggregation as NumericMetricsAggregation.SingleValue
-                    agg.value()
+                    // If value and value_as_string differs (ie. for date fields), transform should be aware of
+                    if (agg.value().toString() == agg.valueAsString) {
+                        agg.value()
+                    } else {
+                        // example of agg - {"min_timestamp":{"value":1.662856742912E12,"value_as_string":"2022-09-11T00:39:02.912Z"}}
+                        agg.convertToMap()[agg.name] ?: agg.value()
+                    }
                 }
                 is Percentiles -> {
                     val percentiles = mutableMapOf<String, Double>()

src/main/resources/mappings/opendistro-ism-config.json

@@ -1,6 +1,6 @@
 {
   "_meta" : {
-    "schema_version": 17
+    "schema_version": 18
   },
   "dynamic": "strict",
   "properties": {
@@ -1197,6 +1197,9 @@
                 },
                 "timezone": {
                   "type": "keyword"
+                },
+                "format": {
+                  "type": "keyword"
                 }
               }
             },

src/test/kotlin/org/opensearch/indexmanagement/IndexManagementRestTestCase.kt

@@ -38,7 +38,7 @@ import kotlin.collections.HashSet
 
 abstract class IndexManagementRestTestCase : ODFERestTestCase() {
 
-    val configSchemaVersion = 17
+    val configSchemaVersion = 18
     val historySchemaVersion = 5
 
     // Having issues with tests leaking into other tests and mappings being incorrect and they are not caught by any pending task wait check as

src/test/resources/mappings/cached-opendistro-ism-config.json

@@ -1,6 +1,6 @@
 {
   "_meta" : {
-    "schema_version": 17
+    "schema_version": 18
   },
   "dynamic": "strict",
   "properties": {
@@ -1197,6 +1197,9 @@
                 },
                 "timezone": {
                   "type": "keyword"
+                },
+                "format": {
+                  "type": "keyword"
                 }
               }
             },