opendistro-for-elasticsearch · akbhatta · Nov 10, 2020 · Nov 10, 2020
diff --git a/reports-scheduler/src/main/config/reports-scheduler.yml b/reports-scheduler/src/main/config/reports-scheduler.yml
@@ -24,3 +24,4 @@ opendistro.reports.scheduler:
     minPollingDurationSeconds: 300 # 5 Minutes, Minimum 60 seconds
     maxPollingDurationSeconds: 900 # 15 Minutes, Minimum 5 Minutes
     maxLockRetries: 1 # Max number of retries to retry locking
+    defaultItemsQueryCount: 100 # default number of items to query
diff --git a/.../com/amazon/opendistroforelasticsearch/reportsscheduler/action/ReportDefinitionActions.kt b/.../com/amazon/opendistroforelasticsearch/reportsscheduler/action/ReportDefinitionActions.kt
@@ -122,9 +122,9 @@ internal object ReportDefinitionActions {
      * @return [GetAllReportDefinitionsResponse]
      */
     fun getAll(request: GetAllReportDefinitionsRequest): GetAllReportDefinitionsResponse {
-        log.info("$LOG_PREFIX:ReportDefinition-getAll ${request.fromIndex}")
+        log.info("$LOG_PREFIX:ReportDefinition-getAll fromIndex:${request.fromIndex} maxItems:${request.maxItems}")
         // TODO verify actual requester ID
-        val reportDefinitionsList = ReportDefinitionsIndex.getAllReportDefinitions(listOf(TEMP_ROLE_ID), request.fromIndex)
+        val reportDefinitionsList = ReportDefinitionsIndex.getAllReportDefinitions(listOf(TEMP_ROLE_ID), request.fromIndex, request.maxItems)
         return GetAllReportDefinitionsResponse(reportDefinitionsList)
     }
 }
diff --git a/...in/com/amazon/opendistroforelasticsearch/reportsscheduler/action/ReportInstanceActions.kt b/...in/com/amazon/opendistroforelasticsearch/reportsscheduler/action/ReportInstanceActions.kt
@@ -143,9 +143,9 @@ internal object ReportInstanceActions {
      * @return [GetAllReportInstancesResponse]
      */
     fun getAll(request: GetAllReportInstancesRequest): GetAllReportInstancesResponse {
-        log.info("$LOG_PREFIX:ReportInstance-getAll ${request.fromIndex}")
+        log.info("$LOG_PREFIX:ReportInstance-getAll fromIndex:${request.fromIndex} maxItems:${request.maxItems}")
         // TODO verify actual requester ID
-        val reportInstanceList = ReportInstancesIndex.getAllReportInstances(listOf(TEMP_ROLE_ID), request.fromIndex)
+        val reportInstanceList = ReportInstancesIndex.getAllReportInstances(listOf(TEMP_ROLE_ID), request.fromIndex, request.maxItems)
         return GetAllReportInstancesResponse(reportInstanceList)
     }
 

diff --git a/...in/com/amazon/opendistroforelasticsearch/reportsscheduler/index/ReportDefinitionsIndex.kt b/...in/com/amazon/opendistroforelasticsearch/reportsscheduler/index/ReportDefinitionsIndex.kt
@@ -18,6 +18,7 @@ package com.amazon.opendistroforelasticsearch.reportsscheduler.index
 
 import com.amazon.opendistroforelasticsearch.reportsscheduler.ReportsSchedulerPlugin.Companion.LOG_PREFIX
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.ReportDefinitionDetails
+import com.amazon.opendistroforelasticsearch.reportsscheduler.model.ReportDefinitionDetailsSearchResults
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.ACCESS_LIST_FIELD
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.UPDATED_TIME_FIELD
 import com.amazon.opendistroforelasticsearch.reportsscheduler.settings.PluginSettings
@@ -50,7 +51,6 @@ internal object ReportDefinitionsIndex {
     private const val REPORT_DEFINITIONS_MAPPING_FILE_NAME = "report-definitions-mapping.yml"
     private const val REPORT_DEFINITIONS_SETTINGS_FILE_NAME = "report-definitions-settings.yml"
     private const val MAPPING_TYPE = "_doc"
-    private const val MAX_ITEMS_TO_QUERY = 10000
 
     private lateinit var client: Client
     private lateinit var clusterService: ClusterService
@@ -149,31 +149,24 @@ internal object ReportDefinitionsIndex {
      * Query index for report definition for given access details
      * @param access the list of access details to search reports for.
      * @param from the paginated start index
-     * @return list of Report definition details
+     * @param maxItems the max items to query
+     * @return search result of Report definition details
      */
-    fun getAllReportDefinitions(access: List<String>, from: Int): List<ReportDefinitionDetails> {
+    fun getAllReportDefinitions(access: List<String>, from: Int, maxItems: Int): ReportDefinitionDetailsSearchResults {
         createIndex()
         val query = QueryBuilders.termsQuery(ACCESS_LIST_FIELD, access)
         val sourceBuilder = SearchSourceBuilder()
             .timeout(TimeValue(PluginSettings.operationTimeoutMs, TimeUnit.MILLISECONDS))
             .sort(UPDATED_TIME_FIELD)
-            .size(MAX_ITEMS_TO_QUERY)
+            .size(maxItems)
             .from(from)
             .query(query)
         val searchRequest = SearchRequest()
             .indices(REPORT_DEFINITIONS_INDEX_NAME)
             .source(sourceBuilder)
         val actionFuture = client.search(searchRequest)
         val response = actionFuture.actionGet(PluginSettings.operationTimeoutMs)
-        val mutableList: MutableList<ReportDefinitionDetails> = mutableListOf()
-        response.hits.forEach {
-            val parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
-                LoggingDeprecationHandler.INSTANCE,
-                it.sourceAsString)
-            parser.nextToken()
-            mutableList.add(ReportDefinitionDetails.parse(parser, it.id))
-        }
-        return mutableList
+        return ReportDefinitionDetailsSearchResults(from.toLong(), response)
     }
 
     /**

diff --git a/...tlin/com/amazon/opendistroforelasticsearch/reportsscheduler/index/ReportInstancesIndex.kt b/...tlin/com/amazon/opendistroforelasticsearch/reportsscheduler/index/ReportInstancesIndex.kt
@@ -20,6 +20,7 @@ import com.amazon.opendistroforelasticsearch.reportsscheduler.ReportsSchedulerPl
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.ReportInstance
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.ReportInstance.Status
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.ReportInstanceDoc
+import com.amazon.opendistroforelasticsearch.reportsscheduler.model.ReportInstanceSearchResults
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.ACCESS_LIST_FIELD
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.STATUS_FIELD
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.UPDATED_TIME_FIELD
@@ -54,7 +55,6 @@ internal object ReportInstancesIndex {
     private const val REPORT_INSTANCES_MAPPING_FILE_NAME = "report-instances-mapping.yml"
     private const val REPORT_INSTANCES_SETTINGS_FILE_NAME = "report-instances-settings.yml"
     private const val MAPPING_TYPE = "_doc"
-    private const val MAX_ITEMS_TO_QUERY = 10000
 
     private lateinit var client: Client
     private lateinit var clusterService: ClusterService
@@ -153,31 +153,24 @@ internal object ReportInstancesIndex {
      * Query index for report instance for given access details
      * @param access the list of access details to search reports for.
      * @param from the paginated start index
-     * @return list of Report instance details
+     * @param maxItems the max items to query
+     * @return search result of Report instance details
      */
-    fun getAllReportInstances(access: List<String>, from: Int): List<ReportInstance> {
+    fun getAllReportInstances(access: List<String>, from: Int, maxItems: Int): ReportInstanceSearchResults {
         createIndex()
         val query = QueryBuilders.termsQuery(ACCESS_LIST_FIELD, access)
         val sourceBuilder = SearchSourceBuilder()
             .timeout(TimeValue(PluginSettings.operationTimeoutMs, TimeUnit.MILLISECONDS))
             .sort(UPDATED_TIME_FIELD)
-            .size(MAX_ITEMS_TO_QUERY)
+            .size(maxItems)
             .from(from)
             .query(query)
         val searchRequest = SearchRequest()
             .indices(REPORT_INSTANCES_INDEX_NAME)
             .source(sourceBuilder)
         val actionFuture = client.search(searchRequest)
         val response = actionFuture.actionGet(PluginSettings.operationTimeoutMs)
-        val mutableList: MutableList<ReportInstance> = mutableListOf()
-        response.hits.forEach {
-            val parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
-                LoggingDeprecationHandler.INSTANCE,
-                it.sourceAsString)
-            parser.nextToken()
-            mutableList.add(ReportInstance.parse(parser, it.id))
-        }
-        return mutableList
+        return ReportInstanceSearchResults(from.toLong(), response)
     }
 
     /**
@@ -252,7 +245,7 @@ internal object ReportInstancesIndex {
         )
         val sourceBuilder = SearchSourceBuilder()
             .timeout(TimeValue(PluginSettings.operationTimeoutMs, TimeUnit.MILLISECONDS))
-            .size(MAX_ITEMS_TO_QUERY)
+            .size(PluginSettings.defaultItemsQueryCount)
             .query(query)
         val searchRequest = SearchRequest()
             .indices(REPORT_INSTANCES_INDEX_NAME)

diff --git a/...mazon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportDefinitionsRequest.kt b/...mazon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportDefinitionsRequest.kt
@@ -18,6 +18,8 @@ package com.amazon.opendistroforelasticsearch.reportsscheduler.model
 
 import com.amazon.opendistroforelasticsearch.reportsscheduler.ReportsSchedulerPlugin.Companion.LOG_PREFIX
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.FROM_INDEX_FIELD
+import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.MAX_ITEMS_FIELD
+import com.amazon.opendistroforelasticsearch.reportsscheduler.settings.PluginSettings
 import com.amazon.opendistroforelasticsearch.reportsscheduler.util.logger
 import org.elasticsearch.action.ActionRequest
 import org.elasticsearch.action.ActionRequestValidationException
@@ -38,17 +40,20 @@ import java.io.IOException
  * <pre> JSON format
  * {@code
  * {
- *   "fromIndex":100
+ *   "fromIndex":100,
+ *   "maxItems":100
  * }
  * }</pre>
  */
 internal class GetAllReportDefinitionsRequest(
-    val fromIndex: Int
+    val fromIndex: Int,
+    val maxItems: Int
 ) : ActionRequest(), ToXContentObject {
 
     @Throws(IOException::class)
     constructor(input: StreamInput) : this(
-        fromIndex = input.readInt()
+        fromIndex = input.readInt(),
+        maxItems = input.readInt()
     )
 
     companion object {
@@ -60,20 +65,22 @@ internal class GetAllReportDefinitionsRequest(
          * @return created [GetAllReportDefinitionsRequest] object
          */
         fun parse(parser: XContentParser): GetAllReportDefinitionsRequest {
-            var reportInstanceId = 0
+            var fromIndex = 0
+            var maxItems = PluginSettings.defaultItemsQueryCount
             XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()
                 when (fieldName) {
-                    FROM_INDEX_FIELD -> reportInstanceId = parser.intValue()
+                    FROM_INDEX_FIELD -> fromIndex = parser.intValue()
+                    MAX_ITEMS_FIELD -> maxItems = parser.intValue()
                     else -> {
                         parser.skipChildren()
                         log.info("$LOG_PREFIX:Skipping Unknown field $fieldName")
                     }
                 }
             }
-            return GetAllReportDefinitionsRequest(reportInstanceId)
+            return GetAllReportDefinitionsRequest(fromIndex, maxItems)
         }
     }
 
@@ -83,6 +90,7 @@ internal class GetAllReportDefinitionsRequest(
     @Throws(IOException::class)
     override fun writeTo(output: StreamOutput) {
         output.writeInt(fromIndex)
+        output.writeInt(maxItems)
     }
 
     /**
@@ -112,6 +120,7 @@ internal class GetAllReportDefinitionsRequest(
     override fun toXContent(builder: XContentBuilder?, params: ToXContent.Params?): XContentBuilder {
         return builder!!.startObject()
             .field(FROM_INDEX_FIELD, fromIndex)
+            .field(MAX_ITEMS_FIELD, maxItems)
             .endObject()
     }
 }
diff --git a/...azon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportDefinitionsResponse.kt b/...azon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportDefinitionsResponse.kt
@@ -16,35 +16,33 @@
 
 package com.amazon.opendistroforelasticsearch.reportsscheduler.model
 
-import com.amazon.opendistroforelasticsearch.reportsscheduler.ReportsSchedulerPlugin.Companion.LOG_PREFIX
-import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.REPORT_DEFINITION_LIST_FIELD
 import com.amazon.opendistroforelasticsearch.reportsscheduler.util.createJsonParser
-import com.amazon.opendistroforelasticsearch.reportsscheduler.util.logger
 import org.elasticsearch.common.io.stream.StreamInput
 import org.elasticsearch.common.io.stream.StreamOutput
 import org.elasticsearch.common.xcontent.ToXContent
 import org.elasticsearch.common.xcontent.XContentBuilder
 import org.elasticsearch.common.xcontent.XContentFactory
 import org.elasticsearch.common.xcontent.XContentParser
-import org.elasticsearch.common.xcontent.XContentParser.Token
-import org.elasticsearch.common.xcontent.XContentParserUtils
 import java.io.IOException
 
 /**
  * Get all report definitions response.
  * <pre> JSON format
  * {@code
  * {
+ *   "startIndex":"0",
+ *   "totalHits":"100",
+ *   "totalHitRelation":"eq",
  *   "reportDefinitionDetailsList":[
  *      // refer [com.amazon.opendistroforelasticsearch.reportsscheduler.model.ReportDefinitionDetails]
  *   ]
  * }
  * }</pre>
  */
 internal class GetAllReportDefinitionsResponse : BaseResponse {
-    val reportDefinitionList: List<ReportDefinitionDetails>
+    val reportDefinitionList: ReportDefinitionDetailsSearchResults
 
-    constructor(reportDefinitionList: List<ReportDefinitionDetails>) : super() {
+    constructor(reportDefinitionList: ReportDefinitionDetailsSearchResults) : super() {
         this.reportDefinitionList = reportDefinitionList
     }
 
@@ -56,39 +54,7 @@ internal class GetAllReportDefinitionsResponse : BaseResponse {
      * @param parser data referenced at parser
      */
     constructor(parser: XContentParser) : super() {
-        var reportDefinitions: List<ReportDefinitionDetails>? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
-        while (Token.END_OBJECT != parser.nextToken()) {
-            val fieldName = parser.currentName()
-            parser.nextToken()
-            when (fieldName) {
-                REPORT_DEFINITION_LIST_FIELD -> reportDefinitions = parseReportDefinitionList(parser)
-                else -> {
-                    parser.skipChildren()
-                    log.info("$LOG_PREFIX:Skipping Unknown field $fieldName")
-                }
-            }
-        }
-        reportDefinitions ?: throw IllegalArgumentException("$REPORT_DEFINITION_LIST_FIELD field absent")
-        this.reportDefinitionList = reportDefinitions
-    }
-
-    companion object {
-        private val log by logger(GetAllReportDefinitionsResponse::class.java)
-
-        /**
-         * Parse the report definition list from parser
-         * @param parser data referenced at parser
-         * @return created list of ReportDefinitionDetails
-         */
-        private fun parseReportDefinitionList(parser: XContentParser): List<ReportDefinitionDetails> {
-            val retList: MutableList<ReportDefinitionDetails> = mutableListOf()
-            XContentParserUtils.ensureExpectedToken(Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation)
-            while (parser.nextToken() != Token.END_ARRAY) {
-                retList.add(ReportDefinitionDetails.parse(parser))
-            }
-            return retList
-        }
+        reportDefinitionList = ReportDefinitionDetailsSearchResults(parser)
     }
 
     /**
@@ -103,9 +69,6 @@ internal class GetAllReportDefinitionsResponse : BaseResponse {
      * {@inheritDoc}
      */
     override fun toXContent(builder: XContentBuilder?, params: ToXContent.Params?): XContentBuilder {
-        builder!!.startObject()
-            .startArray(REPORT_DEFINITION_LIST_FIELD)
-        reportDefinitionList.forEach { it.toXContent(builder, ToXContent.EMPTY_PARAMS, true) }
-        return builder.endArray().endObject()
+        return reportDefinitionList.toXContent(builder, params)
     }
 }
diff --git a/.../amazon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportInstancesRequest.kt b/.../amazon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportInstancesRequest.kt
@@ -18,6 +18,8 @@ package com.amazon.opendistroforelasticsearch.reportsscheduler.model
 
 import com.amazon.opendistroforelasticsearch.reportsscheduler.ReportsSchedulerPlugin.Companion.LOG_PREFIX
 import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.FROM_INDEX_FIELD
+import com.amazon.opendistroforelasticsearch.reportsscheduler.model.RestTag.MAX_ITEMS_FIELD
+import com.amazon.opendistroforelasticsearch.reportsscheduler.settings.PluginSettings
 import com.amazon.opendistroforelasticsearch.reportsscheduler.util.logger
 import org.elasticsearch.action.ActionRequest
 import org.elasticsearch.action.ActionRequestValidationException
@@ -38,17 +40,20 @@ import java.io.IOException
  * <pre> JSON format
  * {@code
  * {
- *   "fromIndex":100
+ *   "fromIndex":100,
+ *   "maxItems":100
  * }
  * }</pre>
  */
 internal data class GetAllReportInstancesRequest(
-    val fromIndex: Int
+    val fromIndex: Int,
+    val maxItems: Int
 ) : ActionRequest(), ToXContentObject {
 
     @Throws(IOException::class)
     constructor(input: StreamInput) : this(
-        fromIndex = input.readInt()
+        fromIndex = input.readInt(),
+        maxItems = input.readInt()
     )
 
     companion object {
@@ -60,20 +65,22 @@ internal data class GetAllReportInstancesRequest(
          * @return created [GetAllReportInstancesRequest] object
          */
         fun parse(parser: XContentParser): GetAllReportInstancesRequest {
-            var reportInstanceId = 0
+            var fromIndex = 0
+            var maxItems = PluginSettings.defaultItemsQueryCount
             XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()
                 when (fieldName) {
-                    FROM_INDEX_FIELD -> reportInstanceId = parser.intValue()
+                    FROM_INDEX_FIELD -> fromIndex = parser.intValue()
+                    MAX_ITEMS_FIELD -> maxItems = parser.intValue()
                     else -> {
                         parser.skipChildren()
                         log.info("$LOG_PREFIX:Skipping Unknown field $fieldName")
                     }
                 }
             }
-            return GetAllReportInstancesRequest(reportInstanceId)
+            return GetAllReportInstancesRequest(fromIndex, maxItems)
         }
     }
 
@@ -83,6 +90,7 @@ internal data class GetAllReportInstancesRequest(
     @Throws(IOException::class)
     override fun writeTo(output: StreamOutput) {
         output.writeInt(fromIndex)
+        output.writeInt(maxItems)
     }
 
     /**
@@ -112,6 +120,7 @@ internal data class GetAllReportInstancesRequest(
     override fun toXContent(builder: XContentBuilder?, params: ToXContent.Params?): XContentBuilder {
         return builder!!.startObject()
             .field(FROM_INDEX_FIELD, fromIndex)
+            .field(MAX_ITEMS_FIELD, maxItems)
             .endObject()
     }
 }