[reports-scheduler] Add support for Elasticsearch 7.10.0

reports-scheduler/.gitignore

@@ -16,7 +16,6 @@
 *.war
 *.nar
 *.ear
-*.zip
 *.tar.gz
 *.rar
 

reports-scheduler/build-tools/esplugin-coverage.gradle

@@ -46,13 +46,13 @@ task dummyIntegTest(type: Test) {
     enabled = false
     workingDir = file("/") // Force absolute path to jacoco agent jar
     jacoco {
-        destinationFile = file("${jacocoDir}/integTestRunner.exec")
+        destinationFile = file("${jacocoDir}/integTest.exec")
         destinationFile.parentFile.mkdirs()
         jmx = true
     }
 }
 
-integTest.runner {
+integTest {
     systemProperty 'jacoco.dir', "${jacocoDir}"
 }
 
@@ -69,7 +69,7 @@ jacocoTestReport {
 
 allprojects{
     afterEvaluate {
-        jacocoTestReport.dependsOn integTest.runner
+        jacocoTestReport.dependsOn integTest
 
         testClusters.integTest {
             jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/')

reports-scheduler/build.gradle

@@ -13,10 +13,13 @@
  * permissions and limitations under the License.
  */
 
+import org.elasticsearch.gradle.test.RestIntegTestTask
+import java.util.concurrent.Callable
+
 buildscript {
     ext {
         es_group = "org.elasticsearch"
-        es_version = System.getProperty("es.version", "7.9.1")
+        es_version = System.getProperty("es.version", "7.10.0")
         kotlin_version = System.getProperty("kotlin.version", "1.4.0")
     }
 
@@ -52,10 +55,6 @@ apply plugin: 'org.jetbrains.kotlin.plugin.allopen'
 
 def usingRemoteCluster = System.properties.containsKey('tests.rest.cluster') || System.properties.containsKey('tests.cluster')
 def usingMultiNode = project.properties.containsKey('numNodes')
-// Only apply jacoco test coverage if we are running a local single node cluster
-if (!usingRemoteCluster && !usingMultiNode) {
-    apply from: 'build-tools/esplugin-coverage.gradle'
-}
 
 check.dependsOn jacocoTestReport
 
@@ -113,7 +112,7 @@ plugins.withId('org.jetbrains.kotlin.jvm') {
 
 allprojects {
     group = "com.amazon.opendistroforelasticsearch"
-    version = "${opendistroVersion}.1"
+    version = "${opendistroVersion}.0"
 
     plugins.withId('java') {
         sourceCompatibility = targetCompatibility = "1.8"
@@ -153,33 +152,15 @@ forbiddenApis.ignoreFailures = true
 // Allow test cases to be named Tests without having to be inherited from LuceneTestCase.
 // see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
 testingConventions.enabled = false
+loggerUsageCheck.enabled = false
 
-integTest.dependsOn(rootProject.assemble)
-integTestRunner {
+test {
     systemProperty 'tests.security.manager', 'false'
     useJUnitPlatform()
 }
 
 File repo = file("$buildDir/testclusters/repo")
 def _numNodes = findProperty('numNodes') as Integer ?: 1
-testClusters.integTest {
-    testDistribution = 'OSS'
-    // need to install job-scheduler first, need to assemble job-scheduler first
-    plugin(fileTree("src/test/resources/job-scheduler").getSingleFile())
-    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
-    if (_numNodes > 1) numberOfNodes = _numNodes
-    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
-    // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
-    // since we also support multi node integration tests we increase debugPort per node
-    if (System.getProperty("cluster.debug") != null) {
-        def debugPort = 5005
-        nodes.forEach { node ->
-            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
-            debugPort += 1
-        }
-    }
-    setting 'path.repo', repo.absolutePath
-}
 
 def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
 es_tmp_dir.mkdirs()
@@ -193,25 +174,36 @@ afterEvaluate {
     testClusters.integTest.nodes.each { node ->
         def plugins = node.plugins
         def firstPlugin = plugins.get(0)
-        if (firstPlugin.provider == project.bundlePlugin.archiveFile) {
-            plugins.remove(0)
-            plugins.add(firstPlugin)
-        }
+        plugins.remove(0)
+        plugins.add(firstPlugin)
     }
 }
 
 tasks.withType(licenseHeaders.class) {
     additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
 }
 
-integTest.runner {
+task integTest(type: RestIntegTestTask) {
+    description = "Run tests against a cluster that has security enabled"
+    testClassesDirs = sourceSets.test.output.classesDirs
+    classpath = sourceSets.test.runtimeClasspath
+}
+tasks.named("check").configure { dependsOn(integTest) }
+
+integTest {
     systemProperty 'tests.security.manager', 'false'
     systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
-    systemProperty 'tests.path.repo', repo.absolutePath
+
+    systemProperty "https", System.getProperty("https")
+    systemProperty "user", System.getProperty("user")
+    systemProperty "password", System.getProperty("password")
     // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
     // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
     doFirst {
-        systemProperty 'cluster.debug', getDebug()
+        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
+        // use longer timeouts for requests.
+        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
+        systemProperty 'cluster.debug', isDebuggingCluster
         // Set number of nodes system property to be used in tests
         systemProperty 'cluster.number_of_nodes', "${_numNodes}"
         // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
@@ -227,6 +219,39 @@ integTest.runner {
     }
 }
 
+Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
+integTest.dependsOn(bundle)
+integTest.getClusters().forEach{c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))}
+
+testClusters.integTest {
+    testDistribution = 'OSS'
+    // need to install job-scheduler first, need to assemble job-scheduler first
+    plugin(provider(new Callable<RegularFile>(){
+        @Override
+        RegularFile call() throws Exception {
+            return new RegularFile() {
+                @Override
+                File getAsFile() {
+                    return fileTree("src/test/resources/job-scheduler").getSingleFile()
+                }
+            }
+        }
+    }))
+    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
+    if (_numNodes > 1) numberOfNodes = _numNodes
+    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
+    // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
+    // since we also support multi node integration tests we increase debugPort per node
+    if (System.getProperty("cluster.debug") != null) {
+        def debugPort = 5005
+        nodes.forEach { node ->
+            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
+            debugPort += 1
+        }
+    }
+    setting 'path.repo', repo.absolutePath
+}
+
 run {
     doFirst {
         // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
@@ -235,6 +260,7 @@ run {
             cluster.waitForAllConditions()
         }
     }
+    useCluster testClusters.integTest
 }
 
 task ktlint(type: JavaExec, group: "verification") {
@@ -258,4 +284,9 @@ task ktlintFormat(type: JavaExec, group: "formatting") {
 
 compileKotlin { kotlinOptions.freeCompilerArgs = ['-Xjsr305=strict'] }
 
+// Only apply jacoco test coverage if we are running a local single node cluster
+if (!usingRemoteCluster && !usingMultiNode) {
+    apply from: 'build-tools/esplugin-coverage.gradle'
+}
+
 apply from: 'build-tools/pkgbuild.gradle'

reports-scheduler/gradle.properties

@@ -13,4 +13,4 @@
 #   permissions and limitations under the License.
 #
 
-version = 1.11.0
+version = 1.12.0

reports-scheduler/gradle/wrapper/gradle-wrapper.properties

@@ -15,7 +15,7 @@
 #
 
 #Wed Jul 29 13:30:55 PDT 2020
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/CreateReportDefinitionRequest.kt

@@ -64,7 +64,7 @@ internal class CreateReportDefinitionRequest : ActionRequest, ToXContentObject {
      */
     constructor(parser: XContentParser) : super() {
         var reportDefinition: ReportDefinition? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
         while (Token.END_OBJECT != parser.nextToken()) {
             val fieldName = parser.currentName()
             parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/CreateReportDefinitionResponse.kt

@@ -56,7 +56,7 @@ internal class CreateReportDefinitionResponse(
          */
         fun parse(parser: XContentParser): CreateReportDefinitionResponse {
             var reportDefinitionId: String? = null
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/DeleteReportDefinitionRequest.kt

@@ -62,7 +62,7 @@ internal class DeleteReportDefinitionRequest(
          */
         fun parse(parser: XContentParser, useReportDefinitionId: String? = null): DeleteReportDefinitionRequest {
             var reportDefinitionId: String? = useReportDefinitionId
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/DeleteReportDefinitionResponse.kt

@@ -56,7 +56,7 @@ internal data class DeleteReportDefinitionResponse(
          */
         fun parse(parser: XContentParser): DeleteReportDefinitionResponse {
             var reportDefinitionId: String? = null
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportDefinitionsRequest.kt

@@ -67,7 +67,7 @@ internal class GetAllReportDefinitionsRequest(
         fun parse(parser: XContentParser): GetAllReportDefinitionsRequest {
             var fromIndex = 0
             var maxItems = PluginSettings.defaultItemsQueryCount
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/GetAllReportInstancesRequest.kt

@@ -67,7 +67,7 @@ internal data class GetAllReportInstancesRequest(
         fun parse(parser: XContentParser): GetAllReportInstancesRequest {
             var fromIndex = 0
             var maxItems = PluginSettings.defaultItemsQueryCount
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/GetReportDefinitionRequest.kt

@@ -62,7 +62,7 @@ internal class GetReportDefinitionRequest(
          */
         fun parse(parser: XContentParser, useReportDefinitionId: String? = null): GetReportDefinitionRequest {
             var reportDefinitionId: String? = useReportDefinitionId
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/GetReportDefinitionResponse.kt

@@ -61,7 +61,7 @@ internal class GetReportDefinitionResponse : BaseResponse {
      */
     constructor(parser: XContentParser) : super() {
         var reportDefinition: ReportDefinitionDetails? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
         while (Token.END_OBJECT != parser.nextToken()) {
             val fieldName = parser.currentName()
             parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/GetReportInstanceRequest.kt

@@ -61,7 +61,7 @@ internal class GetReportInstanceRequest(
          */
         fun parse(parser: XContentParser, useReportInstanceId: String? = null): GetReportInstanceRequest {
             var reportInstanceId: String? = useReportInstanceId
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/GetReportInstanceResponse.kt

@@ -61,7 +61,7 @@ internal class GetReportInstanceResponse : BaseResponse {
      */
     constructor(parser: XContentParser) : super() {
         var reportInstance: ReportInstance? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
         while (Token.END_OBJECT != parser.nextToken()) {
             val fieldName = parser.currentName()
             parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/InContextReportCreateRequest.kt

@@ -99,7 +99,7 @@ internal class InContextReportCreateRequest : ActionRequest, ToXContentObject {
         var status: Status? = null
         var statusText: String? = null
         var inContextDownloadUrlPath: String? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
         while (Token.END_OBJECT != parser.nextToken()) {
             val fieldName = parser.currentName()
             parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/InContextReportCreateResponse.kt

@@ -61,7 +61,7 @@ internal class InContextReportCreateResponse : BaseResponse {
      */
     constructor(parser: XContentParser) : super() {
         var reportInstance: ReportInstance? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
         while (Token.END_OBJECT != parser.nextToken()) {
             val fieldName = parser.currentName()
             parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/OnDemandReportCreateRequest.kt

@@ -62,7 +62,7 @@ internal class OnDemandReportCreateRequest(
          */
         fun parse(parser: XContentParser, useReportDefinitionId: String? = null): OnDemandReportCreateRequest {
             var reportDefinitionId: String? = useReportDefinitionId
-            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+            XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
             while (Token.END_OBJECT != parser.nextToken()) {
                 val fieldName = parser.currentName()
                 parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/OnDemandReportCreateResponse.kt

@@ -61,7 +61,7 @@ internal class OnDemandReportCreateResponse : BaseResponse {
      */
     constructor(parser: XContentParser) : super() {
         var reportInstance: ReportInstance? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
         while (Token.END_OBJECT != parser.nextToken()) {
             val fieldName = parser.currentName()
             parser.nextToken()

reports-scheduler/src/main/kotlin/com/amazon/opendistroforelasticsearch/reportsscheduler/model/PollReportInstanceResponse.kt

@@ -72,7 +72,7 @@ internal class PollReportInstanceResponse : BaseResponse {
     constructor(parser: XContentParser) : super() {
         var retryAfter: Int = PluginSettings.minPollingDurationSeconds
         var reportInstance: ReportInstance? = null
-        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation)
+        XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser)
         while (Token.END_OBJECT != parser.nextToken()) {
             val fieldName = parser.currentName()
             parser.nextToken()