From 8fc719c484d0ee9802445a05b5be9bcb22f870a0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 30 Nov 2023 22:59:01 +0000 Subject: [PATCH] Integration test infrastructure set up (#230) * Initial integ test framework modification, sets up integration test cluster and fixes ./gradlew run Signed-off-by: Joshua Palis * spotless Signed-off-by: Joshua Palis * Updating DEVELOPER_GUIDE Signed-off-by: Joshua Palis --------- Signed-off-by: Joshua Palis (cherry picked from commit aba7dea51f3334db506a2e39d8b511cf352bf81c) Signed-off-by: github-actions[bot] --- DEVELOPER_GUIDE.md | 2 +- build.gradle | 48 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 107386c3d..5e68587e2 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -36,7 +36,7 @@ This package uses the [Gradle](https://docs.gradle.org/current/userguide/usergui #### Building from the command line 1. `./gradlew check` builds and tests. -2. `./gradlew :run` runs the plugin. +2. `./gradlew :run` installs and runs ML-Commons and Flow Framework Plugins into a local cluster 3. `./gradlew spotlessApply` formats code. And/or import formatting rules in [formatterConfig.xml](formatter/formatterConfig.xml) with IDE. 4. `./gradlew test` to run the complete test suite. diff --git a/build.gradle b/build.gradle index af45544f9..e4227a40a 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,6 @@ import java.nio.file.Files import org.opensearch.gradle.test.RestIntegTestTask +import java.util.concurrent.Callable apply plugin: 'java' apply plugin: 'idea' @@ -155,6 +156,9 @@ dependencies { implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1' implementation 'org.bouncycastle:bcprov-jdk18on:1.77' + // ZipArchive dependencies used for integration tests + zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}" + configurations.all { resolutionStrategy { force("com.google.guava:guava:32.1.3-jre") // CVE for 31.1 @@ -164,6 +168,11 @@ dependencies { } } + +def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile +opensearch_tmp_dir.mkdirs() +def _numNodes = findProperty('numNodes') as Integer ?: 1 + test { include '**/*Tests.class' } @@ -176,6 +185,8 @@ jacocoTestReport { } tasks.named("check").configure { dependsOn(jacocoTestReport) } + +// Set up integration tests task integTest(type: RestIntegTestTask) { description = "Run tests against a cluster" testClassesDirs = sourceSets.test.output.classesDirs @@ -183,10 +194,18 @@ task integTest(type: RestIntegTestTask) { } tasks.named("check").configure { dependsOn(integTest) } -def _numNodes = findProperty('numNodes') as Integer ?: 1 - integTest { + dependsOn "bundlePlugin" + systemProperty 'tests.security.manager', 'false' + systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath + systemProperty('project.root', project.rootDir.absolutePath) + systemProperty "https", System.getProperty("https") + systemProperty "user", System.getProperty("user") + systemProperty "password", System.getProperty("password") + + + // doFirst delays this block until execution time doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can // use longer timeouts for requests. @@ -207,10 +226,31 @@ integTest { } } +// Set up integration test clusters, installs all zipArchive dependencies and Flow Framework testClusters.integTest { testDistribution = "ARCHIVE" + + // Installs all registered zipArchive dependencies on integTest cluster nodes + configurations.zipArchive.asFileTree.each { + plugin(provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return it + } + } + } + })) + } + + // Install Flow Framwork Plugin on integTest cluster nodes + plugin(project.tasks.bundlePlugin.archiveFile) + // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1 if (_numNodes > 1) numberOfNodes = _numNodes + // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore // i.e. we have to use a custom property to flag when we want to debug OpenSearch JVM // since we also support multi node integration tests we increase debugPort per node @@ -221,11 +261,9 @@ testClusters.integTest { debugPort += 1 } } - - // This installs our plugin into the testClusters - plugin(project.tasks.bundlePlugin.archiveFile) } +// Automatically sets up the integration test cluster locally run { useCluster testClusters.integTest }