From 7ae3b3b155b3729b5a2475315f9a742ff1c56756 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 27 Apr 2018 16:49:06 +0200 Subject: [PATCH] Move repository-s3 fixture tests to QA test project (#29372) This commit moves the repository-s3 fixture test added in #29296 in a new `repository-s3/qa/amazon-s3` project. This new project allows the REST integration tests to be executed using the real S3 service when all the required environment variables are provided. When no env var is provided, then the tests are executed using the fixture added in #29296. The REST tests located at the `repository-s3`plugin project now only verify that the plugin is correctly loaded. The REST tests have been adapted to allow a bucket name and a base path to be specified as env vars. This way it is possible to run the tests with different base paths (could be anything, like a CI job name or a branch name) without multiplicating buckets. Related to #29349 --- plugins/repository-s3/build.gradle | 22 +-- .../repository-s3/qa/amazon-s3/build.gradle | 83 ++++++++ .../repositories/s3/AmazonS3Fixture.java | 0 ...azonS3RepositoryClientYamlTestSuiteIT.java | 37 ++++ .../repositories/s3/AmazonS3TestServer.java | 0 .../test/repository_s3/10_repository.yml | 183 ++++++++++++++++++ plugins/repository-s3/qa/build.gradle | 0 .../repositories/s3/S3Repository.java | 2 +- .../test/repository_s3/10_basic.yml | 180 ----------------- qa/smoke-test-plugins/build.gradle | 4 +- qa/vagrant/build.gradle | 2 +- x-pack/qa/smoke-test-plugins-ssl/build.gradle | 6 +- x-pack/qa/smoke-test-plugins/build.gradle | 6 +- 13 files changed, 318 insertions(+), 207 deletions(-) create mode 100644 plugins/repository-s3/qa/amazon-s3/build.gradle rename plugins/repository-s3/{ => qa/amazon-s3}/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java (100%) create mode 100644 plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3RepositoryClientYamlTestSuiteIT.java rename plugins/repository-s3/{ => qa/amazon-s3}/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java (100%) create mode 100644 plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml create mode 100644 plugins/repository-s3/qa/build.gradle diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 46988a2dd5107..23252881cd75f 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -1,5 +1,3 @@ -import org.elasticsearch.gradle.test.AntFixture - /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -66,28 +64,14 @@ test { exclude '**/*CredentialsTests.class' } -forbiddenApisTest { - // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage - bundledSignatures -= 'jdk-non-portable' - bundledSignatures += 'jdk-internal' -} - -/** A task to start the AmazonS3Fixture which emulates a S3 service **/ -task s3Fixture(type: AntFixture) { - dependsOn compileTestJava - env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" - executable = new File(project.runtimeJavaHome, 'bin/java') - args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, 'bucket_test' +check { + // also execute the QA tests when testing the plugin + dependsOn 'qa:amazon-s3:check' } integTestCluster { - dependsOn s3Fixture - keystoreSetting 's3.client.integration_test.access_key', "s3_integration_test_access_key" keystoreSetting 's3.client.integration_test.secret_key', "s3_integration_test_secret_key" - - /* Use a closure on the string to delay evaluation until tests are executed */ - setting 's3.client.integration_test.endpoint', "http://${ -> s3Fixture.addressAndPort }" } thirdPartyAudit.excludes = [ diff --git a/plugins/repository-s3/qa/amazon-s3/build.gradle b/plugins/repository-s3/qa/amazon-s3/build.gradle new file mode 100644 index 0000000000000..5e288899021a1 --- /dev/null +++ b/plugins/repository-s3/qa/amazon-s3/build.gradle @@ -0,0 +1,83 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.test.AntFixture + +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':plugins:repository-s3', configuration: 'runtime') +} + +integTestCluster { + plugin ':plugins:repository-s3' +} + +forbiddenApisTest { + // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage + bundledSignatures -= 'jdk-non-portable' + bundledSignatures += 'jdk-internal' +} + +boolean useFixture = false + +String s3AccessKey = System.getenv("amazon_s3_access_key") +String s3SecretKey = System.getenv("amazon_s3_secret_key") +String s3Bucket = System.getenv("amazon_s3_bucket") +String s3BasePath = System.getenv("amazon_s3_base_path") + +if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) { + s3AccessKey = 's3_integration_test_access_key' + s3SecretKey = 's3_integration_test_secret_key' + s3Bucket = 'bucket_test' + s3BasePath = 'integration_test' + useFixture = true +} + +/** A task to start the AmazonS3Fixture which emulates a S3 service **/ +task s3Fixture(type: AntFixture) { + dependsOn compileTestJava + env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }" + executable = new File(project.runtimeJavaHome, 'bin/java') + args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket +} + +Map expansions = [ + 'bucket': s3Bucket, + 'base_path': s3BasePath +] +processTestResources { + inputs.properties(expansions) + MavenFilteringHack.filter(it, expansions) +} + +integTestCluster { + keystoreSetting 's3.client.integration_test.access_key', s3AccessKey + keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey + + if (useFixture) { + dependsOn s3Fixture + /* Use a closure on the string to delay evaluation until tests are executed */ + setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}" + } else { + println "Using an external service to test the repository-s3 plugin" + } +} \ No newline at end of file diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java similarity index 100% rename from plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java rename to plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3RepositoryClientYamlTestSuiteIT.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3RepositoryClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..afcc0fa353482 --- /dev/null +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3RepositoryClientYamlTestSuiteIT.java @@ -0,0 +1,37 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.repositories.s3; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +public class AmazonS3RepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public AmazonS3RepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java similarity index 100% rename from plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java rename to plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml new file mode 100644 index 0000000000000..8b3daccf0a2d7 --- /dev/null +++ b/plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml @@ -0,0 +1,183 @@ +# Integration tests for repository-s3 +--- +"Snapshot/Restore with repository-s3": + + # Register repository + - do: + snapshot.create_repository: + repository: repository + body: + type: s3 + settings: + bucket: ${bucket} + client: integration_test + base_path: ${base_path} + canned_acl: private + storage_class: standard + + - match: { acknowledged: true } + + # Get repository + - do: + snapshot.get_repository: + repository: repository + + - match: { repository.settings.bucket : ${bucket} } + - match: { repository.settings.client : "integration_test" } + - match: { repository.settings.base_path : ${base_path} } + - match: { repository.settings.canned_acl : "private" } + - match: { repository.settings.storage_class : "standard" } + - is_false: repository.settings.access_key + - is_false: repository.settings.secret_key + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 1 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 2 + - snapshot: one + - index: + _index: docs + _type: doc + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _type: doc + _id: 4 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 5 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 6 + - snapshot: two + - index: + _index: docs + _type: doc + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository + snapshot: snapshot-two + + - do: + snapshot.delete: + repository: repository + snapshot: snapshot-one + + # Remove our repository + - do: + snapshot.delete_repository: + repository: repository diff --git a/plugins/repository-s3/qa/build.gradle b/plugins/repository-s3/qa/build.gradle new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index 09d9782aa91f8..cb4f977bae77b 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -156,7 +156,7 @@ class S3Repository extends BlobStoreRepository { String bucket = BUCKET_SETTING.get(metadata.settings()); if (bucket == null) { - throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway"); + throw new RepositoryException(metadata.name(), "No bucket defined for s3 repository"); } boolean serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings()); diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/10_basic.yml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/10_basic.yml index 11f4610f6f7b2..7bb65a508863d 100644 --- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/10_basic.yml +++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/10_basic.yml @@ -11,183 +11,3 @@ nodes.info: {} - match: { nodes.$master.plugins.0.name: repository-s3 } ---- -"Snapshot/Restore with repository-s3": - - # Register repository - - do: - snapshot.create_repository: - repository: repository - body: - type: s3 - settings: - bucket: "bucket_test" - client: "integration_test" - canned_acl: "public-read" - storage_class: "standard" - - - match: { acknowledged: true } - - # Get repository - - do: - snapshot.get_repository: - repository: repository - - - match: {repository.settings.bucket : "bucket_test"} - - match: {repository.settings.client : "integration_test"} - - match: {repository.settings.canned_acl : "public-read"} - - match: {repository.settings.storage_class : "standard"} - - is_false: repository.settings.access_key - - is_false: repository.settings.secret_key - - # Index documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _type: doc - _id: 1 - - snapshot: one - - index: - _index: docs - _type: doc - _id: 2 - - snapshot: one - - index: - _index: docs - _type: doc - _id: 3 - - snapshot: one - - - do: - count: - index: docs - - - match: {count: 3} - - # Create a first snapshot - - do: - snapshot.create: - repository: repository - snapshot: snapshot-one - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-one } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.include_global_state: true } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.status: - repository: repository - snapshot: snapshot-one - - - is_true: snapshots - - match: { snapshots.0.snapshot: snapshot-one } - - match: { snapshots.0.state : SUCCESS } - - # Index more documents - - do: - bulk: - refresh: true - body: - - index: - _index: docs - _type: doc - _id: 4 - - snapshot: two - - index: - _index: docs - _type: doc - _id: 5 - - snapshot: two - - index: - _index: docs - _type: doc - _id: 6 - - snapshot: two - - index: - _index: docs - _type: doc - _id: 7 - - snapshot: two - - - do: - count: - index: docs - - - match: {count: 7} - - # Create a second snapshot - - do: - snapshot.create: - repository: repository - snapshot: snapshot-two - wait_for_completion: true - - - match: { snapshot.snapshot: snapshot-two } - - match: { snapshot.state : SUCCESS } - - match: { snapshot.shards.failed : 0 } - - - do: - snapshot.get: - repository: repository - snapshot: snapshot-one,snapshot-two - - - is_true: snapshots - - match: { snapshots.0.state : SUCCESS } - - match: { snapshots.1.state : SUCCESS } - - # Delete the index - - do: - indices.delete: - index: docs - - # Restore the second snapshot - - do: - snapshot.restore: - repository: repository - snapshot: snapshot-two - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 7} - - # Delete the index again - - do: - indices.delete: - index: docs - - # Restore the first snapshot - - do: - snapshot.restore: - repository: repository - snapshot: snapshot-one - wait_for_completion: true - - - do: - count: - index: docs - - - match: {count: 3} - - # Remove the snapshots - - do: - snapshot.delete: - repository: repository - snapshot: snapshot-two - - - do: - snapshot.delete: - repository: repository - snapshot: snapshot-one - - # Remove our repository - - do: - snapshot.delete_repository: - repository: repository diff --git a/qa/smoke-test-plugins/build.gradle b/qa/smoke-test-plugins/build.gradle index d60216dad194f..602dfa2d6ea4f 100644 --- a/qa/smoke-test-plugins/build.gradle +++ b/qa/smoke-test-plugins/build.gradle @@ -23,9 +23,9 @@ apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' ext.pluginsCount = 0 -project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj -> +project(':plugins').getChildProjects().each { pluginName, pluginProject -> integTestCluster { - plugin subproj.path + plugin pluginProject.path } pluginsCount += 1 } diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index 4086cf2205785..2b1ffb280819c 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -22,7 +22,7 @@ apply plugin: 'elasticsearch.vagrant' List plugins = [] for (Project subproj : project.rootProject.subprojects) { - if (subproj.path.startsWith(':plugins:') || subproj.path.equals(':example-plugins:custom-settings')) { + if (subproj.parent.path == ':plugins' || subproj.path.equals(':example-plugins:custom-settings')) { // add plugin as a dep dependencies { packaging project(path: "${subproj.path}", configuration: 'zip') diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index bc7aa9fd39328..28fd4d2db49ed 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -1,5 +1,7 @@ import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin +import org.elasticsearch.gradle.plugin.PluginBuildPlugin import org.elasticsearch.gradle.test.NodeInfo import javax.net.ssl.HttpsURLConnection @@ -160,9 +162,9 @@ integTestCluster.dependsOn(importClientCertificateInNodeKeyStore, importNodeCert ext.pluginsCount = 0 -project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj -> +project(':plugins').getChildProjects().each { pluginName, pluginProject -> // need to get a non-decorated project object, so must re-lookup the project by path - integTestCluster.plugin(subproj.path) + integTestCluster.plugin(pluginProject.path) pluginsCount += 1 } diff --git a/x-pack/qa/smoke-test-plugins/build.gradle b/x-pack/qa/smoke-test-plugins/build.gradle index 8c232bc5f3a51..207fa8204db00 100644 --- a/x-pack/qa/smoke-test-plugins/build.gradle +++ b/x-pack/qa/smoke-test-plugins/build.gradle @@ -1,4 +1,6 @@ import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin +import org.elasticsearch.gradle.plugin.PluginBuildPlugin apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' @@ -8,9 +10,9 @@ dependencies { } ext.pluginsCount = 0 -project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj -> +project(':plugins').getChildProjects().each { pluginName, pluginProject -> // need to get a non-decorated project object, so must re-lookup the project by path - integTestCluster.plugin(subproj.path) + integTestCluster.plugin(pluginProject.path) pluginsCount += 1 }