Skip to content

Commit

Permalink
Build: Add mavenPlugin cluster configuration method (#30541)
Browse files Browse the repository at this point in the history 
This commit adds the ability to specify a plugin from maven for a
test cluster to use. Currently, only local projects may be used as
plugins, except when testing bwc, where the coordinates of the project
are used. However, that assumes all projects always keep the same
coordinates, or are even still plugins, which is no longer the case for
x-pack. The full cluster and rolling restart tests are changed to use
this new method when pulling x-pack versions before 6.3.0.
  • Loading branch information
@rjernst
rjernst committed May 12, 2018
1 parent d96bc5b commit 80a9838
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 45 deletions.
7 changes: 6 additions & 1 deletion buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ class ClusterConfiguration {
// map from destination path, to source file
Map<String, Object> extraConfigFiles = new HashMap<>()

LinkedHashMap<String, Project> plugins = new LinkedHashMap<>()
LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()

List<Project> modules = new ArrayList<>()

Expand Down Expand Up @@ -185,6 +185,11 @@ class ClusterConfiguration {
plugins.put(pluginProject.name, pluginProject)
}

@Input
void mavenPlugin(String name, String mavenCoords) {
plugins.put(name, mavenCoords)
}

/** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */
@Input
void module(Project moduleProject) {
Expand Down
103 changes: 61 additions & 42 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ class ClusterFormationTasks {
// from mirrors using gradles built-in mechanism etc.

configureDistributionDependency(project, config.distribution, bwcDistro, config.bwcVersion)
for (Map.Entry<String, Project> entry : config.plugins.entrySet()) {
configureBwcPluginDependency("${prefix}_elasticsearchBwcPlugins", project, entry.getValue(), bwcPlugins, config.bwcVersion)
for (Map.Entry<String, Object> entry : config.plugins.entrySet()) {
configureBwcPluginDependency(project, entry.getValue(), bwcPlugins, config.bwcVersion)
}
bwcDistro.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
bwcPlugins.resolutionStrategy.cacheChangingModulesFor(0, TimeUnit.SECONDS)
Expand Down Expand Up @@ -150,10 +150,15 @@ class ClusterFormationTasks {
}

/** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */
static void configureBwcPluginDependency(String name, Project project, Project pluginProject, Configuration configuration, Version elasticsearchVersion) {
verifyProjectHasBuildPlugin(name, elasticsearchVersion, project, pluginProject)
final String pluginName = findPluginName(pluginProject)
project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
static void configureBwcPluginDependency(Project project, Object plugin, Configuration configuration, Version elasticsearchVersion) {
if (plugin instanceof Project) {
Project pluginProject = (Project)plugin
verifyProjectHasBuildPlugin(configuration.name, elasticsearchVersion, project, pluginProject)
final String pluginName = findPluginName(pluginProject)
project.dependencies.add(configuration.name, "org.elasticsearch.plugin:${pluginName}:${elasticsearchVersion}@zip")
} else {
project.dependencies.add(configuration.name, "${plugin}@zip")
}
}

/**
Expand Down Expand Up @@ -210,9 +215,9 @@ class ClusterFormationTasks {
}

// install plugins
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
String actionName = pluginTaskName('install', plugin.getKey(), 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, plugin.getValue(), prefix)
for (String pluginName : node.config.plugins.keySet()) {
String actionName = pluginTaskName('install', pluginName, 'Plugin')
setup = configureInstallPluginTask(taskName(prefix, node, actionName), project, setup, node, pluginName, prefix)
}

// sets up any extra config files that need to be copied over to the ES instance;
Expand Down Expand Up @@ -448,31 +453,40 @@ class ClusterFormationTasks {
Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup)

List<FileCollection> pluginFiles = []
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {

Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = pluginConfigurationName(prefix, pluginProject)
String configurationName = pluginConfigurationName(prefix, plugin.key)
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
}
project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
setup.dependsOn(pluginProject.tasks.bundlePlugin)

// also allow rest tests to use the rest spec from the plugin
String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
File restApiDir = new File(resourceDir, 'rest-api-spec/api')
if (restApiDir.exists() == false) continue
if (copyRestSpec == null) {
copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
copyPlugins.dependsOn(copyRestSpec)
copyRestSpec.into(project.sourceSets.test.output.resourcesDir)

if (plugin.getValue() instanceof Project) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)

project.dependencies.add(configurationName, project.dependencies.project(path: pluginProject.path, configuration: 'zip'))
setup.dependsOn(pluginProject.tasks.bundlePlugin)

// also allow rest tests to use the rest spec from the plugin
String copyRestSpecTaskName = pluginTaskName('copy', plugin.getKey(), 'PluginRestSpec')
Copy copyRestSpec = project.tasks.findByName(copyRestSpecTaskName)
for (File resourceDir : pluginProject.sourceSets.test.resources.srcDirs) {
File restApiDir = new File(resourceDir, 'rest-api-spec/api')
if (restApiDir.exists() == false) continue
if (copyRestSpec == null) {
copyRestSpec = project.tasks.create(name: copyRestSpecTaskName, type: Copy)
copyPlugins.dependsOn(copyRestSpec)
copyRestSpec.into(project.sourceSets.test.output.resourcesDir)
}
copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
}
copyRestSpec.from(resourceDir).include('rest-api-spec/api/**')
} else {
project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
}



pluginFiles.add(configuration)
}

Expand All @@ -481,32 +495,37 @@ class ClusterFormationTasks {
return copyPlugins
}

private static String pluginConfigurationName(final String prefix, final Project project) {
return "_plugin_${prefix}_${project.path}".replace(':', '_')
private static String pluginConfigurationName(final String prefix, final String name) {
return "_plugin_${prefix}_${name}".replace(':', '_')
}

private static String pluginBwcConfigurationName(final String prefix, final Project project) {
return "_plugin_bwc_${prefix}_${project.path}".replace(':', '_')
private static String pluginBwcConfigurationName(final String prefix, final String name) {
return "_plugin_bwc_${prefix}_${name}".replace(':', '_')
}

/** Configures task to copy a plugin based on a zip file resolved using dependencies for an older version */
static Task configureCopyBwcPluginsTask(String name, Project project, Task setup, NodeInfo node, String prefix) {
Configuration bwcPlugins = project.configurations.getByName("${prefix}_elasticsearchBwcPlugins")
for (Map.Entry<String, Project> plugin : node.config.plugins.entrySet()) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)
String configurationName = pluginBwcConfigurationName(prefix, pluginProject)
for (Map.Entry<String, Object> plugin : node.config.plugins.entrySet()) {
String configurationName = pluginBwcConfigurationName(prefix, plugin.key)
Configuration configuration = project.configurations.findByName(configurationName)
if (configuration == null) {
configuration = project.configurations.create(configurationName)
}

final String depName = findPluginName(pluginProject)
if (plugin.getValue() instanceof Project) {
Project pluginProject = plugin.getValue()
verifyProjectHasBuildPlugin(name, node.nodeVersion, project, pluginProject)

Dependency dep = bwcPlugins.dependencies.find {
it.name == depName
final String depName = findPluginName(pluginProject)

Dependency dep = bwcPlugins.dependencies.find {
it.name == depName
}
configuration.dependencies.add(dep)
} else {
project.dependencies.add(configurationName, "${plugin.getValue()}@zip")
}
configuration.dependencies.add(dep)
}

Copy copyPlugins = project.tasks.create(name: name, type: Copy, dependsOn: setup) {
Expand All @@ -531,12 +550,12 @@ class ClusterFormationTasks {
return installModule
}

static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, Project plugin, String prefix) {
static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) {
final FileCollection pluginZip;
if (node.nodeVersion != VersionProperties.elasticsearch) {
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, plugin))
pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName))
} else {
pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, plugin))
pluginZip = project.configurations.getByName(pluginConfigurationName(prefix, pluginName))
}
// delay reading the file location until execution time by wrapping in a closure within a GString
final Object file = "${-> new File(node.pluginsTmpDir, pluginZip.singleFile.getName()).toURI().toURL().toString()}"
Expand Down
2 changes: 1 addition & 1 deletion x-pack/qa/full-cluster-restart/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ subprojects {
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
dependsOn copyTestNodeKeystore
if (version.before('6.3.0')) {
plugin xpackProject('plugin').path
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}"
}
bwcVersion = version
numBwcNodes = 2
Expand Down
2 changes: 1 addition & 1 deletion x-pack/qa/rolling-upgrade/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ subprojects {
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
dependsOn copyTestNodeKeystore
if (version.before('6.3.0')) {
plugin xpackProject('plugin').path
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${version}"
}
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
Expand Down

0 comments on commit 80a9838

Please sign in to comment.