From a02e8ad90be1a419694330ae7196023677b422fb Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 23 Sep 2021 17:18:44 -0500 Subject: [PATCH] Upgrade repository-hdfs to Hadoop 3 (#76897) This upgrades the repository-hdfs plugin to hadoop 3. Tests are performed against both hadoop 2 and hadoop 3 HDFS. The advantages of using the hadoop 3 client are: Over-the-wire encryption works (tests coming in an upcoming PR). We don't have to add (or ask customers to add) additional jvm permissions to the elasticsearch jvm It's compatible with java versions higher than java 8 --- plugins/repository-hdfs/build.gradle | 361 ++++++++++++------ .../build.gradle | 2 +- .../hadoop/util/ShutdownHookManager.java | 0 .../commons-configuration-1.6.jar.sha1 | 1 - .../commons-configuration2-2.7.jar.sha1 | 1 + ...txt => commons-configuration2-LICENSE.txt} | 0 ....txt => commons-configuration2-NOTICE.txt} | 2 +- .../licenses/guava-11.0.2.jar.sha1 | 1 - .../licenses/guava-27.1-jre.jar.sha1 | 1 + .../hadoop-annotations-2.8.5.jar.sha1 | 1 - .../licenses/hadoop-auth-2.8.5.jar.sha1 | 1 - .../licenses/hadoop-client-2.8.5.jar.sha1 | 1 - .../hadoop-client-runtime-3.3.1.jar.sha1 | 1 + .../licenses/hadoop-hdfs-2.8.5.jar.sha1 | 1 - .../licenses/hadoop-hdfs-3.3.1.jar.sha1 | 1 + .../hadoop-hdfs-client-2.8.5.jar.sha1 | 1 - .../hdfs_repository/20_repository_create.yml | 2 +- .../hdfs_repository/20_repository_delete.yml | 4 +- .../hdfs_repository/20_repository_verify.yml | 2 +- .../test/hdfs_repository/30_snapshot.yml | 4 +- .../test/hdfs_repository/30_snapshot_get.yml | 4 +- .../hdfs_repository/30_snapshot_readonly.yml | 2 +- .../test/hdfs_repository/40_restore.yml | 2 +- .../20_repository_create.yml | 2 +- .../20_repository_delete.yml | 4 +- .../20_repository_verify.yml | 2 +- .../secure_hdfs_repository/30_snapshot.yml | 4 +- .../30_snapshot_get.yml | 4 +- .../30_snapshot_readonly.yml | 2 +- .../secure_hdfs_repository/40_restore.yml | 2 +- settings.gradle | 3 +- .../build.gradle | 0 .../src/main/java/hdfs/MiniHDFS.java | 0 .../main/resources/readonly-repository.tar.gz | Bin test/fixtures/hdfs3-fixture/build.gradle | 13 + .../src/main/java/hdfs/MiniHDFS.java | 176 +++++++++ .../main/resources/readonly-repository.tar.gz | Bin 0 -> 1314 bytes .../searchable-snapshots/qa/hdfs/build.gradle | 6 +- .../qa/hdfs/build.gradle | 2 +- 39 files changed, 461 insertions(+), 155 deletions(-) rename plugins/repository-hdfs/{hadoop-common => hadoop-client-api}/build.gradle (69%) rename plugins/repository-hdfs/{hadoop-common => hadoop-client-api}/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java (100%) delete mode 100644 plugins/repository-hdfs/licenses/commons-configuration-1.6.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/commons-configuration2-2.7.jar.sha1 rename plugins/repository-hdfs/licenses/{commons-configuration-LICENSE.txt => commons-configuration2-LICENSE.txt} (100%) rename plugins/repository-hdfs/licenses/{commons-configuration-NOTICE.txt => commons-configuration2-NOTICE.txt} (83%) delete mode 100644 plugins/repository-hdfs/licenses/guava-11.0.2.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/guava-27.1-jre.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.1.jar.sha1 delete mode 100644 plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1 rename test/fixtures/{hdfs-fixture => hdfs2-fixture}/build.gradle (100%) rename test/fixtures/{hdfs-fixture => hdfs2-fixture}/src/main/java/hdfs/MiniHDFS.java (100%) rename test/fixtures/{hdfs-fixture => hdfs2-fixture}/src/main/resources/readonly-repository.tar.gz (100%) create mode 100644 test/fixtures/hdfs3-fixture/build.gradle create mode 100644 test/fixtures/hdfs3-fixture/src/main/java/hdfs/MiniHDFS.java create mode 100644 test/fixtures/hdfs3-fixture/src/main/resources/readonly-repository.tar.gz diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 3ae7b83d34f0c..e1030d0380ca7 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import org.apache.tools.ant.filters.ReplaceTokens import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.internal.info.BuildParams import org.elasticsearch.gradle.internal.test.RestIntegTestTask @@ -26,45 +27,43 @@ esplugin { } versions << [ - 'hadoop2': '2.8.5' + 'hadoop': '3.3.1' ] +final int minTestedHadoopVersion = 2; +final int maxTestedHadoopVersion = 3; + testFixtures.useFixture ":test:fixtures:krb5kdc-fixture", "hdfs" -configurations { - hdfsFixture +for (int hadoopVersion = minTestedHadoopVersion; hadoopVersion <= maxTestedHadoopVersion; hadoopVersion++) { + configurations.create("hdfs" + hadoopVersion + "Fixture") } dependencies { - api "org.apache.hadoop:hadoop-client:${versions.hadoop2}" - api project(path: 'hadoop-common', configuration: 'shadow') + api project(path: 'hadoop-client-api', configuration: 'shadow') if (isEclipse) { /* * Eclipse can't pick up the shadow dependency so we point it at *something* * so it can compile things. */ - api project(path: 'hadoop-common') + api project(path: 'hadoop-client-api') } - api "org.apache.hadoop:hadoop-annotations:${versions.hadoop2}" - api "org.apache.hadoop:hadoop-auth:${versions.hadoop2}" - api "org.apache.hadoop:hadoop-hdfs:${versions.hadoop2}" - api "org.apache.hadoop:hadoop-hdfs-client:${versions.hadoop2}" + runtimeOnly "org.apache.hadoop:hadoop-client-runtime:${versions.hadoop}" + implementation "org.apache.hadoop:hadoop-hdfs:${versions.hadoop}" api 'org.apache.htrace:htrace-core4:4.0.1-incubating' - runtimeOnly 'com.google.guava:guava:11.0.2' + runtimeOnly 'com.google.guava:guava:27.1-jre' api 'com.google.protobuf:protobuf-java:2.5.0' api 'commons-logging:commons-logging:1.1.3' api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" api 'commons-cli:commons-cli:1.2' api "commons-codec:commons-codec:${versions.commonscodec}" api 'commons-collections:commons-collections:3.2.2' - api 'commons-configuration:commons-configuration:1.6' + api 'org.apache.commons:commons-configuration2:2.7' api 'commons-io:commons-io:2.4' api 'commons-lang:commons-lang:2.6' api 'javax.servlet:servlet-api:2.5' api "org.slf4j:slf4j-api:${versions.slf4j}" api "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" - - hdfsFixture project(':test:fixtures:hdfs-fixture') // Set the keytab files in the classpath so that we can access them from test code without the security manager // freaking out. if (isEclipse == false) { @@ -73,6 +72,9 @@ dependencies { } } } +for (int hadoopVersion = minTestedHadoopVersion; hadoopVersion <= maxTestedHadoopVersion; hadoopVersion++) { + dependencies.add("hdfs" + hadoopVersion + "Fixture", project(':test:fixtures:hdfs' + hadoopVersion + '-fixture')) +} restResources { restApi { @@ -95,6 +97,7 @@ tasks.named("dependencyLicenses").configure { tasks.named("integTest").configure { dependsOn(project.tasks.named("bundlePlugin")) + enabled = false } testClusters.matching { it.name == "integTest" }.configureEach { @@ -105,79 +108,96 @@ String realm = "BUILD.ELASTIC.CO" String krb5conf = project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs") // Create HDFS File System Testing Fixtures for HA/Secure combinations -for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) { - project.tasks.register(fixtureName, org.elasticsearch.gradle.internal.test.AntFixture) { - dependsOn project.configurations.hdfsFixture, project(':test:fixtures:krb5kdc-fixture').tasks.postProcessFixture - executable = "${BuildParams.runtimeJavaHome}/bin/java" - env 'CLASSPATH', "${-> project.configurations.hdfsFixture.asPath}" - maxWaitInSeconds 60 - onlyIf { BuildParams.inFipsJvm == false } - waitCondition = { fixture, ant -> - // the hdfs.MiniHDFS fixture writes the ports file when - // it's ready, so we can just wait for the file to exist - return fixture.portsFile.exists() - } - final List miniHDFSArgs = [] +for (int hadoopVersion = minTestedHadoopVersion; hadoopVersion <= maxTestedHadoopVersion; hadoopVersion++) { + for (String fixtureName : ['hdfs' + hadoopVersion + 'Fixture', 'haHdfs' + hadoopVersion + 'Fixture', 'secureHdfs' + hadoopVersion + 'Fixture', 'secureHaHdfs' + hadoopVersion + 'Fixture']) { + final int hadoopVer = hadoopVersion + project.tasks.register(fixtureName, org.elasticsearch.gradle.internal.test.AntFixture) { + executable = "${BuildParams.runtimeJavaHome}/bin/java" + dependsOn project.configurations.getByName("hdfs" + hadoopVer + "Fixture"), project(':test:fixtures:krb5kdc-fixture').tasks.postProcessFixture + env 'CLASSPATH', "${-> project.configurations.getByName("hdfs" + hadoopVer + "Fixture").asPath}" + + maxWaitInSeconds 60 + onlyIf { BuildParams.inFipsJvm == false } + waitCondition = { fixture, ant -> + // the hdfs.MiniHDFS fixture writes the ports file when + // it's ready, so we can just wait for the file to exist + return fixture.portsFile.exists() + } + final List miniHDFSArgs = [] - // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options - if (name.equals('secureHdfsFixture') || name.equals('secureHaHdfsFixture')) { - miniHDFSArgs.add("-Djava.security.krb5.conf=${krb5conf}") - onlyIf { BuildParams.runtimeJavaVersion < JavaVersion.VERSION_16 } - } - // If it's an HA fixture, set a nameservice to use in the JVM options - if (name.equals('haHdfsFixture') || name.equals('secureHaHdfsFixture')) { - miniHDFSArgs.add("-Dha-nameservice=ha-hdfs") - } + // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options + if (name.startsWith('secure')) { + miniHDFSArgs.addAll(["--add-exports", "java.security.jgss/sun.security.krb5=ALL-UNNAMED"]) + miniHDFSArgs.add("-Djava.security.krb5.conf=${krb5conf}") + miniHDFSArgs.add("-Dhdfs.config.port=" + getSecureNamenodePortForVersion(hadoopVer)) + } else { + miniHDFSArgs.add("-Dhdfs.config.port=" + getNonSecureNamenodePortForVersion(hadoopVer)) + } + // If it's an HA fixture, set a nameservice to use in the JVM options + if (name.startsWith('haHdfs') || name.startsWith('secureHaHdfs')) { + miniHDFSArgs.add("-Dha-nameservice=ha-hdfs") + } - // Common options - miniHDFSArgs.add('hdfs.MiniHDFS') - miniHDFSArgs.add(baseDir) + // Common options + miniHDFSArgs.add('hdfs.MiniHDFS') + miniHDFSArgs.add(baseDir) - // If it's a secure fixture, then set the principal name and keytab locations to use for auth. - if (name.equals('secureHdfsFixture') || name.equals('secureHaHdfsFixture')) { - miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}") - miniHDFSArgs.add( - project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") - ) - } + // If it's a secure fixture, then set the principal name and keytab locations to use for auth. + if (name.startsWith('secure')) { + miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}") + miniHDFSArgs.add( + project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") + ) + } - args miniHDFSArgs.toArray() + args miniHDFSArgs.toArray() + } } } +def getSecureNamenodePortForVersion(hadoopVersion) { + return 10002 - (2 * hadoopVersion) +} + +def getNonSecureNamenodePortForVersion(hadoopVersion) { + return 10003 - (2 * hadoopVersion) +} + Set disabledIntegTestTaskNames = [] -for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) { - def testTask = tasks.register(integTestTaskName, RestIntegTestTask) { - description = "Runs rest tests against an elasticsearch cluster with HDFS." - dependsOn("bundlePlugin") +for (int hadoopVersion = minTestedHadoopVersion; hadoopVersion <= maxTestedHadoopVersion; hadoopVersion++) { + final int hadoopVer = hadoopVersion + for (String integTestTaskName : ['integTest' + hadoopVersion, 'integTestHa' + hadoopVersion, 'integTestSecure' + hadoopVersion, + 'integTestSecureHa' + hadoopVersion]) { + def testTask = tasks.register(integTestTaskName, RestIntegTestTask) { + description = "Runs rest tests against an elasticsearch cluster with HDFS" + hadoopVer + dependsOn("bundlePlugin") - if (disabledIntegTestTaskNames.contains(name)) { - enabled = false; - } + if (disabledIntegTestTaskNames.contains(name)) { + enabled = false; + } - if (name.contains("Secure")) { - onlyIf { BuildParams.runtimeJavaVersion < JavaVersion.VERSION_16 } - if (name.contains("Ha")) { - dependsOn "secureHaHdfsFixture" - } else { - dependsOn "secureHdfsFixture" + if (name.contains("Secure")) { + if (name.contains("Ha")) { + dependsOn "secureHaHdfs" + hadoopVer + "Fixture" + } else { + dependsOn "secureHdfs" + hadoopVer + "Fixture" + } } - } onlyIf { BuildParams.inFipsJvm == false } if (name.contains("Ha")) { Path portsFile - File portsFileDir = file("${workingDir}/hdfsFixture") + File portsFileDir = file("${workingDir}/hdfs" + hadoopVer + "Fixture") if (name.contains("Secure")) { portsFile = buildDir.toPath() .resolve("fixtures") - .resolve("secureHaHdfsFixture") + .resolve("secureHaHdfs" + hadoopVer + "Fixture") .resolve("ports") } else { portsFile = buildDir.toPath() .resolve("fixtures") - .resolve("haHdfsFixture") + .resolve("haHdfs" + hadoopVer + "Fixture") .resolve("ports") } nonInputProperties.systemProperty "test.hdfs-fixture.ports", file("$portsFileDir/ports") @@ -196,26 +216,47 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec if (disabledIntegTestTaskNames.contains(name) == false) { nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" - jvmArgs "-Djava.security.krb5.conf=${krb5conf}", - "--add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED" + jvmArgs "-Djava.security.krb5.conf=${krb5conf}" nonInputProperties.systemProperty( "test.krb5.keytab.hdfs", project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") ) } } + } + + testClusters.matching { it.name == testTask.name }.configureEach { + plugin(bundlePlugin.archiveFile) + if (integTestTaskName.contains("Secure")) { + systemProperty "java.security.krb5.conf", krb5conf + extraConfigFile( + "repository-hdfs/krb5.keytab", + file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}"), IGNORE_VALUE + ) + } + } } - testClusters.matching { it.name == testTask.name}.configureEach { - plugin(bundlePlugin.archiveFile) - if (integTestTaskName.contains("Secure")) { - systemProperty "java.security.krb5.conf", krb5conf - extraConfigFile( - "repository-hdfs/krb5.keytab", - file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}"), IGNORE_VALUE - ) + def processHadoopTestResources = tasks.register("processHadoop" + hadoopVer + "TestResources", Copy) + processHadoopTestResources.configure { + Map expansions = [ + 'hdfs_port': getNonSecureNamenodePortForVersion(hadoopVer), + 'secure_hdfs_port': getSecureNamenodePortForVersion(hadoopVer), + ] + inputs.properties(expansions) + filter("tokens" : expansions.collectEntries {k, v -> [k, v + .toString()]} /* must be a map of strings */, ReplaceTokens.class) + it.into("build/resources/test/rest-api-spec/test") + it.into("hdfs_repository_" + hadoopVer) { + from "src/test/resources/rest-api-spec/test/hdfs_repository" + } + it.into("secure_hdfs_repository_" + hadoopVer) { + from "src/test/resources/rest-api-spec/test/secure_hdfs_repository" } } + tasks.named("processTestResources").configure { + dependsOn (processHadoopTestResources) + } } // Determine HDFS Fixture compatibility for the current build environment. @@ -243,63 +284,66 @@ if (legalPath == false) { fixtureSupported = false } -// Always ignore HA integration tests in the normal integration test runner, they are included below as -// part of their own HA-specific integration test tasks. -tasks.named("integTest").configure { - onlyIf { BuildParams.inFipsJvm == false } - exclude('**/Ha*TestSuiteIT.class') -} - -if (fixtureSupported) { - // Check depends on the HA test. Already depends on the standard test. - tasks.named("check").configure { - dependsOn("integTestHa") +for (int hadoopVersion = minTestedHadoopVersion; hadoopVersion <= maxTestedHadoopVersion; hadoopVersion++) { + final int hadoopVer = hadoopVersion + // Always ignore HA integration tests in the normal integration test runner, they are included below as + // part of their own HA-specific integration test tasks. + tasks.named("integTest" + hadoopVer).configure { + onlyIf { BuildParams.inFipsJvm == false } + exclude('**/Ha*TestSuiteIT.class') } - // Both standard and HA tests depend on their respective HDFS fixtures - tasks.named("integTest").configure { - dependsOn "hdfsFixture" + if (fixtureSupported) { + // Check depends on the HA test. Already depends on the standard test. + tasks.named("check").configure { + dependsOn("integTestHa" + hadoopVer) + } - // The normal test runner only runs the standard hdfs rest tests - systemProperty 'tests.rest.suite', 'hdfs_repository' - } - tasks.named("integTestHa").configure { - dependsOn "haHdfsFixture" - // Only include the HA integration tests for the HA test task - setIncludes(['**/Ha*TestSuiteIT.class']) + // Both standard and HA tests depend on their respective HDFS fixtures + tasks.named("integTest" + hadoopVer).configure { + dependsOn "hdfs" + hadoopVer + "Fixture" - } + // The normal test runner only runs the standard hdfs rest tests + systemProperty 'tests.rest.suite', 'hdfs_repository_' + hadoopVer + } + tasks.named("integTestHa" + hadoopVer).configure { + dependsOn "haHdfs" + hadoopVer + "Fixture" + // Only include the HA integration tests for the HA test task + setIncludes(['**/Ha*TestSuiteIT.class']) + + } -} else { - if (legalPath) { - logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH") } else { - logger.warn("hdfsFixture unsupported since there are spaces in the path: '" + rootProject.rootDir.toString() + "'") - } + if (legalPath) { + logger.warn("hdfs" + hadoopVer + "Fixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH") + } else { + logger.warn("hdfs" + hadoopVer + "Fixture unsupported since there are spaces in the path: '" + rootProject.rootDir.toString() + "'") + } - // The normal integration test runner will just test that the plugin loads - tasks.named("integTest").configure { - systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic' - } - // HA fixture is unsupported. Don't run them. - tasks.named("integTestHa").configure { - setEnabled(false) + // The normal integration test runner will just test that the plugin loads + tasks.named("integTest" + hadoopVer).configure { + systemProperty 'tests.rest.suite', 'hdfs_repository_' + hadoopVer + '/10_basic' + } + // HA fixture is unsupported. Don't run them. + tasks.named("integTestHa" + hadoopVer).configure { + setEnabled(false) + } } -} -tasks.named("check").configure { - dependsOn("integTestSecure", "integTestSecureHa") -} + tasks.named("check").configure { + dependsOn("integTest" + hadoopVer, "integTestSecure" + hadoopVer, "integTestSecureHa" + hadoopVer) + } // Run just the secure hdfs rest test suite. -tasks.named("integTestSecure").configure { - systemProperty 'tests.rest.suite', 'secure_hdfs_repository' - // Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner. - exclude('**/Ha*TestSuiteIT.class') -} + tasks.named("integTestSecure" + hadoopVer).configure { + systemProperty 'tests.rest.suite', 'secure_hdfs_repository_' + hadoopVer + // Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner. + exclude('**/Ha*TestSuiteIT.class') + } // Only include the HA integration tests for the HA test task -tasks.named("integTestSecureHa").configure { - setIncludes(['**/Ha*TestSuiteIT.class']) + tasks.named("integTestSecureHa" + hadoopVer).configure { + setIncludes(['**/Ha*TestSuiteIT.class']) + } } tasks.named("thirdPartyAudit").configure { @@ -308,7 +352,82 @@ tasks.named("thirdPartyAudit").configure { // internal java api: sun.misc.Unsafe 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', - 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm', - 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot', + 'com.google.common.cache.Striped64', + 'com.google.common.cache.Striped64$1', + 'com.google.common.cache.Striped64$Cell', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', + 'com.google.common.hash.Striped64', + 'com.google.common.hash.Striped64$1', + 'com.google.common.hash.Striped64$Cell', + 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + 'org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture$UnsafeAtomicHelper', + 'org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture$UnsafeAtomicHelper$1', + 'org.apache.hadoop.shaded.com.google.common.cache.Striped64', + 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$1', + 'org.apache.hadoop.shaded.com.google.common.cache.Striped64$Cell', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'org.apache.hadoop.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', + 'org.apache.hadoop.shaded.com.google.common.hash.Striped64', + 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$1', + 'org.apache.hadoop.shaded.com.google.common.hash.Striped64$Cell', + 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', + 'org.apache.hadoop.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'org.apache.hadoop.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeBooleanField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeByteField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCachedField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCharField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeCustomEncodedField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeDoubleField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeFloatField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeIntField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeLongField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeObjectField', + 'org.apache.hadoop.shaded.org.apache.avro.reflect.FieldAccessUnsafe$UnsafeShortField', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.cache.Striped64$Cell', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.hash.Striped64$Cell', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'org.apache.hadoop.shaded.org.apache.curator.shaded.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + 'org.apache.hadoop.shaded.org.xbill.DNS.spi.DNSJavaNameServiceDescriptor', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.PureJavaSnappy', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.SnappyRawCompressor', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.SnappyRawDecompressor', + 'org.apache.hadoop.shaded.org.xerial.snappy.pure.UnsafeUtil', + 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64', + 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$1', + 'org.apache.hadoop.thirdparty.com.google.common.cache.Striped64$Cell', + 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', + 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', + 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', + 'org.apache.hadoop.thirdparty.com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', + 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64', + 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$1', + 'org.apache.hadoop.thirdparty.com.google.common.hash.Striped64$Cell', + 'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', + 'org.apache.hadoop.thirdparty.com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', + 'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', + 'org.apache.hadoop.thirdparty.com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$1', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$JvmMemoryAccessor', + 'org.apache.hadoop.thirdparty.protobuf.UnsafeUtil$MemoryAccessor' ) } diff --git a/plugins/repository-hdfs/hadoop-common/build.gradle b/plugins/repository-hdfs/hadoop-client-api/build.gradle similarity index 69% rename from plugins/repository-hdfs/hadoop-common/build.gradle rename to plugins/repository-hdfs/hadoop-client-api/build.gradle index ddff6ae503def..345f8605eac90 100644 --- a/plugins/repository-hdfs/hadoop-common/build.gradle +++ b/plugins/repository-hdfs/hadoop-client-api/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.java' apply plugin: 'com.github.johnrengelman.shadow' dependencies { - implementation "org.apache.hadoop:hadoop-common:${project.parent.versions.hadoop2}" + implementation "org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}" } tasks.named('shadowJar').configure { diff --git a/plugins/repository-hdfs/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java b/plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java similarity index 100% rename from plugins/repository-hdfs/hadoop-common/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java rename to plugins/repository-hdfs/hadoop-client-api/src/main/java/org/apache/hadoop/util/ShutdownHookManager.java diff --git a/plugins/repository-hdfs/licenses/commons-configuration-1.6.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration-1.6.jar.sha1 deleted file mode 100644 index 44ad1f6d8da42..0000000000000 --- a/plugins/repository-hdfs/licenses/commons-configuration-1.6.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -32cadde23955d7681b0d94a2715846d20b425235 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-configuration2-2.7.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration2-2.7.jar.sha1 new file mode 100644 index 0000000000000..31e16840e2c4f --- /dev/null +++ b/plugins/repository-hdfs/licenses/commons-configuration2-2.7.jar.sha1 @@ -0,0 +1 @@ +593326399e5fb5e1f986607f06f63c1250ab36b4 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/commons-configuration-LICENSE.txt b/plugins/repository-hdfs/licenses/commons-configuration2-LICENSE.txt similarity index 100% rename from plugins/repository-hdfs/licenses/commons-configuration-LICENSE.txt rename to plugins/repository-hdfs/licenses/commons-configuration2-LICENSE.txt diff --git a/plugins/repository-hdfs/licenses/commons-configuration-NOTICE.txt b/plugins/repository-hdfs/licenses/commons-configuration2-NOTICE.txt similarity index 83% rename from plugins/repository-hdfs/licenses/commons-configuration-NOTICE.txt rename to plugins/repository-hdfs/licenses/commons-configuration2-NOTICE.txt index 3d6dfaec547c6..25c286381506f 100644 --- a/plugins/repository-hdfs/licenses/commons-configuration-NOTICE.txt +++ b/plugins/repository-hdfs/licenses/commons-configuration2-NOTICE.txt @@ -1,4 +1,4 @@ -Apache Commons Configuration +Apache Commons Configuration 2 Copyright 2001-2015 The Apache Software Foundation This product includes software developed at diff --git a/plugins/repository-hdfs/licenses/guava-11.0.2.jar.sha1 b/plugins/repository-hdfs/licenses/guava-11.0.2.jar.sha1 deleted file mode 100644 index 0588c7ee5eaf7..0000000000000 --- a/plugins/repository-hdfs/licenses/guava-11.0.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35a3c69e19d72743cac83778aecbee68680f63eb \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/guava-27.1-jre.jar.sha1 b/plugins/repository-hdfs/licenses/guava-27.1-jre.jar.sha1 new file mode 100644 index 0000000000000..07bf9c19e42b2 --- /dev/null +++ b/plugins/repository-hdfs/licenses/guava-27.1-jre.jar.sha1 @@ -0,0 +1 @@ +e47b59c893079b87743cdcfb6f17ca95c08c592c \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1 deleted file mode 100644 index a808179106498..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-annotations-2.8.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a301159daf9368b05364577c985498857b5c48f \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1 deleted file mode 100644 index 2d9064f4965eb..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-auth-2.8.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -63fe1f9d9ef6bdf2cb52dfeb28ed8faf78e4b85c \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1 deleted file mode 100644 index 83d232c5a295d..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-client-2.8.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -45e11f6004581e53959bc3d38c4d01dbeb5f4b22 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 new file mode 100644 index 0000000000000..feb37ecc90255 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1 @@ -0,0 +1 @@ +f3a55d882328ee87a1054f99d62ba987fa9029a4 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1 deleted file mode 100644 index 241a3ea92ac2b..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-hdfs-2.8.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -36914392fd3e77d46e54c3bb092dfc32d3f4a32b \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.1.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.1.jar.sha1 new file mode 100644 index 0000000000000..66c98cf7ec291 --- /dev/null +++ b/plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.1.jar.sha1 @@ -0,0 +1 @@ +5da7f270cb6564e099e0d2d424285a24fca62bd2 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1 b/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1 deleted file mode 100644 index 2a05eca7320a8..0000000000000 --- a/plugins/repository-hdfs/licenses/hadoop-hdfs-client-2.8.5.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9d72fa62b01c32f1c0587d53c5005fc49f2bd11c \ No newline at end of file diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_create.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_create.yml index 0f942dfdc03d6..c992e42afaeb0 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_create.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_create.yml @@ -10,7 +10,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "test/repository_create" # Get repository diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_delete.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_delete.yml index 34c770a8074c7..0abf71bf7df45 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_delete.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_delete.yml @@ -10,7 +10,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "foo/bar" # Get repository @@ -39,7 +39,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "foo/bar" # Get repository again diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_verify.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_verify.yml index d1695b00d9dbc..ff7357e084577 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_verify.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_verify.yml @@ -9,7 +9,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "test/repository_verify" # Verify repository diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot.yml index 20019686d3db1..69cea05c525a5 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot.yml @@ -11,7 +11,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "test/snapshot" # Create index @@ -36,7 +36,7 @@ - match: { snapshot.shards.failed : 0 } # Remove our snapshot - - do: + - do: snapshot.delete: repository: test_snapshot_repository snapshot: test_snapshot diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml index f38f4783b195b..3a282c2c9942c 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml @@ -11,7 +11,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "test/snapshot_get" # Create index @@ -59,7 +59,7 @@ - match: { snapshots.0.snapshot : test_snapshot_get } # Remove our snapshot - - do: + - do: snapshot.delete: repository: test_snapshot_get_repository snapshot: test_snapshot_get diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml index c2a37964e70a7..1c718d63183dd 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml @@ -11,7 +11,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "/user/elasticsearch/existing/readonly-repository" readonly: true diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/40_restore.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/40_restore.yml index 1e9250ff88069..716857781b758 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/40_restore.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/40_restore.yml @@ -14,7 +14,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9999" + uri: "hdfs://localhost:@hdfs_port@" path: "test/restore" # Create index diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yml index 46cc9e92479c5..117d9a74d87d9 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yml @@ -10,7 +10,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/test/repository_create" security: principal: "elasticsearch@BUILD.ELASTIC.CO" diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yml index 7bff67d3d01b0..26b73174b2205 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yml @@ -10,7 +10,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/foo/bar" security: principal: "elasticsearch@BUILD.ELASTIC.CO" @@ -41,7 +41,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/foo/bar" security: principal: "elasticsearch@BUILD.ELASTIC.CO" diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yml index 4593c13caee23..187d43b7e88d6 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yml @@ -9,7 +9,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/test/repository_verify" security: principal: "elasticsearch@BUILD.ELASTIC.CO" diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yml index e232782084e16..01b137279714a 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yml @@ -11,7 +11,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/test/snapshot" security: principal: "elasticsearch@BUILD.ELASTIC.CO" @@ -38,7 +38,7 @@ - match: { snapshot.shards.failed : 0 } # Remove our snapshot - - do: + - do: snapshot.delete: repository: test_snapshot_repository snapshot: test_snapshot diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yml index 20d988884113f..67f3faf14ed5e 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yml @@ -11,7 +11,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/test/snapshot_get" security: principal: "elasticsearch@BUILD.ELASTIC.CO" @@ -61,7 +61,7 @@ - match: { snapshots.0.snapshot : test_snapshot_get } # Remove our snapshot - - do: + - do: snapshot.delete: repository: test_snapshot_get_repository snapshot: test_snapshot_get diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yml index 8c4c0347a156a..e093424a6548c 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yml @@ -11,7 +11,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/existing/readonly-repository" security: principal: "elasticsearch@BUILD.ELASTIC.CO" diff --git a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml index ce757236ced68..c7a298d72d6c9 100644 --- a/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml +++ b/plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml @@ -14,7 +14,7 @@ body: type: hdfs settings: - uri: "hdfs://localhost:9998" + uri: "hdfs://localhost:@secure_hdfs_port@" path: "/user/elasticsearch/test/restore" security: principal: "elasticsearch@BUILD.ELASTIC.CO" diff --git a/settings.gradle b/settings.gradle index 7dea0e8721077..7c7654c901629 100644 --- a/settings.gradle +++ b/settings.gradle @@ -65,7 +65,8 @@ List projects = [ 'test:framework', 'test:fixtures:azure-fixture', 'test:fixtures:gcs-fixture', - 'test:fixtures:hdfs-fixture', + 'test:fixtures:hdfs2-fixture', + 'test:fixtures:hdfs3-fixture', 'test:fixtures:krb5kdc-fixture', 'test:fixtures:minio-fixture', 'test:fixtures:old-elasticsearch', diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs2-fixture/build.gradle similarity index 100% rename from test/fixtures/hdfs-fixture/build.gradle rename to test/fixtures/hdfs2-fixture/build.gradle diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs2-fixture/src/main/java/hdfs/MiniHDFS.java similarity index 100% rename from test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java rename to test/fixtures/hdfs2-fixture/src/main/java/hdfs/MiniHDFS.java diff --git a/test/fixtures/hdfs-fixture/src/main/resources/readonly-repository.tar.gz b/test/fixtures/hdfs2-fixture/src/main/resources/readonly-repository.tar.gz similarity index 100% rename from test/fixtures/hdfs-fixture/src/main/resources/readonly-repository.tar.gz rename to test/fixtures/hdfs2-fixture/src/main/resources/readonly-repository.tar.gz diff --git a/test/fixtures/hdfs3-fixture/build.gradle b/test/fixtures/hdfs3-fixture/build.gradle new file mode 100644 index 0000000000000..872ab2efd42ab --- /dev/null +++ b/test/fixtures/hdfs3-fixture/build.gradle @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +apply plugin: 'elasticsearch.java' + +dependencies { + api "org.apache.hadoop:hadoop-minicluster:3.3.1" +} diff --git a/test/fixtures/hdfs3-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs3-fixture/src/main/java/hdfs/MiniHDFS.java new file mode 100644 index 0000000000000..0a26f5d82ac17 --- /dev/null +++ b/test/fixtures/hdfs3-fixture/src/main/java/hdfs/MiniHDFS.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package hdfs; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.MiniDFSNNTopology; +import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; +import org.apache.hadoop.security.UserGroupInformation; + +import java.io.File; +import java.lang.management.ManagementFactory; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * MiniHDFS test fixture. There is a CLI tool, but here we can + * easily properly setup logging, avoid parsing JSON, etc. + */ +public class MiniHDFS { + + private static String PORT_FILE_NAME = "ports"; + private static String PID_FILE_NAME = "pid"; + + public static void main(String[] args) throws Exception { + if (args.length != 1 && args.length != 3) { + throw new IllegalArgumentException( + "Expected: MiniHDFS [ ], got: " + Arrays.toString(args) + ); + } + boolean secure = args.length == 3; + + // configure Paths + Path baseDir = Paths.get(args[0]); + // hadoop-home/, so logs will not complain + if (System.getenv("HADOOP_HOME") == null) { + Path hadoopHome = baseDir.resolve("hadoop-home"); + Files.createDirectories(hadoopHome); + System.setProperty("hadoop.home.dir", hadoopHome.toAbsolutePath().toString()); + } + // hdfs-data/, where any data is going + Path hdfsHome = baseDir.resolve("hdfs-data"); + + // configure cluster + Configuration cfg = new Configuration(); + cfg.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsHome.toAbsolutePath().toString()); + // lower default permission: TODO: needed? + cfg.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY, "766"); + + // optionally configure security + if (secure) { + String kerberosPrincipal = args[1]; + String keytabFile = args[2]; + + cfg.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + cfg.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true"); + cfg.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, kerberosPrincipal); + cfg.set(DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, kerberosPrincipal); + cfg.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, kerberosPrincipal); + cfg.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, keytabFile); + cfg.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, keytabFile); + cfg.set(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, "true"); + cfg.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true"); + cfg.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true"); + cfg.set(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY, "true"); + cfg.set(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY, "AES/CTR/NoPadding"); + } + + UserGroupInformation.setConfiguration(cfg); + + MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(cfg); + String explicitPort = System.getProperty("hdfs.config.port"); + if (explicitPort != null) { + builder.nameNodePort(Integer.parseInt(explicitPort)); + } else { + if (secure) { + builder.nameNodePort(9998); + } else { + builder.nameNodePort(9999); + } + } + + // Configure HA mode + String haNameService = System.getProperty("ha-nameservice"); + boolean haEnabled = haNameService != null; + if (haEnabled) { + MiniDFSNNTopology.NNConf nn1 = new MiniDFSNNTopology.NNConf("nn1").setIpcPort(0); + MiniDFSNNTopology.NNConf nn2 = new MiniDFSNNTopology.NNConf("nn2").setIpcPort(0); + MiniDFSNNTopology.NSConf nameservice = new MiniDFSNNTopology.NSConf(haNameService).addNN(nn1).addNN(nn2); + MiniDFSNNTopology namenodeTopology = new MiniDFSNNTopology().addNameservice(nameservice); + builder.nnTopology(namenodeTopology); + } + + MiniDFSCluster dfs = builder.build(); + + // Configure contents of the filesystem + org.apache.hadoop.fs.Path esUserPath = new org.apache.hadoop.fs.Path("/user/elasticsearch"); + + FileSystem fs; + if (haEnabled) { + dfs.transitionToActive(0); + fs = HATestUtil.configureFailoverFs(dfs, cfg); + } else { + fs = dfs.getFileSystem(); + } + + try { + // Set the elasticsearch user directory up + fs.mkdirs(esUserPath); + if (UserGroupInformation.isSecurityEnabled()) { + List acls = new ArrayList<>(); + acls.add(new AclEntry.Builder().setType(AclEntryType.USER).setName("elasticsearch").setPermission(FsAction.ALL).build()); + fs.modifyAclEntries(esUserPath, acls); + } + + // Install a pre-existing repository into HDFS + String directoryName = "readonly-repository"; + String archiveName = directoryName + ".tar.gz"; + URL readOnlyRepositoryArchiveURL = MiniHDFS.class.getClassLoader().getResource(archiveName); + if (readOnlyRepositoryArchiveURL != null) { + Path tempDirectory = Files.createTempDirectory(MiniHDFS.class.getName()); + File readOnlyRepositoryArchive = tempDirectory.resolve(archiveName).toFile(); + FileUtils.copyURLToFile(readOnlyRepositoryArchiveURL, readOnlyRepositoryArchive); + FileUtil.unTar(readOnlyRepositoryArchive, tempDirectory.toFile()); + + fs.copyFromLocalFile( + true, + true, + new org.apache.hadoop.fs.Path(tempDirectory.resolve(directoryName).toAbsolutePath().toUri()), + esUserPath.suffix("/existing/" + directoryName) + ); + + FileUtils.deleteDirectory(tempDirectory.toFile()); + } + } finally { + fs.close(); + } + + // write our PID file + Path tmp = Files.createTempFile(baseDir, null, null); + String pid = ManagementFactory.getRuntimeMXBean().getName().split("@")[0]; + Files.write(tmp, pid.getBytes(StandardCharsets.UTF_8)); + Files.move(tmp, baseDir.resolve(PID_FILE_NAME), StandardCopyOption.ATOMIC_MOVE); + + // write our port file + String portFileContent = Integer.toString(dfs.getNameNodePort(0)); + if (haEnabled) { + portFileContent = portFileContent + "\n" + Integer.toString(dfs.getNameNodePort(1)); + } + tmp = Files.createTempFile(baseDir, null, null); + Files.write(tmp, portFileContent.getBytes(StandardCharsets.UTF_8)); + Files.move(tmp, baseDir.resolve(PORT_FILE_NAME), StandardCopyOption.ATOMIC_MOVE); + } + +} diff --git a/test/fixtures/hdfs3-fixture/src/main/resources/readonly-repository.tar.gz b/test/fixtures/hdfs3-fixture/src/main/resources/readonly-repository.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..2cdb6d77c07d0a06029521d60fdc966e92248d72 GIT binary patch literal 1314 zcmV+-1>O1|iwFQ)qqIjP0S`X&6>Cgu-pu#=zCXvJBxxF2uE(NTlc~08Xdnk?@>NJB3M=Q zuNq0Z+=H672A4-jl4|Q;ka1D}I20B!*MB%1#%+P-J;5nlaQ&|anL)BFW7;e@@cL@d z;9>|4lm`_Y><&7Pn&e|?6cwk4u`qL2pO1@^%W^cX^bhg#Tr@Vn39rp|2l?Jn6O_-j z8cY(KxJU11A_|i_)$|yPjlC8Q%|xOoI-PcJKC*nRC1%&w*3Mf{k0~|nzgOc*Wp*>x zsAvBOqvQ53U@Rc8e*k#c)Ubcohc_(`{O|c+$^MZb2$27^f@=ELsZDyShvHry)Ei!- zp8vZEp*>Lxs&nYBAL`e!Q2^xU8t9 zxanZrk&+7&x9F?WPS5E0Z7t3EQn8ikZJX*@FL5Xw92|W6(ucp7*13QE%6I4Znm`Sz z*}t2Tx6=MaQ9NP)vIO?u3OxHS)nIKU``;>!6xQ8>aN@8WT=%TM`sT; zR+8<$$D9P8jisFO&{m45nq%=L_K&-p$U9SW=P1=jhtu-E>NR`gX<@_tKS2z+{U6Bx z9zJK#KWcObG_HRMi3s%H3eKSahRcI`@n7bf<5>I`F_J`x|E<7{|4;i3{}WZf`mqAw z`}Zx=u~j3j7>-^FzT51Wh6`7v!jq8^nL1I z!6%yI1%;VX`0mrx1Yv{d~0;C*RxOB$`T?TfXdMEmbF0PT?hw z&h~cq`T7dW83~1Gnwjw4j1}HOd-3RKR52$e;xRimsSW4S^NC?`acRGz09ttP&;4`M zb1lx~{}rVn)2wIzf+SSr|B{4du>V%z=l?XVQ^RKbf5*)~^Id=Z@Yz`Z8&|%(+Wpmd zJMQQxUHnA($5XaN45pcBuN&~}(1N+R6pP%N877N}nz9yorSI<2Gv$lY)Zk)uO{N`R z)ycBr7>uN-LLA%V)qD`*?)r#!g}@}s{H;VVgW(_uO(FP|5bMVc07--6cgy(lfAod*4;l3 z-aSm)JqNOzFLv