Upgrade repository-hdfs to Hadoop 3 (#76897)

This upgrades the repository-hdfs plugin to hadoop 3. Tests are performed against both hadoop 2 and hadoop 3 HDFS. The advantages of using the hadoop 3 client are:
Over-the-wire encryption works (tests coming in an upcoming PR).
We don't have to add (or ask customers to add) additional jvm permissions to the elasticsearch jvm
It's compatible with java versions higher than java 8

plugins/repository-hdfs/hadoop-common/build.gradle → plugins/repository-hdfs/hadoop-client-api/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.java'
 apply plugin: 'com.github.johnrengelman.shadow'
 
 dependencies {
-  implementation "org.apache.hadoop:hadoop-common:${project.parent.versions.hadoop2}"
+  implementation "org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}"
 }
 
 tasks.named('shadowJar').configure {

plugins/repository-hdfs/licenses/commons-configuration2-2.7.jar.sha1

@@ -0,0 +1 @@
+593326399e5fb5e1f986607f06f63c1250ab36b4

plugins/repository-hdfs/licenses/commons-configuration-NOTICE.txt → plugins/repository-hdfs/licenses/commons-configuration2-NOTICE.txt

@@ -1,4 +1,4 @@
-Apache Commons Configuration
+Apache Commons Configuration 2
 Copyright 2001-2015 The Apache Software Foundation
 
 This product includes software developed at

plugins/repository-hdfs/licenses/guava-27.1-jre.jar.sha1

@@ -0,0 +1 @@
+e47b59c893079b87743cdcfb6f17ca95c08c592c

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1

@@ -0,0 +1 @@
+f3a55d882328ee87a1054f99d62ba987fa9029a4

plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.1.jar.sha1

@@ -0,0 +1 @@
+5da7f270cb6564e099e0d2d424285a24fca62bd2

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_create.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "test/repository_create"
 
     # Get repository

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_delete.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "foo/bar"
 
     # Get repository
@@ -39,7 +39,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "foo/bar"
 
     # Get repository again

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_verify.yml

@@ -9,7 +9,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "test/repository_verify"
 
     # Verify repository

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "test/snapshot"
 
   # Create index
@@ -36,7 +36,7 @@
   - match: { snapshot.shards.failed : 0 }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_repository
         snapshot: test_snapshot

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "test/snapshot_get"
 
   # Create index
@@ -59,7 +59,7 @@
   - match: { snapshots.0.snapshot : test_snapshot_get }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_get_repository
         snapshot: test_snapshot_get

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "/user/elasticsearch/existing/readonly-repository"
             readonly: true
 

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/40_restore.yml

@@ -14,7 +14,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "test/restore"
 
   # Create index

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/test/repository_create"
               security:
                 principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/foo/bar"
               security:
                 principal: "[email protected]"
@@ -41,7 +41,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/foo/bar"
               security:
                 principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yml

@@ -9,7 +9,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/test/repository_verify"
               security:
                 principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/test/snapshot"
             security:
               principal: "[email protected]"
@@ -38,7 +38,7 @@
   - match: { snapshot.shards.failed : 0 }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_repository
         snapshot: test_snapshot

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/test/snapshot_get"
             security:
               principal: "[email protected]"
@@ -61,7 +61,7 @@
   - match: { snapshots.0.snapshot : test_snapshot_get }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_get_repository
         snapshot: test_snapshot_get

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/existing/readonly-repository"
             security:
               principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml

@@ -14,7 +14,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/test/restore"
             security:
               principal: "[email protected]"

settings.gradle

@@ -65,7 +65,8 @@ List projects = [
   'test:framework',
   'test:fixtures:azure-fixture',
   'test:fixtures:gcs-fixture',
-  'test:fixtures:hdfs-fixture',
+  'test:fixtures:hdfs2-fixture',
+  'test:fixtures:hdfs3-fixture',
   'test:fixtures:krb5kdc-fixture',
   'test:fixtures:minio-fixture',
   'test:fixtures:old-elasticsearch',

test/fixtures/hdfs3-fixture/build.gradle

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+apply plugin: 'elasticsearch.java'
+
+dependencies {
+  api "org.apache.hadoop:hadoop-minicluster:3.3.1"
+}