Upgrade repository-hdfs to Hadoop 3

plugins/repository-hdfs/hadoop-common/build.gradle → plugins/repository-hdfs/hadoop-client-api/build.gradle

@@ -2,7 +2,7 @@ apply plugin: 'elasticsearch.java'
 apply plugin: 'com.github.johnrengelman.shadow'
 
 dependencies {
-  implementation "org.apache.hadoop:hadoop-common:${project.parent.versions.hadoop2}"
+  implementation "org.apache.hadoop:hadoop-client-api:${project.parent.versions.hadoop}"
 }
 
 tasks.named('shadowJar').configure {

plugins/repository-hdfs/licenses/commons-configuration2-2.7.jar.sha1

@@ -0,0 +1 @@
+593326399e5fb5e1f986607f06f63c1250ab36b4

plugins/repository-hdfs/licenses/commons-configuration-NOTICE.txt → plugins/repository-hdfs/licenses/commons-configuration2-NOTICE.txt

@@ -1,4 +1,4 @@
-Apache Commons Configuration
+Apache Commons Configuration 2
 Copyright 2001-2015 The Apache Software Foundation
 
 This product includes software developed at

plugins/repository-hdfs/licenses/guava-27.1-jre.jar.sha1

@@ -0,0 +1 @@
+e47b59c893079b87743cdcfb6f17ca95c08c592c

plugins/repository-hdfs/licenses/hadoop-client-runtime-3.3.1.jar.sha1

@@ -0,0 +1 @@
+f3a55d882328ee87a1054f99d62ba987fa9029a4

plugins/repository-hdfs/licenses/hadoop-hdfs-3.3.1.jar.sha1

@@ -0,0 +1 @@
+5da7f270cb6564e099e0d2d424285a24fca62bd2

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_create.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "test/repository_create"
 
     # Get repository

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_delete.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "foo/bar"
 
     # Get repository
@@ -39,7 +39,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "foo/bar"
 
     # Get repository again

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/20_repository_verify.yml

@@ -9,7 +9,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9999"
+              uri: "hdfs://localhost:@hdfs_port@"
               path: "test/repository_verify"
 
     # Verify repository

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "test/snapshot"
 
   # Create index
@@ -36,7 +36,7 @@
   - match: { snapshot.shards.failed : 0 }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_repository
         snapshot: test_snapshot

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_get.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "test/snapshot_get"
 
   # Create index
@@ -59,7 +59,7 @@
   - match: { snapshots.0.snapshot : test_snapshot_get }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_get_repository
         snapshot: test_snapshot_get

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/30_snapshot_readonly.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "/user/elasticsearch/existing/readonly-repository"
             readonly: true
 

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/hdfs_repository/40_restore.yml

@@ -14,7 +14,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9999"
+            uri: "hdfs://localhost:@hdfs_port@"
             path: "test/restore"
 
   # Create index

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_create.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/test/repository_create"
               security:
                 principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_delete.yml

@@ -10,7 +10,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/foo/bar"
               security:
                 principal: "[email protected]"
@@ -41,7 +41,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/foo/bar"
               security:
                 principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/20_repository_verify.yml

@@ -9,7 +9,7 @@
           body:
             type: hdfs
             settings:
-              uri: "hdfs://localhost:9998"
+              uri: "hdfs://localhost:@secure_hdfs_port@"
               path: "/user/elasticsearch/test/repository_verify"
               security:
                 principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/test/snapshot"
             security:
               principal: "[email protected]"
@@ -38,7 +38,7 @@
   - match: { snapshot.shards.failed : 0 }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_repository
         snapshot: test_snapshot

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_get.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/test/snapshot_get"
             security:
               principal: "[email protected]"
@@ -61,7 +61,7 @@
   - match: { snapshots.0.snapshot : test_snapshot_get }
 
   # Remove our snapshot
-  - do: 
+  - do:
       snapshot.delete:
         repository: test_snapshot_get_repository
         snapshot: test_snapshot_get

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/30_snapshot_readonly.yml

@@ -11,7 +11,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/existing/readonly-repository"
             security:
               principal: "[email protected]"

plugins/repository-hdfs/src/test/resources/rest-api-spec/test/secure_hdfs_repository/40_restore.yml

@@ -14,7 +14,7 @@
         body:
           type: hdfs
           settings:
-            uri: "hdfs://localhost:9998"
+            uri: "hdfs://localhost:@secure_hdfs_port@"
             path: "/user/elasticsearch/test/restore"
             security:
               principal: "[email protected]"

settings.gradle

@@ -65,7 +65,8 @@ List projects = [
   'test:framework',
   'test:fixtures:azure-fixture',
   'test:fixtures:gcs-fixture',
-  'test:fixtures:hdfs-fixture',
+  'test:fixtures:hdfs2-fixture',
+  'test:fixtures:hdfs3-fixture',
   'test:fixtures:krb5kdc-fixture',
   'test:fixtures:minio-fixture',
   'test:fixtures:old-elasticsearch',

test/fixtures/hdfs3-fixture/build.gradle

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+apply plugin: 'elasticsearch.java'
+
+dependencies {
+  api "org.apache.hadoop:hadoop-minicluster:3.3.1"
+}