Backmerging with trunk

BUILDING.txt

@@ -7,7 +7,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.3 or later
 * Boost 1.72 (if compiling native code)
-* Protocol Buffers 3.7.1 (if compiling native code)
+* Protocol Buffers 3.21.12 (if compiling native code)
 * CMake 3.19 or newer (if compiling native code)
 * Zlib devel (if compiling native code)
 * Cyrus SASL devel (if compiling native code)
@@ -74,10 +74,10 @@ Refer to  dev-support/docker/Dockerfile):
   $ ./bootstrap
   $ make -j$(nproc)
   $ sudo make install
-* Protocol Buffers 3.7.1 (required to build native code)
-  $ curl -L -s -S https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz -o protobuf-3.7.1.tar.gz
-  $ mkdir protobuf-3.7-src
-  $ tar xzf protobuf-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7-src && cd protobuf-3.7-src
+* Protocol Buffers 3.21.12 (required to build native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure
   $ make -j$(nproc)
   $ sudo make install
@@ -433,10 +433,10 @@ Installing required dependencies for clean install of macOS 10.14:
 * Install native libraries, only openssl is required to compile native code,
 you may optionally install zlib, lz4, etc.
   $ brew install openssl
-* Protocol Buffers 3.7.1 (required to compile native code)
-  $ wget https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz
-  $ mkdir -p protobuf-3.7 && tar zxvf protobuf-java-3.7.1.tar.gz --strip-components 1 -C protobuf-3.7
-  $ cd protobuf-3.7
+* Protocol Buffers 3.21.12 (required to compile native code)
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure
   $ make
   $ make check
@@ -472,11 +472,10 @@ Building on CentOS 8
 * Install python2 for building documentation.
   $ sudo dnf install python2
 
-* Install Protocol Buffers v3.7.1.
-  $ git clone https://github.com/protocolbuffers/protobuf
-  $ cd protobuf
-  $ git checkout v3.7.1
-  $ autoreconf -i
+* Install Protocol Buffers v3.21.12.
+  $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz
+  $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12
+  $ ./autogen.sh
   $ ./configure --prefix=/usr/local
   $ make
   $ sudo make install
@@ -531,7 +530,7 @@ Requirements:
 * JDK 1.8
 * Maven 3.0 or later (maven.apache.org)
 * Boost 1.72 (boost.org)
-* Protocol Buffers 3.7.1 (https://github.com/protocolbuffers/protobuf/releases)
+* Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags)
 * CMake 3.19 or newer (cmake.org)
 * Visual Studio 2019 (visualstudio.com)
 * Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from

LICENSE-binary

@@ -210,6 +210,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.* (css and js
 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/AbstractFuture.java
 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/checker/TimeoutFuture.java
 
+ch.qos.reload4j:reload4j:1.2.22
 com.aliyun:aliyun-java-sdk-core:4.5.10
 com.aliyun:aliyun-java-sdk-kms:2.11.0
 com.aliyun:aliyun-java-sdk-ram:3.1.0
@@ -294,7 +295,6 @@ io.reactivex:rxjava-string:1.1.1
 io.reactivex:rxnetty:0.4.20
 io.swagger:swagger-annotations:1.5.4
 javax.inject:javax.inject:1
-log4j:log4j:1.2.17
 net.java.dev.jna:jna:5.2.0
 net.minidev:accessors-smart:1.2
 org.apache.avro:avro:1.9.2
@@ -363,7 +363,7 @@ org.objenesis:objenesis:2.6
 org.xerial.snappy:snappy-java:1.1.10.4
 org.yaml:snakeyaml:2.0
 org.wildfly.openssl:wildfly-openssl:1.1.3.Final
-software.amazon.awssdk:bundle:jar:2.23.5
+software.amazon.awssdk:bundle:jar:2.23.19
 
 
 --------------------------------------------------------------------------------
@@ -489,11 +489,10 @@ org.checkerframework:checker-qual:2.5.2
 org.codehaus.mojo:animal-sniffer-annotations:1.21
 org.jruby.jcodings:jcodings:1.0.13
 org.jruby.joni:joni:2.1.2
-org.slf4j:jul-to-slf4j:jar:1.7.25
-org.ojalgo:ojalgo:43.0:compile
-org.slf4j:jul-to-slf4j:1.7.25
-org.slf4j:slf4j-api:1.7.25
-org.slf4j:slf4j-log4j12:1.7.25
+org.ojalgo:ojalgo:43.0
+org.slf4j:jul-to-slf4j:1.7.36
+org.slf4j:slf4j-api:1.7.36
+org.slf4j:slf4j-reload4j:1.7.36
 
 
 CDDL 1.1 + GPLv2 with classpath exception

dev-support/docker/Dockerfile

@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.7.1
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_aarch64

@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.7.1
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_centos_7

@@ -76,7 +76,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_centos_8

@@ -101,7 +101,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/Dockerfile_debian_10

@@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Set env vars for Google Protobuf 3.7.1
+# Set env vars for Google Protobuf 3.21.12
 #######
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"

dev-support/docker/pkg-resolver/install-protobuf.sh

@@ -27,25 +27,26 @@ if [ $? -eq 1 ]; then
   exit 1
 fi
 
-default_version="3.7.1"
+default_version="3.21.12"
 version_to_install=$default_version
 if [ -n "$2" ]; then
   version_to_install="$2"
 fi
 
-if [ "$version_to_install" != "3.7.1" ]; then
+if [ "$version_to_install" != "3.21.12" ]; then
   echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
   version_to_install=$default_version
 fi
 
-if [ "$version_to_install" == "3.7.1" ]; then
+if [ "$version_to_install" == "3.21.12" ]; then
   # hadolint ignore=DL3003
   mkdir -p /opt/protobuf-src &&
     curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
+      https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \
       -o /opt/protobuf.tar.gz &&
     tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src &&
     cd /opt/protobuf-src &&
+    ./autogen.sh &&
     ./configure --prefix=/opt/protobuf &&
     make "-j$(nproc)" &&
     make install &&

hadoop-assemblies/src/main/resources/assemblies/hadoop-dynamometer.xml

@@ -66,7 +66,7 @@
       <excludes>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
       </excludes>
     </dependencySet>
   </dependencySets>

hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml

@@ -40,7 +40,7 @@
         <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
         <exclude>org.hsqldb:hsqldb</exclude>
       </excludes>
     </dependencySet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-httpfs-dist.xml

@@ -69,7 +69,7 @@
         <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
         <exclude>org.hsqldb:hsqldb</exclude>
       </excludes>
     </dependencySet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml

@@ -69,7 +69,7 @@
         <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
         <exclude>org.hsqldb:hsqldb</exclude>
       </excludes>
     </dependencySet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml

@@ -179,7 +179,7 @@
         <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
         <exclude>org.hsqldb:hsqldb</exclude>
         <exclude>jdiff:jdiff:jar</exclude>
       </excludes>

hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml

@@ -40,7 +40,7 @@
         <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
         <exclude>org.hsqldb:hsqldb</exclude>
       </excludes>
     </dependencySet>

hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml

@@ -229,7 +229,7 @@
         <exclude>org.apache.hadoop:hadoop-pipes</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
       </excludes>
     </dependencySet>
   </dependencySets>

hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml

@@ -309,7 +309,7 @@
         <exclude>org.apache.hadoop:*</exclude>
         <!-- use slf4j from common to avoid multiple binding warnings -->
         <exclude>org.slf4j:slf4j-api</exclude>
-        <exclude>org.slf4j:slf4j-log4j12</exclude>
+        <exclude>org.slf4j:slf4j-reload4j</exclude>
         <exclude>org.hsqldb:hsqldb</exclude>
       </excludes>
     </dependencySet>

hadoop-client-modules/hadoop-client-check-invariants/pom.xml

@@ -84,8 +84,8 @@
                     <exclude>org.slf4j:slf4j-api</exclude>
                     <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
                     <exclude>commons-logging:commons-logging</exclude>
-                    <!-- Leave log4j unshaded so downstream users can configure logging. -->
-                    <exclude>log4j:log4j</exclude>
+                    <!-- Leave reload4j unshaded so downstream users can configure logging. -->
+                    <exclude>ch.qos.reload4j:reload4j</exclude>
                     <!-- Leave javax annotations we need exposed -->
                     <exclude>com.google.code.findbugs:jsr305</exclude>
                     <!-- Leave bouncycastle unshaded because it's signed with a special Oracle certificate so it can be a custom JCE security provider -->

hadoop-client-modules/hadoop-client-check-test-invariants/pom.xml

@@ -88,8 +88,8 @@
                     <exclude>org.slf4j:slf4j-api</exclude>
                     <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
                     <exclude>commons-logging:commons-logging</exclude>
-                    <!-- Leave log4j unshaded so downstream users can configure logging. -->
-                    <exclude>log4j:log4j</exclude>
+                    <!-- Leave reload4j unshaded so downstream users can configure logging. -->
+                    <exclude>ch.qos.reload4j:reload4j</exclude>
                     <!-- Leave JUnit unshaded so downstream can use our test helper classes -->
                     <exclude>junit:junit</exclude>
                     <!-- JUnit brings in hamcrest -->

hadoop-client-modules/hadoop-client-integration-tests/pom.xml

@@ -33,20 +33,15 @@
 
   <dependencies>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
+      <groupId>ch.qos.reload4j</groupId>
+      <artifactId>reload4j</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

hadoop-client-modules/hadoop-client-minicluster/pom.xml

@@ -193,8 +193,12 @@
           <artifactId>slf4j-log4j12</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>log4j</groupId>
-          <artifactId>log4j</artifactId>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-reload4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>ch.qos.reload4j</groupId>
+          <artifactId>reload4j</artifactId>
         </exclusion>
         <exclusion>
           <groupId>com.fasterxml.jackson.core</groupId>
@@ -693,7 +697,7 @@
                       <exclude>commons-logging:commons-logging</exclude>
                       <exclude>junit:junit</exclude>
                       <exclude>com.google.code.findbugs:jsr305</exclude>
-                      <exclude>log4j:log4j</exclude>
+                      <exclude>ch.qos.reload4j:reload4j</exclude>
                       <exclude>org.eclipse.jetty.websocket:websocket-common</exclude>
                       <exclude>org.eclipse.jetty.websocket:websocket-api</exclude>
                       <!-- We need a filter that matches just those things that are included in the above artiacts -->

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -103,8 +103,8 @@
          * one of the three custom log4j appenders we have
       -->
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
+      <groupId>ch.qos.reload4j</groupId>
+      <artifactId>reload4j</artifactId>
       <scope>runtime</scope>
       <optional>true</optional>
     </dependency>
@@ -143,8 +143,8 @@
                       <exclude>org.slf4j:slf4j-api</exclude>
                       <!-- Leave commons-logging unshaded so downstream users can configure logging. -->
                       <exclude>commons-logging:commons-logging</exclude>
-                      <!-- Leave log4j unshaded so downstream users can configure logging. -->
-                      <exclude>log4j:log4j</exclude>
+                      <!-- Leave reload4j unshaded so downstream users can configure logging. -->
+                      <exclude>ch.qos.reload4j:reload4j</exclude>
                       <!-- Leave javax APIs that are stable -->
                       <!-- the jdk ships part of the javax.annotation namespace, so if we want to relocate this we'll have to care it out by class :( -->
                       <exclude>com.google.code.findbugs:jsr305</exclude>

hadoop-client-modules/hadoop-client/pom.xml

@@ -214,8 +214,8 @@
           <artifactId>commons-cli</artifactId>
         </exclusion>
         <exclusion>
-          <groupId>log4j</groupId>
-          <artifactId>log4j</artifactId>
+          <groupId>ch.qos.reload4j</groupId>
+          <artifactId>reload4j</artifactId>
         </exclusion>
         <exclusion>
           <groupId>com.sun.jersey</groupId>
@@ -298,11 +298,6 @@
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>
         </exclusion>
-        <!-- No slf4j backends for downstream clients -->
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
 
@@ -331,11 +326,6 @@
           <groupId>io.netty</groupId>
           <artifactId>netty</artifactId>
         </exclusion>
-        <!-- No slf4j backends for downstream clients -->
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
       </exclusions>
     </dependency>
 

hadoop-common-project/hadoop-auth-examples/pom.xml

@@ -47,13 +47,13 @@
       <scope>compile</scope>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
+      <groupId>ch.qos.reload4j</groupId>
+      <artifactId>reload4j</artifactId>
       <scope>runtime</scope>
     </dependency>
     <dependency>
       <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
+      <artifactId>slf4j-reload4j</artifactId>
       <scope>runtime</scope>
     </dependency>
   </dependencies>