Merge branch 'apache:trunk' into YARN-11711

apache · Aug 21, 2024 · 2a60d06 · 2a60d06
2 parents 978e588 + 68fcd72
commit 2a60d06
Show file tree

Hide file tree

Showing 157 changed files with 2,859 additions and 1,209 deletions.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -169,7 +169,7 @@ protected int init(String[] args) throws IOException {
 
   @Override
   public String getCommandUsage() {
-    StringBuffer sbuf = new StringBuffer(USAGE_PREFIX + COMMANDS);
+    StringBuilder sbuf = new StringBuilder(USAGE_PREFIX + COMMANDS);
     String banner = StringUtils.repeat("=", 66);
     sbuf.append(banner + "\n");
     sbuf.append(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC + "\n");

diff --git a/...oject/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/...oject/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.crypto.key.kms;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider;
@@ -561,17 +562,19 @@ private <T> T call(HttpURLConnection conn, Object jsonOutput,
       }
       throw ex;
     }
+
     if ((conn.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN
-        && (conn.getResponseMessage().equals(ANONYMOUS_REQUESTS_DISALLOWED) ||
-            conn.getResponseMessage().contains(INVALID_SIGNATURE)))
+        && (!StringUtils.isEmpty(conn.getResponseMessage())
+            && (conn.getResponseMessage().equals(ANONYMOUS_REQUESTS_DISALLOWED)
+            || conn.getResponseMessage().contains(INVALID_SIGNATURE))))
         || conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
       // Ideally, this should happen only when there is an Authentication
       // failure. Unfortunately, the AuthenticationFilter returns 403 when it
       // cannot authenticate (Since a 401 requires Server to send
       // WWW-Authenticate header as well)..
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Response={}({}), resetting authToken",
-            conn.getResponseCode(), conn.getResponseMessage());
+        LOG.debug("Response={}, resetting authToken",
+            conn.getResponseCode());
       }
       KMSClientProvider.this.authToken =
           new DelegationTokenAuthenticatedURL.Token();
@@ -797,6 +800,7 @@ public EncryptedKeyVersion generateEncryptedKey(
   @SuppressWarnings("rawtypes")
   @Override
   public KeyVersion decryptEncryptedKey(
+
       EncryptedKeyVersion encryptedKeyVersion) throws IOException,
                                                       GeneralSecurityException {
     checkNotNull(encryptedKeyVersion.getEncryptionKeyVersionName(),

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java
@@ -163,7 +163,7 @@ protected void parseExecResult(BufferedReader lines) throws IOException {
   @VisibleForTesting
   protected void parseOutput() throws IOException {
     if (output.size() < 2) {
-      StringBuffer sb = new StringBuffer("Fewer lines of output than expected");
+      StringBuilder sb = new StringBuilder("Fewer lines of output than expected");
       if (output.size() > 0) {
         sb.append(": " + output.get(0));
       }

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -1052,7 +1052,7 @@ private static void unTarUsingTar(InputStream inputStream, File untarDir,
 
   private static void unTarUsingTar(File inFile, File untarDir,
       boolean gzipped) throws IOException {
-    StringBuffer untarCommand = new StringBuffer();
+    StringBuilder untarCommand = new StringBuilder();
     // not using canonical path here; this postpones relative path
     // resolution until bash is executed.
     final String source = "'" + FileUtil.makeSecureShellPath(inFile) + "'";

diff --git a/...op-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java b/...op-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/VectoredReadUtils.java
@@ -308,7 +308,7 @@ public static List<? extends FileRange> validateAndSortRanges(
       validateRangeRequest(input.get(0));
       sortedRanges = input;
     } else {
-      sortedRanges = sortRanges(input);
+      sortedRanges = sortRangeList(input);
       FileRange prev = null;
       for (final FileRange current : sortedRanges) {
         validateRangeRequest(current);
@@ -341,12 +341,25 @@ public static List<? extends FileRange> validateAndSortRanges(
    * @param input input ranges.
    * @return a new list of the ranges, sorted by offset.
    */
-  public static List<? extends FileRange> sortRanges(List<? extends FileRange> input) {
+  public static List<? extends FileRange> sortRangeList(List<? extends FileRange> input) {
     final List<? extends FileRange> l = new ArrayList<>(input);
     l.sort(Comparator.comparingLong(FileRange::getOffset));
     return l;
   }
 
+  /**
+   * Sort the input ranges by offset; no validation is done.
+   * <p>
+   * This method is used externally and must be retained with
+   * the signature unchanged.
+   * @param input input ranges.
+   * @return a new list of the ranges, sorted by offset.
+   */
+  @InterfaceStability.Stable
+  public static FileRange[] sortRanges(List<? extends FileRange> input) {
+    return sortRangeList(input).toArray(new FileRange[0]);
+  }
+
   /**
    * Merge sorted ranges to optimize the access from the underlying file
    * system.

diff --git a/...t/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java b/...t/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
@@ -51,7 +51,6 @@ public class RSRawDecoder extends RawErasureDecoder {
   private byte[] gfTables;
   private int[] cachedErasedIndexes;
   private int[] validIndexes;
-  private int numErasedDataUnits;
   private boolean[] erasureFlags;
 
   public RSRawDecoder(ErasureCoderOptions coderOptions) {
@@ -120,14 +119,10 @@ private void processErasures(int[] erasedIndexes) {
     this.gfTables = new byte[getNumAllUnits() * getNumDataUnits() * 32];
 
     this.erasureFlags = new boolean[getNumAllUnits()];
-    this.numErasedDataUnits = 0;
 
     for (int i = 0; i < erasedIndexes.length; i++) {
       int index = erasedIndexes[i];
       erasureFlags[index] = true;
-      if (index < getNumDataUnits()) {
-        numErasedDataUnits++;
-      }
     }
 
     generateDecodeMatrix(erasedIndexes);
@@ -156,21 +151,22 @@ private void generateDecodeMatrix(int[] erasedIndexes) {
 
     GF256.gfInvertMatrix(tmpMatrix, invertMatrix, getNumDataUnits());
 
-    for (i = 0; i < numErasedDataUnits; i++) {
-      for (j = 0; j < getNumDataUnits(); j++) {
-        decodeMatrix[getNumDataUnits() * i + j] =
-                invertMatrix[getNumDataUnits() * erasedIndexes[i] + j];
-      }
-    }
-
-    for (p = numErasedDataUnits; p < erasedIndexes.length; p++) {
-      for (i = 0; i < getNumDataUnits(); i++) {
-        s = 0;
+    for (p = 0; p < erasedIndexes.length; p++) {
+      int erasedIndex = erasedIndexes[p];
+      if (erasedIndex < getNumDataUnits()) {
         for (j = 0; j < getNumDataUnits(); j++) {
-          s ^= GF256.gfMul(invertMatrix[j * getNumDataUnits() + i],
-                  encodeMatrix[getNumDataUnits() * erasedIndexes[p] + j]);
+          decodeMatrix[getNumDataUnits() * p + j] =
+              invertMatrix[getNumDataUnits() * erasedIndexes[p] + j];
+        }
+      } else {
+        for (i = 0; i < getNumDataUnits(); i++) {
+          s = 0;
+          for (j = 0; j < getNumDataUnits(); j++) {
+            s ^= GF256.gfMul(invertMatrix[j * getNumDataUnits() + i],
+                encodeMatrix[getNumDataUnits() * erasedIndexes[p] + j]);
+          }
+          decodeMatrix[getNumDataUnits() * p + i] = s;
         }
-        decodeMatrix[getNumDataUnits() * p + i] = s;
       }
     }
   }

diff --git a/...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/oncrpc/RpcDeniedReply.java b/...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/oncrpc/RpcDeniedReply.java
@@ -58,7 +58,7 @@ public RejectState getRejectState() {
 
   @Override
   public String toString() {
-    return new StringBuffer().append("xid:").append(xid)
+    return new StringBuilder().append("xid:").append(xid)
         .append(",messageType:").append(messageType).append("verifier_flavor:")
         .append(verifier.getFlavor()).append("rejectState:")
         .append(rejectState).toString();

diff --git a/...-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java b/...-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
@@ -148,7 +148,7 @@ public static Configuration excludeIncompatibleCredentialProviders(
     if (providerPath == null) {
       return config;
     }
-    StringBuffer newProviderPath = new StringBuffer();
+    StringBuilder newProviderPath = new StringBuilder();
     String[] providers = providerPath.split(",");
     Path path = null;
     for (String provider: providers) {

diff --git a/...project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/...project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -127,7 +127,7 @@ protected int init(String[] args) throws IOException {
 
   @Override
   public String getCommandUsage() {
-    StringBuffer sbuf = new StringBuffer(USAGE_PREFIX + COMMANDS);
+    StringBuilder sbuf = new StringBuilder(USAGE_PREFIX + COMMANDS);
     String banner = StringUtils.repeat("=", 66);
     sbuf.append(banner + "\n")
         .append(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC + "\n")

diff --git a/...oject/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java b/...oject/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLHostnameVerifier.java
@@ -370,7 +370,7 @@ public void check(final String[] hosts, final String[] cns,
                     strictWithSubDomains);
             }
             // Build up lists of allowed hosts For logging/debugging purposes.
-            StringBuffer buf = new StringBuffer(32);
+            StringBuilder buf = new StringBuilder(32);
             buf.append('<');
             for (int i = 0; i < hosts.length; i++) {
                 String h = hosts[i];
@@ -408,15 +408,15 @@ public void check(final String[] hosts, final String[] cns,
                 throw new SSLException(msg);
             }
 
-            // StringBuffer for building the error message.
-            buf = new StringBuffer();
+            // StringBuilder for building the error message.
+            buf = new StringBuilder();
 
             boolean match = false;
             out:
             for (Iterator<String> it = names.iterator(); it.hasNext();) {
                 // Don't trim the CN, though!
                 final String cn = StringUtils.toLowerCase(it.next());
-                // Store CN in StringBuffer in case we need to report an error.
+                // Store CN in StringBuilder in case we need to report an error.
             buf.append(" <")
             .append(cn)
             .append('>');

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -1014,7 +1014,7 @@ private void runCommand() throws IOException {
     BufferedReader inReader =
             new BufferedReader(new InputStreamReader(process.getInputStream(),
                 StandardCharsets.UTF_8));
-    final StringBuffer errMsg = new StringBuffer();
+    final StringBuilder errMsg = new StringBuilder();
 
     // read error and input streams as this would free up the buffers
     // free the error stream buffer
@@ -1208,7 +1208,7 @@ public static class ShellCommandExecutor extends Shell
       implements CommandExecutor {
 
     private String[] command;
-    private StringBuffer output;
+    private StringBuilder output;
 
 
     public ShellCommandExecutor(String[] execString) {
@@ -1289,7 +1289,7 @@ public String[] getExecString() {
 
     @Override
     protected void parseExecResult(BufferedReader lines) throws IOException {
-      output = new StringBuffer();
+      output = new StringBuilder();
       char[] buf = new char[512];
       int nRead;
       while ( (nRead = lines.read(buf, 0, buf.length)) > 0 ) {

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -1334,7 +1334,7 @@ public static String wrap(String str, int wrapLength, String newLineStr,
 
       int inputLineLength = str.length();
       int offset = 0;
-      StringBuffer wrappedLine = new StringBuffer(inputLineLength + 32);
+      StringBuilder wrappedLine = new StringBuilder(inputLineLength + 32);
 
       while(inputLineLength - offset > wrapLength) {
         if(str.charAt(offset) == 32) {

diff --git a/...roject/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c b/...roject/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c
@@ -132,9 +132,6 @@ static int processErasures(IsalDecoder* pCoder, unsigned char** inputs,
     index = erasedIndexes[i];
     pCoder->erasedIndexes[i] = index;
     pCoder->erasureFlags[index] = 1;
-    if (index < numDataUnits) {
-      pCoder->numErasedDataUnits++;
-    }
   }
 
   pCoder->numErased = numErased;
@@ -175,7 +172,6 @@ int decode(IsalDecoder* pCoder, unsigned char** inputs,
 
 // Clear variables used per decode call
 void clearDecoder(IsalDecoder* decoder) {
-  decoder->numErasedDataUnits = 0;
   decoder->numErased = 0;
   memset(decoder->gftbls, 0, sizeof(decoder->gftbls));
   memset(decoder->decodeMatrix, 0, sizeof(decoder->decodeMatrix));
@@ -205,24 +201,24 @@ int generateDecodeMatrix(IsalDecoder* pCoder) {
   h_gf_invert_matrix(pCoder->tmpMatrix,
                                 pCoder->invertMatrix, numDataUnits);
 
-  for (i = 0; i < pCoder->numErasedDataUnits; i++) {
+  for (p = 0; p < pCoder->numErased; p++) {
     for (j = 0; j < numDataUnits; j++) {
-      pCoder->decodeMatrix[numDataUnits * i + j] =
-                      pCoder->invertMatrix[numDataUnits *
-                      pCoder->erasedIndexes[i] + j];
-    }
-  }
-
-  for (p = pCoder->numErasedDataUnits; p < pCoder->numErased; p++) {
-    for (i = 0; i < numDataUnits; i++) {
-      s = 0;
-      for (j = 0; j < numDataUnits; j++) {
-        s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i],
-          pCoder->encodeMatrix[numDataUnits *
-                                        pCoder->erasedIndexes[p] + j]);
+      int erasedIndex = pCoder->erasedIndexes[p];
+      if (erasedIndex < numDataUnits) {
+        pCoder->decodeMatrix[numDataUnits * p + j] =
+            pCoder->invertMatrix[numDataUnits *
+                                     pCoder->erasedIndexes[p] + j];
+      } else {
+        for (i = 0; i < numDataUnits; i++) {
+          s = 0;
+          for (j = 0; j < numDataUnits; j++) {
+            s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i],
+                          pCoder->encodeMatrix[numDataUnits *
+                            pCoder->erasedIndexes[p] + j]);
+          }
+          pCoder->decodeMatrix[numDataUnits * p + i] = s;
+        }
       }
-
-      pCoder->decodeMatrix[numDataUnits * p + i] = s;
     }
   }
 

diff --git a/...roject/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h b/...roject/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h
@@ -62,7 +62,6 @@ typedef struct _IsalDecoder {
   unsigned char erasureFlags[MMAX];
   int erasedIndexes[MMAX];
   int numErased;
-  int numErasedDataUnits;
   unsigned char* realInputs[MMAX];
 } IsalDecoder;