apache#7 kap[6250] fix kerberos issue for both FI and CDH, eta ASAP

yuexingri · Oct 15, 2019 · d24b2db · d24b2db
1 parent 6ece7cc
commit d24b2db
Show file tree

Hide file tree

Showing 39 changed files with 80 additions and 40 deletions.
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/tags/pom.xml b/common/tags/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/common/unsafe/pom.xml b/common/unsafe/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/core/pom.xml b/core/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/examples/pom.xml b/examples/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/flume/pom.xml b/external/flume/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-10-assembly/pom.xml b/external/kafka-0-10-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-10-sql/pom.xml b/external/kafka-0-10-sql/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-10/pom.xml b/external/kafka-0-10/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-8-assembly/pom.xml b/external/kafka-0-8-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kafka-0-8/pom.xml b/external/kafka-0-8/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kinesis-asl-assembly/pom.xml b/external/kinesis-asl-assembly/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/kinesis-asl/pom.xml b/external/kinesis-asl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/external/spark-ganglia-lgpl/pom.xml b/external/spark-ganglia-lgpl/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/graphx/pom.xml b/graphx/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/launcher/pom.xml b/launcher/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/mllib/pom.xml b/mllib/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/pom.xml b/pom.xml
@@ -26,7 +26,7 @@
   </parent>
   <groupId>org.apache.spark</groupId>
   <artifactId>spark-parent_2.11</artifactId>
-  <version>2.2.1-kylin-r5</version>
+  <version>2.2.1-kylin-r6</version>
   <packaging>pom</packaging>
   <name>Spark Project Parent POM</name>
   <url>http://spark.apache.org/</url>

diff --git a/repl/pom.xml b/repl/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

diff --git a/resource-managers/mesos/pom.xml b/resource-managers/mesos/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml
@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -405,8 +405,10 @@ private[spark] class Client(
       // slightly later then renewal time (80% of next renewal time). This is to make sure
       // credentials are renewed and updated before expired.
       val currTime = System.currentTimeMillis()
-      val renewalTime = (nearestTimeOfNextRenewal - currTime) * 0.75 + currTime
-      val updateTime = (nearestTimeOfNextRenewal - currTime) * 0.8 + currTime
+      val renewalTime = (nearestTimeOfNextRenewal - currTime) *
+        sparkConf.get(CREDENTIAL_RENEW_TIME_COEFFICIENT) + currTime
+      val updateTime = (nearestTimeOfNextRenewal - currTime) *
+        sparkConf.get(CREDENTIAL_UPDATE_COEFFICIENT) + currTime
 
       sparkConf.set(CREDENTIALS_RENEWAL_TIME, renewalTime.toLong)
       sparkConf.set(CREDENTIALS_UPDATE_TIME, updateTime.toLong)

diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -237,6 +237,21 @@ package object config {
       .createOptional
 
   /* Security configuration. */
+  private[spark] val CREDENTIAL_DRIVER_SKIP_UPDATE =
+    ConfigBuilder("spark.yarn.credentials.driver.skipUpdate")
+      .booleanConf
+      .createWithDefault(true)
+
+
+  private[spark] val CREDENTIAL_UPDATE_COEFFICIENT =
+    ConfigBuilder("spark.yarn.credentials.update.coefficient")
+      .doubleConf
+      .createWithDefault(0.6)
+
+  private[spark] val CREDENTIAL_RENEW_TIME_COEFFICIENT =
+    ConfigBuilder("spark.yarn.credentials.renewTime.coefficient")
+      .doubleConf
+      .createWithDefault(0.5)
 
   private[spark] val CREDENTIAL_FILE_MAX_COUNT =
     ConfigBuilder("spark.yarn.credentials.file.retention.count")

diff --git a/...agers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/AMCredentialRenewer.scala b/...agers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/AMCredentialRenewer.scala
@@ -67,6 +67,8 @@ private[yarn] class AMCredentialRenewer(
   private val credentialsFile = sparkConf.get(CREDENTIALS_FILE_PATH)
   private val daysToKeepFiles = sparkConf.get(CREDENTIALS_FILE_MAX_RETENTION)
   private val numFilesToKeep = sparkConf.get(CREDENTIAL_FILE_MAX_COUNT)
+  private val credentialUpdateCoefficient = sparkConf.get(CREDENTIAL_UPDATE_COEFFICIENT)
+  private val credentialRenewTimeCoefficient = sparkConf.get(CREDENTIAL_RENEW_TIME_COEFFICIENT)
   private val freshHadoopConf =
     hadoopUtil.getConfBypassingFSCache(hadoopConf, new Path(credentialsFile).toUri.getScheme)
 
@@ -193,8 +195,9 @@ private[yarn] class AMCredentialRenewer(
     } else {
       // Next valid renewal time is about 75% of credential renewal time, and update time is
       // slightly later than valid renewal time (80% of renewal time).
-      timeOfNextRenewal = ((nearestNextRenewalTime - currTime) * 0.75 + currTime).toLong
-      ((nearestNextRenewalTime - currTime) * 0.8 + currTime).toLong
+      timeOfNextRenewal = ((nearestNextRenewalTime - currTime) *
+        credentialRenewTimeCoefficient + currTime).toLong
+      ((nearestNextRenewalTime - currTime) * credentialUpdateCoefficient + currTime).toLong
     }
 
     // Add the temp credentials back to the original ones.

diff --git a/...anagers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala b/...anagers/yarn/src/main/scala/org/apache/spark/deploy/yarn/security/CredentialUpdater.scala
@@ -19,18 +19,22 @@ package org.apache.spark.deploy.yarn.security
 
 import java.util.concurrent.{Executors, TimeUnit}
 
+import scala.collection.JavaConverters._
 import scala.util.control.NonFatal
 
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
 import org.apache.hadoop.security.{Credentials, UserGroupInformation}
 
-import org.apache.spark.SparkConf
+import org.apache.spark.{SparkConf, SparkContext, SparkEnv}
 import org.apache.spark.deploy.SparkHadoopUtil
 import org.apache.spark.deploy.yarn.config._
 import org.apache.spark.internal.Logging
 import org.apache.spark.util.{ThreadUtils, Utils}
 
+
+
 private[spark] class CredentialUpdater(
     sparkConf: SparkConf,
     hadoopConf: Configuration,
@@ -39,6 +43,7 @@ private[spark] class CredentialUpdater(
   @volatile private var lastCredentialsFileSuffix = 0
 
   private val credentialsFile = sparkConf.get(CREDENTIALS_FILE_PATH)
+  private val skipUpdate = sparkConf.get(CREDENTIAL_DRIVER_SKIP_UPDATE)
   private val freshHadoopConf =
     SparkHadoopUtil.get.getConfBypassingFSCache(
       hadoopConf, new Path(credentialsFile).toUri.getScheme)
@@ -66,6 +71,10 @@ private[spark] class CredentialUpdater(
   }
 
   private def updateCredentialsIfRequired(): Unit = {
+    if (skipUpdate && (SparkEnv.get.executorId == SparkContext.DRIVER_IDENTIFIER)) {
+      logInfo("Skip update token with driver.")
+      return
+    }
     val timeToNextUpdate = try {
       val credentialsFilePath = new Path(credentialsFile)
       val remoteFs = FileSystem.get(freshHadoopConf)
@@ -78,7 +87,18 @@ private[spark] class CredentialUpdater(
             logInfo("Reading new credentials from " + credentialsStatus.getPath)
             val newCredentials = getCredentialsFromHDFSFile(remoteFs, credentialsStatus.getPath)
             lastCredentialsFileSuffix = suffix
+            newCredentials.getAllTokens.asScala
+              .map(_.decodeIdentifier())
+              .filter(_.isInstanceOf[DelegationTokenIdentifier])
+              .map(_.toString)
+              .foreach(logInfo(_))
             UserGroupInformation.getCurrentUser.addCredentials(newCredentials)
+            val tokens = UserGroupInformation.getCurrentUser.getCredentials.getAllTokens
+            tokens.asScala
+              .map(_.decodeIdentifier())
+              .filter(_.isInstanceOf[DelegationTokenIdentifier])
+              .map(_.toString)
+              .foreach(logInfo(_))
             logInfo("Credentials updated from credentials file.")
 
             val remainingTime = (getTimeOfNextUpdateFromFileName(credentialsStatus.getPath)

diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

diff --git a/sql/core/pom.xml b/sql/core/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.11</artifactId>
-    <version>2.2.1-kylin-r5</version>
+    <version>2.2.1-kylin-r6</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>