[WIP][test-hadoop3.2] Test Hadoop 3.2 on jenkins

.gitignore

@@ -83,6 +83,7 @@ TempStatsStore/
 metastore/
 metastore_db/
 sql/hive-thriftserver/test_warehouses
+sql/hive-thriftserver2/test_warehouses
 warehouse/
 spark-warehouse/
 

assembly/pom.xml

@@ -178,6 +178,16 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>hive-thriftserver2</id>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.spark</groupId>
+          <artifactId>spark-hive-thriftserver2_${scala.binary.version}</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+      </dependencies>
+    </profile>
     <profile>
       <id>spark-ganglia-lgpl</id>
       <dependencies>

dev/deps/spark-deps-hadoop-3.1

@@ -15,6 +15,7 @@ arpack_combined_all-0.1.jar
 arrow-format-0.12.0.jar
 arrow-memory-0.12.0.jar
 arrow-vector-0.12.0.jar
+audience-annotations-0.5.0.jar
 automaton-1.11-8.jar
 avro-1.8.2.jar
 avro-ipc-1.8.2.jar
@@ -29,7 +30,7 @@ commons-cli-1.2.jar
 commons-codec-1.10.jar
 commons-collections-3.2.2.jar
 commons-compiler-3.0.11.jar
-commons-compress-1.8.1.jar
+commons-compress-1.9.jar
 commons-configuration2-2.1.1.jar
 commons-crypto-1.0.0.jar
 commons-daemon-1.0.13.jar
@@ -42,13 +43,14 @@ commons-logging-1.1.3.jar
 commons-math3-3.4.1.jar
 commons-net-3.1.jar
 commons-pool-1.5.4.jar
+commons-text-1.6.jar
 compress-lzf-1.0.3.jar
 core-1.1.2.jar
-curator-client-2.12.0.jar
-curator-framework-2.12.0.jar
-curator-recipes-2.12.0.jar
+curator-client-2.13.0.jar
+curator-framework-2.13.0.jar
+curator-recipes-2.13.0.jar
 datanucleus-api-jdo-3.2.6.jar
-datanucleus-core-3.2.10.jar
+datanucleus-core-4.1.17.jar
 datanucleus-rdbms-3.2.9.jar
 derby-10.12.1.1.jar
 dnsjava-2.1.7.jar
@@ -60,20 +62,30 @@ gson-2.2.4.jar
 guava-14.0.1.jar
 guice-4.0.jar
 guice-servlet-4.0.jar
-hadoop-annotations-3.1.0.jar
-hadoop-auth-3.1.0.jar
-hadoop-client-3.1.0.jar
-hadoop-common-3.1.0.jar
-hadoop-hdfs-client-3.1.0.jar
-hadoop-mapreduce-client-common-3.1.0.jar
-hadoop-mapreduce-client-core-3.1.0.jar
-hadoop-mapreduce-client-jobclient-3.1.0.jar
-hadoop-yarn-api-3.1.0.jar
-hadoop-yarn-client-3.1.0.jar
-hadoop-yarn-common-3.1.0.jar
-hadoop-yarn-registry-3.1.0.jar
-hadoop-yarn-server-common-3.1.0.jar
-hadoop-yarn-server-web-proxy-3.1.0.jar
+hadoop-annotations-3.2.0.jar
+hadoop-auth-3.2.0.jar
+hadoop-client-3.2.0.jar
+hadoop-common-3.2.0.jar
+hadoop-hdfs-client-3.2.0.jar
+hadoop-mapreduce-client-common-3.2.0.jar
+hadoop-mapreduce-client-core-3.2.0.jar
+hadoop-mapreduce-client-jobclient-3.2.0.jar
+hadoop-yarn-api-3.2.0.jar
+hadoop-yarn-client-3.2.0.jar
+hadoop-yarn-common-3.2.0.jar
+hadoop-yarn-registry-3.2.0.jar
+hadoop-yarn-server-common-3.2.0.jar
+hadoop-yarn-server-web-proxy-3.2.0.jar
+hive-common-2.3.4.jar
+hive-llap-client-2.3.4.jar
+hive-llap-common-2.3.4.jar
+hive-serde-2.3.4.jar
+hive-service-rpc-2.3.4.jar
+hive-shims-0.23-2.3.4.jar
+hive-shims-2.3.4.jar
+hive-shims-common-2.3.4.jar
+hive-shims-scheduler-2.3.4.jar
+hive-storage-api-2.6.0.jar
 hk2-api-2.4.0-b34.jar
 hk2-locator-2.4.0-b34.jar
 hk2-utils-2.4.0-b34.jar
@@ -88,8 +100,8 @@ jackson-core-2.9.8.jar
 jackson-core-asl-1.9.13.jar
 jackson-databind-2.9.8.jar
 jackson-dataformat-yaml-2.9.8.jar
-jackson-jaxrs-base-2.7.8.jar
-jackson-jaxrs-json-provider-2.7.8.jar
+jackson-jaxrs-base-2.9.5.jar
+jackson-jaxrs-json-provider-2.9.5.jar
 jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations-2.9.8.jar
 jackson-module-paranamer-2.9.8.jar
@@ -116,12 +128,13 @@ jersey-container-servlet-core-2.22.2.jar
 jersey-guava-2.22.2.jar
 jersey-media-jaxb-2.22.2.jar
 jersey-server-2.22.2.jar
-jetty-webapp-9.4.12.v20180830.jar
-jetty-xml-9.4.12.v20180830.jar
+jetty-webapp-9.3.24.v20180605.jar
+jetty-xml-9.3.24.v20180605.jar
 jline-2.14.6.jar
 joda-time-2.9.3.jar
 jodd-core-3.5.2.jar
 jpam-1.1.jar
+json-1.8.jar
 json-smart-2.3.jar
 json4s-ast_2.12-3.5.3.jar
 json4s-core_2.12-3.5.3.jar
@@ -173,6 +186,7 @@ okhttp-3.8.1.jar
 okio-1.13.0.jar
 opencsv-2.3.jar
 orc-core-1.5.4-nohive.jar
+orc-core-1.5.4.jar
 orc-mapreduce-1.5.4-nohive.jar
 orc-shims-1.5.4.jar
 oro-2.0.8.jar
@@ -214,5 +228,5 @@ woodstox-core-5.0.3.jar
 xbean-asm7-shaded-4.12.jar
 xz-1.5.jar
 zjsonpatch-0.3.0.jar
-zookeeper-3.4.9.jar
+zookeeper-3.4.13.jar
 zstd-jni-1.3.2-2.jar

dev/lint-java

@@ -20,7 +20,7 @@
 SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)"
 
-ERRORS=$($SCRIPT_DIR/../build/mvn -Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive -Phive-thriftserver checkstyle:check | grep ERROR)
+ERRORS=$($SCRIPT_DIR/../build/mvn -Pkinesis-asl -Pmesos -Pkubernetes -Pyarn -Phive -Phive-thriftserver2 checkstyle:check | grep ERROR)
 
 if test ! -z "$ERRORS"; then
     echo -e "Checkstyle checks failed at following occurrences:\n$ERRORS"

dev/mima

@@ -24,7 +24,7 @@ set -e
 FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 cd "$FWDIR"
 
-SPARK_PROFILES="-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
+SPARK_PROFILES="-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver2 -Phive"
 TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)"
 OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)"
 

dev/run-tests-jenkins.py

@@ -173,6 +173,8 @@ def main():
         os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.6"
     if "test-hadoop2.7" in ghprb_pull_title:
         os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop2.7"
+    if "test-hadoop3.1" in ghprb_pull_title:
+        os.environ["AMPLAB_JENKINS_BUILD_PROFILE"] = "hadoop3.1"
 
     build_display_name = os.environ["BUILD_DISPLAY_NAME"]
     build_url = os.environ["BUILD_URL"]

dev/run-tests.py

@@ -112,7 +112,7 @@ def determine_modules_to_test(changed_modules):
     ['graphx', 'examples']
     >>> x = [x.name for x in determine_modules_to_test([modules.sql])]
     >>> x # doctest: +NORMALIZE_WHITESPACE
-    ['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver',
+    ['sql', 'avro', 'hive', 'mllib', 'sql-kafka-0-10', 'examples', 'hive-thriftserver2',
      'pyspark-sql', 'repl', 'sparkr', 'pyspark-mllib', 'pyspark-ml']
     """
     modules_to_test = set()
@@ -274,6 +274,7 @@ def get_hadoop_profiles(hadoop_version):
 
     sbt_maven_hadoop_profiles = {
         "hadoop2.7": ["-Phadoop-2.7"],
+        "hadoop3.1": ["-Phadoop-3.1"],
     }
 
     if hadoop_version in sbt_maven_hadoop_profiles:

dev/sbt-checkstyle

@@ -26,7 +26,8 @@ ERRORS=$(echo -e "q\n" \
         -Pkubernetes \
         -Pyarn \
         -Phive \
-        -Phive-thriftserver \
+        -Phadoop-3.1 \
+        -Phive-thriftserver2 \
         checkstyle test:checkstyle \
     | awk '{if($1~/error/)print}' \
 )

dev/scalastyle

@@ -26,7 +26,7 @@ ERRORS=$(echo -e "q\n" \
         -Pkubernetes \
         -Pyarn \
         -Phive \
-        -Phive-thriftserver \
+        -Phive-thriftserver2 \
         -Pspark-ganglia-lgpl \
         -Pdocker-integration-tests \
 	-Pkubernetes-integration-tests \

dev/sparktestsupport/modules.py

@@ -154,19 +154,18 @@ def __hash__(self):
     ],
 )
 
-
-hive_thriftserver = Module(
-    name="hive-thriftserver",
+hive_thriftserver2 = Module(
+    name="hive-thriftserver2",
     dependencies=[hive],
     source_file_regexes=[
-        "sql/hive-thriftserver",
+        "sql/hive-thriftserver2",
         "sbin/start-thriftserver.sh",
     ],
     build_profile_flags=[
-        "-Phive-thriftserver",
+        "-Phive-thriftserver2",
     ],
     sbt_test_goals=[
-        "hive-thriftserver/test",
+        "hive-thriftserver2/test",
     ]
 )
 

dev/test-dependencies.sh

@@ -29,7 +29,7 @@ export LC_ALL=C
 # TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
 
 # NOTE: These should match those in the release publishing script
-HADOOP2_MODULE_PROFILES="-Phive-thriftserver -Pmesos -Pkubernetes -Pyarn -Phive"
+HADOOP2_MODULE_PROFILES="-Pmesos -Pkubernetes -Pyarn -Phive"
 MVN="build/mvn"
 HADOOP_PROFILES=(
     hadoop-2.7
@@ -67,15 +67,22 @@ $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /de
 
 # Generate manifests for each Hadoop profile:
 for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do
+  HIVE_THRIFTSERVER_MODULE=none
+  if [ "${HADOOP_PROFILE}" = 'hadoop-2.7' ]; then
+    HIVE_THRIFTSERVER_MODULE="-Phive-thriftserver"
+  else
+    HIVE_THRIFTSERVER_MODULE="-Phive-thriftserver2"
+  fi
+
   echo "Performing Maven install for $HADOOP_PROFILE"
-  $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install clean -q
+  $MVN $HADOOP2_MODULE_PROFILES $HIVE_THRIFTSERVER_MODULE -P$HADOOP_PROFILE jar:jar jar:test-jar install:install clean -q
 
   echo "Performing Maven validate for $HADOOP_PROFILE"
-  $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE validate -q
+  $MVN $HADOOP2_MODULE_PROFILES $HIVE_THRIFTSERVER_MODULE -P$HADOOP_PROFILE validate -q
 
   echo "Generating dependency manifest for $HADOOP_PROFILE"
   mkdir -p dev/pr-deps
-  $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly \
+  $MVN $HADOOP2_MODULE_PROFILES $HIVE_THRIFTSERVER_MODULE -P$HADOOP_PROFILE dependency:build-classpath -pl assembly \
     | grep "Dependencies classpath:" -A 1 \
     | tail -n 1 | tr ":" "\n" | rev | cut -d "/" -f 1 | rev | sort \
     | grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE

launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java

@@ -167,6 +167,7 @@ List<String> buildClassPath(String appClassPath) throws IOException {
         "sql/core",
         "sql/hive",
         "sql/hive-thriftserver",
+        "sql/hive-thriftserver2",
         "streaming"
       );
       if (prependClasses) {