[NSE-276] Add option to switch Hadoop version

zhztheplayer · Apr 26, 2021 · f68d834 · f68d834
1 parent ef97b98
commit f68d834
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 50 deletions.
diff --git a/arrow-data-source/pom.xml b/arrow-data-source/pom.xml
@@ -3,7 +3,7 @@
     <groupId>com.intel.oap</groupId>
     <artifactId>native-sql-engine-parent</artifactId>
     <version>1.1.0</version>
-  </parent>  
+  </parent>
 
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.intel.oap</groupId>
@@ -18,12 +18,6 @@
     <module>parquet</module>
   </modules>
   <properties>
-    <scala.version>2.12.10</scala.version>
-    <scala.binary.version>2.12</scala.binary.version>
-    <spark.version>3.0.0</spark.version>
-    <arrow.version>3.0.0</arrow.version>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <script.dir>${arrow.script.dir}</script.dir>
     <datasource.cpp_tests>${cpp_tests}</datasource.cpp_tests>
     <datasource.build_arrow>${build_arrow}</datasource.build_arrow>
@@ -49,44 +43,44 @@
 
   <dependencies>
     <dependency>
-    <groupId>org.apache.hadoop</groupId>
-    <artifactId>hadoop-aws</artifactId>
-    <version>2.7.3</version>
-    <exclusions>
-      <exclusion>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-core</artifactId>
-      </exclusion>
-      <exclusion>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-annotations</artifactId>
-      </exclusion>
-      <exclusion>
-        <groupId>com.fasterxml.jackson.core</groupId>
-        <artifactId>jackson-databind</artifactId>
-      </exclusion>
-      <exclusion>
-        <groupId>javax.servlet</groupId>
-        <artifactId>servlet-api</artifactId>
-      </exclusion>
-      <exclusion>
-        <groupId>com.sun.jersey</groupId>
-        <artifactId>jersey-core</artifactId>
-      </exclusion>
-      <exclusion>
-        <groupId>com.sun.jersey</groupId>
-        <artifactId>jersey-json</artifactId>
-      </exclusion>
-      <exclusion>
-        <groupId>com.sun.jersey</groupId>
-        <artifactId>jersey-server</artifactId>
-      </exclusion>
-      <exclusion>
-        <groupId>commons-httpclient</groupId>
-        <artifactId>commons-httpcore</artifactId>
-      </exclusion>
-    </exclusions>
-  </dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-aws</artifactId>
+      <version>${hadoop.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-annotations</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.fasterxml.jackson.core</groupId>
+          <artifactId>jackson-databind</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.servlet</groupId>
+          <artifactId>servlet-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-json</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-server</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-httpclient</groupId>
+          <artifactId>commons-httpcore</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.apache.httpcomponents</groupId>
       <artifactId>httpcore</artifactId>
@@ -174,7 +168,7 @@
             <configuration>
               <executable>bash</executable>
               <arguments>
-              <argument>${script.dir}/build_arrow.sh</argument>
+                <argument>${script.dir}/build_arrow.sh</argument>
                 <argument>--tests=${datasource.cpp_tests}</argument>
                 <argument>--build_arrow=${datasource.build_arrow}</argument>
                 <argument>--static_arrow=${datasource.static_arrow}</argument>

diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml
@@ -30,10 +30,6 @@
   <properties>
     <cpp.dir>../cpp/</cpp.dir>
     <cpp.build.dir>../cpp/build/releases/</cpp.build.dir>
-    <arrow.version>3.0.0</arrow.version>
-    <spark.version>3.0.0</spark.version>
-    <scala.binary.version>2.12</scala.binary.version>
-    <scala.version>2.12.10</scala.version>
     <build.testJarPhase>none</build.testJarPhase>
     <build.copyDependenciesPhase>package</build.copyDependenciesPhase>
     <hadoop.deps.scope>provided</hadoop.deps.scope>

diff --git a/pom.xml b/pom.xml
@@ -30,6 +30,13 @@
   </licenses>
 
   <properties>
+    <scala.version>2.12.10</scala.version>
+    <scala.binary.version>2.12</scala.binary.version>
+    <spark.version>3.0.0</spark.version>
+    <arrow.version>3.0.0</arrow.version>
+    <hadoop.version>2.7.4</hadoop.version>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <arrow.script.dir>${project.basedir}/script</arrow.script.dir>
     <cpp_tests>OFF</cpp_tests>
     <build_arrow>ON</build_arrow>
@@ -44,4 +51,12 @@
     <module>native-sql-engine/core</module>
   </modules>
 
+  <profiles>
+    <profile>
+      <id>hadoop-3.2</id>
+      <properties>
+        <hadoop.version>3.2.0</hadoop.version>
+      </properties>
+    </profile>
+  </profiles>
 </project>