[Improve][Connector-v2] update the docs of hive on oss in aliyun

apache · Apr 17, 2024 · 2b0a92a · 2b0a92a
1 parent 422d25a
commit 2b0a92a
Show file tree

Hide file tree

Showing 5 changed files with 94 additions and 2 deletions.
diff --git a/docs/en/connector-v2/sink/Hive.md b/docs/en/connector-v2/sink/Hive.md
@@ -263,6 +263,86 @@ sink {
 }
 ```
 
+## Hive on oss
+
+### Step 1
+
+Create the lib dir for hive of emr.
+
+```shell
+mkdir -p ${SEATUNNEL_HOME}/plugins/Hive/lib
+```
+
+### Step 2
+
+Get the jars from maven center to the lib.
+
+```shell
+cd ${SEATUNNEL_HOME}/plugins/Hive/lib
+wget https://repo1.maven.org/maven2/org/apache/hive/hive-exec/2.3.9/hive-exec-2.3.9.jar
+```
+
+### Step 3
+
+Copy the jars from your environment on emr to the lib dir and delete the conflicting jar.
+
+```shell
+cp -r /opt/apps/JINDOSDK/jindosdk-current/lib/jindo-*.jar ${SEATUNNEL_HOME}/plugins/Hive/lib
+rm -f ${SEATUNNEL_HOME}/lib/hadoop-aliyun-*.jar
+```
+
+### Step 4
+
+Run the case.
+
+```shell
+env {
+  parallelism = 1
+  job.mode = "BATCH"
+}
+
+source {
+  FakeSource {
+    schema = {
+      fields {
+        pk_id = bigint
+        name = string
+        score = int
+      }
+      primaryKey {
+        name = "pk_id"
+        columnNames = [pk_id]
+      }
+    }
+    rows = [
+      {
+        kind = INSERT
+        fields = [1, "A", 100]
+      },
+      {
+        kind = INSERT
+        fields = [2, "B", 100]
+      },
+      {
+        kind = INSERT
+        fields = [3, "C", 100]
+      }
+    ]
+  }
+}
+
+sink {
+  Hive {
+    table_name = "test_hive.test_hive_sink_on_oss"
+    metastore_uri = "thrift://master-1-1.c-1009b01725b501f2.cn-wulanchabu.emr.aliyuncs.com:9083"
+    hive.hadoop.conf-path = "/tmp/hadoop"
+    hive.hadoop.conf = {
+        bucket="oss://emr-osshdfs.cn-wulanchabu.oss-dls.aliyuncs.com"
+    }
+  }
+}
+```
+
 ## Changelog
 
 ### 2.2.0-beta 2022-09-26

diff --git a/...seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_on_cos.conf b/...seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_on_cos.conf
@@ -55,5 +55,8 @@ sink {
     table_name = "test_hive.test_hive_sink_on_cos"
     metastore_uri = "thrift://hadoop04:9083"
     hive.hadoop.conf-path = "/tmp/hadoop"
+    hive.hadoop.conf = {
+        bucket="cosn://emr-cosn.com"
+    }
   }
 }
diff --git a/...seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_on_oss.conf b/...seatunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/fake_to_hive_on_oss.conf
@@ -53,7 +53,10 @@ source {
 sink {
   Hive {
     table_name = "test_hive.test_hive_sink_on_oss"
-    metastore_uri = "thrift://hadoop04:9083"
+    metastore_uri = "thrift://master-1-1.c-1009b01725b501f2.cn-wulanchabu.emr.aliyuncs.com:9083"
     hive.hadoop.conf-path = "/tmp/hadoop"
+    hive.hadoop.conf = {
+        bucket="oss://emr-osshdfs.cn-wulanchabu.oss-dls.aliyuncs.com"
+    }
   }
 }
diff --git a/...atunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/hive_on_cos_to_assert.conf b/...atunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/hive_on_cos_to_assert.conf
@@ -25,6 +25,9 @@ source {
     table_name = "test_hive.test_hive_sink_on_cos"
     metastore_uri = "thrift://hadoop04:9083"
     hive.hadoop.conf-path = "/tmp/hadoop"
+    hive.hadoop.conf = {
+        bucket="cosn://emr-cosn.com"
+    }
     result_table_name = hive_source
   }
 }

diff --git a/...atunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/hive_on_oss_to_assert.conf b/...atunnel-connector-v2-e2e/connector-hive-e2e/src/test/resources/hive_on_oss_to_assert.conf
@@ -23,8 +23,11 @@ env {
 source {
   Hive {
     table_name = "test_hive.test_hive_sink_on_oss"
-    metastore_uri = "thrift://hadoop04:9083"
+    metastore_uri = "thrift://master-1-1.c-1009b01725b501f2.cn-wulanchabu.emr.aliyuncs.com:9083"
     hive.hadoop.conf-path = "/tmp/hadoop"
+    hive.hadoop.conf = {
+        bucket="oss://emr-osshdfs.cn-wulanchabu.oss-dls.aliyuncs.com"
+    }
     result_table_name = hive_source
   }
 }