[Feature][Shade] Add seatunnel hadoop3 uber

config/hazelcast-client.yaml

@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast-client:
+  cluster-name: seatunnel
+  properties:
+    hazelcast.logging.type: log4j2
+  network:
+    cluster-members:
+      - localhost:5801

config/hazelcast.yaml

@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast:
+  cluster-name: seatunnel
+  network:
+    rest-api:
+      enabled: true
+      endpoint-groups:
+        CLUSTER_WRITE:
+          enabled: true
+    join:
+      tcp-ip:
+        enabled: true
+        member-list:
+          - localhost
+    port:
+      auto-increment: false
+      port: 5801
+  properties:
+    hazelcast.invocation.max.retry.count: 20
+    hazelcast.tcp.join.port.try.count: 30
+    hazelcast.logging.type: log4j2
+

config/log4j2.properties

@@ -25,11 +25,11 @@ property.file_ttl = 7d
 rootLogger.level = INFO
 
 ############################ log output to console #############################
-rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
-rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
+#rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
+#rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
 ############################ log output to console #############################
 ############################ log output to file    #############################
-#rootLogger.appenderRef.file.ref = fileAppender
+rootLogger.appenderRef.file.ref = fileAppender
 ############################ log output to file    #############################
 
 appender.consoleStdout.name = consoleStdoutAppender
@@ -75,4 +75,4 @@ appender.file.strategy.action.condition.nested_condition.type = IfAny
 appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
 appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
 appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
-appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

config/log4j2_client.properties

@@ -0,0 +1,78 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+property.file_path = ${sys:seatunnel.logs.path:-/tmp/seatunnel/logs}
+property.file_name = ${sys:seatunnel.logs.file_name:-seatunnel}
+property.file_split_size = 100MB
+property.file_count = 100
+property.file_ttl = 7d
+
+rootLogger.level = INFO
+
+############################ log output to console #############################
+rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
+rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
+############################ log output to console #############################
+############################ log output to file    #############################
+#rootLogger.appenderRef.file.ref = fileAppender
+############################ log output to file    #############################
+
+appender.consoleStdout.name = consoleStdoutAppender
+appender.consoleStdout.type = CONSOLE
+appender.consoleStdout.target = SYSTEM_OUT
+appender.consoleStdout.layout.type = PatternLayout
+appender.consoleStdout.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
+appender.consoleStdout.filter.acceptLtWarn.type = ThresholdFilter
+appender.consoleStdout.filter.acceptLtWarn.level = WARN
+appender.consoleStdout.filter.acceptLtWarn.onMatch = DENY
+appender.consoleStdout.filter.acceptLtWarn.onMismatch = ACCEPT
+
+appender.consoleStderr.name = consoleStderrAppender
+appender.consoleStderr.type = CONSOLE
+appender.consoleStderr.target = SYSTEM_ERR
+appender.consoleStderr.layout.type = PatternLayout
+appender.consoleStderr.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
+appender.consoleStderr.filter.acceptGteWarn.type = ThresholdFilter
+appender.consoleStderr.filter.acceptGteWarn.level = WARN
+appender.consoleStderr.filter.acceptGteWarn.onMatch = ACCEPT
+appender.consoleStderr.filter.acceptGteWarn.onMismatch = DENY
+
+appender.file.name = fileAppender
+appender.file.type = RollingFile
+appender.file.fileName = ${file_path}/${file_name}.log
+appender.file.filePattern = ${file_path}/${file_name}.log.%d{yyyy-MM-dd}-%i
+appender.file.append = true
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c - %m%n
+appender.file.policies.type = Policies
+appender.file.policies.time.type = TimeBasedTriggeringPolicy
+appender.file.policies.time.modulate = true
+appender.file.policies.size.type = SizeBasedTriggeringPolicy
+appender.file.policies.size.size = ${file_split_size}
+appender.file.strategy.type = DefaultRolloverStrategy
+appender.file.strategy.fileIndex = nomax
+appender.file.strategy.action.type = Delete
+appender.file.strategy.action.basepath = ${file_path}
+appender.file.strategy.action.maxDepth = 1
+appender.file.strategy.action.condition.type = IfFileName
+appender.file.strategy.action.condition.glob = ${file_name}.log*
+appender.file.strategy.action.condition.nested_condition.type = IfAny
+appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
+appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
+appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

config/seatunnel.yaml

@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+seatunnel:
+  engine:
+    backup-count: 1
+    print-execution-info-interval: 10
+    slot-service:
+      dynamic-slot: true
+    checkpoint:
+      interval: 300000
+      timeout: 10000
+      max-concurrent: 5
+      tolerable-failure: 2
+      storage:
+        type: hdfs
+        max-retained: 3
+        namespace: /tmp/seatunnel/checkpoint_snapshot
+        plugin-config:
+          storage-type: hdfs
+          fs.defaultFS: /tmp/ # Ensure that the directory has written permission

config/v2.batch.config.template

@@ -22,6 +22,7 @@ env {
   # You can set SeaTunnel environment configuration here
   execution.parallelism = 2
   job.mode = "BATCH"
+  checkpoint.interval = 10000
   #execution.checkpoint.interval = 10000
   #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
 }
@@ -44,20 +45,10 @@ source {
   # please go to https://seatunnel.apache.org/docs/category/source-v2
 }
 
-transform {
-  sql {
-    sql = "select name,age from fake"
-  }
-
-  # If you would like to get more information about how to configure Seatunnel and see full list of transform plugins,
-  # please go to https://seatunnel.apache.org/docs/category/transform
-}
-
 sink {
   Console {
-    parallelism = 3
   }
 
   # If you would like to get more information about how to configure Seatunnel and see full list of sink plugins,
   # please go to https://seatunnel.apache.org/docs/category/sink-v2
-}
+}

config/v2.streaming.conf.template

@@ -22,6 +22,7 @@ env {
   # You can set flink configuration here
   execution.parallelism = 2
   job.mode = "STREAMING"
+  checkpoint.interval = 2000
   #execution.checkpoint.interval = 10000
   #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
 }
@@ -44,20 +45,10 @@ source {
   # please go to https://seatunnel.apache.org/docs/category/source-v2
 }
 
-transform {
-  sql {
-    sql = "select name,age from fake"
-  }
-
-  # If you would like to get more information about how to configure Seatunnel and see full list of transform plugins,
-  # please go to https://seatunnel.apache.org/docs/category/transform
-}
-
 sink {
   Console {
-    parallelism = 3
   }
 
   # If you would like to get more information about how to configure Seatunnel and see full list of sink plugins,
   # please go to https://seatunnel.apache.org/docs/category/sink-v2
-}
+}

docs/en/seatunnel-engine/checkpoint-storage.md

@@ -116,6 +116,6 @@ seatunnel:
         max-retained: 3
         plugin-config:
           storage-type: hdfs
-            fs.defaultFS: /tmp/ # Ensure that the directory has written permission 
+          fs.defaultFS: /tmp/ # Ensure that the directory has written permission 
 
 ```

pom.xml

@@ -90,6 +90,7 @@
         <module>seatunnel-engine</module>
         <module>seatunnel-examples</module>
         <module>seatunnel-e2e</module>
+        <module>seatunnel-shade</module>
     </modules>
 
     <profiles>

seatunnel-connectors-v2/connector-file/connector-file-base/pom.xml

@@ -34,16 +34,15 @@
         <orc.version>1.5.6</orc.version>
         <commons.collecton4.version>4.4</commons.collecton4.version>
         <commons.lang3.version>3.4</commons.lang3.version>
-        <flink.hadoop.version>2.7.5-7.0</flink.hadoop.version>
         <parquet-avro.version>1.12.3</parquet-avro.version>
     </properties>
 
     <dependencyManagement>
         <dependencies>
             <dependency>
-                <groupId>org.apache.flink</groupId>
-                <artifactId>flink-shaded-hadoop-2</artifactId>
-                <version>${flink.hadoop.version}</version>
+                <groupId>org.apache.seatunnel</groupId>
+                <artifactId>seatunnel-hadoop3-uber</artifactId>
+                <version>${version}</version>
                 <scope>provided</scope>
             </dependency>
         </dependencies>
@@ -117,8 +116,8 @@
         </dependency>
 
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2</artifactId>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-uber</artifactId>
             <scope>provided</scope>
             <exclusions>
                 <exclusion>

seatunnel-connectors-v2/connector-file/connector-file-ftp/pom.xml

@@ -36,8 +36,8 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2</artifactId>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-uber</artifactId>
             <scope>provided</scope>
             <exclusions>
                 <exclusion>

seatunnel-connectors-v2/connector-file/connector-file-hadoop/pom.xml

@@ -36,8 +36,8 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2</artifactId>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-uber</artifactId>
             <scope>provided</scope>
             <exclusions>
                 <exclusion>

seatunnel-connectors-v2/connector-file/connector-file-local/pom.xml

@@ -36,8 +36,8 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2</artifactId>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-uber</artifactId>
             <scope>provided</scope>
             <exclusions>
                 <exclusion>

seatunnel-connectors-v2/connector-file/connector-file-s3/pom.xml

@@ -42,8 +42,8 @@
         </dependency>
 
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2</artifactId>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-uber</artifactId>
             <scope>provided</scope>
             <exclusions>
                 <exclusion>

seatunnel-connectors-v2/connector-file/connector-file-sftp/pom.xml

@@ -36,8 +36,8 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2</artifactId>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-uber</artifactId>
             <scope>provided</scope>
         </dependency>
     </dependencies>

seatunnel-connectors-v2/connector-hive/pom.xml

@@ -41,14 +41,14 @@
             <version>${project.version}</version>
             <exclusions>
                 <exclusion>
-                    <groupId>org.apache.flink</groupId>
-                    <artifactId>flink-shaded-hadoop-2</artifactId>
+                    <groupId>org.apache.seatunnel</groupId>
+                    <artifactId>seatunnel-hadoop3-uber</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
         <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-shaded-hadoop-2</artifactId>
+            <groupId>org.apache.seatunnel</groupId>
+            <artifactId>seatunnel-hadoop3-uber</artifactId>
             <scope>provided</scope>
             <exclusions>
                 <exclusion>

seatunnel-core/seatunnel-flink-starter/pom.xml

@@ -139,6 +139,7 @@
                             <exclude>org.apache.logging.log4j:log4j-slf4j-impl</exclude>
                             <exclude>org.apache.logging.log4j:log4j-1.2-api</exclude>
                             <exclude>org.apache.logging.log4j:log4j-to-slf4j</exclude>
+                            <exclude>org.apache.seatunnel:seatunnel-hadoop3-uber</exclude>
                         </excludes>
                     </artifactSet>
                 </configuration>