feat: merge 090 features to main (#3929)

* Set s3 and aws dependencies ad provided (#3897) * feat: execlude zookeeper for curator (#3899) * Execlude zookeeper when using curator * Fix local build java
4paradigm · May 16, 2024 · ba817e4 · ba817e4
1 parent 63d3a17
commit ba817e4
Show file tree

Hide file tree

Showing 5 changed files with 36 additions and 16 deletions.
diff --git a/java/openmldb-batch/pom.xml b/java/openmldb-batch/pom.xml
@@ -167,7 +167,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
+        <dependency>
+            <groupId>org.apache.zookeeper</groupId>
+            <artifactId>zookeeper</artifactId>
+            <version>3.4.14</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-framework</artifactId>
@@ -182,6 +186,12 @@
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
             <version>4.2.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <!-- OpenMLDB -->
@@ -241,6 +251,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-aws</artifactId>
             <version>${hadoop.version}</version>
+            <scope>provided</scope>
         </dependency>
     </dependencies>
 

diff --git a/java/openmldb-common/pom.xml b/java/openmldb-common/pom.xml
@@ -40,6 +40,12 @@
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
             <version>4.2.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.testng</groupId>

diff --git a/java/openmldb-taskmanager/pom.xml b/java/openmldb-taskmanager/pom.xml
@@ -134,6 +134,12 @@
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
             <version>4.2.0</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>
@@ -142,9 +148,6 @@
             <scope>provided</scope>
         </dependency>
 
-
-
-
         <!-- Kubernetes -->
         <dependency>
             <groupId>io.fabric8</groupId>

diff --git a/...-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java b/...-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java
@@ -53,7 +53,7 @@
  */
 @Slf4j
 public class JobResultSaver {
-    private static final Log log = LogFactory.getLog(JobResultSaver.class);
+    private static final Log logger = LogFactory.getLog(JobResultSaver.class);
 
     // false: unused, true: using
     // 0: unused, 1: saving, 2: finished but still in use
@@ -92,8 +92,8 @@ public String genUniqueFileName() {
     public boolean saveFile(int resultId, String jsonData) {
         // No need to wait, cuz id status must have been changed by genResultId before.
         // It's a check.
-        if (log.isDebugEnabled()) {
-            log.debug("save result " + resultId + ", data " + jsonData);
+        if (logger.isDebugEnabled()) {
+            logger.debug("save result " + resultId + ", data " + jsonData);
         }
         int status = idStatus.get(resultId);
         if (status != 1) {
@@ -105,7 +105,7 @@ public boolean saveFile(int resultId, String jsonData) {
                 idStatus.set(resultId, 2);
                 idStatus.notifyAll();
             }
-            log.info("saved all result of result " + resultId);
+            logger.info("saved all result of result " + resultId);
             return true;
         }
         // save to <log path>/tmp_result/<result_id>/<unique file name>
@@ -114,7 +114,7 @@ public boolean saveFile(int resultId, String jsonData) {
             File saveP = new File(savePath);
             if (!saveP.exists()) {
                 boolean res = saveP.mkdirs();
-                log.info("create save path " + savePath + ", status " + res);
+                logger.info("create save path " + savePath + ", status " + res);
             }
         }
         String fileFullPath = String.format("%s/%s", savePath, genUniqueFileName());
@@ -125,7 +125,7 @@ public boolean saveFile(int resultId, String jsonData) {
                         + fileFullPath);
             }
         } catch (IOException e) {
-            log.error("create file failed, path " + fileFullPath, e);
+            logger.error("create file failed, path " + fileFullPath, e);
             return false;
         }
 
@@ -135,7 +135,7 @@ public boolean saveFile(int resultId, String jsonData) {
         } catch (IOException e) {
             // Write failed, we'll lost a part of result, but it's ok for show sync job
             // output. So we just log it, and response the http request.
-            log.error("write result to file failed, path " + fileFullPath, e);
+            logger.error("write result to file failed, path " + fileFullPath, e);
             return false;
         }
         return true;
@@ -151,7 +151,7 @@ public String readResult(int resultId, long timeoutMs) throws InterruptedExcepti
             }
         }
         if (idStatus.get(resultId) != 2) {
-            log.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId);
+            logger.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId);
         }
         String output = "";
         // all finished, read csv from savePath
@@ -163,7 +163,7 @@ public String readResult(int resultId, long timeoutMs) throws InterruptedExcepti
             output = printFilesTostr(savePath);
             FileUtils.forceDelete(saveP);
         } else {
-            log.info("empty result for " + resultId + ", show empty string");
+            logger.info("empty result for " + resultId + ", show empty string");
         }
         // reset id
         synchronized (idStatus) {
@@ -189,7 +189,7 @@ public String printFilesTostr(String fileDir) {
             }
             return stringWriter.toString();
         } catch (Exception e) {
-            log.warn("read result met exception when read " + fileDir + ", " + e.getMessage());
+            logger.warn("read result met exception when read " + fileDir + ", " + e.getMessage());
             e.printStackTrace();
             return "read met exception, check the taskmanager log";
         }
@@ -219,7 +219,7 @@ private void printFile(String file, StringWriter stringWriter, boolean printHead
                 csvPrinter.printRecord(iter.next());
             }
         } catch (Exception e) {
-            log.warn("error when print result file " + file + ", ignore it");
+            logger.warn("error when print result file " + file + ", ignore it");
             e.printStackTrace();
         }
     }

diff --git a/...askmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java b/...askmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java
@@ -62,7 +62,7 @@ public class RecoverableZooKeeper {
     private final String quorumServers;
     private final int maxMultiSize; // unused now
 
-    @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.")
+    //@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.")
     public RecoverableZooKeeper(String quorumServers, int sessionTimeout, Watcher watcher) throws IOException {
         // TODO: Add support for zk 'chroot'; we don't add it to the quorumServers
         // String as we should.