From 8637ff7f408ea222f2bbc674c5992203b475bb27 Mon Sep 17 00:00:00 2001 From: tobe Date: Fri, 26 Apr 2024 19:46:33 +0800 Subject: [PATCH] feat: execlude zookeeper for curator (#3899) * Execlude zookeeper when using curator * Fix local build java --- java/openmldb-batch/pom.xml | 12 +++++++++- java/openmldb-common/pom.xml | 6 +++++ java/openmldb-taskmanager/pom.xml | 9 +++++--- .../taskmanager/server/JobResultSaver.java | 22 +++++++++---------- .../taskmanager/zk/RecoverableZooKeeper.java | 2 +- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/java/openmldb-batch/pom.xml b/java/openmldb-batch/pom.xml index 09ca222034c..9f38b8d86d8 100644 --- a/java/openmldb-batch/pom.xml +++ b/java/openmldb-batch/pom.xml @@ -167,7 +167,11 @@ - + + org.apache.zookeeper + zookeeper + 3.4.14 + org.apache.curator curator-framework @@ -182,6 +186,12 @@ org.apache.curator curator-recipes 4.2.0 + + + org.apache.zookeeper + zookeeper + + diff --git a/java/openmldb-common/pom.xml b/java/openmldb-common/pom.xml index c4ad2e2e6eb..da240a18783 100644 --- a/java/openmldb-common/pom.xml +++ b/java/openmldb-common/pom.xml @@ -40,6 +40,12 @@ org.apache.curator curator-recipes 4.2.0 + + + org.apache.zookeeper + zookeeper + + org.testng diff --git a/java/openmldb-taskmanager/pom.xml b/java/openmldb-taskmanager/pom.xml index 309b51224a4..59573b4320b 100644 --- a/java/openmldb-taskmanager/pom.xml +++ b/java/openmldb-taskmanager/pom.xml @@ -134,6 +134,12 @@ org.apache.curator curator-recipes 4.2.0 + + + org.apache.zookeeper + zookeeper + + org.projectlombok @@ -142,9 +148,6 @@ provided - - - io.fabric8 diff --git a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java index 570bc035603..0e9825d0423 100644 --- a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java +++ b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/server/JobResultSaver.java @@ -53,7 +53,7 @@ */ @Slf4j public class JobResultSaver { - private static final Log log = LogFactory.getLog(JobResultSaver.class); + private static final Log logger = LogFactory.getLog(JobResultSaver.class); // false: unused, true: using // 0: unused, 1: saving, 2: finished but still in use @@ -92,8 +92,8 @@ public String genUniqueFileName() { public boolean saveFile(int resultId, String jsonData) { // No need to wait, cuz id status must have been changed by genResultId before. // It's a check. - if (log.isDebugEnabled()) { - log.debug("save result " + resultId + ", data " + jsonData); + if (logger.isDebugEnabled()) { + logger.debug("save result " + resultId + ", data " + jsonData); } int status = idStatus.get(resultId); if (status != 1) { @@ -105,7 +105,7 @@ public boolean saveFile(int resultId, String jsonData) { idStatus.set(resultId, 2); idStatus.notifyAll(); } - log.info("saved all result of result " + resultId); + logger.info("saved all result of result " + resultId); return true; } // save to /tmp_result// @@ -114,7 +114,7 @@ public boolean saveFile(int resultId, String jsonData) { File saveP = new File(savePath); if (!saveP.exists()) { boolean res = saveP.mkdirs(); - log.info("create save path " + savePath + ", status " + res); + logger.info("create save path " + savePath + ", status " + res); } } String fileFullPath = String.format("%s/%s", savePath, genUniqueFileName()); @@ -125,7 +125,7 @@ public boolean saveFile(int resultId, String jsonData) { + fileFullPath); } } catch (IOException e) { - log.error("create file failed, path " + fileFullPath, e); + logger.error("create file failed, path " + fileFullPath, e); return false; } @@ -135,7 +135,7 @@ public boolean saveFile(int resultId, String jsonData) { } catch (IOException e) { // Write failed, we'll lost a part of result, but it's ok for show sync job // output. So we just log it, and response the http request. - log.error("write result to file failed, path " + fileFullPath, e); + logger.error("write result to file failed, path " + fileFullPath, e); return false; } return true; @@ -151,7 +151,7 @@ public String readResult(int resultId, long timeoutMs) throws InterruptedExcepti } } if (idStatus.get(resultId) != 2) { - log.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId); + logger.warn("read result timeout, result saving may be still running, try read anyway, id " + resultId); } String output = ""; // all finished, read csv from savePath @@ -163,7 +163,7 @@ public String readResult(int resultId, long timeoutMs) throws InterruptedExcepti output = printFilesTostr(savePath); FileUtils.forceDelete(saveP); } else { - log.info("empty result for " + resultId + ", show empty string"); + logger.info("empty result for " + resultId + ", show empty string"); } // reset id synchronized (idStatus) { @@ -189,7 +189,7 @@ public String printFilesTostr(String fileDir) { } return stringWriter.toString(); } catch (Exception e) { - log.warn("read result met exception when read " + fileDir + ", " + e.getMessage()); + logger.warn("read result met exception when read " + fileDir + ", " + e.getMessage()); e.printStackTrace(); return "read met exception, check the taskmanager log"; } @@ -219,7 +219,7 @@ private void printFile(String file, StringWriter stringWriter, boolean printHead csvPrinter.printRecord(iter.next()); } } catch (Exception e) { - log.warn("error when print result file " + file + ", ignore it"); + logger.warn("error when print result file " + file + ", ignore it"); e.printStackTrace(); } } diff --git a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java index 9ff2b9349b4..10bc226ef50 100644 --- a/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java +++ b/java/openmldb-taskmanager/src/main/java/com/_4paradigm/openmldb/taskmanager/zk/RecoverableZooKeeper.java @@ -62,7 +62,7 @@ public class RecoverableZooKeeper { private final String quorumServers; private final int maxMultiSize; // unused now - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.") + //@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE", justification = "None. Its always been this way.") public RecoverableZooKeeper(String quorumServers, int sessionTimeout, Watcher watcher) throws IOException { // TODO: Add support for zk 'chroot'; we don't add it to the quorumServers // String as we should.