EC Yarn app id logs should be printed to a separate log close #2628

apache · Aug 8, 2022 · a7bfca4 · a7bfca4
1 parent bf68ef4
commit a7bfca4
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 38 deletions.
diff --git a/...is-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/utils/MessageUtils.java b/...is-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/message/utils/MessageUtils.java
@@ -44,8 +44,7 @@ public static <T> T getBean(Class<T> tClass) {
             try {
                 t = applicationContext.getBean(tClass);
             } catch (NoSuchBeanDefinitionException e) {
-                LOGGER.warn(
-                        String.format("can not get bean from spring ioc:%s", tClass.getName()));
+                LOGGER.warn(String.format("can not get bean from spring ioc:%s", tClass.getName()));
             }
         }
         return t;

diff --git a/...rc/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/...rc/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java
@@ -94,46 +94,50 @@ public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest
         return response;
     }
 
-    private synchronized void killYarnAppIdOfOneEc(EngineConn engineConn) {
+    private  void killYarnAppIdOfOneEc(EngineConn engineConn) {
         String engineConnInstance = engineConn.getServiceInstance().toString();
         logger.info("try to kill yarn app ids in the engine of ({}).", engineConnInstance);
         String engineLogDir = engineConn.getEngineConnManagerEnv().engineConnLogDirs();
-        final String errEngineLogPath = engineLogDir.concat(File.separator).concat("stderr");
+        final String errEngineLogPath = engineLogDir.concat(File.separator).concat("yarnApp.log");
         logger.info("try to parse the yarn app id from the engine err log file path: {}", errEngineLogPath);
-        ecYarnAppKillService.execute(() -> {
-            BufferedReader in = null;
-            try {
-                in = new BufferedReader(new FileReader(errEngineLogPath));
-                String line;
-                String regex = getYarnAppRegexByEngineType(engineConn);
-                if (StringUtils.isBlank(regex)) {
-                    return;
-                }
-                Pattern pattern = Pattern.compile(regex);
-                List<String> appIds = new ArrayList<>();
-                while ((line = in.readLine()) != null) {
-                    if (StringUtils.isNotBlank(line)) {
-                        Matcher mApp = pattern.matcher(line);
-                        if (mApp.find()) {
-                            String candidate1 = mApp.group(mApp.groupCount());
-                            if (!appIds.contains(candidate1)) {
-                                appIds.add(candidate1);
+        File file = new File(errEngineLogPath);
+        if (file.exists())
+        {
+            ecYarnAppKillService.execute(() -> {
+                BufferedReader in = null;
+                try {
+                    in = new BufferedReader(new FileReader(errEngineLogPath));
+                    String line;
+                    String regex = getYarnAppRegexByEngineType(engineConn);
+                    if (StringUtils.isBlank(regex)) {
+                        return;
+                    }
+                    Pattern pattern = Pattern.compile(regex);
+                    List<String> appIds = new ArrayList<>();
+                    while ((line = in.readLine()) != null) {
+                        if (StringUtils.isNotBlank(line)) {
+                            Matcher mApp = pattern.matcher(line);
+                            if (mApp.find()) {
+                                String candidate1 = mApp.group(mApp.groupCount());
+                                if (!appIds.contains(candidate1)) {
+                                    appIds.add(candidate1);
+                                }
                             }
                         }
                     }
+                    GovernanceUtils.killYarnJobApp(appIds);
+                    logger.info("finished kill yarn app ids in the engine of ({}).", engineConnInstance);
+                } catch (IOException ioEx) {
+                    if (ioEx instanceof FileNotFoundException) {
+                        logger.error("the engine log file {} not found.", errEngineLogPath);
+                    } else {
+                        logger.error("the engine log file parse failed. the reason is {}", ioEx.getMessage());
+                    }
+                } finally {
+                    IOUtils.closeQuietly(in);
                 }
-                GovernanceUtils.killYarnJobApp(appIds);
-                logger.info("finished kill yarn app ids in the engine of ({})." , engineConnInstance);
-            } catch (IOException ioEx) {
-                if (ioEx instanceof FileNotFoundException) {
-                    logger.error("the engine log file {} not found.", errEngineLogPath);
-                } else {
-                    logger.error("the engine log file parse failed. the reason is {}", ioEx.getMessage());
-                }
-            } finally {
-                IOUtils.closeQuietly(in);
-            }
-        });
+            });
+    }
     }
 
     private String getYarnAppRegexByEngineType(EngineConn engineConn) {

diff --git a/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/hive/src/main/resources/log4j2.xml
@@ -27,7 +27,7 @@
             <DefaultRolloverStrategy max="10"/>
         </RollingFile>
 
-        <RollingFile name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/stderr">
+        <RollingFile name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/yarnApp.log">
             <RegexFilter regex=".* application .*" onMatch="ACCEPT" onMismatch="DENY"/>
             <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
         </RollingFile>

diff --git a/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml b/linkis-engineconn-plugins/shell/src/main/resources/conf/log4j2.xml
@@ -27,7 +27,7 @@
             <DefaultRolloverStrategy max="10"/>
         </RollingFile>
 
-        <File name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/stderr">
+        <File name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/yarnApp.log">
             <RegexFilter regex=".* application .*" onMatch="ACCEPT" onMismatch="DENY"/>
             <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
         </File>

diff --git a/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/spark/src/main/resources/log4j2.xml
@@ -27,7 +27,7 @@
             <DefaultRolloverStrategy max="10"/>
         </RollingFile>
 
-        <File name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/stderr">
+        <File name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/yarnApp.log">
             <RegexFilter regex=".* application .*" onMatch="ACCEPT" onMismatch="DENY"/>
             <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
         </File>

diff --git a/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml b/linkis-engineconn-plugins/sqoop/src/main/resources/log4j2.xml
@@ -34,7 +34,7 @@
             <PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %class{36} %L %M - %msg%xEx%n"/>
         </File>
 
-        <File name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/stderr">
+        <File name="YarnAppIdOutputFile" append="true" fileName="${env:LOG_DIRS}/yarnApp.log">
             <RegexFilter regex=".* application .*" onMatch="ACCEPT" onMismatch="DENY"/>
             <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
         </File>