From eeba2994011a0ba9be5bb536253bdf8e2515a9d8 Mon Sep 17 00:00:00 2001 From: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:59:59 +0800 Subject: [PATCH] Dev 1.9.0 bug fix (#629) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Code optimization * Code optimization * Code optimization * Code optimization --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> --- .../StarrocksTimeExceedAlterSender.scala | 6 +-- .../jobtime/StarrocksTimeExceedRule.scala | 4 +- .../exception/WorkspaceExceptionManager.java | 1 + .../filesystem/restful/api/FsRestfulApi.java | 17 +++--- .../filesystem/util/FilesystemUtils.java | 52 +++++++++---------- 5 files changed, 40 insertions(+), 40 deletions(-) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala index 9334cc88b6..5f1f42311c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala @@ -58,11 +58,11 @@ class StarrocksTimeExceedAlterSender extends Observer with Logging { } else { val jobHistory = a.asInstanceOf[JobHistory] val timeValue = - HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alter.time") + HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alert.time") val userValue = - HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alter.user") + HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alert.user") var levelValue = - HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alter.level") + HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alert.level") if (StringUtils.isNotBlank(timeValue) && StringUtils.isNotBlank(userValue)) { val replaceParm: util.HashMap[String, String] = new util.HashMap[String, String] replaceParm.put("$id", String.valueOf(jobHistory.getId)) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala index 6268a37768..4741727ed9 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala @@ -69,7 +69,7 @@ class StarrocksTimeExceedRule(hitObserver: Observer) MapUtils.getString(datasourceTypeMap, "name", "").toLowerCase() // 获取管理台配置需要使用的数据源类型（默认starrocks） var datasourceType = HttpsUntils - .getJDBCConf(job.getSubmitUser, "linkis.jdbc.task.timeout.alter.datasource.type") + .getJDBCConf(job.getSubmitUser, "linkis.jdbc.task.timeout.alert.datasource.type") .toLowerCase() if (StringUtils.isBlank(datasourceType)) datasourceType = "starrocks" if (datasourceType.contains(jobDatasourceType)) { @@ -79,7 +79,7 @@ class StarrocksTimeExceedRule(hitObserver: Observer) val timeValue = HttpsUntils.getJDBCConf( job.getSubmitUser, - "linkis.jdbc.task.timeout.alter.time" + "linkis.jdbc.task.timeout.alert.time" ) if (StringUtils.isNotBlank(timeValue)) { val timeoutInSeconds = timeValue.toDouble diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java index 3b664f86fc..735960d21f 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java @@ -93,6 +93,7 @@ public class WorkspaceExceptionManager { "80038", "The name directory {0} specified by PKG-INFO does not exist. Please confirm that the {0} specified by PKG-INFO in the package matches the actual folder name (PKG-INFO指定Name目录{0}不存在，请确认包中PKG-INFO指定{0}和实际文件夹名称一致)"); put("80039", "File upload failed, error message: {0} (文件上传失败，错误信息：{0})"); + put("80040", "{0} file not found in the archive ({0}文件不存在，请确认包中包含{0}文件)"); } }; diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 74aeec15b7..53ded50bb6 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -1444,8 +1444,7 @@ public Message pythonUpload( return Message.error("文件名称不能为空"); } // 获取文件名称 - String fileNameSuffix = fileName.substring(0, fileName.lastIndexOf(".")); - if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_.-]{0,49}$")) { + if (!fileName.matches("^[a-zA-Z][a-zA-Z0-9_.-]{0,49}$")) { return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50"); } @@ -1485,7 +1484,7 @@ public Message pythonUpload( } // 构建新的文件路径 - String newPath = fsPath.getPath() + "/" + file.getOriginalFilename(); + String newPath = fsPath.getPath() + FsPath.SEPARATOR + file.getOriginalFilename(); // 上传文件,tar包需要单独解压处理 if (!file.getOriginalFilename().endsWith(".tar.gz")) { FsPath fsPathNew = new FsPath(newPath); @@ -1504,15 +1503,19 @@ public Message pythonUpload( return Message.error("文件上传失败：PKG-INFO 文件不存在"); } is = FilesystemUtils.getZipInputStreamByTarInputStream(file, packageName); - newPath = fsPath.getPath() + FsPath.SEPARATOR + packageName + FsPath.CUR_DIR + "zip"; + newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName.replace(".tar.gz", ".zip"); FsPath fsPathNew = new FsPath(newPath); outputStream = fileSystem.write(fsPathNew, true); IOUtils.copy(is, outputStream); - } catch (IOException e) { + } catch (Exception e) { return Message.error("文件上传失败：" + e.getMessage()); } finally { - outputStream.close(); - is.close(); + if (outputStream != null) { + IOUtils.closeQuietly(outputStream); + } + if (is != null) { + IOUtils.closeQuietly(is); + } } } // 返回成功消息并包含文件地址 diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java index 9cf2f27f85..cee25219aa 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -36,6 +36,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Stack; @@ -84,36 +85,24 @@ public static void traverseFolder(FsPath fsPath, FileSystem fileSystem, Stack getInstallRequestPythonModules(MultipartFile file) th List modules = new ArrayList<>(); String originalFilename = file.getOriginalFilename(); if (StringUtils.isNotBlank(originalFilename) && originalFilename.endsWith(".tar.gz")) { + int findSetup = 0; // 读取 setup.py 文件的内容，并使用正则表达式提取 install_requires 字段。 // 解析 install_requires 字段中的依赖包信息 try (TarArchiveInputStream tarInput = @@ -239,19 +236,18 @@ public static List getInstallRequestPythonModules(MultipartFile file) th TarArchiveEntry entry; while ((entry = tarInput.getNextTarEntry()) != null) { if (entry.getName().endsWith("setup.py")) { - StringBuilder content = new StringBuilder(); - byte[] buffer = new byte[1024]; - int length; - while ((length = tarInput.read(buffer)) != -1) { - content.append(new String(buffer, 0, length)); - } - modules = extractDependencies(content.toString()); + findSetup = 1; + String content = IOUtils.toString(tarInput, StandardCharsets.UTF_8); + modules = extractDependencies(content); break; } } } catch (Exception e) { throw WorkspaceExceptionManager.createException(80039, e.getMessage()); } + if (findSetup == 0) { + throw WorkspaceExceptionManager.createException(80040, "setup.py"); + } } return modules; }