From 82fcf60cae799a20f9ff12b399b48b436009c137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Wed, 30 Oct 2024 11:25:22 +0800 Subject: [PATCH 1/7] Code optimization --- .../restful/api/ConfigurationRestfulApi.java | 6 ++-- .../restful/DataSourceCoreRestfulApi.java | 28 ++++++------------- .../core/restful/RestfulApiHelper.java | 13 +++++---- .../filesystem/util/FilesystemUtils.java | 2 +- .../apache/linkis/udf/api/UDFRestfulApi.java | 6 ++-- 5 files changed, 22 insertions(+), 33 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index c9c1840739..e4cb9cdbb2 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -311,14 +311,12 @@ public Message saveFullTree(HttpServletRequest req, @RequestBody JsonNode json) configKeyValue.setConfigValue(sparkConf); } if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue() - && (configKeyValue.getKey().equals("linkis.nebula.password") - || configKeyValue.getKey().equals("wds.linkis.jdbc.password")) + && configKeyValue.getKey().equals("linkis.nebula.password") && StringUtils.isNotBlank(configKeyValue.getConfigValue())) { List configByLabelIds = configurationService.getConfigByLabelId(configKeyValue.getConfigLabelId(), null); for (ConfigKeyValue configByLabelId : configByLabelIds) { - if ((configByLabelId.getKey().equals("linkis.nebula.password") - || configByLabelId.getKey().equals("wds.linkis.jdbc.password")) + if (configByLabelId.getKey().equals("linkis.nebula.password") && !configByLabelId.getConfigValue().equals(configKeyValue.getConfigValue())) { configKeyValue.setConfigValue( AESUtils.encrypt( diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index 350fffe010..cf57612c03 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -389,9 +389,7 @@ public Message getInfoByDataSourceId( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); - } + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -427,9 +425,8 @@ public Message getInfoByDataSourceName( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); - } + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -464,9 +461,8 @@ public Message getPublishedInfoByDataSourceName( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); - } + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -510,9 +506,7 @@ public Message getInfoByDataSourceIdAndVersion( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); - } + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -550,7 +544,7 @@ public Message getVersionList( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue() && null != versions) { + if (null != versions) { versions.forEach( version -> { RestfulApiHelper.decryptPasswordKey( @@ -700,9 +694,7 @@ public Message getConnectParams( Map connectParams = dataSource.getConnectParams(); List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); - if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); - } + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); return Message.ok().data("connectParams", connectParams); }, "Fail to connect data source[连接数据源失败]"); @@ -738,9 +730,7 @@ public Message getConnectParams( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); - if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); - } + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); return Message.ok().data("connectParams", connectParams); }, "Fail to connect data source[连接数据源失败]"); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java index ff2d48121f..8726fd9a23 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java @@ -74,11 +74,12 @@ public static void encryptPasswordKey( Object password = connectParams.get(keyDefinition.getKey()); if (null != password) { String passwordStr = String.valueOf(password); - if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue() - && !connectParams.containsKey("isEncrypt")) { - passwordStr = - AESUtils.encrypt(passwordStr, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()); - connectParams.put("isEncrypt", "1"); + if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + if (!connectParams.containsKey("isEncrypt")) { + passwordStr = + AESUtils.encrypt(passwordStr, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()); + connectParams.put("isEncrypt", "1"); + } } else { passwordStr = CryptoUtils.object2String(passwordStr); } @@ -106,7 +107,7 @@ public static void decryptPasswordKey( passwordStr = AESUtils.decrypt(passwordStr, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()); } else { - passwordStr = CryptoUtils.object2String(passwordStr); + passwordStr = String.valueOf(CryptoUtils.string2Object(passwordStr)); } connectParams.put(keyDefinition.getKey(), passwordStr); } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java index 82f760326c..9cf2f27f85 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -205,7 +205,7 @@ public static String checkModuleFile(MultipartFile file, String username) throws Utils.exec( (new String[] { "python", - Configuration.getLinkisHome() + "/admin/" + "check_python_module.py", + Configuration.getLinkisHome() + "/admin/" + "check_modules.py", module })); return !Boolean.parseBoolean(exec); diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 61d40dcceb..3ff6fb8c90 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -1286,9 +1286,9 @@ public Message pythonFileExist( if (org.apache.commons.lang3.StringUtils.isBlank(fileName)) { return Message.error("参数fileName不能为空"); } - String fileNameWithoutExtension = fileName.substring(0, fileName.lastIndexOf(".")); - if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_.-]{0,49}$")) { - return Message.error("只支持数字字母下划线，且以字母开头，长度最大50"); + String fileNameWithoutExtension = fileName.split("\\.")[0]; + if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_-]{0,49}$")) { + return Message.error("只支持数字字母下划线，中划线，且以字母开头，长度最大50"); } // 封装PythonModuleInfo对象并查询数据库 From b15d84ca4ec944b8b05e964f4347fc68eca7f57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Wed, 30 Oct 2024 11:28:16 +0800 Subject: [PATCH 2/7] Code optimization --- .../configuration/restful/api/ConfigurationRestfulApi.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index e4cb9cdbb2..c9c1840739 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -311,12 +311,14 @@ public Message saveFullTree(HttpServletRequest req, @RequestBody JsonNode json) configKeyValue.setConfigValue(sparkConf); } if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue() - && configKeyValue.getKey().equals("linkis.nebula.password") + && (configKeyValue.getKey().equals("linkis.nebula.password") + || configKeyValue.getKey().equals("wds.linkis.jdbc.password")) && StringUtils.isNotBlank(configKeyValue.getConfigValue())) { List configByLabelIds = configurationService.getConfigByLabelId(configKeyValue.getConfigLabelId(), null); for (ConfigKeyValue configByLabelId : configByLabelIds) { - if (configByLabelId.getKey().equals("linkis.nebula.password") + if ((configByLabelId.getKey().equals("linkis.nebula.password") + || configByLabelId.getKey().equals("wds.linkis.jdbc.password")) && !configByLabelId.getConfigValue().equals(configKeyValue.getConfigValue())) { configKeyValue.setConfigValue( AESUtils.encrypt( From 97049f592b8952537ab2608f2cb1331d4ee4c30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Wed, 30 Oct 2024 18:00:11 +0800 Subject: [PATCH 3/7] Code optimization --- .../exception/WorkspaceExceptionManager.java | 1 + .../filesystem/restful/api/FsRestfulApi.java | 10 +++-- .../filesystem/util/FilesystemUtils.java | 41 +++++++------------ 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java index 3b664f86fc..735960d21f 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/exception/WorkspaceExceptionManager.java @@ -93,6 +93,7 @@ public class WorkspaceExceptionManager { "80038", "The name directory {0} specified by PKG-INFO does not exist. Please confirm that the {0} specified by PKG-INFO in the package matches the actual folder name (PKG-INFO指定Name目录{0}不存在，请确认包中PKG-INFO指定{0}和实际文件夹名称一致)"); put("80039", "File upload failed, error message: {0} (文件上传失败，错误信息：{0})"); + put("80040", "{0} file not found in the archive ({0}文件不存在，请确认包中包含{0}文件)"); } }; diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 74aeec15b7..4cbb28931b 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -1508,11 +1508,15 @@ public Message pythonUpload( FsPath fsPathNew = new FsPath(newPath); outputStream = fileSystem.write(fsPathNew, true); IOUtils.copy(is, outputStream); - } catch (IOException e) { + } catch (Exception e) { return Message.error("文件上传失败：" + e.getMessage()); } finally { - outputStream.close(); - is.close(); + if (outputStream != null) { + IOUtils.closeQuietly(outputStream); + } + if (is != null) { + IOUtils.closeQuietly(is); + } } } // 返回成功消息并包含文件地址 diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java index 9cf2f27f85..2172be8d62 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -36,6 +36,7 @@ import org.springframework.web.multipart.MultipartFile; import java.io.*; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Stack; @@ -84,36 +85,24 @@ public static void traverseFolder(FsPath fsPath, FileSystem fileSystem, Stack getInstallRequestPythonModules(MultipartFile file) th List modules = new ArrayList<>(); String originalFilename = file.getOriginalFilename(); if (StringUtils.isNotBlank(originalFilename) && originalFilename.endsWith(".tar.gz")) { + int findSetup = 0; // 读取 setup.py 文件的内容，并使用正则表达式提取 install_requires 字段。 // 解析 install_requires 字段中的依赖包信息 try (TarArchiveInputStream tarInput = @@ -239,19 +229,18 @@ public static List getInstallRequestPythonModules(MultipartFile file) th TarArchiveEntry entry; while ((entry = tarInput.getNextTarEntry()) != null) { if (entry.getName().endsWith("setup.py")) { - StringBuilder content = new StringBuilder(); - byte[] buffer = new byte[1024]; - int length; - while ((length = tarInput.read(buffer)) != -1) { - content.append(new String(buffer, 0, length)); - } - modules = extractDependencies(content.toString()); + findSetup = 1; + String content = IOUtils.toString(tarInput, StandardCharsets.UTF_8); + modules = extractDependencies(content); break; } } } catch (Exception e) { throw WorkspaceExceptionManager.createException(80039, e.getMessage()); } + if (findSetup == 0) { + throw WorkspaceExceptionManager.createException(80040, "setup.py"); + } } return modules; } From 2b72272f0e9befbe3d9cdca9f5e31466495bbad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Thu, 31 Oct 2024 14:43:18 +0800 Subject: [PATCH 4/7] Code optimization --- .../jobtime/StarrocksTimeExceedAlterSender.scala | 6 +++--- .../jobhistory/jobtime/StarrocksTimeExceedRule.scala | 4 ++-- .../linkis/filesystem/restful/api/FsRestfulApi.java | 7 +++---- .../linkis/filesystem/util/FilesystemUtils.java | 11 +++++++++-- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala index 9334cc88b6..5f1f42311c 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedAlterSender.scala @@ -58,11 +58,11 @@ class StarrocksTimeExceedAlterSender extends Observer with Logging { } else { val jobHistory = a.asInstanceOf[JobHistory] val timeValue = - HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alter.time") + HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alert.time") val userValue = - HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alter.user") + HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alert.user") var levelValue = - HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alter.level") + HttpsUntils.getJDBCConf(jobHistory.getSubmitUser, "linkis.jdbc.task.timeout.alert.level") if (StringUtils.isNotBlank(timeValue) && StringUtils.isNotBlank(userValue)) { val replaceParm: util.HashMap[String, String] = new util.HashMap[String, String] replaceParm.put("$id", String.valueOf(jobHistory.getId)) diff --git a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala index 6268a37768..4741727ed9 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala +++ b/linkis-extensions/linkis-et-monitor/src/main/scala/org/apache/linkis/monitor/jobhistory/jobtime/StarrocksTimeExceedRule.scala @@ -69,7 +69,7 @@ class StarrocksTimeExceedRule(hitObserver: Observer) MapUtils.getString(datasourceTypeMap, "name", "").toLowerCase() // 获取管理台配置需要使用的数据源类型（默认starrocks） var datasourceType = HttpsUntils - .getJDBCConf(job.getSubmitUser, "linkis.jdbc.task.timeout.alter.datasource.type") + .getJDBCConf(job.getSubmitUser, "linkis.jdbc.task.timeout.alert.datasource.type") .toLowerCase() if (StringUtils.isBlank(datasourceType)) datasourceType = "starrocks" if (datasourceType.contains(jobDatasourceType)) { @@ -79,7 +79,7 @@ class StarrocksTimeExceedRule(hitObserver: Observer) val timeValue = HttpsUntils.getJDBCConf( job.getSubmitUser, - "linkis.jdbc.task.timeout.alter.time" + "linkis.jdbc.task.timeout.alert.time" ) if (StringUtils.isNotBlank(timeValue)) { val timeoutInSeconds = timeValue.toDouble diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 4cbb28931b..53ded50bb6 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -1444,8 +1444,7 @@ public Message pythonUpload( return Message.error("文件名称不能为空"); } // 获取文件名称 - String fileNameSuffix = fileName.substring(0, fileName.lastIndexOf(".")); - if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_.-]{0,49}$")) { + if (!fileName.matches("^[a-zA-Z][a-zA-Z0-9_.-]{0,49}$")) { return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50"); } @@ -1485,7 +1484,7 @@ public Message pythonUpload( } // 构建新的文件路径 - String newPath = fsPath.getPath() + "/" + file.getOriginalFilename(); + String newPath = fsPath.getPath() + FsPath.SEPARATOR + file.getOriginalFilename(); // 上传文件,tar包需要单独解压处理 if (!file.getOriginalFilename().endsWith(".tar.gz")) { FsPath fsPathNew = new FsPath(newPath); @@ -1504,7 +1503,7 @@ public Message pythonUpload( return Message.error("文件上传失败：PKG-INFO 文件不存在"); } is = FilesystemUtils.getZipInputStreamByTarInputStream(file, packageName); - newPath = fsPath.getPath() + FsPath.SEPARATOR + packageName + FsPath.CUR_DIR + "zip"; + newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName.replace(".tar.gz", ".zip"); FsPath fsPathNew = new FsPath(newPath); outputStream = fileSystem.write(fsPathNew, true); IOUtils.copy(is, outputStream); diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java index 2172be8d62..cee25219aa 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -118,9 +118,15 @@ private static String getRootPath(InputStream inputStream, String folder) throws new TarArchiveInputStream(new GzipCompressorInputStream(inputStream))) { TarArchiveEntry entry; while ((entry = tarInput.getNextTarEntry()) != null) { - if (entry.isDirectory() && entry.getName().endsWith("/" + folder + "/")) { + if (entry.isDirectory() + && entry.getName().endsWith(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) { return entry.getName(); } + if (entry.getName().contains(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) { + String delimiter = FsPath.SEPARATOR + folder + FsPath.SEPARATOR; + int delimiterIndex = entry.getName().indexOf(delimiter); + return entry.getName().substring(0, delimiterIndex + delimiter.length()); + } } } catch (Exception e) { throw WorkspaceExceptionManager.createException(80039, e.getMessage()); @@ -146,7 +152,8 @@ private static InputStream createZipFile(InputStream inputStream, String folder, try { TarArchiveEntry entry; while ((entry = tarInput.getNextTarEntry()) != null) { - if (!entry.isDirectory() && entry.getName().contains("/" + folder + "/")) { + if (!entry.isDirectory() + && entry.getName().contains(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) { // \dist\py_mysql-1.0.tar\py_mysql-1.0\py_mysql\lib\__init__.py ZipEntry zipEntry = new ZipEntry(entry.getName().substring(rootPath.length())); zos.putNextEntry(zipEntry); From d997a03b24369a2aff3ec93cf48b9575c509dda7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Thu, 31 Oct 2024 19:07:15 +0800 Subject: [PATCH 5/7] Code optimization --- .../restful/api/ConfigurationRestfulApi.java | 16 +++++++++++ .../restful/DataSourceCoreRestfulApi.java | 28 +++++++++++++------ .../filesystem/restful/api/FsRestfulApi.java | 21 ++++++++++---- .../filesystem/util/FilesystemUtils.java | 7 ++--- .../apache/linkis/udf/api/UDFRestfulApi.java | 4 +-- 5 files changed, 56 insertions(+), 20 deletions(-) diff --git a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java index c9c1840739..ad23c782f0 100644 --- a/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java +++ b/linkis-public-enhancements/linkis-configuration/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java @@ -570,6 +570,22 @@ public Message saveKeyValue(HttpServletRequest req, @RequestBody Map userConfigValue = + configKeyService.getUserConfigValue(engineType, configKeyValue.getKey(), creator, user); + for (ConfigUserValue configUserValue : userConfigValue) { + if ((configUserValue.getKey().equals("linkis.nebula.password") + || configUserValue.getKey().equals("wds.linkis.jdbc.password")) + && !configUserValue.getConfigValue().equals(configKeyValue.getConfigValue())) { + configKeyValue.setConfigValue( + AESUtils.encrypt( + configKeyValue.getConfigValue(), AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue())); + } + } + } ConfigValue configValue = configKeyService.saveConfigValue(configKeyValue, labelList); configurationService.clearAMCacheConf(username, creator, engineType, version); return message.data("configValue", configValue); diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java index cf57612c03..350fffe010 100644 --- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java +++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java @@ -389,7 +389,9 @@ public Message getInfoByDataSourceId( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + } return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -425,8 +427,9 @@ public Message getInfoByDataSourceName( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); - + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + } return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -461,8 +464,9 @@ public Message getPublishedInfoByDataSourceName( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); - + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + } return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -506,7 +510,9 @@ public Message getInfoByDataSourceIdAndVersion( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams()); + } return Message.ok().data("info", dataSource); }, "Fail to access data source[获取数据源信息失败]"); @@ -544,7 +550,7 @@ public Message getVersionList( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); // Decrypt - if (null != versions) { + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue() && null != versions) { versions.forEach( version -> { RestfulApiHelper.decryptPasswordKey( @@ -694,7 +700,9 @@ public Message getConnectParams( Map connectParams = dataSource.getConnectParams(); List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); + } return Message.ok().data("connectParams", connectParams); }, "Fail to connect data source[连接数据源失败]"); @@ -730,7 +738,9 @@ public Message getConnectParams( List keyDefinitionList = dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId()); - RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); + if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) { + RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams); + } return Message.ok().data("connectParams", connectParams); }, "Fail to connect data source[连接数据源失败]"); diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 8fa05496cc..f660a7e095 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -45,6 +45,7 @@ import org.apache.linkis.storage.source.FileSource$; import org.apache.linkis.storage.utils.StorageUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.BOMInputStream; import org.apache.commons.lang3.StringUtils; @@ -1461,9 +1462,15 @@ public Message pythonUpload( } // tar.gz包依赖检查 - String errorMsg = FilesystemUtils.checkModuleFile(file, username); - if (StringUtils.isNotBlank(errorMsg)) { - return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg); + // 获取install_requires中的python模块 + List pythonModules = FilesystemUtils.getInstallRequestPythonModules(file); + String dependencies = ""; + if (CollectionUtils.isNotEmpty(pythonModules)) { + dependencies = String.join(",", pythonModules); + String errorMsg = FilesystemUtils.checkModuleFile(pythonModules, username); + if (StringUtils.isNotBlank(errorMsg)) { + return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg); + } } // 定义目录路径 @@ -1499,11 +1506,12 @@ public Message pythonUpload( OutputStream outputStream = null; try { String packageName = FilesystemUtils.findPackageName(file.getInputStream()); + fileName = packageName + FsPath.CUR_DIR + "zip"; if (StringUtils.isBlank(packageName)) { return Message.error("文件上传失败：PKG-INFO 文件不存在"); } is = FilesystemUtils.getZipInputStreamByTarInputStream(file, packageName); - newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName.replace(".tar.gz", ".zip"); + newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName; FsPath fsPathNew = new FsPath(newPath); outputStream = fileSystem.write(fsPathNew, true); IOUtils.copy(is, outputStream); @@ -1519,6 +1527,9 @@ public Message pythonUpload( } } // 返回成功消息并包含文件地址 - return Message.ok().data("filePath", newPath); + return Message.ok() + .data("filePath", newPath) + .data("dependencies", dependencies) + .data("fileName", fileName); } } diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java index cee25219aa..c013891738 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -184,13 +184,12 @@ public static InputStream getZipInputStreamByTarInputStream( * * 检查python环境中模块是否存在，使用程序调用脚本实现 脚本保存在admin文件夹中，脚本名为check_python_module.py * 脚本执行存在返回ture，不存在返回false * - * @param file 传入需要检查的模块列表 + * @param pythonModules 传入需要检查的模块列表 * @param username * @return 返回不存在的模块列表 */ - public static String checkModuleFile(MultipartFile file, String username) throws IOException { - // 获取install_requires中的python模块 - List pythonModules = getInstallRequestPythonModules(file); + public static String checkModuleFile(List pythonModules, String username) + throws IOException { StringJoiner joiner = new StringJoiner(","); // 检查机器上pyhton环境中模块是否存在 List notExistModules = diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 2d0401d93d..f69072027f 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -1290,10 +1290,10 @@ public Message pythonFileExist( if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_-]{0,49}$")) { return Message.error("只支持数字字母下划线，中划线，且以字母开头，长度最大50"); } - + String fileNameWithoutVersion = fileNameWithoutExtension.split("-")[0]; // 封装PythonModuleInfo对象并查询数据库 PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); - pythonModuleInfo.setName(fileNameWithoutExtension); + pythonModuleInfo.setName(fileNameWithoutVersion); pythonModuleInfo.setCreateUser(userName); PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); From 381f6be87e2f7e314b325743ff584abcedbeca5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Fri, 1 Nov 2024 16:14:14 +0800 Subject: [PATCH 6/7] Code optimization --- .../filesystem/restful/api/FsRestfulApi.java | 3 +- .../filesystem/util/FilesystemUtils.java | 63 ++++++++++++------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index f660a7e095..f7b4625d2b 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -67,6 +67,7 @@ import java.nio.file.Paths; import java.text.MessageFormat; import java.util.*; +import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; @@ -1466,7 +1467,7 @@ public Message pythonUpload( List pythonModules = FilesystemUtils.getInstallRequestPythonModules(file); String dependencies = ""; if (CollectionUtils.isNotEmpty(pythonModules)) { - dependencies = String.join(",", pythonModules); + dependencies = pythonModules.stream().distinct().collect(Collectors.joining(",")); String errorMsg = FilesystemUtils.checkModuleFile(pythonModules, username); if (StringUtils.isNotBlank(errorMsg)) { return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg); diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java index c013891738..cdf0af5ea6 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -37,10 +37,7 @@ import java.io.*; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Stack; -import java.util.StringJoiner; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -120,12 +117,12 @@ private static String getRootPath(InputStream inputStream, String folder) throws while ((entry = tarInput.getNextTarEntry()) != null) { if (entry.isDirectory() && entry.getName().endsWith(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) { - return entry.getName(); + return entry.getName().replace(folder + FsPath.SEPARATOR, ""); } if (entry.getName().contains(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) { String delimiter = FsPath.SEPARATOR + folder + FsPath.SEPARATOR; int delimiterIndex = entry.getName().indexOf(delimiter); - return entry.getName().substring(0, delimiterIndex + delimiter.length()); + return entry.getName().substring(0, delimiterIndex); } } } catch (Exception e) { @@ -199,7 +196,7 @@ public static String checkModuleFile(List pythonModules, String username String exec = Utils.exec( (new String[] { - "python", + "python3", Configuration.getLinkisHome() + "/admin/" + "check_modules.py", module })); @@ -234,7 +231,7 @@ public static List getInstallRequestPythonModules(MultipartFile file) th new TarArchiveInputStream(new GzipCompressorInputStream(file.getInputStream()))) { TarArchiveEntry entry; while ((entry = tarInput.getNextTarEntry()) != null) { - if (entry.getName().endsWith("setup.py")) { + if (entry.getName().endsWith("setup.py") || entry.getName().endsWith("pyproject.toml")) { findSetup = 1; String content = IOUtils.toString(tarInput, StandardCharsets.UTF_8); modules = extractDependencies(content); @@ -245,31 +242,49 @@ public static List getInstallRequestPythonModules(MultipartFile file) th throw WorkspaceExceptionManager.createException(80039, e.getMessage()); } if (findSetup == 0) { - throw WorkspaceExceptionManager.createException(80040, "setup.py"); + throw WorkspaceExceptionManager.createException(80040, "setup.py or pyproject.toml"); } } return modules; } public static List extractDependencies(String content) { + String trim = + content + .replaceAll("#.*?\\n", "") + .replaceAll("\\n", "") + .replaceAll("'", "") + .replaceAll(" ", "") + .trim(); List modules = new ArrayList<>(); - Pattern pattern = Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL); - Matcher matcher = pattern.matcher(content); - if (matcher.find()) { - String requirements = matcher.group(1); - String[] packages = requirements.split(","); - for (String pkg : packages) { - pkg = pkg.replaceAll("#.*?\\n", "").replaceAll("\\n", "").replaceAll("'", "").trim(); - for (String operator : OPERATORS) { - if (pkg.contains(operator)) { - String[] parts = pkg.split(operator); - pkg = parts[0].trim(); - } - } - if (StringUtils.isNotBlank(pkg)) { - modules.add(pkg); + String moduleStr = ""; + Matcher setupMatcher = + Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim); + if (setupMatcher.find()) { + moduleStr = setupMatcher.group(1); + } + Matcher pyprojectMatcher = + Pattern.compile("dependencies=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim); + if (pyprojectMatcher.find()) { + moduleStr = pyprojectMatcher.group(1); + } + String[] packages = moduleStr.split(","); + for (String pkg : packages) { + pkg = + pkg.replaceAll("#.*?\\n", "") + .replaceAll("\\n", "") + .replaceAll("'", "") + .replace("\"", "") + .trim(); + for (String operator : OPERATORS) { + if (pkg.contains(operator)) { + String[] parts = pkg.split(operator); + pkg = parts[0].trim(); } } + if (StringUtils.isNotBlank(pkg)) { + modules.add(pkg); + } } return modules; } From d5e88618db695a596e8a8bbad87b47fa641175e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Fri, 1 Nov 2024 17:58:34 +0800 Subject: [PATCH 7/7] Code optimization --- .../monitor/department/dao/UserDepartmentInfoMapper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java index 9f263136ea..420012f8fd 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java @@ -21,6 +21,7 @@ import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; +import org.springframework.transaction.annotation.Transactional; import java.util.List; @@ -28,13 +29,13 @@ public interface UserDepartmentInfoMapper { void insertUser(UserDepartmentInfo user); - + @Transactional(rollbackFor = Exception.class) int batchInsertUsers(@Param("userDepartmentInfos") List userDepartmentInfos); void updateUser(UserDepartmentInfo user); UserDepartmentInfo selectUser(@Param("userName") String userName); - + @Transactional(rollbackFor = Exception.class) void deleteUser(); List selectAllUsers();