diff --git a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java index 9f263136ea..420012f8fd 100644 --- a/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java +++ b/linkis-extensions/linkis-et-monitor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java @@ -21,6 +21,7 @@ import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; +import org.springframework.transaction.annotation.Transactional; import java.util.List; @@ -28,13 +29,13 @@ public interface UserDepartmentInfoMapper { void insertUser(UserDepartmentInfo user); - + @Transactional(rollbackFor = Exception.class) int batchInsertUsers(@Param("userDepartmentInfos") List userDepartmentInfos); void updateUser(UserDepartmentInfo user); UserDepartmentInfo selectUser(@Param("userName") String userName); - + @Transactional(rollbackFor = Exception.class) void deleteUser(); List selectAllUsers(); diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index f660a7e095..f7b4625d2b 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -67,6 +67,7 @@ import java.nio.file.Paths; import java.text.MessageFormat; import java.util.*; +import java.util.stream.Collectors; import com.fasterxml.jackson.databind.JsonNode; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; @@ -1466,7 +1467,7 @@ public Message pythonUpload( List pythonModules = FilesystemUtils.getInstallRequestPythonModules(file); String dependencies = ""; if (CollectionUtils.isNotEmpty(pythonModules)) { - dependencies = String.join(",", pythonModules); + dependencies = pythonModules.stream().distinct().collect(Collectors.joining(",")); String errorMsg = FilesystemUtils.checkModuleFile(pythonModules, username); if (StringUtils.isNotBlank(errorMsg)) { return Message.error("部分依赖未加载,请检查并重新上传依赖包,依赖信息:" + errorMsg); diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java index c013891738..cdf0af5ea6 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java @@ -37,10 +37,7 @@ import java.io.*; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Stack; -import java.util.StringJoiner; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -120,12 +117,12 @@ private static String getRootPath(InputStream inputStream, String folder) throws while ((entry = tarInput.getNextTarEntry()) != null) { if (entry.isDirectory() && entry.getName().endsWith(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) { - return entry.getName(); + return entry.getName().replace(folder + FsPath.SEPARATOR, ""); } if (entry.getName().contains(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) { String delimiter = FsPath.SEPARATOR + folder + FsPath.SEPARATOR; int delimiterIndex = entry.getName().indexOf(delimiter); - return entry.getName().substring(0, delimiterIndex + delimiter.length()); + return entry.getName().substring(0, delimiterIndex); } } } catch (Exception e) { @@ -199,7 +196,7 @@ public static String checkModuleFile(List pythonModules, String username String exec = Utils.exec( (new String[] { - "python", + "python3", Configuration.getLinkisHome() + "/admin/" + "check_modules.py", module })); @@ -234,7 +231,7 @@ public static List getInstallRequestPythonModules(MultipartFile file) th new TarArchiveInputStream(new GzipCompressorInputStream(file.getInputStream()))) { TarArchiveEntry entry; while ((entry = tarInput.getNextTarEntry()) != null) { - if (entry.getName().endsWith("setup.py")) { + if (entry.getName().endsWith("setup.py") || entry.getName().endsWith("pyproject.toml")) { findSetup = 1; String content = IOUtils.toString(tarInput, StandardCharsets.UTF_8); modules = extractDependencies(content); @@ -245,31 +242,49 @@ public static List getInstallRequestPythonModules(MultipartFile file) th throw WorkspaceExceptionManager.createException(80039, e.getMessage()); } if (findSetup == 0) { - throw WorkspaceExceptionManager.createException(80040, "setup.py"); + throw WorkspaceExceptionManager.createException(80040, "setup.py or pyproject.toml"); } } return modules; } public static List extractDependencies(String content) { + String trim = + content + .replaceAll("#.*?\\n", "") + .replaceAll("\\n", "") + .replaceAll("'", "") + .replaceAll(" ", "") + .trim(); List modules = new ArrayList<>(); - Pattern pattern = Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL); - Matcher matcher = pattern.matcher(content); - if (matcher.find()) { - String requirements = matcher.group(1); - String[] packages = requirements.split(","); - for (String pkg : packages) { - pkg = pkg.replaceAll("#.*?\\n", "").replaceAll("\\n", "").replaceAll("'", "").trim(); - for (String operator : OPERATORS) { - if (pkg.contains(operator)) { - String[] parts = pkg.split(operator); - pkg = parts[0].trim(); - } - } - if (StringUtils.isNotBlank(pkg)) { - modules.add(pkg); + String moduleStr = ""; + Matcher setupMatcher = + Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim); + if (setupMatcher.find()) { + moduleStr = setupMatcher.group(1); + } + Matcher pyprojectMatcher = + Pattern.compile("dependencies=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim); + if (pyprojectMatcher.find()) { + moduleStr = pyprojectMatcher.group(1); + } + String[] packages = moduleStr.split(","); + for (String pkg : packages) { + pkg = + pkg.replaceAll("#.*?\\n", "") + .replaceAll("\\n", "") + .replaceAll("'", "") + .replace("\"", "") + .trim(); + for (String operator : OPERATORS) { + if (pkg.contains(operator)) { + String[] parts = pkg.split(operator); + pkg = parts[0].trim(); } } + if (StringUtils.isNotBlank(pkg)) { + modules.add(pkg); + } } return modules; }