Dev 1.9.0 bug fix (#635)

* Code optimization * Code optimization * Code optimization * Code optimization * Code optimization * Code optimization * Code optimization --------- Co-authored-by: “v_kkhuang” <“[email protected]”>
WeDataSphere · Nov 4, 2024 · a102334 · a102334
1 parent 18613ce
commit a102334
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 27 deletions.
diff --git a/...itor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java b/...itor/src/main/java/org/apache/linkis/monitor/department/dao/UserDepartmentInfoMapper.java
@@ -21,20 +21,21 @@
 
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
 
 @Mapper
 public interface UserDepartmentInfoMapper {
 
   void insertUser(UserDepartmentInfo user);
-
+  @Transactional(rollbackFor = Exception.class)
   int batchInsertUsers(@Param("userDepartmentInfos") List<UserDepartmentInfo> userDepartmentInfos);
 
   void updateUser(UserDepartmentInfo user);
 
   UserDepartmentInfo selectUser(@Param("userName") String userName);
-
+  @Transactional(rollbackFor = Exception.class)
   void deleteUser();
 
   List<UserDepartmentInfo> selectAllUsers();

diff --git a/...es-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/...es-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
@@ -67,6 +67,7 @@
 import java.nio.file.Paths;
 import java.text.MessageFormat;
 import java.util.*;
+import java.util.stream.Collectors;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
@@ -1466,7 +1467,7 @@ public Message pythonUpload(
     List<String> pythonModules = FilesystemUtils.getInstallRequestPythonModules(file);
     String dependencies = "";
     if (CollectionUtils.isNotEmpty(pythonModules)) {
-      dependencies = String.join(",", pythonModules);
+      dependencies = pythonModules.stream().distinct().collect(Collectors.joining(","));
       String errorMsg = FilesystemUtils.checkModuleFile(pythonModules, username);
       if (StringUtils.isNotBlank(errorMsg)) {
         return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg);

diff --git a/...is-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java b/...is-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java
@@ -37,10 +37,7 @@
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Stack;
-import java.util.StringJoiner;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -120,12 +117,12 @@ private static String getRootPath(InputStream inputStream, String folder) throws
       while ((entry = tarInput.getNextTarEntry()) != null) {
         if (entry.isDirectory()
             && entry.getName().endsWith(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) {
-          return entry.getName();
+          return entry.getName().replace(folder + FsPath.SEPARATOR, "");
         }
         if (entry.getName().contains(FsPath.SEPARATOR + folder + FsPath.SEPARATOR)) {
           String delimiter = FsPath.SEPARATOR + folder + FsPath.SEPARATOR;
           int delimiterIndex = entry.getName().indexOf(delimiter);
-          return entry.getName().substring(0, delimiterIndex + delimiter.length());
+          return entry.getName().substring(0, delimiterIndex);
         }
       }
     } catch (Exception e) {
@@ -199,7 +196,7 @@ public static String checkModuleFile(List<String> pythonModules, String username
                   String exec =
                       Utils.exec(
                           (new String[] {
-                            "python",
+                            "python3",
                             Configuration.getLinkisHome() + "/admin/" + "check_modules.py",
                             module
                           }));
@@ -234,7 +231,7 @@ public static List<String> getInstallRequestPythonModules(MultipartFile file) th
           new TarArchiveInputStream(new GzipCompressorInputStream(file.getInputStream()))) {
         TarArchiveEntry entry;
         while ((entry = tarInput.getNextTarEntry()) != null) {
-          if (entry.getName().endsWith("setup.py")) {
+          if (entry.getName().endsWith("setup.py") || entry.getName().endsWith("pyproject.toml")) {
             findSetup = 1;
             String content = IOUtils.toString(tarInput, StandardCharsets.UTF_8);
             modules = extractDependencies(content);
@@ -245,31 +242,49 @@ public static List<String> getInstallRequestPythonModules(MultipartFile file) th
         throw WorkspaceExceptionManager.createException(80039, e.getMessage());
       }
       if (findSetup == 0) {
-        throw WorkspaceExceptionManager.createException(80040, "setup.py");
+        throw WorkspaceExceptionManager.createException(80040, "setup.py  or pyproject.toml");
       }
     }
     return modules;
   }
 
   public static List<String> extractDependencies(String content) {
+    String trim =
+        content
+            .replaceAll("#.*?\\n", "")
+            .replaceAll("\\n", "")
+            .replaceAll("'", "")
+            .replaceAll(" ", "")
+            .trim();
     List<String> modules = new ArrayList<>();
-    Pattern pattern = Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL);
-    Matcher matcher = pattern.matcher(content);
-    if (matcher.find()) {
-      String requirements = matcher.group(1);
-      String[] packages = requirements.split(",");
-      for (String pkg : packages) {
-        pkg = pkg.replaceAll("#.*?\\n", "").replaceAll("\\n", "").replaceAll("'", "").trim();
-        for (String operator : OPERATORS) {
-          if (pkg.contains(operator)) {
-            String[] parts = pkg.split(operator);
-            pkg = parts[0].trim();
-          }
-        }
-        if (StringUtils.isNotBlank(pkg)) {
-          modules.add(pkg);
+    String moduleStr = "";
+    Matcher setupMatcher =
+        Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim);
+    if (setupMatcher.find()) {
+      moduleStr = setupMatcher.group(1);
+    }
+    Matcher pyprojectMatcher =
+        Pattern.compile("dependencies=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim);
+    if (pyprojectMatcher.find()) {
+      moduleStr = pyprojectMatcher.group(1);
+    }
+    String[] packages = moduleStr.split(",");
+    for (String pkg : packages) {
+      pkg =
+          pkg.replaceAll("#.*?\\n", "")
+              .replaceAll("\\n", "")
+              .replaceAll("'", "")
+              .replace("\"", "")
+              .trim();
+      for (String operator : OPERATORS) {
+        if (pkg.contains(operator)) {
+          String[] parts = pkg.split(operator);
+          pkg = parts[0].trim();
         }
       }
+      if (StringUtils.isNotBlank(pkg)) {
+        modules.add(pkg);
+      }
     }
     return modules;
   }