From 6c639be47ef791a88c18d682fbfc5d650be9c47a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Sat, 24 Aug 2024 14:35:17 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=94=B1AI=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E7=94=9F=E6=88=90=E6=8E=A5=E5=8F=A3=E5=92=8CAI?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E7=94=9F=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../filesystem/restful/api/FsRestfulApi.java | 59 ++++ .../apache/linkis/udf/api/UDFRestfulApi.java | 266 ++++++++++++++++++ .../udf/dao/PythonModuleInfoMapper.java | 43 +++ .../linkis/udf/entity/PythonModuleInfo.java | 158 +++++++++++ .../udf/service/PythonModuleInfoService.java | 40 +++ .../impl/PythonModuleInfoServiceImpl.java | 63 +++++ .../mapper/common/PythonModuleInfoMapper.xml | 90 ++++++ 7 files changed, 719 insertions(+) create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 4d9a3fa651..7dd01e8c36 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -1423,6 +1423,65 @@ public Message encryptPath( return Message.ok().data("data", fileMD5Str); } + @ApiOperation(value = "Python模块上传", notes = "上传Python模块文件并返回文件地址", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "file", required = true, dataType = "MultipartFile", value = "上传的文件"), + @ApiImplicitParam(name = "fileName", required = true, dataType = "String", value = "文件名称") + }) + @RequestMapping(path = "/filesystem/python-upload", method = RequestMethod.POST) + public Message pythonUpload( + HttpServletRequest req, + @RequestParam("file") MultipartFile file, + @RequestParam("fileName") String fileName) throws IOException { + + // 获取登录用户 + String username = ModuleUserUtils.getOperationUser(req, "pythonUpload"); + + // 校验文件名称 + if (!fileName.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50"); + } + + // 校验文件类型 + if (!file.getOriginalFilename().endsWith(".py") && !file.getOriginalFilename().endsWith(".zip")) { + return Message.error("仅支持.py和.zip格式模块文件"); + } + + // 校验文件大小 + if (file.getSize() > 50 * 1024 * 1024) { + return Message.error("限制最大单个文件50M"); + } + + // 定义目录路径 + String path = "/appcom/linkis/udf/" + username; + FsPath fsPath = new FsPath(path); + + // 获取文件系统实例 + FileSystem fileSystem = fsService.getFileSystem(username, fsPath); + + // 确认目录是否存在，不存在则创建新目录 + if (!fileSystem.exists(fsPath)) { + try { + fileSystem.createNewFile(fsPath); + } catch (IOException e) { + return Message.error("创建目录失败：" + e.getMessage()); + } + } + + // 构建新的文件路径 + String newPath = fsPath.getPath() + "/" + fileName; + FsPath fsPathNew = new FsPath(newPath); + + // 上传文件 + try (InputStream is = file.getInputStream(); + OutputStream outputStream = fileSystem.write(fsPathNew, true)) { + IOUtils.copy(is, outputStream); + } catch (IOException e) { + return Message.error("文件上传失败：" + e.getMessage()); + } + // 返回成功消息并包含文件地址 + return Message.ok().data("filePath", newPath); + } /** * * * diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index c659e4aa93..064f2d5390 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -17,11 +17,14 @@ package org.apache.linkis.udf.api; +import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; +import org.apache.linkis.udf.entity.PythonModuleInfo; import org.apache.linkis.udf.entity.UDFInfo; import org.apache.linkis.udf.entity.UDFTree; import org.apache.linkis.udf.excepiton.UDFException; +import org.apache.linkis.udf.service.PythonModuleInfoService; import org.apache.linkis.udf.service.UDFService; import org.apache.linkis.udf.service.UDFTreeService; import org.apache.linkis.udf.utils.ConstantVar; @@ -49,6 +52,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import com.google.common.collect.Lists; @@ -73,6 +77,7 @@ public class UDFRestfulApi { @Autowired private UDFService udfService; @Autowired private UDFTreeService udfTreeService; + @Autowired private PythonModuleInfoService pythonModuleInfoService; ObjectMapper mapper = new ObjectMapper(); @@ -1013,4 +1018,265 @@ public Message versionInfo( } return message; } + + /** + * Python物料查询 + * + * @param name python模块名称 + * @param engineType 引擎类型(all,spark,python) + * @param username 用户名 + * @param isLoad 是否加载（0-未加载，1-已加载） + * @param isExpire 是否过期（0-未过期，1-已过期） + * @param pageNow 页码 + * @param pageSize 每页展示数据条数 + */ + @RequestMapping(path = "/python-list", method = RequestMethod.GET) + @ApiOperation(value = "查询Python模块列表", notes = "根据条件查询Python模块信息") + @ApiImplicitParams({ + @ApiImplicitParam(name = "name", value = "Python模块名称", required = false, dataType = "String"), + @ApiImplicitParam( + name = "engineType", + value = "引擎类型（all, spark, python）", + required = false, + dataType = "String"), + @ApiImplicitParam(name = "username", value = "用户名", required = false, dataType = "String"), + @ApiImplicitParam( + name = "isLoad", + value = "是否加载（0-未加载，1-已加载）", + required = false, + dataType = "Integer"), + @ApiImplicitParam( + name = "isExpire", + value = "是否过期（0-未过期，1-已过期）", + required = false, + dataType = "Integer"), + @ApiImplicitParam(name = "pageNow", value = "页码", required = false, dataType = "Integer"), + @ApiImplicitParam(name = "pageSize", value = "每页展示数据条数", required = false, dataType = "Integer") + }) + public Message pythonList( + @RequestParam(value = "name", required = false) String name, + @RequestParam(value = "engineType", required = false) String engineType, + @RequestParam(value = "username", required = false) String username, + @RequestParam(value = "isLoad", required = false) Integer isLoad, + @RequestParam(value = "isExpire", required = false) Integer isExpire, + @RequestParam(value = "pageNow", required = false) Integer pageNow, + @RequestParam(value = "pageSize", required = false) Integer pageSize, + HttpServletRequest req) { + + // 获取登录用户 + String user = ModuleUserUtils.getOperationUser(req, "pythonList"); + + // 参数校验 + if (name == null) name = null; + if (engineType == null) engineType = null; + if (pageNow == null) pageNow = 1; + if (pageSize == null) pageSize = 10; + + // 判断是否是管理员 + boolean isAdmin = Configuration.isAdmin(user); + + // 根据管理员权限设置username + if (isAdmin) { + if (username == null) username = null; + } else { + username = user; + } + + // 分页设置 + PageHelper.startPage(pageNow, pageSize); + try { + // 执行数据库查询 + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setName(name); + pythonModuleInfo.setEngineType(engineType); + pythonModuleInfo.setCreateUser(username); + pythonModuleInfo.setIsLoad(isLoad); + pythonModuleInfo.setIsExpire(isExpire); + List pythonList = pythonModuleInfoService.getByConditions(pythonModuleInfo); + + // 封装返回结果 + return Message.ok().data("pythonList", pythonList); + } finally { + // 关闭分页 + PageHelper.clearPage(); + } + } + + /** + * Python物料删除 + * + * @param id id + * @param isExpire 0-未过期，1-已过期 + */ + @RequestMapping(path = "/python-delete", method = RequestMethod.GET) + @ApiOperation(value = "删除Python模块", notes = "根据模块ID删除Python模块��管理员可以删除任何模块，普通用户只能删除自己创建的模块") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "模块ID", required = true, dataType = "Long"), + @ApiImplicitParam( + name = "isExpire", + value = "模块是否过期（0：未过期，1：已过期）", + required = true, + dataType = "int") + }) + public Message pythonDelete( + @RequestParam(value = "id", required = false) Long id, + @RequestParam(value = "isExpire", required = false) int isExpire, + HttpServletRequest req, + HttpServletResponse resp) { + // 打印审计日志并获取登录用户 + String user = ModuleUserUtils.getOperationUser(req, "pythonDelete"); + + // 参数校验 + if (id == null || isExpire < 0 || isExpire > 1) { + return Message.error("Invalid parameters"); + } + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setId(id); + // 根据id查询Python模块信息 + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + if (moduleInfo == null) { + return Message.ok(); // 如果不存在则直接返回成功 + } + + // 判断是否是管理员 + if (!Configuration.isAdmin(user)) { + // 如果不是管理员，检查创建用户是否与当前用户一致 + if (!moduleInfo.getCreateUser().equals(user)) { + return Message.error("无权删除他人Python模块"); + } + } + + // 更新Python模块信息 + moduleInfo.setIsExpire(1); + moduleInfo.setUpdateUser(user); + moduleInfo.setUpdateTime(new Date()); + // 修改数据库中的模块名称和文件名称 + String newName = moduleInfo.getName() + "_" + System.currentTimeMillis(); + String newPath = moduleInfo.getPath() + "_" + System.currentTimeMillis(); + moduleInfo.setPath(newPath); + moduleInfo.setName(newName); + pythonModuleInfoService.updatePythonModuleInfo(moduleInfo); + return Message.ok(); + } + + /** Python物料新增/更新 */ + @ApiOperation(value = "Python物料新增/更新", notes = "根据传入的Python物料信息新增或更新") + @ApiImplicitParams({ + @ApiImplicitParam( + name = "Python物料新增/更新Request", + value = "Python物料新增/更新请求体", + required = false, + dataType = "PythonModuleInfo") + }) + @RequestMapping(value = "/python-save", method = RequestMethod.POST) + public Message request( + @RequestBody PythonModuleInfo pythonModuleInfo, + HttpServletRequest httpReq, + HttpServletResponse httpResp) { + + // 获取登录用户 + String userName = ModuleUserUtils.getOperationUser(httpReq, "pythonSave"); + + // 入参校验 + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getName())) { + return Message.error("模块名称：不能为空"); + } + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getPath())) { + return Message.error("模块物料：不能为空"); + } + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getEngineType())) { + return Message.error("引擎类型：不能为空"); + } + if (pythonModuleInfo.getIsLoad() == null) { + return Message.error("是否加载：不能为空"); + } + if (pythonModuleInfo.getIsExpire() == null) { + return Message.error("是否过期：不能为空"); + } + String path = pythonModuleInfo.getPath(); + String fileName = path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf(".")); + if (!pythonModuleInfo.getName().equals(fileName)) { + return Message.error("模块名称与物料文件名称必须一样"); + } + // 根据id判断是插入还是更新 + if (pythonModuleInfo.getId() == null) { + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + // 插入逻辑 + if (moduleInfo != null) { + return Message.error("模块" + moduleInfo.getName() + "已存在"); + } + pythonModuleInfo.setCreateTime(new Date()); + pythonModuleInfo.setUpdateTime(new Date()); + pythonModuleInfo.setCreateUser(userName); + pythonModuleInfo.setUpdateUser(userName); + pythonModuleInfoService.insertPythonModuleInfo(pythonModuleInfo); + return Message.ok().data("id", pythonModuleInfo.getId()); + } else { + PythonModuleInfo pythonModuleTmp = new PythonModuleInfo(); + pythonModuleTmp.setId(pythonModuleInfo.getId()); + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleTmp); + // 更新逻辑 + if (moduleInfo == null) { + return Message.error("未找到该Python模块"); + } + if (!Configuration.isAdmin(userName) && !userName.equals(moduleInfo.getCreateUser())) { + return Message.error("无权编辑他人Python模块"); + } + if (moduleInfo.getIsExpire() != 0) { + return Message.error("当前模块已过期，不允许进行修改操作"); + } + pythonModuleInfo.setUpdateUser(userName); + pythonModuleInfo.setUpdateTime(new Date()); + pythonModuleInfoService.updatePythonModuleInfo(pythonModuleInfo); + } + return Message.ok(); + } + + /** + * python文件是否存在查询 + * + * @param fileName 文件名称 + */ + @RequestMapping(path = "/python-file-exist", method = RequestMethod.GET) + @ApiOperation(value = "查询Python文件是否存在", notes = "根据用户名和文件名查询Python模块信息，如果存在则返回true，否则返回false") + @ApiImplicitParams({ + @ApiImplicitParam( + name = "fileName", + value = "Python文件名", + required = true, + dataType = "string", + paramType = "query"), + @ApiImplicitParam( + name = "Authorization", + value = "Bearer token", + required = true, + dataType = "string", + paramType = "header") + }) + public Message pythonFileExist( + @RequestParam("fileName") String fileName, HttpServletRequest req) { + // 审计日志打印并获取登录用户 + String userName = ModuleUserUtils.getOperationUser(req, "pythonFileExist"); + + // 参数校验 + if (org.apache.commons.lang3.StringUtils.isBlank(fileName)) { + return Message.error("参数fileName不能为空"); + } + if (!fileName.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + return Message.error("只支持数字字母下划线，且以字母开头，长度最大50"); + } + + // 封装PythonModuleInfo对象并查询数据库 + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setName(fileName); + pythonModuleInfo.setCreateUser(userName); + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + + // 根据查询结果返回相应信息 + if (moduleInfo == null) { + return Message.ok().data("result", true); + } else { + return Message.error("模块" + fileName + "已存在，如需重新上传请先删除旧的模块"); + } + } } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java new file mode 100644 index 0000000000..3326a3c367 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.dao; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; + +@Mapper +public interface PythonModuleInfoMapper { + + // SQL 1: 模糊查询 + List selectByConditions(PythonModuleInfo pythonModuleInfo); + + // SQL 2: 更新 + int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 3: 新增 + int insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 4: 带有判断的查询 + PythonModuleInfo selectByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); + + // SQL 5: 查询包含多个引擎类型的hdfs路径 + List selectPathsByUsernameAndEnginetypes(String username, List enginetypes); +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java new file mode 100644 index 0000000000..727b323cb6 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.entity; + +import java.util.Date; + +public class PythonModuleInfo { + private Long id; + private String name; + private String description; + private String path; + private String engineType; + private String createUser; + private String updateUser; + private Integer isLoad; + private Integer isExpire; + private Date createTime; + private Date updateTime; + + public PythonModuleInfo() {} + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Integer getIsLoad() { + return isLoad; + } + + public void setIsLoad(Integer isLoad) { + this.isLoad = isLoad; + } + + public Integer getIsExpire() { + return isExpire; + } + + public void setIsExpire(Integer isExpire) { + this.isExpire = isExpire; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "PythonModuleInfo{" + + "id=" + + id + + ", name='" + + name + + '\'' + + ", description='" + + description + + '\'' + + ", path='" + + path + + '\'' + + ", engineType='" + + engineType + + '\'' + + ", createUser='" + + createUser + + '\'' + + ", updateUser='" + + updateUser + + '\'' + + ", isLoad=" + + isLoad + + ", isExpire=" + + isExpire + + ", createTime=" + + createTime + + ", updateTime=" + + updateTime + + '}'; + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java new file mode 100644 index 0000000000..dce8fa864a --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import java.util.List; + +public interface PythonModuleInfoService { + + // SQL 1: 模糊查询 + List getByConditions(PythonModuleInfo pythonModuleInfo); + + // SQL 2: 更新 + int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 3: 新增 + int insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 4: 带有判断的查询 + PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); + + // SQL 5: 查询包含多个引擎类型的hdfs路径 + List getPathsByUsernameAndEnginetypes(String username, List enginetypes); +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java new file mode 100644 index 0000000000..6180ecd849 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service.impl; + +import org.apache.linkis.udf.dao.PythonModuleInfoMapper; +import org.apache.linkis.udf.entity.PythonModuleInfo; +import org.apache.linkis.udf.service.PythonModuleInfoService; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class PythonModuleInfoServiceImpl implements PythonModuleInfoService { + + private final PythonModuleInfoMapper pythonModuleInfoMapper; + + @Autowired + public PythonModuleInfoServiceImpl(PythonModuleInfoMapper pythonModuleInfoMapper) { + this.pythonModuleInfoMapper = pythonModuleInfoMapper; + } + + @Override + public List getByConditions(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); + } + + @Override + public int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); + } + + @Override + public int insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); + } + + @Override + public PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); + } + + @Override + public List getPathsByUsernameAndEnginetypes(String username, List enginetypes) { + return pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml new file mode 100644 index 0000000000..66967b8053 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml @@ -0,0 +1,90 @@ + + + + + + + + + + + + UPDATE linkis_ps_python_module_info + + name = #{name}, + description = #{description}, + path = #{path}, + engine_type = #{engineType}, + create_user = #{createUser}, + update_user = #{updateUser}, + is_load = #{isLoad}, + is_expire = #{isExpire}, + create_time = #{createTime}, + update_time = #{updateTime}, + + WHERE id = #{id} + + + + + INSERT INTO linkis_ps_python_module_info + (name, description, path, engine_type, create_user, update_user, is_load, is_expire, create_time, update_time) + VALUES + (#{name}, #{description}, #{path}, #{engineType}, #{createUser}, #{updateUser}, #{isLoad}, #{isExpire}, #{createTime}, #{updateTime}) + + + + + + + + + + From cdf741ce49d6441f14eb05fc27f0af8813e262f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Sat, 24 Aug 2024 15:36:02 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=94=B1AI=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E7=94=9F=E6=88=90=E6=8E=A5=E5=8F=A3=E5=92=8CAI?= =?UTF-8?q?=E6=8F=92=E4=BB=B6=E7=94=9F=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../linkis/udf/api/rpc/UdfReceiver.scala | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index 501e4d3bc3..227e8ebfd6 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -18,10 +18,9 @@ package org.apache.linkis.udf.api.rpc import org.apache.linkis.rpc.{Receiver, Sender} -import org.apache.linkis.udf.service.{UDFService, UDFTreeService} +import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} import java.lang - import scala.concurrent.duration.Duration class UdfReceiver extends Receiver { @@ -38,6 +37,17 @@ class UdfReceiver extends Receiver { override def receive(message: Any, sender: Sender): Unit = {} + // 注⼊PythonModuleInfoService + val pythonModuleInfoService: PythonModuleInfoService = context.system.asInstanceOf[ExtendedActorSystem].lifecycle.systemManager.asInstanceOf[SystemManager].pythonModuleInfoService + + def parseModuleInfoFromPath(path: String): PythonModuleInfoVO = { + // 假设路径格式为 "username/module_name/module_version" + val parts = path.split("/") + var vo = PythonModuleInfoVO() + vo.setPath(path) + vo + } + override def receiveAndReply(message: Any, sender: Sender): Any = { message match { case RequestUdfTree(userName, treeType, treeId, treeCategory) => @@ -46,10 +56,16 @@ class UdfReceiver extends Receiver { case RequestUdfIds(userName, udfIds, treeCategory) => val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) + case RequestPythonModuleProtocol(userName, engineTypes) => + // 获取Python模块路径列表 + val paths = pythonModuleInfoService.getPathsByUsernameAndEnginetypes(userName, engineTypes) + // 将路径列表转换为PythonModuleInfo列表 + val pythonModuleInfoList = paths.map(parseModuleInfoFromPath) + new ResponsePythonModuleProtocol(pythonModuleInfoList) case _ => } } override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} -} +} \ No newline at end of file From 37f86b16dd64f57ffcc881206e903c9c6305a63f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Sat, 24 Aug 2024 17:42:52 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=94=B1=E4=BA=BA?= =?UTF-8?q?=E5=B7=A5=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../filesystem/restful/api/FsRestfulApi.java | 31 +++++++++++------- .../apache/linkis/udf/api/UDFRestfulApi.java | 32 ++++++++++--------- .../udf/dao/PythonModuleInfoMapper.java | 5 +-- .../udf/service/PythonModuleInfoService.java | 5 +-- .../impl/PythonModuleInfoServiceImpl.java | 5 +-- .../mapper/common/PythonModuleInfoMapper.xml | 3 ++ .../linkis/udf/api/rpc/UdfReceiver.scala | 30 ++++++++--------- 7 files changed, 63 insertions(+), 48 deletions(-) diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 7dd01e8c36..db4a46f375 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -1425,25 +1425,32 @@ public Message encryptPath( @ApiOperation(value = "Python模块上传", notes = "上传Python模块文件并返回文件地址", response = Message.class) @ApiImplicitParams({ - @ApiImplicitParam(name = "file", required = true, dataType = "MultipartFile", value = "上传的文件"), - @ApiImplicitParam(name = "fileName", required = true, dataType = "String", value = "文件名称") + @ApiImplicitParam(name = "file", required = true, dataType = "MultipartFile", value = "上传的文件"), + @ApiImplicitParam(name = "fileName", required = true, dataType = "String", value = "文件名称") }) - @RequestMapping(path = "/filesystem/python-upload", method = RequestMethod.POST) + @RequestMapping(path = "/python-upload", method = RequestMethod.POST) public Message pythonUpload( - HttpServletRequest req, - @RequestParam("file") MultipartFile file, - @RequestParam("fileName") String fileName) throws IOException { + HttpServletRequest req, + @RequestParam("file") MultipartFile file, + @RequestParam(value = "fileName", required = false) String fileName) + throws WorkSpaceException, IOException { // 获取登录用户 String username = ModuleUserUtils.getOperationUser(req, "pythonUpload"); // 校验文件名称 - if (!fileName.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + if (StringUtils.isBlank(fileName)) { + return Message.error("文件名称不能为空"); + } + // 获取文件名称 + String fileNameSuffix = fileName.substring(0, fileName.lastIndexOf(".")); + if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50"); } // 校验文件类型 - if (!file.getOriginalFilename().endsWith(".py") && !file.getOriginalFilename().endsWith(".zip")) { + if (!file.getOriginalFilename().endsWith(".py") + && !file.getOriginalFilename().endsWith(".zip")) { return Message.error("仅支持.py和.zip格式模块文件"); } @@ -1453,7 +1460,7 @@ public Message pythonUpload( } // 定义目录路径 - String path = "/appcom/linkis/udf/" + username; + String path = "hdfs:///appcom/linkis/udf/" + username; FsPath fsPath = new FsPath(path); // 获取文件系统实例 @@ -1462,19 +1469,19 @@ public Message pythonUpload( // 确认目录是否存在，不存在则创建新目录 if (!fileSystem.exists(fsPath)) { try { - fileSystem.createNewFile(fsPath); + fileSystem.mkdirs(fsPath); } catch (IOException e) { return Message.error("创建目录失败：" + e.getMessage()); } } // 构建新的文件路径 - String newPath = fsPath.getPath() + "/" + fileName; + String newPath = fsPath.getPath() + "/" + file.getOriginalFilename(); FsPath fsPathNew = new FsPath(newPath); // 上传文件 try (InputStream is = file.getInputStream(); - OutputStream outputStream = fileSystem.write(fsPathNew, true)) { + OutputStream outputStream = fileSystem.write(fsPathNew, true)) { IOUtils.copy(is, outputStream); } catch (IOException e) { return Message.error("文件上传失败：" + e.getMessage()); diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 064f2d5390..2bebc661dd 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -42,6 +42,7 @@ import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; +import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -1067,16 +1068,13 @@ public Message pythonList( String user = ModuleUserUtils.getOperationUser(req, "pythonList"); // 参数校验 - if (name == null) name = null; - if (engineType == null) engineType = null; + if (org.apache.commons.lang3.StringUtils.isBlank(name)) name = null; + if (org.apache.commons.lang3.StringUtils.isBlank(engineType)) engineType = null; if (pageNow == null) pageNow = 1; if (pageSize == null) pageSize = 10; - // 判断是否是管理员 - boolean isAdmin = Configuration.isAdmin(user); - // 根据管理员权限设置username - if (isAdmin) { + if (Configuration.isAdmin(user)) { if (username == null) username = null; } else { username = user; @@ -1093,9 +1091,9 @@ public Message pythonList( pythonModuleInfo.setIsLoad(isLoad); pythonModuleInfo.setIsExpire(isExpire); List pythonList = pythonModuleInfoService.getByConditions(pythonModuleInfo); - + PageInfo pageInfo = new PageInfo<>(pythonList); // 封装返回结果 - return Message.ok().data("pythonList", pythonList); + return Message.ok().data("pythonList", pythonList).data("totalPage", pageInfo.getTotal()); } finally { // 关闭分页 PageHelper.clearPage(); @@ -1109,7 +1107,7 @@ public Message pythonList( * @param isExpire 0-未过期，1-已过期 */ @RequestMapping(path = "/python-delete", method = RequestMethod.GET) - @ApiOperation(value = "删除Python模块", notes = "根据模块ID删除Python模块��管理员可以删除任何模块，普通用户只能删除自己创建的模块") + @ApiOperation(value = "删除Python模块", notes = "根据模块ID删除Python模块,管理员可以删除任何模块，普通用户只能删除自己创建的模块") @ApiImplicitParams({ @ApiImplicitParam(name = "id", value = "模块ID", required = true, dataType = "Long"), @ApiImplicitParam( @@ -1127,8 +1125,11 @@ public Message pythonDelete( String user = ModuleUserUtils.getOperationUser(req, "pythonDelete"); // 参数校验 - if (id == null || isExpire < 0 || isExpire > 1) { - return Message.error("Invalid parameters"); + if (id == null) { + return Message.error("Invalid parameters: id is null"); + } + if (isExpire != 0 && isExpire != 1) { + return Message.error("Invalid parameters: isExpire must be 0 or 1"); } PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); pythonModuleInfo.setId(id); @@ -1170,7 +1171,7 @@ public Message pythonDelete( }) @RequestMapping(value = "/python-save", method = RequestMethod.POST) public Message request( - @RequestBody PythonModuleInfo pythonModuleInfo, + @Nullable @RequestBody PythonModuleInfo pythonModuleInfo, HttpServletRequest httpReq, HttpServletResponse httpResp) { @@ -1254,7 +1255,7 @@ public Message request( paramType = "header") }) public Message pythonFileExist( - @RequestParam("fileName") String fileName, HttpServletRequest req) { + @RequestParam(value = "fileName", required = false) String fileName, HttpServletRequest req) { // 审计日志打印并获取登录用户 String userName = ModuleUserUtils.getOperationUser(req, "pythonFileExist"); @@ -1262,13 +1263,14 @@ public Message pythonFileExist( if (org.apache.commons.lang3.StringUtils.isBlank(fileName)) { return Message.error("参数fileName不能为空"); } - if (!fileName.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + String fileNameWithoutExtension = fileName.substring(0, fileName.lastIndexOf(".")); + if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { return Message.error("只支持数字字母下划线，且以字母开头，长度最大50"); } // 封装PythonModuleInfo对象并查询数据库 PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); - pythonModuleInfo.setName(fileName); + pythonModuleInfo.setName(fileNameWithoutExtension); pythonModuleInfo.setCreateUser(userName); PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java index 3326a3c367..f32432ef18 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java @@ -33,11 +33,12 @@ public interface PythonModuleInfoMapper { int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); // SQL 3: 新增 - int insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); // SQL 4: 带有判断的查询 PythonModuleInfo selectByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); // SQL 5: 查询包含多个引擎类型的hdfs路径 - List selectPathsByUsernameAndEnginetypes(String username, List enginetypes); + List selectPathsByUsernameAndEnginetypes( + String username, List enginetypes); } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java index dce8fa864a..0e94309baf 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java @@ -30,11 +30,12 @@ public interface PythonModuleInfoService { int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); // SQL 3: 新增 - int insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); // SQL 4: 带有判断的查询 PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); // SQL 5: 查询包含多个引擎类型的hdfs路径 - List getPathsByUsernameAndEnginetypes(String username, List enginetypes); + List getPathsByUsernameAndEnginetypes( + String username, List enginetypes); } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java index 6180ecd849..91f5839416 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java @@ -47,7 +47,7 @@ public int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo) { } @Override - public int insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo) { + public Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo) { return pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); } @@ -57,7 +57,8 @@ public PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo) } @Override - public List getPathsByUsernameAndEnginetypes(String username, List enginetypes) { + public List getPathsByUsernameAndEnginetypes( + String username, List enginetypes) { return pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); } } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml index 66967b8053..0b513ba2f2 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml +++ b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml @@ -64,6 +64,9 @@ (name, description, path, engine_type, create_user, update_user, is_load, is_expire, create_time, update_time) VALUES (#{name}, #{description}, #{path}, #{engineType}, #{createUser}, #{updateUser}, #{isLoad}, #{isExpire}, #{createTime}, #{updateTime}) + + SELECT LAST_INSERT_ID() + diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index 227e8ebfd6..662952f5d4 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -38,15 +38,15 @@ class UdfReceiver extends Receiver { override def receive(message: Any, sender: Sender): Unit = {} // 注⼊PythonModuleInfoService - val pythonModuleInfoService: PythonModuleInfoService = context.system.asInstanceOf[ExtendedActorSystem].lifecycle.systemManager.asInstanceOf[SystemManager].pythonModuleInfoService - - def parseModuleInfoFromPath(path: String): PythonModuleInfoVO = { - // 假设路径格式为 "username/module_name/module_version" - val parts = path.split("/") - var vo = PythonModuleInfoVO() - vo.setPath(path) - vo - } +// val pythonModuleInfoService: PythonModuleInfoService = context.system.asInstanceOf[ExtendedActorSystem].lifecycle.systemManager.asInstanceOf[SystemManager].pythonModuleInfoService +// +// def parseModuleInfoFromPath(path: String): PythonModuleInfoVO = { +// // 假设路径格式为 "username/module_name/module_version" +// val parts = path.split("/") +// var vo = PythonModuleInfoVO() +// vo.setPath(path) +// vo +// } override def receiveAndReply(message: Any, sender: Sender): Any = { message match { @@ -56,12 +56,12 @@ class UdfReceiver extends Receiver { case RequestUdfIds(userName, udfIds, treeCategory) => val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) - case RequestPythonModuleProtocol(userName, engineTypes) => - // 获取Python模块路径列表 - val paths = pythonModuleInfoService.getPathsByUsernameAndEnginetypes(userName, engineTypes) - // 将路径列表转换为PythonModuleInfo列表 - val pythonModuleInfoList = paths.map(parseModuleInfoFromPath) - new ResponsePythonModuleProtocol(pythonModuleInfoList) +// case RequestPythonModuleProtocol(userName, engineTypes) => +// // 获取Python模块路径列表 +// val paths = pythonModuleInfoService.getPathsByUsernameAndEnginetypes(userName, engineTypes) +// // 将路径列表转换为PythonModuleInfo列表 +// val pythonModuleInfoList = paths.map(parseModuleInfoFromPath) +// new ResponsePythonModuleProtocol(pythonModuleInfoList) case _ => } } From 462509b91a0d17993edd653b80c1279f91219c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Sat, 24 Aug 2024 22:46:18 +0800 Subject: [PATCH 4/5] bug fix --- .../apache/linkis/udf/api/UDFRestfulApi.java | 2 +- .../service/impl/PythonModuleInfoMapper.java | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index 2bebc661dd..0db971d6f5 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -1201,6 +1201,7 @@ public Message request( } // 根据id判断是插入还是更新 if (pythonModuleInfo.getId() == null) { + pythonModuleInfo.setCreateUser(userName); PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); // 插入逻辑 if (moduleInfo != null) { @@ -1208,7 +1209,6 @@ public Message request( } pythonModuleInfo.setCreateTime(new Date()); pythonModuleInfo.setUpdateTime(new Date()); - pythonModuleInfo.setCreateUser(userName); pythonModuleInfo.setUpdateUser(userName); pythonModuleInfoService.insertPythonModuleInfo(pythonModuleInfo); return Message.ok().data("id", pythonModuleInfo.getId()); diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java new file mode 100644 index 0000000000..d2868cce2f --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service.impl; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface PythonModuleInfoMapper { + + // SQL 1: 模糊查询 + List selectByConditions(PythonModuleInfo pythonModuleInfo); + + // SQL 2: 更新 + int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 3: 新增 + Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 4: 带有判断的查询 + PythonModuleInfo selectByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); + + // SQL 5: 查询包含多个引擎类型的hdfs路径 + List selectPathsByUsernameAndEnginetypes( + @Param("username") String username, @Param("enginetypes") List enginetypes); +} From 39de399081a542570281b50e761fbbcddc75efb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cv=5Fkkhuang=E2=80=9D?= <“420895376@qq.com”> Date: Sat, 24 Aug 2024 22:47:14 +0800 Subject: [PATCH 5/5] bug fix --- .../udf/dao/PythonModuleInfoMapper.java | 6 +-- .../service/impl/PythonModuleInfoMapper.java | 45 ------------------- 2 files changed, 3 insertions(+), 48 deletions(-) delete mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java index f32432ef18..0f52960e17 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java @@ -17,9 +17,9 @@ package org.apache.linkis.udf.dao; -import org.apache.linkis.udf.entity.PythonModuleInfo; - import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.linkis.udf.entity.PythonModuleInfo; import java.util.List; @@ -40,5 +40,5 @@ public interface PythonModuleInfoMapper { // SQL 5: 查询包含多个引擎类型的hdfs路径 List selectPathsByUsernameAndEnginetypes( - String username, List enginetypes); + @Param("username") String username, @Param("enginetypes") List enginetypes); } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java deleted file mode 100644 index d2868cce2f..0000000000 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoMapper.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.udf.service.impl; - -import org.apache.linkis.udf.entity.PythonModuleInfo; - -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -@Mapper -public interface PythonModuleInfoMapper { - - // SQL 1: 模糊查询 - List selectByConditions(PythonModuleInfo pythonModuleInfo); - - // SQL 2: 更新 - int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); - - // SQL 3: 新增 - Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); - - // SQL 4: 带有判断的查询 - PythonModuleInfo selectByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); - - // SQL 5: 查询包含多个引擎类型的hdfs路径 - List selectPathsByUsernameAndEnginetypes( - @Param("username") String username, @Param("enginetypes") List enginetypes); -}