From d221ce5622356738290c179f9fd5d7cc8525365d Mon Sep 17 00:00:00 2001 From: aiceflower Date: Fri, 30 Aug 2024 11:00:12 +0800 Subject: [PATCH] merge 1.7.1 to 1.8.0 (#592) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Dev 1.7.1 add python module load hook (#584) * ai generate code init * 人工修改代码 * 人工修改提交 * 【1.7.1】python物料管理 (#583) * 代码由AI自动生成接口和AI插件生成 * 代码由AI自动生成接口和AI插件生成 * 代码由人工修改 --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> * 人工修改 code format and add rpc func * fix bug * Dev 1.7.0 python udf manager (#586) * 代码由AI自动生成接口和AI插件生成 * 代码由AI自动生成接口和AI插件生成 * 代码由人工修改 * bug fix * bug fix --------- Co-authored-by: “v_kkhuang” <“420895376@qq.com”> * fix bug * code format * 使用ai生成单元测试案例 * 人工修改单元测试 * 人工修改单元测试 * Dev 1.7.1 webank test (#587) * 使用ai生成单元测试案例 * 人工修改单元测试 * 人工修改单元测试 * add instance info * update file permission (#590) Co-authored-by: “v_kkhuang” <“420895376@qq.com”> * chore: 1.7.1 (#591) * chore: 1.7.1 * upd: config file --------- Co-authored-by: v-kkhuang <62878639+v-kkhuang@users.noreply.github.com> Co-authored-by: “v_kkhuang” <“420895376@qq.com”> Co-authored-by: Yonghao Mei <73584269+mayinrain@users.noreply.github.com> --- .../linkis/common/conf/Configuration.scala | 2 + .../executor/hook/PythonModuleLoad.scala | 161 ++++ .../hook/PythonModuleLoadEngineConnHook.scala | 64 ++ .../executor/hook/PythonSparkEngineHook.scala | 45 + .../PythonModuleLoadEngineConnHookTest.scala | 82 ++ .../executor/hook/PythonModuleLoadTest.scala | 63 ++ .../hook/PythonSparkEngineHookTest.scala | 61 ++ .../resources/linkis-engineconn.properties | 2 +- .../linkis/jobhistory/util/QueryUtils.scala | 6 +- .../linkis/udf/entity/PythonModuleInfoVO.java | 209 +++++ .../api/rpc/RequestPythonModuleProtocol.scala | 28 + .../rpc/ResponsePythonModuleProtocol.scala | 33 + .../filesystem/restful/api/FsRestfulApi.java | 67 ++ .../apache/linkis/udf/api/UDFRestfulApi.java | 280 +++++++ .../udf/dao/PythonModuleInfoMapper.java | 45 + .../linkis/udf/entity/PythonModuleInfo.java | 158 ++++ .../udf/service/PythonModuleInfoService.java | 41 + .../impl/PythonModuleInfoServiceImpl.java | 64 ++ .../mapper/common/PythonModuleInfoMapper.xml | 93 +++ .../linkis/udf/api/rpc/UdfReceiver.scala | 52 +- .../udf/api/rpc/UdfReceiverChooser.scala | 9 +- .../udf/api/PythonModuleRestfulApiTest.java | 132 +++ .../udf/dao/PythonModuleInfoMapperTest.java | 113 +++ .../service/PythonModuleInfoServiceTest.java | 129 +++ linkis-web/.gitignore | 3 +- linkis-web/package.json | 25 +- linkis-web/src/apps/PythonModule/.env | 0 linkis-web/src/apps/PythonModule/.fes.js | 13 + linkis-web/src/apps/PythonModule/.fes.prod.js | 4 + linkis-web/src/apps/PythonModule/.fes.test.js | 4 + .../src/apps/PythonModule/.gitattributes | 9 + linkis-web/src/apps/PythonModule/.npmrc | 2 + linkis-web/src/apps/PythonModule/index.html | 15 + linkis-web/src/apps/PythonModule/package.json | 31 + .../src/apps/PythonModule/public/logo.svg | 33 + .../PythonModule/src/.fes/configType.d.ts | 5 + .../PythonModule/src/.fes/core/coreExports.js | 19 + .../apps/PythonModule/src/.fes/core/plugin.js | 7 + .../src/.fes/core/pluginExports.js | 2 + .../src/.fes/core/pluginRegister.js | 25 + .../src/.fes/core/routes/routeExports.js | 105 +++ .../src/.fes/core/routes/routes.js | 20 + .../src/.fes/core/routes/runtime.js | 6 + .../src/.fes/defaultContainer.vue | 3 + .../src/apps/PythonModule/src/.fes/fes.js | 65 ++ .../PythonModule/src/.fes/initialState.js | 7 + .../src/.fes/plugin-model/core.js | 24 + .../.fes/plugin-model/models/initialState.js | 5 + .../src/.fes/plugin-request/request.js | 69 ++ .../src/apps/PythonModule/src/global.less | 11 + .../PythonModule/src/letgo/cache-control.js | 232 ++++++ .../src/letgo/components/pageLoading.vue | 24 + .../apps/PythonModule/src/letgo/global.css | 4 + .../apps/PythonModule/src/letgo/globalBase.js | 21 + .../PythonModule/src/letgo/letgoConstants.js | 16 + .../PythonModule/src/letgo/letgoRequest.js | 2 + .../apps/PythonModule/src/letgo/pageBase.js | 9 + .../apps/PythonModule/src/letgo/reactive.js | 74 ++ .../src/apps/PythonModule/src/letgo/shared.js | 36 + .../PythonModule/src/letgo/useComputed.js | 15 + .../PythonModule/src/letgo/useInstance.js | 24 + .../apps/PythonModule/src/letgo/useJSQuery.js | 124 +++ .../PythonModule/src/letgo/useLetgoGlobal.js | 78 ++ .../src/letgo/useTemporaryState.js | 23 + .../PythonModule/src/pages/index/index.jsx | 781 ++++++++++++++++++ .../apps/PythonModule/src/pages/index/main.js | 385 +++++++++ .../src/apps/PythonModule/tsconfig.json | 33 + .../src/apps/linkis/components/tag/index.vue | 6 +- .../src/apps/linkis/i18n/common/en.json | 14 + .../src/apps/linkis/i18n/common/zh.json | 14 + .../src/apps/linkis/module/ECM/index.scss | 8 +- .../apps/linkis/module/codeQuery/index.scss | 2 +- .../apps/linkis/module/codeQuery/index.vue | 2 +- .../module/globalHistoryManagement/index.vue | 208 ++++- .../globalHistoryManagement/viewHistory.vue | 95 ++- .../module/microServiceManagement/index.scss | 4 +- .../apps/linkis/module/pythonModule/index.js | 23 + .../apps/linkis/module/pythonModule/index.vue | 13 + .../module/resourceManagement/index.scss | 12 +- .../apps/linkis/module/setting/setting.vue | 36 +- .../linkis/module/udfTree/EditForm/index.vue | 2 +- linkis-web/src/apps/linkis/router.js | 10 + linkis-web/src/apps/linkis/view/layout.vue | 23 + .../src/apps/linkis/view/linkis/index.vue | 82 +- .../components/consoleComponent/result.vue | 2 +- .../components/consoleComponent/toolbar.vue | 2 +- linkis-web/src/dss/module/header/index.scss | 2 +- .../src/dss/module/resourceSimple/job.vue | 54 +- linkis-web/src/router.js | 46 +- 89 files changed, 4853 insertions(+), 139 deletions(-) create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala create mode 100644 linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala create mode 100644 linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java create mode 100644 linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java create mode 100644 linkis-web/src/apps/PythonModule/.env create mode 100644 linkis-web/src/apps/PythonModule/.fes.js create mode 100644 linkis-web/src/apps/PythonModule/.fes.prod.js create mode 100644 linkis-web/src/apps/PythonModule/.fes.test.js create mode 100644 linkis-web/src/apps/PythonModule/.gitattributes create mode 100644 linkis-web/src/apps/PythonModule/.npmrc create mode 100644 linkis-web/src/apps/PythonModule/index.html create mode 100644 linkis-web/src/apps/PythonModule/package.json create mode 100644 linkis-web/src/apps/PythonModule/public/logo.svg create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/configType.d.ts create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/core/coreExports.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/core/plugin.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/core/pluginExports.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/core/pluginRegister.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/core/routes/routeExports.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/core/routes/routes.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/core/routes/runtime.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/defaultContainer.vue create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/fes.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/initialState.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/plugin-model/core.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/plugin-model/models/initialState.js create mode 100644 linkis-web/src/apps/PythonModule/src/.fes/plugin-request/request.js create mode 100644 linkis-web/src/apps/PythonModule/src/global.less create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/cache-control.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/components/pageLoading.vue create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/global.css create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/globalBase.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/letgoConstants.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/letgoRequest.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/pageBase.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/reactive.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/shared.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/useComputed.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/useInstance.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/useJSQuery.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/useLetgoGlobal.js create mode 100644 linkis-web/src/apps/PythonModule/src/letgo/useTemporaryState.js create mode 100644 linkis-web/src/apps/PythonModule/src/pages/index/index.jsx create mode 100644 linkis-web/src/apps/PythonModule/src/pages/index/main.js create mode 100644 linkis-web/src/apps/PythonModule/tsconfig.json create mode 100644 linkis-web/src/apps/linkis/module/pythonModule/index.js create mode 100644 linkis-web/src/apps/linkis/module/pythonModule/index.vue diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala index 417c377038..163d7aa4db 100644 --- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala +++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala @@ -70,6 +70,8 @@ object Configuration extends Logging { val VARIABLE_OPERATION: Boolean = CommonVars("wds.linkis.variable.operation", false).getValue + val IS_VIEW_FS_ENV = CommonVars("wds.linkis.env.is.viewfs", true) + val ERROR_MSG_TIP = CommonVars( "linkis.jobhistory.error.msg.tip", diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala new file mode 100644 index 0000000000..34928d8525 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoad.scala @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf +import org.apache.linkis.engineconn.computation.executor.execute.{ + ComputationExecutor, + EngineExecutionContext +} +import org.apache.linkis.engineconn.core.engineconn.EngineConnManager +import org.apache.linkis.engineconn.core.executor.ExecutorManager +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.rpc.Sender +import org.apache.linkis.udf.UDFClientConfiguration +import org.apache.linkis.udf.api.rpc.{RequestPythonModuleProtocol, ResponsePythonModuleProtocol} +import org.apache.linkis.udf.entity.PythonModuleInfoVO + +import org.apache.commons.lang3.StringUtils + +import java.util + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** + * The PythonModuleLoad class is designed to load Python modules into the execution environment + * dynamically. This class is not an extension of UDFLoad, but shares a similar philosophy of + * handling dynamic module loading based on user preferences and system configurations. + */ +abstract class PythonModuleLoad extends Logging { + + /** Abstract properties to be defined by the subclass */ + protected val engineType: String + protected val runType: RunType + + protected def getEngineType(): String = engineType + + protected def constructCode(pythonModuleInfo: PythonModuleInfoVO): String + + private def queryPythonModuleRpc( + userName: String, + engineType: String + ): java.util.List[PythonModuleInfoVO] = { + val infoList = Sender + .getSender(UDFClientConfiguration.UDF_SERVICE_NAME.getValue) + .ask(RequestPythonModuleProtocol(userName, engineType)) + .asInstanceOf[ResponsePythonModuleProtocol] + .getModulesInfo() + infoList + } + + protected def getLoadPythonModuleCode: Array[String] = { + val engineCreationContext = + EngineConnManager.getEngineConnManager.getEngineConn.getEngineCreationContext + val user = engineCreationContext.getUser + + var infoList: util.List[PythonModuleInfoVO] = + Utils.tryAndWarn(queryPythonModuleRpc(user, getEngineType())) + if (infoList == null) { + logger.info("rpc get info is empty.") + infoList = new util.ArrayList[PythonModuleInfoVO]() + } + + // 替换Viewfs + if (IS_VIEW_FS_ENV.getValue) { + infoList.asScala.foreach { info => + val path = info.getPath + logger.info(s"python path: ${path}") + if (path.startsWith("hdfs") || path.startsWith("viewfs")) { + info.setPath(path.replace("hdfs://", "viewfs://")) + } else { + info.setPath("viewfs://" + path) + } + } + } else { + + infoList.asScala.foreach { info => + val path = info.getPath + logger.info(s"hdfs python path: ${path}") + if (!path.startsWith("hdfs")) { + info.setPath("hdfs://" + path) + } + } + } + + logger.info(s"${user} load python modules: ") + infoList.asScala.foreach(l => logger.info(s"module name:${l.getName}, path:${l.getPath}\n")) + + // 创建加载code + val codes: mutable.Buffer[String] = infoList.asScala + .filter { info => StringUtils.isNotEmpty(info.getPath) } + .map(constructCode) + // 打印codes + val str: String = codes.mkString("\n") + logger.info(s"python codes: $str") + codes.toArray + } + + private def executeFunctionCode(codes: Array[String], executor: ComputationExecutor): Unit = { + if (null == codes || null == executor) { + return + } + codes.foreach { code => + logger.info("Submit function registration to engine, code: " + code) + Utils.tryCatch(executor.executeLine(new EngineExecutionContext(executor), code)) { + t: Throwable => + logger.error("Failed to load python module", t) + null + } + } + } + + /** + * Generate and execute the code necessary for loading Python modules. + * + * @param executor + * An object capable of executing code in the current engine context. + */ + protected def loadPythonModules(labels: Array[Label[_]]): Unit = { + + val codes = getLoadPythonModuleCode + logger.info(s"codes length: ${codes.length}") + if (null != codes && codes.nonEmpty) { + val executor = ExecutorManager.getInstance.getExecutorByLabels(labels) + if (executor != null) { + val className = executor.getClass.getName + logger.info(s"executor class: ${className}") + } else { + logger.error(s"Failed to load python, executor is null") + } + + executor match { + case computationExecutor: ComputationExecutor => + executeFunctionCode(codes, computationExecutor) + case _ => + } + } + logger.info(s"Successful to load python, engineType : ${engineType}") + } + +} + +// Note: The actual implementation of methods like `executeFunctionCode` and `construct diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala new file mode 100644 index 0000000000..80eaa888b8 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHook.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.common.hook.EngineConnHook +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel + +abstract class PythonModuleLoadEngineConnHook + extends PythonModuleLoad + with EngineConnHook + with Logging { + + override def afterExecutionExecute( + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): Unit = { + Utils.tryAndWarnMsg { + val codeLanguageLabel = new CodeLanguageLabel + codeLanguageLabel.setCodeType(runType.toString) + logger.info(s"engineType: ${engineType}") + val labels = Array[Label[_]](codeLanguageLabel) + loadPythonModules(labels) + }(s"Failed to load Python Modules: ${engineType}") + + } + + override def afterEngineServerStartFailed( + engineCreationContext: EngineCreationContext, + throwable: Throwable + ): Unit = { + logger.error(s"Failed to start Engine Server: ${throwable.getMessage}", throwable) + } + + override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = { + logger.info("Preparing to load Python Module...") + } + + override def beforeExecutionExecute( + engineCreationContext: EngineCreationContext, + engineConn: EngineConn + ): Unit = { + logger.info(s"Before executing command on load Python Module.") + } + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala new file mode 100644 index 0000000000..0fe554f93d --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHook.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.manager.label.entity.engine.RunType +import org.apache.linkis.manager.label.entity.engine.RunType.RunType +import org.apache.linkis.udf.entity.PythonModuleInfoVO + +/** + * 定义一个用于Spark引擎的Python模块加载与执行挂钩的类 + */ +class PythonSparkEngineHook extends PythonModuleLoadEngineConnHook { + + // 设置engineType属性为"spark"，表示此挂钩适用于Spark数据处理引擎 + override val engineType: String = "spark" + + // 设置runType属性为RunType.PYSPARK，表示此挂钩将执行PySpark类型的代码 + override protected val runType: RunType = RunType.PYSPARK + + // 重写constructCode方法，用于根据Python模块信息构造加载模块的代码 + override protected def constructCode(pythonModuleInfo: PythonModuleInfoVO): String = { + // 使用pythonModuleInfo的path属性，构造SparkContext.addPyFile的命令字符串 + // 这个命令在PySpark环境中将模块文件添加到所有worker上，以便在代码中可以使用 + val path: String = pythonModuleInfo.getPath + val loadCode = s"sc.addPyFile('${path}')" + logger.info(s"pythonLoadCode: ${loadCode}") + loadCode + } + +} diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala new file mode 100644 index 0000000000..e507a7b22f --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadEngineConnHookTest.scala @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.engineconn.common.creation.{DefaultEngineCreationContext, EngineCreationContext} +import org.apache.linkis.engineconn.common.engineconn.DefaultEngineConn +import org.apache.linkis.manager.label.entity.engine.CodeLanguageLabel +import org.junit.jupiter.api.Test +import org.mockito.Mockito.{mock, verify, when} + + +// 单元测试案例 +class PythonModuleLoadEngineConnHookTest { + + @Test + def testAfterExecutionExecute(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = new DefaultEngineCreationContext + val mockEngineConn = mock[DefaultEngineConn] + val hook = new PythonSparkEngineHook + + // 设置模拟行为 + var labels = new CodeLanguageLabel + labels.setCodeType("spark") + + // 执行测试方法 + hook.afterExecutionExecute(mockEngineCreationContext, mockEngineConn) + + } + + @Test + def testAfterEngineServerStartFailed(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = mock[EngineCreationContext] + val mockThrowable = mock[Throwable] + val hook = new PythonSparkEngineHook + + // 设置模拟行为 + var labels = new CodeLanguageLabel + labels.setCodeType("spark") + + // 执行测试方法 + hook.afterEngineServerStartFailed(mockEngineCreationContext, mockThrowable) + + } + + @Test + def testBeforeCreateEngineConn(): Unit = { + // 创建模拟对象 + + // 验证调用 + + } + + @Test + def testBeforeExecutionExecute(): Unit = { + // 创建模拟对象 + val mockEngineCreationContext = mock[EngineCreationContext] + val mockEngineConn = mock[DefaultEngineConn] + val hook = new PythonSparkEngineHook + + // 执行测试方法 + hook.beforeExecutionExecute(mockEngineCreationContext, mockEngineConn) + + + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala new file mode 100644 index 0000000000..18970a593b --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonModuleLoadTest.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconn.computation.executor.hook + +import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.junit.jupiter.api.Test + +import java.util + +class PythonModuleLoadTest { + /** + * 测试getEngineType方法，确保返回正确的引擎类型。 + */ + @Test def testGetEngineType(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + override protected def getEngineType = "Spark" + } + } + + /** + * 测试constructCode方法，确保构建的代码字符串正确。 + */ + @Test def testConstructCode(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + protected def constructCode(pythonModuleInfo: Nothing): String = "import " + } + val moduleInfo = new Nothing("numpy", "/path/to/numpy") + val expectedCode = "import numpy" + } + + /** + * 测试loadPythonModules方法，确保模块加载逻辑正确。 + */ + @Test def testLoadPythonModules(): Unit = { + val pythonModuleLoad: PythonModuleLoad = new PythonSparkEngineHook() { + override protected def getEngineType = "Spark" + + protected def constructCode(pythonModuleInfo: Nothing): String = "import " + } + val moduleInfoList = new util.ArrayList[PythonModuleInfoVO]() + moduleInfoList.add(new Nothing("numpy", "/path/to/numpy")) + moduleInfoList.add(new Nothing("pandas", "/path/to/pandas")) + // val labels = new Array[Label[_]] + // pythonModuleLoad.loadPythonModules(labels) + // 如果loadPythonModules方法有副作用，例如修改外部状态或调用其他方法， + // 那么这里应该添加相应的断言或验证。 + } +} \ No newline at end of file diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala new file mode 100644 index 0000000000..d3ff351070 --- /dev/null +++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/test/scala/org/apache/linkis/engineconn/computation/executor/hook/PythonSparkEngineHookTest.scala @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This test suite contains unit tests for the PythonSparkEngineHook class. + * It ensures that the hook constructs the correct code for loading Python modules + * and logs the appropriate information. + */ +import org.apache.linkis.engineconn.computation.executor.hook.PythonSparkEngineHook +import org.apache.linkis.udf.entity.PythonModuleInfoVO +import org.junit.jupiter.api.Test +import org.mockito.Mockito + +class PythonSparkEngineHookTest { + + /** + * Test to verify that the constructCode method returns the correct code for loading a Python module. + */ + @Test + def testConstructCode(): Unit = { + val pythonModuleInfo = new PythonModuleInfoVO + pythonModuleInfo.setPath("file:///path/to/module.py") + + val hook = new PythonSparkEngineHook + // val result = hook.constructCode(pythonModuleInfo) + + // assert(result == "sc.addPyFile('file:///path/to/module.py')") + } + + /** + * Test to verify that the constructCode method logs the correct information when constructing the code. + */ + @Test + def testConstructCodeReturn(): Unit = { + val pythonModuleInfo = new PythonModuleInfoVO + pythonModuleInfo.setPath("file:///path/to/module.py") + + val hook = new PythonSparkEngineHook + val logger = Mockito.mock(classOf[org.slf4j.Logger]) + // hook.logger = logger + + // hook.constructCode(pythonModuleInfo) + + val expectedLog = "pythonLoadCode: sc.addPyFile('file:///path/to/module.py')" + Mockito.verify(logger).info(expectedLog) + } +} \ No newline at end of file diff --git a/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties b/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties index 3de8a6512b..a535e31ea0 100644 --- a/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties +++ b/linkis-engineconn-plugins/spark/src/main/resources/linkis-engineconn.properties @@ -24,7 +24,7 @@ wds.linkis.engineconn.debug.enable=true wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.spark.SparkEngineConnPlugin -wds.linkis.engine.connector.hooks=org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook,org.apache.linkis.engineconn.computation.executor.hook.PyUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.ScalaUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.JarUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.SparkInitSQLHook +wds.linkis.engine.connector.hooks=org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook,org.apache.linkis.engineconn.computation.executor.hook.PyUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.ScalaUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.JarUdfEngineHook,org.apache.linkis.engineconn.computation.executor.hook.SparkInitSQLHook,org.apache.linkis.engineconn.computation.executor.hook.PythonSparkEngineHook linkis.spark.once.yarn.restful.url=http://127.0.0.1:8088 diff --git a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala index 762f008abc..582183d07f 100644 --- a/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala +++ b/linkis-public-enhancements/linkis-jobhistory/src/main/scala/org/apache/linkis/jobhistory/util/QueryUtils.scala @@ -18,10 +18,10 @@ package org.apache.linkis.jobhistory.util import org.apache.linkis.common.conf.CommonVars +import org.apache.linkis.common.conf.Configuration.IS_VIEW_FS_ENV import org.apache.linkis.common.io.FsPath import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.entity.job.{JobRequest, SubJobDetail} -import org.apache.linkis.jobhistory.conf.JobhistoryConfiguration import org.apache.linkis.jobhistory.entity.JobHistory import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem @@ -32,8 +32,7 @@ import org.apache.commons.lang3.time.DateFormatUtils import java.io.{BufferedReader, InputStream, InputStreamReader, OutputStream} import java.text.SimpleDateFormat -import java.util -import java.util.{Arrays, Date} +import java.util.Date import java.util.regex.Pattern object QueryUtils extends Logging { @@ -44,7 +43,6 @@ object QueryUtils extends Logging { private val CODE_STORE_PREFIX_VIEW_FS = CommonVars("wds.linkis.query.store.prefix.viewfs", "hdfs:///apps-data/") - private val IS_VIEW_FS_ENV = CommonVars("wds.linkis.env.is.viewfs", true) private val CODE_STORE_SUFFIX = CommonVars("wds.linkis.query.store.suffix", "") private val CODE_STORE_LENGTH = CommonVars("wds.linkis.query.code.store.length", 50000) private val CHARSET = "utf-8" diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java new file mode 100644 index 0000000000..1c6a2af99a --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfoVO.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.entity; + +import java.sql.Timestamp; + +/** PythonModuleInfo实体类，用于表示Python模块包信息。 这个类包含了模块的详细信息，如名称、描述、路径、引擎类型、加载状态、过期状态等。 */ +public class PythonModuleInfoVO { + // 自增id，用于唯一标识每一个模块 + private Long id; + + // Python模块名称 + private String name; + + // Python模块描述 + private String description; + + // HDFS路径，存储模块的物理位置 + private String path; + + // 引擎类型，例如：python, spark 或 all + private String engineType; + + // 创建用户，记录创建模块的用户信息 + private String createUser; + + // 修改用户，记录最后修改模块的用户信息 + private String updateUser; + + // 是否加载，0-未加载，1-已加载 + private boolean isLoad; + + // 是否过期，0-未过期，1-已过期 + private Boolean isExpire; + + // 创建时间，记录模块创建的时间 + private Timestamp createTime; + + // 修改时间，记录模块最后修改的时间 + private Timestamp updateTime; + + // 默认构造函数 + public PythonModuleInfoVO() {} + + // 具有所有参数的构造函数 + public PythonModuleInfoVO( + Long id, + String name, + String description, + String path, + String engineType, + String createUser, + String updateUser, + boolean isLoad, + Boolean isExpire, + Timestamp createTime, + Timestamp updateTime) { + this.id = id; + this.name = name; + this.description = description; + this.path = path; + this.engineType = engineType; + this.createUser = createUser; + this.updateUser = updateUser; + this.isLoad = isLoad; + this.isExpire = isExpire; + this.createTime = createTime; + this.updateTime = updateTime; + } + + // Getter和Setter方法 + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public boolean isLoad() { + return isLoad; + } + + public void setLoad(boolean isLoad) { + this.isLoad = isLoad; + } + + public Boolean isExpire() { + return isExpire; + } + + public void setExpire(Boolean isExpire) { + this.isExpire = isExpire; + } + + public Timestamp getCreateTime() { + return createTime; + } + + public void setCreateTime(Timestamp createTime) { + this.createTime = createTime; + } + + public Timestamp getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Timestamp updateTime) { + this.updateTime = updateTime; + } + + // 重写toString方法，用于调试和日志记录 + @Override + public String toString() { + return "PythonModuleInfo{" + + "id=" + + id + + ", name='" + + name + + '\'' + + ", description='" + + description + + '\'' + + ", path='" + + path + + '\'' + + ", engineType='" + + engineType + + '\'' + + ", createUser='" + + createUser + + '\'' + + ", updateUser='" + + updateUser + + '\'' + + ", isLoad=" + + isLoad + + ", isExpire=" + + isExpire + + ", createTime=" + + createTime + + ", updateTime=" + + updateTime + + '}'; + } +} diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala new file mode 100644 index 0000000000..27cd071fb7 --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/RequestPythonModuleProtocol.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.api.rpc + +import org.apache.linkis.protocol.{CacheableProtocol, RetryableProtocol} + +trait PythonModuleProtocol + +case class RequestPythonModuleProtocol(userName: String, engineType: String) + extends RetryableProtocol + with CacheableProtocol + with PythonModuleProtocol + with UdfProtocol diff --git a/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala new file mode 100644 index 0000000000..4ff5c0f8db --- /dev/null +++ b/linkis-public-enhancements/linkis-pes-common/src/main/scala/org/apache/linkis/udf/api/rpc/ResponsePythonModuleProtocol.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.api.rpc + +import org.apache.linkis.udf.entity.PythonModuleInfoVO + +import scala.collection.JavaConverters._ + +class ResponsePythonModuleProtocol(val pythonModules: java.util.List[PythonModuleInfoVO]) + extends PythonModuleProtocol { + + // 如果PythonModuleProtocol需要实现某些方法，你可以在这里实现或覆盖它们 + // 例如，下面是一个假设的示例，展示如何可能实现或覆盖一个方法 + def getModulesInfo(): java.util.List[PythonModuleInfoVO] = { + pythonModules + } + +} diff --git a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java index 4d9a3fa651..28dc65dcc5 100644 --- a/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/linkis-public-enhancements/linkis-pes-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java @@ -1423,6 +1423,73 @@ public Message encryptPath( return Message.ok().data("data", fileMD5Str); } + @ApiOperation(value = "Python模块上传", notes = "上传Python模块文件并返回文件地址", response = Message.class) + @ApiImplicitParams({ + @ApiImplicitParam(name = "file", required = true, dataType = "MultipartFile", value = "上传的文件"), + @ApiImplicitParam(name = "fileName", required = true, dataType = "String", value = "文件名称") + }) + @RequestMapping(path = "/python-upload", method = RequestMethod.POST) + public Message pythonUpload( + HttpServletRequest req, + @RequestParam("file") MultipartFile file, + @RequestParam(value = "fileName", required = false) String fileName) + throws WorkSpaceException, IOException { + + // 获取登录用户 + String username = ModuleUserUtils.getOperationUser(req, "pythonUpload"); + + // 校验文件名称 + if (StringUtils.isBlank(fileName)) { + return Message.error("文件名称不能为空"); + } + // 获取文件名称 + String fileNameSuffix = fileName.substring(0, fileName.lastIndexOf(".")); + if (!fileNameSuffix.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + return Message.error("模块名称错误，仅支持数字字母下划线，且以字母开头，长度最大50"); + } + + // 校验文件类型 + if (!file.getOriginalFilename().endsWith(".py") + && !file.getOriginalFilename().endsWith(".zip")) { + return Message.error("仅支持.py和.zip格式模块文件"); + } + + // 校验文件大小 + if (file.getSize() > 50 * 1024 * 1024) { + return Message.error("限制最大单个文件50M"); + } + + // 定义目录路径 + String path = "hdfs:///appcom/linkis/udf/" + username; + FsPath fsPath = new FsPath(path); + + // 获取文件系统实例 + FileSystem fileSystem = fsService.getFileSystem(username, fsPath); + + // 确认目录是否存在，不存在则创建新目录 + if (!fileSystem.exists(fsPath)) { + try { + fileSystem.mkdirs(fsPath); + fileSystem.setPermission(fsPath, "770"); + } catch (IOException e) { + return Message.error("创建目录失败：" + e.getMessage()); + } + } + + // 构建新的文件路径 + String newPath = fsPath.getPath() + "/" + file.getOriginalFilename(); + FsPath fsPathNew = new FsPath(newPath); + + // 上传文件 + try (InputStream is = file.getInputStream(); + OutputStream outputStream = fileSystem.write(fsPathNew, true)) { + IOUtils.copy(is, outputStream); + } catch (IOException e) { + return Message.error("文件上传失败：" + e.getMessage()); + } + // 返回成功消息并包含文件地址 + return Message.ok().data("filePath", newPath); + } /** * * * diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java index c659e4aa93..0a84e0dcc1 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java @@ -17,11 +17,14 @@ package org.apache.linkis.udf.api; +import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.server.Message; import org.apache.linkis.server.utils.ModuleUserUtils; +import org.apache.linkis.udf.entity.PythonModuleInfo; import org.apache.linkis.udf.entity.UDFInfo; import org.apache.linkis.udf.entity.UDFTree; import org.apache.linkis.udf.excepiton.UDFException; +import org.apache.linkis.udf.service.PythonModuleInfoService; import org.apache.linkis.udf.service.UDFService; import org.apache.linkis.udf.service.UDFTreeService; import org.apache.linkis.udf.utils.ConstantVar; @@ -39,6 +42,7 @@ import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; +import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -49,6 +53,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import com.google.common.collect.Lists; @@ -73,6 +78,7 @@ public class UDFRestfulApi { @Autowired private UDFService udfService; @Autowired private UDFTreeService udfTreeService; + @Autowired private PythonModuleInfoService pythonModuleInfoService; ObjectMapper mapper = new ObjectMapper(); @@ -1013,4 +1019,278 @@ public Message versionInfo( } return message; } + + /** + * Python物料查询 + * + * @param name python模块名称 + * @param engineType 引擎类型(all,spark,python) + * @param username 用户名 + * @param isLoad 是否加载（0-未加载，1-已加载） + * @param isExpire 是否过期（0-未过期，1-已过期） + * @param pageNow 页码 + * @param pageSize 每页展示数据条数 + */ + @RequestMapping(path = "/python-list", method = RequestMethod.GET) + @ApiOperation(value = "查询Python模块列表", notes = "根据条件查询Python模块信息") + @ApiImplicitParams({ + @ApiImplicitParam(name = "name", value = "Python模块名称", required = false, dataType = "String"), + @ApiImplicitParam( + name = "engineType", + value = "引擎类型（all, spark, python）", + required = false, + dataType = "String"), + @ApiImplicitParam(name = "username", value = "用户名", required = false, dataType = "String"), + @ApiImplicitParam( + name = "isLoad", + value = "是否加载（0-未加载，1-已加载）", + required = false, + dataType = "Integer"), + @ApiImplicitParam( + name = "isExpire", + value = "是否过期（0-未过期，1-已过期）", + required = false, + dataType = "Integer"), + @ApiImplicitParam(name = "pageNow", value = "页码", required = false, dataType = "Integer"), + @ApiImplicitParam(name = "pageSize", value = "每页展示数据条数", required = false, dataType = "Integer") + }) + public Message pythonList( + @RequestParam(value = "name", required = false) String name, + @RequestParam(value = "engineType", required = false) String engineType, + @RequestParam(value = "username", required = false) String username, + @RequestParam(value = "isLoad", required = false) Integer isLoad, + @RequestParam(value = "isExpire", required = false) Integer isExpire, + @RequestParam(value = "pageNow", required = false) Integer pageNow, + @RequestParam(value = "pageSize", required = false) Integer pageSize, + HttpServletRequest req) { + + // 获取登录用户 + String user = ModuleUserUtils.getOperationUser(req, "pythonList"); + + // 参数校验 + if (org.apache.commons.lang3.StringUtils.isBlank(name)) name = null; + if (org.apache.commons.lang3.StringUtils.isBlank(engineType)) engineType = null; + if (pageNow == null) pageNow = 1; + if (pageSize == null) pageSize = 10; + + // 根据管理员权限设置username + if (Configuration.isAdmin(user)) { + if (username == null) username = null; + } else { + username = user; + } + + // 分页设置 + PageHelper.startPage(pageNow, pageSize); + try { + // 执行数据库查询 + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setName(name); + pythonModuleInfo.setEngineType(engineType); + pythonModuleInfo.setCreateUser(username); + pythonModuleInfo.setIsLoad(isLoad); + pythonModuleInfo.setIsExpire(isExpire); + List pythonList = pythonModuleInfoService.getByConditions(pythonModuleInfo); + PageInfo pageInfo = new PageInfo<>(pythonList); + // 封装返回结果 + return Message.ok().data("pythonList", pythonList).data("totalPage", pageInfo.getTotal()); + } finally { + // 关闭分页 + PageHelper.clearPage(); + } + } + + /** + * Python物料删除 + * + * @param id id + * @param isExpire 0-未过期，1-已过期 + */ + @RequestMapping(path = "/python-delete", method = RequestMethod.GET) + @ApiOperation(value = "删除Python模块", notes = "根据模块ID删除Python模块,管理员可以删除任何模块，普通用户只能删除自己创建的模块") + @ApiImplicitParams({ + @ApiImplicitParam(name = "id", value = "模块ID", required = true, dataType = "Long"), + @ApiImplicitParam( + name = "isExpire", + value = "模块是否过期（0：未过期，1：已过期）", + required = true, + dataType = "int") + }) + public Message pythonDelete( + @RequestParam(value = "id", required = false) Long id, + @RequestParam(value = "isExpire", required = false) int isExpire, + HttpServletRequest req, + HttpServletResponse resp) { + // 打印审计日志并获取登录用户 + String user = ModuleUserUtils.getOperationUser(req, "pythonDelete"); + + // 参数校验 + if (id == null) { + return Message.error("Invalid parameters: id is null"); + } + if (isExpire != 0 && isExpire != 1) { + return Message.error("Invalid parameters: isExpire must be 0 or 1"); + } + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setId(id); + // 根据id查询Python模块信息 + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + if (moduleInfo == null) { + return Message.ok(); // 如果不存在则直接返回成功 + } + + // 判断是否是管理员 + if (!Configuration.isAdmin(user)) { + // 如果不是管理员，检查创建用户是否与当前用户一致 + if (!moduleInfo.getCreateUser().equals(user)) { + return Message.error("无权删除他人Python模块"); + } + } + + // 更新Python模块信息 + moduleInfo.setIsExpire(1); + moduleInfo.setUpdateUser(user); + moduleInfo.setUpdateTime(new Date()); + // 修改数据库中的模块名称和文件名称 + String newName = moduleInfo.getName() + "_" + System.currentTimeMillis(); + String newPath = moduleInfo.getPath() + "_" + System.currentTimeMillis(); + moduleInfo.setPath(newPath); + moduleInfo.setName(newName); + pythonModuleInfoService.updatePythonModuleInfo(moduleInfo); + return Message.ok(); + } + + /** Python物料新增/更新 */ + @ApiOperation(value = "Python物料新增/更新", notes = "根据传入的Python物料信息新增或更新") + @ApiImplicitParams({ + @ApiImplicitParam( + name = "Python物料新增/更新Request", + value = "Python物料新增/更新请求体", + required = false, + dataType = "PythonModuleInfo") + }) + @RequestMapping(value = "/python-save", method = RequestMethod.POST) + public Message request( + @Nullable @RequestBody PythonModuleInfo pythonModuleInfo, + HttpServletRequest httpReq, + HttpServletResponse httpResp) { + + // 获取登录用户 + String userName = ModuleUserUtils.getOperationUser(httpReq, "pythonSave"); + + // 入参校验 + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getName())) { + return Message.error("模块名称：不能为空"); + } + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getPath())) { + return Message.error("模块物料：不能为空"); + } + if (org.apache.commons.lang3.StringUtils.isBlank(pythonModuleInfo.getEngineType())) { + return Message.error("引擎类型：不能为空"); + } + if (pythonModuleInfo.getIsLoad() == null) { + return Message.error("是否加载：不能为空"); + } + if (pythonModuleInfo.getIsExpire() == null) { + return Message.error("是否过期：不能为空"); + } + String path = pythonModuleInfo.getPath(); + String fileName = path.substring(path.lastIndexOf("/") + 1, path.lastIndexOf(".")); + if (!pythonModuleInfo.getName().equals(fileName)) { + return Message.error("模块名称与物料文件名称必须一样"); + } + // 根据id判断是插入还是更新 + if (pythonModuleInfo.getId() == null) { + Integer newExpire = pythonModuleInfo.getIsExpire(); + pythonModuleInfo.setCreateUser(userName); + // 查询未过期的 + pythonModuleInfo.setIsExpire(0); + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + // 插入逻辑 + if (moduleInfo != null) { + return Message.error("模块" + moduleInfo.getName() + "已存在"); + } + pythonModuleInfo.setCreateTime(new Date()); + pythonModuleInfo.setUpdateTime(new Date()); + pythonModuleInfo.setIsExpire(newExpire); + pythonModuleInfo.setUpdateUser(userName); + pythonModuleInfoService.insertPythonModuleInfo(pythonModuleInfo); + return Message.ok().data("id", pythonModuleInfo.getId()); + } else { + PythonModuleInfo pythonModuleTmp = new PythonModuleInfo(); + pythonModuleTmp.setId(pythonModuleInfo.getId()); + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleTmp); + // 更新逻辑 + if (moduleInfo == null) { + return Message.error("未找到该Python模块"); + } + if (!Configuration.isAdmin(userName) && !userName.equals(moduleInfo.getCreateUser())) { + return Message.error("无权编辑他人Python模块"); + } + if (moduleInfo.getIsExpire() != 0) { + return Message.error("当前模块已过期，不允许进行修改操作"); + } + // 如果模块过期，则修改数据库中的模块名称和文件名称 + if (pythonModuleInfo.getIsExpire() == 1) { + // 修改数据库中的模块名称和文件名称 + String newName = moduleInfo.getName() + "_" + System.currentTimeMillis(); + String newPath = moduleInfo.getPath() + "_" + System.currentTimeMillis(); + pythonModuleInfo.setPath(newPath); + pythonModuleInfo.setName(newName); + } + pythonModuleInfo.setUpdateUser(userName); + pythonModuleInfo.setUpdateTime(new Date()); + pythonModuleInfoService.updatePythonModuleInfo(pythonModuleInfo); + } + return Message.ok(); + } + + /** + * python文件是否存在查询 + * + * @param fileName 文件名称 + */ + @RequestMapping(path = "/python-file-exist", method = RequestMethod.GET) + @ApiOperation(value = "查询Python文件是否存在", notes = "根据用户名和文件名查询Python模块信息，如果存在则返回true，否则返回false") + @ApiImplicitParams({ + @ApiImplicitParam( + name = "fileName", + value = "Python文件名", + required = true, + dataType = "string", + paramType = "query"), + @ApiImplicitParam( + name = "Authorization", + value = "Bearer token", + required = true, + dataType = "string", + paramType = "header") + }) + public Message pythonFileExist( + @RequestParam(value = "fileName", required = false) String fileName, HttpServletRequest req) { + // 审计日志打印并获取登录用户 + String userName = ModuleUserUtils.getOperationUser(req, "pythonFileExist"); + + // 参数校验 + if (org.apache.commons.lang3.StringUtils.isBlank(fileName)) { + return Message.error("参数fileName不能为空"); + } + String fileNameWithoutExtension = fileName.substring(0, fileName.lastIndexOf(".")); + if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_]{0,49}$")) { + return Message.error("只支持数字字母下划线，且以字母开头，长度最大50"); + } + + // 封装PythonModuleInfo对象并查询数据库 + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + pythonModuleInfo.setName(fileNameWithoutExtension); + pythonModuleInfo.setCreateUser(userName); + PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo); + + // 根据查询结果返回相应信息 + if (moduleInfo == null) { + return Message.ok().data("result", true); + } else { + return Message.error("模块" + fileName + "已存在，如需重新上传请先删除旧的模块"); + } + } } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java new file mode 100644 index 0000000000..54c79bcee0 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/PythonModuleInfoMapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.dao; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface PythonModuleInfoMapper { + + // SQL 1: 模糊查询 + List selectByConditions(PythonModuleInfo pythonModuleInfo); + + // SQL 2: 更新 + int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 3: 新增 + Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 4: 带有判断的查询 + PythonModuleInfo selectByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); + + // SQL 5: 查询包含多个引擎类型的hdfs路径 + List selectPathsByUsernameAndEnginetypes( + @Param("username") String username, @Param("enginetypes") List enginetypes); +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java new file mode 100644 index 0000000000..727b323cb6 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/entity/PythonModuleInfo.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.entity; + +import java.util.Date; + +public class PythonModuleInfo { + private Long id; + private String name; + private String description; + private String path; + private String engineType; + private String createUser; + private String updateUser; + private Integer isLoad; + private Integer isExpire; + private Date createTime; + private Date updateTime; + + public PythonModuleInfo() {} + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public Integer getIsLoad() { + return isLoad; + } + + public void setIsLoad(Integer isLoad) { + this.isLoad = isLoad; + } + + public Integer getIsExpire() { + return isExpire; + } + + public void setIsExpire(Integer isExpire) { + this.isExpire = isExpire; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + @Override + public String toString() { + return "PythonModuleInfo{" + + "id=" + + id + + ", name='" + + name + + '\'' + + ", description='" + + description + + '\'' + + ", path='" + + path + + '\'' + + ", engineType='" + + engineType + + '\'' + + ", createUser='" + + createUser + + '\'' + + ", updateUser='" + + updateUser + + '\'' + + ", isLoad=" + + isLoad + + ", isExpire=" + + isExpire + + ", createTime=" + + createTime + + ", updateTime=" + + updateTime + + '}'; + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java new file mode 100644 index 0000000000..0e94309baf --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/PythonModuleInfoService.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import java.util.List; + +public interface PythonModuleInfoService { + + // SQL 1: 模糊查询 + List getByConditions(PythonModuleInfo pythonModuleInfo); + + // SQL 2: 更新 + int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 3: 新增 + Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo); + + // SQL 4: 带有判断的查询 + PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo); + + // SQL 5: 查询包含多个引擎类型的hdfs路径 + List getPathsByUsernameAndEnginetypes( + String username, List enginetypes); +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java new file mode 100644 index 0000000000..91f5839416 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/service/impl/PythonModuleInfoServiceImpl.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service.impl; + +import org.apache.linkis.udf.dao.PythonModuleInfoMapper; +import org.apache.linkis.udf.entity.PythonModuleInfo; +import org.apache.linkis.udf.service.PythonModuleInfoService; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class PythonModuleInfoServiceImpl implements PythonModuleInfoService { + + private final PythonModuleInfoMapper pythonModuleInfoMapper; + + @Autowired + public PythonModuleInfoServiceImpl(PythonModuleInfoMapper pythonModuleInfoMapper) { + this.pythonModuleInfoMapper = pythonModuleInfoMapper; + } + + @Override + public List getByConditions(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); + } + + @Override + public int updatePythonModuleInfo(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); + } + + @Override + public Long insertPythonModuleInfo(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); + } + + @Override + public PythonModuleInfo getByUserAndNameAndId(PythonModuleInfo pythonModuleInfo) { + return pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); + } + + @Override + public List getPathsByUsernameAndEnginetypes( + String username, List enginetypes) { + return pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml new file mode 100644 index 0000000000..0b513ba2f2 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/main/resources/mapper/common/PythonModuleInfoMapper.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + UPDATE linkis_ps_python_module_info + + name = #{name}, + description = #{description}, + path = #{path}, + engine_type = #{engineType}, + create_user = #{createUser}, + update_user = #{updateUser}, + is_load = #{isLoad}, + is_expire = #{isExpire}, + create_time = #{createTime}, + update_time = #{updateTime}, + + WHERE id = #{id} + + + + + INSERT INTO linkis_ps_python_module_info + (name, description, path, engine_type, create_user, update_user, is_load, is_expire, create_time, update_time) + VALUES + (#{name}, #{description}, #{path}, #{engineType}, #{createUser}, #{updateUser}, #{isLoad}, #{isExpire}, #{createTime}, #{updateTime}) + + SELECT LAST_INSERT_ID() + + + + + + + + + + + diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala index 501e4d3bc3..2e0eac3ca2 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiver.scala @@ -17,28 +17,56 @@ package org.apache.linkis.udf.api.rpc +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.rpc.utils.RPCUtils import org.apache.linkis.rpc.{Receiver, Sender} -import org.apache.linkis.udf.service.{UDFService, UDFTreeService} - -import java.lang +import org.apache.linkis.udf.entity.{PythonModuleInfo, PythonModuleInfoVO} +import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} +import java.{lang, util} +import scala.collection.JavaConverters.asScalaBufferConverter import scala.concurrent.duration.Duration +import scala.tools.nsc.interactive.Logger -class UdfReceiver extends Receiver { +class UdfReceiver extends Receiver with Logging { private var udfTreeService: UDFTreeService = _ private var udfService: UDFService = _ + // 注⼊PythonModuleInfoService + private var pythonModuleInfoService: PythonModuleInfoService = _ + def this(udfTreeService: UDFTreeService, udfService: UDFService) = { this() this.udfTreeService = udfTreeService this.udfService = udfService } + def this( + udfTreeService: UDFTreeService, + udfService: UDFService, + pythonModuleInfoService: PythonModuleInfoService + ) = { + this(udfTreeService, udfService) + this.pythonModuleInfoService = pythonModuleInfoService + } + override def receive(message: Any, sender: Sender): Unit = {} + def parseModuleInfoVO(info: PythonModuleInfo): PythonModuleInfoVO = { + // 假设路径格式为 "username/module_name/module_version" + val vo = new PythonModuleInfoVO() + vo.setPath(info.getPath) + vo.setName(info.getName) + vo.setId(info.getId) + vo.setCreateUser(info.getCreateUser) + vo + } + override def receiveAndReply(message: Any, sender: Sender): Any = { + logger.info(s"udfPython message: ${message.getClass.getName}") message match { case RequestUdfTree(userName, treeType, treeId, treeCategory) => val udfTree = udfTreeService.getTreeById(treeId, userName, treeType, treeCategory) @@ -46,6 +74,22 @@ class UdfReceiver extends Receiver { case RequestUdfIds(userName, udfIds, treeCategory) => val udfs = udfService.getUDFInfoByIds(udfIds.map(id => new lang.Long(id)), treeCategory) new ResponseUdfs(udfs) + case RequestPythonModuleProtocol(userName, engineType) => + val instance: ServiceInstance = RPCUtils.getServiceInstanceFromSender(sender) + logger.info(s"RequestPythonModuleProtocol: userName: $userName, engineType: $engineType, sendInstance: $instance .") + // 获取Python模块路径列表 + var list = new java.util.ArrayList[String]() + list.add(engineType) + list.add("all") + val infoes: util.List[PythonModuleInfo] = + pythonModuleInfoService.getPathsByUsernameAndEnginetypes(userName, list) + // 将路径列表转换为PythonModuleInfo列表 + var voList = new java.util.ArrayList[PythonModuleInfoVO]() + infoes.asScala.foreach(info => { + val vo: PythonModuleInfoVO = parseModuleInfoVO(info) + voList.add(vo) + }) + new ResponsePythonModuleProtocol(voList) case _ => } } diff --git a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala index de75580102..e5621c3bf5 100644 --- a/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala +++ b/linkis-public-enhancements/linkis-udf-service/src/main/scala/org/apache/linkis/udf/api/rpc/UdfReceiverChooser.scala @@ -18,7 +18,7 @@ package org.apache.linkis.udf.api.rpc import org.apache.linkis.rpc.{Receiver, ReceiverChooser, RPCMessageEvent} -import org.apache.linkis.udf.service.{UDFService, UDFTreeService} +import org.apache.linkis.udf.service.{PythonModuleInfoService, UDFService, UDFTreeService} import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @@ -34,10 +34,15 @@ class UdfReceiverChooser extends ReceiverChooser { @Autowired private var udfService: UDFService = _ + @Autowired + private var pythonModuleInfoService: PythonModuleInfoService = _ + private var udfReceiver: Option[UdfReceiver] = None @PostConstruct - def init(): Unit = udfReceiver = Some(new UdfReceiver(udfTreeService, udfService)) + def init(): Unit = udfReceiver = Some( + new UdfReceiver(udfTreeService, udfService, pythonModuleInfoService) + ) override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match { case _: UdfProtocol => udfReceiver diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java new file mode 100644 index 0000000000..6ba1d96745 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/api/PythonModuleRestfulApiTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.api; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +import org.junit.jupiter.api.Test; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** PythonModuleRestfulApiTest 类用于对 PythonModuleRestfulApi 进行单元测试。 */ +public class PythonModuleRestfulApiTest { + @Autowired protected MockMvc mockMvc; + /** 测试Python模块列表功能 */ + @Test + public void testPythonList() throws Exception { + // 测试获取Python模块列表 + mockMvc + .perform( + get("/python-list") + .param("name", "testModule") + .param("engineType", "spark") + .param("username", "testUser") + .param("isLoad", "0") + .param("isExpire", "1") + .param("pageNow", "1") + .param("pageSize", "10")) + .andExpect(status().isOk()); + + // 测试获取Python模块列表（无参数） + mockMvc.perform(get("/python-list")).andExpect(status().isOk()); + + // 测试获取Python模块列表（空参数） + mockMvc + .perform( + get("/python-list") + .param("name", "") + .param("engineType", "") + .param("username", "") + .param("isLoad", "") + .param("isExpire", "") + .param("pageNow", "") + .param("pageSize", "")) + .andExpect(status().isOk()); + } + + /** 测试删除Python模块功能 */ + @Test + public void testPythonDelete() throws Exception { + // 测试删除Python模块 + mockMvc + .perform(get("/python-delete").param("id", "1").param("isExpire", "0")) + .andExpect(status().isOk()); + + // 测试删除不存在的Python模块 + mockMvc + .perform(get("/python-delete").param("id", "999").param("isExpire", "0")) + .andExpect(status().isNotFound()); + + // 测试删除Python模块时传入无效参数 + mockMvc + .perform(get("/python-delete").param("id", "1").param("isExpire", "2")) + .andExpect(status().isBadRequest()); + } + + /** 测试保存Python模块功能 */ + @Test + public void testPythonSave() throws Exception { + // 测试保存Python模块 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"testModule\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isOk()); + + // 测试保存Python模块时传入空名称 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"\",\"path\":\"/path/to/module.py\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + + // 测试保存Python模块时传入空路径 + mockMvc + .perform( + post("/python-save") + .contentType(MediaType.APPLICATION_JSON) + .content( + "{\"name\":\"testModule\",\"path\":\"\",\"engineType\":\"python\",\"isLoad\":1,\"isExpire\":0}")) + .andExpect(status().isBadRequest()); + } + + /** 测试检查Python模块文件是否存在功能 */ + @Test + public void testPythonFileExist() throws Exception { + // 测试检查Python模块文件是否存在 + mockMvc + .perform(get("/python-file-exist").param("fileName", "testModule.py")) + .andExpect(status().isOk()); + + // 测试检查Python模块文件是否存在时传入空文件名 + mockMvc + .perform(get("/python-file-exist").param("fileName", "")) + .andExpect(status().isBadRequest()); + + // 测试检查Python模块文件是否存在时未传入文件名 + mockMvc.perform(get("/python-file-exist")).andExpect(status().isBadRequest()); + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java new file mode 100644 index 0000000000..a68309dbf5 --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/dao/PythonModuleInfoMapperTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.dao; + +import org.apache.linkis.udf.entity.PythonModuleInfo; + +import org.springframework.test.context.event.annotation.BeforeTestClass; + +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** PythonModuleInfoMapperTest 类用于测试 PythonModuleInfoMapper 的功能。 */ +public class PythonModuleInfoMapperTest { + + private PythonModuleInfoMapper pythonModuleInfoMapper; // PythonModuleInfoMapper 的模拟对象 + + /** 在每个测试方法执行前执行，用于初始化测试环境。 */ + @BeforeTestClass + public void setUp() { + pythonModuleInfoMapper = mock(PythonModuleInfoMapper.class); + } + + /** 测试 selectByConditions 方法的功能。 */ + @Test + public void testSelectByConditions() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByConditions(pythonModuleInfo)) + .thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = pythonModuleInfoMapper.selectByConditions(pythonModuleInfo); + assertEquals(1, result.size()); + // 验证结果的属性 + } + + /** 测试 updatePythonModuleInfo 方法的功能。 */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo)).thenReturn(1); + + int result = pythonModuleInfoMapper.updatePythonModuleInfo(pythonModuleInfo); + assertEquals(1, result); + } + + /** 测试 insertPythonModuleInfo 方法的功能。 */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo)).thenReturn(1L); + + Long result = pythonModuleInfoMapper.insertPythonModuleInfo(pythonModuleInfo); + assertEquals(1L, result.longValue()); + } + + /** 测试 selectByUserAndNameAndId 方法的功能。 */ + @Test + public void testSelectByUserAndNameAndId() { + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo)) + .thenReturn(pythonModuleInfo); + + PythonModuleInfo result = pythonModuleInfoMapper.selectByUserAndNameAndId(pythonModuleInfo); + assertNotNull(result); + // 验证结果的属性 + } + + /** 测试 selectPathsByUsernameAndEnginetypes 方法的功能。 */ + @Test + public void testSelectPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("type1", "type2"); + PythonModuleInfo pythonModuleInfo = new PythonModuleInfo(); + // 设置 pythonModuleInfo 的属性 + + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) + .thenReturn(Arrays.asList(pythonModuleInfo)); + + List result = + pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes); + assertEquals(1, result.size()); + // 验证结果的属性 + } +} diff --git a/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java new file mode 100644 index 0000000000..9fc050938a --- /dev/null +++ b/linkis-public-enhancements/linkis-udf-service/src/test/java/org/apache/linkis/udf/service/PythonModuleInfoServiceTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.udf.service; + +import org.apache.linkis.udf.dao.PythonModuleInfoMapper; +import org.apache.linkis.udf.entity.PythonModuleInfo; +import org.apache.linkis.udf.service.impl.PythonModuleInfoServiceImpl; + +import java.util.Arrays; +import java.util.List; + +import org.aspectj.lang.annotation.Before; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +/** PythonModuleInfoServiceImplTest 类用于对 PythonModuleInfoServiceImpl 进行单元测试。 */ +public class PythonModuleInfoServiceTest { + + @Mock private PythonModuleInfoMapper pythonModuleInfoMapper; + + @InjectMocks private PythonModuleInfoServiceImpl pythonModuleInfoServiceImpl; + + /** 在每个测试方法执行前执行，用于初始化测试环境。 */ + @Before("") + public void setUp() { + MockitoAnnotations.openMocks(this); + } + + /** 测试 getByConditions 方法的功能。 */ + @Test + public void testGetByConditions() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("TestModule"); + when(pythonModuleInfoMapper.selectByConditions(mockInfo)).thenReturn(Arrays.asList(mockInfo)); + + List result = pythonModuleInfoServiceImpl.getByConditions(mockInfo); + + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals(mockInfo.getId(), result.get(0).getId()); + assertEquals(mockInfo.getName(), result.get(0).getName()); + } + + /** 测试 updatePythonModuleInfo 方法的功能。 */ + @Test + public void testUpdatePythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UpdatedModule"); + when(pythonModuleInfoMapper.updatePythonModuleInfo(mockInfo)).thenReturn(1); + + int result = pythonModuleInfoServiceImpl.updatePythonModuleInfo(mockInfo); + + assertEquals(1, result); + } + + /** 测试 insertPythonModuleInfo 方法的功能。 */ + @Test + public void testInsertPythonModuleInfo() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("NewModule"); + when(pythonModuleInfoMapper.insertPythonModuleInfo(mockInfo)).thenReturn(1L); + + Long result = pythonModuleInfoServiceImpl.insertPythonModuleInfo(mockInfo); + + assertNotNull(result); + assertEquals(1L, result.longValue()); + } + + /** 测试 getByUserAndNameAndId 方法的功能。 */ + @Test + public void testGetByUserAndNameAndId() { + PythonModuleInfo mockInfo = new PythonModuleInfo(); + mockInfo.setId(1L); + mockInfo.setName("UniqueModule"); + when(pythonModuleInfoMapper.selectByUserAndNameAndId(mockInfo)).thenReturn(mockInfo); + + PythonModuleInfo result = pythonModuleInfoServiceImpl.getByUserAndNameAndId(mockInfo); + + assertNotNull(result); + assertEquals(mockInfo.getId(), result.getId()); + assertEquals(mockInfo.getName(), result.getName()); + } + + /** 测试 getPathsByUsernameAndEnginetypes 方法的功能。 */ + @Test + public void testGetPathsByUsernameAndEnginetypes() { + String username = "testUser"; + List enginetypes = Arrays.asList("Engine1", "Engine2"); + PythonModuleInfo mockInfo1 = new PythonModuleInfo(); + mockInfo1.setId(1L); + mockInfo1.setName("Module1"); + PythonModuleInfo mockInfo2 = new PythonModuleInfo(); + mockInfo2.setId(2L); + mockInfo2.setName("Module2"); + when(pythonModuleInfoMapper.selectPathsByUsernameAndEnginetypes(username, enginetypes)) + .thenReturn(Arrays.asList(mockInfo1, mockInfo2)); + + List result = + pythonModuleInfoServiceImpl.getPathsByUsernameAndEnginetypes(username, enginetypes); + + assertNotNull(result); + assertEquals(2, result.size()); + assertTrue(result.contains(mockInfo1)); + assertTrue(result.contains(mockInfo2)); + } +} diff --git a/linkis-web/.gitignore b/linkis-web/.gitignore index 89cc5767eb..8f5fdeb918 100644 --- a/linkis-web/.gitignore +++ b/linkis-web/.gitignore @@ -2,11 +2,12 @@ .vscode .cache .idea/ +.fes node_modules/ dist/ -package-lock.json apache-linkis-*.tar.gz ./cn.json .env.* +package-lock.json diff --git a/linkis-web/package.json b/linkis-web/package.json index 47828a70c2..de2ff942cc 100644 --- a/linkis-web/package.json +++ b/linkis-web/package.json @@ -4,11 +4,13 @@ "private": true, "scripts": { "serve": "vue-cli-service serve", - "build": "vue-cli-service build", + "build": "vue-cli-service build && wnpm run buildSubModule", "lint": "vue-cli-service lint --no-fix", "fix": "eslint --ext .js,.vue src --fix", "precommit": "lint-staged", - "preinstall": "npm install --package-lock-only --ignore-scripts && npx npm-force-resolutions" + "preinstall": "wnpm install --package-lock-only --ignore-scripts && npx npm-force-resolutions", + "buildSubModule": "cd src/apps/PythonModule && npm run build:prod && cd ../../..", + "installAll": "wnpm install && cd src/apps/PythonModule && wnpm install && cd ../../.." }, "husky": { "hooks": { @@ -23,26 +25,22 @@ }, "dependencies": { "@form-create/iview": "2.5.27", - "axios": "0.21.4", - "babel-polyfill": "6.26.0", - "core-js": "3.27.2", + "axios": "0.28.1", "dexie": "3.2.3", "dt-sql-parser": "3.0.5", - "eslint": "7.21.0", - "eslint-plugin-vue": "9.6.0", "highlight.js": "10.7.0", + "hint.css": "^2.7.0", "iview": "3.5.4", "jsencrypt": "3.2.1", "lodash": "4.17.21", "md5": "2.3.0", - "mitt": "1.2.0", "moment": "2.29.4", "monaco-editor": "0.30.1", "object-to-formdata": "4.2.2", "path-browserify": "1.0.1", - "postcss": "8.4.21", "qs": "6.11.0", "reconnecting-websocket": "4.4.0", + "sass": "1.77.8", "sql-formatter": "2.3.3", "svgo": "3.0.2", "v-jsoneditor": "1.4.5", @@ -55,22 +53,27 @@ }, "devDependencies": { "@intlify/vue-i18n-loader": "1.0.0", - "@vue/cli-plugin-babel": "5.0.8", + "@vue/cli-plugin-babel": "5.0.1", "@vue/cli-plugin-eslint": "5.0.8", "@vue/cli-service": "5.0.8", "@vue/eslint-config-standard": "4.0.0", "archiver": "3.1.1", "autoprefixer": "10.4.14", "babel-eslint": "10.1.0", + "babel-polyfill": "6.26.0", "copy-webpack-plugin": "9.1.0", + "core-js": "3.27.2", "csp-html-webpack-plugin": "5.1.0", + "eslint": "7.21.0", + "eslint-plugin-vue": "9.6.0", "filemanager-webpack-plugin": "7.0.0", "husky": "1.3.1", "lint-staged": "13.1.1", "material-design-icons": "3.0.1", + "mitt": "1.2.0", "monaco-editor-webpack-plugin": "6.0.0", - "node-sass": "8.0.0", "npm-force-resolutions": "0.0.10", + "postcss": "8.4.21", "sass-loader": "10.4.1", "svg-sprite-loader": "6.0.0", "vue-cli-plugin-mockjs": "0.1.3", diff --git a/linkis-web/src/apps/PythonModule/.env b/linkis-web/src/apps/PythonModule/.env new file mode 100644 index 0000000000..e69de29bb2 diff --git a/linkis-web/src/apps/PythonModule/.fes.js b/linkis-web/src/apps/PythonModule/.fes.js new file mode 100644 index 0000000000..9844402aac --- /dev/null +++ b/linkis-web/src/apps/PythonModule/.fes.js @@ -0,0 +1,13 @@ +import { defineBuildConfig } from '@fesjs/fes'; +export default defineBuildConfig({ + // base: './', + proxy: { + base: './' + }, + viteOption: { + build: { + outDir: '../../../dist/dist/dist' + } + } + +}); diff --git a/linkis-web/src/apps/PythonModule/.fes.prod.js b/linkis-web/src/apps/PythonModule/.fes.prod.js new file mode 100644 index 0000000000..58a3946fa5 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/.fes.prod.js @@ -0,0 +1,4 @@ +import { defineBuildConfig } from '@fesjs/fes'; +export default defineBuildConfig({ + publicPath: './', +}); diff --git a/linkis-web/src/apps/PythonModule/.fes.test.js b/linkis-web/src/apps/PythonModule/.fes.test.js new file mode 100644 index 0000000000..58a3946fa5 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/.fes.test.js @@ -0,0 +1,4 @@ +import { defineBuildConfig } from '@fesjs/fes'; +export default defineBuildConfig({ + publicPath: './', +}); diff --git a/linkis-web/src/apps/PythonModule/.gitattributes b/linkis-web/src/apps/PythonModule/.gitattributes new file mode 100644 index 0000000000..ec8935df7d --- /dev/null +++ b/linkis-web/src/apps/PythonModule/.gitattributes @@ -0,0 +1,9 @@ +* text=auto +* text eol=lf +*.png binary +*.gif binary +*.ttf binary +*.woff binary +*.eot binary +*.woff binary +*.otf binary \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/.npmrc b/linkis-web/src/apps/PythonModule/.npmrc new file mode 100644 index 0000000000..dd4617f0fa --- /dev/null +++ b/linkis-web/src/apps/PythonModule/.npmrc @@ -0,0 +1,2 @@ +registry=http://wnpm.weoa.com:8001 +shamefully-hoist=true \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/index.html b/linkis-web/src/apps/PythonModule/index.html new file mode 100644 index 0000000000..ca59d3d3e0 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/index.html @@ -0,0 +1,15 @@ + + + + + + + + + + + + +
+ + \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/package.json b/linkis-web/src/apps/PythonModule/package.json new file mode 100644 index 0000000000..44fc1a6447 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/package.json @@ -0,0 +1,31 @@ +{ + "dependencies": { + "core-js": "3.35.1", + "vue": "3.3.9", + "@vueuse/core": "10.9.0", + "lodash-es": "4.17.21", + "@qlin/request": "0.2.6", + "@webank/letgo-components": "1.1.4", + "@fesjs/fes-design": "0.8.60", + "@webank/fes-design-material": "0.2.31", + "@fesjs/traction-widget": "1.9.1", + "dayjs": "1.11.9", + "lodash": "4.17.21", + "@fesjs/fes": "3.1.10", + "@fesjs/plugin-model": "3.0.1", + "@fesjs/plugin-request": "4.0.0-rc.3", + "@fesjs/builder-vite": "4.0.2" + }, + "name": "linkis", + "version": "1.0.0", + "scripts": { + "build:test": "cross-env FES_ENV=test fes build", + "build:prod": "cross-env FES_ENV=prod fes build", + "analyze": "cross-env ANALYZE=1 fes build", + "dev": "fes dev" + }, + "devDependencies": { + "typescript": "5.3.3", + "cross-env": "7.0.3" + } +} \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/public/logo.svg b/linkis-web/src/apps/PythonModule/public/logo.svg new file mode 100644 index 0000000000..210841be54 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/public/logo.svg @@ -0,0 +1,33 @@ + + + 编组 11 + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/src/.fes/configType.d.ts b/linkis-web/src/apps/PythonModule/src/.fes/configType.d.ts new file mode 100644 index 0000000000..e28a69ea37 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/configType.d.ts @@ -0,0 +1,5 @@ + +export * from '@fesjs/preset-built-in'; +export * from '@fesjs/builder-vite'; +export * from '@fesjs/plugin-model'; +export * from '@fesjs/plugin-request'; diff --git a/linkis-web/src/apps/PythonModule/src/.fes/core/coreExports.js b/linkis-web/src/apps/PythonModule/src/.fes/core/coreExports.js new file mode 100644 index 0000000000..289dd7853b --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/core/coreExports.js @@ -0,0 +1,19 @@ +export { + useRoute, + useRouter, + onBeforeRouteUpdate, + onBeforeRouteLeave, + RouterLink, + RouterView, + useLink, + createWebHashHistory, + createWebHistory, + createMemoryHistory, + createRouter, + Plugin, + ApplyPluginsType +} from 'C:/Users/chandlermei/Desktop/letgo-code (1)/node_modules/@fesjs/runtime'; + +export { plugin } from '../core/plugin.js'; +export { getRouter, getHistory, destroyRouter, defineRouteMeta } from '../core/routes/routeExports.js'; + diff --git a/linkis-web/src/apps/PythonModule/src/.fes/core/plugin.js b/linkis-web/src/apps/PythonModule/src/.fes/core/plugin.js new file mode 100644 index 0000000000..3f89c73d17 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/core/plugin.js @@ -0,0 +1,7 @@ +import { Plugin } from 'C:/Users/chandlermei/Desktop/letgo-code (1)/node_modules/@fesjs/runtime'; + +const plugin = new Plugin({ + validKeys: ['modifyClientRenderOpts','rootContainer','onAppCreated','render','patchRoutes','modifyCreateHistory','modifyRoute','beforeRender','onRouterCreated','request',], +}); + +export { plugin }; diff --git a/linkis-web/src/apps/PythonModule/src/.fes/core/pluginExports.js b/linkis-web/src/apps/PythonModule/src/.fes/core/pluginExports.js new file mode 100644 index 0000000000..f3122607bd --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/core/pluginExports.js @@ -0,0 +1,2 @@ +export { useModel } from '../plugin-model/core.js'; +export * from '../plugin-request/request.js'; diff --git a/linkis-web/src/apps/PythonModule/src/.fes/core/pluginRegister.js b/linkis-web/src/apps/PythonModule/src/.fes/core/pluginRegister.js new file mode 100644 index 0000000000..4e3ab9aa4c --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/core/pluginRegister.js @@ -0,0 +1,25 @@ +import { plugin } from './plugin'; +import * as Plugin_0 from 'C:/Users/chandlermei/Desktop/letgo-code (1)/src/app.jsx'; +import * as Plugin_1 from '@@/core/routes/runtime.js'; + +function handleDefaultExport(pluginExports) { + // 避免编译警告 + const defaultKey = 'default'; + if (pluginExports[defaultKey]) { + const {default: defaultExport, ...otherExports} = pluginExports; + return { + ...defaultExport, + ...otherExports + } + } + return pluginExports; +} + + plugin.register({ + apply: handleDefaultExport(Plugin_0), + path: 'C:/Users/chandlermei/Desktop/letgo-code (1)/src/app.jsx', + }); + plugin.register({ + apply: handleDefaultExport(Plugin_1), + path: '@@/core/routes/runtime.js', + }); diff --git a/linkis-web/src/apps/PythonModule/src/.fes/core/routes/routeExports.js b/linkis-web/src/apps/PythonModule/src/.fes/core/routes/routeExports.js new file mode 100644 index 0000000000..4a5dd6fe73 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/core/routes/routeExports.js @@ -0,0 +1,105 @@ +import { createApp } from 'vue'; +import { createRouter as createVueRouter, createWebHashHistory, ApplyPluginsType } from 'C:/Users/chandlermei/Desktop/letgo-code (1)/node_modules/@fesjs/runtime'; +import { plugin } from '../plugin'; +import { updateInitialState } from '../../initialState'; + +const ROUTER_BASE = ''; +let router = null; +let history = null; +export const createRouter = (routes) => { + const createHistory = plugin.applyPlugins({ + key: 'modifyCreateHistory', + type: ApplyPluginsType.modify, + args: { + base: ROUTER_BASE + }, + initialValue: createWebHashHistory, + }); + // 修改routes + plugin.applyPlugins({ + key: 'patchRoutes', + type: ApplyPluginsType.event, + args: { routes }, + }); + const route = plugin.applyPlugins({ + key: 'modifyRoute', + type: ApplyPluginsType.modify, + initialValue: { + base: ROUTER_BASE, + routes: routes, + createHistory: createHistory + }, + }); + + history = route['createHistory']?.(route.base); + router = createVueRouter({ + history, + routes: route.routes + }); + + let isInit = false + router.beforeEach(async (to, from, next) => { + if(isInit){ + return next() + } + isInit = true + const beforeRenderConfig = plugin.applyPlugins({ + key: "beforeRender", + type: ApplyPluginsType.modify, + initialValue: { + loading: null, + action: null + }, + }); + if (typeof beforeRenderConfig.action !== "function") { + return next(); + } + const rootElement = document.createElement('div'); + document.body.appendChild(rootElement) + const app = createApp(beforeRenderConfig.loading); + app.mount(rootElement); + try { + const initialState = await beforeRenderConfig.action({router, history}); + updateInitialState(initialState || {}) + next(); + } catch(e){ + next(false); + window.console.error(`[fes] beforeRender执行出现异常：`); + window.console.error(e); + } + app.unmount(); + app._container.innerHTML = ''; + document.body.removeChild(rootElement); + }) + + plugin.applyPlugins({ + key: 'onRouterCreated', + type: ApplyPluginsType.event, + args: { router, history }, + }); + + return router; +}; + +export const getRouter = ()=>{ + if(!router){ + window.console.warn(`[preset-build-in] router is null`) + } + return router; +} + +export const getHistory = ()=>{ + if(!history){ + window.console.warn(`[preset-build-in] history is null`) + } + return history; +} + +export const destroyRouter = ()=>{ + router = null; + history = null; +} + +export const defineRouteMeta = (param)=>{ + return param +} diff --git a/linkis-web/src/apps/PythonModule/src/.fes/core/routes/routes.js b/linkis-web/src/apps/PythonModule/src/.fes/core/routes/routes.js new file mode 100644 index 0000000000..9da8b96e49 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/core/routes/routes.js @@ -0,0 +1,20 @@ + + +import pythonModuleAgentIndexJsx from 'C:/Users/chandlermei/Desktop/letgo-code (1)/src/pages/pythonModuleAgent/index.jsx' + +export function getRoutes() { + const routes = [ + { + "path": "/pythonModuleAgent", + "component": pythonModuleAgentIndexJsx, + "name": "pythonModuleAgent", + "meta": { + "name": "pythonModuleAgent", + "title": "agent生成" + }, + "count": 7 + } +]; + return routes; +} + diff --git a/linkis-web/src/apps/PythonModule/src/.fes/core/routes/runtime.js b/linkis-web/src/apps/PythonModule/src/.fes/core/routes/runtime.js new file mode 100644 index 0000000000..5b3deaf9d3 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/core/routes/runtime.js @@ -0,0 +1,6 @@ +import { createRouter } from "./routeExports"; + +export function onAppCreated({ app, routes }) { + const router = createRouter(routes); + app.use(router); +} diff --git a/linkis-web/src/apps/PythonModule/src/.fes/defaultContainer.vue b/linkis-web/src/apps/PythonModule/src/.fes/defaultContainer.vue new file mode 100644 index 0000000000..2785996d86 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/defaultContainer.vue @@ -0,0 +1,3 @@ + diff --git a/linkis-web/src/apps/PythonModule/src/.fes/fes.js b/linkis-web/src/apps/PythonModule/src/.fes/fes.js new file mode 100644 index 0000000000..2253048309 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/fes.js @@ -0,0 +1,65 @@ + +import { + createApp, +} from 'vue'; +import { plugin } from './core/plugin'; +import './core/pluginRegister'; +import { ApplyPluginsType } from 'C:/Users/chandlermei/Desktop/letgo-code (1)/node_modules/@fesjs/runtime'; +import { getRoutes } from './core/routes/routes'; +import DefaultContainer from './defaultContainer.vue'; + + + +import '../global.less'; + +const renderClient = (opts = {}) => { + const { plugin, routes, rootElement } = opts; + const rootContainer = plugin.applyPlugins({ + type: ApplyPluginsType.modify, + key: 'rootContainer', + initialValue: DefaultContainer, + args: { + routes: routes, + plugin: plugin + } + }); + + const app = createApp(rootContainer); + + plugin.applyPlugins({ + key: 'onAppCreated', + type: ApplyPluginsType.event, + args: { app, routes }, + }); + + if (rootElement) { + app.mount(rootElement); + } + return app; +} + +const getClientRender = (args = {}) => plugin.applyPlugins({ + key: 'render', + type: ApplyPluginsType.compose, + initialValue: () => { + const opts = plugin.applyPlugins({ + key: 'modifyClientRenderOpts', + type: ApplyPluginsType.modify, + initialValue: { + routes: args.routes || getRoutes(), + plugin, + rootElement: '#app', + defaultTitle: `fes.js`, + }, + }); + return renderClient(opts); + }, + args, +}); + +const clientRender = getClientRender(); + +const app = clientRender(); + + + diff --git a/linkis-web/src/apps/PythonModule/src/.fes/initialState.js b/linkis-web/src/apps/PythonModule/src/.fes/initialState.js new file mode 100644 index 0000000000..0d2f439a16 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/initialState.js @@ -0,0 +1,7 @@ +import { reactive } from 'vue'; + +export const initialState = reactive({}); + +export const updateInitialState = (obj) => { + Object.assign(initialState, obj); +}; diff --git a/linkis-web/src/apps/PythonModule/src/.fes/plugin-model/core.js b/linkis-web/src/apps/PythonModule/src/.fes/plugin-model/core.js new file mode 100644 index 0000000000..8e4b219b88 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/plugin-model/core.js @@ -0,0 +1,24 @@ + + +import initialState from 'C:/Users/chandlermei/Desktop/letgo-code (1)/src/.fes/plugin-model/models/initialState'; + +export const models = { + '@@initialState': initialState, +}; + + +const cache = new Map(); + +export const useModel = (name) => { + const modelFunc = models[name]; + if (modelFunc === undefined) { + throw new Error('[plugin-model]: useModel, name is undefined.'); + } + if (typeof modelFunc !== 'function') { + throw new Error('[plugin-model]: useModel is not a function.'); + } + if (!cache.has(name)) { + cache.set(name, modelFunc()); + } + return cache.get(name); +}; diff --git a/linkis-web/src/apps/PythonModule/src/.fes/plugin-model/models/initialState.js b/linkis-web/src/apps/PythonModule/src/.fes/plugin-model/models/initialState.js new file mode 100644 index 0000000000..34ab20c12e --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/plugin-model/models/initialState.js @@ -0,0 +1,5 @@ +import { initialState } from '@@/initialState'; + +export default function initialStateModel() { + return initialState; +} diff --git a/linkis-web/src/apps/PythonModule/src/.fes/plugin-request/request.js b/linkis-web/src/apps/PythonModule/src/.fes/plugin-request/request.js new file mode 100644 index 0000000000..47a044d859 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/.fes/plugin-request/request.js @@ -0,0 +1,69 @@ +import { ApplyPluginsType, plugin } from '@fesjs/fes'; +import { ref, shallowRef } from 'vue'; + +import { createRequest } from '@qlin/request'; + +function getRequestInstance() { + const defaultConfig = plugin.applyPlugins({ + key: 'request', + type: ApplyPluginsType.modify, + initialValue: { + timeout: 10000, + }, + }); + + return createRequest(defaultConfig); +} + +let currentRequest; + +export function rawRequest(url, data, options = {}) { + if (typeof options === 'string') { + options = { + method: options, + }; + } + if (!currentRequest) { + currentRequest = getRequestInstance(); + } + return currentRequest(url, data, options); +} + +export async function request(url, data, options = {}) { + const response = await rawRequest(url, data, options); + return response.data; +} + +request.version = '4.0.0'; + +function isPromiseLike(obj) { + return !!obj && typeof obj === 'object' && typeof obj.then === 'function'; +} + +export function useRequest(url, data, options = {}) { + const loadingRef = ref(true); + const errorRef = ref(null); + const dataRef = shallowRef(null); + let promise; + if (isPromiseLike(url)) { + promise = url; + } + else { + promise = request(url, data, options); + } + promise + .then((res) => { + dataRef.value = res; + }) + .catch((error) => { + errorRef.value = error; + }) + .finally(() => { + loadingRef.value = false; + }); + return { + loading: loadingRef, + error: errorRef, + data: dataRef, + }; +} diff --git a/linkis-web/src/apps/PythonModule/src/global.less b/linkis-web/src/apps/PythonModule/src/global.less new file mode 100644 index 0000000000..67cc7b0090 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/global.less @@ -0,0 +1,11 @@ +@import './letgo/global.css'; + +html, +body { + margin: 0; + padding: 0; +} + +#app { + height: 100vh; +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/cache-control.js b/linkis-web/src/apps/PythonModule/src/letgo/cache-control.js new file mode 100644 index 0000000000..b9b52e880c --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/cache-control.js @@ -0,0 +1,232 @@ +import { isPlainObject, isString } from 'lodash-es'; +import { IEnumCacheType } from './letgoConstants'; + +const CACHE_KEY_PREFIX = 'letgo-query_'; + +function isURLSearchParams(obj) { + return Object.prototype.toString.call(obj) === '[object URLSearchParams]'; +} + +function stringifyParams(params) { + if (isURLSearchParams(params)) return params.toString(); + + if (typeof params === 'string') return params; + + if (isPlainObject(params)) return JSON.stringify(params); + + return ''; +} + +function genInnerKey(config) { + const key = `${config.id}_${stringifyParams(config.params)}`; + if (config.type !== IEnumCacheType.RAM) return `${CACHE_KEY_PREFIX}${key}`; + + return key; +} + +function getFormattedCache(config) { + return { + id: config.id, + enableCaching: config.enableCaching || true, + type: config.type || IEnumCacheType.RAM, + cacheDuration: (config.cacheDuration || 0) * 1000, + }; +} + +function canCache(data) { + return ( + isPlainObject(data) || + isString(data) || + Array.isArray(data) || + isURLSearchParams(data) + ); +} + +function isExpire(cacheData) { + if (!cacheData.cacheDuration || cacheData.expire >= Date.now()) + return false; + + return true; +} + +class RamCache { + constructor() { + this.data = new Map(); + } + + get(key) { + const result = this.data.get(key); + if (result && isExpire(result)) { + this.data.delete(key); + return null; + } + return result ? result.data : null; + } + + set(key, value) { + // 超时清理数据 + this.data.forEach((value, key, map) => { + if (isExpire(value)) map.delete(key); + }); + if (this.data.size > 1000) { + window.console.warn( + 'Request: ram cache is exceed 1000 item, please check cache size', + ); + return; + } + + this.data.set(key, value); + } + + deleteWithPrefix(prefix) { + this.data.forEach((value, key, map) => { + if (key.startsWith(prefix)) map.delete(key); + }); + } + + delete(key) { + this.data.delete(key); + } +} + +const rawCacheImpl = new RamCache(); + +function setCacheData(key, response, config) { + const currentCacheData = { + cacheType: config.type, + data: response, + cacheDuration: config.cacheDuration, + expire: Date.now() + config.cacheDuration || 0, + }; + if (config.type !== IEnumCacheType.RAM) { + const cacheInstance = window[config.type]; + try { + cacheInstance.setItem(key, JSON.stringify(currentCacheData)); + } catch (e) { + // setItem 出现异常，清理缓存 + for (const item in cacheInstance) { + if ( + item.startsWith(CACHE_KEY_PREFIX) && + Object.prototype.hasOwnProperty.call(cacheInstance, item) + ) + cacheInstance.removeItem(item); + } + } + } else { + rawCacheImpl.set(key, currentCacheData); + } +} + +function getCacheData(key, config) { + if (config.type !== IEnumCacheType.RAM) { + const cacheInstance = window[config.type]; + const text = cacheInstance.getItem(key) || null; + try { + const currentCacheData = JSON.parse(text); + if (currentCacheData && !isExpire(currentCacheData)) + return currentCacheData.data; + + cacheInstance.removeItem(key); + return null; + } catch (e) { + cacheInstance.removeItem(key); + return null; + } + } else { + return rawCacheImpl.get(key); + } +} + +// 存储缓存队列 +const cacheStartFlag = new Map(); +const cachingQueue = new Map(); + +/** + * 等上一次请求结果 + * 1. 如果上一次请求成功，直接使用上一次的请求结果 + * 2. 如果上一次请求失败，重启本次请求 + */ +function handleCachingStart(key) { + const caching = cacheStartFlag.get(key); + if (caching) { + return new Promise((resolve) => { + const queue = cachingQueue.get(key) || []; + cachingQueue.set(key, queue.concat(resolve)); + }); + } + cacheStartFlag.set(key, true); +} + +// 有请求成功的 +function handleCachingQueueSuccess(key, response) { + // 移除首次缓存 flag + const queue = cachingQueue.get(key); + if (queue && queue.length > 0) { + queue.forEach((resolve) => { + resolve(response); + }); + } + cachingQueue.delete(key); + cacheStartFlag.delete(key); +} + +// 处理请求失败 +function handleCachingQueueError(key) { + const queue = cachingQueue.get(key); + if (queue && queue.length > 0) { + const firstResolve = queue.shift(); + firstResolve(); + cachingQueue.set(key, queue); + } else { + cachingQueue.delete(key); + cacheStartFlag.delete(key); + } +} + +export function clearCache(id, type) { + if (type !== IEnumCacheType.RAM) { + const prefix = `${CACHE_KEY_PREFIX}${id}_`; + const storage = window[type]; + for (const key in storage) { + if ( + key.startsWith(prefix) && + Object.prototype.hasOwnProperty.call(storage, key) + ) + storage.removeItem(key); + } + } else { + rawCacheImpl.deleteWithPrefix(`${id}_`); + } +} + +export async function cacheControl(config, fn) { + if (config.enableCaching) { + const key = genInnerKey(config); + const cacheConfig = getFormattedCache(config); + const cacheData = getCacheData(key, cacheConfig); + if (cacheData) return cacheData; + + let response = await handleCachingStart(key); + if (response) return response; + + try { + response = await fn(); + + if (canCache(response)) { + handleCachingQueueSuccess(key, response); + setCacheData(key, response, cacheConfig); + } else { + window.console.warn( + `[query cache]: ${key} 响应数据无法序列化，无法缓存，请移除相关配置`, + ); + } + + return response; + } catch (err) { + handleCachingQueueError(key); + throw err; + } + } else { + return fn(); + } +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/components/pageLoading.vue b/linkis-web/src/apps/PythonModule/src/letgo/components/pageLoading.vue new file mode 100644 index 0000000000..2d396d549f --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/components/pageLoading.vue @@ -0,0 +1,24 @@ + + + + + \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/src/letgo/global.css b/linkis-web/src/apps/PythonModule/src/letgo/global.css new file mode 100644 index 0000000000..db80f5d8e0 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/global.css @@ -0,0 +1,4 @@ +.buttons .fes-form-item-content { + display: flex; + justify-content: space-between; +} \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/src/letgo/globalBase.js b/linkis-web/src/apps/PythonModule/src/letgo/globalBase.js new file mode 100644 index 0000000000..93178b2463 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/globalBase.js @@ -0,0 +1,21 @@ +import { omit } from 'lodash-es'; +import { letgoRequest } from './letgoRequest'; + +export class LetgoGlobalBase { + constructor(globalCtx) { + this._globalCtx = globalCtx; + this.$request = letgoRequest; + } + + get $utils() { + return this._globalCtx.$utils; + } + + get $context() { + return this._globalCtx.$context; + } + + get $globalCode() { + return omit(this._globalCtx, '$utils', '$context'); + } +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/letgoConstants.js b/linkis-web/src/apps/PythonModule/src/letgo/letgoConstants.js new file mode 100644 index 0000000000..79ee660c8d --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/letgoConstants.js @@ -0,0 +1,16 @@ +export const IEnumRunCondition = { + MANUAL: 0, + PageLoads: 1, +}; + +export const IEnumCodeType = { + JAVASCRIPT_QUERY: 'query', + JAVASCRIPT_COMPUTED: 'computed', + TEMPORARY_STATE: 'temporaryState', +}; + +export const IEnumCacheType = { + RAM: 'ram', + SESSION_STORAGE: 'sessionStorage', + LOCAL_STORAGE: 'localStorage', +}; diff --git a/linkis-web/src/apps/PythonModule/src/letgo/letgoRequest.js b/linkis-web/src/apps/PythonModule/src/letgo/letgoRequest.js new file mode 100644 index 0000000000..2711e51bd5 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/letgoRequest.js @@ -0,0 +1,2 @@ +import { rawRequest } from '@fesjs/fes'; +export const letgoRequest = rawRequest; diff --git a/linkis-web/src/apps/PythonModule/src/letgo/pageBase.js b/linkis-web/src/apps/PythonModule/src/letgo/pageBase.js new file mode 100644 index 0000000000..6ccce754a6 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/pageBase.js @@ -0,0 +1,9 @@ +import { LetgoGlobalBase } from './globalBase'; + +export class LetgoPageBase extends LetgoGlobalBase { + constructor(ctx) { + super(ctx.globalCtx); + this.$pageCode = ctx.codes; + this.$refs = ctx.instances; + } +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/reactive.js b/linkis-web/src/apps/PythonModule/src/letgo/reactive.js new file mode 100644 index 0000000000..48cd8d10ff --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/reactive.js @@ -0,0 +1,74 @@ +import { computed, reactive, shallowReactive } from 'vue'; +import { getAllMethodAndProperties } from './shared.js'; + +export function markClassReactive(target, filter) { + const members = getAllMethodAndProperties(target).filter((member) => { + if (filter) return filter(member); + + return true; + }); + + const state = reactive( + members.reduce((acc, cur) => { + acc[cur] = target[cur]; + return acc; + }, {}), + ); + members.forEach((key) => { + Object.defineProperty(target, key, { + get() { + return state[key]; + }, + set(value) { + state[key] = value; + }, + }); + }); + return target; +} + +export function markShallowReactive(target, properties) { + const state = shallowReactive(properties); + Object.keys(properties).forEach((key) => { + Object.defineProperty(target, key, { + get() { + return state[key]; + }, + set(value) { + state[key] = value; + }, + }); + }); + return target; +} + +export function markReactive(target, properties) { + const state = reactive(properties); + Object.keys(properties).forEach((key) => { + Object.defineProperty(target, key, { + get() { + return state[key]; + }, + set(value) { + state[key] = value; + }, + }); + }); + return target; +} + +export function markComputed(target, properties) { + const prototype = Object.getPrototypeOf(target); + properties.forEach((key) => { + const descriptor = Object.getOwnPropertyDescriptor(prototype, key); + if (descriptor?.get) { + const tmp = computed(descriptor.get.bind(target)); + Object.defineProperty(target, key, { + get() { + return tmp.value; + }, + }); + } + }); + return target; +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/shared.js b/linkis-web/src/apps/PythonModule/src/letgo/shared.js new file mode 100644 index 0000000000..751813dafb --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/shared.js @@ -0,0 +1,36 @@ +export function isGetterProp(instance, key) { + if (key in instance) { + let p = instance; + let propDesc; + while (p && !propDesc) { + propDesc = Object.getOwnPropertyDescriptor(p, key); + p = Object.getPrototypeOf(p); + } + // only getter + return !!propDesc && propDesc.get && !propDesc.set; + } + return false; +} + +export function getAllMethodAndProperties(obj) { + let props = []; + + do { + const l = Object.getOwnPropertyNames(obj) + .concat(Object.getOwnPropertySymbols(obj).map((s) => s.toString())) + .sort() + .filter( + (p, i, arr) => + !['constructor', '_globalCtx'].includes(p) && // not the constructor + (i === 0 || p !== arr[i - 1]) && // not overriding in this prototype + !props.includes(p), // not overridden in a child + ); + props = props.concat(l); + } while ( + // eslint-disable-next-line no-cond-assign + (obj = Object.getPrototypeOf(obj)) && // walk-up the prototype chain + Object.getPrototypeOf(obj) // not the the Object prototype methods (hasOwnProperty, etc...) + ); + + return props.reverse(); +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/useComputed.js b/linkis-web/src/apps/PythonModule/src/letgo/useComputed.js new file mode 100644 index 0000000000..3522a95a25 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/useComputed.js @@ -0,0 +1,15 @@ +import { computed, reactive } from 'vue'; + +export function useComputed({ id, func }) { + return reactive({ + id, + value: computed(() => { + try { + return func(); + } catch (_) { + window.console.warn(_); + return null; + } + }), + }); +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/useInstance.js b/linkis-web/src/apps/PythonModule/src/letgo/useInstance.js new file mode 100644 index 0000000000..a734ef6fb0 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/useInstance.js @@ -0,0 +1,24 @@ +import { ref } from 'vue'; + +export function useInstance() { + const refEl = ref(); + const proxy = new Proxy( + {}, + { + get(_, key) { + if ( + typeof refEl.value?.[key] === 'function' && + refEl.value.hasOwnProperty && + !refEl.value.hasOwnProperty(key) + ) { + return refEl.value[key].bind(refEl.value); + } + return refEl.value?.[key]; + }, + set(_, key, value) { + if (refEl.value) refEl.value[key] = value; + }, + }, + ); + return [refEl, proxy]; +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/useJSQuery.js b/linkis-web/src/apps/PythonModule/src/letgo/useJSQuery.js new file mode 100644 index 0000000000..ca38075942 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/useJSQuery.js @@ -0,0 +1,124 @@ +import { isPlainObject } from 'lodash-es'; +import { markReactive } from './reactive'; +import { IEnumRunCondition, IEnumCacheType } from './letgoConstants'; +import { cacheControl, clearCache } from './cache-control'; + +class JSQuery { + constructor(data) { + markReactive(this, { + id: data.id, + query: data.query, + params: data.params, + + enableCaching: data.enableCaching || false, + cacheDuration: data.cacheDuration || null, + cacheType: data.cacheType || IEnumCacheType.RAM, + + enableTransformer: data.enableTransformer || false, + transformer: data.transformer, + runWhenPageLoads: data.runWhenPageLoads || false, + queryTimeout: data.queryTimeout, + runCondition: data.runCondition || IEnumRunCondition.MANUAL, + successEvent: data.successEvent || [], + failureEvent: data.failureEvent || [], + data: null, + response: null, + error: null, + loading: false, + }); + + this.hasBeenCalled = false; + } + + timeoutPromise(timeout) { + return new Promise((resolve, reject) => { + setTimeout(() => { + reject({ + type: 'TIMEOUT', + msg: '请求超时', + }); + }, timeout); + }); + } + + formatParams(extraParams) { + const params = this.params; + if (!params) { + return extraParams || null; + } + + if (isPlainObject(params) && isPlainObject(extraParams)) + return { ...params, ...extraParams }; + + return params; + } + + async trigger(extraParams) { + if (this.query) { + try { + this.hasBeenCalled = true; + this.loading = true; + const params = this.formatParams(extraParams); + const response = await cacheControl( + { + id: this.id, + enableCaching: this.enableCaching, + cacheDuration: this.cacheDuration, + type: this.cacheType, + params, + }, + async () => { + if (this.queryTimeout) + return Promise.race([ + this.timeoutPromise(this.queryTimeout), + this.query(params), + ]); + + return this.query(params); + }, + ); + + this.response = response; + let data = response?.data; + if (this.enableTransformer && this.transformer) + data = await this.transformer(data); + + this.data = data; + this.error = null; + this.successEvent.forEach((eventHandler) => { + eventHandler(data); + }); + return this.data; + } catch (err) { + this.data = null; + this.failureEvent.forEach((eventHandler) => { + eventHandler(err); + }); + if (err instanceof Error) this.error = err.message; + + window.console.warn(err); + throw err; + } finally { + this.loading = false; + } + } + } + + clearCache() { + clearCache(this.id, this.cacheType); + } + + reset() { + this.clearCache(); + this.data = null; + this.error = null; + } +} + +export function useJSQuery(data) { + const result = new JSQuery(data); + + if (!data._isGlobalQuery && data.runWhenPageLoads) result.trigger(); + + return result; +} diff --git a/linkis-web/src/apps/PythonModule/src/letgo/useLetgoGlobal.js b/linkis-web/src/apps/PythonModule/src/letgo/useLetgoGlobal.js new file mode 100644 index 0000000000..50f0814934 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/useLetgoGlobal.js @@ -0,0 +1,78 @@ +import { reactive, computed } from 'vue'; +import { isPlainObject } from 'lodash-es'; +import dayjs from 'dayjs'; +import _ from 'lodash'; +import { createGlobalState } from '@vueuse/core'; +import { useRouter, useRoute } from '@fesjs/fes'; +import { FMessage, FModal } from '@fesjs/fes-design'; + +function useLetgoGlobal() { + const router = useRouter(); + const route = useRoute(); + + const $context = reactive({}); + const letgoContext = $context; + + $context.userInfo = { + user: '', + uriList: [], + roleList: [], + }; + $context.publicPath = process.env.BASE_URL; + $context.urlParams = computed(() => route.query || {}); + $context.navigateTo = (routeName, params) => { + if (params && isPlainObject(params)) { + router.push({ + name: routeName, + query: params, + }); + } else { + router.push({ + name: routeName, + }); + } + }; + $context.navigateBack = (routeName, params) => { + router.back(); + }; + + const $utils = { + FMessage, + FModal, + dayjs, + _, + }; + const utils = $utils; + + const __globalCtx = { + $context, + letgoContext, + $utils, + utils, + }; + + const __query_deps = {}; + + return new Proxy(__globalCtx, { + get(obj, prop) { + if ([].includes(prop) && !__globalCtx[prop].hasBeenCalled) + __globalCtx[prop].trigger(); + + const currentQuery = __query_deps[prop]; + if (currentQuery) { + currentQuery.forEach((d) => { + if ( + __globalCtx[d].runWhenPageLoads && + !__globalCtx[d].hasBeenCalled + ) { + __globalCtx[d].trigger(); + } + }); + } + + return obj[prop]; + }, + }); +} + +export const useSharedLetgoGlobal = createGlobalState(useLetgoGlobal); diff --git a/linkis-web/src/apps/PythonModule/src/letgo/useTemporaryState.js b/linkis-web/src/apps/PythonModule/src/letgo/useTemporaryState.js new file mode 100644 index 0000000000..280f458a7f --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/letgo/useTemporaryState.js @@ -0,0 +1,23 @@ +import { reactive } from 'vue'; +import { isPlainObject, set } from 'lodash-es'; + +export function useTemporaryState({ id, initValue }) { + const result = reactive({ + id, + value: initValue, + setValue, + setIn, + }); + + function setIn(path, val) { + if (isPlainObject(result.value)) { + set(result.value, path, val); + } + } + + function setValue(val) { + result.value = val; + } + + return result; +} diff --git a/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx b/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx new file mode 100644 index 0000000000..a4191e1f65 --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx @@ -0,0 +1,781 @@ +import { onMounted, defineComponent } from 'vue'; +import { defineRouteMeta } from '@fesjs/fes'; +import { FForm, FFormItem, FButton, FModal, FText } from '@fesjs/fes-design'; +import { + WInput, + WSelect, + WRadioGroup, + WUpload, + WTextarea, + WTable, +} from '@webank/fes-design-material'; +import { useSharedLetgoGlobal } from '../../letgo/useLetgoGlobal'; +import { useJSQuery } from '../../letgo/useJSQuery'; +import { letgoRequest } from '../../letgo/letgoRequest'; +import { Main } from './main'; +import { markClassReactive } from '../../letgo/reactive.js'; +import { isGetterProp } from '../../letgo/shared.js'; + +defineRouteMeta({ + name: 'index', + title: 'agent生成', +}); + +export default defineComponent({ + name: 'index', + setup() { + const { $context, $utils } = useSharedLetgoGlobal(); + + const apiPythonfileexistUdf = useJSQuery({ + id: 'apiPythonfileexistUdf', + query(params) { + return letgoRequest( + '/api/rest_j/v1/udf/python-file-exist', + params, + { + method: 'GET', + headers: {}, + }, + ); + }, + + runCondition: 'manual', + + successEvent: [], + failureEvent: [], + }); + + const apiPythonuploadFilesystem = useJSQuery({ + id: 'apiPythonuploadFilesystem', + query(params) { + return letgoRequest( + '/api/rest_j/v1/filesystem/python-upload', + params, + { + method: 'POST' + }, + ); + }, + + runCondition: 'manual', + + successEvent: [], + failureEvent: [], + }); + + const apiPythonsaveUdf = useJSQuery({ + id: 'apiPythonsaveUdf', + query(params) { + return letgoRequest('/api/rest_j/v1/udf/python-save', params, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }); + }, + + runCondition: 'manual', + + successEvent: [], + failureEvent: [], + }); + + const apiPythondeleteUdf = useJSQuery({ + id: 'apiPythondeleteUdf', + query(params) { + return letgoRequest( + '/api/rest_j/v1/udf/python-delete', + params, + { + method: 'GET', + headers: {}, + }, + ); + }, + + runCondition: 'manual', + + successEvent: [], + failureEvent: [], + }); + + const apiPythonlistUdf = useJSQuery({ + id: 'apiPythonlistUdf', + query(params) { + return letgoRequest('/api/rest_j/v1/udf/python-list', params, { + method: 'GET', + headers: {}, + }); + }, + + runCondition: 'manual', + + successEvent: [], + failureEvent: [], + }); + + const __instance__ = new Main({ + globalCtx: { + $utils, + $context, + }, + instances: {}, + codes: { + apiPythonlistUdf, + apiPythonfileexistUdf, + apiPythonuploadFilesystem, + apiPythonsaveUdf, + apiPythondeleteUdf, + }, + }); + const $$ = markClassReactive(__instance__, (member) => { + if ( + member.startsWith('_') || + member.startsWith('$') || + isGetterProp(__instance__, member) || + typeof __instance__[member] === 'function' + ) + return false; + + return true; + }); + + onMounted($$.onMounted.bind($$)); + + const wTable2Columns9RenderSlots = (slotProps) => { + return ( +
+ { + $$.showEditModuleModal(slotProps.row); + }, + ]} + type="link" + style={{padding: '0px', width: '45px'}} + > + 编辑 + + { + $$.showLoadStatusChangeConfirmation( + slotProps.row, + ); + }, + ]} + type="link" + style={{padding: '0px', width: '45px'}} + > + {slotProps.row.isLoad === 1 ? '已加载' : '未加载'} + + { + $$.showDeleteConfirmation(slotProps.row); + }, + ]} + type="link" + style={{ color: slotProps.row.isExpire === 1 ? '#CFD0D3' : 'red', padding: '0px', width: '45px'}} + > + 删除 + +
+ ); + }; + + return () => { + return ( +
+ + + + + + + + { + $$.currentPage = 1 + $$.loadPythonModuleList( + $$.pythonModuleName, + $$.userName, + $$.engineType, + $$.isExpired, + $$.isLoaded, + $$.currentPage, + $$.pageSize, + ); + }, + ]} + > + 搜索 + + { + $$.currentPage = 1; + $$.resetQueryParameters(); + }, + ]} + type="primary" + style={{ backgroundColor: '#ff9900', color: '#fff', borderColor: '#ff9900' }} + > + 清空 + + { + $$.showAddModuleModal(); + }, + ]} + type="primary" + style={{ backgroundColor: '#19be6b', color: '#fff', borderColor: '#19be6b' }} + > + 新增 + + + + document.body} + onUpdate:show={[ + () => { + $$.closeEditModuleModal(); + }, + ]} + onOk={[ + () => { + $$.handleEditModule(); + }, + ]} + onCancel={[ + () => { + $$.closeEditModuleModal(); + }, + ]} + > + $$.editFormRef = el} model={$$.selectedModule} labelWidth={80}> + + + { + if($$.selectedModule.path && $$.selectedModule.name) { + return true; + } + return false + } + }, + ]} + v-slots={{ + tip: () => { + if(!$$.selectedModule.name) { + return ( + + {'仅支持上传 .py和.zip格式文件'} + + ); + } + else { + return '' + } + + }, + fileList: () => { + if($$.selectedModule.name && $$.selectedModule.path) { + return ( +
+ {$$.selectedModule.name + '.' + $$.selectedModule.path?.split('.')[1] || ''} +
+ ); + } + else { + return '' + } + + } + }} + beforeUpload={(...args) => $$.validateModuleFile(...args)} + accept={['.zip', '.py']} + httpRequest={(...args) => $$.handleUploadHttpRequest(...args)} + /> + + + + + + { + $$.closeAddModuleModal(); + }, + ]} + onOk={() => { + $$.handleAddModule(); + }} + > + $$.addFormRef = el} model={$$.selectedModule} labelWidth={80}> + + + { + if($$.selectedModule.path && $$.selectedModule.name) { + return true; + } + return false + } + }, + ]} + v-slots={{ + tip: () => { + if(!$$.selectedModule.name) { + return ( + + {'仅支持上传 .py和.zip格式文件'} + + ); + } + else { + return '' + } + + }, + fileList: () => { + if($$.selectedModule.name && $$.selectedModule.path) { + return ( +
+ {$$.selectedModule.name + '.' + $$.selectedModule.path?.split('.')[1] || ''} +
+ ); + } + else { + return '' + } + + } + }} + beforeUpload={(...args) => $$.validateModuleFile(...args)} + accept={['.zip', '.py']} + httpRequest={(...args) => $$.handleUploadHttpRequest(...args)} + /> + + + + + + { + return row.engineType === 'all' + ? '通用' : row.engineType === 'spark' ? 'Spark' + : 'Python'; + }, + }, + { + prop: 'status', + label: '状态', + formatter: ({ + row, + column, + rowIndex, + coloumIndex, + cellValue, + }) => { + return row.isExpire === 0 + ? '正常' + : '过期'; + }, + }, + { + prop: 'path', + label: '路径信息', + }, + { + prop: 'description', + label: '模块描述', + }, + { + prop: 'createTime', + label: '创建时间', + formatter: ({ + row, + column, + rowIndex, + columnIndex, + cellValue, + }) => { + return $utils + .dayjs(row.createTime) + .format('YYYY-MM-DD HH:mm:ss'); + }, + }, + { + prop: 'updateTime', + label: '修改时间', + formatter: ({ + row, + column, + rowIndex, + columnIndex, + cellValue, + }) => { + return $utils + .dayjs(row.updateTime) + .format('YYYY-MM-DD HH:mm:ss'); + }, + }, + { + prop: 'createUser', + label: '创建人', + }, + { + prop: 'updateUser', + label: '修改人', + }, + { + prop: '', + label: '操作', + width: 200, + render: wTable2Columns9RenderSlots, + }, + ]} + data={$$.pythonModuleList} + pagination={{ + currentPage: $$.currentPage, + pageSize: $$.pageSize, + totalCount: $$.totalRecords, + showTotal: true, + alwayShow: true, + showQuickJumper: true, + }} + onChange={[(...args) => $$.handlePageChange(...args)]} + /> +
+ ); + }; + }, +}); diff --git a/linkis-web/src/apps/PythonModule/src/pages/index/main.js b/linkis-web/src/apps/PythonModule/src/pages/index/main.js new file mode 100644 index 0000000000..7e629aff7e --- /dev/null +++ b/linkis-web/src/apps/PythonModule/src/pages/index/main.js @@ -0,0 +1,385 @@ +import { LetgoPageBase } from '../../letgo/pageBase'; + +export class Main extends LetgoPageBase { + constructor (ctx) { + super(ctx); + this.pythonModuleName = ''; + this.userName = ''; + this.engineType = ''; + this.isExpired = 0; + this.isLoaded = null; + this.currentPage = 1; + this.pageSize = 10; + this.totalPages = 0; + this.totalRecords = 0; + this.pythonModuleList = []; + this.selectedModule = {}; + this.newModuleName = ''; + this.selectedEngineType = 'spark'; + this.selectedModuleDescription = ''; + this.selectedModulePath = ''; + this.selectedModuleIsLoad = 1; + this.selectedModuleIsExpire = 0; + this.selectedModuleId = null; + this.selectedModuleFile = null; + this.selectedModuleFileError = ''; + this.selectedModuleFileUploadStatus = false; + this.addPythonModuleVisible = false; + this.editPythonModuleVisible = false; + this.deleteConfirmationVisible = false; + this.loadStatusChangeConfirmationVisible = false; + this.addModuleModalVisible = false; + this.editModuleModalVisible = false; + this.addFormRef = null; + this.editFormRef = null; + } + + onMounted () { + this.loadPythonModuleList(); + } + + async loadPythonModuleList () { + try { + const params = { + name: this.pythonModuleName, + engineType: this.engineType, + username: this.userName, + // isLoad: this.isLoaded, + isExpire: this.isExpired, + pageNow: this.currentPage, + pageSize: this.pageSize + }; + if (this.isLoaded === 0 || this.isLoaded === 1) { + params.isLoad = this.isLoaded; + } + const response = + await this.$pageCode.apiPythonlistUdf.trigger(params); + this.pythonModuleList = response.data.pythonList; + this.totalRecords = response.data.totalPage; + return response; + } catch (error) { + window.console.error(error); + // throw error; + } + } + + handlePageChange (currentPage, pageSize) { + this.currentPage = currentPage; + this.loadPythonModuleList(); + } + + resetQueryParameters () { + this.pythonModuleName = ''; + this.userName = ''; + this.engineType = ''; + this.isExpired = 0; + this.isLoaded = null; + this.currentPage = 1; + this.pageSize = 10; + this.loadPythonModuleList(); + } + + showAddModuleModal () { + this.addPythonModuleVisible = true; + this.selectedModule.name = ''; + this.selectedModule.engineType = 'spark'; + this.selectedModule.isExpire = 0; + this.selectedModule.isLoad = 1; + this.selectedModule.path = ''; + this.selectedModule.fileList = []; + this.selectedModule.description = ''; + } + + showEditModuleModal (selectedModule) { + if (selectedModule && typeof selectedModule === 'object') { + this.selectedModule = { + ...selectedModule + }; + this.selectedModule.fileList = [{ uid: '12345', name: '123' }]; + this.editPythonModuleVisible = true; + } else { + this.$utils.FMessage.error({ + content: '模块信息无效，请重新选择模块。' + }); + } + } + + showDeleteConfirmation (selectedModule) { + this.selectedModule = selectedModule; + this.$utils.FModal.confirm({ + title: '确认删除', + content: `您确定要删除模块"${selectedModule.name}"吗？`, + mask: false, + onOk: async () => { + try { + await this.expirePythonModule( + selectedModule.id, + selectedModule.isExpire + ); + this.$utils.FMessage.success({ + content: '删除成功' + }); + await this.loadPythonModuleList(); + } catch (error) { + // this.$utils.FMessage.error({ + // content: '删除模块时发生错误，请检查网络或稍后重试。' + // }); + window.console.error(error); + } + }, + onCancel: () => { + this.deleteConfirmationVisible = false; + } + }); + this.deleteConfirmationVisible = true; + } + + showLoadStatusChangeConfirmation (selectedModule) { + this.selectedModule = { ...selectedModule }; + this.loadStatusChangeConfirmationVisible = true; + this.$utils.FModal.confirm({ + title: '确认加载状态变更', + mask: false, + content: `您确定要修改模块 ${selectedModule.name} 的加载状态吗？`, + onOk: async () => { + await this.handleLoadStatusChange(); + this.loadStatusChangeConfirmationVisible = false; + }, + onCancel: () => { + this.loadStatusChangeConfirmationVisible = false; + } + }); + } + + async validateModuleName (newModuleName) { + if (newModuleName.split('.')[0].length > 50) { + this.$utils.FMessage.error('模块名称长度不能超过50个字符'); + throw new Error('模块名称长度不能超过50个字符'); + } + // 名称只支持数字字母下划线，且以字母开头 + if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(newModuleName.split('.')[0])) { + this.$utils.FMessage.error('模块名称只支持数字字母下划线，且以字母开头'); + throw new Error('模块名称只支持数字字母下划线，且以字母开头'); + } + try { + const response = await this.$pageCode.apiPythonfileexistUdf.trigger( + { + fileName: newModuleName + } + ); + if (response.status === 0) { + return response.data; + } else { + this.$utils.FMessage.error(`模块名称${newModuleName}已存在，如需重新上传请先删除旧的模块`); + throw new Error(response.message); + } + } catch (err) { + // this.$utils.FMessage.error(`模块名称${newModuleName}已存在，如需重新上传请先删除旧的模块`); + throw new Error(`模块名称${newModuleName}已存在，如需重新上传请先删除旧的模块`); + } + } + + async validateModuleSize (size) { + if (size > 52428800) { + this.$utils.FMessage.error('模块大小不能超过50MB'); + throw new Error('模块大小不能超过50MB'); + } + } + + async validateModuleFile (file) { + try { + await this.validateModuleSize(file.size); + await this.validateModuleName(file.name); + this.selectedModule.fileList = this.selectedModule.fileList.filter(item => item.uid !== file.uid); + } catch (err) { + window.console.error(err); + return false; + } + } + + async handleUploadHttpRequest (options) { + try { + const formData = new FormData(); + window.console.log('options:', options); + formData.append('file', options.file); + formData.append('fileName', options.file.name); + const response = await this.$pageCode.apiPythonuploadFilesystem.trigger(formData); + this.selectedModule.path = response.data.filePath; + this.selectedModule.name = options.file.name.split('.')[0]; + this.selectedModule.fileList = [options.file]; + } catch (err) { + window.console.error(err); + // this.$utils.FMessage.error('上传失败'); + } + } + + async savePythonModule ( + newModuleName, + selectedEngineType, + selectedModuleDescription, + selectedModulePath, + selectedModuleIsLoad, + selectedModuleIsExpire, + selectedModuleId + ) { + const params = { + name: newModuleName, + description: selectedModuleDescription, + path: selectedModulePath, + engineType: selectedEngineType, + isLoad: selectedModuleIsLoad, + isExpire: selectedModuleIsExpire + }; + if (selectedModuleId) { + params.id = selectedModuleId; + } + try { + const response = + await this.$pageCode.apiPythonsaveUdf.trigger(params); + if (response.status === 0) { + await this.$utils.FMessage.success({ + content: '保存成功。' + }); + } else { + await this.$utils.FMessage.error({ + content: response.message + }); + } + return response; + } catch (error) { + // await this.$utils.FMessage.error({ + // content: '保存失败，请检查网络或稍后重试。' + // }); + window.console.error(error); + throw error; + } + } + + async expirePythonModule (selectedModuleId, isExpired) { + const response = await this.$pageCode.apiPythondeleteUdf.trigger({ + id: selectedModuleId, + isExpire: isExpired + }); + return response; + } + + async handleAddModule () { + await this.addFormRef.validate(); + try { + await this.savePythonModule( + this.selectedModule.name, + this.selectedModule.engineType, + this.selectedModule.description, + this.selectedModule.path, + this.selectedModule.isLoad, + this.selectedModule.isExpire + ); + this.addPythonModuleVisible = false; + this.loadPythonModuleList(); + } catch (err) { + window.console.error(err); + } + } + + async handleEditModule () { + await this.editFormRef.validate(); + try { + await this.savePythonModule( + this.selectedModule.name, + this.selectedModule.engineType, + this.selectedModule.description, + this.selectedModule.path, + this.selectedModule.isLoad, + this.selectedModule.isExpire, + this.selectedModule.id + ); + this.closeEditModuleModal(); + this.loadPythonModuleList(); + } catch (err) { + window.console.error(err); + } + } + + async handleDeleteModule () { + try { + if (this.selectedModuleId === null) { + throw new Error('模块ID未设置，无法执行删除操作'); + } + const response = await this.expirePythonModule( + this.selectedModuleId + ); + if (response.status === 0) { + this.$utils.FMessage.success({ + content: response.message + }); + } else { + this.$utils.FMessage.error({ + content: response.message + }); + } + } catch (error) { + // this.$utils.FMessage.error({ + // content: '模块删除失败，请检查网络或稍后重试。' + // }); + window.console.error('Error during delete module:', error); + } + } + + async handleLoadStatusChange () { + const { id, name, path, isExpire, isLoad, engineType, description } = + this.selectedModule; + window.console.log({ + id, + name, + path, + isExpire, + isLoad, + engineType, + description + }); + const targetLoadStatus = isLoad === 1 ? 0 : 1; + if (id === null) { + this.$utils.FMessage.error({ + content: '模块ID未设置，无法进行加载状态变更。' + }); + return; + } + try { + await this.savePythonModule( + name, + engineType, + description, + path, + targetLoadStatus, + isExpire, + id + ); + await this.loadPythonModuleList(); + } catch (error) { + // this.$utils.FMessage.error({ + // content: '模块加载状态更新时发生错误：' + error.message + // }); + window.console.error(error); + } + } + + closeAddModuleModal () { + this.addPythonModuleVisible = false; + } + + handleFileListChange ({ file, fileList }) { + this.selectedModule.fileList = []; + } + + closeEditModuleModal () { + this.editPythonModuleVisible = false; + } + + closeDeleteConfirmation () { + this.deleteConfirmationVisible = false; + } + + closeLoadStatusChangeConfirmation () { + this.loadStatusChangeConfirmationVisible = false; + } +} diff --git a/linkis-web/src/apps/PythonModule/tsconfig.json b/linkis-web/src/apps/PythonModule/tsconfig.json new file mode 100644 index 0000000000..5ccb046d8a --- /dev/null +++ b/linkis-web/src/apps/PythonModule/tsconfig.json @@ -0,0 +1,33 @@ +{ + "compilerOptions": { + "module": "esnext", + "target": "esnext", + "lib": [ + "esnext", + "dom" + ], + "sourceMap": true, + "baseUrl": ".", + "jsx": "preserve", + "allowSyntheticDefaultImports": true, + "moduleResolution": "node", + "forceConsistentCasingInFileNames": true, + "noImplicitReturns": true, + "noUnusedLocals": true, + "allowJs": true, + "experimentalDecorators": true, + "strict": true, + "paths": { + "@/*": [ + "./src/*" + ], + "@@/*": [ + "./src/.fes/*" + ] + } + }, + "include": [ + "src", + "package.json" + ] +} \ No newline at end of file diff --git a/linkis-web/src/apps/linkis/components/tag/index.vue b/linkis-web/src/apps/linkis/components/tag/index.vue index 108353cf72..33eb930935 100644 --- a/linkis-web/src/apps/linkis/components/tag/index.vue +++ b/linkis-web/src/apps/linkis/components/tag/index.vue @@ -182,9 +182,9 @@ export default { + diff --git a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue index 30a02bfc42..d52fd6b65f 100644 --- a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue +++ b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue @@ -55,7 +55,7 @@ import api from '@/common/service/api' import mixin from '@/common/service/mixin' import util from '@/common/util' import ViewLog from '@/apps/linkis/module/resourceManagement/log.vue' -import { isUndefined } from 'lodash' +import { cloneDeep, isUndefined } from 'lodash' import storage from '@/common/helper/storage'; export default { @@ -109,11 +109,17 @@ export default { hasEngine: false, param: {}, yarnAddress: '', + logTimer: null, } }, created() { this.hasResultData = false }, + unmouted() { + if(this.logTimer) { + clearTimeout(this.logTimer); + } + }, async mounted() { let taskID = this.$route.query.taskID let engineInstance = this.$route.query.engineInstance @@ -147,7 +153,7 @@ export default { this.foldFlag = true; }, // The request is triggered when the tab is clicked, and the log is requested at the beginning, and no judgment is made.(点击tab时触发请求，log初始就请求了，不做判断) - onClickTabs(name) { + async onClickTabs(name) { this.tabName = name if (name === 'result') { // Determine whether it is a result set(判断是否为结果集) @@ -174,8 +180,11 @@ export default { } } else if(name === 'code') { - window.console.log('click code') - + // window.console.log('click code') + if(this.codes.code) return; + const res = await api.fetch(`/jobhistory/job-extra-info?jobId=${this.$route.query.taskID}`, 'get') + const { executionCode } = res.metricsMap; + this.codes = { code: executionCode }; } else { this.$nextTick(() => { this.$refs.logRef.fold(); @@ -372,16 +381,53 @@ export default { return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; }, + async getLogs(jobhistory) { + const params = { + path: jobhistory.task.logPath + } + if (this.$route.query.proxyUser) { + params.proxyUser = this.$route.query.proxyUser + } + let openLog = {} + if (this.$route.query.status === 'Scheduled' || this.$route.query.status === 'Running') { + const tempParams = { + fromLine: this.fromLine, + size: 10000, + } + openLog = await api.fetch(`/entrance/${this.$route.query.execID}/log`, tempParams, 'get') + } else { + openLog = await api.fetch('/filesystem/openLog', params, 'get') + } + if (openLog) { + const log = cloneDeep(this.logs) + const convertLogs = util.convertLog(openLog.log) + Object.keys(convertLogs).forEach(key => { + if (convertLogs[key]) { + log[key] += convertLogs[key] + '\n' + } + }) + this.logs = log + this.fromLine = openLog.fromLine + this.$nextTick(() => { + this.$refs.logRef.fold(); + this.foldFlag = true; + }) + if (convertLogs['all'].split('\n').length >= 10000 && ['Scheduled', 'Running'].includes(this.$route.query.status)) { + if(this.logTimer) clearTimeout(this.logTimer); + this.logTimer = setTimeout(async () => await this.getLogs(jobhistory), 2000); + } + } + }, // Get historical details(获取历史详情) async initHistory(jobId) { try { let jobhistory = await api.fetch(`/jobhistory/${jobId}/get`, 'get') const option = jobhistory.task - const executionCode = option.executionCode; + // const executionCode = option.executionCode; this.jobhistoryTask = option this.script.runType = option.runType this.yarnAddress = option.yarnAddress - this.codes = { code: executionCode } + // this.codes = { code: executionCode } if (!jobhistory.task.logPath) { const errCode = jobhistory.task.errCode ? `\n${this.$t('message.linkis.errorCode')}：${ @@ -398,37 +444,8 @@ export default { this.fromLine = 1 return } - const params = { - path: jobhistory.task.logPath - } - if (this.$route.query.proxyUser) { - params.proxyUser = this.$route.query.proxyUser - } - let openLog = {} - if (this.$route.query.status === 'Scheduled' || this.$route.query.status === 'Running') { - const tempParams = { - fromLine: this.fromLine, - size: -1, - } - openLog = await api.fetch(`/entrance/${this.$route.query.execID}/log`, tempParams, 'get') - } else { - openLog = await api.fetch('/filesystem/openLog', params, 'get') - } - if (openLog) { - const log = { all: '', error: '', warning: '', info: '' } - const convertLogs = util.convertLog(openLog.log) - Object.keys(convertLogs).forEach(key => { - if (convertLogs[key]) { - log[key] += convertLogs[key] + '\n' - } - }) - this.logs = log - this.fromLine = log['all'].split('\n').length - } - this.$nextTick(() => { - this.$refs.logRef.fold(); - this.foldFlag = true; - }) + await this.getLogs(jobhistory); + this.isLoading = false } catch (errorMsg) { window.console.error(errorMsg) @@ -561,10 +578,10 @@ export default { top: 0; right: 20px; } -/deep/ .table-div { +::v-deep .table-div { height: 100% !important; } -/deep/ .log { +::v-deep .log { height: calc(100% - 70px) } diff --git a/linkis-web/src/apps/linkis/module/microServiceManagement/index.scss b/linkis-web/src/apps/linkis/module/microServiceManagement/index.scss index 663e993b23..47b5362217 100644 --- a/linkis-web/src/apps/linkis/module/microServiceManagement/index.scss +++ b/linkis-web/src/apps/linkis/module/microServiceManagement/index.scss @@ -59,13 +59,13 @@ } .microService { height: 100%; - /deep/ .ivu-tag { + ::v-deep .ivu-tag { max-width: 140px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } - /deep/ .ivu-tooltip-inner { + ::v-deep .ivu-tooltip-inner { max-width: 100%; } } \ No newline at end of file diff --git a/linkis-web/src/apps/linkis/module/pythonModule/index.js b/linkis-web/src/apps/linkis/module/pythonModule/index.js new file mode 100644 index 0000000000..126a73fa17 --- /dev/null +++ b/linkis-web/src/apps/linkis/module/pythonModule/index.js @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +export default { + name: 'pythonModule', + events: [], + component: () => import('./index.vue'), +}; \ No newline at end of file diff --git a/linkis-web/src/apps/linkis/module/pythonModule/index.vue b/linkis-web/src/apps/linkis/module/pythonModule/index.vue new file mode 100644 index 0000000000..99960929c2 --- /dev/null +++ b/linkis-web/src/apps/linkis/module/pythonModule/index.vue @@ -0,0 +1,13 @@ + + diff --git a/linkis-web/src/apps/linkis/module/resourceManagement/index.scss b/linkis-web/src/apps/linkis/module/resourceManagement/index.scss index b45cfa80db..732b2a29d4 100644 --- a/linkis-web/src/apps/linkis/module/resourceManagement/index.scss +++ b/linkis-web/src/apps/linkis/module/resourceManagement/index.scss @@ -45,7 +45,7 @@ td.table-name-column { cursor: pointer; } } -/deep/.table-project-column { +::v-deep .table-project-column { text-align: center; .ivu-table-cell { display: inline-block; @@ -78,7 +78,7 @@ td.table-project-column { padding-right: 10px; } } -/deep/.addTagClass { +::v-deep .addTagClass { position: relative; z-index: 99; } @@ -110,7 +110,7 @@ td.table-name-column { cursor: pointer; } } -/deep/.table-project-column { +::v-deep .table-project-column { text-align: center; .ivu-table-cell { display: inline-block; @@ -124,20 +124,20 @@ td.table-project-column { text-align: center; padding: 10px 0; } -/deep/.addTagClass { +::v-deep .addTagClass { position: relative; z-index: 99; } .ecm, .ecmEngine { height: 100%; overflow: hidden; - /deep/ .ivu-tag { + ::v-deep .ivu-tag { max-width: 140px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } - /deep/ .ivu-tooltip-inner { + ::v-deep .ivu-tooltip-inner { max-width: 100%; } } diff --git a/linkis-web/src/apps/linkis/module/setting/setting.vue b/linkis-web/src/apps/linkis/module/setting/setting.vue index 91c4118e3a..9842ef045b 100644 --- a/linkis-web/src/apps/linkis/module/setting/setting.vue +++ b/linkis-web/src/apps/linkis/module/setting/setting.vue @@ -398,8 +398,40 @@ export default { cb(true); }, 200); }, + preCheckConfig() { + try { + this.fullTree.forEach((item) => { + if (item.settings) { + item.settings.forEach((s) => { + if ( + s.key === "spark.conf" && s.configValue + ) { + const kvList = s.configValue.split(';').map(kv => kv.replace(/^\s+|\s+$/g,'')); + kvList.forEach((kv) => { + if(kv && !/^[\w.-]+=[\w.-]+$/.test(kv)) { + this.unValidMsg = {key: s.key, msg: s.description} + throw new Error('invalid') + } + }); + const formattedKvString = kvList.join(';\n'); + s.configValue = formattedKvString.trim(); + } + }); + } + }); + return true; + } catch(err) { + window.console.warn(err); + this.isAdvancedShow = true; + return false; + } + }, save() { this.loading = true; + if(!this.preCheckConfig()) { + this.loading = false; + return; + } this.checkValid(); api .fetch("/configuration/saveFullTree", { @@ -761,7 +793,7 @@ export default { font-size: 14px; } .tabs { - /deep/.ivu-tabs-bar { + ::v-deep .ivu-tabs-bar { &::before { content: ''; display: block; @@ -770,7 +802,7 @@ export default { float: left; } } - /deep/.ivu-tabs-tab:not(:last-of-type) { + ::v-deep .ivu-tabs-tab:not(:last-of-type) { position: relative; &::after { content: ''; diff --git a/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue b/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue index 382bebe97d..3dac9914be 100644 --- a/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue +++ b/linkis-web/src/apps/linkis/module/udfTree/EditForm/index.vue @@ -252,7 +252,7 @@ export default {