diff --git a/linkis-web/src/apps/PythonModule/public/tutorial.html b/linkis-web/src/apps/PythonModule/public/tutorial.html new file mode 100644 index 0000000000..794b2bc0ed --- /dev/null +++ b/linkis-web/src/apps/PythonModule/public/tutorial.html @@ -0,0 +1,87 @@ + + + + + + Python 模块使用文档 + + + +
+
+ 注意事项:当前linkis支持的python版本为2.7.13和3.6.4两个版本,用户上传的python文件或模块必须兼容对应的python版本才能正常使用。 +
+

1. 首先你需要准备好对应的模块物料,支持.py文件、.tar.gz和.zip文件

+

a. 对于py 文件你的模块名一定要和文件名一致,如hello_world.py 模块名就是hello_world,您可以在里面定义你的python方法和class等,如下:

+
+            
#文件名一定是 hello_world.py
+def add_hello_world(name):
+  return "hello world {}".format(name) +
+
+

b. 对于zip文件需要稍微复杂点,你得遵循Python模块包的文件格式,如下:

+
+            
#zip文件名hello_world.zip
+hello_world.zip
+├── hello_world # 包目录,必须为模块名
+├── __init__.py # 必须有该文件
+├── hello_world.py # 其它python文件
+
#hello_world.py文件内容如下:
+def add_hello_world(name):
+  return "hello world {}".format(name) +
+
+

需要有 __init__.py 文件才能让 Python 将包含该文件的目录当作包来处理,可以只是一个空文件。你也可以直接去pip网站上面下载对应的包,然后通过zip压缩即可

+

2. 在linkis 管理台建立对应的模块:

+

第一步点击新增

+

第二步选择对应的引擎,分别对于Spark(.py脚本),python(.python脚本),通用(.py脚本和.python脚本)

+

第三步上传之前准备的模块物料,如果要更新模块内容需要先将旧的模块删除后再重新上传

+

第四步选择加载才可以进行使用

+

第五步如果你的模块不需要了,可以选择过期,等同于删除

+ +

3. 在scritpis进行使用

+

1. 如果你的模块已经加好了,可以直接在脚本中通过import进行使用如下:

+
+            
from hello_world import add_hello_world
+add_hello_world("peace") +
+
+

2. 可以将对应的模块注册成Spark的UDF进行使用

+
+            
from pyspark.sql.functions import udf
+from pyspark.sql.types import StringType
+from hello_world import add_hello_world
+add_hello_world_udf = udf(add_hello_world, StringType())
+df=spark.sql("select * from dws_demo.demo_user_info limit 100")
+df_transformed = df.withColumn("username", add_hello_world_udf(df["username"]))
+df_transformed.show() +
+
+
+ + \ No newline at end of file diff --git a/linkis-web/src/apps/PythonModule/src/global.less b/linkis-web/src/apps/PythonModule/src/global.less index 67cc7b0090..94e665c23b 100644 --- a/linkis-web/src/apps/PythonModule/src/global.less +++ b/linkis-web/src/apps/PythonModule/src/global.less @@ -9,3 +9,6 @@ body { #app { height: 100vh; } +.fes-form .fes-form-item-label { + justify-content: flex-end; +} diff --git a/linkis-web/src/apps/PythonModule/src/locales/en-US.js b/linkis-web/src/apps/PythonModule/src/locales/en-US.js index fa159589f3..d714976e4f 100644 --- a/linkis-web/src/apps/PythonModule/src/locales/en-US.js +++ b/linkis-web/src/apps/PythonModule/src/locales/en-US.js @@ -25,7 +25,7 @@ export default { cancel: 'Cancel', moduleResource: 'Module Resource', uploadResource: 'Please upload module resource', - onlyPyAndZip: 'Only .py and .zip files are supported', + onlyPyAndZip: 'Only .py, .zip and tar.gz files are supported', selectIsLoaded: 'Please select if loaded', selectIsExpire: 'Please select if expired', moduleDescription: 'Module Description', @@ -46,12 +46,18 @@ export default { confirmStatusChange: 'Confirm Load Status Change', confirmStatusChangeContent: 'Are you sure you want to change the load status of module {name}?', moduleNameTooLong: 'Module name cannot exceed 50 characters', - moduleNameNotFormat: 'Module name can only contain letters, numbers, and underscores, and must start with a letter', + // 支持横线 + moduleNameNotFormat: 'Module name can only contain letters, numbers, dash, and underscores, and must start with a letter', moduleNameExist: 'Module name {name} already exists, please delete the old module before re-uploading', moduleSizeExceed: 'Module size cannot exceed 50MB', saveSuccess: 'Saved successfully', normal: 'Normal', expire: 'Expired', known: 'Got it', - uploadFile: 'Upload File' + uploadFile: 'Upload File', + fullScreen: 'Full Screen View', + pythonModule: 'Module Dependencies', + placeholderPyModules: 'Please enter the module dependencies.', + noDeps: 'There is no module dependencies', + messageForPyModules: 'Please enter the module dependencies, separated by commas, Module name cannot exceed 50 characters and can only contain letters, numbers, dash, and underscores, and must start with a letter' }; diff --git a/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js b/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js index 7607a233da..a555064503 100644 --- a/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js +++ b/linkis-web/src/apps/PythonModule/src/locales/zh-CN.js @@ -25,7 +25,7 @@ export default { cancel: '取消', moduleResource: '模块物料', uploadResource: '请上传模块物料', - onlyPyAndZip: '仅支持.py和.zip格式文件', + onlyPyAndZip: '仅支持.py、.zip和.tar.gz格式文件', selectIsLoaded: '请选择是否加载', selectIsExpire: '请选择是否过期', moduleDescription: '模块描述', @@ -46,12 +46,17 @@ export default { confirmStatusChange: '确认加载状态变更', confirmStatusChangeContent: '您确定要修改模块 {name} 的加载状态吗?', moduleNameTooLong: '模块名称长度不能超过50个字符', - moduleNameNotFormat: '模块名称只支持数字字母下划线,且以字母开头', + moduleNameNotFormat: '模块名称只支持数字字母横线下划线,且以字母开头', moduleNameExist: '模块名称{name}已存在,如需重新上传请先删除旧的模块', moduleSizeExceed: '模块大小不能超过50MB', saveSuccess: '保存成功', normal: '正常', expire: '过期', known: '知道了', - uploadFile: '上传文件' + uploadFile: '上传文件', + fullScreen: '全屏查看', + pythonModule: '模块依赖', + placeholderPyModules: '请输入模块依赖', + noDeps: '无依赖模块', + messageForPyModules: '模块名长度最大为50,支持数字字母横线下划线,且以英文开头', }; diff --git a/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx b/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx index 672375d8b8..5ec93dec9e 100644 --- a/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx +++ b/linkis-web/src/apps/PythonModule/src/pages/index/index.jsx @@ -51,10 +51,11 @@ export default defineComponent({ id: 'apiPythonuploadFilesystem', query(params) { return letgoRequest( - '/api/rest_j/v1/filesystem/python-upload', + '/api/rest_j/v1/udf/python-upload', params, { - method: 'POST' + method: 'POST', + timeout: 60000 }, ); }, @@ -204,7 +205,7 @@ export default defineComponent({ _label={$t('moduleName')} placeholder={$t('moduleNamePlaceholder')} v-model={$$.pythonModuleName} - span={5} + span={6} style={{ width: '260px', }} @@ -214,7 +215,7 @@ export default defineComponent({ _label={$t('userName')} placeholder={$t('userNamePlaceholder')} v-model={$$.userName} - span={5} + span={6} style={{ width: '260px', }} @@ -223,7 +224,7 @@ export default defineComponent({ - + /> */} + {$t('onlyPyAndZip')} @@ -463,7 +470,7 @@ export default defineComponent({ if($$.selectedModule.name && $$.selectedModule.path) { return (
- {$$.selectedModule.name + '.' + $$.selectedModule.path?.split('.')[1] || ''} + {$$.selectedModule?.path?.split('/')[$$.selectedModule?.path?.split('/').length - 1] || ''}
); } @@ -474,7 +481,7 @@ export default defineComponent({ } }} beforeUpload={(...args) => $$.validateModuleFile(...args)} - accept={['.zip', '.py']} + accept={['.zip', '.py', '.tar.gz']} httpRequest={(...args) => $$.handleUploadHttpRequest(...args)} > }} + loading={$$.isUploading} > {$t('uploadFile')} @@ -507,7 +515,7 @@ export default defineComponent({ }, ]} /> - + /> */} + {$t('onlyPyAndZip')} @@ -638,7 +652,7 @@ export default defineComponent({ if($$.selectedModule.name && $$.selectedModule.path) { return (
- {$$.selectedModule.name + '.' + $$.selectedModule.path?.split('.')[1] || ''} + {$$.selectedModule?.path?.split('/')[$$.selectedModule?.path?.split('/').length - 1] || ''}
); } @@ -649,7 +663,7 @@ export default defineComponent({ } }} beforeUpload={(...args) => $$.validateModuleFile(...args)} - accept={['.zip', '.py']} + accept={['.zip', '.py', '.tar.gz']} httpRequest={(...args) => $$.handleUploadHttpRequest(...args)} > }} + loading={$$.isUploading} > {$t('uploadFile')} @@ -682,7 +697,7 @@ export default defineComponent({ }, ]} /> - + /> */} document.body} v-slots={{ title: () => { - return(

{$t('useTutorial')}

) - } + return( +
+

{$t('useTutorial')}

+ $$.openNewTab()}>{$t('fullScreen')} +
+ ) + }, + }} onUpdate:show={[ () => { @@ -754,7 +775,10 @@ export default defineComponent({ ]} >
-

1. 首先你需要准备好对应的模块物料,支持.py文件和.zip文件

+
+ 注意事项:当前linkis支持的python版本为2.7.13和3.6.4两个版本,用户上传的python文件或模块必须兼容对应的python版本才能正常使用。 +
+

1. 首先你需要准备好对应的模块物料,支持.py文件, tar.gz文件, .zip文件

a. 对于py 文件你的模块名一定要和文件名一致,如hello_world.py 模块名就是hello_world,您可以在里面定义你的python方法和class等,如下:

                   #文件名一定是 hello_world.py
@@ -823,28 +847,42 @@ df_transformed.show() : 'Python'; }, }, - { - prop: 'status', - label: $t('status'), - formatter: ({ - row, - column, - rowIndex, - coloumIndex, - cellValue, - }) => { - return row.isExpire === 0 - ? $t('normal') - : $t('expire'); - }, - }, + // { + // prop: 'status', + // label: $t('status'), + // formatter: ({ + // row, + // column, + // rowIndex, + // coloumIndex, + // cellValue, + // }) => { + // return row.isExpire === 0 + // ? $t('normal') + // : $t('expire'); + // }, + // }, { prop: 'path', label: $t('pathInfo'), }, + { + prop: 'pythonModule', + label: $t('pythonModule'), + formatter: ({row, column ,rowIndex, columnIndex, cellValue}) => { + if(!row.pythonModule) { + return '- -' + } + } + }, { prop: 'description', label: $t('moduleDescription'), + formatter: ({row, column, rowIndex, columnIndex, cellValue}) => { + if(!row.description) { + return '- -' + } + } }, { prop: 'createTime', diff --git a/linkis-web/src/apps/PythonModule/src/pages/index/main.js b/linkis-web/src/apps/PythonModule/src/pages/index/main.js index 14a8942948..c829101d01 100644 --- a/linkis-web/src/apps/PythonModule/src/pages/index/main.js +++ b/linkis-web/src/apps/PythonModule/src/pages/index/main.js @@ -6,7 +6,7 @@ export class Main extends LetgoPageBase { this.pythonModuleName = ''; this.userName = ''; this.engineType = ''; - this.isExpired = 0; + // this.isExpired = 0; this.isLoaded = null; this.currentPage = 1; this.pageSize = 10; @@ -17,9 +17,10 @@ export class Main extends LetgoPageBase { this.newModuleName = ''; this.selectedEngineType = 'spark'; this.selectedModuleDescription = ''; + this.selectedModulePythonModule = ''; this.selectedModulePath = ''; this.selectedModuleIsLoad = 1; - this.selectedModuleIsExpire = 0; + // this.selectedModuleIsExpire = 0; this.selectedModuleId = null; this.selectedModuleFile = null; this.selectedModuleFileError = ''; @@ -33,6 +34,8 @@ export class Main extends LetgoPageBase { this.addFormRef = null; this.editFormRef = null; this.tutorialVisible = false; + this.isUploading = false; + this.getDep = false; } onMounted () { @@ -49,7 +52,7 @@ export class Main extends LetgoPageBase { engineType: this.engineType, username: this.userName, // isLoad: this.isLoaded, - isExpire: this.isExpired, + isExpire: 0, pageNow: this.currentPage, pageSize: this.pageSize }; @@ -58,7 +61,7 @@ export class Main extends LetgoPageBase { } const response = await this.$pageCode.apiPythonlistUdf.trigger(params); - this.pythonModuleList = response.data.pythonList; + this.pythonModuleList = response.data.pythonList this.totalRecords = response.data.totalPage; return response; } catch (error) { @@ -76,7 +79,7 @@ export class Main extends LetgoPageBase { this.pythonModuleName = ''; this.userName = ''; this.engineType = ''; - this.isExpired = 0; + // this.isExpired = 0; this.isLoaded = null; this.currentPage = 1; this.pageSize = 10; @@ -87,11 +90,12 @@ export class Main extends LetgoPageBase { this.addPythonModuleVisible = true; this.selectedModule.name = ''; this.selectedModule.engineType = 'spark'; - this.selectedModule.isExpire = 0; + // this.selectedModule.isExpire = 0; this.selectedModule.isLoad = 1; this.selectedModule.path = ''; this.selectedModule.fileList = []; this.selectedModule.description = ''; + this.selectedModule.pythonModule = ''; } showTutorial () { @@ -169,7 +173,7 @@ export class Main extends LetgoPageBase { throw new Error(this.$pageCode.$t('moduleNameTooLong')); } // 名称只支持数字字母下划线,且以字母开头 - if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(newModuleName.split('.')[0])) { + if (!/^[a-zA-Z][a-zA-Z0-9_-]*$/.test(newModuleName.split('.')[0])) { this.$utils.FMessage.error(this.$pageCode.$t('moduleNameNotFormat')); throw new Error(this.$pageCode.$t('moduleNameNotFormat')); } @@ -209,18 +213,32 @@ export class Main extends LetgoPageBase { } } + cutExtension(fileName) { + const pyIndex = fileName.indexOf('.py'); + const zipIndex = fileName.indexOf('.zip'); + const targzIndex = fileName.indexOf('.tar.gz'); + if(Math.max(pyIndex, zipIndex, targzIndex) !== -1) { + return fileName.substring(0, Math.max(pyIndex, zipIndex, targzIndex)); + } else { + return fileName; + } + } async handleUploadHttpRequest (options) { try { + this.isUploading = true; const formData = new FormData(); - window.console.log('options:', options); formData.append('file', options.file); formData.append('fileName', options.file.name); const response = await this.$pageCode.apiPythonuploadFilesystem.trigger(formData); this.selectedModule.path = response.data.filePath; - this.selectedModule.name = options.file.name.split('.')[0]; + this.selectedModule.name = this.cutExtension(response.data.fileName); + this.selectedModule.pythonModule = response.data.dependencies; + this.getDep = true; this.selectedModule.fileList = [options.file]; + this.isUploading = false; } catch (err) { window.console.error(err); + this.isUploading = false; // this.$utils.FMessage.error('上传失败'); } } @@ -231,8 +249,9 @@ export class Main extends LetgoPageBase { selectedModuleDescription, selectedModulePath, selectedModuleIsLoad, - selectedModuleIsExpire, - selectedModuleId + // selectedModuleIsExpire, + selectedModulePythonModule, + selectedModuleId, ) { const params = { name: newModuleName, @@ -240,7 +259,8 @@ export class Main extends LetgoPageBase { path: selectedModulePath, engineType: selectedEngineType, isLoad: selectedModuleIsLoad, - isExpire: selectedModuleIsExpire + pythonModule: selectedModulePythonModule, + isExpire: 0 }; if (selectedModuleId) { params.id = selectedModuleId; @@ -284,7 +304,9 @@ export class Main extends LetgoPageBase { this.selectedModule.description, this.selectedModule.path, this.selectedModule.isLoad, - this.selectedModule.isExpire + this.selectedModule.pythonModule, + + // this.selectedModule.isExpire ); this.addPythonModuleVisible = false; this.loadPythonModuleList(); @@ -302,8 +324,9 @@ export class Main extends LetgoPageBase { this.selectedModule.description, this.selectedModule.path, this.selectedModule.isLoad, - this.selectedModule.isExpire, - this.selectedModule.id + // this.selectedModule.isExpire, + this.selectedModule.pythonModule, + this.selectedModule.id, ); this.closeEditModuleModal(); this.loadPythonModuleList(); @@ -338,16 +361,19 @@ export class Main extends LetgoPageBase { } async handleLoadStatusChange () { - const { id, name, path, isExpire, isLoad, engineType, description } = + const { id, name, path, + // isExpire, + isLoad, engineType, description, pythonModule } = this.selectedModule; window.console.log({ id, name, path, - isExpire, + // isExpire, isLoad, engineType, - description + description, + pythonModule }); const targetLoadStatus = isLoad === 1 ? 0 : 1; if (id === null) { @@ -363,8 +389,9 @@ export class Main extends LetgoPageBase { description, path, targetLoadStatus, - isExpire, - id + // isExpire, + pythonModule, + id, ); await this.loadPythonModuleList(); } catch (error) { @@ -377,6 +404,7 @@ export class Main extends LetgoPageBase { closeAddModuleModal () { this.addPythonModuleVisible = false; + this.getDep = false; } handleFileListChange ({ file, fileList }) { @@ -385,6 +413,7 @@ export class Main extends LetgoPageBase { closeEditModuleModal () { this.editPythonModuleVisible = false; + this.getDep = false; } closeDeleteConfirmation () { @@ -394,4 +423,9 @@ export class Main extends LetgoPageBase { closeLoadStatusChangeConfirmation () { this.loadStatusChangeConfirmationVisible = false; } + + openNewTab() { + window.open('./tutorial.html', '_blank'); + this.tutorialVisible = false; + } } diff --git a/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue b/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue index c19af012d3..14f749c6ab 100644 --- a/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue +++ b/linkis-web/src/apps/linkis/module/datasource/datasourceForm/index.vue @@ -22,6 +22,7 @@ v-model="fApi" :option="options" :value.sync="formData" + @change="handleFormChange" />
@@ -160,6 +161,7 @@ export default { options: { submitBtn: false, }, + isEncrypt: '', rule: [ { type: 'input', @@ -219,11 +221,17 @@ export default { }, }, methods: { + handleFormChange(field, value, rule, api, setFlag) { + if(field === 'password' && !setFlag) { + this.isEncrypt = ''; + } + }, getDataSource(newV) { if (this.data.id) { getDataSourceByIdAndVersion(newV.id, newV.versionId || 0).then( (result) => { const mConnect = result.info.connectParams + this.isEncrypt = result.info.connectParams?.isEncrypt || ''; this.sourceConnectData = mConnect delete result.info.connectParams this.dataSrc = { ...result.info, ...mConnect } diff --git a/linkis-web/src/apps/linkis/module/datasource/index.vue b/linkis-web/src/apps/linkis/module/datasource/index.vue index 1ad41841bc..82d6fd524a 100644 --- a/linkis-web/src/apps/linkis/module/datasource/index.vue +++ b/linkis-web/src/apps/linkis/module/datasource/index.vue @@ -588,7 +588,9 @@ export default { realFormData.connectParams = formData realFormData.createSystem = 'Linkis' realFormData.dataSourceTypeId = this.currentSourceData.dataSourceTypeId - + if(this.$refs.datasourceForm.isEncrypt) { + realFormData.connectParams.isEncrypt = this.$refs.datasourceForm.isEncrypt; + } let postDataSource = createDataSource let commentMsg = this.$t('message.linkis.datasource.initVersion') if (!this.currentSourceData.id) { @@ -694,8 +696,12 @@ export default { realFormData[key] = formData[key] delete formData[key] }) - realFormData.connectParams = formData + // console.log(this.$refs.datasourceForm.isEncrypt); + if(this.$refs.datasourceForm.isEncrypt) { + realFormData.connectParams.isEncrypt = this.$refs.datasourceForm.isEncrypt; + } + realFormData.createSystem = 'Linkis' realFormData.dataSourceTypeId = this.currentSourceData.dataSourceTypeId diff --git a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue index d52fd6b65f..56725cbf15 100644 --- a/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue +++ b/linkis-web/src/apps/linkis/module/globalHistoryManagement/viewHistory.vue @@ -17,7 +17,7 @@