Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev 1.9.0 bug fix #634

Merged
merged 7 commits into from
Oct 31, 2024
Merged

Dev 1.9.0 bug fix #634

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
82fcf60
Code optimization
Oct 30, 2024
b15d84c
Code optimization
Oct 30, 2024
096c0d5
Merge branch 'dev-1.9.0-webank' into dev-1.9.0-bug-fix
Oct 30, 2024
97049f5
Code optimization
Oct 30, 2024
2b72272
Code optimization
Oct 31, 2024
81d9cae
Merge branch 'dev-1.9.0-webank' into dev-1.9.0-bug-fix
Oct 31, 2024
d997a03
Code optimization
Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions ...on/src/main/java/org/apache/linkis/configuration/restful/api/ConfigurationRestfulApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,22 @@ public Message saveKeyValue(HttpServletRequest req, @RequestBody Map<String, Obj
return Message.error(e.getMessage());
}
}
if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()
&& (configKeyValue.getKey().equals("linkis.nebula.password")
|| configKeyValue.getKey().equals("wds.linkis.jdbc.password"))
&& StringUtils.isNotBlank(configKeyValue.getConfigValue())) {
List<ConfigUserValue> userConfigValue =
configKeyService.getUserConfigValue(engineType, configKeyValue.getKey(), creator, user);
for (ConfigUserValue configUserValue : userConfigValue) {
if ((configUserValue.getKey().equals("linkis.nebula.password")
|| configUserValue.getKey().equals("wds.linkis.jdbc.password"))
&& !configUserValue.getConfigValue().equals(configKeyValue.getConfigValue())) {
configKeyValue.setConfigValue(
AESUtils.encrypt(
configKeyValue.getConfigValue(), AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()));
}
}
}
ConfigValue configValue = configKeyService.saveConfigValue(configKeyValue, labelList);
configurationService.clearAMCacheConf(username, creator, engineType, version);
return message.data("configValue", configValue);
Expand Down
28 changes: 19 additions & 9 deletions .../main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,9 @@ public Message getInfoByDataSourceId(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -425,8 +427,9 @@ public Message getInfoByDataSourceName(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());

if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -461,8 +464,9 @@ public Message getPublishedInfoByDataSourceName(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());

if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -506,7 +510,9 @@ public Message getInfoByDataSourceIdAndVersion(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, dataSource.getConnectParams());
}
return Message.ok().data("info", dataSource);
},
"Fail to access data source[获取数据源信息失败]");
Expand Down Expand Up @@ -544,7 +550,7 @@ public Message getVersionList(
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
// Decrypt
if (null != versions) {
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue() && null != versions) {
versions.forEach(
version -> {
RestfulApiHelper.decryptPasswordKey(
Expand Down Expand Up @@ -694,7 +700,9 @@ public Message getConnectParams(
Map<String, Object> connectParams = dataSource.getConnectParams();
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
}
return Message.ok().data("connectParams", connectParams);
},
"Fail to connect data source[连接数据源失败]");
Expand Down Expand Up @@ -730,7 +738,9 @@ public Message getConnectParams(

List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
if (!AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
RestfulApiHelper.decryptPasswordKey(keyDefinitionList, connectParams);
}
return Message.ok().data("connectParams", connectParams);
},
"Fail to connect data source[连接数据源失败]");
Expand Down
21 changes: 16 additions & 5 deletions ...es-publicservice/src/main/java/org/apache/linkis/filesystem/restful/api/FsRestfulApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.linkis.storage.source.FileSource$;
import org.apache.linkis.storage.utils.StorageUtils;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.BOMInputStream;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -1461,9 +1462,15 @@ public Message pythonUpload(
}

// tar.gz包依赖检查
String errorMsg = FilesystemUtils.checkModuleFile(file, username);
if (StringUtils.isNotBlank(errorMsg)) {
return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg);
// 获取install_requires中的python模块
List<String> pythonModules = FilesystemUtils.getInstallRequestPythonModules(file);
String dependencies = "";
if (CollectionUtils.isNotEmpty(pythonModules)) {
dependencies = String.join(",", pythonModules);
String errorMsg = FilesystemUtils.checkModuleFile(pythonModules, username);
if (StringUtils.isNotBlank(errorMsg)) {
return Message.error("部分依赖未加载，请检查并重新上传依赖包，依赖信息：" + errorMsg);
}
}

// 定义目录路径
Expand Down Expand Up @@ -1499,11 +1506,12 @@ public Message pythonUpload(
OutputStream outputStream = null;
try {
String packageName = FilesystemUtils.findPackageName(file.getInputStream());
fileName = packageName + FsPath.CUR_DIR + "zip";
if (StringUtils.isBlank(packageName)) {
return Message.error("文件上传失败：PKG-INFO 文件不存在");
}
is = FilesystemUtils.getZipInputStreamByTarInputStream(file, packageName);
newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName.replace(".tar.gz", ".zip");
newPath = fsPath.getPath() + FsPath.SEPARATOR + fileName;
FsPath fsPathNew = new FsPath(newPath);
outputStream = fileSystem.write(fsPathNew, true);
IOUtils.copy(is, outputStream);
Expand All @@ -1519,6 +1527,9 @@ public Message pythonUpload(
}
}
// 返回成功消息并包含文件地址
return Message.ok().data("filePath", newPath);
return Message.ok()
.data("filePath", newPath)
.data("dependencies", dependencies)
.data("fileName", fileName);
}
}
7 changes: 3 additions & 4 deletions ...is-pes-publicservice/src/main/java/org/apache/linkis/filesystem/util/FilesystemUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,13 +184,12 @@ public static InputStream getZipInputStreamByTarInputStream(
* * 检查python环境中模块是否存在，使用程序调用脚本实现 脚本保存在admin文件夹中，脚本名为check_python_module.py
* 脚本执行存在返回ture，不存在返回false
*
* @param file 传入需要检查的模块列表
* @param pythonModules 传入需要检查的模块列表
* @param username
* @return 返回不存在的模块列表
*/
public static String checkModuleFile(MultipartFile file, String username) throws IOException {
// 获取install_requires中的python模块
List<String> pythonModules = getInstallRequestPythonModules(file);
public static String checkModuleFile(List<String> pythonModules, String username)
throws IOException {
StringJoiner joiner = new StringJoiner(",");
// 检查机器上pyhton环境中模块是否存在
List<String> notExistModules =
Expand Down
4 changes: 2 additions & 2 deletions ...nhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/api/UDFRestfulApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1290,10 +1290,10 @@ public Message pythonFileExist(
if (!fileNameWithoutExtension.matches("^[a-zA-Z][a-zA-Z0-9_-]{0,49}$")) {
return Message.error("只支持数字字母下划线，中划线，且以字母开头，长度最大50");
}

String fileNameWithoutVersion = fileNameWithoutExtension.split("-")[0];
// 封装PythonModuleInfo对象并查询数据库
PythonModuleInfo pythonModuleInfo = new PythonModuleInfo();
pythonModuleInfo.setName(fileNameWithoutExtension);
pythonModuleInfo.setName(fileNameWithoutVersion);
pythonModuleInfo.setCreateUser(userName);
PythonModuleInfo moduleInfo = pythonModuleInfoService.getByUserAndNameAndId(pythonModuleInfo);

Expand Down
Loading