Skip to content

Commit

Permalink
Dev 1.9.0 code view fix (#644)
Browse files Browse the repository at this point in the history 
* code reivew fix

* code reivew fix

* code reivew fix

* code reivew fix

* code reivew fix

* code reivew fix

* code reivew fix

* code reivew fix

* code reivew fix

---------

Co-authored-by: “v_kkhuang” <“[email protected]”>
  • Loading branch information
@v-kkhuang
v-kkhuang and “v_kkhuang” authored Nov 13, 2024
1 parent 05f4197 commit b50f2fc
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 28 deletions.
6 changes: 6 additions & 0 deletions linkis-commons/linkis-common/src/main/java/org/apache/linkis/common/utils/AESUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ public class AESUtils {

public static final String PASSWORD = "password";

public static final String IS_ENCRYPT = "isEncrypt";

public static final String DECRYPT = "0";

public static final String ENCRYPT = "1";

public static final CommonVars<String> LINKIS_DATASOURCE_AES_KEY =
CommonVars.apply("linkis.datasource.aes.secretkey", "");

Expand Down
34 changes: 22 additions & 12 deletions .../main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ public Message insertJsonInfo(@RequestBody DataSource dataSource, HttpServletReq
connectParams.get(AESUtils.PASSWORD).toString(),
AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()));
// 标记密码已经加密
dataSource.getConnectParams().put("isEncrypt", "1");
dataSource.getConnectParams().put(AESUtils.IS_ENCRYPT, AESUtils.ENCRYPT);
}
insertDataSource(dataSource);
return Message.ok().data("insertId", dataSource.getId());
Expand Down Expand Up @@ -284,7 +284,7 @@ public Message updateDataSourceInJson(
connectParams.get(AESUtils.PASSWORD).toString(),
AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()));
// 标记密码已经加密
dataSource.getConnectParams().put("isEncrypt", "1");
dataSource.getConnectParams().put(AESUtils.IS_ENCRYPT, AESUtils.ENCRYPT);
}

// add default value filed
Expand Down Expand Up @@ -948,23 +948,29 @@ private void dealDatasoueceData(DataSource dataSourceInfo, String isEncrypt) {
Map datasourceParmMap =
BDPJettyServerHelper.gson()
.fromJson(dataSourceInfoBrief.getParameter().toString(), Map.class);
if (!datasourceParmMap.getOrDefault("isEncrypt", "0").equals("1") && isEncrypt.equals("1")) {
if (!datasourceParmMap
.getOrDefault(AESUtils.IS_ENCRYPT, AESUtils.DECRYPT)
.equals(AESUtils.ENCRYPT)
&& isEncrypt.equals(AESUtils.ENCRYPT)) {
datasourceParmMap.put(
AESUtils.PASSWORD,
AESUtils.encrypt(
datasourceParmMap.get(AESUtils.PASSWORD).toString(),
AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()));
datasourceParmMap.put("isEncrypt", "1");
datasourceParmMap.put(AESUtils.IS_ENCRYPT, AESUtils.ENCRYPT);
dataSourceInfoBrief.setParameter(BDPJettyServerHelper.gson().toJson(datasourceParmMap));
dataSourceInfoService.updateDataSourceInfo(dataSourceInfoBrief);
}
if (datasourceParmMap.getOrDefault("isEncrypt", "0").equals("1") && isEncrypt.equals("0")) {
if (datasourceParmMap
.getOrDefault(AESUtils.IS_ENCRYPT, AESUtils.DECRYPT)
.equals(AESUtils.ENCRYPT)
&& isEncrypt.equals(AESUtils.DECRYPT)) {
datasourceParmMap.put(
AESUtils.PASSWORD,
AESUtils.decrypt(
datasourceParmMap.get(AESUtils.PASSWORD).toString(),
AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()));
datasourceParmMap.remove("isEncrypt");
datasourceParmMap.remove(AESUtils.IS_ENCRYPT);
dataSourceInfoBrief.setParameter(BDPJettyServerHelper.gson().toJson(datasourceParmMap));
dataSourceInfoService.updateDataSourceInfo(dataSourceInfoBrief);
}
Expand All @@ -983,8 +989,10 @@ private void dealDatasoueceVersionData(DataSource dataSourceInfo, String isEncry
&& datasourceVersion.getParameter().contains(AESUtils.PASSWORD)) {
Map datasourceVersionMap =
BDPJettyServerHelper.gson().fromJson(datasourceVersion.getParameter(), Map.class);
if (!datasourceVersionMap.getOrDefault("isEncrypt", "0").equals("1")
&& isEncrypt.equals("1")) {
if (!datasourceVersionMap
.getOrDefault(AESUtils.IS_ENCRYPT, AESUtils.DECRYPT)
.equals(AESUtils.ENCRYPT)
&& isEncrypt.equals(AESUtils.ENCRYPT)) {
try {
Object password =
CryptoUtils.string2Object(
Expand All @@ -993,7 +1001,7 @@ private void dealDatasoueceVersionData(DataSource dataSourceInfo, String isEncry
AESUtils.PASSWORD,
AESUtils.encrypt(
password.toString(), AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue()));
datasourceVersionMap.put("isEncrypt", "1");
datasourceVersionMap.put(AESUtils.IS_ENCRYPT, AESUtils.ENCRYPT);
datasourceVersion.setParameter(
BDPJettyServerHelper.gson().toJson(datasourceVersionMap));
dataSourceVersionDao.updateByDatasourceVersion(datasourceVersion);
Expand All @@ -1006,15 +1014,17 @@ private void dealDatasoueceVersionData(DataSource dataSourceInfo, String isEncry
}
}
// 解密
if (datasourceVersionMap.getOrDefault("isEncrypt", "0").equals("1")
&& isEncrypt.equals("0")) {
if (datasourceVersionMap
.getOrDefault(AESUtils.IS_ENCRYPT, AESUtils.DECRYPT)
.equals(AESUtils.ENCRYPT)
&& isEncrypt.equals(AESUtils.DECRYPT)) {
try {
String password = datasourceVersionMap.get(AESUtils.PASSWORD).toString();
String decryptPassword =
AESUtils.decrypt(password, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue());
datasourceVersionMap.put(
AESUtils.PASSWORD, CryptoUtils.object2String(decryptPassword));
datasourceVersionMap.remove("isEncrypt");
datasourceVersionMap.remove(AESUtils.IS_ENCRYPT);
datasourceVersion.setParameter(
BDPJettyServerHelper.gson().toJson(datasourceVersionMap));
dataSourceVersionDao.updateByDatasourceVersion(datasourceVersion);
Expand Down
2 changes: 1 addition & 1 deletion ...in/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceOperateRestfulApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ protected void doConnect(String operator, DataSource dataSource) throws ErrorExc
}
}
if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()
&& !dataSource.getConnectParams().containsKey("isEncrypt")
&& !dataSource.getConnectParams().containsKey(AESUtils.IS_ENCRYPT)
&& dataSource.getConnectParams().containsKey("password")) {
String password = dataSource.getConnectParams().get("password").toString();
String encrypt = AESUtils.encrypt(password, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue());
Expand Down
4 changes: 2 additions & 2 deletions ...rver/src/main/java/org/apache/linkis/datasourcemanager/core/restful/RestfulApiHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ public static void encryptPasswordKey(
if (null != password) {
String passwordStr = String.valueOf(password);
if (AESUtils.LINKIS_DATASOURCE_AES_SWITCH.getValue()) {
if (!connectParams.containsKey("isEncrypt")) {
if (!connectParams.containsKey(AESUtils.IS_ENCRYPT)) {
passwordStr =
AESUtils.encrypt(passwordStr, AESUtils.LINKIS_DATASOURCE_AES_KEY.getValue());
connectParams.put("isEncrypt", "1");
connectParams.put(AESUtils.IS_ENCRYPT, AESUtils.ENCRYPT);
}
} else {
passwordStr = CryptoUtils.object2String(passwordStr);
Expand Down
7 changes: 7 additions & 0 deletions ...c-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/conf/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.linkis.udf.conf;

import org.apache.linkis.common.conf.CommonVars;

public class Constants {
public static final String FILE_EXTENSION_PY = ".py";
public static final String FILE_EXTENSION_ZIP = ".zip";
Expand All @@ -26,4 +28,9 @@ public class Constants {
public static final String HDFS_PATH_UDF = "hdfs:///appcom/linkis/udf/";
public static final int MAX_FILE_SIZE_MB = 50 * 1024 * 1024;
public static final String[] OPERATORS = {"==", ">=", "<=", ">", "<", "~="};

public static final CommonVars<String> PYTHON_COMMAND =
CommonVars.apply("linkis.execution.command.python.version", "python3");
public static final CommonVars<String> PYTHON_PATH =
CommonVars.apply("linkis.python3.path", "/appcom/Install/anaconda3/bin/python");
}
37 changes: 24 additions & 13 deletions ...c-enhancements/linkis-udf-service/src/main/java/org/apache/linkis/udf/utils/UdfUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ public class UdfUtils {
private static final Logger logger = LoggerFactory.getLogger(UdfUtils.class);
private static Set<String> moduleSet = new HashSet<>();

private static final String setuppyFileName = "setup.py";
private static final String pyprojecttomlFileName = "pyproject.toml";

/**
* 从 tar.gz 文件中查找PKG-INFO文件，并获取其需要打包的文件夹名称
*
Expand Down Expand Up @@ -184,7 +187,10 @@ public static Boolean checkModuleIsExistEnv(String module) {
String exec =
Utils.exec(
(new String[] {
"python3", Configuration.getLinkisHome() + "/admin/" + "check_modules.py", module
Constants.PYTHON_COMMAND.getValue(),
Configuration.getLinkisHome() + "/admin/" + "check_modules.py",
Constants.PYTHON_PATH.getValue(),
module
}));
return Boolean.parseBoolean(exec);
} catch (Exception e) {
Expand All @@ -205,10 +211,11 @@ public static List<String> getInstallRequestPythonModules(MultipartFile file) th
new TarArchiveInputStream(new GzipCompressorInputStream(file.getInputStream()))) {
TarArchiveEntry entry;
while ((entry = tarInput.getNextTarEntry()) != null) {
if (entry.getName().endsWith("setup.py") || entry.getName().endsWith("pyproject.toml")) {
if (entry.getName().endsWith(setuppyFileName)
|| entry.getName().endsWith(pyprojecttomlFileName)) {
findSetup = 1;
String content = IOUtils.toString(tarInput, StandardCharsets.UTF_8);
modules = extractDependencies(content);
modules = extractDependencies(content, entry.getName());
break;
}
}
Expand All @@ -224,7 +231,7 @@ public static List<String> getInstallRequestPythonModules(MultipartFile file) th
return modules;
}

public static List<String> extractDependencies(String content) {
public static List<String> extractDependencies(String content, String name) {
String trim =
content
.replaceAll("#.*?\\n", "")
Expand All @@ -234,15 +241,19 @@ public static List<String> extractDependencies(String content) {
.trim();
List<String> modules = new ArrayList<>();
String moduleStr = "";
Matcher setupMatcher =
Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim);
if (setupMatcher.find()) {
moduleStr = setupMatcher.group(1);
if (name.endsWith(setuppyFileName)) {
Matcher setupMatcher =
Pattern.compile("install_requires=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim);
if (setupMatcher.find()) {
moduleStr = setupMatcher.group(1);
}
}
Matcher pyprojectMatcher =
Pattern.compile("dependencies=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim);
if (pyprojectMatcher.find()) {
moduleStr = pyprojectMatcher.group(1);
if (name.endsWith(pyprojecttomlFileName)) {
Matcher pyprojectMatcher =
Pattern.compile("dependencies=\\[(.*?)\\]", Pattern.DOTALL).matcher(trim);
if (pyprojectMatcher.find()) {
moduleStr = pyprojectMatcher.group(1);
}
}
String[] packages = moduleStr.split(",");
for (String pkg : packages) {
Expand All @@ -259,7 +270,7 @@ public static List<String> extractDependencies(String content) {
}
}
if (StringUtils.isNotBlank(pkg)) {
modules.add(pkg);
modules.add(pkg.toLowerCase());
}
}
return modules;
Expand Down

0 comments on commit b50f2fc

Please sign in to comment.