From 784d8ce4f8d37a1add621488b773fcbe409fa5fe Mon Sep 17 00:00:00 2001 From: litiliu <38579068+litiliu@users.noreply.github.com> Date: Fri, 17 Jan 2025 19:10:29 +0800 Subject: [PATCH] [config][enhance]support use properties when encrypt/decrypt config (#8527) Co-authored-by: litiliu --- .../Config-Encryption-Decryption.md | 39 +++++++++++++++ .../Config-Encryption-Decryption.md | 40 +++++++++++++++ .../api/configuration/ConfigShade.java | 13 +++++ .../common/config/TypesafeConfigUtils.java | 3 ++ .../core/starter/utils/ConfigShadeUtils.java | 35 ++++++++++--- .../core/starter/utils/ConfigShadeTest.java | 50 +++++++++++++++++++ ...he.seatunnel.api.configuration.ConfigShade | 3 +- .../resources/config.shade_with_props.json | 44 ++++++++++++++++ 8 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_props.json diff --git a/docs/en/connector-v2/Config-Encryption-Decryption.md b/docs/en/connector-v2/Config-Encryption-Decryption.md index edb8061b46a..7574c53919c 100644 --- a/docs/en/connector-v2/Config-Encryption-Decryption.md +++ b/docs/en/connector-v2/Config-Encryption-Decryption.md @@ -183,3 +183,42 @@ If you want to customize the encryption method and the configuration of the encr 5. Package it to jar and add jar to `${SEATUNNEL_HOME}/lib` 6. Change the option `shade.identifier` to the value that you defined in `ConfigShade#getIdentifier`of you config file, please enjoy it \^_\^ +### How to encrypt and decrypt with customized params + +If you want to encrypt and decrypt with customized params, you can follow the steps below: +1. Add a configuration named `shade.properties` in the env part of the configuration file, the value of this configuration is in the form of key-value pairs (the type of the key must be a string), as shown below: + + ```hocon + env { + shade.properties = { + suffix = "666" + } + } + + ``` + +2. Override the `ConfigShade` interface's `open` method, as shown below: + + ```java + public static class ConfigShadeWithProps implements ConfigShade { + + private String suffix; + private String identifier = "withProps"; + + @Override + public void open(Map props) { + this.suffix = String.valueOf(props.get("suffix")); + } + } + ``` +3. Use the parameters passed in the open method in the encryption and decryption methods, as shown below: + + ```java + public String encrypt(String content) { + return content + suffix; + } + + public String decrypt(String content) { + return content.substring(0, content.length() - suffix.length()); + } + ``` \ No newline at end of file diff --git a/docs/zh/connector-v2/Config-Encryption-Decryption.md b/docs/zh/connector-v2/Config-Encryption-Decryption.md index 9293d0e71e6..7664d792fd9 100644 --- a/docs/zh/connector-v2/Config-Encryption-Decryption.md +++ b/docs/zh/connector-v2/Config-Encryption-Decryption.md @@ -183,3 +183,43 @@ Base64编码支持加密以下参数： 5. 将其打成 jar 包, 并添加到 `${SEATUNNEL_HOME}/lib` 目录下。 6. 将选项 `shade.identifier` 的值更改为上面定义在配置文件中的 `ConfigShade#getIdentifier` 的值。 +### 在加密解密方法中使用自定义参数 + +如果您想要使用自定义参数进行加密和解密，可以按照以下步骤操作： +1. 在配置文件的env 中添加`shade.properties`配置，该配置的值是键值对形式（键的类型必须是字符串） ，如下所示： + + ```hocon + env { + shade.properties = { + suffix = "666" + } + } + + ``` +2. 覆写 `ConfigShade` 接口的 `open` 方法，如下所示： + + ```java + public static class ConfigShadeWithProps implements ConfigShade { + + private String suffix; + private String identifier = "withProps"; + + @Override + public void open(Map props) { + this.suffix = String.valueOf(props.get("suffix")); + } + } + ``` + 3. 在加密和解密方法中使用open 方法中传入的参数，如下所示： + + ```java + @Override + public String encrypt(String content) { + return content + suffix; + } + + @Override + public String decrypt(String content) { + return content.substring(0, content.length() - suffix.length()); + } + ``` \ No newline at end of file diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigShade.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigShade.java index 5532f48e064..d7a8a2f3aa8 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigShade.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigShade.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.api.configuration; +import java.util.Map; + /** * The interface that provides the ability to encrypt and decrypt {@link * org.apache.seatunnel.shade.com.typesafe.config.Config} @@ -47,4 +49,15 @@ public interface ConfigShade { default String[] sensitiveOptions() { return new String[0]; } + + /** + * this method will be called before the encrypt/decrpyt method. Users can use the props to + * control the behavior of the encrypt/decrypt + * + * @param props the additional properties defined with the key `shade.props` in the + * configuration + */ + default void open(Map props) { + // default do nothing + } } diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/TypesafeConfigUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/TypesafeConfigUtils.java index d80273ece0e..6f001a3da6d 100644 --- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/TypesafeConfigUtils.java +++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/TypesafeConfigUtils.java @@ -77,6 +77,9 @@ public static T getConfig( ? (T) Boolean.valueOf(config.getString(configKey)) : defaultValue; } + if (defaultValue instanceof Map) { + return config.hasPath(configKey) ? (T) config.getAnyRef(configKey) : defaultValue; + } throw new RuntimeException("Unsupported config type, configKey: " + configKey); } diff --git a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java index 2772454efb2..f3a3013a066 100644 --- a/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java +++ b/seatunnel-core/seatunnel-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java @@ -46,6 +46,7 @@ public final class ConfigShadeUtils { private static final String SHADE_IDENTIFIER_OPTION = "shade.identifier"; + private static final String SHADE_PROPS_OPTION = "shade.properties"; public static final String[] DEFAULT_SENSITIVE_KEYWORDS = new String[] {"password", "username", "auth", "token", "access_key", "secret_key"}; @@ -101,7 +102,14 @@ public static Config decryptConfig(Config config) { : ConfigFactory.empty(), SHADE_IDENTIFIER_OPTION, DEFAULT_SHADE.getIdentifier()); - return decryptConfig(identifier, config); + Map props = + TypesafeConfigUtils.getConfig( + config.hasPath(Constants.ENV) + ? config.getConfig(Constants.ENV) + : ConfigFactory.empty(), + SHADE_PROPS_OPTION, + new HashMap<>()); + return decryptConfig(identifier, config, props); } public static Config encryptConfig(Config config) { @@ -112,20 +120,33 @@ public static Config encryptConfig(Config config) { : ConfigFactory.empty(), SHADE_IDENTIFIER_OPTION, DEFAULT_SHADE.getIdentifier()); - return encryptConfig(identifier, config); + Map props = + TypesafeConfigUtils.getConfig( + config.hasPath(Constants.ENV) + ? config.getConfig(Constants.ENV) + : ConfigFactory.empty(), + SHADE_PROPS_OPTION, + new HashMap<>()); + return encryptConfig(identifier, config, props); } - public static Config decryptConfig(String identifier, Config config) { - return processConfig(identifier, config, true); + private static Config decryptConfig( + String identifier, Config config, Map props) { + return processConfig(identifier, config, true, props); } - public static Config encryptConfig(String identifier, Config config) { - return processConfig(identifier, config, false); + private static Config encryptConfig( + String identifier, Config config, Map props) { + return processConfig(identifier, config, false, props); } @SuppressWarnings("unchecked") - private static Config processConfig(String identifier, Config config, boolean isDecrypted) { + private static Config processConfig( + String identifier, Config config, boolean isDecrypted, Map props) { ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier, DEFAULT_SHADE); + // call open method before the encrypt/decrypt + configShade.open(props); + List sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS)); sensitiveOptions.addAll(Arrays.asList(configShade.sensitiveOptions())); BiFunction processFunction = diff --git a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java index b62df816083..4cd8b9c8717 100644 --- a/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java +++ b/seatunnel-core/seatunnel-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.List; +import java.util.Map; import static org.apache.seatunnel.core.starter.utils.ConfigBuilder.CONFIG_RENDER_OPTIONS; @@ -274,6 +275,55 @@ public void testDecryptAndEncrypt() { Assertions.assertEquals(decryptPassword, PASSWORD); } + @Test + public void testDecryptWithProps() throws URISyntaxException { + URL resource = ConfigShadeTest.class.getResource("/config.shade_with_props.json"); + Assertions.assertNotNull(resource); + Config decryptedProps = ConfigBuilder.of(Paths.get(resource.toURI()), Lists.newArrayList()); + + String suffix = "666"; + String rawUsername = "un"; + String rawPassword = "pd"; + Assertions.assertEquals( + rawUsername, decryptedProps.getConfigList("source").get(0).getString("username")); + Assertions.assertEquals( + rawPassword, decryptedProps.getConfigList("source").get(0).getString("password")); + + Config encryptedConfig = ConfigShadeUtils.encryptConfig(decryptedProps); + Assertions.assertEquals( + rawUsername + suffix, + encryptedConfig.getConfigList("source").get(0).getString("username")); + Assertions.assertEquals( + rawPassword + suffix, + encryptedConfig.getConfigList("source").get(0).getString("password")); + } + + public static class ConfigShadeWithProps implements ConfigShade { + + private String suffix; + private String identifier = "withProps"; + + @Override + public void open(Map props) { + this.suffix = String.valueOf(props.get("suffix")); + } + + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public String encrypt(String content) { + return content + suffix; + } + + @Override + public String decrypt(String content) { + return content.substring(0, content.length() - suffix.length()); + } + } + public static class Base64ConfigShade implements ConfigShade { private static final Base64.Encoder ENCODER = Base64.getEncoder(); diff --git a/seatunnel-core/seatunnel-core-starter/src/test/resources/META-INF/services/org.apache.seatunnel.api.configuration.ConfigShade b/seatunnel-core/seatunnel-core-starter/src/test/resources/META-INF/services/org.apache.seatunnel.api.configuration.ConfigShade index 6d7378028f9..87b02ff318b 100644 --- a/seatunnel-core/seatunnel-core-starter/src/test/resources/META-INF/services/org.apache.seatunnel.api.configuration.ConfigShade +++ b/seatunnel-core/seatunnel-core-starter/src/test/resources/META-INF/services/org.apache.seatunnel.api.configuration.ConfigShade @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.seatunnel.core.starter.utils.ConfigShadeTest$Base64ConfigShade \ No newline at end of file +org.apache.seatunnel.core.starter.utils.ConfigShadeTest$Base64ConfigShade +org.apache.seatunnel.core.starter.utils.ConfigShadeTest$ConfigShadeWithProps \ No newline at end of file diff --git a/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_props.json b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_props.json new file mode 100644 index 00000000000..c6f48bf6f7e --- /dev/null +++ b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_props.json @@ -0,0 +1,44 @@ +{ + "env" : { + "shade.identifier" : "withProps", + "parallelism" : 1, + "shade.properties" : { + "suffix" : "666" + } + }, + "source" : [ + { + "plugin_name" : "MySQL-CDC", + "base-url" : "jdbc:mysql://localhost:56725", + "username" : "un666", + "password" : "pd666", + "hostname" : "127.0.0.1", + "port" : 56725, + "database-name" : "inventory_vwyw0n", + "parallelism" : 1, + "table-name" : "products", + "server-id" : 5656, + "schema" : { + "fields" : { + "name" : "string", + "age" : "int", + "sex" : "boolean" + } + }, + "plugin_output" : "fake" + } + ], + "transform" : [], + "sink" : [ + { + "plugin_name" : "Clickhouse", + "host" : "localhost:8123", + "username" : "un666", + "password" : "pd666", + "database" : "default", + "table" : "fake_all", + "support_upsert" : true, + "primary_key" : "id" + } + ] +}