[config][enhance]support use properties when encrypt/decrypt config (#…

…8527) Co-authored-by: litiliu <[email protected]>
apache · Jan 17, 2025 · 784d8ce · 784d8ce
1 parent 8cfd8b4
commit 784d8ce
Show file tree

Hide file tree

Showing 8 changed files with 219 additions and 8 deletions.
diff --git a/docs/en/connector-v2/Config-Encryption-Decryption.md b/docs/en/connector-v2/Config-Encryption-Decryption.md
@@ -183,3 +183,42 @@ If you want to customize the encryption method and the configuration of the encr
 5. Package it to jar and add jar to `${SEATUNNEL_HOME}/lib`
 6. Change the option `shade.identifier` to the value that you defined in `ConfigShade#getIdentifier`of you config file, please enjoy it \^_\^
 
+### How to encrypt and decrypt with customized params
+
+If you want to encrypt and decrypt with customized params, you can follow the steps below:
+1. Add a configuration named `shade.properties` in the env part of the configuration file, the value of this configuration is in the form of key-value pairs (the type of the key must be a string), as shown below:
+
+   ```hocon
+    env {
+        shade.properties = {
+           suffix = "666"
+        }
+    }
+
+   ```
+
+2. Override the `ConfigShade` interface's `open` method, as shown below:
+
+   ```java
+       public static class ConfigShadeWithProps implements ConfigShade {
+
+        private String suffix;
+        private String identifier = "withProps";
+
+        @Override
+        public void open(Map<String, Object> props) {
+            this.suffix = String.valueOf(props.get("suffix"));
+        }
+   }
+   ```
+3. Use the parameters passed in the open method in the encryption and decryption methods, as shown below:
+
+   ```java
+       public String encrypt(String content) {
+           return content + suffix;
+       }
+
+       public String decrypt(String content) {
+           return content.substring(0, content.length() - suffix.length());
+       }
+   ```
diff --git a/docs/zh/connector-v2/Config-Encryption-Decryption.md b/docs/zh/connector-v2/Config-Encryption-Decryption.md
@@ -183,3 +183,43 @@ Base64编码支持加密以下参数：
 5. 将其打成 jar 包, 并添加到 `${SEATUNNEL_HOME}/lib` 目录下。
 6. 将选项 `shade.identifier` 的值更改为上面定义在配置文件中的 `ConfigShade#getIdentifier` 的值。
 
+### 在加密解密方法中使用自定义参数
+
+如果您想要使用自定义参数进行加密和解密，可以按照以下步骤操作：
+1. 在配置文件的env 中添加`shade.properties`配置，该配置的值是键值对形式（键的类型必须是字符串） ，如下所示：
+
+   ```hocon
+    env {
+        shade.properties = {
+           suffix = "666"
+        }
+    }
+
+   ```
+2. 覆写 `ConfigShade` 接口的 `open` 方法，如下所示：
+
+   ```java
+    public static class ConfigShadeWithProps implements ConfigShade {
+
+        private String suffix;
+        private String identifier = "withProps";
+
+        @Override
+        public void open(Map<String, Object> props) {
+            this.suffix = String.valueOf(props.get("suffix"));
+        }
+   }
+   ```
+   3. 在加密和解密方法中使用open 方法中传入的参数，如下所示：
+
+   ```java
+    @Override
+    public String encrypt(String content) {
+        return content + suffix;
+    }
+
+    @Override
+    public String decrypt(String content) {
+        return content.substring(0, content.length() - suffix.length());
+    }
+   ```
diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigShade.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/configuration/ConfigShade.java
@@ -17,6 +17,8 @@
 
 package org.apache.seatunnel.api.configuration;
 
+import java.util.Map;
+
 /**
  * The interface that provides the ability to encrypt and decrypt {@link
  * org.apache.seatunnel.shade.com.typesafe.config.Config}
@@ -47,4 +49,15 @@ public interface ConfigShade {
     default String[] sensitiveOptions() {
         return new String[0];
     }
+
+    /**
+     * this method will be called before the encrypt/decrpyt method. Users can use the props to
+     * control the behavior of the encrypt/decrypt
+     *
+     * @param props the additional properties defined with the key `shade.props` in the
+     *     configuration
+     */
+    default void open(Map<String, Object> props) {
+        // default do nothing
+    }
 }
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/TypesafeConfigUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/TypesafeConfigUtils.java
@@ -77,6 +77,9 @@ public static <T> T getConfig(
                     ? (T) Boolean.valueOf(config.getString(configKey))
                     : defaultValue;
         }
+        if (defaultValue instanceof Map) {
+            return config.hasPath(configKey) ? (T) config.getAnyRef(configKey) : defaultValue;
+        }
         throw new RuntimeException("Unsupported config type, configKey: " + configKey);
     }
 

diff --git a/...-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java b/...-core-starter/src/main/java/org/apache/seatunnel/core/starter/utils/ConfigShadeUtils.java
@@ -46,6 +46,7 @@
 public final class ConfigShadeUtils {
 
     private static final String SHADE_IDENTIFIER_OPTION = "shade.identifier";
+    private static final String SHADE_PROPS_OPTION = "shade.properties";
 
     public static final String[] DEFAULT_SENSITIVE_KEYWORDS =
             new String[] {"password", "username", "auth", "token", "access_key", "secret_key"};
@@ -101,7 +102,14 @@ public static Config decryptConfig(Config config) {
                                 : ConfigFactory.empty(),
                         SHADE_IDENTIFIER_OPTION,
                         DEFAULT_SHADE.getIdentifier());
-        return decryptConfig(identifier, config);
+        Map<String, Object> props =
+                TypesafeConfigUtils.getConfig(
+                        config.hasPath(Constants.ENV)
+                                ? config.getConfig(Constants.ENV)
+                                : ConfigFactory.empty(),
+                        SHADE_PROPS_OPTION,
+                        new HashMap<>());
+        return decryptConfig(identifier, config, props);
     }
 
     public static Config encryptConfig(Config config) {
@@ -112,20 +120,33 @@ public static Config encryptConfig(Config config) {
                                 : ConfigFactory.empty(),
                         SHADE_IDENTIFIER_OPTION,
                         DEFAULT_SHADE.getIdentifier());
-        return encryptConfig(identifier, config);
+        Map<String, Object> props =
+                TypesafeConfigUtils.getConfig(
+                        config.hasPath(Constants.ENV)
+                                ? config.getConfig(Constants.ENV)
+                                : ConfigFactory.empty(),
+                        SHADE_PROPS_OPTION,
+                        new HashMap<>());
+        return encryptConfig(identifier, config, props);
     }
 
-    public static Config decryptConfig(String identifier, Config config) {
-        return processConfig(identifier, config, true);
+    private static Config decryptConfig(
+            String identifier, Config config, Map<String, Object> props) {
+        return processConfig(identifier, config, true, props);
     }
 
-    public static Config encryptConfig(String identifier, Config config) {
-        return processConfig(identifier, config, false);
+    private static Config encryptConfig(
+            String identifier, Config config, Map<String, Object> props) {
+        return processConfig(identifier, config, false, props);
     }
 
     @SuppressWarnings("unchecked")
-    private static Config processConfig(String identifier, Config config, boolean isDecrypted) {
+    private static Config processConfig(
+            String identifier, Config config, boolean isDecrypted, Map<String, Object> props) {
         ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier, DEFAULT_SHADE);
+        // call open method before the encrypt/decrypt
+        configShade.open(props);
+
         List<String> sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_KEYWORDS));
         sensitiveOptions.addAll(Arrays.asList(configShade.sensitiveOptions()));
         BiFunction<String, Object, String> processFunction =

diff --git a/...l-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java b/...l-core-starter/src/test/java/org/apache/seatunnel/core/starter/utils/ConfigShadeTest.java
@@ -41,6 +41,7 @@
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
+import java.util.Map;
 
 import static org.apache.seatunnel.core.starter.utils.ConfigBuilder.CONFIG_RENDER_OPTIONS;
 
@@ -274,6 +275,55 @@ public void testDecryptAndEncrypt() {
         Assertions.assertEquals(decryptPassword, PASSWORD);
     }
 
+    @Test
+    public void testDecryptWithProps() throws URISyntaxException {
+        URL resource = ConfigShadeTest.class.getResource("/config.shade_with_props.json");
+        Assertions.assertNotNull(resource);
+        Config decryptedProps = ConfigBuilder.of(Paths.get(resource.toURI()), Lists.newArrayList());
+
+        String suffix = "666";
+        String rawUsername = "un";
+        String rawPassword = "pd";
+        Assertions.assertEquals(
+                rawUsername, decryptedProps.getConfigList("source").get(0).getString("username"));
+        Assertions.assertEquals(
+                rawPassword, decryptedProps.getConfigList("source").get(0).getString("password"));
+
+        Config encryptedConfig = ConfigShadeUtils.encryptConfig(decryptedProps);
+        Assertions.assertEquals(
+                rawUsername + suffix,
+                encryptedConfig.getConfigList("source").get(0).getString("username"));
+        Assertions.assertEquals(
+                rawPassword + suffix,
+                encryptedConfig.getConfigList("source").get(0).getString("password"));
+    }
+
+    public static class ConfigShadeWithProps implements ConfigShade {
+
+        private String suffix;
+        private String identifier = "withProps";
+
+        @Override
+        public void open(Map<String, Object> props) {
+            this.suffix = String.valueOf(props.get("suffix"));
+        }
+
+        @Override
+        public String getIdentifier() {
+            return identifier;
+        }
+
+        @Override
+        public String encrypt(String content) {
+            return content + suffix;
+        }
+
+        @Override
+        public String decrypt(String content) {
+            return content.substring(0, content.length() - suffix.length());
+        }
+    }
+
     public static class Base64ConfigShade implements ConfigShade {
 
         private static final Base64.Encoder ENCODER = Base64.getEncoder();

diff --git a/...r/src/test/resources/META-INF/services/org.apache.seatunnel.api.configuration.ConfigShade b/...r/src/test/resources/META-INF/services/org.apache.seatunnel.api.configuration.ConfigShade
@@ -13,4 +13,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-org.apache.seatunnel.core.starter.utils.ConfigShadeTest$Base64ConfigShade
+org.apache.seatunnel.core.starter.utils.ConfigShadeTest$Base64ConfigShade
+org.apache.seatunnel.core.starter.utils.ConfigShadeTest$ConfigShadeWithProps
diff --git a/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_props.json b/seatunnel-core/seatunnel-core-starter/src/test/resources/config.shade_with_props.json
@@ -0,0 +1,44 @@
+{
+  "env" : {
+    "shade.identifier" : "withProps",
+    "parallelism" : 1,
+    "shade.properties" : {
+      "suffix" : "666"
+    }
+  },
+  "source" : [
+    {
+      "plugin_name" : "MySQL-CDC",
+      "base-url" : "jdbc:mysql://localhost:56725",
+      "username" : "un666",
+      "password" : "pd666",
+      "hostname" : "127.0.0.1",
+      "port" : 56725,
+      "database-name" : "inventory_vwyw0n",
+      "parallelism" : 1,
+      "table-name" : "products",
+      "server-id" : 5656,
+      "schema" : {
+        "fields" : {
+          "name" : "string",
+          "age" : "int",
+          "sex" : "boolean"
+        }
+      },
+      "plugin_output" : "fake"
+    }
+  ],
+  "transform" : [],
+  "sink" : [
+    {
+      "plugin_name" : "Clickhouse",
+      "host" : "localhost:8123",
+      "username" : "un666",
+      "password" : "pd666",
+      "database" : "default",
+      "table" : "fake_all",
+      "support_upsert" : true,
+      "primary_key" : "id"
+    }
+  ]
+}