feat(core): secret function from environment (#1685)

closes #1679

build.gradle

@@ -168,6 +168,8 @@ subprojects {
         // configure en_US default locale for tests
         systemProperty 'user.language', 'en'
         systemProperty 'user.country', 'US'
+
+        environment 'SECRETS_MY_SECRET', "{\"secretKey\":\"secretValue\"}".bytes.encodeBase64().toString()
     }
 
     testlogger {

core/src/main/java/io/kestra/core/runners/pebble/Extension.java

@@ -1,5 +1,6 @@
 package io.kestra.core.runners.pebble;
 
+import io.kestra.core.runners.pebble.functions.SecretFunction;
 import io.pebbletemplates.pebble.extension.*;
 import io.pebbletemplates.pebble.operator.Associativity;
 import io.pebbletemplates.pebble.operator.BinaryOperator;
@@ -17,12 +18,17 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
+import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
 
 import static io.pebbletemplates.pebble.operator.BinaryOperatorType.NORMAL;
 
 @Singleton
 public class Extension extends AbstractExtension {
+    @Inject
+    private SecretFunction secretFunction;
+
     @Override
     public List<TokenParser> getTokenParsers() {
         return null;
@@ -82,6 +88,7 @@ public Map<String, Function> getFunctions() {
         tests.put("now", new NowFunction());
         tests.put("json", new JsonFunction());
         tests.put("currentEachOutput", new CurrentEachOutputFunction());
+        tests.put("secret", secretFunction);
 
         return tests;
     }

core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java

@@ -0,0 +1,46 @@
+package io.kestra.core.runners.pebble.functions;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import io.kestra.core.secret.SecretService;
+import io.pebbletemplates.pebble.error.PebbleException;
+import io.pebbletemplates.pebble.extension.Function;
+import io.pebbletemplates.pebble.template.EvaluationContext;
+import io.pebbletemplates.pebble.template.PebbleTemplate;
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+@Singleton
+public class SecretFunction implements Function {
+    @Inject
+    private SecretService secretService;
+
+    @Override
+    public List<String> getArgumentNames() {
+        return List.of("key");
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Object execute(Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
+        String key = getSecretKey(args, self, lineNumber);
+        Map<String, String> flow = (Map<String, String>) context.getVariable("flow");
+
+        try {
+            return secretService.findSecret(flow.get("namespace"), key);
+        } catch (IllegalVariableEvaluationException | IOException e) {
+            throw new PebbleException(e, e.getMessage(), lineNumber, self.getName());
+        }
+    }
+
+    protected String getSecretKey(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
+        if (!args.containsKey("key")) {
+            throw new PebbleException(null, "The 'secret' function expects an argument 'key'.", lineNumber, self.getName());
+        }
+
+        return (String) args.get("key");
+    }
+}

core/src/main/java/io/kestra/core/secret/SecretService.java

@@ -0,0 +1,34 @@
+package io.kestra.core.secret;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import jakarta.inject.Singleton;
+
+import javax.annotation.PostConstruct;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Singleton
+public class SecretService {
+    private static final String SECRET_PREFIX = "SECRETS_";
+    private Map<String, String> decodedSecrets;
+
+    @PostConstruct
+    private void postConstruct() {
+        decodedSecrets = System.getenv().entrySet().stream().filter(entry -> entry.getKey().startsWith(SECRET_PREFIX)).collect(Collectors.toMap(
+            entry -> entry.getKey().substring(SECRET_PREFIX.length()).toUpperCase(),
+            entry -> new String(Base64.getDecoder().decode(entry.getValue()))
+        ));
+    }
+
+    public String findSecret(String namespace, String key) throws IOException, IllegalVariableEvaluationException {
+        return Optional
+            .ofNullable(decodedSecrets.get(key.toUpperCase()))
+            .orElseThrow(() -> new IllegalVariableEvaluationException("Unable to find secret '" + key + "'. " +
+                "You should add it in your environment variables as 'SECRETS_" + key.toUpperCase() +
+                "' with base64-encoded value."
+            ));
+    }
+}

core/src/test/java/io/kestra/core/secret/SecretFunctionTest.java

@@ -0,0 +1,39 @@
+package io.kestra.core.secret;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import io.kestra.core.models.executions.Execution;
+import io.kestra.core.runners.AbstractMemoryRunnerTest;
+import io.kestra.core.runners.RunnerUtils;
+import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.TimeoutException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+@MicronautTest
+public class SecretFunctionTest extends AbstractMemoryRunnerTest {
+    @Inject
+    private RunnerUtils runnerUtils;
+
+    @Inject
+    private SecretService secretService;
+
+    @Test
+    void getSecret() throws TimeoutException {
+        Execution execution = runnerUtils.runOne("io.kestra.tests", "secrets");
+        assertThat(execution.getTaskRunList().get(0).getOutputs().get("value"), is("secretValue"));
+    }
+
+    @Test
+    void getUnknownSecret() {
+        IllegalVariableEvaluationException exception = Assertions.assertThrows(IllegalVariableEvaluationException.class, () ->
+            secretService.findSecret(null, "unknown_secret_key")
+        );
+
+        assertThat(exception.getMessage(), is("Unable to find secret 'unknown_secret_key'"));
+    }
+}

core/src/test/resources/flows/valids/secrets.yaml

@@ -0,0 +1,7 @@
+id: secrets
+namespace: io.kestra.tests
+
+tasks:
+  - id: get-secret
+    type: io.kestra.core.tasks.debugs.Return
+    format: "{{json(secret('my_secret')).secretKey}}"