Feature - Jinja2 rendering (#5)

* Working basic Jinja2 rendering

* Better import for Jinja2

* Fixing security hotspot for Jinja2

docs/features.rst

@@ -5,19 +5,22 @@
 .. _sources:
 
 ================
+Features
+================
+
 Files sources
 ================
 
 AWS Common configuration
-=========================
+-------------------------
 
 As described in :ref:`iam_override`, you can override the IAM role you want to use when attempting to retrieve files
 from an AWS Service.
 
 The IAM override defined the closest to the resource to retrieve is used. See :ref:`iam_priority` for more details.
 
 AWS S3 Source
-===============
+---------------
 
 
 This allows you to define an S3 source with the Bucket name and Object key that you want to retrieve.
@@ -34,7 +37,7 @@ This allows you to define an S3 source with the Bucket name and Object key that
 
 
 AWS SSM Source
-==============
+---------------
 
 Similarly to AWS S3, this allows to retrieve the content of a SSM Parameter and store it as file.
 This can be useful simple credentials syntax and otherwise String defined parameters.
@@ -44,7 +47,7 @@ This can be useful simple credentials syntax and otherwise String defined parame
     If you are using a SecureString, make sure that you IAM role has kms:Decrypt permissions on the KMS Key.
 
 AWS Secrets Manager Source
-===========================
+---------------------------
 
 .. attention::
 
@@ -57,7 +60,7 @@ AWS Secrets Manager Source
 
 
 Url Source
-=============
+------------
 
 Allows you to download a file from an arbitrary URL. You can specify basic auth credentials if the file is not publicly
 accessible.
@@ -67,5 +70,60 @@ accessible.
     We do not recommend to put the basic auth credentials in plain text in the configuration, unless the source
     of the configuration for ECS Files Composer comes from AWS Secrets manager.
 
+Files Rendering
+====================
+
+To allow further flexibility, you have the possibility to set a **context** which indicates whether the file should
+be used as a template for a supported renderer.
+
+.. note::
+
+    The default value is plain, which means no alteration at all is to be done and the file should be used as-is.
+
+.. warning::
+
+    Do not attempt to perform rendering on a file that is not text (i.e. Images/ZIP etc.)
+
+Jinja2 & Custom filters
+-------------------------
+
+Used in a lot of very well known frameworks and Applications, such as Ansible, Jinja2 is a very powerful template
+rendering engine. Therefore, that will allow you to use Jinja filters to alter the file as you need it to.
+
+.. hint::
+
+    When using the jinja2 context, the file is placed into a randomly generated folder. That folder then auto-destroys
+    itself once the processing of the given file is complete, and the file is then rendered and written at the defined
+    location.
+
+.. seealso::
+
+    More about `Jinja2`_ and `Jinja2 filters`_
+
+env_override filter
+"""""""""""""""""""""
+
+This filter allows you to very simply interpolate an environment variable value from the key of that env var.
+Take the following example
+
+.. code-block:: yaml
+
+    files:
+      /tmp/test.txt:
+        content: >-
+          this is a test {{ "default" | env_override('ENV_VAR_TO_CHANGE') }}
+        owner: john
+        group: root
+        mode: 600
+        context: jinja2
+
+Files composer will use the content as template, which has been written to a temporary directory.
+It then invokes Jinja, with the custom filter **env_override**. If the filter finds an environment variable
+named *ENV_VAR_TO_CHANGE*, it then retrieves the value and pass it to Jinja. If not, Jinja will use *default* as the
+value.
+
+
 .. _AWS ECS Task Definition Secrets: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ecs-taskdefinition-containerdefinitions.html#cfn-ecs-taskdefinition-containerdefinition-secrets
 .. _Secrets usage in ECS Compose-X: https://docs.compose-x.io/syntax/docker-compose/secrets.html
+.. _Jinja2: https://jinja.palletsprojects.com/en/3.0.x/
+.. _Jinja2 filters: https://jinja.palletsprojects.com/en/3.0.x/templates/#filters

ecs-files-input.json

@@ -24,7 +24,7 @@
     "definitions": {
         "FileDef": {
             "type": "object",
-            "additionalProperties": false,
+            "additionalProperties": true,
             "properties": {
                 "path": {
                     "type": "string"

ecs_files_composer/ecs_files_composer.py

@@ -10,6 +10,7 @@
 import subprocess
 import warnings
 from os import environ, path
+from tempfile import TemporaryDirectory
 from typing import Any
 
 import boto3
@@ -18,11 +19,13 @@
 from boto3 import session
 from botocore.exceptions import ClientError
 from botocore.response import StreamingBody
+from jinja2 import Environment, FileSystemLoader
 from yaml import Loader
 
 from ecs_files_composer import input
 from ecs_files_composer.common import LOG, keyisset
 from ecs_files_composer.envsubst import expandvars
+from ecs_files_composer.jinja2_filters import env
 
 
 def create_session_from_creds(tmp_creds, region=None):
@@ -170,6 +173,7 @@ class File(input.FileDef, object):
 
     def __init__(self, iam_override=None, **data: Any):
         super().__init__(**data)
+        self.templates_dir = None
 
     def handler(self, iam_override):
         """
@@ -178,13 +182,17 @@ def handler(self, iam_override):
         :param input.IamOverrideDef iam_override:
         :return:
         """
+        if self.context and isinstance(self.context, input.Context) and self.context.value == "jinja2":
+            self.templates_dir = TemporaryDirectory()
         if self.commands and self.commands.pre:
             warnings.warn("Commands are not yet implemented", Warning)
         if self.source and not self.content:
             self.handle_sources(iam_override=iam_override)
             self.write_content()
         if not self.source and self.content:
             self.write_content(decode=True)
+        if self.templates_dir:
+            self.render_jinja()
         self.set_unix_settings()
         if self.commands and self.commands.post:
             warnings.warn("Commands are not yet implemented", Warning)
@@ -272,6 +280,17 @@ def handle_http_content(self):
             LOG.error(e)
             raise
 
+    def render_jinja(self):
+        """
+        Allows to use the temp directory as environment base, the original file as source template, and render
+        a final template.
+        """
+        jinja_env = Environment(loader=FileSystemLoader(self.templates_dir.name), autoescape=True, auto_reload=False)
+        jinja_env.filters['env_override'] = env
+        template = jinja_env.get_template(path.basename(self.path))
+        self.content = template.render()
+        self.write_content(is_template=False)
+
     def set_unix_settings(self):
         """
         Applies UNIX settings to given file
@@ -298,19 +317,34 @@ def set_unix_settings(self):
             else:
                 raise
 
-    def write_content(self, decode=False, as_bytes=False, bytes_content=None):
+    def write_content(self, is_template=True, decode=False, as_bytes=False, bytes_content=None):
+        """
+        Function to write the content retrieved to path.
+
+        :param bool is_template: Whether the content should be considered to be a template.
+        :param decode:
+        :param as_bytes:
+        :param bytes_content:
+        :return:
+        """
+        file_path = (
+            f"{self.templates_dir.name}/{path.basename(self.path)}"
+            if (self.templates_dir and is_template)
+            else self.path
+        )
+        LOG.info(f"Outputting {self.path} to {file_path}")
         if isinstance(self.content, str):
             if decode and self.encoding == input.Encoding["base64"]:
-                with open(self.path, "wb") as file_fd:
+                with open(file_path, "wb") as file_fd:
                     file_fd.write(base64.b64decode(self.content))
             else:
-                with open(self.path, "w") as file_fd:
+                with open(file_path, "w") as file_fd:
                     file_fd.write(self.content)
         elif isinstance(self.content, StreamingBody):
-            with open(self.path, "wb") as file_fd:
+            with open(file_path, "wb") as file_fd:
                 file_fd.write(self.content.read())
         elif as_bytes and bytes_content:
-            with open(self.path, "wb") as file_fd:
+            with open(file_path, "wb") as file_fd:
                 file_fd.write(bytes_content)
 
 

ecs_files_composer/input.py

@@ -1,13 +1,13 @@
 # generated by datamodel-codegen:
 #   filename:  ecs-files-input.json
-#   timestamp: 2021-07-16T13:56:02+00:00
+#   timestamp: 2021-07-20T08:20:14+00:00
 
 from __future__ import annotations
 
 from enum import Enum
 from typing import Any, Dict, List, Optional
 
-from pydantic import AnyUrl, BaseModel, Field
+from pydantic import AnyUrl, BaseModel, Extra, Field
 
 
 class Encoding(Enum):
@@ -84,6 +84,9 @@ class SourceDef(BaseModel):
 
 
 class FileDef(BaseModel):
+    class Config:
+        extra = Extra.allow
+
     path: Optional[str] = None
     content: Optional[str] = Field(None, description='The raw content of the file to use')
     source: Optional[SourceDef] = None

ecs_files_composer/jinja2_filters/__init__.py

@@ -0,0 +1,19 @@
+#  -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2021 John Mille<[email protected]>
+
+"""
+Package allowing to expand the Jinja filters to use.
+"""
+
+from os import environ
+
+
+def env(value, key):
+    """
+    Function to use in new Jinja filter
+    :param value:
+    :param key:
+    :return:
+    """
+    return environ.get(key, value)

requirements.txt

@@ -3,3 +3,4 @@ PyYAML~=5.4.1
 boto3>=1.16.35
 jsonschema~=3.2
 requests>=2.25.1
+Jinja2~=3.0.1