diff --git a/docs/features.rst b/docs/features.rst index 2145f39..1d751c1 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -5,11 +5,14 @@ .. _sources: ================ +Features +================ + Files sources ================ AWS Common configuration -========================= +------------------------- As described in :ref:`iam_override`, you can override the IAM role you want to use when attempting to retrieve files from an AWS Service. @@ -17,7 +20,7 @@ from an AWS Service. The IAM override defined the closest to the resource to retrieve is used. See :ref:`iam_priority` for more details. AWS S3 Source -=============== +--------------- This allows you to define an S3 source with the Bucket name and Object key that you want to retrieve. @@ -34,7 +37,7 @@ This allows you to define an S3 source with the Bucket name and Object key that AWS SSM Source -============== +--------------- Similarly to AWS S3, this allows to retrieve the content of a SSM Parameter and store it as file. This can be useful simple credentials syntax and otherwise String defined parameters. @@ -44,7 +47,7 @@ This can be useful simple credentials syntax and otherwise String defined parame If you are using a SecureString, make sure that you IAM role has kms:Decrypt permissions on the KMS Key. AWS Secrets Manager Source -=========================== +--------------------------- .. attention:: @@ -57,7 +60,7 @@ AWS Secrets Manager Source Url Source -============= +------------ Allows you to download a file from an arbitrary URL. You can specify basic auth credentials if the file is not publicly accessible. @@ -67,5 +70,60 @@ accessible. We do not recommend to put the basic auth credentials in plain text in the configuration, unless the source of the configuration for ECS Files Composer comes from AWS Secrets manager. +Files Rendering +==================== + +To allow further flexibility, you have the possibility to set a **context** which indicates whether the file should +be used as a template for a supported renderer. + +.. note:: + + The default value is plain, which means no alteration at all is to be done and the file should be used as-is. + +.. warning:: + + Do not attempt to perform rendering on a file that is not text (i.e. Images/ZIP etc.) + +Jinja2 & Custom filters +------------------------- + +Used in a lot of very well known frameworks and Applications, such as Ansible, Jinja2 is a very powerful template +rendering engine. Therefore, that will allow you to use Jinja filters to alter the file as you need it to. + +.. hint:: + + When using the jinja2 context, the file is placed into a randomly generated folder. That folder then auto-destroys + itself once the processing of the given file is complete, and the file is then rendered and written at the defined + location. + +.. seealso:: + + More about `Jinja2`_ and `Jinja2 filters`_ + +env_override filter +""""""""""""""""""""" + +This filter allows you to very simply interpolate an environment variable value from the key of that env var. +Take the following example + +.. code-block:: yaml + + files: + /tmp/test.txt: + content: >- + this is a test {{ "default" | env_override('ENV_VAR_TO_CHANGE') }} + owner: john + group: root + mode: 600 + context: jinja2 + +Files composer will use the content as template, which has been written to a temporary directory. +It then invokes Jinja, with the custom filter **env_override**. If the filter finds an environment variable +named *ENV_VAR_TO_CHANGE*, it then retrieves the value and pass it to Jinja. If not, Jinja will use *default* as the +value. + + .. _AWS ECS Task Definition Secrets: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ecs-taskdefinition-containerdefinitions.html#cfn-ecs-taskdefinition-containerdefinition-secrets .. _Secrets usage in ECS Compose-X: https://docs.compose-x.io/syntax/docker-compose/secrets.html +.. _Jinja2: https://jinja.palletsprojects.com/en/3.0.x/ +.. _Jinja2 filters: https://jinja.palletsprojects.com/en/3.0.x/templates/#filters diff --git a/ecs-files-input.json b/ecs-files-input.json index da94a0c..05be302 100644 --- a/ecs-files-input.json +++ b/ecs-files-input.json @@ -24,7 +24,7 @@ "definitions": { "FileDef": { "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "path": { "type": "string" diff --git a/ecs_files_composer/chmod.py b/ecs_files_composer/chmod.py deleted file mode 100644 index 8c0b43f..0000000 --- a/ecs_files_composer/chmod.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import re -import stat - -RD, WD, XD = 4, 2, 1 -BNS = [RD, WD, XD] -MDS = [ - [stat.S_IRUSR, stat.S_IRGRP, stat.S_IROTH], - [stat.S_IWUSR, stat.S_IWGRP, stat.S_IWOTH], - [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH], -] - - -def chmod(path, mode): - if isinstance(mode, int): - mode = str(mode) - if not re.match("^[0-7]{1,3}$", mode): - raise Exception("mode does not conform to ^[0-7]{1,3}$ pattern") - mode = "{0:0>3}".format(mode) - mode_num = 0 - for midx, m in enumerate(mode): - for bnidx, bn in enumerate(BNS): - if (int(m) & bn) > 0: - mode_num += MDS[bnidx][midx] - os.chmod(path, mode_num) diff --git a/ecs_files_composer/ecs_files_composer.py b/ecs_files_composer/ecs_files_composer.py index bbad94a..1cf4a7b 100644 --- a/ecs_files_composer/ecs_files_composer.py +++ b/ecs_files_composer/ecs_files_composer.py @@ -10,6 +10,7 @@ import subprocess import warnings from os import environ, path +from tempfile import TemporaryDirectory from typing import Any import boto3 @@ -18,11 +19,13 @@ from boto3 import session from botocore.exceptions import ClientError from botocore.response import StreamingBody +from jinja2 import Environment, FileSystemLoader from yaml import Loader from ecs_files_composer import input from ecs_files_composer.common import LOG, keyisset from ecs_files_composer.envsubst import expandvars +from ecs_files_composer.jinja2_filters import env def create_session_from_creds(tmp_creds, region=None): @@ -170,6 +173,7 @@ class File(input.FileDef, object): def __init__(self, iam_override=None, **data: Any): super().__init__(**data) + self.templates_dir = None def handler(self, iam_override): """ @@ -178,6 +182,8 @@ def handler(self, iam_override): :param input.IamOverrideDef iam_override: :return: """ + if self.context and isinstance(self.context, input.Context) and self.context.value == "jinja2": + self.templates_dir = TemporaryDirectory() if self.commands and self.commands.pre: warnings.warn("Commands are not yet implemented", Warning) if self.source and not self.content: @@ -185,6 +191,8 @@ def handler(self, iam_override): self.write_content() if not self.source and self.content: self.write_content(decode=True) + if self.templates_dir: + self.render_jinja() self.set_unix_settings() if self.commands and self.commands.post: warnings.warn("Commands are not yet implemented", Warning) @@ -272,6 +280,17 @@ def handle_http_content(self): LOG.error(e) raise + def render_jinja(self): + """ + Allows to use the temp directory as environment base, the original file as source template, and render + a final template. + """ + jinja_env = Environment(loader=FileSystemLoader(self.templates_dir.name), autoescape=True, auto_reload=False) + jinja_env.filters['env_override'] = env + template = jinja_env.get_template(path.basename(self.path)) + self.content = template.render() + self.write_content(is_template=False) + def set_unix_settings(self): """ Applies UNIX settings to given file @@ -298,19 +317,34 @@ def set_unix_settings(self): else: raise - def write_content(self, decode=False, as_bytes=False, bytes_content=None): + def write_content(self, is_template=True, decode=False, as_bytes=False, bytes_content=None): + """ + Function to write the content retrieved to path. + + :param bool is_template: Whether the content should be considered to be a template. + :param decode: + :param as_bytes: + :param bytes_content: + :return: + """ + file_path = ( + f"{self.templates_dir.name}/{path.basename(self.path)}" + if (self.templates_dir and is_template) + else self.path + ) + LOG.info(f"Outputting {self.path} to {file_path}") if isinstance(self.content, str): if decode and self.encoding == input.Encoding["base64"]: - with open(self.path, "wb") as file_fd: + with open(file_path, "wb") as file_fd: file_fd.write(base64.b64decode(self.content)) else: - with open(self.path, "w") as file_fd: + with open(file_path, "w") as file_fd: file_fd.write(self.content) elif isinstance(self.content, StreamingBody): - with open(self.path, "wb") as file_fd: + with open(file_path, "wb") as file_fd: file_fd.write(self.content.read()) elif as_bytes and bytes_content: - with open(self.path, "wb") as file_fd: + with open(file_path, "wb") as file_fd: file_fd.write(bytes_content) diff --git a/ecs_files_composer/input.py b/ecs_files_composer/input.py index ad5354d..e535cec 100644 --- a/ecs_files_composer/input.py +++ b/ecs_files_composer/input.py @@ -1,13 +1,13 @@ # generated by datamodel-codegen: # filename: ecs-files-input.json -# timestamp: 2021-07-16T13:56:02+00:00 +# timestamp: 2021-07-20T08:20:14+00:00 from __future__ import annotations from enum import Enum from typing import Any, Dict, List, Optional -from pydantic import AnyUrl, BaseModel, Field +from pydantic import AnyUrl, BaseModel, Extra, Field class Encoding(Enum): @@ -84,6 +84,9 @@ class SourceDef(BaseModel): class FileDef(BaseModel): + class Config: + extra = Extra.allow + path: Optional[str] = None content: Optional[str] = Field(None, description='The raw content of the file to use') source: Optional[SourceDef] = None diff --git a/ecs_files_composer/jinja2_filters/__init__.py b/ecs_files_composer/jinja2_filters/__init__.py new file mode 100644 index 0000000..2dff151 --- /dev/null +++ b/ecs_files_composer/jinja2_filters/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MPL-2.0 +# Copyright 2020-2021 John Mille + +""" +Package allowing to expand the Jinja filters to use. +""" + +from os import environ + + +def env(value, key): + """ + Function to use in new Jinja filter + :param value: + :param key: + :return: + """ + return environ.get(key, value) diff --git a/requirements.txt b/requirements.txt index ca8b3b8..22a6b86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ PyYAML~=5.4.1 boto3>=1.16.35 jsonschema~=3.2 requests>=2.25.1 +Jinja2~=3.0.1