ECS specific improvements

* On init, context should not be Jinja by default * Added some ECS specific functions to retrieve metadata information
compose-x · Sep 28, 2021 · 9c24a69 · 9c24a69
1 parent 78e8863
commit 9c24a69
Show file tree

Hide file tree

Showing 6 changed files with 278 additions and 9 deletions.
diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml
@@ -12,13 +12,16 @@ services:
       ECS_LOCAL_METADATA_PORT: "51679"
       HOME: "/home"
       AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1}
-      AWS_PROFILE: ${AWS_PROFILE}
+      AWS_PROFILE: ${AWS_PROFILE:-default}
     ports:
       - 51679:51679
+    container_name: ecs-local-endpoints
 
   files-sidecar:
+    container_name: files-sidecar
     environment:
       AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: "/creds"
+      ECS_CONTAINER_METADATA_URI: http://169.254.170.2/v3/containers/files-sidecar
       AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1}
       ECS_CONFIG_CONTENT: |
 
@@ -47,7 +50,30 @@ services:
             source:
               Secret:
                 SecretId: GHToken
-#    command: --from-s3 s3://sacrificial-lamb/test.yaml
+
+          /opt/files/test_ecs_property.txt:
+            content: |
+              {{ ecs_container_metadata('PrivateDNSName') }}
+            context: jinja2
+
+          /opt/files/test_ecs_properties.yaml:
+            content: |
+              {{ ecs_container_metadata() | to_yaml }}
+            context: jinja2
+          /opt/files/test_ecs_properties.json:
+            content: |
+              {{ ecs_task_metadata() | tojson }}
+              # HELLO
+              {{ ecs_container_metadata('ImageID')}}
+            context: jinja2
 
     depends_on:
       - ecs-local-endpoints
+
+volumes:
+  localshared:
+    driver: local
+    driver_opts:
+      device: /tmp/files
+      o: bind
+      type: none
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -10,7 +10,7 @@ services:
   files-sidecar:
     volumes:
       - localshared:/opt/files/
-    image: public.ecr.aws/compose-x/ecs-files-composer:v0.1.1
+    image: public.ecr.aws/compose-x/ecs-files-composer:latest
     deploy:
       resources:
         # Smallest RAM size for a lambda

diff --git a/ecs_files_composer/ecs_files_composer.py b/ecs_files_composer/ecs_files_composer.py
@@ -93,7 +93,7 @@ def init_config(
     }
     if decode_base64:
         initial_config["encoding"] = "base64"
-    initial_config["context"] = "jinja2"
+    # initial_config["context"] = "jinja2"
     start_jobs(jobs_input_def)
     with open(config_path, "r") as config_fd:
         try:

diff --git a/ecs_files_composer/files_mgmt.py b/ecs_files_composer/files_mgmt.py
@@ -21,7 +21,7 @@
 from ecs_files_composer.common import LOG
 from ecs_files_composer.envsubst import expandvars
 from ecs_files_composer.input import Context, Encoding, FileDef
-from ecs_files_composer.jinja2_filters import env as env_override
+from ecs_files_composer.jinja2_filters import JINJA_FILTERS, JINJA_FUNCTIONS
 
 
 class File(FileDef, object):
@@ -177,13 +177,13 @@ def render_jinja(self):
         a final template.
         """
         LOG.info(f"Rendering Jinja for {self.path} - {self.templates_dir.name}")
-        print(self.content)
         jinja_env = Environment(
             loader=FileSystemLoader(self.templates_dir.name),
             autoescape=True,
             auto_reload=False,
         )
-        jinja_env.filters["env_override"] = env_override
+        jinja_env.filters.update(JINJA_FILTERS)
+        jinja_env.globals.update(JINJA_FUNCTIONS)
         template = jinja_env.get_template(path.basename(self.path))
         self.content = template.render(env=os.environ)
         self.write_content(is_template=False)

diff --git a/ecs_files_composer/jinja2_filters/__init__.py b/ecs_files_composer/jinja2_filters/__init__.py
@@ -5,15 +5,140 @@
 """
 Package allowing to expand the Jinja filters to use.
 """
-
+import json
+import re
 from os import environ
 
+import requests
+import yaml
+
 
-def env(value, key):
+def env_override(value, key):
     """
     Function to use in new Jinja filter
     :param value:
     :param key:
     :return:
     """
     return environ.get(key, value)
+
+
+def define_metadata(for_task=False):
+    meta_v4 = "ECS_CONTAINER_METADATA_URI_V4"
+    meta_v3 = "ECS_CONTAINER_METADATA_URI"
+
+    if environ.get(meta_v3, None):
+        meta_url = environ.get(meta_v3)
+    elif environ.get(meta_v4, None):
+        meta_url = environ.get(meta_v4)
+    else:
+        raise EnvironmentError(
+            "No ECS Metadata URL provided. This filter only works on ECS"
+        )
+    if for_task:
+        return requests.get(f"{meta_url}/task")
+    else:
+        return requests.get(meta_url)
+
+
+def from_list_to_dict(top_key, new_mapping, to_convert):
+    """
+
+    :param str top_key:
+    :param dict new_mapping:
+    :param list to_convert:
+    :return:
+    """
+    for count, item in enumerate(to_convert):
+        list_key = f"{top_key}_{count}"
+        if isinstance(item, dict):
+            from_dict_to_simple_keys(list_key, new_mapping, item)
+        elif isinstance(item, list):
+            from_list_to_dict(list_key, new_mapping, item)
+
+
+def from_dict_to_simple_keys(top_key, new_mapping, to_convert):
+    """
+
+    :param str top_key:
+    :param dict new_mapping:
+    :param dict to_convert:
+    :return:
+    """
+    for key, value in to_convert.items():
+        if isinstance(value, str):
+            new_mapping[f"{top_key}_{key}"] = value
+        elif isinstance(value, dict):
+            from_dict_to_simple_keys(f"{top_key}_{key}", new_mapping, value)
+        elif isinstance(value, list):
+            from_list_to_dict(f"{top_key}_{key}", new_mapping, value)
+
+
+def from_metadata_to_flat_keys(metadata):
+    """
+    Function to transform the metadata into simplified structure
+    :param dict metadata:
+    :return:
+    """
+    new_metadata = {}
+    for key, value in metadata.items():
+        if isinstance(value, str):
+            new_metadata[key] = value
+        elif isinstance(value, dict):
+            from_dict_to_simple_keys(key, new_metadata, value)
+        elif isinstance(value, list):
+            from_list_to_dict(key, new_metadata, value)
+    return new_metadata
+
+
+def get_property(metadata, property_key):
+    metadata_mapping = from_metadata_to_flat_keys(metadata)
+    property_re = re.compile(property_key)
+    for key, value in metadata_mapping.items():
+        if property_re.findall(key):
+            return value
+    return None
+
+
+def ecs_container_metadata(property_key=None, fallback_value=None):
+    metadata_raw = define_metadata()
+    metadata = metadata_raw.json()
+    if property_key:
+        value = get_property(metadata, property_key)
+        if value is None:
+            print(f"No task property found matching {property_key}")
+            return fallback_value
+        return value
+    return metadata
+
+
+def ecs_task_metadata(property_key=None, fallback_value=None):
+    metadata_raw = define_metadata(for_task=True)
+    metadata = metadata_raw.json()
+    if property_key:
+        value = get_property(metadata, property_key)
+        if value is None:
+            print(f"No task property found matching {property_key}")
+            return fallback_value
+        return value
+    return metadata
+
+
+def to_yaml(value):
+    """
+    Filter to render input to YAML formatted content
+    :return:
+    """
+    return yaml.dump(value, Dumper=yaml.Dumper)
+
+
+def to_json(value, indent=2):
+    return json.dumps(value, indent=indent)
+
+
+JINJA_FUNCTIONS = {
+    "ecs_container_metadata": ecs_container_metadata,
+    "ecs_task_metadata": ecs_task_metadata,
+}
+
+JINJA_FILTERS = {"to_yaml": to_yaml, "to_json": to_json, "env_override": env_override}
diff --git a/tests/test_ecs_metadata.py b/tests/test_ecs_metadata.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+#  -*- coding: utf-8 -*-
+# SPDX-License-Identifier: MPL-2.0
+# Copyright 2020-2021 John Mille<[email protected]>
+
+import json
+import uuid
+from base64 import b64encode
+from os import path
+
+import boto3.session
+import pytest
+
+from ecs_files_composer import input
+from ecs_files_composer.ecs_files_composer import start_jobs
+
+HERE = path.abspath(path.dirname(__file__))
+
+test_container = {
+    "DockerId": "cd189a933e5849daa93386466019ab50-2495160603",
+    "Name": "curl",
+    "DockerName": "curl",
+    "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+    "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
+    "Labels": {
+        "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
+        "com.amazonaws.ecs.container-name": "curl",
+        "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50",
+        "com.amazonaws.ecs.task-definition-family": "curltest",
+        "com.amazonaws.ecs.task-definition-version": "2",
+    },
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {"CPU": 10, "Memory": 128},
+    "CreatedAt": "2020-10-08T20:09:11.44527186Z",
+    "StartedAt": "2020-10-08T20:09:11.44527186Z",
+    "Type": "NORMAL",
+    "Networks": [
+        {
+            "NetworkMode": "awsvpc",
+            "IPv4Addresses": ["192.0.2.3"],
+            "AttachmentIndex": 0,
+            "MACAddress": "0a:de:f6:10:51:e5",
+            "IPv4SubnetCIDRBlock": "192.0.2.0/24",
+            "DomainNameServers": ["192.0.2.2"],
+            "DomainNameSearchList": ["us-west-2.compute.internal"],
+            "PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal",
+            "SubnetGatewayIpv4Address": "192.0.2.0/24",
+        }
+    ],
+    "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1",
+    "LogOptions": {
+        "awslogs-create-group": "true",
+        "awslogs-group": "/ecs/containerlogs",
+        "awslogs-region": "us-west-2",
+        "awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50",
+    },
+    "LogDriver": "awslogs",
+}
+
+test_task = {
+    "Cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
+    "TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3",
+    "Family": "curltest",
+    "Revision": "3",
+    "DesiredStatus": "RUNNING",
+    "KnownStatus": "RUNNING",
+    "Limits": {"CPU": 0.25, "Memory": 512},
+    "PullStartedAt": "2020-10-08T20:47:16.053330955Z",
+    "PullStoppedAt": "2020-10-08T20:47:19.592684631Z",
+    "AvailabilityZone": "us-west-2a",
+    "Containers": [
+        {
+            "DockerId": "e9028f8d5d8e4f258373e7b93ce9a3c3-2495160603",
+            "Name": "curl",
+            "DockerName": "curl",
+            "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest",
+            "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb",
+            "Labels": {
+                "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default",
+                "com.amazonaws.ecs.container-name": "curl",
+                "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3",
+                "com.amazonaws.ecs.task-definition-family": "curltest",
+                "com.amazonaws.ecs.task-definition-version": "3",
+            },
+            "DesiredStatus": "RUNNING",
+            "KnownStatus": "RUNNING",
+            "Limits": {"CPU": 10, "Memory": 128},
+            "CreatedAt": "2020-10-08T20:47:20.567813946Z",
+            "StartedAt": "2020-10-08T20:47:20.567813946Z",
+            "Type": "NORMAL",
+            "Networks": [
+                {
+                    "NetworkMode": "awsvpc",
+                    "IPv4Addresses": ["192.0.2.3"],
+                    "IPv6Addresses": ["2001:dB8:10b:1a00:32bf:a372:d80f:e958"],
+                    "AttachmentIndex": 0,
+                    "MACAddress": "02:b7:20:19:72:39",
+                    "IPv4SubnetCIDRBlock": "192.0.2.0/24",
+                    "IPv6SubnetCIDRBlock": "2600:1f13:10b:1a00::/64",
+                    "DomainNameServers": ["192.0.2.2"],
+                    "DomainNameSearchList": ["us-west-2.compute.internal"],
+                    "PrivateDNSName": "ip-172-31-30-173.us-west-2.compute.internal",
+                    "SubnetGatewayIpv4Address": "192.0.2.0/24",
+                }
+            ],
+            "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/1bdcca8b-f905-4ee6-885c-4064cb70f6e6",
+            "LogOptions": {
+                "awslogs-create-group": "true",
+                "awslogs-group": "/ecs/containerlogs",
+                "awslogs-region": "us-west-2",
+                "awslogs-stream": "ecs/curl/e9028f8d5d8e4f258373e7b93ce9a3c3",
+            },
+            "LogDriver": "awslogs",
+        }
+    ],
+    "LaunchType": "FARGATE",
+}