diff --git a/docker-compose.override.yaml b/docker-compose.override.yaml index a47dfa1..0123766 100644 --- a/docker-compose.override.yaml +++ b/docker-compose.override.yaml @@ -12,13 +12,16 @@ services: ECS_LOCAL_METADATA_PORT: "51679" HOME: "/home" AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1} - AWS_PROFILE: ${AWS_PROFILE} + AWS_PROFILE: ${AWS_PROFILE:-default} ports: - 51679:51679 + container_name: ecs-local-endpoints files-sidecar: + container_name: files-sidecar environment: AWS_CONTAINER_CREDENTIALS_RELATIVE_URI: "/creds" + ECS_CONTAINER_METADATA_URI: http://169.254.170.2/v3/containers/files-sidecar AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-eu-west-1} ECS_CONFIG_CONTENT: | @@ -47,7 +50,30 @@ services: source: Secret: SecretId: GHToken -# command: --from-s3 s3://sacrificial-lamb/test.yaml + + /opt/files/test_ecs_property.txt: + content: | + {{ ecs_container_metadata('PrivateDNSName') }} + context: jinja2 + + /opt/files/test_ecs_properties.yaml: + content: | + {{ ecs_container_metadata() | to_yaml }} + context: jinja2 + /opt/files/test_ecs_properties.json: + content: | + {{ ecs_task_metadata() | tojson }} + # HELLO + {{ ecs_container_metadata('ImageID')}} + context: jinja2 depends_on: - ecs-local-endpoints + +volumes: + localshared: + driver: local + driver_opts: + device: /tmp/files + o: bind + type: none diff --git a/docker-compose.yaml b/docker-compose.yaml index 438da8d..f5dc1f8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,7 +10,7 @@ services: files-sidecar: volumes: - localshared:/opt/files/ - image: public.ecr.aws/compose-x/ecs-files-composer:v0.1.1 + image: public.ecr.aws/compose-x/ecs-files-composer:latest deploy: resources: # Smallest RAM size for a lambda diff --git a/ecs_files_composer/ecs_files_composer.py b/ecs_files_composer/ecs_files_composer.py index 38aeb43..f1533d1 100644 --- a/ecs_files_composer/ecs_files_composer.py +++ b/ecs_files_composer/ecs_files_composer.py @@ -93,7 +93,7 @@ def init_config( } if decode_base64: initial_config["encoding"] = "base64" - initial_config["context"] = "jinja2" + # initial_config["context"] = "jinja2" start_jobs(jobs_input_def) with open(config_path, "r") as config_fd: try: diff --git a/ecs_files_composer/files_mgmt.py b/ecs_files_composer/files_mgmt.py index 83b9361..303d448 100644 --- a/ecs_files_composer/files_mgmt.py +++ b/ecs_files_composer/files_mgmt.py @@ -21,7 +21,7 @@ from ecs_files_composer.common import LOG from ecs_files_composer.envsubst import expandvars from ecs_files_composer.input import Context, Encoding, FileDef -from ecs_files_composer.jinja2_filters import env as env_override +from ecs_files_composer.jinja2_filters import JINJA_FILTERS, JINJA_FUNCTIONS class File(FileDef, object): @@ -177,13 +177,13 @@ def render_jinja(self): a final template. """ LOG.info(f"Rendering Jinja for {self.path} - {self.templates_dir.name}") - print(self.content) jinja_env = Environment( loader=FileSystemLoader(self.templates_dir.name), autoescape=True, auto_reload=False, ) - jinja_env.filters["env_override"] = env_override + jinja_env.filters.update(JINJA_FILTERS) + jinja_env.globals.update(JINJA_FUNCTIONS) template = jinja_env.get_template(path.basename(self.path)) self.content = template.render(env=os.environ) self.write_content(is_template=False) diff --git a/ecs_files_composer/jinja2_filters/__init__.py b/ecs_files_composer/jinja2_filters/__init__.py index 2dff151..8159e18 100644 --- a/ecs_files_composer/jinja2_filters/__init__.py +++ b/ecs_files_composer/jinja2_filters/__init__.py @@ -5,11 +5,15 @@ """ Package allowing to expand the Jinja filters to use. """ - +import json +import re from os import environ +import requests +import yaml + -def env(value, key): +def env_override(value, key): """ Function to use in new Jinja filter :param value: @@ -17,3 +21,124 @@ def env(value, key): :return: """ return environ.get(key, value) + + +def define_metadata(for_task=False): + meta_v4 = "ECS_CONTAINER_METADATA_URI_V4" + meta_v3 = "ECS_CONTAINER_METADATA_URI" + + if environ.get(meta_v3, None): + meta_url = environ.get(meta_v3) + elif environ.get(meta_v4, None): + meta_url = environ.get(meta_v4) + else: + raise EnvironmentError( + "No ECS Metadata URL provided. This filter only works on ECS" + ) + if for_task: + return requests.get(f"{meta_url}/task") + else: + return requests.get(meta_url) + + +def from_list_to_dict(top_key, new_mapping, to_convert): + """ + + :param str top_key: + :param dict new_mapping: + :param list to_convert: + :return: + """ + for count, item in enumerate(to_convert): + list_key = f"{top_key}_{count}" + if isinstance(item, dict): + from_dict_to_simple_keys(list_key, new_mapping, item) + elif isinstance(item, list): + from_list_to_dict(list_key, new_mapping, item) + + +def from_dict_to_simple_keys(top_key, new_mapping, to_convert): + """ + + :param str top_key: + :param dict new_mapping: + :param dict to_convert: + :return: + """ + for key, value in to_convert.items(): + if isinstance(value, str): + new_mapping[f"{top_key}_{key}"] = value + elif isinstance(value, dict): + from_dict_to_simple_keys(f"{top_key}_{key}", new_mapping, value) + elif isinstance(value, list): + from_list_to_dict(f"{top_key}_{key}", new_mapping, value) + + +def from_metadata_to_flat_keys(metadata): + """ + Function to transform the metadata into simplified structure + :param dict metadata: + :return: + """ + new_metadata = {} + for key, value in metadata.items(): + if isinstance(value, str): + new_metadata[key] = value + elif isinstance(value, dict): + from_dict_to_simple_keys(key, new_metadata, value) + elif isinstance(value, list): + from_list_to_dict(key, new_metadata, value) + return new_metadata + + +def get_property(metadata, property_key): + metadata_mapping = from_metadata_to_flat_keys(metadata) + property_re = re.compile(property_key) + for key, value in metadata_mapping.items(): + if property_re.findall(key): + return value + return None + + +def ecs_container_metadata(property_key=None, fallback_value=None): + metadata_raw = define_metadata() + metadata = metadata_raw.json() + if property_key: + value = get_property(metadata, property_key) + if value is None: + print(f"No task property found matching {property_key}") + return fallback_value + return value + return metadata + + +def ecs_task_metadata(property_key=None, fallback_value=None): + metadata_raw = define_metadata(for_task=True) + metadata = metadata_raw.json() + if property_key: + value = get_property(metadata, property_key) + if value is None: + print(f"No task property found matching {property_key}") + return fallback_value + return value + return metadata + + +def to_yaml(value): + """ + Filter to render input to YAML formatted content + :return: + """ + return yaml.dump(value, Dumper=yaml.Dumper) + + +def to_json(value, indent=2): + return json.dumps(value, indent=indent) + + +JINJA_FUNCTIONS = { + "ecs_container_metadata": ecs_container_metadata, + "ecs_task_metadata": ecs_task_metadata, +} + +JINJA_FILTERS = {"to_yaml": to_yaml, "to_json": to_json, "env_override": env_override} diff --git a/tests/test_ecs_metadata.py b/tests/test_ecs_metadata.py new file mode 100644 index 0000000..ce80abf --- /dev/null +++ b/tests/test_ecs_metadata.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MPL-2.0 +# Copyright 2020-2021 John Mille + +import json +import uuid +from base64 import b64encode +from os import path + +import boto3.session +import pytest + +from ecs_files_composer import input +from ecs_files_composer.ecs_files_composer import start_jobs + +HERE = path.abspath(path.dirname(__file__)) + +test_container = { + "DockerId": "cd189a933e5849daa93386466019ab50-2495160603", + "Name": "curl", + "DockerName": "curl", + "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest", + "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "curl", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/cd189a933e5849daa93386466019ab50", + "com.amazonaws.ecs.task-definition-family": "curltest", + "com.amazonaws.ecs.task-definition-version": "2", + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": {"CPU": 10, "Memory": 128}, + "CreatedAt": "2020-10-08T20:09:11.44527186Z", + "StartedAt": "2020-10-08T20:09:11.44527186Z", + "Type": "NORMAL", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": ["192.0.2.3"], + "AttachmentIndex": 0, + "MACAddress": "0a:de:f6:10:51:e5", + "IPv4SubnetCIDRBlock": "192.0.2.0/24", + "DomainNameServers": ["192.0.2.2"], + "DomainNameSearchList": ["us-west-2.compute.internal"], + "PrivateDNSName": "ip-10-0-0-222.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "192.0.2.0/24", + } + ], + "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/05966557-f16c-49cb-9352-24b3a0dcd0e1", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/containerlogs", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/curl/cd189a933e5849daa93386466019ab50", + }, + "LogDriver": "awslogs", +} + +test_task = { + "Cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "TaskARN": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3", + "Family": "curltest", + "Revision": "3", + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": {"CPU": 0.25, "Memory": 512}, + "PullStartedAt": "2020-10-08T20:47:16.053330955Z", + "PullStoppedAt": "2020-10-08T20:47:19.592684631Z", + "AvailabilityZone": "us-west-2a", + "Containers": [ + { + "DockerId": "e9028f8d5d8e4f258373e7b93ce9a3c3-2495160603", + "Name": "curl", + "DockerName": "curl", + "Image": "111122223333.dkr.ecr.us-west-2.amazonaws.com/curltest:latest", + "ImageID": "sha256:25f3695bedfb454a50f12d127839a68ad3caf91e451c1da073db34c542c4d2cb", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:111122223333:cluster/default", + "com.amazonaws.ecs.container-name": "curl", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/e9028f8d5d8e4f258373e7b93ce9a3c3", + "com.amazonaws.ecs.task-definition-family": "curltest", + "com.amazonaws.ecs.task-definition-version": "3", + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": {"CPU": 10, "Memory": 128}, + "CreatedAt": "2020-10-08T20:47:20.567813946Z", + "StartedAt": "2020-10-08T20:47:20.567813946Z", + "Type": "NORMAL", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": ["192.0.2.3"], + "IPv6Addresses": ["2001:dB8:10b:1a00:32bf:a372:d80f:e958"], + "AttachmentIndex": 0, + "MACAddress": "02:b7:20:19:72:39", + "IPv4SubnetCIDRBlock": "192.0.2.0/24", + "IPv6SubnetCIDRBlock": "2600:1f13:10b:1a00::/64", + "DomainNameServers": ["192.0.2.2"], + "DomainNameSearchList": ["us-west-2.compute.internal"], + "PrivateDNSName": "ip-172-31-30-173.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "192.0.2.0/24", + } + ], + "ContainerARN": "arn:aws:ecs:us-west-2:111122223333:container/1bdcca8b-f905-4ee6-885c-4064cb70f6e6", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/containerlogs", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/curl/e9028f8d5d8e4f258373e7b93ce9a3c3", + }, + "LogDriver": "awslogs", + } + ], + "LaunchType": "FARGATE", +}