Working composable environment variables

docs/syntax/docker-compose/environment.rst

@@ -0,0 +1,31 @@
+
+.. _environment_syntax_reference:
+
+===================
+environment
+===================
+
+Environment variables play a crucial role in configuring the services.
+You can define environment variables to set properties from resources or AWS Intrinsic functions.
+
+For example, you can do
+
+.. code-block:: yaml
+
+    services:
+      web-server:
+        environment:
+          ENV_VAR: value_01
+          SIMPLE_PROPERTY: x-s3::storage-bucket::BucketName
+          AWS_REGION: x-aws::AWS::Region
+          COMPLEX_ENV_VAR: s3://x-s3::storage-bucket::BucketName/x-aws::AWS::Provider/cluster_01
+
+    x-s3:
+      storage-bucket:
+        Lookup:
+          Tags:
+            Name: my-docs
+
+* SIMPLE_PROPERTY will resolve into the value of the BucketName. We assume the bucket name is ``my-docs``
+* AWS_REGION will result to the AWS Region the AWS Stack is deployed in.
+* COMPLEX_ENV_VAR will result in ``s3://my-docs/eu-west-1/cluster_01``

ecs_composex/common/envsubst.py

@@ -5,6 +5,8 @@
 Module to do a better env variables handling.
 """
 
+from __future__ import annotations
+
 import os
 import re
 

ecs_composex/common/settings.py

@@ -15,12 +15,9 @@
 
 from copy import deepcopy
 from datetime import datetime as dt
-from json import loads
-from os import path
 from re import compile, sub
 
 import boto3
-import jsonschema
 import yaml
 
 try:

ecs_composex/compose/compose_services/__init__.py

@@ -16,11 +16,13 @@
 
 if TYPE_CHECKING:
     from ecs_composex.ecs.ecs_family import ComposeFamily
+    from ecs_composex.common.settings import ComposeXSettings
 
 from compose_x_common.compose_x_common import keyisset, keypresent, set_else_none
 from troposphere import AWSHelperFn, If, NoValue, Ref, Sub
 from troposphere.ecs import (
     ContainerDefinition,
+    Environment,
     HealthCheck,
     KernelCapabilities,
     LinuxParameters,
@@ -42,6 +44,7 @@
 from ecs_composex.compose.compose_services.helpers import (
     define_ingress_mappings,
     import_env_variables,
+    sub_generate,
     validate_healthcheck,
 )
 from ecs_composex.compose.compose_volumes.services_helpers import map_volumes
@@ -1034,3 +1037,18 @@ def import_x_aws_settings(self):
                 )
             elif keyisset(setting[0], self.definition) and callable(setting[2]):
                 setting[2](setting[0])
+
+    def composed_env_processing(self, settings: ComposeXSettings):
+
+        env_vars: list[Environment] = getattr(
+            self.container_definition, "Environment", []
+        )
+        for env_var in env_vars:
+            if not isinstance(env_var, Environment) or not isinstance(
+                env_var.Value, str
+            ):
+                continue
+            sub_name, sub_params = sub_generate(
+                env_var.Value, {}, settings, self.family
+            )
+            env_var.Value = Sub(sub_name, **sub_params)

ecs_composex/compose/compose_services/helpers.py

@@ -3,8 +3,12 @@
 
 from __future__ import annotations
 
+import re
 from typing import TYPE_CHECKING
 
+if TYPE_CHECKING:
+    from ecs_composex.common import ComposeXSettings
+
 if TYPE_CHECKING:
     from troposphere import Template
     from troposphere.ecs import ContainerDefinition, Secret
@@ -15,6 +19,7 @@
 from troposphere import FindInMap, GetAtt, ImportValue, NoValue, Ref, Sub
 from troposphere.ecs import ContainerDefinition, Environment
 
+from ecs_composex.common import NONALPHANUM
 from ecs_composex.common.logging import LOG
 
 
@@ -227,3 +232,39 @@ def validate_healthcheck(healthcheck, valid_keys, required_keys):
         raise AttributeError(
             f"Expected at least {required_keys}. Got", healthcheck.keys()
         )
+
+
+def sub_generate(
+    input_s: str, sub_args: dict, settings: ComposeXSettings, service_family
+) -> tuple:
+    eval_r = re.compile(
+        r"(?P<intrinsic>x-aws::(?P<pseudo>AWS::[a-zA-Z0-9]+))|"
+        r"(?P<res_key>x-[a-zA-Z0-9-_]+)::(?P<res_name>[a-zA-Z0-9-_]+)::(?P<return_value>[a-zA-Z0-9-_]+)"
+    )
+    for _inter in eval_r.findall(input_s):
+        first_match = eval_r.search(input_s)
+        if not first_match:
+            break
+        aws, resource = first_match.group("pseudo"), first_match.group("res_key")
+
+        if resource:
+            res_key: str = first_match.group("res_key")
+            res_name: str = first_match.group("res_name")
+            return_value: str = first_match.group("return_value")
+            resource, parameter = settings.get_resource_attribute(
+                f"{res_key}::{res_name}::{return_value}"
+            )
+            value, params_to_add = resource.get_resource_attribute_value(
+                parameter, service_family
+            )
+            sub_arg_name_r: str = (
+                NONALPHANUM.sub("", res_key)
+                + NONALPHANUM.sub("", res_name)
+                + NONALPHANUM.sub("", return_value)
+            )
+            sub_arg_name: str = f"${{{sub_arg_name_r}}}"
+            sub_args[sub_arg_name_r] = value
+        else:
+            sub_arg_name: str = f"${{{first_match.group('pseudo')}}}"
+        input_s = eval_r.sub(sub_arg_name, input_s, count=1)
+    return input_s, sub_args

ecs_composex/compose/x_resources/__init__.py

@@ -11,21 +11,19 @@
 
 if TYPE_CHECKING:
     from ecs_composex.common.settings import ComposeXSettings
+    from ecs_composex.ecs.ecs_family import ComposeFamily
 
 import json
 import re
 from copy import deepcopy
-from os import path
 
 import jsonschema
 from compose_x_common.aws import get_account_id
 from compose_x_common.compose_x_common import (
     attributes_to_mapping,
     keyisset,
-    keypresent,
     set_else_none,
 )
-from importlib_resources import files as pkg_files
 from troposphere import AWSObject, Export, FindInMap, GetAtt, Join, Output, Ref, Sub
 from troposphere.ecs import Environment
 
@@ -324,56 +322,61 @@ def generate_cfn_mappings_from_lookup_properties(self):
             else:
                 self.mappings[parameter.title] = value
 
-    def set_update_container_env_var(
-        self, target: tuple, parameter, env_var_name: str
-    ) -> list:
-        """
-        Function that will set or update the value of a given env var from Return value of a resource.
-        :param tuple target:
-        :param parameter:
-        """
+    def get_resource_attribute_value(
+        self, parameter, family: ComposeFamily
+    ) -> tuple | None:
+        """Finds the value"""
         if isinstance(parameter, str):
             try:
                 attr_parameter = self.property_to_parameter_mapping[parameter]
             except KeyError:
                 LOG.error(
                     f"{self.module.res_key}.{self.name} - No return value {parameter} available."
                 )
-                return []
+                return
         elif isinstance(parameter, Parameter):
             attr_parameter = parameter
         else:
             raise TypeError(
                 "parameter is", type(parameter), "must be one of", [str, Parameter]
             )
-        env_vars = []
         params_to_add = []
         attr_id = self.attributes_outputs[attr_parameter]
         if self.cfn_resource:
-            env_vars.append(
-                Environment(
-                    Name=env_var_name,
-                    Value=Ref(attr_id["ImportParameter"]),
-                )
-            )
+            value = Ref(attr_id["ImportParameter"])
             params_to_add.append(attr_parameter)
         elif self.lookup_properties:
-            env_vars.append(
-                Environment(
-                    Name=env_var_name,
-                    Value=attr_id["ImportValue"],
-                )
-            )
+            value = attr_id["ImportValue"]
+        else:
+            raise LookupError("Unable to find attribute of resource")
         if params_to_add:
             params_values = {}
             settings = [get_parameter_settings(self, param) for param in params_to_add]
             resource_params_to_add = []
             for setting in settings:
                 resource_params_to_add.append(setting[1])
                 params_values[setting[0]] = setting[2]
-            add_parameters(target[0].template, resource_params_to_add)
-            target[0].stack.Parameters.update(params_values)
-        return env_vars
+            add_parameters(family.template, resource_params_to_add)
+            family.stack.Parameters.update(params_values)
+        return value, params_to_add
+
+    def set_update_container_env_var(
+        self, family: ComposeFamily, parameter: str | Parameter, env_var_name: str
+    ) -> list:
+        """
+        Function that will set or update the value of a given env var from Return value of a resource.
+        If the resource is new, adding the parameter to the top stack
+        If the resource is lookup, point to the mapping.
+        """
+        env_vars: list[Environment] = []
+        try:
+            value, params_to_add = self.get_resource_attribute_value(parameter, family)
+            env_vars.append(Environment(Name=env_var_name, Value=value))
+            return env_vars
+        except Exception as error:
+            print(error)
+            print("Failed to define env vars")
+            return []
 
     def generate_resource_service_env_vars(
         self, target: tuple, target_definition: dict

ecs_composex/ecs/ecs_family/__init__.py

@@ -114,7 +114,7 @@ def logical_name(self) -> str:
         return re.sub(r"[^a-zA-Z0-9]+", "", self.name)
 
     @property
-    def services(self) -> list:
+    def services(self) -> list[ComposeService]:
         return list(chain(self.managed_sidecars, self.ordered_services))
 
     @property
@@ -491,6 +491,10 @@ def finalize_family_settings(self):
         self.set_add_region_when_external()
         self.sort_secrets_env_vars()
 
+    def composed_env_processing(self, settings: ComposeXSettings) -> None:
+        for service in self.services:
+            service.composed_env_processing(settings)
+
     def set_add_region_when_external(self):
         from troposphere.ecs import Environment
 

ecs_composex/ecs_composex.py

@@ -299,6 +299,7 @@ def generate_full_template(settings: ComposeXSettings):
         map_resource_return_value_to_services_command(family, settings)
         family.state_facts()
         family.x_environment_processing()
+        family.composed_env_processing(settings)
 
     set_ecs_cluster_identifier(settings.root_stack, settings)
     add_all_tags(settings.root_stack.stack_template, settings)

ecs_composex/resource_settings.py

@@ -336,11 +336,12 @@ def set_update_container_env_vars_from_resource_attribute(
             and parts.group("res_key") == resource.module.res_key
         ):
             continue
+        env_vars: list[Environment] = resource.set_update_container_env_var(
+            target[0], parts.group("return_value"), defined_env_var.Name
+        )
         extend_container_envvars(
             svc.container_definition,
-            resource.set_update_container_env_var(
-                target, parts.group("return_value"), defined_env_var.Name
-            ),
+            env_vars,
             replace=True,
         )
 

tests/features/features/community.feature

@@ -0,0 +1,9 @@
+Feature: community
+
+    @warpstream
+    Scenario Outline: WarpStream
+        Given I use <file_path> as my docker-compose file
+        Then I render the docker-compose to composex to validate
+        Examples:
+            | file_path                 |
+            | use-cases/warpstream.yaml |