diff --git a/docs/syntax/docker-compose/environment.rst b/docs/syntax/docker-compose/environment.rst new file mode 100644 index 00000000..19c1cb51 --- /dev/null +++ b/docs/syntax/docker-compose/environment.rst @@ -0,0 +1,31 @@ + +.. _environment_syntax_reference: + +=================== +environment +=================== + +Environment variables play a crucial role in configuring the services. +You can define environment variables to set properties from resources or AWS Intrinsic functions. + +For example, you can do + +.. code-block:: yaml + + services: + web-server: + environment: + ENV_VAR: value_01 + SIMPLE_PROPERTY: x-s3::storage-bucket::BucketName + AWS_REGION: x-aws::AWS::Region + COMPLEX_ENV_VAR: s3://x-s3::storage-bucket::BucketName/x-aws::AWS::Provider/cluster_01 + + x-s3: + storage-bucket: + Lookup: + Tags: + Name: my-docs + +* SIMPLE_PROPERTY will resolve into the value of the BucketName. We assume the bucket name is ``my-docs`` +* AWS_REGION will result to the AWS Region the AWS Stack is deployed in. +* COMPLEX_ENV_VAR will result in ``s3://my-docs/eu-west-1/cluster_01`` diff --git a/ecs_composex/common/envsubst.py b/ecs_composex/common/envsubst.py index 8e96f8aa..91af3ec2 100644 --- a/ecs_composex/common/envsubst.py +++ b/ecs_composex/common/envsubst.py @@ -5,6 +5,8 @@ Module to do a better env variables handling. """ +from __future__ import annotations + import os import re diff --git a/ecs_composex/common/settings.py b/ecs_composex/common/settings.py index e93084f8..b57e5c58 100644 --- a/ecs_composex/common/settings.py +++ b/ecs_composex/common/settings.py @@ -15,12 +15,9 @@ from copy import deepcopy from datetime import datetime as dt -from json import loads -from os import path from re import compile, sub import boto3 -import jsonschema import yaml try: diff --git a/ecs_composex/compose/compose_services/__init__.py b/ecs_composex/compose/compose_services/__init__.py index acbbb6fc..0c2e8058 100644 --- a/ecs_composex/compose/compose_services/__init__.py +++ b/ecs_composex/compose/compose_services/__init__.py @@ -16,11 +16,13 @@ if TYPE_CHECKING: from ecs_composex.ecs.ecs_family import ComposeFamily + from ecs_composex.common.settings import ComposeXSettings from compose_x_common.compose_x_common import keyisset, keypresent, set_else_none from troposphere import AWSHelperFn, If, NoValue, Ref, Sub from troposphere.ecs import ( ContainerDefinition, + Environment, HealthCheck, KernelCapabilities, LinuxParameters, @@ -42,6 +44,7 @@ from ecs_composex.compose.compose_services.helpers import ( define_ingress_mappings, import_env_variables, + sub_generate, validate_healthcheck, ) from ecs_composex.compose.compose_volumes.services_helpers import map_volumes @@ -1034,3 +1037,18 @@ def import_x_aws_settings(self): ) elif keyisset(setting[0], self.definition) and callable(setting[2]): setting[2](setting[0]) + + def composed_env_processing(self, settings: ComposeXSettings): + + env_vars: list[Environment] = getattr( + self.container_definition, "Environment", [] + ) + for env_var in env_vars: + if not isinstance(env_var, Environment) or not isinstance( + env_var.Value, str + ): + continue + sub_name, sub_params = sub_generate( + env_var.Value, {}, settings, self.family + ) + env_var.Value = Sub(sub_name, **sub_params) diff --git a/ecs_composex/compose/compose_services/helpers.py b/ecs_composex/compose/compose_services/helpers.py index 30ede3ca..b0418165 100644 --- a/ecs_composex/compose/compose_services/helpers.py +++ b/ecs_composex/compose/compose_services/helpers.py @@ -3,8 +3,12 @@ from __future__ import annotations +import re from typing import TYPE_CHECKING +if TYPE_CHECKING: + from ecs_composex.common import ComposeXSettings + if TYPE_CHECKING: from troposphere import Template from troposphere.ecs import ContainerDefinition, Secret @@ -15,6 +19,7 @@ from troposphere import FindInMap, GetAtt, ImportValue, NoValue, Ref, Sub from troposphere.ecs import ContainerDefinition, Environment +from ecs_composex.common import NONALPHANUM from ecs_composex.common.logging import LOG @@ -227,3 +232,39 @@ def validate_healthcheck(healthcheck, valid_keys, required_keys): raise AttributeError( f"Expected at least {required_keys}. Got", healthcheck.keys() ) + + +def sub_generate( + input_s: str, sub_args: dict, settings: ComposeXSettings, service_family +) -> tuple: + eval_r = re.compile( + r"(?Px-aws::(?PAWS::[a-zA-Z0-9]+))|" + r"(?Px-[a-zA-Z0-9-_]+)::(?P[a-zA-Z0-9-_]+)::(?P[a-zA-Z0-9-_]+)" + ) + for _inter in eval_r.findall(input_s): + first_match = eval_r.search(input_s) + if not first_match: + break + aws, resource = first_match.group("pseudo"), first_match.group("res_key") + + if resource: + res_key: str = first_match.group("res_key") + res_name: str = first_match.group("res_name") + return_value: str = first_match.group("return_value") + resource, parameter = settings.get_resource_attribute( + f"{res_key}::{res_name}::{return_value}" + ) + value, params_to_add = resource.get_resource_attribute_value( + parameter, service_family + ) + sub_arg_name_r: str = ( + NONALPHANUM.sub("", res_key) + + NONALPHANUM.sub("", res_name) + + NONALPHANUM.sub("", return_value) + ) + sub_arg_name: str = f"${{{sub_arg_name_r}}}" + sub_args[sub_arg_name_r] = value + else: + sub_arg_name: str = f"${{{first_match.group('pseudo')}}}" + input_s = eval_r.sub(sub_arg_name, input_s, count=1) + return input_s, sub_args diff --git a/ecs_composex/compose/x_resources/__init__.py b/ecs_composex/compose/x_resources/__init__.py index 6ba47d1e..6cc14b7b 100644 --- a/ecs_composex/compose/x_resources/__init__.py +++ b/ecs_composex/compose/x_resources/__init__.py @@ -11,21 +11,19 @@ if TYPE_CHECKING: from ecs_composex.common.settings import ComposeXSettings + from ecs_composex.ecs.ecs_family import ComposeFamily import json import re from copy import deepcopy -from os import path import jsonschema from compose_x_common.aws import get_account_id from compose_x_common.compose_x_common import ( attributes_to_mapping, keyisset, - keypresent, set_else_none, ) -from importlib_resources import files as pkg_files from troposphere import AWSObject, Export, FindInMap, GetAtt, Join, Output, Ref, Sub from troposphere.ecs import Environment @@ -324,14 +322,10 @@ def generate_cfn_mappings_from_lookup_properties(self): else: self.mappings[parameter.title] = value - def set_update_container_env_var( - self, target: tuple, parameter, env_var_name: str - ) -> list: - """ - Function that will set or update the value of a given env var from Return value of a resource. - :param tuple target: - :param parameter: - """ + def get_resource_attribute_value( + self, parameter, family: ComposeFamily + ) -> tuple | None: + """Finds the value""" if isinstance(parameter, str): try: attr_parameter = self.property_to_parameter_mapping[parameter] @@ -339,31 +333,22 @@ def set_update_container_env_var( LOG.error( f"{self.module.res_key}.{self.name} - No return value {parameter} available." ) - return [] + return elif isinstance(parameter, Parameter): attr_parameter = parameter else: raise TypeError( "parameter is", type(parameter), "must be one of", [str, Parameter] ) - env_vars = [] params_to_add = [] attr_id = self.attributes_outputs[attr_parameter] if self.cfn_resource: - env_vars.append( - Environment( - Name=env_var_name, - Value=Ref(attr_id["ImportParameter"]), - ) - ) + value = Ref(attr_id["ImportParameter"]) params_to_add.append(attr_parameter) elif self.lookup_properties: - env_vars.append( - Environment( - Name=env_var_name, - Value=attr_id["ImportValue"], - ) - ) + value = attr_id["ImportValue"] + else: + raise LookupError("Unable to find attribute of resource") if params_to_add: params_values = {} settings = [get_parameter_settings(self, param) for param in params_to_add] @@ -371,9 +356,27 @@ def set_update_container_env_var( for setting in settings: resource_params_to_add.append(setting[1]) params_values[setting[0]] = setting[2] - add_parameters(target[0].template, resource_params_to_add) - target[0].stack.Parameters.update(params_values) - return env_vars + add_parameters(family.template, resource_params_to_add) + family.stack.Parameters.update(params_values) + return value, params_to_add + + def set_update_container_env_var( + self, family: ComposeFamily, parameter: str | Parameter, env_var_name: str + ) -> list: + """ + Function that will set or update the value of a given env var from Return value of a resource. + If the resource is new, adding the parameter to the top stack + If the resource is lookup, point to the mapping. + """ + env_vars: list[Environment] = [] + try: + value, params_to_add = self.get_resource_attribute_value(parameter, family) + env_vars.append(Environment(Name=env_var_name, Value=value)) + return env_vars + except Exception as error: + print(error) + print("Failed to define env vars") + return [] def generate_resource_service_env_vars( self, target: tuple, target_definition: dict diff --git a/ecs_composex/ecs/ecs_family/__init__.py b/ecs_composex/ecs/ecs_family/__init__.py index 7a81ce7b..f789bc73 100644 --- a/ecs_composex/ecs/ecs_family/__init__.py +++ b/ecs_composex/ecs/ecs_family/__init__.py @@ -114,7 +114,7 @@ def logical_name(self) -> str: return re.sub(r"[^a-zA-Z0-9]+", "", self.name) @property - def services(self) -> list: + def services(self) -> list[ComposeService]: return list(chain(self.managed_sidecars, self.ordered_services)) @property @@ -491,6 +491,10 @@ def finalize_family_settings(self): self.set_add_region_when_external() self.sort_secrets_env_vars() + def composed_env_processing(self, settings: ComposeXSettings) -> None: + for service in self.services: + service.composed_env_processing(settings) + def set_add_region_when_external(self): from troposphere.ecs import Environment diff --git a/ecs_composex/ecs_composex.py b/ecs_composex/ecs_composex.py index e8eab13c..d00e210a 100644 --- a/ecs_composex/ecs_composex.py +++ b/ecs_composex/ecs_composex.py @@ -299,6 +299,7 @@ def generate_full_template(settings: ComposeXSettings): map_resource_return_value_to_services_command(family, settings) family.state_facts() family.x_environment_processing() + family.composed_env_processing(settings) set_ecs_cluster_identifier(settings.root_stack, settings) add_all_tags(settings.root_stack.stack_template, settings) diff --git a/ecs_composex/resource_settings.py b/ecs_composex/resource_settings.py index 4b2f37ca..aa1df37c 100644 --- a/ecs_composex/resource_settings.py +++ b/ecs_composex/resource_settings.py @@ -336,11 +336,12 @@ def set_update_container_env_vars_from_resource_attribute( and parts.group("res_key") == resource.module.res_key ): continue + env_vars: list[Environment] = resource.set_update_container_env_var( + target[0], parts.group("return_value"), defined_env_var.Name + ) extend_container_envvars( svc.container_definition, - resource.set_update_container_env_var( - target, parts.group("return_value"), defined_env_var.Name - ), + env_vars, replace=True, ) diff --git a/tests/features/features/community.feature b/tests/features/features/community.feature new file mode 100644 index 00000000..e9ab2e8b --- /dev/null +++ b/tests/features/features/community.feature @@ -0,0 +1,9 @@ +Feature: community + + @warpstream + Scenario Outline: WarpStream + Given I use as my docker-compose file + Then I render the docker-compose to composex to validate + Examples: + | file_path | + | use-cases/warpstream.yaml | diff --git a/use-cases/warpstream.yaml b/use-cases/warpstream.yaml new file mode 100644 index 00000000..11ccc6fd --- /dev/null +++ b/use-cases/warpstream.yaml @@ -0,0 +1,138 @@ +secrets: + CLUSTER_SECRETS: + external: true + x-secrets: + Name: /kafka/warpstream/${CLUSTER_NAME} + JsonKeys: + - SecretKey: cluster_id + VarName: WARPSTREAM_DEFAULT_VIRTUAL_CLUSTER_ID + - SecretKey: api_key + VarName: WARPSTREAM_API_KEY + - SecretKey: agent_pool_name + VarName: WARPSTREAM_AGENT_POOL_NAME + +services: + warp: + secrets: + - CLUSTER_SECRETS + ports: + - 9092/tcp + - 9999/tcp + - 8080/tcp + x-network: + Ingress: + Myself: true + ExtSources: + - IPv4: 0.0.0.0/0 + Name: ANY + Ports: + - 9092 + + expose: + - 8080/tcp + image: public.ecr.aws/warpstream-labs/warpstream_agent:latest + x-docker_opts: + InterpolateWithDigest: true + environment: + WARPSTREAM_DISCOVERY_KAFKA_HOSTNAME_OVERRIDE: "warp.bdd-testing.compose-x.io" + WARPSTREAM_BUCKET_URL: s3://x-s3::warp-cluster-storage::BucketName?region=x-aws::AWS::Region&prefix=cluster-01/ + WARPSTREAM_REQUIRE_AUTHENTICATION: true + command: + - agent + - -requireAuthentication + +x-tags: + project: warpstream + +x-route53: + PublicZone: + ZoneName: bdd-testing.compose-x.io + Lookup: true + +x-acm: + warp-cert: + MacroParameters: + DomainNames: + - "warp.bdd-testing.compose-x.io" + - "*.warp.bdd-testing.compose-x.io" + HostedZoneId: x-route53::PublicZone + +x-elbv2: + public-ingress: + DnsAliases: + - Route53Zone: x-route53::PublicZone + Names: + - "warp.bdd-testing.compose-x.io" + - "*.warp.bdd-testing.compose-x.io" + + Properties: + Scheme: internet-facing + Type: network + LoadBalancerAttributes: + dns_record.client_routing_policy: partial_availability_zone_affinity + MacroParameters: + cross_zone: false + Settings: + NoAllocateEips: true + Listeners: + - Port: 9092 + Protocol: TLS + Certificates: + - x-acm: warp-cert + Targets: + - name: warp:warp:9092 + access: / + Services: + warp:warp: + port: 9092 + protocol: TCP + healthcheck: 9092:TCP:2:2:15:5 + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: "15" + - Key: deregistration_delay.connection_termination.enabled + Value: "false" + - Key: preserve_client_ip.enabled + Value: "true" + +x-s3: + warp-cluster-storage: + Services: + warp: + Access: + bucket: ListOnly + objects: CRUD + s3-bucket-ssl-requests-only: true + MacroParameters: + ExpandAccountIdToBucket: true + ExpandRegionToBucket: true + Properties: + AccelerateConfiguration: + AccelerationStatus: Suspended + AccessControl: BucketOwnerFullControl + BucketEncryption: + ServerSideEncryptionConfiguration: + - BucketKeyEnabled: true + ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + LifecycleConfiguration: + AbortIncompleteMultipartUpload: + DaysAfterInitiation: 1 + Rules: + - Id: InfrequentAccess + Prefix: cluster-01/ + Status: Enabled + Transitions: + - StorageClass: STANDARD_IA + TransitionInDays: '31' + - StorageClass: INTELLIGENT_TIERING + TransitionInDays: '90' + + MetricsConfigurations: + - Id: EntireBucket + ObjectLockEnabled: true + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true