From 3449ad7732d6000852edeccdf6730f224b9ef6bc Mon Sep 17 00:00:00 2001 From: John Preston <1236150+JohnPreston@users.noreply.github.com> Date: Thu, 5 Aug 2021 15:33:19 +0100 Subject: [PATCH] Adding self signed certificates generation (#9) --- ecs-files-input.json | 460 +++++++++++++---------- ecs_files_composer/aws_mgmt.py | 153 ++++++++ ecs_files_composer/certificates_mgmt.py | 113 ++++++ ecs_files_composer/common.py | 34 +- ecs_files_composer/ecs_files_composer.py | 348 +---------------- ecs_files_composer/files_mgmt.py | 204 ++++++++++ ecs_files_composer/input.py | 39 +- examples/nginx.config.yaml | 9 + pyproject.toml | 12 + tests/test_ecs_config_composer.py | 26 ++ 10 files changed, 834 insertions(+), 564 deletions(-) create mode 100644 ecs_files_composer/aws_mgmt.py create mode 100644 ecs_files_composer/certificates_mgmt.py create mode 100644 ecs_files_composer/files_mgmt.py create mode 100644 examples/nginx.config.yaml diff --git a/ecs-files-input.json b/ecs-files-input.json index 05be302..d0f33e2 100644 --- a/ecs-files-input.json +++ b/ecs-files-input.json @@ -1,207 +1,285 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "typeName": "input", - "description": "Configuration input definition for ECS Files Composer", - "required": [ - "files" - ], - "properties": { - "files": { - "type": "object", - "uniqueItems": true, - "patternProperties": { - "^/[\\x00-\\x7F]+$": { - "$ref": "#/definitions/FileDef" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "typeName": "input", + "description": "Configuration input definition for ECS Files Composer", + "required": [ + "files" + ], + "properties": { + "files": { + "type": "object", + "uniqueItems": true, + "patternProperties": { + "^/[\\x00-\\x7F]+$": { + "$ref": "#/definitions/FileDef" + } + } + }, + "certificates": { + "type": "object", + "additionalProperties": false, + "properties": { + "x509": { + "uniqueItems": true, + "patternProperties": { + "^/[\\x00-\\x7F]+$": { + "$ref": "#/definitions/X509CertDef" } + } + } + } + }, + "IamOverride": { + "type": "object", + "$ref": "#/definitions/IamOverrideDef" + } + }, + "definitions": { + "FileDef": { + "type": "object", + "additionalProperties": true, + "properties": { + "path": { + "type": "string" + }, + "content": { + "type": "string", + "description": "The raw content of the file to use" + }, + "source": { + "$ref": "#/definitions/SourceDef" + }, + "encoding": { + "type": "string", + "enum": [ + "base64", + "plain" + ], + "default": "plain" + }, + "group": { + "type": "string", + "description": "UNIX group name or GID owner of the file. Default to root(0)", + "default": "root" + }, + "owner": { + "type": "string", + "description": "UNIX user or UID owner of the file. Default to root(0)", + "default": "root" + }, + "mode": { + "type": "string", + "description": "UNIX file mode", + "default": "0644" + }, + "context": { + "type": "string", + "enum": [ + "plain", + "jinja2" + ], + "default": "plain" + }, + "ignore_if_failed": { + "type": "boolean", + "description": "Whether or not the failure to retrieve the file should stop the execution", + "default": false + }, + "commands": { + "type": "object", + "properties": { + "post": { + "$ref": "#/definitions/CommandsDef", + "description": "Commands to run after the file was retrieved" + }, + "pre": { + "$ref": "#/definitions/CommandsDef", + "description": "Commands executed prior to the file being fetched, after `depends_on` completed" + } + } + } + } + }, + "SourceDef": { + "type": "object", + "properties": { + "Url": { + "$ref": "#/definitions/UrlDef" + }, + "Ssm": { + "$ref": "#/definitions/SsmDef" + }, + "S3": { + "$ref": "#/definitions/S3Def" + }, + "Secret": { + "$ref": "#/definitions/SecretDef" + } + } + }, + "UrlDef": { + "type": "object", + "properties": { + "Url": { + "type": "string", + "format": "uri" + }, + "Username": { + "type": "string" + }, + "Password": { + "type": "string" + } + } + }, + "SsmDef": { + "type": "object", + "properties": { + "ParameterName": { + "type": "string" }, "IamOverride": { - "type": "object", - "$ref": "#/definitions/IamOverrideDef" + "$ref": "#/definitions/IamOverrideDef" } + } }, - "definitions": { - "FileDef": { - "type": "object", - "additionalProperties": true, - "properties": { - "path": { - "type": "string" - }, - "content": { - "type": "string", - "description": "The raw content of the file to use" - }, - "source": { - "$ref": "#/definitions/SourceDef" - }, - "encoding": { - "type": "string", - "enum": [ - "base64", - "plain" - ], - "default": "plain" - }, - "group": { - "type": "string", - "description": "UNIX group name or GID owner of the file. Default to root(0)", - "default": "root" - }, - "owner": { - "type": "string", - "description": "UNIX user or UID owner of the file. Default to root(0)", - "default": "root" - }, - "mode": { - "type": "string", - "description": "UNIX file mode", - "default": "0644" - }, - "context": { - "type": "string", - "enum": [ - "plain", - "jinja2" - ], - "default": "plain" - }, - "ignore_if_failed": { - "type": "boolean", - "description": "Whether or not the failure to retrieve the file should stop the execution", - "default": false - }, - "commands": { - "type": "object", - "properties": { - "post": { - "$ref": "#/definitions/CommandsDef", - "description": "Commands to run after the file was retrieved" - }, - "pre": { - "$ref": "#/definitions/CommandsDef", - "description": "Commands executed prior to the file being fetched, after `depends_on` completed" - } - } - } - } + "SecretDef": { + "type": "object", + "required": [ + "SecretId" + ], + "properties": { + "SecretId": { + "type": "string" }, - "SourceDef": { - "type": "object", - "properties": { - "Url": { - "$ref": "#/definitions/UrlDef" - }, - "Ssm": { - "$ref": "#/definitions/SsmDef" - }, - "S3": { - "$ref": "#/definitions/S3Def" - }, - "Secret": { - "$ref": "#/definitions/SecretDef" - } - } + "VersionId": { + "type": "string" }, - "UrlDef": { - "type": "object", - "properties": { - "Url": { - "type": "string", - "format": "uri" - }, - "Username": { - "type": "string" - }, - "Password": { - "type": "string" - } - } + "VersionStage": { + "type": "string" }, - "SsmDef": { - "type": "object", - "properties": { - "ParameterName": { - "type": "string" - }, - "IamOverride": { - "$ref": "#/definitions/IamOverrideDef" - } - } + "IamOverride": { + "$ref": "#/definitions/IamOverrideDef" + } + } + }, + "S3Def": { + "type": "object", + "required": [ + "BucketName", + "Key" + ], + "properties": { + "BucketName": { + "type": "string", + "description": "Name of the S3 Bucket" }, - "SecretDef": { - "type": "object", - "required": [ - "SecretId" - ], - "properties": { - "SecretId": { - "type": "string" - }, - "VersionId": { - "type": "string" - }, - "VersionStage": { - "type": "string" - }, - "IamOverride": { - "$ref": "#/definitions/IamOverrideDef" - } - } + "BucketRegion": { + "type": "string", + "description": "S3 Region to use. Default will ignore or retrieve via s3:GetBucketLocation" }, - "S3Def": { - "type": "object", - "required": [ - "BucketName", - "Key" - ], - "properties": { - "BucketName": { - "type": "string", - "description": "Name of the S3 Bucket" - }, - "BucketRegion": { - "type": "string", - "description": "S3 Region to use. Default will ignore or retrieve via s3:GetBucketLocation" - }, - "Key": { - "type": "string", - "description": "Full path to the file to retrieve" - }, - "IamOverride": { - "$ref": "#/definitions/IamOverrideDef" - } - } + "Key": { + "type": "string", + "description": "Full path to the file to retrieve" }, - "IamOverrideDef": { - "type": "object", - "description": "When source points to AWS, allows to indicate if another role should be used", - "properties": { - "RoleArn": { - "type": "string" - }, - "SessionName": { - "type": "string", - "default": "S3File@EcsConfigComposer", - "description": "Name of the IAM session" - }, - "ExternalId": { - "type": "string", - "description": "The External ID to use when using sts:AssumeRole" - }, - "RegionName": { - "type": "string" - } - } + "IamOverride": { + "$ref": "#/definitions/IamOverrideDef" + } + } + }, + "IamOverrideDef": { + "type": "object", + "description": "When source points to AWS, allows to indicate if another role should be used", + "properties": { + "RoleArn": { + "type": "string" }, - "CommandsDef": { - "type": "array", - "description": "List of commands to run", - "items": { - "type": "string", - "description": "Shell command to run" - } + "SessionName": { + "type": "string", + "default": "S3File@EcsConfigComposer", + "description": "Name of the IAM session" + }, + "ExternalId": { + "type": "string", + "description": "The External ID to use when using sts:AssumeRole" + }, + "RegionName": { + "type": "string" + } + } + }, + "CommandsDef": { + "type": "array", + "description": "List of commands to run", + "items": { + "type": "string", + "description": "Shell command to run" + } + }, + "X509CertDef": { + "type": "object", + "additionalProperties": true, + "required": [ + "certFileName", + "keyFileName" + ], + "properties": { + "dir_path": { + "type": "string" + }, + "emailAddress": { + "type": "string", + "format": "idn-email", + "default": "files-composer@compose-x.tld" + }, + "commonName": { + "type": "string", + "format": "hostname" + }, + "countryName": { + "type": "string", + "pattern": "^[A-Z]+$", + "default": "AW" + }, + "localityName": { + "type": "string", + "default": "AWS" + }, + "stateOrProvinceName": { + "type": "string", + "default": "AWS" + }, + "organizationName": { + "type": "string", + "default": "AWS" + }, + "organizationUnitName": { + "type": "string", + "default": "AWS" + }, + "validityEndInSeconds": { + "type": "number", + "default": 8035200, + "description": "Validity before cert expires, in seconds. Default 3*31*24*60*60=3Months" + }, + "keyFileName": { + "type": "string" + }, + "certFileName": { + "type": "string" + }, + "group": { + "type": "string", + "description": "UNIX group name or GID owner of the file. Default to root(0)", + "default": "root" + }, + "owner": { + "type": "string", + "description": "UNIX user or UID owner of the file. Default to root(0)", + "default": "root" } + } } + } } diff --git a/ecs_files_composer/aws_mgmt.py b/ecs_files_composer/aws_mgmt.py new file mode 100644 index 0000000..7f9ab07 --- /dev/null +++ b/ecs_files_composer/aws_mgmt.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MPL-2.0 +# Copyright 2020-2021 John Mille + +"""AWS module.""" + +import re + +import boto3 +from boto3 import session +from botocore.exceptions import ClientError + +from ecs_files_composer import input +from ecs_files_composer.common import LOG +from ecs_files_composer.envsubst import expandvars + + +def create_session_from_creds(tmp_creds, region=None): + """ + Function to easily convert the AssumeRole reply into a boto3 session + :param tmp_creds: + :return: + :rtype boto3.session.Session + """ + creds = tmp_creds["Credentials"] + params = { + "aws_access_key_id": creds["AccessKeyId"], + "aws_secret_access_key": creds["SecretAccessKey"], + "aws_session_token": creds["SessionToken"], + } + if region: + params["region_name"] = region + return boto3.session.Session(**params) + + +class AwsResourceHandler(object): + """ + Class to handle all AWS related credentials init. + """ + + def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): + """ + :param str role_arn: + :param str external_id: + :param str region: + :param ecs_files_composer.input.IamOverrideDef iam_config_object: + """ + self.session = session.Session() + self.client_session = session.Session() + if role_arn or iam_config_object: + if role_arn and not iam_config_object: + params = {"RoleArn": role_arn, "RoleSessionName": "EcsConfigComposer@AwsResourceHandlerInit"} + if external_id: + params["ExternalId"] = external_id + tmp_creds = self.session.client("sts").assume_role(**params) + self.client_session = create_session_from_creds(tmp_creds, region=region) + elif iam_config_object: + params = { + "RoleArn": iam_config_object.role_arn, + "RoleSessionName": f"{iam_config_object.session_name}@AwsResourceHandlerInit", + } + if iam_config_object.external_id: + params["ExternalId"] = iam_config_object.external_id + tmp_creds = self.session.client("sts").assume_role(**params) + self.client_session = create_session_from_creds(tmp_creds, region=iam_config_object.region_name) + + +class S3Fetcher(AwsResourceHandler): + """ + Class to handle S3 actions + """ + + def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): + super().__init__(role_arn, external_id, region, iam_config_object) + self.client = self.client_session.client("s3") + + def get_content(self, s3_uri=None, s3_bucket=None, s3_key=None): + """ + Retrieves a file in a temp dir and returns content + + :param str s3_uri: + :param str s3_bucket: + :param str s3_key: + :return: The Stream Body for the file, allowing to do various things + """ + bucket_re = re.compile(r"(?:s3://)(?P[a-z0-9-.]+)/(?P[\S]+$)") + if s3_uri and bucket_re.match(s3_uri).groups(): + s3_bucket = bucket_re.match(s3_uri).group("bucket") + s3_key = bucket_re.match(s3_uri).group("key") + try: + file_r = self.client.get_object(Bucket=s3_bucket, Key=s3_key) + file_content = file_r["Body"] + return file_content + except self.client.exceptions.NoSuchKey: + LOG.error(f"Failed to download the file {s3_key} from bucket {s3_bucket}") + raise + + +class SsmFetcher(AwsResourceHandler): + """ + Class to handle SSM actions + """ + + def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): + super().__init__(role_arn, external_id, region, iam_config_object) + self.client = self.client_session.client("ssm") + + def get_content(self, parameter_name): + """ + Import the Content of a given parameter + + :param parameter_name: + :return: + """ + try: + parameter = self.client.get_parameter(Name=parameter_name, WithDecryption=True) + return parameter["Parameter"]["Value"] + except self.client.exceptions: + raise + except ClientError: + raise + + +class SecretFetcher(AwsResourceHandler): + """ + Class to handle Secret Manager actions + """ + + def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): + super().__init__(role_arn, external_id, region, iam_config_object) + self.client = self.client_session.client("secretsmanager") + + def get_content(self, secret): + """ + Import the Content of a given parameter + + :param input.SecretDef secret: + :return: + """ + secret_id = expandvars(secret.secret_id) + params = {"SecretId": secret_id} + LOG.debug(f"Retrieving secretsmanager://{secret_id}") + if secret.version_id: + params["VersionId"] = secret.version_id + if secret.version_stage: + params["VersionStage"] = secret.version_stage + try: + parameter = self.client.get_secret_value(**params) + return parameter["SecretString"] + except self.client.exceptions: + raise + except ClientError: + raise diff --git a/ecs_files_composer/certificates_mgmt.py b/ecs_files_composer/certificates_mgmt.py new file mode 100644 index 0000000..77d5420 --- /dev/null +++ b/ecs_files_composer/certificates_mgmt.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MPL-2.0 +# Copyright 2020-2021 John Mille + +import pathlib +import socket +from os import path +from typing import Any + +from OpenSSL import crypto + +from ecs_files_composer.files_mgmt import File +from ecs_files_composer.input import X509CertDef + + +class X509Certificate(X509CertDef, object): + """ + Class to wrap actions around a new X509 certificate + """ + + def __init__(self, **data: Any): + super().__init__(**data) + self.key = None + self.cert = None + self.key_content = None + self.cert_content = None + self.cert_file = None + self.key_file = None + self.cert_file_path = None + self.key_file_path = None + + def init_cert_paths(self): + self.cert_file_path = path.abspath(f"{self.dir_path}/{self.cert_file_name}") + self.key_file_path = path.abspath(f"{self.dir_path}/{self.key_file_name}") + print(f"Creating {self.dir_path} folder") + dir_path = pathlib.Path(path.abspath(self.dir_path)) + dir_path.mkdir(parents=True, exist_ok=True) + + def generate_key(self): + self.key = crypto.PKey() + self.key.generate_key(crypto.TYPE_RSA, 4096) + + def set_common_name(self): + if self.common_name is None: + self.common_name = socket.gethostname() + + def generate_cert(self): + if not self.common_name: + self.set_common_name() + self.cert = crypto.X509() + self.cert.get_subject().C = self.country_name + self.cert.get_subject().ST = self.state_or_province_name + self.cert.get_subject().L = self.locality_name + self.cert.get_subject().O = self.organization_name + self.cert.get_subject().OU = self.organization_unit_name + self.cert.get_subject().CN = self.common_name + self.cert.get_subject().emailAddress = self.email_address + self.cert.set_serial_number(0) + self.cert.gmtime_adj_notBefore(0) + self.cert.gmtime_adj_notAfter(int(self.validity_end_in_seconds)) + self.cert.set_issuer(self.cert.get_subject()) + self.cert.set_pubkey(self.key) + self.cert.sign(self.key, 'sha512') + + def generate_cert_content(self): + if not self.key: + self.generate_key() + if not self.cert: + self.generate_cert() + self.cert_content = crypto.dump_certificate(crypto.FILETYPE_PEM, self.cert).decode("utf-8") + self.key_content = crypto.dump_privatekey(crypto.FILETYPE_PEM, self.key).decode("utf-8") + + def set_cert_files(self): + if not self.cert_content or not self.key_content: + self.generate_cert_content() + self.key_file = File().parse_obj( + { + "content": self.key_content, + "path": self.key_file_path, + "mode": "0600", + "owner": self.owner, + "group": self.group, + } + ) + self.cert_file = File().parse_obj( + { + "content": self.cert_content, + "path": self.cert_file_path, + "mode": "0600", + "owner": self.owner, + "group": self.group, + } + ) + + +def process_x509_certs(job): + """ + + :param ecs_files_composer.input.Model job: + :return: + """ + if not hasattr(job.certificates, "x509") or not job.certificates.x509: + return + for cert_path, cert_def in job.certificates.x509.items(): + cert_obj = X509Certificate( + keyFileName=cert_def["keyFileName"], certFileName=cert_def["certFileName"] + ).parse_obj(cert_def) + cert_obj.dir_path = cert_path + cert_obj.init_cert_paths() + cert_obj.set_cert_files() + job.certificates.x509[cert_path] = cert_obj + job.files[cert_obj.cert_file.path] = cert_obj.cert_file + job.files[cert_obj.key_file_path] = cert_obj.key_file diff --git a/ecs_files_composer/common.py b/ecs_files_composer/common.py index 5ff226e..ad77f6e 100644 --- a/ecs_files_composer/common.py +++ b/ecs_files_composer/common.py @@ -6,39 +6,7 @@ import sys from os import environ - -def keyisset(x, y): - """ - Macro to figure if the the dictionary contains a key and that the key is not empty - - :param x: The key to check presence in the dictionary - :type x: str - :param y: The dictionary to check for - :type y: dict - - :returns: True/False - :rtype: bool - """ - if isinstance(y, dict) and x in y.keys() and y[x]: - return True - return False - - -def keypresent(x, y): - """ - Macro to figure if the the dictionary contains a key and that the key is not empty - - :param x: The key to check presence in the dictionary - :type x: str - :param y: The dictionary to check for - :type y: dict - - :returns: True/False - :rtype: bool - """ - if isinstance(y, dict) and x in y.keys(): - return True - return False +from compose_x_common.compose_x_common import keyisset def setup_logging(): diff --git a/ecs_files_composer/ecs_files_composer.py b/ecs_files_composer/ecs_files_composer.py index 1cf4a7b..2062226 100644 --- a/ecs_files_composer/ecs_files_composer.py +++ b/ecs_files_composer/ecs_files_composer.py @@ -4,348 +4,18 @@ """Main module.""" -import base64 import json -import re -import subprocess -import warnings from os import environ, path -from tempfile import TemporaryDirectory -from typing import Any -import boto3 -import requests import yaml -from boto3 import session -from botocore.exceptions import ClientError -from botocore.response import StreamingBody -from jinja2 import Environment, FileSystemLoader +from compose_x_common.compose_x_common import keyisset from yaml import Loader from ecs_files_composer import input -from ecs_files_composer.common import LOG, keyisset -from ecs_files_composer.envsubst import expandvars -from ecs_files_composer.jinja2_filters import env - - -def create_session_from_creds(tmp_creds, region=None): - """ - Function to easily convert the AssumeRole reply into a boto3 session - :param tmp_creds: - :return: - :rtype boto3.session.Session - """ - creds = tmp_creds["Credentials"] - params = { - "aws_access_key_id": creds["AccessKeyId"], - "aws_secret_access_key": creds["SecretAccessKey"], - "aws_session_token": creds["SessionToken"], - } - if region: - params["region_name"] = region - return boto3.session.Session(**params) - - -class AwsResourceHandler(object): - """ - Class to handle all AWS related credentials init. - """ - - def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): - """ - :param str role_arn: - :param str external_id: - :param str region: - :param ecs_files_composer.input.IamOverrideDef iam_config_object: - """ - self.session = session.Session() - self.client_session = session.Session() - if role_arn or iam_config_object: - if role_arn and not iam_config_object: - params = {"RoleArn": role_arn, "RoleSessionName": "EcsConfigComposer@AwsResourceHandlerInit"} - if external_id: - params["ExternalId"] = external_id - tmp_creds = self.session.client("sts").assume_role(**params) - self.client_session = create_session_from_creds(tmp_creds, region=region) - elif iam_config_object: - params = { - "RoleArn": iam_config_object.role_arn, - "RoleSessionName": f"{iam_config_object.session_name}@AwsResourceHandlerInit", - } - if iam_config_object.external_id: - params["ExternalId"] = iam_config_object.external_id - tmp_creds = self.session.client("sts").assume_role(**params) - self.client_session = create_session_from_creds(tmp_creds, region=iam_config_object.region_name) - - -class S3Fetcher(AwsResourceHandler): - """ - Class to handle S3 actions - """ - - def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): - super().__init__(role_arn, external_id, region, iam_config_object) - self.client = self.client_session.client("s3") - - def get_content(self, s3_uri=None, s3_bucket=None, s3_key=None): - """ - Retrieves a file in a temp dir and returns content - - :param str s3_uri: - :param str s3_bucket: - :param str s3_key: - :return: The Stream Body for the file, allowing to do various things - """ - bucket_re = re.compile(r"(?:s3://)(?P[a-z0-9-.]+)/(?P[\S]+$)") - if s3_uri and bucket_re.match(s3_uri).groups(): - s3_bucket = bucket_re.match(s3_uri).group("bucket") - s3_key = bucket_re.match(s3_uri).group("key") - try: - file_r = self.client.get_object(Bucket=s3_bucket, Key=s3_key) - file_content = file_r["Body"] - return file_content - except self.client.exceptions.NoSuchKey: - LOG.error(f"Failed to download the file {s3_key} from bucket {s3_bucket}") - raise - - -class SsmFetcher(AwsResourceHandler): - """ - Class to handle SSM actions - """ - - def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): - super().__init__(role_arn, external_id, region, iam_config_object) - self.client = self.client_session.client("ssm") - - def get_content(self, parameter_name): - """ - Import the Content of a given parameter - - :param parameter_name: - :return: - """ - try: - parameter = self.client.get_parameter(Name=parameter_name, WithDecryption=True) - return parameter["Parameter"]["Value"] - except self.client.exceptions: - raise - except ClientError: - raise - - -class SecretFetcher(AwsResourceHandler): - """ - Class to handle Secret Manager actions - """ - - def __init__(self, role_arn=None, external_id=None, region=None, iam_config_object=None): - super().__init__(role_arn, external_id, region, iam_config_object) - self.client = self.client_session.client("secretsmanager") - - def get_content(self, secret): - """ - Import the Content of a given parameter - - :param input.SecretDef secret: - :return: - """ - secret_id = expandvars(secret.secret_id) - params = {"SecretId": secret_id} - LOG.debug(f"Retrieving secretsmanager://{secret_id}") - if secret.version_id: - params["VersionId"] = secret.version_id - if secret.version_stage: - params["VersionStage"] = secret.version_stage - try: - parameter = self.client.get_secret_value(**params) - return parameter["SecretString"] - except self.client.exceptions: - raise - except ClientError: - raise - - -class File(input.FileDef, object): - """ - Class to wrap common files actions around - """ - - def __init__(self, iam_override=None, **data: Any): - super().__init__(**data) - self.templates_dir = None - - def handler(self, iam_override): - """ - Main entrypoint for files to relate - - :param input.IamOverrideDef iam_override: - :return: - """ - if self.context and isinstance(self.context, input.Context) and self.context.value == "jinja2": - self.templates_dir = TemporaryDirectory() - if self.commands and self.commands.pre: - warnings.warn("Commands are not yet implemented", Warning) - if self.source and not self.content: - self.handle_sources(iam_override=iam_override) - self.write_content() - if not self.source and self.content: - self.write_content(decode=True) - if self.templates_dir: - self.render_jinja() - self.set_unix_settings() - if self.commands and self.commands.post: - warnings.warn("Commands are not yet implemented", Warning) - - def handle_sources(self, iam_override=None): - """ - Handles files from external sources - - :param input.IamOverrideDef iam_override: - :return: - """ - if self.source.url: - self.handle_http_content() - elif self.source.ssm: - self.handle_ssm_source(iam_override) - elif self.source.s3: - self.handle_s3_source(iam_override) - elif self.source.secret: - LOG.warn("When using ECS, we recommend to use the Secrets in Task Definition") - self.handle_secret_source(iam_override) - - def handle_url_source(self): - """ - Handles retrieving files from URLs - :return: - """ - - def handle_ssm_source(self, iam_override=None): - """ - Handles retrieving the content from SSM Parameter - - :param input.IamOverrideDef iam_override: - :return: - """ - parameter_name = expandvars(self.source.ssm.parameter_name) - LOG.debug(f"Retrieving ssm://{parameter_name}") - if self.source.ssm.iam_override: - fetcher = SsmFetcher(iam_config_object=self.source.ssm.iam_override) - else: - fetcher = SsmFetcher(iam_config_object=iam_override) - self.content = fetcher.get_content(parameter_name=parameter_name) - - def handle_s3_source(self, iam_override=None): - """ - Handles retrieving the content from S3 - - :param input.IamOverrideDef iam_override: - :return: - """ - bucket_name = expandvars(self.source.s3.bucket_name) - key = expandvars(self.source.s3.key) - LOG.debug(f"Retrieving s3://{bucket_name}/{key}") - if self.source.s3.iam_override: - fetcher = S3Fetcher(iam_config_object=self.source.s3.iam_override) - else: - fetcher = S3Fetcher(iam_config_object=iam_override) - self.content = fetcher.get_content(s3_bucket=bucket_name, s3_key=key) - - def handle_secret_source(self, iam_override=None): - """ - Handles retrieving secrets from AWS Secrets Manager - - :param input.IamOverrideDef iam_override: - :return: - """ - if self.source.secret.iam_override: - fetcher = SecretFetcher(iam_config_object=self.source.secret.iam_override) - else: - fetcher = SecretFetcher(iam_config_object=iam_override) - self.content = fetcher.get_content(self.source.secret) - - def handle_http_content(self): - """ - Fetches the content from a provided URI - - """ - if not self.source.url.username or not self.source.url.password: - req = requests.get(self.source.url.url) - else: - req = requests.get(self.source.url.url, auth=(self.source.url.username, self.source.url.password)) - try: - req.raise_for_status() - self.write_content(as_bytes=True, bytes_content=req.content) - except requests.exceptions.HTTPError as e: - LOG.error(e) - raise - - def render_jinja(self): - """ - Allows to use the temp directory as environment base, the original file as source template, and render - a final template. - """ - jinja_env = Environment(loader=FileSystemLoader(self.templates_dir.name), autoescape=True, auto_reload=False) - jinja_env.filters['env_override'] = env - template = jinja_env.get_template(path.basename(self.path)) - self.content = template.render() - self.write_content(is_template=False) - - def set_unix_settings(self): - """ - Applies UNIX settings to given file - - """ - cmd = ["chmod", self.mode, self.path] - try: - res = subprocess.run( - cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False - ) - except subprocess.CalledProcessError: - if self.ignore_if_failed: - LOG.error(res.stderr) - else: - raise - cmd = ["chown", f"{self.owner}:{self.group}", self.path] - try: - res = subprocess.run( - cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False - ) - except subprocess.CalledProcessError: - if self.ignore_if_failed: - LOG.error(res.stderr) - else: - raise - - def write_content(self, is_template=True, decode=False, as_bytes=False, bytes_content=None): - """ - Function to write the content retrieved to path. - - :param bool is_template: Whether the content should be considered to be a template. - :param decode: - :param as_bytes: - :param bytes_content: - :return: - """ - file_path = ( - f"{self.templates_dir.name}/{path.basename(self.path)}" - if (self.templates_dir and is_template) - else self.path - ) - LOG.info(f"Outputting {self.path} to {file_path}") - if isinstance(self.content, str): - if decode and self.encoding == input.Encoding["base64"]: - with open(file_path, "wb") as file_fd: - file_fd.write(base64.b64decode(self.content)) - else: - with open(file_path, "w") as file_fd: - file_fd.write(self.content) - elif isinstance(self.content, StreamingBody): - with open(file_path, "wb") as file_fd: - file_fd.write(self.content.read()) - elif as_bytes and bytes_content: - with open(file_path, "wb") as file_fd: - file_fd.write(bytes_content) +from ecs_files_composer.aws_mgmt import S3Fetcher, SecretFetcher, SsmFetcher +from ecs_files_composer.certificates_mgmt import process_x509_certs +from ecs_files_composer.common import LOG +from ecs_files_composer.files_mgmt import File def init_config( @@ -417,10 +87,12 @@ def start_jobs(config): if not keyisset("files", config): raise KeyError("Missing required files from configuration input") job = input.Model(files=config["files"]).parse_obj(config) + process_x509_certs(job) for file_path, file in job.files.items(): - file_def = File().parse_obj(file) - job.files[file_path] = file_def - file_def.path = file_path + if not isinstance(file, File): + file_def = File().parse_obj(file) + job.files[file_path] = file_def + file_def.path = file_path for file in job.files.values(): file.handler(job.iam_override) LOG.info(f"Tasks for {file.path} completed.") diff --git a/ecs_files_composer/files_mgmt.py b/ecs_files_composer/files_mgmt.py new file mode 100644 index 0000000..229b3e6 --- /dev/null +++ b/ecs_files_composer/files_mgmt.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: MPL-2.0 +# Copyright 2020-2021 John Mille + +"""Main module.""" + +import base64 +import subprocess +import warnings +from os import path +from tempfile import TemporaryDirectory +from typing import Any + +import requests +from botocore.response import StreamingBody +from jinja2 import Environment, FileSystemLoader + +from ecs_files_composer.aws_mgmt import S3Fetcher, SecretFetcher, SsmFetcher +from ecs_files_composer.common import LOG +from ecs_files_composer.envsubst import expandvars +from ecs_files_composer.input import Context, Encoding, FileDef +from ecs_files_composer.jinja2_filters import env + + +class File(FileDef, object): + """ + Class to wrap common files actions around + """ + + def __init__(self, iam_override=None, **data: Any): + super().__init__(**data) + self.templates_dir = None + + def handler(self, iam_override): + """ + Main entrypoint for files to relate + + :param ecs_files_composer.input.IamOverrideDef iam_override: + :return: + """ + if self.context and isinstance(self.context, Context) and self.context.value == "jinja2": + self.templates_dir = TemporaryDirectory() + if self.commands and self.commands.pre: + warnings.warn("Commands are not yet implemented", Warning) + if self.source and not self.content: + self.handle_sources(iam_override=iam_override) + self.write_content() + if not self.source and self.content: + self.write_content(decode=True) + if self.templates_dir: + self.render_jinja() + self.set_unix_settings() + if self.commands and self.commands.post: + warnings.warn("Commands are not yet implemented", Warning) + + def handle_sources(self, iam_override=None): + """ + Handles files from external sources + + :param ecs_files_composer.input.IamOverrideDef iam_override: + :return: + """ + if self.source.url: + self.handle_http_content() + elif self.source.ssm: + self.handle_ssm_source(iam_override) + elif self.source.s3: + self.handle_s3_source(iam_override) + elif self.source.secret: + LOG.warn("When using ECS, we recommend to use the Secrets in Task Definition") + self.handle_secret_source(iam_override) + + def handle_url_source(self): + """ + Handles retrieving files from URLs + :return: + """ + + def handle_ssm_source(self, iam_override=None): + """ + Handles retrieving the content from SSM Parameter + + :param ecs_files_composer.input.IamOverrideDef iam_override: + :return: + """ + parameter_name = expandvars(self.source.ssm.parameter_name) + LOG.debug(f"Retrieving ssm://{parameter_name}") + if self.source.ssm.iam_override: + fetcher = SsmFetcher(iam_config_object=self.source.ssm.iam_override) + else: + fetcher = SsmFetcher(iam_config_object=iam_override) + self.content = fetcher.get_content(parameter_name=parameter_name) + + def handle_s3_source(self, iam_override=None): + """ + Handles retrieving the content from S3 + + :param ecs_files_composer.input.IamOverrideDef iam_override: + :return: + """ + bucket_name = expandvars(self.source.s3.bucket_name) + key = expandvars(self.source.s3.key) + LOG.debug(f"Retrieving s3://{bucket_name}/{key}") + if self.source.s3.iam_override: + fetcher = S3Fetcher(iam_config_object=self.source.s3.iam_override) + else: + fetcher = S3Fetcher(iam_config_object=iam_override) + self.content = fetcher.get_content(s3_bucket=bucket_name, s3_key=key) + + def handle_secret_source(self, iam_override=None): + """ + Handles retrieving secrets from AWS Secrets Manager + + :param ecs_files_composer.input.IamOverrideDef iam_override: + :return: + """ + if self.source.secret.iam_override: + fetcher = SecretFetcher(iam_config_object=self.source.secret.iam_override) + else: + fetcher = SecretFetcher(iam_config_object=iam_override) + self.content = fetcher.get_content(self.source.secret) + + def handle_http_content(self): + """ + Fetches the content from a provided URI + + """ + if not self.source.url.username or not self.source.url.password: + req = requests.get(self.source.url.url) + else: + req = requests.get(self.source.url.url, auth=(self.source.url.username, self.source.url.password)) + try: + req.raise_for_status() + self.write_content(as_bytes=True, bytes_content=req.content) + except requests.exceptions.HTTPError as e: + LOG.error(e) + raise + + def render_jinja(self): + """ + Allows to use the temp directory as environment base, the original file as source template, and render + a final template. + """ + jinja_env = Environment(loader=FileSystemLoader(self.templates_dir.name), autoescape=True, auto_reload=False) + jinja_env.filters['env_override'] = env + template = jinja_env.get_template(path.basename(self.path)) + self.content = template.render() + self.write_content(is_template=False) + + def set_unix_settings(self): + """ + Applies UNIX settings to given file + + """ + cmd = ["chmod", self.mode, self.path] + try: + res = subprocess.run( + cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False + ) + except subprocess.CalledProcessError: + if self.ignore_if_failed: + LOG.error(res.stderr) + else: + raise + cmd = ["chown", f"{self.owner}:{self.group}", self.path] + try: + res = subprocess.run( + cmd, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False + ) + except subprocess.CalledProcessError: + if self.ignore_if_failed: + LOG.error(res.stderr) + else: + raise + + def write_content(self, is_template=True, decode=False, as_bytes=False, bytes_content=None): + """ + Function to write the content retrieved to path. + + :param bool is_template: Whether the content should be considered to be a template. + :param decode: + :param as_bytes: + :param bytes_content: + :return: + """ + file_path = ( + f"{self.templates_dir.name}/{path.basename(self.path)}" + if (self.templates_dir and is_template) + else self.path + ) + LOG.info(f"Outputting {self.path} to {file_path}") + if isinstance(self.content, str): + if decode and self.encoding == Encoding["base64"]: + with open(file_path, "wb") as file_fd: + file_fd.write(base64.b64decode(self.content)) + else: + with open(file_path, "w") as file_fd: + file_fd.write(self.content) + elif isinstance(self.content, StreamingBody): + with open(file_path, "wb") as file_fd: + file_fd.write(self.content.read()) + elif as_bytes and bytes_content: + with open(file_path, "wb") as file_fd: + file_fd.write(bytes_content) diff --git a/ecs_files_composer/input.py b/ecs_files_composer/input.py index e535cec..b4b5d35 100644 --- a/ecs_files_composer/input.py +++ b/ecs_files_composer/input.py @@ -1,13 +1,20 @@ # generated by datamodel-codegen: # filename: ecs-files-input.json -# timestamp: 2021-07-20T08:20:14+00:00 +# timestamp: 2021-08-05T14:06:00+00:00 from __future__ import annotations from enum import Enum from typing import Any, Dict, List, Optional -from pydantic import AnyUrl, BaseModel, Extra, Field +from pydantic import AnyUrl, BaseModel, EmailStr, Extra, Field, constr + + +class Certificates(BaseModel): + class Config: + extra = Extra.forbid + + x509: Optional[Any] = None class Encoding(Enum): @@ -41,8 +48,36 @@ class CommandsDef(BaseModel): __root__: List[str] = Field(..., description='List of commands to run') +class X509CertDef(BaseModel): + class Config: + extra = Extra.allow + + dir_path: Optional[str] = None + email_address: Optional[EmailStr] = Field('files-composer@compose-x.tld', alias='emailAddress') + common_name: Optional[ + constr( + regex=r'^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]{0,61}[A-Za-z0-9])\Z' + ) + ] = Field(None, alias='commonName') + country_name: Optional[str] = Field('AW', alias='countryName', regex='^[A-Z]+$') + locality_name: Optional[str] = Field('AWS', alias='localityName') + state_or_province_name: Optional[str] = Field('AWS', alias='stateOrProvinceName') + organization_name: Optional[str] = Field('AWS', alias='organizationName') + organization_unit_name: Optional[str] = Field('AWS', alias='organizationUnitName') + validity_end_in_seconds: Optional[float] = Field( + 8035200, + alias='validityEndInSeconds', + description='Validity before cert expires, in seconds. Default 3*31*24*60*60=3Months', + ) + key_file_name: str = Field(..., alias='keyFileName') + cert_file_name: str = Field(..., alias='certFileName') + group: Optional[str] = Field('root', description='UNIX group name or GID owner of the file. Default to root(0)') + owner: Optional[str] = Field('root', description='UNIX user or UID owner of the file. Default to root(0)') + + class Model(BaseModel): files: Dict[str, Any] + certificates: Optional[Certificates] = None iam_override: Optional[IamOverrideDef] = Field(None, alias='IamOverride') diff --git a/examples/nginx.config.yaml b/examples/nginx.config.yaml new file mode 100644 index 0000000..dcde5f4 --- /dev/null +++ b/examples/nginx.config.yaml @@ -0,0 +1,9 @@ +files: + /tmp/required: + content: "" + +certificates: + x509: + /tmp/nginx: + keyFileName: nginx.key + certFileName: nginx.crt diff --git a/pyproject.toml b/pyproject.toml index 76f0859..c4d3949 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,10 +28,22 @@ requests = "^2.26.0" PyYAML = "^5.4.1" Jinja2 = "^3.0.1" jsonschema = "^3.2.0" +compose-x-common = "^0.0.1" [tool.poetry.dev-dependencies] placebo = "^0.9.0" datamodel-code-generator = {version = "^0.11.8", extras = ["http"]} +Sphinx = "^4.1.2" +black = "^21.7b0" +isort = "^5.9.3" +sphinx-material = "^0.0.34" +coverage = "^5.5" +behave = "^1.2.6" +pytest = "^6.2.4" +flake8 = "^3.9.2" +watchdog = "^2.1.3" +pre-commit = "^2.13.0" +tbump = "6.3.1" [tool.poetry.scripts] files_composer = "ecs_files_composer.cli:main" diff --git a/tests/test_ecs_config_composer.py b/tests/test_ecs_config_composer.py index d6f0609..07a5442 100644 --- a/tests/test_ecs_config_composer.py +++ b/tests/test_ecs_config_composer.py @@ -41,9 +41,35 @@ def s3_files_config(): } +@pytest.fixture +def simple_json_config_with_certs(): + with open(f"{HERE}/RAW_CONTENT.txt", "r") as raw_fd: + raw_content = raw_fd.read() + return { + "files": { + "/tmp/test_raw.txt": {"content": raw_content}, + "/tmp/test.txt": {"content": "THIS IS A TEST"}, + "/tmp/dedoded.txt": {"content": "VEhJUyBJUyBFTkRPREVEIE1FU1NBR0U=", "encoding": "base64"}, + }, + "certificates": { + "x509": { + "/tmp/webserver": { + "keyFileName": "server.key", + "certFileName": "server.crt", + "commonName": "nowhere.tld", + } + } + }, + } + + def test_simple_job_import(simple_json_config): start_jobs(simple_json_config) def test_s3_files_simple(s3_files_config): start_jobs(s3_files_config) + + +def test_simple_cert(simple_json_config_with_certs): + start_jobs(simple_json_config_with_certs)